Merge origin/main into fix/net-send-nosignal

.github/workflows/build.yml

@@ -421,7 +421,7 @@ jobs:
 
   linux:
     needs: eligibility
-    if: ${{ (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.linux)) && needs.eligibility.outputs.full == 'true' }}
+    if: ${{ !github.event.repository.fork && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.linux)) && needs.eligibility.outputs.full == 'true' }}
     runs-on: [self-hosted, linux, x64, anyps5]
     timeout-minutes: 60
     env:

.github/workflows/input-mapper.yml

@@ -0,0 +1,29 @@
+name: Input mapper
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+    paths:
+      - '.github/workflows/input-mapper.yml'
+      - 'tools/input-mapper/**'
+      - 'core/libs/prx/libScePad/include/**'
+      - 'core/libs/prx/libSceVideoOut/src/InputMapping.cpp'
+
+permissions:
+  contents: read
+
+jobs:
+  input-mapper:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@v7
+        with:
+          node-version: '22'
+          package-manager-cache: false
+      - run: npm --prefix tools/input-mapper test

.github/workflows/pages-progress.yml

@@ -18,6 +18,7 @@ concurrency:
 
 jobs:
   progress:
+    if: ${{ !github.event.repository.fork }}
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:

.github/workflows/pages.yml

@@ -33,7 +33,7 @@ concurrency:
 
 jobs:
   pages:
-    if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
+    if: ${{ !github.event.repository.fork && (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success') }}
     runs-on: ubuntu-latest
     environment:
       name: github-pages

.github/workflows/web-tests.yml

@@ -36,4 +36,4 @@ jobs:
           node --check demo/web/output-directory.mjs
           node --check demo/web/worker.mjs
       - name: Test browser converter helpers
-        run: node --test demo/web/tests/batch.test.mjs demo/web/tests/output-directory.test.mjs
+        run: node --test demo/web/tests/app.test.mjs demo/web/tests/batch.test.mjs demo/web/tests/output-directory.test.mjs

CMakeLists.txt

@@ -136,10 +136,17 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/tests/test_shader_cache_native_paths.py
             ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ShaderCacheVersion.cmake ${CMAKE_COMMAND})
         set_tests_properties(shader_cache_native_paths PROPERTIES TIMEOUT 20)
-        add_test(NAME shader_cache_runtime_sources COMMAND ${Python3_EXECUTABLE}
-            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_shader_cache_runtime_sources.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler ${CMAKE_COMMAND})
-        set_tests_properties(shader_cache_runtime_sources PROPERTIES TIMEOUT 180)
+        foreach(shard RANGE 0 6)
+            if(shard EQUAL 0)
+                set(shader_cache_sources_test_name shader_cache_runtime_sources)
+            else()
+                set(shader_cache_sources_test_name shader_cache_runtime_sources_shard_${shard})
+            endif()
+            add_test(NAME ${shader_cache_sources_test_name} COMMAND ${Python3_EXECUTABLE}
+                ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_shader_cache_runtime_sources.py
+                ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler ${CMAKE_COMMAND} --shard ${shard}/7)
+            set_tests_properties(${shader_cache_sources_test_name} PROPERTIES TIMEOUT 180)
+        endforeach()
         add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py)
         add_test(NAME pr_overlap_dependencies COMMAND ${Python3_EXECUTABLE}
@@ -164,6 +171,8 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_import_audit.py)
         add_test(NAME unwrap_self COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_unwrap_self.py)
+        add_test(NAME audit_dump COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_audit_dump.py)
         foreach(shard RANGE 0 11)
             if(shard EQUAL 0)
                 set(conventions_test_name conventions_function_bodies)
@@ -252,6 +261,8 @@ if(BUILD_TESTING)
         endforeach()
         add_test(NAME trace_digest COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_trace_digest.py)
+        add_test(NAME symbolize_samples COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_symbolize_samples.py)
         add_test(NAME conventions_bom_shebang COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_bom_shebang.py)
         set_tests_properties(conventions_bom_shebang PROPERTIES TIMEOUT 20)
@@ -277,8 +288,13 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_alignment.py)
         add_test(NAME conventions_pr_template COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_pr_template.py)
+        add_test(NAME check_compile_preprocessor COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile_preprocessor.py)
+        set_tests_properties(check_compile_preprocessor PROPERTIES TIMEOUT 20)
         add_test(NAME check_compile COMMAND ${Python3_EXECUTABLE}
-            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile.py)
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile.py CheckCompileTests)
+        add_test(NAME check_compile_outputs COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile.py CheckCompileOutputTests)
         add_test(NAME performance_compare COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_performance_compare.py)
         set_tests_properties(performance_compare PROPERTIES TIMEOUT 20)

core/libs/CMakeLists.txt

@@ -269,6 +269,15 @@ if(BUILD_TESTING)
         set_tests_properties(guest_kernel_sys_sdk_version PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
     endif()
+    add_test_executable(guest_kernel_sys_console_identity_tests tests/GuestKernelSysConsoleIdentity.cpp)
+    target_include_directories(guest_kernel_sys_console_identity_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_console_identity_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_console_identity_tests)
+    add_test(NAME guest_kernel_sys_console_identity COMMAND guest_kernel_sys_console_identity_tests)
+    if(WIN32)
+        set_tests_properties(guest_kernel_sys_console_identity PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+    endif()
     add_test_executable(nid_buffer_validation_tests nid/tests/BufferValidation.cpp)
     target_include_directories(nid_buffer_validation_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/nid/include)
     target_compile_features(nid_buffer_validation_tests PRIVATE cxx_std_20)
@@ -278,6 +287,32 @@ if(BUILD_TESTING)
     add_test(NAME nid_buffer_validation COMMAND nid_buffer_validation_tests)
     set_tests_properties(nid_buffer_validation PROPERTIES TIMEOUT 10)
 
+    add_test_executable(guest_dinkumware_fetch_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/GuestDinkumwareFetch.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/nid/src/NidCompute.cpp ${CMAKE_CURRENT_SOURCE_DIR}/nid/src/Sha1.cpp)
+    target_include_directories(guest_dinkumware_fetch_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/nid/include)
+    target_link_libraries(guest_dinkumware_fetch_tests PRIVATE ${CMAKE_DL_LIBS})
+    configure_windows_unwind(guest_dinkumware_fetch_tests)
+    if(MINGW)
+        target_link_libraries(guest_dinkumware_fetch_tests PRIVATE libc)
+    endif()
+    add_custom_target(guest_dinkumware_fetch_libraries DEPENDS ${LIBC_PATCHED})
+    add_dependencies(guest_dinkumware_fetch_tests guest_dinkumware_fetch_libraries)
+    foreach(operation IN ITEMS and or xor)
+        foreach(width IN ITEMS 1 2 4)
+            add_test(NAME guest_dinkumware_fetch_${operation}_${width} COMMAND guest_dinkumware_fetch_tests
+                $<TARGET_FILE_DIR:libc> unpatched ${operation} ${width})
+            add_test(NAME guest_dinkumware_fetch_${operation}_${width}_patched COMMAND guest_dinkumware_fetch_tests
+                ${PATCHED_DIR} patched ${operation} ${width})
+            set_tests_properties(guest_dinkumware_fetch_${operation}_${width}
+                guest_dinkumware_fetch_${operation}_${width}_patched PROPERTIES TIMEOUT 20)
+            if(WIN32)
+                set_tests_properties(guest_dinkumware_fetch_${operation}_${width} PROPERTIES ENVIRONMENT_MODIFICATION
+                    "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+                set_tests_properties(guest_dinkumware_fetch_${operation}_${width}_patched PROPERTIES ENVIRONMENT_MODIFICATION
+                    "PATH=path_list_prepend:${PATCHED_DIR}")
+            endif()
+        endforeach()
+    endforeach()
     add_test_executable(pe_import_first_tests allocator/tests/PeImportFirstTests.cpp allocator/PeImportFirst.cpp)
     target_include_directories(pe_import_first_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_compile_features(pe_import_first_tests PRIVATE cxx_std_20)
@@ -372,6 +407,16 @@ if(BUILD_TESTING)
     configure_windows_unwind(guest_kernel_sys_constants_tests)
     add_test(NAME guest_kernel_sys_constants COMMAND guest_kernel_sys_constants_tests)
     set_tests_properties(guest_kernel_sys_constants PROPERTIES TIMEOUT 20)
+    add_test_executable(guest_firmware_constant_returns_tests tests/GuestFirmwareConstantReturns.cpp)
+    target_include_directories(guest_firmware_constant_returns_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_firmware_constant_returns_tests PRIVATE libSceComposite libSceFsInternalForVsh libSceHttpCache libScePlayReady libSceSdma libSceSsl libSceVisionManager libmdbg_syscore libkernel libc)
+    configure_windows_unwind(guest_firmware_constant_returns_tests)
+    add_test(NAME guest_firmware_constant_returns COMMAND guest_firmware_constant_returns_tests)
+    set_tests_properties(guest_firmware_constant_returns PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_firmware_constant_returns PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceComposite>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceFsInternalForVsh>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceHttpCache>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libScePlayReady>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceSdma>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceSsl>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceVisionManager>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libmdbg_syscore>")
+    endif()
     if(Python3_Interpreter_FOUND)
         add_test(NAME nid_verneed COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_verneed.py $<TARGET_FILE:nid_patcher>)
@@ -416,6 +461,18 @@ if(BUILD_TESTING)
         add_subdirectory(tests/write_watch)
         add_subdirectory(tests/guest_arena)
     endif()
+    add_test_executable(guest_kernel_sys_pthread_np_tests tests/GuestKernelSysPthreadNp.cpp)
+    target_include_directories(guest_kernel_sys_pthread_np_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_pthread_np_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_pthread_np_tests)
+    foreach(pthreadNpMode IN ITEMS rwlockattr kind)
+        add_test(NAME guest_kernel_sys_pthread_np_${pthreadNpMode} COMMAND guest_kernel_sys_pthread_np_tests ${pthreadNpMode})
+        set_tests_properties(guest_kernel_sys_pthread_np_${pthreadNpMode} PROPERTIES TIMEOUT 20)
+        if(WIN32)
+            set_tests_properties(guest_kernel_sys_pthread_np_${pthreadNpMode} PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+        endif()
+    endforeach()
     add_test_executable(guest_kernel_sys_module_internal_tests tests/GuestKernelSysModuleInternal.cpp)
     target_include_directories(guest_kernel_sys_module_internal_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_kernel_sys_module_internal_tests PRIVATE libkernel_sys libkernel libc)
@@ -435,6 +492,23 @@ if(BUILD_TESTING)
         endif()
         add_test(NAME windows_write_watch COMMAND windows_write_watch_tests)
         set_tests_properties(windows_write_watch PROPERTIES TIMEOUT 20)
+        add_test_executable(windows_shared_map_cost_tests tests/WindowsSharedMapCost.cpp)
+        target_include_directories(windows_shared_map_cost_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+        target_compile_definitions(windows_shared_map_cost_tests PRIVATE _WIN32_WINNT=0x0A00)
+        if(MINGW)
+            target_compile_options(windows_shared_map_cost_tests PRIVATE -fasynchronous-unwind-tables)
+        endif()
+        add_test(NAME windows_shared_map_cost COMMAND windows_shared_map_cost_tests)
+        set_tests_properties(windows_shared_map_cost PROPERTIES TIMEOUT 120)
+    endif()
+    add_test_executable(guest_kernel_sys_login_tests tests/GuestKernelSysLogin.cpp)
+    target_include_directories(guest_kernel_sys_login_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_login_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_login_tests)
+    add_test(NAME guest_kernel_sys_login COMMAND guest_kernel_sys_login_tests)
+    if(WIN32)
+        set_tests_properties(guest_kernel_sys_login PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
     endif()
 
     if(WIN32)
@@ -517,6 +591,17 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceMove>")
     endif()
 
+    add_test_executable(guest_pthread_create_suspended_tests tests/GuestPthreadCreateSuspended.cpp)
+    target_include_directories(guest_pthread_create_suspended_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_pthread_create_suspended_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_pthread_create_suspended_tests)
+    add_test(NAME guest_pthread_create_suspended COMMAND guest_pthread_create_suspended_tests)
+    set_tests_properties(guest_pthread_create_suspended PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_pthread_create_suspended PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+    endif()
+
     add_test_executable(guest_np_party_tests tests/GuestNpParty.cpp)
     target_include_directories(guest_np_party_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_np_party_tests PRIVATE libSceNpParty libc)
@@ -887,6 +972,18 @@ if(BUILD_TESTING)
     endif()
     add_test(NAME MemoryPoolTest COMMAND MemoryPoolTest)
 
+    add_test_executable(guest_kernel_sys_swapcontext_tests tests/GuestKernelSysSwapcontext.cpp)
+    target_include_directories(guest_kernel_sys_swapcontext_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_compile_options(guest_kernel_sys_swapcontext_tests PRIVATE -fno-omit-frame-pointer)
+    target_link_libraries(guest_kernel_sys_swapcontext_tests PRIVATE libkernel_sys libSceLibcInternal libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_swapcontext_tests)
+    add_test(NAME guest_kernel_sys_swapcontext COMMAND guest_kernel_sys_swapcontext_tests)
+    set_tests_properties(guest_kernel_sys_swapcontext PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_kernel_sys_swapcontext PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+    endif()
+
     add_test_executable(guest_acm_tests tests/GuestAcm.cpp)
     target_include_directories(guest_acm_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_acm_tests PRIVATE libSceAcm libc)

core/libs/SceTypes.hpp

@@ -1502,6 +1502,15 @@ struct SharePlayConnectionInfoA {
     std::int32_t visitorUserId;
 };
 static_assert(sizeof(SharePlayConnectionInfoA) == 72 && offsetof(SharePlayConnectionInfoA, hostAccountId) == 48 && offsetof(SharePlayConnectionInfoA, hostUserId) == 64);
+struct SharePlayConnectionInfo {
+    std::int32_t status;
+    std::int32_t mode;
+    NpOnlineId hostOnlineId;
+    NpOnlineId visitorOnlineId;
+    std::int32_t hostUserId;
+    std::int32_t visitorUserId;
+};
+static_assert(sizeof(SharePlayConnectionInfo) == 56 && offsetof(SharePlayConnectionInfo, hostUserId) == 48);
 struct NpId { NpOnlineId online_id; std::uint8_t opaque[4]; };
 struct NpCreateAsyncRequestParameter { std::uint8_t opaque[64]; };
 struct NpCheckPremiumParameter { std::uint8_t opaque[64]; };

core/libs/nid/include/nid/ElfPatcher.hpp

@@ -22,6 +22,17 @@ struct Elf64_Ehdr {
     std::uint16_t e_shstrndx;
 };
 
+struct Elf64_Phdr {
+    std::uint32_t p_type;
+    std::uint32_t p_flags;
+    std::uint64_t p_offset;
+    std::uint64_t p_vaddr;
+    std::uint64_t p_paddr;
+    std::uint64_t p_filesz;
+    std::uint64_t p_memsz;
+    std::uint64_t p_align;
+};
+
 struct Elf64_Shdr {
     std::uint32_t sh_name;
     std::uint32_t sh_type;
@@ -103,7 +114,7 @@ constexpr std::uint16_t kShnUndef = 0u;
 
 class ElfNidPatcher final : public IBinaryPatcher {
 public:
-    void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const override;
+    void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options = {}) const override;
 };
 
 }

core/libs/nid/include/nid/IBinaryPatcher.hpp

@@ -8,11 +8,16 @@
 
 namespace Nid {
 
+struct PatchOptions {
+    bool executable = false;
+    bool onlySce = false;
+};
+
 class IBinaryPatcher {
 public:
     virtual ~IBinaryPatcher() = default;
 
-    virtual void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const = 0;
+    virtual void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options = {}) const = 0;
 };
 
 }

core/libs/nid/include/nid/MachONidPatcher.hpp

@@ -22,7 +22,7 @@ std::vector<MachOExport> ReadMachOExports(const std::vector<std::uint8_t>& binar
 
 class MachONidPatcher final : public IBinaryPatcher {
 public:
-    void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const override;
+    void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options = {}) const override;
 };
 
 }

core/libs/nid/include/nid/NidResolver.hpp

@@ -1,6 +1,7 @@
 #ifndef NID_NIDRESOLVER_HPP
 #define NID_NIDRESOLVER_HPP
 
+#include <nid/IBinaryPatcher.hpp>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -8,6 +9,10 @@
 
 namespace Nid {
 
+bool IsSceImport(const std::string& name);
+
+std::unordered_map<std::string, std::string> ResolveImportNids(const std::vector<std::string>& names, const std::unordered_set<std::string>& excluded, const PatchOptions& options);
+
 std::string ResolveOneName(const std::string& funcName);
 
 std::unordered_map<std::string, std::string> ResolveNids(const std::vector<std::string>& exportedNames, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports);

core/libs/nid/include/nid/PeNidPatcher.hpp

@@ -39,7 +39,7 @@ struct PeSectionHeader {
 
 class PeNidPatcher final : public IBinaryPatcher {
 public:
-    void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const override;
+    void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options = {}) const override;
 };
 
 }

core/libs/nid/src/ElfNidPatcher.cpp

@@ -354,7 +354,7 @@ std::vector<std::uint32_t> CollectVerneedNameOffsets(const std::vector<std::uint
 
 }
 
-void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const {
+void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options) const {
     using namespace Internal;
 
     if (elf.size() < sizeof(Elf64_Ehdr)) throw std::runtime_error("file too small");
@@ -370,11 +370,13 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
     std::size_t sysvHashOffset = 0u, sysvHashSize = 0u;
     std::uint32_t sysvHashLink = 0u;
     std::size_t versymOffset = 0u;
+    bool hasDynamic = false;
     std::uint32_t dynSymSectionIndex = 0u;
     std::uint32_t gnuHashSymOffset = 0u;
     for (std::uint16_t i = 0u; i < ehdr.e_shnum; ++i) {
         const std::size_t shOffset = static_cast<std::size_t>(ehdr.e_shoff) + i * sizeof(Elf64_Shdr);
         const auto shdr = Read<Elf64_Shdr>(elf, shOffset);
+        if (shdr.sh_type == kShtDynamic) hasDynamic = true;
         if (shdr.sh_type == kShtDynsym) {
             if (shdr.sh_entsize != sizeof(Elf64_Sym) || shdr.sh_size % sizeof(Elf64_Sym) || shdr.sh_link >= ehdr.e_shnum)
                 throw std::runtime_error("invalid ELF dynsym layout");
@@ -398,6 +400,19 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
             throw std::runtime_error("unsupported .gnu.version_d section: symbol version definitions cannot be preserved");
     }
 
+    if (options.executable && dynSymOffset == 0u && !hasDynamic) {
+        if (ehdr.e_phnum) {
+            if (!ehdr.e_phoff || ehdr.e_phentsize != sizeof(Elf64_Phdr) || ehdr.e_phnum == 0xffffu)
+                throw std::runtime_error("unsupported ELF program header layout");
+            RequireElfRange(elf, ehdr.e_phoff, std::uint64_t{ehdr.e_phnum} * sizeof(Elf64_Phdr), "program headers");
+            for (std::uint16_t index = 0u; index < ehdr.e_phnum; ++index) {
+                const auto segment = Read<Elf64_Phdr>(elf, ehdr.e_phoff + index * sizeof(Elf64_Phdr));
+                if (segment.p_type == 2u)
+                    throw std::runtime_error("unsupported ELF PT_DYNAMIC without a .dynsym section");
+            }
+        }
+        return;
+    }
     if (dynSymOffset == 0u) throw std::runtime_error("no .dynsym section");
 
     const std::size_t symCount = dynSymSize / sizeof(Elf64_Sym);
@@ -437,19 +452,26 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
         elf.begin() + static_cast<std::ptrdiff_t>(dynStrOffset + dynStrSize)
     );
 
+    std::vector<std::string> importedNames;
     std::vector<std::string> exportedNames;
     for (std::size_t i = 1u; i < symCount; ++i) {
         const std::size_t symOffset = dynSymOffset + i * sizeof(Elf64_Sym);
         const auto sym = Read<Elf64_Sym>(elf, symOffset);
         if (sym.st_name == 0u) continue;
         const std::uint8_t binding = sym.st_info >> 4u;
-        if (binding == kStbLocal || sym.st_shndx == kShnUndef) continue;
+        if (binding == kStbLocal) continue;
+        if (sym.st_shndx == kShnUndef) {
+            const auto name = ReadCStr(origDynStr, sym.st_name);
+            if (!name.empty()) importedNames.push_back(name);
+            continue;
+        }
         const std::string symName = ReadCStr(origDynStr, sym.st_name);
         if (symName.empty()) continue;
         exportedNames.push_back(symName);
     }
 
-    const auto nidMap = ResolveNids(exportedNames, libraryName, excludedExports);
+    const auto nidMap = options.executable ? std::unordered_map<std::string, std::string>{} : ResolveNids(exportedNames, libraryName, excludedExports);
+    const auto importMap = ResolveImportNids(importedNames, excludedExports, options);
 
     struct SymbolNameUse {
         std::size_t symIndex;
@@ -481,14 +503,11 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
         } else if (binding != kStbLocal && sym.st_shndx == kShnUndef) {
             const bool isUnversioned = versymOffset == 0u ||
                 (Read<std::uint16_t>(elf, versymOffset + i * sizeof(std::uint16_t)) <= 1u);
-            const bool hasNidPostfix = symName.size() >= kNidPostfixLen &&
-                symName.compare(symName.size() - kNidPostfixLen, kNidPostfixLen, kNidPostfix) == 0;
-            const bool hasScePrefix = symName.size() >= 3u &&
-                std::tolower(static_cast<unsigned char>(symName[0])) == 's' &&
-                std::tolower(static_cast<unsigned char>(symName[1])) == 'c' &&
-                std::tolower(static_cast<unsigned char>(symName[2])) == 'e';
-            if (isUnversioned && (hasNidPostfix || hasScePrefix))
-                newValue = ResolveOneName(symName);
+            if (options.executable || isUnversioned) {
+                newValue = importMap.at(symName);
+                if (options.executable && newValue != symName && versymOffset != 0u)
+                    Write(elf, versymOffset + i * sizeof(std::uint16_t), std::uint16_t{1});
+            }
         }
 
         uses.push_back(SymbolNameUse{i, sym.st_name, newValue});
@@ -538,11 +557,69 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
     for (const auto offset : preservedOffsets)
         preservedToNewOffset.emplace_back(offset, valueOffsets.at(ReadCStr(origDynStr, offset)));
 
+    if (newDynStr.size() > dynStrSize && options.executable) {
+        if (ehdr.e_phentsize != sizeof(Elf64_Phdr) || ehdr.e_phnum == 0 || ehdr.e_phnum >= 0xfffdu)
+            throw std::runtime_error("invalid ELF program header table for string growth");
+        RequireElfRange(elf, ehdr.e_phoff, std::uint64_t{ehdr.e_phnum} * sizeof(Elf64_Phdr), "program headers");
+        const auto align = [](std::uint64_t value) {
+            if (value > UINT64_MAX - 4095u) throw std::runtime_error("ELF address overflow");
+            return (value + 4095u) & ~std::uint64_t{4095u};
+        };
+        std::vector<Elf64_Phdr> segments;
+        std::uint64_t virtualEnd = 0;
+        bool hasPhdr = false;
+        for (std::size_t index = 0; index < ehdr.e_phnum; ++index) {
+            const auto segment = Read<Elf64_Phdr>(elf, ehdr.e_phoff + index * sizeof(Elf64_Phdr));
+            segments.push_back(segment);
+            hasPhdr |= segment.p_type == 6;
+            if (segment.p_type != 1) continue;
+            if (segment.p_memsz > UINT64_MAX - segment.p_vaddr) throw std::runtime_error("ELF segment address overflow");
+            virtualEnd = std::max(virtualEnd, segment.p_vaddr + segment.p_memsz);
+        }
+        if (!virtualEnd) throw std::runtime_error("ELF executable has no load segment");
+        if (!hasPhdr) segments.insert(segments.begin(), Elf64_Phdr{6, 4, 0, 0, 0, 0, 0, 8});
+        const auto fileOffset = align(elf.size());
+        const auto address = align(virtualEnd);
+        const auto tableSize = (segments.size() + 1u) * sizeof(Elf64_Phdr);
+        const auto segmentSize = tableSize + newDynStr.size();
+        if (segmentSize > UINT64_MAX - address || segmentSize > SIZE_MAX - fileOffset) throw std::runtime_error("ELF string segment overflow");
+        for (auto& segment : segments) {
+            if (segment.p_type != 6) continue;
+            segment = {6, 4, fileOffset, address, address, tableSize, tableSize, 8};
+        }
+        segments.push_back({1, 4, fileOffset, address, address, segmentSize, segmentSize, 4096});
+        elf.resize(static_cast<std::size_t>(fileOffset + segmentSize), 0);
+        for (std::size_t index = 0; index < segments.size(); ++index)
+            Write(elf, fileOffset + index * sizeof(Elf64_Phdr), segments[index]);
+        Write(elf, offsetof(Elf64_Ehdr, e_phoff), fileOffset);
+        Write(elf, offsetof(Elf64_Ehdr, e_phnum), static_cast<std::uint16_t>(segments.size()));
+        dynStrOffset = static_cast<std::size_t>(fileOffset + tableSize);
+        dynStrSize = newDynStr.size();
+        const auto stringAddress = address + tableSize;
+        const auto sectionOffset = ehdr.e_shoff + dynStrSectionLink * sizeof(Elf64_Shdr);
+        auto strings = Read<Elf64_Shdr>(elf, sectionOffset);
+        strings.sh_offset = dynStrOffset;
+        strings.sh_addr = stringAddress;
+        strings.sh_size = dynStrSize;
+        Write(elf, sectionOffset, strings);
+        bool foundAddress = false;
+        bool foundSize = false;
+        for (std::size_t index = 0; index < ehdr.e_shnum; ++index) {
+            const auto section = Read<Elf64_Shdr>(elf, ehdr.e_shoff + index * sizeof(Elf64_Shdr));
+            if (section.sh_type != kShtDynamic) continue;
+            RequireElfRange(elf, section.sh_offset, section.sh_size, "dynamic");
+            for (std::uint64_t offset = 0; offset + sizeof(Elf64_Dyn) <= section.sh_size; offset += sizeof(Elf64_Dyn)) {
+                auto entry = Read<Elf64_Dyn>(elf, section.sh_offset + offset);
+                if (entry.d_tag == kDtNull) break;
+                if (entry.d_tag == 5) { entry.d_val = stringAddress; foundAddress = true; }
+                if (entry.d_tag == 10) { entry.d_val = dynStrSize; foundSize = true; }
+                Write(elf, section.sh_offset + offset, entry);
+            }
+        }
+        if (!foundAddress || !foundSize) throw std::runtime_error("ELF executable lacks DT_STRTAB or DT_STRSZ");
+    }
     if (newDynStr.size() > dynStrSize) {
-        throw std::runtime_error(
-            "rebuilt .dynstr (" + std::to_string(newDynStr.size()) + " bytes) exceeds original section size (" +
-            std::to_string(dynStrSize) + " bytes) - relinking with a larger section is required"
-        );
+        throw std::runtime_error("rebuilt .dynstr exceeds original section size; relinking with a larger section is required");
     }
     newDynStr.resize(dynStrSize, 0u);
 

core/libs/nid/src/MachONidPatcher.cpp

@@ -341,15 +341,6 @@ std::string StripUnderscore(const std::string& name) {
     return !name.empty() && name[0] == '_' ? name.substr(1) : name;
 }
 
-bool IsGuestFacingImport(const std::string& name) {
-    using namespace Internal;
-    const bool hasNidPostfix = name.size() >= kNidPostfixLen && name.compare(name.size() - kNidPostfixLen, kNidPostfixLen, kNidPostfix) == 0;
-    const bool hasScePrefix = name.size() >= 3u &&
-        std::tolower(static_cast<unsigned char>(name[0])) == 's' &&
-        std::tolower(static_cast<unsigned char>(name[1])) == 'c' &&
-        std::tolower(static_cast<unsigned char>(name[2])) == 'e';
-    return hasNidPostfix || hasScePrefix;
-}
 
 std::vector<bool> DylibOrdinalsArePrx(const std::vector<std::uint8_t>& binary, const Image& image) {
     std::vector<bool> result;
@@ -364,11 +355,48 @@ std::vector<bool> DylibOrdinalsArePrx(const std::vector<std::uint8_t>& binary, c
     return result;
 }
 
-void PatchChainedImports(std::vector<std::uint8_t>& binary, const Image& image) {
+void WriteLinkeditData(std::vector<std::uint8_t>& binary, const Image& image, const TrieLocation& location, const std::vector<std::uint8_t>& trie) {
+    const auto oldOffset = Read<std::uint32_t>(binary, location.OffsetField);
+    const auto oldSize = Read<std::uint32_t>(binary, location.SizeField);
+    if (trie.size() <= oldSize) {
+        std::copy(trie.begin(), trie.end(), binary.begin() + oldOffset);
+        std::fill(binary.begin() + oldOffset + static_cast<std::ptrdiff_t>(trie.size()), binary.begin() + oldOffset + oldSize, 0);
+        Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
+        return;
+    }
+    std::optional<std::size_t> linkedit;
+    for (const auto offset : image.Commands) {
+        const auto cmd = Read<LoadCommand>(binary, offset).Cmd;
+        if (cmd == kLcCodeSignature) throw std::runtime_error("signed Mach-O images cannot grow linkedit data");
+        if (cmd != kLcSegment64) continue;
+        const auto segment = Read<SegmentCommand64>(binary, offset);
+        if (std::strncmp(segment.Name, "__LINKEDIT", sizeof(segment.Name)) == 0) linkedit = offset;
+    }
+    if (!linkedit) throw std::runtime_error("Mach-O image has no __LINKEDIT segment");
+    auto segment = Read<SegmentCommand64>(binary, *linkedit);
+    if (segment.FileOffset + segment.FileSize != binary.size()) throw std::runtime_error("__LINKEDIT does not end the file");
+    const auto newOffset = (binary.size() + 7) & ~std::size_t{7};
+    if (newOffset + trie.size() > 0xffffffffu) throw std::runtime_error("linkedit data offset overflow");
+    binary.resize(newOffset, 0);
+    binary.insert(binary.end(), trie.begin(), trie.end());
+    std::fill(binary.begin() + oldOffset, binary.begin() + oldOffset + oldSize, 0);
+    segment.FileSize = binary.size() - segment.FileOffset;
+    segment.VmSize = std::max(segment.VmSize, (segment.FileSize + kPageSize - 1) & ~(kPageSize - 1));
+    Write(binary, *linkedit, segment);
+    Write(binary, location.OffsetField, static_cast<std::uint32_t>(newOffset));
+    Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
+}
+
+void PatchChainedImports(std::vector<std::uint8_t>& binary, const Image& image, const std::unordered_set<std::string>& excluded, const PatchOptions& options) {
+    std::size_t commandOffset = 0;
     std::optional<LinkeditDataCommand> fixups;
     for (const auto offset : image.Commands) {
         const auto cmd = Read<LoadCommand>(binary, offset).Cmd;
-        if (cmd == kLcDyldChainedFixups) fixups = Read<LinkeditDataCommand>(binary, offset);
+        if (cmd == kLcDyldChainedFixups) {
+            fixups = Read<LinkeditDataCommand>(binary, offset);
+            commandOffset = offset;
+        }
+        if (options.executable && cmd == kLcCodeSignature) throw std::runtime_error("remove the Mach-O code signature before patching and sign again afterwards");
         if ((cmd == kLcDyldInfo || cmd == kLcDyldInfoOnly)) {
             const auto info = Read<DyldInfoCommand>(binary, offset);
             if (info.BindSize != 0 || info.LazyBindSize != 0 || info.WeakBindSize != 0) throw std::runtime_error("Mach-O bind opcodes are not supported; link with chained fixups");
@@ -400,61 +428,72 @@ void PatchChainedImports(std::vector<std::uint8_t>& binary, const Image& image)
             imports.push_back({raw >> 9, raw & 0xff});
         }
     }
-    std::set<std::uint32_t> nameOffsets;
-    for (const auto& item : imports) nameOffsets.insert(item.NameOffset);
-    std::set<std::uint32_t> rewritten;
+    std::vector<std::string> names;
+    std::vector<std::string> originalNames;
+    std::vector<bool> selected;
     for (const auto& item : imports) {
-        const bool special = header.ImportsFormat == kImportFormatAddend64 ? item.Ordinal >= 0xfff0 : item.Ordinal >= 0xf0;
-        if (item.Ordinal == 0 || special || item.Ordinal > prx.size() || !prx[item.Ordinal - 1]) continue;
-        if (rewritten.contains(item.NameOffset)) continue;
         const auto at = symbols + item.NameOffset;
         if (at >= symbolsEnd) throw std::runtime_error("chained import name out of bounds");
         const auto name = Internal::ReadCStr(binary, at);
         if (at + name.size() >= symbolsEnd) throw std::runtime_error("unterminated chained import name");
         const auto bare = StripUnderscore(name);
-        if (bare == name || !IsGuestFacingImport(bare)) continue;
-        const auto patched = "_" + ResolveOneName(bare);
-        if (patched.size() > name.size()) throw std::runtime_error("import NID longer than original name: " + name);
-        const auto inside = nameOffsets.upper_bound(item.NameOffset);
-        if (inside != nameOffsets.end() && *inside <= item.NameOffset + name.size()) throw std::runtime_error("chained import names share storage: " + name);
-        std::memcpy(binary.data() + at, patched.data(), patched.size());
-        std::fill(binary.begin() + static_cast<std::ptrdiff_t>(at + patched.size()), binary.begin() + static_cast<std::ptrdiff_t>(at + name.size() + 1), 0);
-        rewritten.insert(item.NameOffset);
+        const bool special = header.ImportsFormat == kImportFormatAddend64 ? item.Ordinal >= 0xfff0 : item.Ordinal >= 0xf0;
+        const bool eligible = options.executable || (bare != name && item.Ordinal != 0 && !special && item.Ordinal <= prx.size() && prx[item.Ordinal - 1]);
+        originalNames.push_back(name);
+        selected.push_back(eligible);
+        if (eligible) names.push_back(bare);
     }
-}
-
-void WriteTrie(std::vector<std::uint8_t>& binary, const Image& image, const TrieLocation& location, const std::vector<std::uint8_t>& trie) {
-    const auto oldOffset = Read<std::uint32_t>(binary, location.OffsetField);
-    const auto oldSize = Read<std::uint32_t>(binary, location.SizeField);
-    if (trie.size() <= oldSize) {
-        std::copy(trie.begin(), trie.end(), binary.begin() + oldOffset);
-        std::fill(binary.begin() + oldOffset + static_cast<std::ptrdiff_t>(trie.size()), binary.begin() + oldOffset + oldSize, 0);
-        Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
+    const auto mapping = ResolveImportNids(names, excluded, options);
+    if (!options.executable) {
+        std::set<std::uint32_t> nameOffsets;
+        for (const auto& item : imports) nameOffsets.insert(item.NameOffset);
+        std::set<std::uint32_t> rewritten;
+        for (const auto& item : imports) {
+            const bool special = header.ImportsFormat == kImportFormatAddend64 ? item.Ordinal >= 0xfff0 : item.Ordinal >= 0xf0;
+            if (item.Ordinal == 0 || special || item.Ordinal > prx.size() || !prx[item.Ordinal - 1]) continue;
+            if (rewritten.contains(item.NameOffset)) continue;
+            const auto at = symbols + item.NameOffset;
+            if (at >= symbolsEnd) throw std::runtime_error("chained import name out of bounds");
+            const auto name = Internal::ReadCStr(binary, at);
+            if (at + name.size() >= symbolsEnd) throw std::runtime_error("unterminated chained import name");
+            const auto bare = StripUnderscore(name);
+            if (bare == name) continue;
+            const auto patched = "_" + mapping.at(bare);
+            if (patched.size() > name.size()) throw std::runtime_error("import NID longer than original name: " + name);
+            const auto inside = nameOffsets.upper_bound(item.NameOffset);
+            if (inside != nameOffsets.end() && *inside <= item.NameOffset + name.size()) throw std::runtime_error("chained import names share storage: " + name);
+            std::memcpy(binary.data() + at, patched.data(), patched.size());
+            std::fill(binary.begin() + static_cast<std::ptrdiff_t>(at + patched.size()), binary.begin() + static_cast<std::ptrdiff_t>(at + name.size() + 1), 0);
+            rewritten.insert(item.NameOffset);
+        }
         return;
     }
-    std::optional<std::size_t> linkedit;
-    for (const auto offset : image.Commands) {
-        const auto cmd = Read<LoadCommand>(binary, offset).Cmd;
-        if (cmd == kLcCodeSignature) throw std::runtime_error("signed Mach-O images cannot grow their export trie");
-        if (cmd != kLcSegment64) continue;
-        const auto segment = Read<SegmentCommand64>(binary, offset);
-        if (std::strncmp(segment.Name, "__LINKEDIT", sizeof(segment.Name)) == 0) linkedit = offset;
+    if (imports.empty()) return;
+    std::vector<std::uint8_t> data(binary.begin() + base, binary.begin() + symbolsEnd);
+    const auto newSymbols = data.size();
+    if (newSymbols > 0xffffffffu) throw std::runtime_error("chained symbol offset overflow");
+    Write(data, offsetof(ChainedFixupsHeader, SymbolsOffset), static_cast<std::uint32_t>(newSymbols));
+    for (std::size_t index = 0; index < imports.size(); ++index) {
+        const auto& original = originalNames[index];
+        const auto name = selected[index] ? (original.starts_with("_") ? "_" : "") + mapping.at(StripUnderscore(original)) : original;
+        const auto offset = data.size() - newSymbols;
+        const auto at = header.ImportsOffset + index * entrySize;
+        if (header.ImportsFormat == kImportFormatAddend64) {
+            if (offset > 0xffffffffu) throw std::runtime_error("chained symbol offset overflow");
+            const auto raw = Read<std::uint64_t>(data, at);
+            Write(data, at, (raw & 0xffffffffu) | (std::uint64_t{offset} << 32));
+        } else {
+            if (offset > 0x7fffffu) throw std::runtime_error("chained symbol offset overflow");
+            const auto raw = Read<std::uint32_t>(data, at);
+            Write(data, at, (raw & 0x1ffu) | (static_cast<std::uint32_t>(offset) << 9));
+        }
+        data.insert(data.end(), name.begin(), name.end());
+        data.push_back(0);
     }
-    if (!linkedit) throw std::runtime_error("Mach-O image has no __LINKEDIT segment");
-    auto segment = Read<SegmentCommand64>(binary, *linkedit);
-    if (segment.FileOffset + segment.FileSize != binary.size()) throw std::runtime_error("__LINKEDIT does not end the file");
-    const auto newOffset = (binary.size() + 7) & ~std::size_t{7};
-    if (newOffset + trie.size() > 0xffffffffu) throw std::runtime_error("export trie offset overflow");
-    binary.resize(newOffset, 0);
-    binary.insert(binary.end(), trie.begin(), trie.end());
-    std::fill(binary.begin() + oldOffset, binary.begin() + oldOffset + oldSize, 0);
-    segment.FileSize = binary.size() - segment.FileOffset;
-    segment.VmSize = std::max(segment.VmSize, (segment.FileSize + kPageSize - 1) & ~(kPageSize - 1));
-    Write(binary, *linkedit, segment);
-    Write(binary, location.OffsetField, static_cast<std::uint32_t>(newOffset));
-    Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
+    WriteLinkeditData(binary, image, {commandOffset + 8, commandOffset + 12}, data);
 }
 
+
 }
 
 std::vector<MachOExport> ReadMachOExports(const std::vector<std::uint8_t>& binary) {
@@ -464,8 +503,12 @@ std::vector<MachOExport> ReadMachOExports(const std::vector<std::uint8_t>& binar
     return ParseTrie(binary, Read<std::uint32_t>(binary, location->OffsetField), Read<std::uint32_t>(binary, location->SizeField));
 }
 
-void MachONidPatcher::PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const {
+void MachONidPatcher::PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options) const {
     const auto image = ParseImage(binary);
+    if (options.executable) {
+        PatchChainedImports(binary, image, excludedExports, options);
+        return;
+    }
     const auto location = FindTrie(binary, image);
     if (!location) throw std::runtime_error("no export trie");
     auto exports = ParseTrie(binary, Read<std::uint32_t>(binary, location->OffsetField), Read<std::uint32_t>(binary, location->SizeField));
@@ -486,8 +529,8 @@ void MachONidPatcher::PatchNids(std::vector<std::uint8_t>& binary, const std::st
         if (found == nidMap.end()) throw std::runtime_error("symbol not in nid map: " + name);
         exports[index].Name = "_" + found->second;
     }
-    WriteTrie(binary, image, *location, TrieBuilder(exports).Build());
-    PatchChainedImports(binary, ParseImage(binary));
+    WriteLinkeditData(binary, image, *location, TrieBuilder(exports).Build());
+    PatchChainedImports(binary, ParseImage(binary), excludedExports, options);
 }
 
 }

core/libs/nid/src/NidResolver.cpp

@@ -3,21 +3,32 @@
 #include <nid/NidCompute.hpp>
 #include <nid/NidPatcherUtils.hpp>
 #include <cstdint>
+#include <cctype>
 #include <stdexcept>
 #include <unordered_set>
 #include <vector>
 
 namespace Nid {
 
+bool IsSceImport(const std::string& name) {
+    return name.ends_with(Internal::kNidPostfix) || (name.size() >= 3 &&
+        std::tolower(static_cast<unsigned char>(name[0])) == 's' &&
+        std::tolower(static_cast<unsigned char>(name[1])) == 'c' &&
+        std::tolower(static_cast<unsigned char>(name[2])) == 'e');
+}
+
+std::unordered_map<std::string, std::string> ResolveImportNids(const std::vector<std::string>& names, const std::unordered_set<std::string>& excluded, const PatchOptions& options) {
+    const std::unordered_set<std::string> unique(names.begin(), names.end());
+    auto result = ResolveNids(std::vector<std::string>(unique.begin(), unique.end()), "", excluded);
+    for (const auto& name : unique) {
+        if ((!options.executable || options.onlySce) && !IsSceImport(name)) result.at(name) = name;
+        else if (!options.executable) result.at(name) = ResolveOneName(name);
+    }
+    return result;
+}
+
 std::string ResolveOneName(const std::string& funcName) {
-    using namespace Internal;
-
-    if (IsNidNoPatchCut(funcName))
-        return StripNidNoPatchCut(funcName);
-    if (IsNidNoPatch(funcName))
-        return funcName;
-
-    return ComputeNid(StripNidPostfix(funcName), "");
+    return ResolveNids({funcName}, "", {}).at(funcName);
 }
 
 std::unordered_map<std::string, std::string> ResolveNids(const std::vector<std::string>& exportedNames, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) {

core/libs/nid/src/PeNidPatcher.cpp

@@ -36,7 +36,7 @@ std::size_t RvaToOffset(const std::vector<std::uint8_t>& pe, std::uint32_t rva,
 
 }
 
-void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const {
+void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options) const {
     using namespace Internal;
 
     if (pe.size() < 0x40) throw std::runtime_error("file too small");
@@ -54,7 +54,7 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
     const std::size_t sectionTableOffset = coffHeaderOffset + 20u + sizeOfOptionalHeader;
     const auto symbolTable = Read<std::uint32_t>(pe, coffHeaderOffset + 8u);
     const auto symbolCount = Read<std::uint32_t>(pe, coffHeaderOffset + 12u);
-    for (std::uint16_t i = 0; i < numberOfSections; ++i) {
+    for (std::uint16_t i = 0; !options.executable && i < numberOfSections; ++i) {
         const auto offset = sectionTableOffset + i * sizeof(PeSectionHeader);
         auto section = Read<PeSectionHeader>(pe, offset);
         if (pe[offset] != '/' || !symbolTable) continue;
@@ -79,159 +79,162 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
         throw std::runtime_error("unsupported optional header magic");
     }
 
-    const auto exportDir = Read<PeDataDirectory>(pe, dataDirectoryOffset);
-    if (exportDir.VirtualAddress == 0u) throw std::runtime_error("no export directory");
+    if (!options.executable) {
+        const auto exportDir = Read<PeDataDirectory>(pe, dataDirectoryOffset);
+        if (exportDir.VirtualAddress == 0u) throw std::runtime_error("no export directory");
 
-    const std::size_t exportDirOffset = RvaToOffset(pe, exportDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
-    const auto exportTable = Read<PeExportDirectory>(pe, exportDirOffset);
+        const std::size_t exportDirOffset = RvaToOffset(pe, exportDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+        const auto exportTable = Read<PeExportDirectory>(pe, exportDirOffset);
 
-    if (exportTable.NumberOfNames == 0u) throw std::runtime_error("no exported names");
+        if (exportTable.NumberOfNames == 0u) throw std::runtime_error("no exported names");
 
-    const std::size_t namesArrayOffset = RvaToOffset(pe, exportTable.AddressOfNames, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
-    const std::size_t ordinalsArrayOffset = RvaToOffset(pe, exportTable.AddressOfNameOrdinals, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+        const std::size_t namesArrayOffset = RvaToOffset(pe, exportTable.AddressOfNames, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+        const std::size_t ordinalsArrayOffset = RvaToOffset(pe, exportTable.AddressOfNameOrdinals, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
 
-    std::vector<std::uint32_t> nameRvas(exportTable.NumberOfNames);
-    std::vector<std::string> names(exportTable.NumberOfNames);
-    for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
-        const auto nameRva = Read<std::uint32_t>(pe, namesArrayOffset + i * 4u);
-        const std::size_t nameOffset = RvaToOffset(pe, nameRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
-        const std::string name = ReadCStr(pe, nameOffset);
-        if (name.empty()) throw std::runtime_error("empty exported name");
-        nameRvas[i] = nameRva;
-        names[i] = name;
-    }
-
-    const auto nidMap = ResolveNids(names, libraryName, excludedExports);
-
-    const std::size_t edataSectionOffset = FindSectionOffsetByRva(pe, exportDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
-    const auto edataSection = Read<PeSectionHeader>(pe, edataSectionOffset);
-
-    const auto fileAlignment = Read<std::uint32_t>(pe, optionalHeaderOffset + 36u);
-    if (fileAlignment == 0u) throw std::runtime_error("invalid file alignment");
-
-    std::uint32_t stringsRegionStart = std::numeric_limits<std::uint32_t>::max();
-    for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
-        const std::uint32_t rvaFromSection = nameRvas[i] - edataSection.VirtualAddress;
-        if (rvaFromSection < stringsRegionStart) stringsRegionStart = rvaFromSection;
-    }
-    if (stringsRegionStart == std::numeric_limits<std::uint32_t>::max())
-        throw std::runtime_error("could not determine strings region start");
-
-    std::uint32_t rawLimit = ((edataSection.SizeOfRawData + fileAlignment - 1u) / fileAlignment) * fileAlignment;
-    const auto dllName = ReadCStr(pe, RvaToOffset(pe, exportTable.Name, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
-    const bool repackDllName = exportTable.Name >= edataSection.VirtualAddress &&
-        exportTable.Name - edataSection.VirtualAddress < rawLimit;
-    if (repackDllName) stringsRegionStart = std::min(stringsRegionStart, exportTable.Name - edataSection.VirtualAddress);
-    struct Forwarder {
-        std::size_t EntryOffset;
-        std::string Target;
-    };
-    std::vector<Forwarder> forwarders;
-    std::vector<std::uint32_t> directRvas;
-    const auto functionsArrayOffset = RvaToOffset(pe, exportTable.AddressOfFunctions, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
-    for (std::uint32_t index = 0u; index < exportTable.NumberOfFunctions; ++index) {
-        const auto entryOffset = functionsArrayOffset + index * sizeof(std::uint32_t);
-        const auto rva = Read<std::uint32_t>(pe, entryOffset);
-        if (rva < exportDir.VirtualAddress || std::uint64_t{rva} - exportDir.VirtualAddress >= exportDir.Size) {
-            directRvas.push_back(rva);
-            continue;
+        std::vector<std::uint32_t> nameRvas(exportTable.NumberOfNames);
+        std::vector<std::string> names(exportTable.NumberOfNames);
+        for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
+            const auto nameRva = Read<std::uint32_t>(pe, namesArrayOffset + i * 4u);
+            const std::size_t nameOffset = RvaToOffset(pe, nameRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+            const std::string name = ReadCStr(pe, nameOffset);
+            if (name.empty()) throw std::runtime_error("empty exported name");
+            nameRvas[i] = nameRva;
+            names[i] = name;
         }
-        const auto target = ReadCStr(pe, RvaToOffset(pe, rva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
-        if (target.size() + 1u > exportDir.Size - (rva - exportDir.VirtualAddress))
-            throw std::runtime_error("forwarder string outside export directory");
-        forwarders.push_back({entryOffset, target});
-        stringsRegionStart = std::min(stringsRegionStart, rva - edataSection.VirtualAddress);
-    }
-    for (const auto rva : directRvas) {
-        if (rva >= edataSection.VirtualAddress && rva - edataSection.VirtualAddress >= stringsRegionStart)
-            rawLimit = std::min(rawLimit, rva - edataSection.VirtualAddress);
-    }
-    const auto reserveTable = [&](std::uint32_t rva, std::uint64_t size) {
-        if (rva < edataSection.VirtualAddress || rva - edataSection.VirtualAddress >= rawLimit) return;
-        const auto end = std::uint64_t{rva - edataSection.VirtualAddress} + size;
-        if (end > rawLimit) throw std::runtime_error("export table outside edata section");
-        stringsRegionStart = std::max(stringsRegionStart, static_cast<std::uint32_t>(end));
-    };
-    reserveTable(exportDir.VirtualAddress, sizeof(PeExportDirectory));
-    reserveTable(exportTable.AddressOfFunctions, std::uint64_t{exportTable.NumberOfFunctions} * 4u);
-    reserveTable(exportTable.AddressOfNames, std::uint64_t{exportTable.NumberOfNames} * 4u);
-    reserveTable(exportTable.AddressOfNameOrdinals, std::uint64_t{exportTable.NumberOfNames} * 2u);
 
-    std::vector<std::string> finalNames(exportTable.NumberOfNames);
-    for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
-        const auto it = nidMap.find(names[i]);
-        if (it == nidMap.end()) throw std::runtime_error("symbol not in nid map: " + names[i]);
-        finalNames[i] = it->second;
-    }
+        const auto nidMap = ResolveNids(names, libraryName, excludedExports);
 
-    std::vector<std::uint8_t> strings;
-    std::unordered_map<std::string, std::uint32_t> stringRvas;
-    const auto storeString = [&](const std::string& name) {
-        const auto existing = stringRvas.find(name);
-        if (existing != stringRvas.end()) return existing->second;
-        const auto rva = edataSection.VirtualAddress + stringsRegionStart + static_cast<std::uint32_t>(strings.size());
-        stringRvas.emplace(name, rva);
-        strings.insert(strings.end(), name.begin(), name.end());
-        strings.push_back(0u);
-        return rva;
-    };
-    std::vector<std::uint32_t> forwarderRvas;
-    for (const auto& forwarder : forwarders) forwarderRvas.push_back(storeString(forwarder.Target));
-    if (!forwarders.empty() && (std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart < exportDir.VirtualAddress ||
-        std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart + strings.size() > std::uint64_t{exportDir.VirtualAddress} + exportDir.Size))
-        throw std::runtime_error("not enough export directory space to repack forwarders");
-    const auto dllNameRva = repackDllName ? storeString(dllName) : exportTable.Name;
-    std::vector<std::uint32_t> finalNameRvas;
-    for (const auto& name : finalNames) finalNameRvas.push_back(storeString(name));
-    const auto requiredSize = std::uint64_t{stringsRegionStart} + strings.size();
-    if (requiredSize > rawLimit)
-        throw std::runtime_error("not enough raw space in edata section to repack export names");
-    const auto directoryEnd = std::max(std::uint64_t{exportDir.VirtualAddress} + exportDir.Size,
-        std::uint64_t{edataSection.VirtualAddress} + requiredSize);
-    for (const auto rva : directRvas) {
-        if (rva != 0u && rva >= exportDir.VirtualAddress && std::uint64_t{rva} < directoryEnd)
-            throw std::runtime_error("repacked export directory would cover a direct export");
-    }
-    const auto directorySize = directoryEnd - exportDir.VirtualAddress;
-    if (directorySize > std::numeric_limits<std::uint32_t>::max()) throw std::runtime_error("export directory too large");
-    const auto stringFileOffset = static_cast<std::size_t>(edataSection.PointerToRawData) + stringsRegionStart;
-    if (stringFileOffset > pe.size() || strings.size() > pe.size() - stringFileOffset)
-        throw std::runtime_error("repack write out of file bounds");
-    std::memcpy(pe.data() + stringFileOffset, strings.data(), strings.size());
-    for (std::size_t index = 0u; index < finalNameRvas.size(); ++index)
-        Write(pe, namesArrayOffset + index * 4u, finalNameRvas[index]);
-    for (std::size_t index = 0u; index < forwarders.size(); ++index)
-        Write(pe, forwarders[index].EntryOffset, forwarderRvas[index]);
-    Write(pe, exportDirOffset + offsetof(PeExportDirectory, Name), dllNameRva);
-    Write(pe, dataDirectoryOffset + offsetof(PeDataDirectory, Size), static_cast<std::uint32_t>(directorySize));
+        const std::size_t edataSectionOffset = FindSectionOffsetByRva(pe, exportDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+        const auto edataSection = Read<PeSectionHeader>(pe, edataSectionOffset);
 
-    const std::uint32_t newVirtualSize = std::max(edataSection.VirtualSize, static_cast<std::uint32_t>(requiredSize));
-    auto patchedSection = edataSection;
-    patchedSection.VirtualSize = newVirtualSize;
-    Write(pe, edataSectionOffset, patchedSection);
+        const auto fileAlignment = Read<std::uint32_t>(pe, optionalHeaderOffset + 36u);
+        if (fileAlignment == 0u) throw std::runtime_error("invalid file alignment");
 
-    struct ExportName {
-        std::string Name;
-        std::uint32_t Rva;
-        std::uint16_t Ordinal;
-    };
+        std::uint32_t stringsRegionStart = std::numeric_limits<std::uint32_t>::max();
+        for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
+            const std::uint32_t rvaFromSection = nameRvas[i] - edataSection.VirtualAddress;
+            if (rvaFromSection < stringsRegionStart) stringsRegionStart = rvaFromSection;
+        }
+        if (stringsRegionStart == std::numeric_limits<std::uint32_t>::max())
+            throw std::runtime_error("could not determine strings region start");
+
+        std::uint32_t rawLimit = ((edataSection.SizeOfRawData + fileAlignment - 1u) / fileAlignment) * fileAlignment;
+        const auto dllName = ReadCStr(pe, RvaToOffset(pe, exportTable.Name, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
+        const bool repackDllName = exportTable.Name >= edataSection.VirtualAddress &&
+            exportTable.Name - edataSection.VirtualAddress < rawLimit;
+        if (repackDllName) stringsRegionStart = std::min(stringsRegionStart, exportTable.Name - edataSection.VirtualAddress);
+        struct Forwarder {
+            std::size_t EntryOffset;
+            std::string Target;
+        };
+        std::vector<Forwarder> forwarders;
+        std::vector<std::uint32_t> directRvas;
+        const auto functionsArrayOffset = RvaToOffset(pe, exportTable.AddressOfFunctions, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+        for (std::uint32_t index = 0u; index < exportTable.NumberOfFunctions; ++index) {
+            const auto entryOffset = functionsArrayOffset + index * sizeof(std::uint32_t);
+            const auto rva = Read<std::uint32_t>(pe, entryOffset);
+            if (rva < exportDir.VirtualAddress || std::uint64_t{rva} - exportDir.VirtualAddress >= exportDir.Size) {
+                directRvas.push_back(rva);
+                continue;
+            }
+            const auto target = ReadCStr(pe, RvaToOffset(pe, rva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
+            if (target.size() + 1u > exportDir.Size - (rva - exportDir.VirtualAddress))
+                throw std::runtime_error("forwarder string outside export directory");
+            forwarders.push_back({entryOffset, target});
+            stringsRegionStart = std::min(stringsRegionStart, rva - edataSection.VirtualAddress);
+        }
+        for (const auto rva : directRvas) {
+            if (rva >= edataSection.VirtualAddress && rva - edataSection.VirtualAddress >= stringsRegionStart)
+                rawLimit = std::min(rawLimit, rva - edataSection.VirtualAddress);
+        }
+        const auto reserveTable = [&](std::uint32_t rva, std::uint64_t size) {
+            if (rva < edataSection.VirtualAddress || rva - edataSection.VirtualAddress >= rawLimit) return;
+            const auto end = std::uint64_t{rva - edataSection.VirtualAddress} + size;
+            if (end > rawLimit) throw std::runtime_error("export table outside edata section");
+            stringsRegionStart = std::max(stringsRegionStart, static_cast<std::uint32_t>(end));
+        };
+        reserveTable(exportDir.VirtualAddress, sizeof(PeExportDirectory));
+        reserveTable(exportTable.AddressOfFunctions, std::uint64_t{exportTable.NumberOfFunctions} * 4u);
+        reserveTable(exportTable.AddressOfNames, std::uint64_t{exportTable.NumberOfNames} * 4u);
+        reserveTable(exportTable.AddressOfNameOrdinals, std::uint64_t{exportTable.NumberOfNames} * 2u);
+
+        std::vector<std::string> finalNames(exportTable.NumberOfNames);
+        for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
+            const auto it = nidMap.find(names[i]);
+            if (it == nidMap.end()) throw std::runtime_error("symbol not in nid map: " + names[i]);
+            finalNames[i] = it->second;
+        }
+
+        std::vector<std::uint8_t> strings;
+        std::unordered_map<std::string, std::uint32_t> stringRvas;
+        const auto storeString = [&](const std::string& name) {
+            const auto existing = stringRvas.find(name);
+            if (existing != stringRvas.end()) return existing->second;
+            const auto rva = edataSection.VirtualAddress + stringsRegionStart + static_cast<std::uint32_t>(strings.size());
+            stringRvas.emplace(name, rva);
+            strings.insert(strings.end(), name.begin(), name.end());
+            strings.push_back(0u);
+            return rva;
+        };
+        std::vector<std::uint32_t> forwarderRvas;
+        for (const auto& forwarder : forwarders) forwarderRvas.push_back(storeString(forwarder.Target));
+        if (!forwarders.empty() && (std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart < exportDir.VirtualAddress ||
+            std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart + strings.size() > std::uint64_t{exportDir.VirtualAddress} + exportDir.Size))
+            throw std::runtime_error("not enough export directory space to repack forwarders");
+        const auto dllNameRva = repackDllName ? storeString(dllName) : exportTable.Name;
+        std::vector<std::uint32_t> finalNameRvas;
+        for (const auto& name : finalNames) finalNameRvas.push_back(storeString(name));
+        const auto requiredSize = std::uint64_t{stringsRegionStart} + strings.size();
+        if (requiredSize > rawLimit)
+            throw std::runtime_error("not enough raw space in edata section to repack export names");
+        const auto directoryEnd = std::max(std::uint64_t{exportDir.VirtualAddress} + exportDir.Size,
+            std::uint64_t{edataSection.VirtualAddress} + requiredSize);
+        for (const auto rva : directRvas) {
+            if (rva != 0u && rva >= exportDir.VirtualAddress && std::uint64_t{rva} < directoryEnd)
+                throw std::runtime_error("repacked export directory would cover a direct export");
+        }
+        const auto directorySize = directoryEnd - exportDir.VirtualAddress;
+        if (directorySize > std::numeric_limits<std::uint32_t>::max()) throw std::runtime_error("export directory too large");
+        const auto stringFileOffset = static_cast<std::size_t>(edataSection.PointerToRawData) + stringsRegionStart;
+        if (stringFileOffset > pe.size() || strings.size() > pe.size() - stringFileOffset)
+            throw std::runtime_error("repack write out of file bounds");
+        std::memcpy(pe.data() + stringFileOffset, strings.data(), strings.size());
+        for (std::size_t index = 0u; index < finalNameRvas.size(); ++index)
+            Write(pe, namesArrayOffset + index * 4u, finalNameRvas[index]);
+        for (std::size_t index = 0u; index < forwarders.size(); ++index)
+            Write(pe, forwarders[index].EntryOffset, forwarderRvas[index]);
+        Write(pe, exportDirOffset + offsetof(PeExportDirectory, Name), dllNameRva);
+        Write(pe, dataDirectoryOffset + offsetof(PeDataDirectory, Size), static_cast<std::uint32_t>(directorySize));
+
+        const std::uint32_t newVirtualSize = std::max(edataSection.VirtualSize, static_cast<std::uint32_t>(requiredSize));
+        auto patchedSection = edataSection;
+        patchedSection.VirtualSize = newVirtualSize;
+        Write(pe, edataSectionOffset, patchedSection);
+
+        struct ExportName {
+            std::string Name;
+            std::uint32_t Rva;
+            std::uint16_t Ordinal;
+        };
+
+        std::vector<ExportName> exportNames;
+        exportNames.reserve(exportTable.NumberOfNames);
+        for (std::size_t i = 0; i < exportTable.NumberOfNames; ++i) {
+            const auto nameRva = Read<std::uint32_t>(pe, namesArrayOffset + i * sizeof(std::uint32_t));
+            const auto ordinal = Read<std::uint16_t>(pe, ordinalsArrayOffset + i * sizeof(std::uint16_t));
+            if (ordinal >= exportTable.NumberOfFunctions)
+                throw std::runtime_error("export name ordinal out of bounds");
+            const auto nameOffset = RvaToOffset(pe, nameRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+            exportNames.push_back({ReadCStr(pe, nameOffset), nameRva, ordinal});
+        }
+        std::sort(exportNames.begin(), exportNames.end(), [](const ExportName& left, const ExportName& right) { return left.Name < right.Name; });
+        for (std::size_t i = 0; i < exportNames.size(); ++i) {
+            if (i != 0 && exportNames[i - 1].Name == exportNames[i].Name)
+                throw std::runtime_error("duplicate patched export name: " + exportNames[i].Name);
+            Write(pe, namesArrayOffset + i * sizeof(std::uint32_t), exportNames[i].Rva);
+            Write(pe, ordinalsArrayOffset + i * sizeof(std::uint16_t), exportNames[i].Ordinal);
+        }
 
-    std::vector<ExportName> exportNames;
-    exportNames.reserve(exportTable.NumberOfNames);
-    for (std::size_t i = 0; i < exportTable.NumberOfNames; ++i) {
-        const auto nameRva = Read<std::uint32_t>(pe, namesArrayOffset + i * sizeof(std::uint32_t));
-        const auto ordinal = Read<std::uint16_t>(pe, ordinalsArrayOffset + i * sizeof(std::uint16_t));
-        if (ordinal >= exportTable.NumberOfFunctions)
-            throw std::runtime_error("export name ordinal out of bounds");
-        const auto nameOffset = RvaToOffset(pe, nameRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
-        exportNames.push_back({ReadCStr(pe, nameOffset), nameRva, ordinal});
-    }
-    std::sort(exportNames.begin(), exportNames.end(), [](const ExportName& left, const ExportName& right) { return left.Name < right.Name; });
-    for (std::size_t i = 0; i < exportNames.size(); ++i) {
-        if (i != 0 && exportNames[i - 1].Name == exportNames[i].Name)
-            throw std::runtime_error("duplicate patched export name: " + exportNames[i].Name);
-        Write(pe, namesArrayOffset + i * sizeof(std::uint32_t), exportNames[i].Rva);
-        Write(pe, ordinalsArrayOffset + i * sizeof(std::uint16_t), exportNames[i].Ordinal);
     }
 
     constexpr std::size_t kImportDirIndex = 1u;
@@ -250,6 +253,20 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
         std::uint32_t FirstThunk;
     };
 
+    if (options.executable) {
+        for (const auto index : {4u, 11u, 13u}) {
+            const auto directory = Read<PeDataDirectory>(pe, dataDirectoryOffset + index * sizeof(PeDataDirectory));
+            if (directory.VirtualAddress || directory.Size) throw std::runtime_error("signed, bound or delay-loaded PE imports are not supported in --exec mode");
+        }
+    }
+    struct ImportUse {
+        std::size_t NameOffset;
+        std::size_t LookupOffset;
+        std::size_t AddressOffset;
+        std::string Name;
+    };
+    std::vector<ImportUse> imports;
+    std::vector<std::string> importNames;
     std::size_t importDescOffset = RvaToOffset(pe, importDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
 
     while (true) {
@@ -286,19 +303,9 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
 
                 const std::string funcName = ReadCStr(pe, hintNameOffset + 2u);
                 if (!funcName.empty()) {
-                    const bool hasNidPostfix = funcName.size() >= kNidPostfixLen &&
-                        funcName.compare(funcName.size() - kNidPostfixLen, kNidPostfixLen, kNidPostfix) == 0;
-                    const bool hasScePrefix = funcName.size() >= 3u &&
-                        std::tolower(static_cast<unsigned char>(funcName[0])) == 's' &&
-                        std::tolower(static_cast<unsigned char>(funcName[1])) == 'c' &&
-                        std::tolower(static_cast<unsigned char>(funcName[2])) == 'e';
-                    if (hasNidPostfix || hasScePrefix) {
-                        const std::string nid = ResolveOneName(funcName);
-                        if (nid.size() > funcName.size())
-                            throw std::runtime_error("import NID longer than original name: " + funcName);
-                        std::memcpy(pe.data() + hintNameOffset + 2u, nid.data(), nid.size());
-                        pe[hintNameOffset + 2u + nid.size()] = 0u;
-                    }
+                    const auto addressOffset = RvaToOffset(pe, desc.FirstThunk, peHeaderOffset, numberOfSections, sizeOfOptionalHeader) + thunkOffset - RvaToOffset(pe, thunkRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+                    imports.push_back({hintNameOffset + 2u, thunkOffset, addressOffset, funcName});
+                    importNames.push_back(funcName);
                 }
             }
 
@@ -307,6 +314,72 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
 
         importDescOffset += sizeof(PeImportDescriptor);
     }
+    const auto importMap = ResolveImportNids(importNames, excludedExports, options);
+    if (!options.executable) {
+        for (const auto& item : imports) {
+            const auto& name = importMap.at(item.Name);
+            if (name.size() > item.Name.size()) throw std::runtime_error("import NID longer than original name: " + item.Name);
+            std::memcpy(pe.data() + item.NameOffset, name.c_str(), name.size() + 1);
+        }
+        return;
+    }
+    std::vector<std::uint8_t> strings;
+    std::unordered_map<std::string, std::uint32_t> stringOffsets;
+    for (const auto& item : imports) {
+        const auto& name = importMap.at(item.Name);
+        if (name == item.Name || stringOffsets.contains(name)) continue;
+        if (strings.size() & 1u) strings.push_back(0);
+        if (strings.size() > 0xffffffffu - name.size() - 3u) throw std::runtime_error("PE import string table overflow");
+        stringOffsets.emplace(name, static_cast<std::uint32_t>(strings.size()));
+        strings.insert(strings.end(), {0, 0});
+        strings.insert(strings.end(), name.begin(), name.end());
+        strings.push_back(0);
+    }
+    if (strings.empty()) return;
+    const auto fileAlignment = Read<std::uint32_t>(pe, optionalHeaderOffset + 36u);
+    const auto sectionAlignment = Read<std::uint32_t>(pe, optionalHeaderOffset + 32u);
+    const auto align = [](std::uint64_t value, std::uint32_t alignment) {
+        if (!alignment || (alignment & (alignment - 1u))) throw std::runtime_error("invalid PE alignment");
+        const auto result = (value + alignment - 1u) & ~std::uint64_t{alignment - 1u};
+        if (result > 0xffffffffu) throw std::runtime_error("PE section overflow");
+        return static_cast<std::uint32_t>(result);
+    };
+    const auto newHeader = sectionTableOffset + numberOfSections * sizeof(PeSectionHeader);
+    auto headerLimit = std::size_t{Read<std::uint32_t>(pe, optionalHeaderOffset + 60u)};
+    std::uint64_t virtualEnd = headerLimit;
+    for (std::uint16_t index = 0; index < numberOfSections; ++index) {
+        const auto section = Read<PeSectionHeader>(pe, sectionTableOffset + index * sizeof(PeSectionHeader));
+        if (section.PointerToRawData) headerLimit = std::min(headerLimit, std::size_t{section.PointerToRawData});
+        virtualEnd = std::max(virtualEnd, std::uint64_t{section.VirtualAddress} + std::max(section.VirtualSize, section.SizeOfRawData));
+    }
+    if (numberOfSections == 0xffffu || newHeader + sizeof(PeSectionHeader) > headerLimit || headerLimit > pe.size()) throw std::runtime_error("no room for PE import section header");
+    PeSectionHeader section{};
+    const char sectionName[] = ".nidimp";
+    std::memcpy(section.Name, sectionName, sizeof(sectionName));
+    section.VirtualSize = static_cast<std::uint32_t>(strings.size());
+    section.VirtualAddress = align(virtualEnd, sectionAlignment);
+    section.PointerToRawData = align(pe.size(), fileAlignment);
+    section.SizeOfRawData = align(strings.size(), fileAlignment);
+    section.Characteristics = 0x40000040u;
+    const auto imageSize = align(std::uint64_t{section.VirtualAddress} + section.VirtualSize, sectionAlignment);
+    const auto rawEnd = std::uint64_t{section.PointerToRawData} + section.SizeOfRawData;
+    if (rawEnd > 0xffffffffu) throw std::runtime_error("PE file size overflow");
+    pe.resize(static_cast<std::size_t>(rawEnd), 0);
+    std::copy(strings.begin(), strings.end(), pe.begin() + section.PointerToRawData);
+    Write(pe, newHeader, section);
+    Write(pe, coffHeaderOffset + 2u, static_cast<std::uint16_t>(numberOfSections + 1u));
+    Write(pe, optionalHeaderOffset + 56u, imageSize);
+    Write(pe, optionalHeaderOffset + 64u, std::uint32_t{0});
+    for (const auto& item : imports) {
+        const auto& name = importMap.at(item.Name);
+        if (name == item.Name) continue;
+        const auto rva = section.VirtualAddress + stringOffsets.at(name);
+        for (const auto offset : {item.LookupOffset, item.AddressOffset}) {
+            if (magic == 0x20bu) Write(pe, offset, std::uint64_t{rva});
+            else Write(pe, offset, rva);
+        }
+    }
+
 }
 
 }

core/libs/nid/src/main.cpp

@@ -28,6 +28,7 @@ void _writeFile(const std::string& path, const std::vector<std::uint8_t>& data)
 int Run(int argc, char* argv[]) {
     if (argc < 3) {
         std::cerr << "Usage: nid_patcher <library_name> [--preserve-exports <unpatched-library> | --preserve-export-list <export-list>] <file> [file2 ...]\n"
+                     "       nid_patcher --exec [--only-sce] [--preserve-exports <unpatched-library> | --preserve-export-list <export-list>] <file> [file2 ...]\n"
                      "       nid_patcher --write-export-list <unpatched-library> <export-list>\n";
         return 1;
     }
@@ -46,14 +47,20 @@ int Run(int argc, char* argv[]) {
         return 0;
     }
 
-    const std::string libraryName = argv[1];
+    Nid::PatchOptions options;
+    options.executable = std::string(argv[1]) == "--exec";
+    const std::string libraryName = options.executable ? "" : argv[1];
     std::unordered_set<std::string> excludedExports;
     std::vector<std::string> paths;
     try {
+        if (!options.executable && libraryName.starts_with("--")) throw std::runtime_error("unknown option: " + libraryName);
         bool hasReference = false;
         for (int i = 2; i < argc; ++i) {
             const std::string argument = argv[i];
-            if (argument == "--preserve-exports" || argument == "--preserve-export-list") {
+            if (argument == "--only-sce") {
+                if (!options.executable || options.onlySce) throw std::runtime_error("--only-sce requires --exec and may appear once");
+                options.onlySce = true;
+            } else if (argument == "--preserve-exports" || argument == "--preserve-export-list") {
                 if (hasReference || i + 1 == argc) throw std::runtime_error(argument + " requires exactly one reference");
                 hasReference = true;
                 excludedExports = argument == "--preserve-exports" ? Nid::ReadExportExclusions(argv[++i]) : Nid::ReadExportList(argv[++i]);
@@ -72,7 +79,7 @@ int Run(int argc, char* argv[]) {
         try {
             auto binary = _readFile(path);
             const auto patcher = Nid::MakePatcher(binary);
-            patcher->PatchNids(binary, libraryName, excludedExports);
+            patcher->PatchNids(binary, libraryName, excludedExports, options);
             _writeFile(path, binary);
             std::cout << "OK: " << path << "\n";
         } catch (const std::exception& e) {

core/libs/nid/tests/test_macho_nids.py

@@ -263,7 +263,7 @@ def check_export_list(patcher, directory):
 def check_refusals(patcher, directory):
     growing = [("_ab", 0, 0x100), ("_cd", 0, 0x108)]
     result, _ = patch(patcher, directory, "signed.dylib", build_dylib(growing, [], signed=True))
-    assert result.returncode != 0 and "signed Mach-O images cannot grow their export trie" in result.stdout + result.stderr, (result.stdout, result.stderr)
+    assert result.returncode != 0 and "signed Mach-O images cannot grow linkedit data" in result.stdout + result.stderr, (result.stdout, result.stderr)
     result, _ = patch(patcher, directory, "opcodes.dylib", build_dylib([("_puts_nid_postfix", 0, 0x100)], [], bind_opcodes=True))
     assert result.returncode != 0 and "bind opcodes are not supported" in result.stdout + result.stderr, (result.stdout, result.stderr)
     result, _ = patch(patcher, directory, "long.dylib", build_dylib([("_puts_nid_postfix", 0, 0x100)], [("_sceXy", 1)]))

core/libs/nid/tests/test_pe_references.py

@@ -243,5 +243,203 @@ class ForwardingExportTests(unittest.TestCase):
         self.assertEqual(after, image)
 
 
+class ExecutableImportTests(unittest.TestCase):
+    names = ["foo", "foo_nid_postfix", "z", "sceExample", "helper_nid_no_patch", "plain_nid_no_patch_cut", "SDL_Init", "scePreserved", "hostFunctionLongName", "alias_nid_disambig12_nid_postfix"]
+
+    def pe(self, bits=64):
+        image = bytearray(2048)
+        image[:2] = b"MZ"
+        struct.pack_into("<I", image, 0x3c, 0x40)
+        optional_size = 240 if bits == 64 else 224
+        struct.pack_into("<IHHIIIHH", image, 0x40, 0x4550, 0x8664 if bits == 64 else 0x14c, 1, 0, 0, 0, optional_size, 0x22)
+        struct.pack_into("<H", image, 0x58, 0x20b if bits == 64 else 0x10b)
+        struct.pack_into("<II", image, 0x58 + 32, 4096, 512)
+        struct.pack_into("<II", image, 0x58 + 56, 8192, 512)
+        directory = 0x58 + (112 if bits == 64 else 96)
+        struct.pack_into("<I", image, directory - 4, 16)
+        struct.pack_into("<II", image, directory + 8, 0x1000, 40)
+        struct.pack_into("<8sIIIIIIHHI", image, 0x58 + optional_size, b".idata", 1536, 0x1000, 1536, 512, 0, 0, 0, 0, 0xc0000040)
+        struct.pack_into("<IIIII", image, 512, 0x1040, 0, 0, 0x1028, 0x10c0)
+        image[552:561] = b"test.prx\0"
+        offset = 0x380
+        for index, name in enumerate(self.names):
+            entry = name.encode() + b"\0"
+            struct.pack_into("<H", image, offset, 0)
+            image[offset + 2:offset + 2 + len(entry)] = entry
+            for table in (576, 704):
+                struct.pack_into("<Q" if bits == 64 else "<I", image, table + index * (bits // 8), offset - 512 + 0x1000)
+            offset += len(entry) + 2
+        return image
+
+    def elf(self):
+        from test_symbol_versions import build_image, SHOFF
+        entries = [(None, 1, 0, 0, 0)] + [(name, 1, 0, 2, 0) for name in self.names] + [("application_export", 1, 9, 2, 0x400100)]
+        import test_symbol_versions
+        from unittest.mock import patch
+        with patch.object(test_symbol_versions, "DYNSYM_OFFSET", 0xa00):
+            image = build_image(entries, False)
+        struct.pack_into("<H", image, 16, 2)
+        struct.pack_into("<Q", image, 32, 64)
+        struct.pack_into("<HH", image, 54, 56, 2)
+        struct.pack_into("<IIQQQQQQ", image, 64, 1, 5, 0, 0x400000, 0x400000, 4096, 8192, 4096)
+        struct.pack_into("<IIQQQQQQ", image, 120, 2, 6, 0x700, 0x400700, 0x400700, 48, 48, 8)
+        struct.pack_into("<H", image, 60, 5)
+        struct.pack_into("<IIQQQQIIQQ", image, SHOFF + 4 * 64, 0, 6, 3, 0x400700, 0x700, 48, 2, 0, 8, 16)
+        size = struct.unpack_from("<Q", image, SHOFF + 2 * 64 + 32)[0]
+        struct.pack_into("<qQqQqQ", image, 0x700, 5, 0x400300, 10, size, 0, 0)
+        return image
+
+    def macho(self):
+        image = bytearray(build_dylib([("_application_export", 0, 0x100)], [("_" + name, 1 if index % 2 else 2) for index, name in enumerate(self.names)]))
+        struct.pack_into("<I", image, 12, 2)
+        return image
+
+    def read_names(self, image, kind):
+        if kind == "elf":
+            from test_symbol_versions import read_dynsym
+            return [name for name, section, _ in read_dynsym(image) if name and section == 0]
+        if kind == "macho":
+            from test_macho_nids import chained_imports
+            return [name[1:] for name, _ in chained_imports(image)]
+        bits = 64 if kind == "pe64" else 32
+        optional_size = 240 if bits == 64 else 224
+        count = struct.unpack_from("<H", image, 0x46)[0]
+        sections = [struct.unpack_from("<8sIIIIIIHHI", image, 0x58 + optional_size + index * 40) for index in range(count)]
+        result = []
+        for index in range(len(self.names)):
+            rva = struct.unpack_from("<Q" if bits == 64 else "<I", image, 576 + index * (bits // 8))[0]
+            self.assertEqual(rva, struct.unpack_from("<Q" if bits == 64 else "<I", image, 704 + index * (bits // 8))[0])
+            section = next(s for s in sections if s[2] <= rva < s[2] + s[3])
+            offset = section[4] + rva - section[2] + 2
+            result.append(image[offset:image.index(0, offset)].decode())
+        return result
+
+    def test_shared_import_rules_and_filter_in_all_formats(self):
+        for kind, build in (("pe64", self.pe), ("pe32", lambda: self.pe(32)), ("elf", self.elf), ("macho", self.macho)):
+            for only_sce in (False, True):
+                with self.subTest(kind=kind, only_sce=only_sce), tempfile.TemporaryDirectory() as directory:
+                    path = Path(directory) / "arbitrary.extension"
+                    image = build()
+                    path.write_bytes(image)
+                    excluded = Path(directory) / "preserved.txt"
+                    excluded.write_bytes(b"scePreserved\n")
+                    args = [str(PATCHER), "--exec", "--preserve-export-list", str(excluded)]
+                    if only_sce:
+                        args.append("--only-sce")
+                    result = subprocess.run([*args, str(path)], capture_output=True, text=True, timeout=20)
+                    self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+                    expected = ["foo", nid("foo"), nid("z"), nid("sceExample"), "helper_nid_no_patch", "plain", "SDL_Init", "scePreserved", nid("hostFunctionLongName"), nid("alias")]
+                    if only_sce:
+                        expected[2] = "z"
+                        expected[5] = "plain_nid_no_patch_cut"
+                        expected[8] = "hostFunctionLongName"
+                    self.assertCountEqual(self.read_names(path.read_bytes(), kind), expected)
+                    if kind == "macho":
+                        self.assertEqual(exports_of(path.read_bytes()), exports_of(image))
+                    if kind == "elf":
+                        from test_symbol_versions import read_dynsym
+                        self.assertIn(("application_export", 9, 0x400100), read_dynsym(path.read_bytes()))
+
+    def test_exec_accepts_macho_without_exports(self):
+        from test_macho_nids import LC_DYLD_EXPORTS_TRIE, command_fields
+        image = self.macho()
+        command = command_fields(image)[LC_DYLD_EXPORTS_TRIE][0]
+        struct.pack_into("<II", image, command + 8, 0, 0)
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "program"
+            path.write_bytes(image)
+            result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
+            self.assertEqual(result.returncode, 0, result.stderr)
+
+    def test_elf_without_sections_rejects_dynamic_segment_without_writing(self):
+        image = self.elf()
+        struct.pack_into("<Q", image, 40, 0)
+        struct.pack_into("<HH", image, 60, 0, 0)
+        for flags in ([], ["--only-sce"]):
+            with self.subTest(flags=flags), tempfile.TemporaryDirectory() as directory:
+                path = Path(directory) / "program"
+                path.write_bytes(image)
+                result = subprocess.run([str(PATCHER), "--exec", *flags, str(path)], capture_output=True, text=True, timeout=20)
+                self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
+                self.assertIn("PT_DYNAMIC without a .dynsym section", result.stderr)
+                self.assertEqual(path.read_bytes(), image)
+
+    def test_elf_without_sections_or_dynamic_segment_is_unchanged(self):
+        image = self.elf()
+        struct.pack_into("<Q", image, 40, 0)
+        struct.pack_into("<HH", image, 60, 0, 0)
+        struct.pack_into("<H", image, 56, 1)
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "program"
+            path.write_bytes(image)
+            result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
+            self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+            self.assertEqual(path.read_bytes(), image)
+
+    def test_elf_short_imports_grow_without_overwriting_bss(self):
+        from test_symbol_versions import sections
+        self.names = ["a", "b", "c", "d"]
+        image = self.elf()
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "program"
+            path.write_bytes(image)
+            result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            after = path.read_bytes()
+            self.assertCountEqual(self.read_names(after, "elf"), [nid(name) for name in self.names])
+            offset = struct.unpack_from("<Q", after, 32)[0]
+            count = struct.unpack_from("<H", after, 56)[0]
+            segments = [struct.unpack_from("<IIQQQQQQ", after, offset + index * 56) for index in range(count)]
+            original = next(segment for segment in segments if segment[0] == 1 and segment[3] == 0x400000)
+            self.assertEqual(original[5:7], (4096, 8192))
+            phdr = next(segment for segment in segments if segment[0] == 6)
+            load = next(segment for segment in segments if segment[0] == 1 and segment[2] == offset)
+            self.assertEqual(phdr[2:4], load[2:4])
+            strings = next(section for section in sections(after) if section[1] == 3)
+            self.assertEqual(struct.unpack_from("<Q", after, 0x708)[0], strings[3])
+            self.assertEqual(struct.unpack_from("<Q", after, 0x718)[0], strings[5])
+            self.assertGreater(strings[4], len(image))
+
+    def test_pe_own_exports_remain_unchanged(self):
+        image = self.pe()
+        struct.pack_into("<II", image, 0x58 + 112, 0x1500, 96)
+        struct.pack_into("<IIHHIIIIIII", image, 0x700, 0, 0, 0, 0, 0x1540, 1, 1, 1, 0x1528, 0x152c, 0x1530)
+        struct.pack_into("<IIH", image, 0x728, 0x1100, 0x1550, 0)
+        image[0x740:0x748] = b"app.exe\0"
+        image[0x750:0x75b] = b"own_export\0"
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "program"
+            path.write_bytes(image)
+            result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            self.assertEqual(path.read_bytes()[0x700:0x760], image[0x700:0x760])
+
+    def test_unsupported_exec_layouts_fail_without_writing(self):
+        signed = bytearray(build_dylib([], [("_sceExample", 1)], signed=True))
+        struct.pack_into("<I", signed, 12, 2)
+        pe = self.pe()
+        struct.pack_into("<II", pe, 0x58 + 112 + 13 * 8, 0x1400, 32)
+        full_headers = self.pe()
+        struct.pack_into("<I", full_headers, 0x58 + 60, 0x58 + 240 + 40)
+        for image, message in ((signed, "signature"), (pe, "delay-loaded"), (full_headers, "section header"), (b"invalid", "unrecognized")):
+            with self.subTest(message=message), tempfile.TemporaryDirectory() as directory:
+                path = Path(directory) / "program"
+                path.write_bytes(image)
+                result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
+                self.assertEqual(result.returncode, 2)
+                self.assertIn(message, result.stderr)
+                self.assertEqual(path.read_bytes(), image)
+
+    def test_only_sce_requires_exec_and_failure_does_not_write(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "program"
+            image = self.pe()
+            path.write_bytes(image)
+            for arguments in (["libtest", "--only-sce"], ["--only-sce"], ["--exec", "--only-sce", "--only-sce"]):
+                result = subprocess.run([str(PATCHER), *arguments, str(path)], capture_output=True, timeout=20)
+                self.assertNotEqual(result.returncode, 0)
+                self.assertEqual(path.read_bytes(), image)
+
+
 if __name__ == "__main__":
     unittest.main()

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -327,6 +327,7 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineCache.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineJournal.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Resources.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DeferredDestruction.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PlaceholderImages.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Recorder.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ParallelCompare.cpp
@@ -1279,6 +1280,20 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_deferred_destruction_tests EXCLUDE_FROM_ALL tests/DeferredDestruction.cpp)
+target_include_directories(agc_deferred_destruction_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
+target_link_libraries(agc_deferred_destruction_tests PRIVATE libSceAgcDriver libc)
+set_target_properties(agc_deferred_destruction_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_deferred_destruction_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_deferred_destruction_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_deferred_destruction COMMAND agc_deferred_destruction_tests)
+    if(WIN32)
+        set_tests_properties(agc_deferred_destruction PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_device_memory_fallback_tests EXCLUDE_FROM_ALL tests/DeviceMemoryFallback.cpp)
 target_include_directories(agc_device_memory_fallback_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
 target_link_libraries(agc_device_memory_fallback_tests PRIVATE libSceAgcDriver libc)
@@ -1572,8 +1587,12 @@ if(BUILD_TESTING)
     set_tests_properties(agc_pipeline_library_off PROPERTIES ENVIRONMENT "APS5_NO_GPL=1")
     add_test(NAME agc_pipeline_library_fast_only COMMAND agc_driver_recorder_tests --pipeline-library-only)
     set_tests_properties(agc_pipeline_library_fast_only PROPERTIES ENVIRONMENT "APS5_NO_PIPELINE_LTO=1" SKIP_RETURN_CODE 77)
+    add_test(NAME agc_pipeline_library_lto_info COMMAND agc_driver_recorder_tests --pipeline-library-lto-info-only)
+    add_test(NAME agc_pipeline_library_lto_info_fast_only COMMAND agc_driver_recorder_tests --pipeline-library-lto-info-only)
+    set_tests_properties(agc_pipeline_library_lto_info agc_pipeline_library_lto_info_fast_only PROPERTIES SKIP_RETURN_CODE 77)
+    set_tests_properties(agc_pipeline_library_lto_info_fast_only PROPERTIES ENVIRONMENT "APS5_NO_PIPELINE_LTO=1")
     if(WIN32)
-        set_tests_properties(agc_pipeline_library agc_pipeline_library_off agc_pipeline_library_fast_only PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_pipeline_library agc_pipeline_library_off agc_pipeline_library_fast_only agc_pipeline_library_lto_info agc_pipeline_library_lto_info_fast_only PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     set_tests_properties(agc_pipeline_cache agc_pipeline_cache_file PROPERTIES SKIP_RETURN_CODE 77)
@@ -2181,6 +2200,12 @@ if(BUILD_TESTING)
         set_tests_properties(agc_driver_address_passes_targets_split PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_driver_address_passes_draw_transitions COMMAND agc_driver_address_passes_tests --targets)
+    set_tests_properties(agc_driver_address_passes_draw_transitions PROPERTIES ENVIRONMENT "APS5_NO_ADDRESS_READ_PASSES=1;APS5_DRAW_TRANSITIONS=1" SKIP_RETURN_CODE 77 TIMEOUT 60)
+    if(WIN32)
+        set_tests_properties(agc_driver_address_passes_draw_transitions PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     set_target_properties(agc_driver_rcp_hardware_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
     add_test(NAME agc_driver_rcp_hardware_exact COMMAND agc_driver_rcp_hardware_tests)
     set_tests_properties(agc_driver_rcp_hardware_exact PROPERTIES ENVIRONMENT "APS5_EXACT_RCP=1;ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 60)
@@ -2225,6 +2250,7 @@ endif()
 
 target_link_libraries(agc_driver_rect_list_corner_tests PRIVATE shader_recompiler_agc_driver)
 target_link_libraries(agc_driver_queue_zero_late_interrupt_tests PRIVATE libkernel)
+target_link_libraries(agc_driver_linear_target_completion_points_tests PRIVATE libkernel)
 
 target_link_libraries(agc_driver_storage_file_write_tests PRIVATE libkernel)
 
@@ -2239,6 +2265,16 @@ target_include_directories(agc_driver_image_table_cache_bounds_tests PRIVATE $<T
 target_include_directories(agc_driver_draw_cache_relocation_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
 if(BUILD_TESTING)
+    add_test(NAME agc_driver_linear_target_completion_points_sync_flip COMMAND agc_driver_linear_target_completion_points_tests --flip)
+    set_tests_properties(agc_driver_linear_target_completion_points_sync_flip PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_SYNC_FLIP=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_driver_linear_target_completion_points_unpipelined COMMAND agc_driver_linear_target_completion_points_tests --label --interrupt)
+    set_tests_properties(agc_driver_linear_target_completion_points_unpipelined PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=0" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_linear_target_completion_points_sync_flip agc_driver_linear_target_completion_points_unpipelined PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_submit_progress COMMAND agc_driver_submit_progress_tests)
     set_tests_properties(agc_submit_progress PROPERTIES
             ENVIRONMENT "APS5_SUBMIT_PROGRESS=1;ANYPS5_REQUIRE_VULKAN=1" TIMEOUT 30 SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2369,6 +2405,8 @@ target_include_directories(agc_driver_color565_target_tests PRIVATE $<TARGET_PRO
 
 target_include_directories(agc_driver_depth_range_remap_draw_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_depth_bounds_emulation_draw_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
 foreach(stage IN ITEMS vert frag)
     set(agcProvokingShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/ProvokingVertex.${stage})
     set(agcProvokingShaderSpv ${agcTestShaderGeneratedRoot}/ProvokingVertex_${stage}.spv)
@@ -2736,6 +2774,11 @@ if(BUILD_TESTING)
     set_tests_properties(agc_pipeline_journal_long_paths PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_pipeline_library_clear_waits_for_build COMMAND agc_driver_graphics_tests --pipeline-library-clear-waits-for-build)
     add_test(NAME agc_pipeline_store_lazy_buffer_pool COMMAND agc_driver_graphics_tests --pipeline-store-lazy-buffer-pool)
+    add_test(NAME agc_pipeline_store_growth COMMAND agc_driver_graphics_tests --pipeline-store-growth)
+    add_test(NAME agc_pipeline_store_fixed_bound COMMAND agc_driver_graphics_tests --pipeline-store-growth)
+    add_test(NAME agc_pipeline_store_replay_growth COMMAND agc_driver_graphics_tests --pipeline-store-replay-growth)
+    set_tests_properties(agc_pipeline_store_growth agc_pipeline_store_fixed_bound agc_pipeline_store_replay_growth PROPERTIES TIMEOUT 60 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    set_tests_properties(agc_pipeline_store_fixed_bound PROPERTIES ENVIRONMENT "APS5_PIPELINE_STORE_ENTRIES=1024")
     set_tests_properties(agc_pipeline_journal PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     set_tests_properties(agc_pipeline_library_clear_waits_for_build PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     set_tests_properties(agc_pipeline_store_lazy_buffer_pool PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2743,7 +2786,7 @@ if(BUILD_TESTING)
         add_test(NAME agc_pipeline_journal_extended_paths COMMAND agc_driver_graphics_tests --pipeline-journal-extended-paths)
         set_tests_properties(agc_pipeline_journal_extended_paths PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
         set_tests_properties(agc_pipeline_journal_long_paths agc_pipeline_journal_extended_paths PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
-        set_tests_properties(agc_pipeline_journal agc_pipeline_library_clear_waits_for_build agc_pipeline_store_lazy_buffer_pool PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_pipeline_journal agc_pipeline_library_clear_waits_for_build agc_pipeline_store_lazy_buffer_pool agc_pipeline_store_growth agc_pipeline_store_fixed_bound agc_pipeline_store_replay_growth PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     set_tests_properties(agc_driver_sparse_buffer_holes PROPERTIES TIMEOUT 30)
     if(WIN32)
@@ -2875,12 +2918,16 @@ if(BUILD_TESTING)
     set_tests_properties(agc_structurizer_iteration_selection_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_ITERATION_SELECTION=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_shader_control_flow_iteration_selection_off COMMAND agc_shader_control_flow_tests)
     set_tests_properties(agc_shader_control_flow_iteration_selection_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_ITERATION_SELECTION=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_structurizer_dominator_tree_off COMMAND agc_structurizer_tests)
+    set_tests_properties(agc_structurizer_dominator_tree_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_DOMINATOR_TREE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_shader_control_flow_dominator_tree_off COMMAND agc_shader_control_flow_tests)
+    set_tests_properties(agc_shader_control_flow_dominator_tree_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_DOMINATOR_TREE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_structurizer_forced_fallback COMMAND agc_structurizer_tests)
     set_tests_properties(agc_structurizer_forced_fallback PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FORCE_STRUCTURIZER_FALLBACK=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_shader_control_flow_forced_fallback COMMAND agc_shader_control_flow_tests)
     set_tests_properties(agc_shader_control_flow_forced_fallback PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FORCE_STRUCTURIZER_FALLBACK=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     if(WIN32)
-        set_tests_properties(agc_structurizer_intersection_off agc_structurizer_iteration_selection_off agc_shader_control_flow_iteration_selection_off agc_structurizer_forced_fallback agc_shader_control_flow_forced_fallback PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_structurizer_intersection_off agc_structurizer_iteration_selection_off agc_shader_control_flow_iteration_selection_off agc_structurizer_dominator_tree_off agc_shader_control_flow_dominator_tree_off agc_structurizer_forced_fallback agc_shader_control_flow_forced_fallback PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_driver_gated_dispatch_capture_short_timeout COMMAND agc_driver_gated_dispatch_capture_tests)
     set_tests_properties(agc_driver_gated_dispatch_capture_short_timeout PROPERTIES
@@ -3443,8 +3490,7 @@ if(BUILD_TESTING)
         endif()
     endforeach()
 
-    option(ANYPS5_RUN_QUARANTINED_FRAME_REPLAY "Build and register agc_driver_frame_replay on Windows" OFF)
-    if(WIN32 AND ANYPS5_RUN_QUARANTINED_FRAME_REPLAY)
+    if(WIN32)
         add_test_executable(agc_driver_frame_replay_tests tests/FrameReplay.cpp Execution/src/FrameCapture.cpp)
         target_include_directories(agc_driver_frame_replay_tests PRIVATE ${LIBS_INCLUDE_DIR})
         target_link_libraries(agc_driver_frame_replay_tests PRIVATE libSceAgc libSceAgcDriver libkernel libc libzstd_static)
@@ -3457,6 +3503,12 @@ if(BUILD_TESTING)
                 TIMEOUT 120)
         set_tests_properties(agc_driver_frame_replay PROPERTIES ENVIRONMENT_MODIFICATION
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        add_test(NAME agc_driver_frame_replay_concurrent COMMAND agc_driver_frame_replay_tests $<TARGET_FILE:agc_frame_replay> concurrent)
+        set_tests_properties(agc_driver_frame_replay_concurrent PROPERTIES
+                ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FRAME_CAPTURE=${CMAKE_CURRENT_BINARY_DIR}/frame-replay-concurrent-capture;APS5_FRAME_CAPTURE_AT_FLIP=1;APS5_FRAME_CAPTURE_FRAMES=8"
+                TIMEOUT 120)
+        set_tests_properties(agc_driver_frame_replay_concurrent PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 endif()
 
@@ -3980,6 +4032,6 @@ endif()
 
 if(BUILD_TESTING)
     get_directory_property(agcRegisteredTests TESTS)
-    list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal(_long_paths|_extended_paths)?|agc_driver_pipeline_journal)$")
+    list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal(_long_paths|_extended_paths)?|agc_driver_pipeline_journal|agc_pipeline_store_replay_growth)$")
     set_property(TEST ${agcRegisteredTests} APPEND PROPERTY ENVIRONMENT "APS5_NO_PIPELINE_JOURNAL=1")
 endif()

core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp

@@ -33,6 +33,8 @@ extern "C" void AgcDriverSuspendPoint_nid_postfix();
 extern "C" void AgcDriverSerialProgress_nid_postfix(std::uint64_t* accepted, std::uint64_t* completed);
 extern "C" std::size_t AgcDriverAwaitedWaits_nid_postfix(AgcDriver::AwaitedWait* waits, std::size_t capacity);
 extern "C" std::uint64_t AgcDriverDispatchesAdoptedAhead_nid_postfix();
+extern "C" std::uint64_t AgcDriverDispatchesRejectedAhead_nid_postfix();
+extern "C" void AgcDriverSetAheadAdoptHook_nid_postfix(void (*hook)());
 extern "C" void AgcDriverRegisterShader_nid_postfix(const Shader* shader);
 extern "C" void AgcDriverResolveShaderAbi_nid_postfix(const Shader* shader, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);
 extern "C" void AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span<const Shader* const> stages, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp

@@ -55,6 +55,7 @@ public:
     std::uint64_t DecodeDrains() const { return decodeDrains.load(std::memory_order_relaxed); }
     std::optional<std::uint64_t> PendingLabel(std::uint64_t address, std::size_t bytes);
     static bool HeldReads(std::uint64_t address, std::size_t bytes);
+    static std::uint64_t HeldReadQueries();
     void ReleaseRunningReads();
 
     static bool MetadataPassesPipelined();

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -59,6 +59,8 @@ public:
     void Shutdown();
     void WaitIdle();
     std::uint64_t DispatchesAdoptedAhead() const;
+    std::uint64_t DispatchesRejectedAhead() const;
+    void SetAheadAdoptHook(void (*hook)());
     void CheckFailure();
     void ReportFailure(std::exception_ptr error);
     void Submit(const Packet* packet, std::uint32_t queue, bool cpuFlip = false);
@@ -317,7 +319,7 @@ private:
         std::chrono::steady_clock::time_point resolvedAt{};
     };
     struct GroupCaptureStats {
-        std::uint64_t runs = 0, resolved = 0, resolvedHits = 0, adopted = 0, addressMismatches = 0, keyMismatches = 0, failures = 0, stale = 0, labelOverlaps = 0, writtenBefore = 0, deviceRejects = 0, argumentMismatches = 0;
+        std::uint64_t runs = 0, resolved = 0, resolvedHits = 0, adopted = 0, addressMismatches = 0, keyMismatches = 0, memoryMismatches = 0, failures = 0, stale = 0, labelOverlaps = 0, writtenBefore = 0, deviceRejects = 0, argumentMismatches = 0;
         double resolveMs = 0, resolveMaxMs = 0, ageMs = 0, ageMaxMs = 0;
         std::chrono::steady_clock::time_point reported = std::chrono::steady_clock::now();
     };
@@ -474,6 +476,8 @@ private:
     std::atomic<std::uint64_t> evidenceReads{0};
     std::atomic<std::uint64_t> evidenceValidations{0};
     std::atomic<std::uint64_t> dispatchesAdoptedAhead{0};
+    std::atomic<std::uint64_t> dispatchesRejectedAhead{0};
+    std::atomic<void (*)()> aheadAdoptHook{nullptr};
 
     AgcDriver::Mutex writtenBuffersMutex;
     std::deque<WrittenBuffer> writtenBuffers;

core/libs/prx/libSceAgcDriver/Execution/include/FrameRecorder.hpp

@@ -10,6 +10,7 @@
 #include <cstdint>
 #include <deque>
 #include <filesystem>
+#include <functional>
 #include <map>
 #include <memory>
 #include <mutex>
@@ -35,10 +36,10 @@ public:
     void ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType);
     void RegisterVideoOutput(std::uint32_t handle);
     void UnregisterVideoOutput(std::uint32_t handle);
-    void Submit(const Packet& packet, std::uint32_t queue);
-    void Flip(const FlipInfo& info);
+    void Submit(const Packet& packet, std::uint32_t queue, const std::function<void()>& accept);
+    void Flip(const FlipInfo& info, const std::function<void()>& accept);
     void ReserveEopFlip(const FlipInfo& info, std::uint32_t token);
-    void Suspend();
+    void Suspend(const std::function<void()>& accept);
     void PresentBuffer(const DisplayBuffer& buffer);
     void FrameBoundary();
 
@@ -62,7 +63,8 @@ private:
     };
 
     FrameCall make(FrameCallKind kind);
-    void record(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders = {});
+    void record(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders = {}, const std::function<void()>& accept = {});
+    void recordLocked(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders);
     void emitDelta();
     void emitShaderDelta(std::span<const Shader* const> shaders);
     void writeDelta(std::span<const Span> spans, std::span<const std::uint64_t> hostPages, std::span<const Range> added = {});
@@ -72,6 +74,7 @@ private:
     void snapshot(const Range& range);
     void writeReferenceFrames();
     bool abandonIfDiskLow();
+    void stop(const char* reason);
     std::string header() const;
     std::vector<Range> committedRanges();
     std::vector<Range> newRanges();

core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp

@@ -63,7 +63,7 @@ void WriteChanged(std::uint64_t address, std::span<const std::byte> current, std
 bool WriteWatched();
 void Unwatch(std::uint64_t address, std::size_t bytes);
 void Rewatch(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges);
-bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched = true);
+bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched = true, bool importMayWriteGuest = true);
 void SetUnwatchHook(void (*hook)(std::uint64_t address, std::size_t bytes));
 #ifdef _WIN32
 using FreshCollectHook = bool (*)(std::uintptr_t address, std::size_t bytes, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context);

core/libs/prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp

@@ -46,6 +46,8 @@ private:
     std::uint32_t sourceHeight = 0;
     VkImage sourceImage = VK_NULL_HANDLE;
     VkDeviceMemory sourceMemory = VK_NULL_HANDLE;
+    VkDeviceSize sourceBytes = 0;
+    std::uint32_t sourceMemoryType = 0;
     VkImage blackImage = VK_NULL_HANDLE;
     VkDeviceMemory blackMemory = VK_NULL_HANDLE;
 };

core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp

@@ -275,8 +275,8 @@ public:
     // blit-submit times after the flip packet; noted in-place reads checked
     // (APS5_FLIP_READ_CHECK=1) and found overwritten before the blit's fence.
     struct PresentStatistics {
-        std::uint64_t presents, gpuUnread, batchesAheadOfBlit, batchesAfterFlip, readsChecked, readsOverwritten, acquireDrops;
-        double gpuBusyMs, gpuGapMs, lastSubmitAfterFlipMs, blitSubmitAfterFlipMs;
+        std::uint64_t presents, gpuUnread, batchesAheadOfBlit, batchesAfterFlip, readsChecked, readsOverwritten, acquireDrops, keyWriterWaits;
+        double gpuBusyMs, gpuGapMs, lastSubmitAfterFlipMs, blitSubmitAfterFlipMs, keyWriterWaitMs;
     };
     static PresentStatistics PresentCounts();
     struct PresyncStatistics {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp

@@ -16,6 +16,7 @@
 #include "Optimization/ResourceProgram.hpp"
 #include <algorithm>
 #include <cstdlib>
+#include <cstring>
 #include <optional>
 #include <stdexcept>
 #include <utility>
@@ -57,6 +58,19 @@ void ReportDispatchItems(bool profile) {
     std::fprintf(stderr, "[dispatch-item] %llu commit items (%llu recipes rebuilt on the committer); drained instead, cumulative:%s\n", static_cast<unsigned long long>(itemCommits.load(std::memory_order_relaxed)), static_cast<unsigned long long>(itemRebuilds.load(std::memory_order_relaxed)), drained.c_str());
 }
 
+bool CapturedCurrent(std::span<const ShaderRecompiler::MemoryRegion> captured, const ShaderSnapshot& code) {
+    for (const auto& region : captured) {
+        const auto address = region.guestAddress;
+        const auto bytes = region.bytes.size();
+        if (bytes == 0 || (address >= code.codeAddress && address < code.codeAddress + code.code.size() * sizeof(std::uint32_t)) || (!code.header.empty() && address >= code.headerAddress && address < code.headerAddress + code.header.size())) continue;
+        if (Driver::QueryPendingSpan(address, bytes, {}) != ShaderMemory::PendingWrite::None || !GuestMemory::Accessible(reinterpret_cast<const void*>(address), bytes)) return false;
+        GuestMemory::GuardRead(address, bytes);
+        GuestMemory::FlushGpuWrites(address, bytes);
+        if (std::memcmp(region.bytes.data(), reinterpret_cast<const void*>(address), bytes) != 0) return false;
+    }
+    return true;
+}
+
 }
 
 bool Driver::pipelineDispatch() {
@@ -281,6 +295,14 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
         fromAhead = false;
         ++groupStats.keyMismatches;
     }
+    if (fromAhead) {
+        if (const auto hook = aheadAdoptHook.load(std::memory_order_acquire)) hook();
+        if (!CapturedCurrent(ahead.captured, snapshot)) {
+            fromAhead = false;
+            ++dispatchesRejectedAhead;
+            ++groupStats.memoryMismatches;
+        }
+    }
     phaseTiming.Phase(PhaseKey);
     timing.Mark("decode_key_device_setup");
     bool readsClear = false;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp

@@ -42,6 +42,14 @@ std::uint64_t Driver::DispatchesAdoptedAhead() const {
     return dispatchesAdoptedAhead.load(std::memory_order_relaxed);
 }
 
+std::uint64_t Driver::DispatchesRejectedAhead() const {
+    return dispatchesRejectedAhead.load(std::memory_order_relaxed);
+}
+
+void Driver::SetAheadAdoptHook(void (*hook)()) {
+    aheadAdoptHook.store(hook, std::memory_order_release);
+}
+
 namespace {
 
 std::size_t& PreviewedUntil() {
@@ -326,9 +334,9 @@ void Driver::reportGroupCapture(std::uint32_t queue) {
     auto& stats = groupCaptureStats();
     const auto now = std::chrono::steady_clock::now();
     if (now - stats.reported < std::chrono::seconds(10)) return;
-    AgcDriver::ProfilePrint_nid_no_patch("[groupcap] queue 0x%x (10 s): %llu runs resolved %llu dispatches (%llu cache hits) in %.1f ms (max %.2f ms); %llu adopted %.2f ms after the resolve on average, max %.2f ms; mismatches: address %llu key %llu; %llu resolve failures, %llu stale, %llu label overlaps, %llu read an earlier dispatch's output, %llu rejected by the device, %llu dropped because the argument re-read differed\n",
+    AgcDriver::ProfilePrint_nid_no_patch("[groupcap] queue 0x%x (10 s): %llu runs resolved %llu dispatches (%llu cache hits) in %.1f ms (max %.2f ms); %llu adopted %.2f ms after the resolve on average, max %.2f ms; mismatches: address %llu key %llu memory %llu; %llu resolve failures, %llu stale, %llu label overlaps, %llu read an earlier dispatch's output, %llu rejected by the device, %llu dropped because the argument re-read differed\n",
                                          queue, static_cast<unsigned long long>(stats.runs), static_cast<unsigned long long>(stats.resolved), static_cast<unsigned long long>(stats.resolvedHits), stats.resolveMs, stats.resolveMaxMs,
-                                         static_cast<unsigned long long>(stats.adopted), stats.adopted != 0 ? stats.ageMs / static_cast<double>(stats.adopted) : 0.0, stats.ageMaxMs, static_cast<unsigned long long>(stats.addressMismatches), static_cast<unsigned long long>(stats.keyMismatches),
+                                         static_cast<unsigned long long>(stats.adopted), stats.adopted != 0 ? stats.ageMs / static_cast<double>(stats.adopted) : 0.0, stats.ageMaxMs, static_cast<unsigned long long>(stats.addressMismatches), static_cast<unsigned long long>(stats.keyMismatches), static_cast<unsigned long long>(stats.memoryMismatches),
                                          static_cast<unsigned long long>(stats.failures), static_cast<unsigned long long>(stats.stale), static_cast<unsigned long long>(stats.labelOverlaps), static_cast<unsigned long long>(stats.writtenBefore), static_cast<unsigned long long>(stats.deviceRejects), static_cast<unsigned long long>(stats.argumentMismatches));
     stats = GroupCaptureStats{};
     stats.reported = now;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp

@@ -183,6 +183,7 @@ std::shared_ptr<DrawDecode> Driver::decodeDraw(const QueueState& queue, const Su
     product->pixel = Graphics::DecodePixelStageInfo(queue.context, Graphics::ExportMappings(product->state), Graphics::PixelProgramSkipped(queue));
     Graphics::NormalizeUnusedViewportDepth(product->state, product->pixel);
     Graphics::ApplyDepthRemap(context, product->state);
+    Graphics::ApplyDepthBoundsEmulation(context, product->state, product->pixel);
     product->pixel.targetExportPacking = Graphics::ExportPackings(product->state);
     product->pixel.dualSourceBlend = product->state.dualSourceBlend;
     product->pixel.reversedBlend = product->state.reversedBlend;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp

@@ -482,6 +482,10 @@ bool DrawPipeline::HeldReads(std::uint64_t address, std::size_t bytes) {
     return false;
 }
 
+std::uint64_t DrawPipeline::HeldReadQueries() {
+    return heldReadQueries.load(std::memory_order_relaxed);
+}
+
 void DrawPipeline::ReleaseRunningReads() {
     std::lock_guard lock(mutex);
     if (!items.empty()) items.front().reads.clear();

core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp

@@ -19,9 +19,8 @@ void Submit(const Packet* packet, std::uint32_t queue) try {
         return;
     }
     GuestMemory::CheckRange(packet, sizeof(Packet), alignof(Packet));
-    recorder->Submit(*packet, queue);
     const auto flips = driver.FlipSubmissions();
-    driver.Submit(packet, queue);
+    recorder->Submit(*packet, queue, [&] { driver.Submit(packet, queue); });
     if (driver.FlipSubmissions() != flips) recorder->FrameBoundary();
 } catch (const ProcessShutdown&) {
     LibcAwaitExit_nid_postfix();
@@ -41,8 +40,12 @@ void RegisterShader(const Shader* shader) {
 }
 
 void SuspendPoint() {
-    if (auto* const recorder = FrameRecorder::Active()) recorder->Suspend();
-    DriverDetail::Driver::Get().SuspendPoint();
+    auto* const recorder = FrameRecorder::Active();
+    if (recorder == nullptr) {
+        DriverDetail::Driver::Get().SuspendPoint();
+        return;
+    }
+    recorder->Suspend([] { DriverDetail::Driver::Get().SuspendPoint(); });
 }
 
 void RegisterVideoOutput(std::uint32_t handle, const std::shared_ptr<IVideoOutput>& output) {
@@ -146,9 +149,12 @@ extern "C" void AgcDriverSerialProgress_nid_postfix(std::uint64_t* accepted, std
 
 extern "C" void AgcDriverSubmitFlip_nid_postfix(const AgcDriver::FlipInfo& info) try {
     auto* const recorder = AgcDriver::FrameRecorder::Active();
-    if (recorder != nullptr) recorder->Flip(info);
-    AgcDriver::DriverDetail::Driver::Get().SubmitFlip(info);
-    if (recorder != nullptr) recorder->FrameBoundary();
+    if (recorder == nullptr) {
+        AgcDriver::DriverDetail::Driver::Get().SubmitFlip(info);
+        return;
+    }
+    recorder->Flip(info, [&] { AgcDriver::DriverDetail::Driver::Get().SubmitFlip(info); });
+    recorder->FrameBoundary();
 } catch (const ProcessShutdown&) {
     LibcAwaitExit_nid_postfix();
 }
@@ -157,6 +163,14 @@ extern "C" std::uint64_t AgcDriverDispatchesAdoptedAhead_nid_postfix() {
     return AgcDriver::DriverDetail::Driver::Get().DispatchesAdoptedAhead();
 }
 
+extern "C" std::uint64_t AgcDriverDispatchesRejectedAhead_nid_postfix() {
+    return AgcDriver::DriverDetail::Driver::Get().DispatchesRejectedAhead();
+}
+
+extern "C" void AgcDriverSetAheadAdoptHook_nid_postfix(void (*hook)()) {
+    AgcDriver::DriverDetail::Driver::Get().SetAheadAdoptHook(hook);
+}
+
 extern "C" void AgcDriverRegisterVideoOutput_nid_postfix(std::uint32_t handle, const std::shared_ptr<AgcDriver::IVideoOutput>& output) {
     AgcDriver::RegisterVideoOutput(handle, output);
 }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp

@@ -11,6 +11,7 @@
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "prx/libSceAgcDriver/Eq/include/Event.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
 #include <atomic>
 #include <cstdio>
 #include <cstdlib>
@@ -394,6 +395,7 @@ void Driver::execute(const Submission& submission) {
                 GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush);
                 std::lock_guard gpuLock(GuestMemory::GpuMutex());
                 const auto localDevice = device.Load();
+                if (localDevice != nullptr) static_cast<void>(Graphics::StorageTexture::StoreDrawnLinear("flip"));
 
                 if (!recordLabelsForPacket(localDevice.get(), submission.queue) && localDevice != nullptr) localDevice->SubmitRecorded(submission.queue == 0);
                 if (localDevice != nullptr) {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp

@@ -38,11 +38,12 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
             std::lock_guard gpuLock(GuestMemory::GpuMutex());
             const auto localDevice = device.Load();
             recordDeferredLabels(localDevice.get(), submission.queue);
-            const bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0;
+            bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0;
             int reason = localDevice != nullptr ? 0 : 1;
             if (label.has_value()) {
                 const auto bytes = label->Bytes();
                 const auto stamp = ++eventSerial;
+                if (localDevice != nullptr && submission.queue == 0 && Graphics::StorageTexture::StoreDrawnLinear("label") != 0) workOpen = true;
                 reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, stamp, submission.queue) : 4;
                 if (reason == 1) GuestMemory::Write(label->address, bytes, 4);
                 if (reason == 0 || reason == 1 || reason == 5) {
@@ -52,6 +53,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
                 countLabelOutcome(reason);
                 ++immediateLabels;
             } else {
+                if (localDevice != nullptr && submission.queue == 0 && Graphics::StorageTexture::StoreDrawnLinear("interrupt") != 0) workOpen = true;
                 if (localDevice != nullptr) Graphics::StorageTexture::StoreReadbackBeforeLabel("readback before interrupt");
                 ++noOpLabels;
             }
@@ -249,6 +251,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
 
             recordDeferredLabels(draining.get(), submission.queue);
             if (draining != nullptr) {
+                static_cast<void>(Graphics::StorageTexture::StoreDrawnLinear("drain"));
                 if (unlockedDrain && draining->CanWaitUnlocked()) epoch = draining->SubmitAndEpoch();
                 else draining->WaitIdle();
             }
@@ -437,10 +440,11 @@ void Driver::commitLabel(std::uint32_t queue, std::uint64_t address, std::span<c
     std::lock_guard gpuLock(GuestMemory::GpuMutex());
     const auto localDevice = device.Load();
     recordDeferredLabels(localDevice.get(), queue);
-    const bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0;
+    bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0;
     int reason = localDevice != nullptr ? 0 : 1;
     if (!bytes.empty()) {
         const auto stamp = ++eventSerial;
+        if (localDevice != nullptr && queue == 0 && Graphics::StorageTexture::StoreDrawnLinear("label") != 0) workOpen = true;
         reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(address, bytes, stamp, queue) : 4;
         if (reason != 0 && reason != 5 && reason != 6) {
             if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle();
@@ -455,6 +459,7 @@ void Driver::commitLabel(std::uint32_t queue, std::uint64_t address, std::span<c
     }
     if (endOfPipeInterrupt) {
         bool deferred = false;
+        if (bytes.empty() && localDevice != nullptr && queue == 0 && Graphics::StorageTexture::StoreDrawnLinear("interrupt") != 0) workOpen = true;
         if (localDevice != nullptr && (reason == 0 || reason == 5 || reason == 6)) {
             deferred = localDevice->AfterRecordedWork([this, queue] { deliverGuestEopInterrupt(queue); }, queue == 0);
             if (deferred && workOpen) localDevice->SubmitRecorded(queue == 0);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Presentation.cpp

@@ -206,7 +206,7 @@ void Driver::reportPresents(double waitedMs, std::size_t inFlight) {
     const auto flips = flipsCounted.load() - previousFlips;
     const double perFlip = flips != 0 ? 1.0 / static_cast<double>(flips) : 0.0;
     const auto serial = flipSerial.load();
-    AgcDriver::ProfilePrint_nid_no_patch( "[present] %llu presents over 10 s (%llu dropped: no free swapchain image; %zu may trail on the GPU); %s p50 %.2f ms, p90 %.2f ms, max %.2f ms over %zu; per frame, presented or dropped: GPU busy %.1f ms, idle gaps %.1f ms (%llu batches without a completion record); per flip: %.1f batches recorded (%.1f unsignaled at the flip); per present: %.1f batches ahead of the blit, %.1f after it; after the flip packet: last submit %.1f ms, blit submit %.1f ms; in-place reads overwritten before execution: %llu of %llu checked\n", static_cast<unsigned long long>(presents), static_cast<unsigned long long>(dropped), inFlight, inFlight != 0 ? "inflight_wait" : "render_fence_wait", percentile(0.5), percentile(0.9), waits.empty() ? 0.0 : waits.back(), waits.size(), (counts.gpuBusyMs - previous.gpuBusyMs) * perFrame, (counts.gpuGapMs - previous.gpuGapMs) * perFrame, static_cast<unsigned long long>(counts.gpuUnread - previous.gpuUnread), static_cast<double>(serial - windowSerial) * perFlip, static_cast<double>(flipBatchesUnsignaled.load() - previousUnsignaled) * perFlip, static_cast<double>(counts.batchesAheadOfBlit - previous.batchesAheadOfBlit) * perPresent, static_cast<double>(counts.batchesAfterFlip - previous.batchesAfterFlip) * perPresent, (counts.lastSubmitAfterFlipMs - previous.lastSubmitAfterFlipMs) * perPresent, (counts.blitSubmitAfterFlipMs - previous.blitSubmitAfterFlipMs) * perPresent, static_cast<unsigned long long>(counts.readsOverwritten - previous.readsOverwritten), static_cast<unsigned long long>(counts.readsChecked - previous.readsChecked));
+    AgcDriver::ProfilePrint_nid_no_patch( "[present] %llu presents over 10 s (%llu dropped: no free swapchain image; %zu may trail on the GPU); %s p50 %.2f ms, p90 %.2f ms, max %.2f ms over %zu; per frame, presented or dropped: GPU busy %.1f ms, idle gaps %.1f ms (%llu batches without a completion record); per flip: %.1f batches recorded (%.1f unsignaled at the flip); per present: %.1f batches ahead of the blit, %.1f after it; after the flip packet: last submit %.1f ms, blit submit %.1f ms; in-place reads overwritten before execution: %llu of %llu checked; waits for the display keys' writer outside the GPU mutex: %llu, %.1f ms\n", static_cast<unsigned long long>(presents), static_cast<unsigned long long>(dropped), inFlight, inFlight != 0 ? "inflight_wait" : "render_fence_wait", percentile(0.5), percentile(0.9), waits.empty() ? 0.0 : waits.back(), waits.size(), (counts.gpuBusyMs - previous.gpuBusyMs) * perFrame, (counts.gpuGapMs - previous.gpuGapMs) * perFrame, static_cast<unsigned long long>(counts.gpuUnread - previous.gpuUnread), static_cast<double>(serial - windowSerial) * perFlip, static_cast<double>(flipBatchesUnsignaled.load() - previousUnsignaled) * perFlip, static_cast<double>(counts.batchesAheadOfBlit - previous.batchesAheadOfBlit) * perPresent, static_cast<double>(counts.batchesAfterFlip - previous.batchesAfterFlip) * perPresent, (counts.lastSubmitAfterFlipMs - previous.lastSubmitAfterFlipMs) * perPresent, (counts.blitSubmitAfterFlipMs - previous.blitSubmitAfterFlipMs) * perPresent, static_cast<unsigned long long>(counts.readsOverwritten - previous.readsOverwritten), static_cast<unsigned long long>(counts.readsChecked - previous.readsChecked), static_cast<unsigned long long>(counts.keyWriterWaits - previous.keyWriterWaits), counts.keyWriterWaitMs - previous.keyWriterWaitMs);
     previous = counts;
     previousFlips = flipsCounted.load();
     previousUnsignaled = flipBatchesUnsignaled.load();

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp

@@ -138,7 +138,7 @@ std::shared_ptr<const ShaderSnapshot> ReadRawShader(std::uint64_t address) {
                 cache.erase(found);
             }
             const auto bytes = result->code.size() * sizeof(std::uint32_t);
-            while (!cache.empty() && (cache.size() >= 64 || cacheBytes + bytes > 8 * 1024 * 1024)) {
+            while (!cache.empty() && (cache.size() >= 4096 || cacheBytes + bytes > 64 * 1024 * 1024)) {
                 cacheBytes -= cache.back()->code.size() * sizeof(std::uint32_t);
                 cache.pop_back();
             }

core/libs/prx/libSceAgcDriver/Execution/src/FrameRecorder.cpp

@@ -163,20 +163,20 @@ void FrameRecorder::UnregisterVideoOutput(std::uint32_t handle) {
     record(std::move(call), true);
 }
 
-void FrameRecorder::Submit(const Packet& packet, std::uint32_t queue) {
+void FrameRecorder::Submit(const Packet& packet, std::uint32_t queue, const std::function<void()>& accept) {
     auto call = make(FrameCallKind::Submit);
     call.queue = queue;
     call.address = pointer(packet.addr);
     call.size = packet.dw_num;
     call.flags = packet.flags;
-    record(std::move(call), false);
+    record(std::move(call), false, {}, accept);
 }
 
-void FrameRecorder::Flip(const FlipInfo& info) {
+void FrameRecorder::Flip(const FlipInfo& info, const std::function<void()>& accept) {
     auto call = make(FrameCallKind::Flip);
     call.cpuFlip = true;
     call.values = {info.handle, flipValue(info.index), info.mode, flipValue(info.argument)};
-    record(std::move(call), false);
+    record(std::move(call), false, {}, accept);
 }
 
 void FrameRecorder::ReserveEopFlip(const FlipInfo& info, std::uint32_t token) {
@@ -198,8 +198,8 @@ void FrameRecorder::ReserveEopFlip(const FlipInfo& info, std::uint32_t token) {
     ++recordedCalls;
 }
 
-void FrameRecorder::Suspend() {
-    record(make(FrameCallKind::Suspend), false);
+void FrameRecorder::Suspend(const std::function<void()>& accept) {
+    record(make(FrameCallKind::Suspend), false, {}, accept);
 }
 
 void FrameRecorder::PresentBuffer(const DisplayBuffer& buffer) {
@@ -236,8 +236,13 @@ void FrameRecorder::writeReferenceFrames() {
     }
 }
 
-void FrameRecorder::record(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders) {
+void FrameRecorder::record(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders, const std::function<void()>& accept) {
     std::lock_guard lock(mutex);
+    recordLocked(std::move(call), prologueCall, shaders);
+    if (accept) accept();
+}
+
+void FrameRecorder::recordLocked(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders) {
     if (state == State::Waiting) {
         if (!prologueCall) return;
         call.ordinal = ++ordinal;
@@ -360,10 +365,23 @@ void FrameRecorder::emitShaderDelta(std::span<const Shader* const> shaders) {
 
 void FrameRecorder::writeDelta(std::span<const Span> spans, std::span<const std::uint64_t> hostPages, std::span<const Range> added) {
     if (abandonIfDiskLow()) return;
-    const auto [accepted, completed] = DriverDetail::Driver::Get().SerialProgress();
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> blocks;
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> written;
+    for (const auto& span : spans) {
+        GuestMemory::CollectWrittenBlocks(span.address, static_cast<std::size_t>(span.bytes), span.since, written);
+        blocks.insert(blocks.end(), written.begin(), written.end());
+    }
+    auto& driver = DriverDetail::Driver::Get();
+    auto [accepted, completed] = driver.SerialProgress();
+    if (completed < accepted && (!added.empty() || !blocks.empty())) {
+        if (!driver.WaitIdleFor(std::chrono::seconds(10))) {
+            stop("stopped: the GPU work submitted before a memory delta did not finish within 10 s");
+            return;
+        }
+        std::tie(accepted, completed) = driver.SerialProgress();
+    }
     const auto group = deltaGroups + 1;
     bool any = false;
-    std::vector<std::pair<std::uint64_t, std::uint64_t>> written;
     std::vector<std::byte> bytes;
     for (const auto& range : added) {
         writer->WriteRange(range.address, range.bytes, range.kind, range.protection, range.access);
@@ -397,15 +415,12 @@ void FrameRecorder::writeDelta(std::span<const Span> spans, std::span<const std:
         deltaBytes += bytes.size();
         any = true;
     }
-    for (const auto& span : spans) {
-        GuestMemory::CollectWrittenBlocks(span.address, static_cast<std::size_t>(span.bytes), span.since, written);
-        for (const auto& [begin, end] : written) {
-            bytes.resize(static_cast<std::size_t>(end - begin));
-            GuestMemory::ReadCommittedUnsynced(begin, bytes);
-            writer->WriteDelta(group, completed, begin, bytes);
-            deltaBytes += bytes.size();
-            any = true;
-        }
+    for (const auto& [begin, end] : blocks) {
+        bytes.resize(static_cast<std::size_t>(end - begin));
+        GuestMemory::ReadCommittedUnsynced(begin, bytes);
+        writer->WriteDelta(group, completed, begin, bytes);
+        deltaBytes += bytes.size();
+        any = true;
     }
     if (!any) return;
     deltaGroups = group;
@@ -436,8 +451,13 @@ bool FrameRecorder::abandonIfDiskLow() {
     std::error_code error;
     const auto space = std::filesystem::space(directory, error);
     if (error || space.available >= MinimumFreeBytes) return false;
-    warnings.push_back("stopped: less than 4 GiB free on the capture volume");
-    std::fprintf(stderr, "[frame-capture] stopped: less than 4 GiB free on the capture volume\n");
+    stop("stopped: less than 4 GiB free on the capture volume");
+    return true;
+}
+
+void FrameRecorder::stop(const char* reason) {
+    warnings.push_back(reason);
+    std::fprintf(stderr, "[frame-capture] %s\n", reason);
     try {
         writer->Finish(header());
     } catch (const std::exception& failure) {
@@ -445,7 +465,6 @@ bool FrameRecorder::abandonIfDiskLow() {
     }
     writer.reset();
     state = State::Done;
-    return true;
 }
 
 void FrameRecorder::start() {

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -24,6 +24,7 @@
 #include <map>
 #include <memory>
 #include <new>
+#include <optional>
 #include <stdexcept>
 #include <string>
 #include <tuple>
@@ -1770,7 +1771,7 @@ void Rewatch(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges) {
 #endif
 }
 
-bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched) {
+bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched, bool importMayWriteGuest) {
     if (bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return import();
     if (!staysWatched) {
         constexpr std::uint64_t page = 4096;
@@ -1790,7 +1791,11 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function
     const bool covered = tracker.covers(first, static_cast<std::size_t>(stop - first));
     const auto runImport = [&] {
 #ifdef __APPLE__
-        const GuestArena::HostWrite open(reinterpret_cast<void*>(first), static_cast<std::size_t>(stop - first));
+        std::optional<GuestArena::HostWrite> open;
+        if (importMayWriteGuest) open.emplace(reinterpret_cast<void*>(first), static_cast<std::size_t>(stop - first));
+#endif
+#ifndef __APPLE__
+        static_cast<void>(importMayWriteGuest);
 #endif
         return import();
     };
@@ -1820,12 +1825,26 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function
             hashes[static_cast<std::size_t>((cursor - first) / page)] = HashPage(reinterpret_cast<const std::byte*>(cursor));
         }
     }
-    std::erase_if(spans, [&](const auto& span) { return !walkWrites(tracker, span.first, span.second, StampKind::Cpu, true); });
+    const auto saveBytes = [&](std::uint64_t begin, std::uint64_t end) {
+        lock.Suspend();
+        beforeUnwatch(begin, static_cast<std::size_t>(end - begin));
+        lock.Resume();
+    };
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> leaving;
+    std::erase_if(spans, [&](const auto& span) {
+        if (walkWrites(tracker, span.first, span.second, StampKind::Cpu, true)) return false;
+        leaving.push_back(span);
+        return true;
+    });
+    for (const auto& [begin, end] : leaving) saveBytes(begin, end);
     if (!runImport()) return false;
     const WindowCopy window{first, hashes};
     auto unwatchedFrom = first;
     for (const auto& [begin, end] : spans) {
-        if (!walkWrites(tracker, begin, end, StampKind::Cpu, true, &window)) continue;
+        if (!walkWrites(tracker, begin, end, StampKind::Cpu, true, &window)) {
+            saveBytes(begin, end);
+            continue;
+        }
 #ifdef _WIN32
         GuestArena::GuestArenaCompareResidentWhenShared_nid_no_patch(static_cast<std::uintptr_t>(begin), static_cast<std::size_t>(end - begin));
 #endif

core/libs/prx/libSceAgcDriver/Execution/src/PresentationScaler.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp"
 #include "prx/libSceAgcDriver/Execution/include/AspectFit.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
 
 namespace AgcDriver {
@@ -23,7 +24,7 @@ PresentationScaler::~PresentationScaler() {
 
 void PresentationScaler::release() noexcept {
     if (sourceImage != VK_NULL_HANDLE) context.Function<PFN_vkDestroyImage>("vkDestroyImage")(context.device, sourceImage, nullptr);
-    if (sourceMemory != VK_NULL_HANDLE) context.Function<PFN_vkFreeMemory>("vkFreeMemory")(context.device, sourceMemory, nullptr);
+    if (sourceMemory != VK_NULL_HANDLE) Graphics::ReleaseImageMemory(context, sourceMemory, sourceBytes, sourceMemoryType);
     sourceImage = VK_NULL_HANDLE;
     sourceMemory = VK_NULL_HANDLE;
     sourceWidth = 0;
@@ -50,8 +51,9 @@ void PresentationScaler::EnsureSourceImage(std::uint32_t width, std::uint32_t he
     context.Function<PFN_vkGetImageMemoryRequirements>("vkGetImageMemoryRequirements")(context.device, sourceImage, &requirements);
     VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
     allocation.allocationSize = requirements.size;
-    allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
-    Graphics::Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &sourceMemory), "vkAllocateMemory presentation source");
+    allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "presentation source");
+    sourceMemory = Graphics::AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory presentation source", &sourceMemoryType, true);
+    sourceBytes = requirements.size;
     Graphics::Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, sourceImage, sourceMemory, 0), "vkBindImageMemory presentation source");
     sourceWidth = width;
     sourceHeight = height;

core/libs/prx/libSceAgcDriver/Execution/src/ThreadPriority.cpp

@@ -12,6 +12,12 @@
 
 #ifdef _WIN32
 #include <windows.h>
+#elif defined(__APPLE__)
+#include <mach/mach.h>
+#include <mach/mach_time.h>
+#include <mach/thread_policy.h>
+#include <pthread.h>
+#include <sys/qos.h>
 #else
 #include <dlfcn.h>
 #include <sched.h>
@@ -42,7 +48,30 @@ void LogOnce(const char* role, const std::string& text) {
     std::fprintf(stderr, "[thread-priority] %s: %s\n", role, text.c_str());
 }
 
-#ifndef _WIN32
+#if defined(__APPLE__)
+
+void Raise(const char* role) {
+    if (RequestedMode() == ThreadPriorityMode::Realtime) {
+        mach_timebase_info_data_t timebase{};
+        mach_timebase_info(&timebase);
+        const auto ticks = [&](std::uint64_t nanos) {
+            return static_cast<std::uint32_t>(nanos * timebase.denom / timebase.numer);
+        };
+        thread_time_constraint_policy_data_t policy{ticks(1000000), ticks(500000), ticks(1000000), TRUE};
+        const auto status = thread_policy_set(pthread_mach_thread_np(pthread_self()), THREAD_TIME_CONSTRAINT_POLICY,
+                                              reinterpret_cast<thread_policy_t>(&policy), THREAD_TIME_CONSTRAINT_POLICY_COUNT);
+        if (status != KERN_SUCCESS)
+            throw std::runtime_error(std::string("APS5_THREAD_PRIORITY=rt: ") + role + " time-constraint policy refused (kern_return " + std::to_string(status) + ")");
+        LogOnce(role, "Mach time-constraint policy");
+        return;
+    }
+    const int error = pthread_set_qos_class_self_np(QOS_CLASS_USER_INTERACTIVE, 0);
+    if (error != 0)
+        throw std::runtime_error(std::string("APS5_THREAD_PRIORITY=high: ") + role + " user-interactive QoS refused: " + std::strerror(error));
+    LogOnce(role, "user-interactive QoS");
+}
+
+#elif !defined(_WIN32)
 
 struct DBusConnection;
 struct DBusMessage;

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -8,6 +8,7 @@
 #include "SpirvBackend/SpirvOptimizer.hpp"
 #endif
 #include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
 #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
 #include "prx/libSceAgcDriver/Execution/include/FloatFlushProbe.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
@@ -369,6 +370,8 @@ struct VulkanDevice::State {
     void* markerMapping = nullptr;
     BufferMarkerLog markerLog;
     bool deviceLostHook = false;
+    bool deferDestruction = false;
+    PFN_vkGetDeviceProcAddr contextDeviceProc = nullptr;
     VkTimeDomainKHR hostTimeDomain = VK_TIME_DOMAIN_DEVICE_KHR;
     VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES};
     std::vector<std::uint32_t> capabilities{1};
@@ -945,6 +948,7 @@ struct VulkanDevice::State {
                 destroyPool(device, pool, nullptr);
             }
             if (settlePool != VK_NULL_HANDLE) destroyPool(device, settlePool, nullptr);
+            Graphics::DestroyDeferredObjects(device);
             destroyDevice(device, nullptr);
         }
         if (instance != VK_NULL_HANDLE) {
@@ -1283,6 +1287,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         VkPhysicalDeviceProperties2 driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &driver};
         state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &driverProperties);
         driverId = driver.driverID;
+        state->deferDestruction = driver.driverID == VK_DRIVER_ID_MOLTENVK && std::getenv("APS5_NO_DEFERRED_DESTRUCTION") == nullptr;
         state->driverInfo = std::string(driver.driverName) + " " + driver.driverInfo;
         state->driverId = driver.driverID;
         state->individuallyFreeDescriptorSets = driver.driverID == VK_DRIVER_ID_MESA_LLVMPIPE;
@@ -2095,6 +2100,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     check(state->InstanceFunction<PFN_vkCreateDevice>("vkCreateDevice")(selected, &deviceInfo, nullptr, &state->device), "vkCreateDevice");
     state->shaderProfile = std::move(shaderProfile);
     if (compileOnly) return;
+    if (state->deferDestruction) state->contextDeviceProc = Graphics::DeferDestructionToIdle(state->device, state->deviceProc);
     if (calibratedTimestamps != nullptr) {
         const bool khr = std::strcmp(calibratedTimestamps, VK_KHR_CALIBRATED_TIMESTAMPS_EXTENSION_NAME) == 0;
         const auto domains = reinterpret_cast<PFN_vkGetPhysicalDeviceCalibrateableTimeDomainsKHR>(state->instanceProc(state->instance, khr ? "vkGetPhysicalDeviceCalibrateableTimeDomainsKHR" : "vkGetPhysicalDeviceCalibrateableTimeDomainsEXT"));
@@ -2362,6 +2368,7 @@ void VulkanDevice::PrepareForReplacement() {
 
 void VulkanDevice::WaitIdle() {
     APS5_LOG_CHARS_OUT_DEBUG("VulkanDevice::WaitIdle begin");
+    static_cast<void>(Graphics::StorageTexture::StoreDrawnLinear("wait idle"));
     RetirePresents(0);
     if (state->recorder) {
         // Announced with this call's return address: the [recorder] site table then names the
@@ -2468,6 +2475,7 @@ bool VulkanDevice::StoresPendingOver(std::uint64_t address, std::size_t bytes) c
 int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst) {
     static const bool storeAtLabel = Graphics::StorageTexture::StoreAtLabelRequested(std::getenv("APS5_STORE_AT_LABEL"));
     if (storeAtLabel) Graphics::StorageTexture::FlushAllPending("label");
+    if (queue == 0) static_cast<void>(Graphics::StorageTexture::StoreDrawnLinear("label"));
     Graphics::StorageTexture::StoreReadbackBeforeLabel("readback before label");
     if (!state->recorder || bytes.empty() || bytes.size() > 65536 || address % 4 != 0 || bytes.size() % 4 != 0) return 4;
     auto& recorder = *state->recorder;
@@ -3355,6 +3363,15 @@ FrameDumps& Dumps() {
 }
 
 bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) {
+    if (buffer.dccAddress != 0 && buffer.tilingMode == 0) {
+        static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
+        const auto before = profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0;
+        state->recorder->SyncThrough(buffer.dccAddress, DisplayBufferKeyBytes(buffer), true);
+        if (const auto waited = profile ? Graphics::Recorder::ThreadWaitedMs() - before : 0.0; waited > 0.0) {
+            ++presentCounters.keyWriterWaits;
+            presentCounters.keyWriterWaitMs += waited;
+        }
+    }
     static const bool storeAtFlip = Graphics::StorageTexture::StoreAtFlipRequested(std::getenv("APS5_STORE_AT_FLIP"));
     if (storeAtFlip) Graphics::StorageTexture::FlushAllPending("flip");
     else Graphics::StorageTexture::FlushDumpSelected("flip");
@@ -3975,7 +3992,7 @@ Graphics::Context VulkanDevice::buildContext() const {
         state->physical,
         state->queue,
         state->pool,
-        state->deviceProc,
+        state->contextDeviceProc != nullptr ? state->contextDeviceProc : state->deviceProc,
         state->InstanceFunction<PFN_vkGetPhysicalDeviceFormatProperties>("vkGetPhysicalDeviceFormatProperties"),
         state->InstanceFunction<PFN_vkGetPhysicalDeviceImageFormatProperties>("vkGetPhysicalDeviceImageFormatProperties"),
         state->memoryProperties,
@@ -5021,6 +5038,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
     };
     if (recorder.HasQueuedKeyStores() && (resources.HoldsLease() || recorder.AnyQueuedKeyStore(touches))) recorder.FlushKeyStores();
     if (recorder.HasQueuedStores() && (resources.HoldsLease() || recorder.AnyQueuedStore(touches))) recorder.FlushStores();
+    static_cast<void>(Graphics::StorageTexture::StoreDrawnLinearTouching("dispatch access", touches, resources.HoldsLease()));
     VkAccessFlags covered = 0;
     const auto commands = recorder.Commands(&covered);
     recordStep(PhaseRecordCommands);

core/libs/prx/libSceAgcDriver/Execution/src/WorkerSampler.cpp

@@ -163,18 +163,29 @@ struct Sampler {
         return frame;
     }
 
+    static void writeModules(std::FILE* file, const std::map<std::uint64_t, std::string>& loaded) {
+        for (const auto& [base, name] : loaded) {
+            const auto* dos = reinterpret_cast<const IMAGE_DOS_HEADER*>(base);
+            const auto* nt = reinterpret_cast<const IMAGE_NT_HEADERS*>(base + dos->e_lfanew);
+            std::fprintf(file, "# module %s 0x%lx 0x%lx\n", name.c_str(), static_cast<unsigned long>(nt->OptionalHeader.SizeOfImage), static_cast<unsigned long>(nt->FileHeader.TimeDateStamp));
+        }
+    }
+
     void write() {
         std::lock_guard lock(mutex);
         std::FILE* file = std::fopen(path.c_str(), "w");
         if (file == nullptr) return;
         std::fprintf(file, "# %llu samples\n", static_cast<unsigned long long>(samples));
+        std::map<std::uint64_t, std::string> loaded;
         for (const auto& [address, count] : counts) {
             HMODULE module = nullptr;
             char name[MAX_PATH] = "?";
             if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast<LPCSTR>(address), &module)) GetModuleFileNameA(module, name, sizeof(name));
             const char* base = std::strrchr(name, '\\');
+            if (module != nullptr) loaded.emplace(reinterpret_cast<std::uint64_t>(module), base ? base + 1 : name);
             std::fprintf(file, "%s 0x%llx %llu %llu\n", base ? base + 1 : name, static_cast<unsigned long long>(address - reinterpret_cast<std::uint64_t>(module)), static_cast<unsigned long long>(count.first), static_cast<unsigned long long>(count.second));
         }
+        writeModules(file, loaded);
         std::fclose(file);
         if (keepStacks) writeStacks();
     }
@@ -195,11 +206,7 @@ struct Sampler {
         const double tickUs = 1e6 / static_cast<double>(frequency.QuadPart);
         std::fprintf(file, "# samples=%llu depth=%d truncated=%llu suspended_us_mean=%.1f suspended_us_max=%.1f\n", static_cast<unsigned long long>(samples), depth, static_cast<unsigned long long>(stacks.Truncated()),
             samples != 0 ? static_cast<double>(suspendedTicks) * tickUs / static_cast<double>(samples) : 0.0, static_cast<double>(longestSuspendedTicks) * tickUs);
-        for (const auto& [base, name] : loaded) {
-            const auto* dos = reinterpret_cast<const IMAGE_DOS_HEADER*>(base);
-            const auto* nt = reinterpret_cast<const IMAGE_NT_HEADERS*>(base + dos->e_lfanew);
-            std::fprintf(file, "# module %s 0x%lx 0x%lx\n", name.c_str(), static_cast<unsigned long>(nt->OptionalHeader.SizeOfImage), static_cast<unsigned long>(nt->FileHeader.TimeDateStamp));
-        }
+        writeModules(file, loaded);
         const auto text = stacks.Format([&](std::uint64_t address) { return names.at(address); });
         const bool written = std::fwrite(text.data(), 1, text.size(), file) == text.size();
         if (std::fclose(file) != 0 || !written) return;
@@ -246,6 +253,7 @@ struct ProcessSampler {
         std::FILE* file = std::fopen(path.c_str(), "w");
         if (file == nullptr) return;
         std::fprintf(file, "# %llu rounds\n", static_cast<unsigned long long>(rounds));
+        std::map<std::uint64_t, std::string> loaded;
         for (auto& [id, thread] : threads) {
             std::lock_guard lock(thread.mutex);
             std::vector<std::pair<std::uint64_t, std::pair<std::uint64_t, std::uint64_t>>> hot(thread.counts.begin(), thread.counts.end());
@@ -260,9 +268,11 @@ struct ProcessSampler {
                 char name[MAX_PATH] = "?";
                 if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast<LPCSTR>(address), &module)) GetModuleFileNameA(module, name, sizeof(name));
                 const char* base = std::strrchr(name, '\\');
+                if (module != nullptr) loaded.emplace(reinterpret_cast<std::uint64_t>(module), base ? base + 1 : name);
                 std::fprintf(file, "  %s 0x%llx %llu %llu\n", base ? base + 1 : name, static_cast<unsigned long long>(address - reinterpret_cast<std::uint64_t>(module)), static_cast<unsigned long long>(hot[i].second.first), static_cast<unsigned long long>(hot[i].second.second));
             }
         }
+        Sampler::writeModules(file, loaded);
         std::fclose(file);
     }
 };

core/libs/prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp

@@ -0,0 +1,17 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DEFERREDDESTRUCTION_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DEFERREDDESTRUCTION_HPP
+
+#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
+#include <chrono>
+#include <cstddef>
+
+namespace AgcDriver::Graphics {
+
+PFN_vkGetDeviceProcAddr DeferDestructionToIdle(VkDevice device, PFN_vkGetDeviceProcAddr resolver);
+void DestroyDeferredAtIdle(const Context& context, std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now());
+void DestroyDeferredObjects(VkDevice device) noexcept;
+std::size_t DeferredObjects(VkDevice device);
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp

@@ -109,8 +109,10 @@ private:
 // stages and every pipeline state field. A configuration whose stage has no module gets a private
 // pipeline. A miss builds the pipeline outside the store's mutex (a concurrent build of the same
 // key keeps the first inserted) and records its recipe in the pipeline journal. Entries are
-// LRU-bounded (APS5_PIPELINE_STORE_ENTRIES, 1024) and only evicted once no recorded draw holds
-// them. Debug aid: APS5_NO_PIPELINE_CACHE=1 builds one per draw as before.
+// LRU-bounded and only evicted once no recorded draw holds them. The bound starts at 1024 and
+// doubles up to 4096 for a journal replay that does not fit and while pipelines it evicted keep
+// coming back; it never shrinks during a run. APS5_PIPELINE_STORE_ENTRIES fixes it. Debug aid:
+// APS5_NO_PIPELINE_CACHE=1 builds one per draw as before.
 std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
 // `created` reports whether this call built the pipeline (false: served from the store or private).
 std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, bool* created = nullptr);
@@ -124,6 +126,8 @@ struct PipelineStoreCounters {
     std::uint64_t bound = 0;
 };
 PipelineStoreCounters PipelineStoreTotals();
+// Whether the store may still raise its bound (not fixed by APS5_PIPELINE_STORE_ENTRIES, below 4096).
+bool PipelineStoreCanGrow();
 // The pipeline store's key of a configuration (empty when the configuration gets a private pipeline).
 void BuildPipelineStoreKey(std::vector<std::byte>& key, const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout);
 struct DescribedSetLayouts {

core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp

@@ -183,6 +183,7 @@ public:
     // Whether the OPEN (unsubmitted) batch writes the range: a wait on such a range must submit it.
     bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const;
     std::uint64_t PendingWriterSerial(std::uint64_t address, std::size_t bytes) const;
+    std::uint64_t PendingWriterSerial(std::uint64_t address, std::size_t bytes, std::span<const std::uint64_t> exceptNotes) const;
     // What SyncThrough(address, bytes) would wait for, for the [hooksync] attribution: the newest
     // batch whose noted range overlaps the access (the open batch counts first, with the serial it
     // will get), whether that batch's fence has already signaled (its GPU work is done, so the

core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp

@@ -32,6 +32,7 @@ void FlushCachedTextures(VkDevice device);
 void ClearCachedTextures(VkDevice device);
 std::uint64_t ReleaseTextureCaches(std::uint64_t bytes, bool newImage = false);
 std::uint64_t TextureCacheBudget(const VkPhysicalDeviceMemoryProperties& memory);
+std::uint64_t StorageTextureCeiling(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported);
 std::uint64_t StorageTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t sampledBytes, std::uint64_t storageBytes);
 std::uint64_t SampledTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t textureBytes, std::uint64_t storageBytes = 0);
 bool SampledBudgetReportDue(std::uint64_t reported, std::uint64_t budget, std::chrono::steady_clock::duration sinceReport);

core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp

@@ -181,6 +181,7 @@ ShaderStages DecodeShaderStages(const QueueState& queue, bool vertexWorkgroup =
 void ApplyPointSize(const State& state, ShaderRecompiler::ShaderVertexStageInfo& vertex);
 State DecodeState(const QueueState& queue, bool vertexWorkgroup = false);
 void ApplyDepthRemap(const Context& context, State& state);
+void ApplyDepthBoundsEmulation(const Context& context, State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel);
 void NormalizeUnusedViewportDepth(State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel);
 std::array<VkSampleLocationEXT, 32> SampleLocations(const State& state);
 std::array<std::uint8_t, 32> PixelSampleLocations(const State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel);

core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp

@@ -16,6 +16,7 @@
 #include <memory>
 #include <mutex>
 #include <optional>
+#include <functional>
 #include <span>
 #include <tuple>
 #include <utility>
@@ -204,6 +205,7 @@ public:
     bool BlockUnits() const { return blockUnits; }
     std::uint32_t TrackedUnits() const { return trackedLayers; }
     void MarkSliceDirty(std::uint32_t slice, std::uint32_t count = 1);
+    bool MarkDrawn(std::uint32_t slice, std::uint32_t count, VkRect2D scissor);
     void Flush();
     bool Dirty() const;
     // Stores every pending image overlapping the range, except `except`; returns whether any was.
@@ -220,6 +222,8 @@ public:
     static constexpr std::uint64_t SmallStoreBytes = 64 * 1024;
     static std::size_t StoreSmallPending(const char* reason);
     static void StoreReadbackBeforeLabel(const char* reason);
+    static std::size_t StoreDrawnLinear(const char* reason);
+    static std::size_t StoreDrawnLinearTouching(const char* reason, const std::function<bool(std::uint64_t, std::uint64_t)>& touches, bool all);
     static bool AnyPending();
     static void FlushDumpSelected(const char* reason);
     static bool StoreAtFlipRequested(const char* value);
@@ -482,6 +486,27 @@ private:
     // (`shadowed` receives those guest ranges, `imported` the pieces written to the import).
     std::uint64_t uploadWindows(const HostImport& import, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, bool discard = false);
     std::uint64_t writeBackWindows(std::span<const ImportSegment> segments, std::span<const std::pair<std::uint64_t, std::uint64_t>> keep, std::uint64_t firstStored, std::uint64_t lastStored, std::vector<ShadowedRange>& shadowed, std::vector<std::pair<std::uint64_t, std::uint64_t>>& imported, Buffer* save = nullptr);
+    struct DrawnRect {
+        std::uint32_t left = 0;
+        std::uint32_t top = 0;
+        std::uint32_t right = 0;
+        std::uint32_t bottom = 0;
+    };
+    struct DrawnArea {
+        bool drawn = false;
+        bool rect = false;
+        std::uint32_t left = 0;
+        std::uint32_t top = 0;
+        std::uint32_t right = 0;
+        std::uint32_t bottom = 0;
+        std::vector<DrawnRect> rects;
+    };
+    DrawnArea drawnArea() const;
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> drawnSpans(const DrawnArea& area, std::span<const std::pair<std::uint64_t, std::uint64_t>> keep) const;
+    std::uint64_t storeDrawnRect(const DrawnArea& area, std::span<const ImportSegment> segments, std::span<const std::pair<std::uint64_t, std::uint64_t>> stores, const std::vector<std::uint32_t>& saveUnits);
+    void addDrawnRect(DrawnRect rect);
+    void forgetDrawn();
+    void noteStore(std::uint64_t note);
     // Per-layer validity (thin array surfaces whose layers are 64 KiB-aligned guest slices; every
     // other surface is one tracked layer): a layer's generation is the write generation its guest
     // bytes were last known to match the image at, and a pending layer holds results (a clear, a
@@ -518,6 +543,7 @@ private:
     // Per 64 KiB tracker block of the surface (block 0 holds the base address): the generation of
     // the tracked layer it belongs to, for GuestMemory::ChangedBlocks.
     void blockGenerations(std::vector<std::uint64_t>& generations) const;
+    std::uint64_t unitBlockGeneration(std::uint32_t unit, std::uint64_t at) const;
     // Another image of exactly this surface under another storage format with results pending
     // (block units, uncompressed, still cached), if any: Refresh takes its pending units from its
     // image on the device (borrowUnits) instead of having them stored and re-read, and the alias
@@ -602,6 +628,7 @@ private:
     std::uint32_t trackedLayers = 1;
     std::uint64_t trackedLayerBytes = 0;
     std::vector<std::uint64_t> layerGeneration;
+    std::vector<std::uint64_t> completionKept;
     std::vector<bool> layerPending;
     std::vector<bool> layerBehind;
     bool blockUnits = false;
@@ -641,6 +668,12 @@ private:
     VkFormat storageFormat = VK_FORMAT_UNDEFINED;
     // Results are on the GPU only (guarded by the pending-write registry lock).
     bool dirty = false;
+    bool drawnListed = false;
+    bool drawnWhole = false;
+    bool drawingTarget = false;
+    bool pendingFromOthers = false;
+    std::vector<DrawnRect> drawnRects;
+    std::vector<std::uint64_t> storeNotes;
     std::uint64_t version = 0;
     std::uint64_t writes = 0;
     // `original` holds the guest bytes; false after a GPU-side clear, which never read them.

core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp

@@ -75,7 +75,7 @@ struct SurfaceGeometry {
 };
 SurfaceGeometry DescribeSurface(const GuestTextureResource& descriptor);
 bool LevelsFitAllocation(const GuestTextureResource& surface, std::uint32_t levels);
-GuestTextureResource ViewedArrayWindow(const GuestTextureResource& viewed);
+GuestTextureResource ViewedArrayWindow(const GuestTextureResource& viewed, bool keysReadAsStored);
 GuestTextureResource ArrayLayerWindow(const GuestTextureResource& viewed, std::uint32_t maxArrayLayers);
 
 }

core/libs/prx/libSceAgcDriver/Graphics/src/DeferredDestruction.cpp

@@ -0,0 +1,134 @@
+#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
+#include <atomic>
+#include <cstring>
+#include <mutex>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace AgcDriver::Graphics {
+
+namespace {
+
+constexpr std::size_t DrainObjects = 512;
+constexpr auto DrainAge = std::chrono::milliseconds(100);
+
+struct Deferred {
+    PFN_vkDestroyBuffer destroyBuffer = nullptr;
+    PFN_vkFreeMemory freeMemory = nullptr;
+    std::vector<VkBuffer> buffers;
+    std::vector<VkDeviceMemory> memory;
+    std::chrono::steady_clock::time_point oldest{};
+};
+
+struct Registry {
+    std::mutex mutex;
+    std::unordered_map<VkDevice, Deferred> devices;
+    std::atomic<PFN_vkGetDeviceProcAddr> resolver{nullptr};
+};
+
+Registry& registry() {
+    static auto& state = *new Registry();
+    return state;
+}
+
+void destroyAll(VkDevice device, const Deferred& deferred) noexcept {
+    for (const auto buffer : deferred.buffers) deferred.destroyBuffer(device, buffer, nullptr);
+    for (const auto memory : deferred.memory) deferred.freeMemory(device, memory, nullptr);
+}
+
+template<typename THandle>
+bool defer(VkDevice device, THandle handle, std::vector<THandle> Deferred::*list) noexcept {
+    try {
+        auto& state = registry();
+        std::lock_guard lock(state.mutex);
+        const auto found = state.devices.find(device);
+        if (found == state.devices.end()) return false;
+        auto& deferred = found->second;
+        if (deferred.buffers.empty() && deferred.memory.empty()) deferred.oldest = std::chrono::steady_clock::now();
+        (deferred.*list).push_back(handle);
+        return true;
+    } catch (...) {
+        return false;
+    }
+}
+
+VKAPI_ATTR void VKAPI_CALL deferredDestroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks* allocator) {
+    if (buffer == VK_NULL_HANDLE || defer(device, buffer, &Deferred::buffers)) return;
+    reinterpret_cast<PFN_vkDestroyBuffer>(registry().resolver.load(std::memory_order_acquire)(device, "vkDestroyBuffer"))(device, buffer, allocator);
+}
+
+VKAPI_ATTR void VKAPI_CALL deferredFreeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks* allocator) {
+    if (memory == VK_NULL_HANDLE || defer(device, memory, &Deferred::memory)) return;
+    reinterpret_cast<PFN_vkFreeMemory>(registry().resolver.load(std::memory_order_acquire)(device, "vkFreeMemory"))(device, memory, allocator);
+}
+
+VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL deferringDeviceProc(VkDevice device, const char* name) {
+    if (std::strcmp(name, "vkDestroyBuffer") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&deferredDestroyBuffer);
+    if (std::strcmp(name, "vkFreeMemory") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&deferredFreeMemory);
+    return registry().resolver.load(std::memory_order_acquire)(device, name);
+}
+
+}
+
+PFN_vkGetDeviceProcAddr DeferDestructionToIdle(VkDevice device, PFN_vkGetDeviceProcAddr resolver) {
+    if (device == VK_NULL_HANDLE || resolver == nullptr) throw std::invalid_argument("AGC graphics: deferred destruction needs a device and its resolver");
+    Deferred deferred;
+    deferred.destroyBuffer = reinterpret_cast<PFN_vkDestroyBuffer>(resolver(device, "vkDestroyBuffer"));
+    deferred.freeMemory = reinterpret_cast<PFN_vkFreeMemory>(resolver(device, "vkFreeMemory"));
+    if (deferred.destroyBuffer == nullptr || deferred.freeMemory == nullptr) throw std::runtime_error("AGC graphics: missing Vulkan function: vkDestroyBuffer or vkFreeMemory");
+    auto& state = registry();
+    std::lock_guard lock(state.mutex);
+    const auto current = state.resolver.load(std::memory_order_acquire);
+    if (current != nullptr && current != resolver) throw std::runtime_error("AGC graphics: deferred destruction found two different device resolvers");
+    if (!state.devices.emplace(device, std::move(deferred)).second) throw std::runtime_error("AGC graphics: deferred destruction was set up twice for one device");
+    state.resolver.store(resolver, std::memory_order_release);
+    return &deferringDeviceProc;
+}
+
+void DestroyDeferredAtIdle(const Context& context, std::chrono::steady_clock::time_point now) {
+    auto& state = registry();
+    {
+        std::lock_guard lock(state.mutex);
+        const auto found = state.devices.find(context.device);
+        if (found == state.devices.end()) return;
+        const auto objects = found->second.buffers.size() + found->second.memory.size();
+        if (objects == 0 || (objects < DrainObjects && now - found->second.oldest < DrainAge)) return;
+    }
+    const auto idle = context.Function<PFN_vkQueueWaitIdle>("vkQueueWaitIdle")(context.queue);
+    if (idle != VK_SUCCESS && idle != VK_ERROR_DEVICE_LOST) throw std::runtime_error("AGC graphics: vkQueueWaitIdle before destroying deferred objects: Vulkan result " + std::to_string(idle));
+    Deferred taken;
+    {
+        std::lock_guard lock(state.mutex);
+        const auto found = state.devices.find(context.device);
+        if (found == state.devices.end()) return;
+        taken.destroyBuffer = found->second.destroyBuffer;
+        taken.freeMemory = found->second.freeMemory;
+        taken.buffers.swap(found->second.buffers);
+        taken.memory.swap(found->second.memory);
+    }
+    destroyAll(context.device, taken);
+}
+
+void DestroyDeferredObjects(VkDevice device) noexcept {
+    Deferred taken;
+    {
+        auto& state = registry();
+        std::lock_guard lock(state.mutex);
+        const auto found = state.devices.find(device);
+        if (found == state.devices.end()) return;
+        taken = std::move(found->second);
+        state.devices.erase(found);
+    }
+    destroyAll(device, taken);
+}
+
+std::size_t DeferredObjects(VkDevice device) {
+    auto& state = registry();
+    std::lock_guard lock(state.mutex);
+    const auto found = state.devices.find(device);
+    return found == state.devices.end() ? 0 : found->second.buffers.size() + found->second.memory.size();
+}
+
+}

core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp

@@ -719,10 +719,30 @@ public:
         RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_MEMORY_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
         context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, storage->Image(), VK_IMAGE_LAYOUT_GENERAL, depthPlane.staging->Handle(), 1, &colorRegion);
         RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
-        context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, depthPlane.staging->Handle(), image, VK_IMAGE_LAYOUT_GENERAL, 1, &depthRegion);
-        RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT);
+        if (target.stencilAddress == 0) {
+            VkImageMemoryBarrier discard{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
+            discard.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
+            discard.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+            discard.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+            discard.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+            discard.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+            discard.image = image;
+            discard.subresourceRange = {VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, 1};
+            const auto barrier = context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier");
+            barrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &discard);
+            context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, depthPlane.staging->Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &depthRegion);
+            VkImageMemoryBarrier toGeneral = discard;
+            toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
+            toGeneral.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
+            toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+            toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL;
+            barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral);
+        } else {
+            context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, depthPlane.staging->Handle(), image, VK_IMAGE_LAYOUT_GENERAL, 1, &depthRegion);
+            RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT);
+        }
         if (batch) batch->SubmitAndWait();
-        else Recorder::CountBarriers(Recorder::CommandClass::Copy, 3);
+        else Recorder::CountBarriers(Recorder::CommandClass::Copy, target.stencilAddress == 0 ? 4 : 3);
     }
 
     std::shared_ptr<StorageTexture> Mirror(const GuestTextureResource& resource) {
@@ -1227,8 +1247,10 @@ public:
                 continue;
             }
             surface = {};
-            const GuestLayer current{resource.baseAddress + geometry.GuestLayerOffset(layer), geometry.layerBytes, resource.tileMode, geometry.thick, GuestMemory::CollectWrites(resource.baseAddress + geometry.GuestLayerOffset(layer), static_cast<std::size_t>(geometry.layerBytes))};
-            if (held.generation != 0 && held.address == current.address && held.bytes == current.bytes && held.tileMode == current.tileMode && held.thick == current.thick && !Recorder::SnapshotWriteOverlaps(current.address, static_cast<std::size_t>(current.bytes)) && GuestMemory::UnchangedSince(current.address, static_cast<std::size_t>(current.bytes), held.generation)) {
+            GuestLayer current{resource.baseAddress + geometry.GuestLayerOffset(layer), geometry.layerBytes, resource.tileMode, geometry.thick, GuestMemory::CollectWrites(resource.baseAddress + geometry.GuestLayerOffset(layer), static_cast<std::size_t>(geometry.layerBytes))};
+            current.uncommitted = current.generation == 0 && GuestMemory::CommittedRanges(current.address, static_cast<std::size_t>(current.bytes)).empty();
+            const bool unchanged = current.uncommitted ? held.uncommitted : held.generation != 0 && GuestMemory::UnchangedSince(current.address, static_cast<std::size_t>(current.bytes), held.generation);
+            if (held.address == current.address && held.bytes == current.bytes && held.tileMode == current.tileMode && held.thick == current.thick && !Recorder::SnapshotWriteOverlaps(current.address, static_cast<std::size_t>(current.bytes)) && unchanged) {
                 held.generation = current.generation;
                 continue;
             }
@@ -1266,6 +1288,7 @@ private:
         TextureTileMode tileMode = TextureTileMode::kLinear;
         bool thick = false;
         std::uint64_t generation = 0;
+        bool uncommitted = false;
     };
 
     VkImage image = VK_NULL_HANDLE;

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -1921,7 +1921,7 @@ struct Kept {
 // The completion side of a recorded draw: the kept objects, the record check, the lease outcome,
 // the GPU write notes, the copied-buffer write-back (listed in DrawCopiedWriters or run as a
 // completion action) and the targets marked dirty.
-void keepRecordedDraw(Recorder& recorder, const std::shared_ptr<ShaderResources>& resources, std::shared_ptr<Pipeline> pipeline, std::shared_ptr<Framebuffer> framebuffer, DrawInputs& inputs, std::vector<std::shared_ptr<StorageTexture>> targets, std::vector<std::pair<std::uint32_t, std::uint32_t>> slices, std::unique_ptr<DeviceBuffer> scratch, std::function<void()> checkRecords, bool listed, bool completion, const DrawOutcome& outcome) {
+void keepRecordedDraw(Recorder& recorder, const std::shared_ptr<ShaderResources>& resources, std::shared_ptr<Pipeline> pipeline, std::shared_ptr<Framebuffer> framebuffer, DrawInputs& inputs, std::vector<std::shared_ptr<StorageTexture>> targets, std::vector<std::pair<std::uint32_t, std::uint32_t>> slices, VkRect2D scissor, std::unique_ptr<DeviceBuffer> scratch, std::function<void()> checkRecords, bool listed, bool completion, const DrawOutcome& outcome) {
     auto kept = std::make_shared<Kept>();
     kept->resources = resources;
     kept->pipeline = std::move(pipeline);
@@ -1961,14 +1961,15 @@ void keepRecordedDraw(Recorder& recorder, const std::shared_ptr<ShaderResources>
         if (resources->Reusable()) recorder.OnComplete([resources] { resources->CheckFaults(); });
         else recorder.OnComplete([resources] { resources->WriteBackBuffers(); });
     }
+    std::vector<bool> storeNow(residents.size(), false);
     for (std::size_t index = 0; index < residents.size(); ++index) {
         if (residents[index] == nullptr) continue;
-        if (index < slices.size()) residents[index]->MarkSliceDirty(slices[index].first, slices[index].second);
+        if (residents[index]->Descriptor().tileMode == TextureTileMode::kLinear) storeNow[index] = !residents[index]->MarkDrawn(index < slices.size() ? slices[index].first : 0u, index < slices.size() ? slices[index].second : 0u, scissor) || !residents[index]->Cached();
+        else if (index < slices.size()) residents[index]->MarkSliceDirty(slices[index].first, slices[index].second);
         else residents[index]->MarkDirty();
     }
-    for (const auto& resident : residents) {
-        if (resident == nullptr || resident->Descriptor().tileMode != TextureTileMode::kLinear) continue;
-        StorageTexture::FlushPending(resident->Descriptor().baseAddress, static_cast<std::size_t>(resident->GuestBytes()), nullptr, "linear target");
+    for (std::size_t index = 0; index < residents.size(); ++index) {
+        if (storeNow[index]) StorageTexture::FlushPending(residents[index]->Descriptor().baseAddress, static_cast<std::size_t>(residents[index]->GuestBytes()), nullptr, "linear target");
     }
 }
 
@@ -2195,7 +2196,6 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
         recorder->FlushStores();
         outcome.storesFlushed = true;
     }
-    const auto drawBindings = resources.PrepareDrawBindings(*recorder, record.moved);
     const bool capture = CaptureInputsEnabled();
     const bool meshIndirect = state.stages.mesh && args != nullptr;
     const bool addressBased = resources.HoldsLease();
@@ -2217,6 +2217,8 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
     }
     for (const auto image : record.multisampleImages) passAttachments.emplace_back(image, true);
     const bool unknownMemory = addressBased || resources.UsesAddressTable() || resources.HasCopiedWrites();
+    static_cast<void>(StorageTexture::StoreDrawnLinearTouching("draw access", [&](std::uint64_t begin, std::uint64_t end) { return std::any_of(passReads.begin(), passReads.end(), [&](const auto& read) { return read.first < end && begin < read.second; }) || resources.WritesOverlap(begin, static_cast<std::size_t>(end - begin)); }, unknownMemory));
+    const auto drawBindings = resources.PrepareDrawBindings(*recorder, record.moved);
     static const bool addressReadPasses = std::getenv("APS5_NO_ADDRESS_READ_PASSES") == nullptr;
     const bool readOnlyBda = addressReadPasses && resources.BdaReadOnly();
     outcome.addressDraw = addressBased;
@@ -2361,7 +2363,7 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
     }
     recorder->LeaveRenderPassOpen(passKey, drawTiming, fenced, passAccess, std::move(storeProxies));
     timer.phase(PhaseRecord);
-    keepRecordedDraw(*recorder, record.resources, std::move(record.pipeline), std::move(record.framebuffer), inputs, std::move(record.targets), std::move(record.targetSlices), std::move(scratch), std::move(checkRecords), record.listed, record.completion, outcome);
+    keepRecordedDraw(*recorder, record.resources, std::move(record.pipeline), std::move(record.framebuffer), inputs, std::move(record.targets), std::move(record.targetSlices), state.scissor, std::move(scratch), std::move(checkRecords), record.listed, record.completion, outcome);
     timer.phase(PhaseKeep);
     if (record.waited) {
         // The wait the dispatch path makes for such work (source 3, "address-based"): the
@@ -2952,6 +2954,14 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D
         report(outcome.waited ? (outcome.reason == SyncLease ? " recorded then waited (lease)" : " recorded then waited (copied writes)") : completion ? " recorded with completion" : " recorded");
         return;
     }
+    const bool gpuIndirect = args != nullptr && indirect.path == IndirectDrawPath::Gpu;
+    const auto touches = [&](std::uint64_t begin, std::uint64_t end) {
+        const auto bytes = static_cast<std::size_t>(end - begin);
+        if (resources->WritesOverlap(begin, bytes) || resources->ReadsOverlap(begin, bytes)) return true;
+        if (!gpuIndirect) return false;
+        return (begin < args->arguments + args->RangeBytes() && args->arguments < end) || (args->countIndirect && begin < args->countAddress + 4 && args->countAddress < end);
+    };
+    static_cast<void>(StorageTexture::StoreDrawnLinearTouching("draw access", [&](std::uint64_t begin, std::uint64_t end) { return touches(begin, end) || std::any_of(inputs.inPlaceReads.begin(), inputs.inPlaceReads.end(), [&](const auto& read) { return read.first < end && begin < read.second; }); }, resources->HoldsLease() || resources->UsesAddressTable() || resources->HasCopiedWrites()));
     APS5_LOG_CHARS_OUT_DEBUG("Creating CommandBatch");
     std::optional<CommandBatch> batch;
     if (!recorded) {
@@ -2961,16 +2971,9 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D
     }
     using CommandClass = Recorder::CommandClass;
     const auto countBarrier = [&](std::uint32_t count = 1) { if (recorded) Recorder::CountBarriers(CommandClass::Draw, count); };
-    const bool gpuIndirect = args != nullptr && indirect.path == IndirectDrawPath::Gpu;
     // A queued DCC key store over memory the draw writes or reads in place (unknown for an
     // address-based build), or over its GPU-side records, lands before it, as before a dispatch
     // (VulkanDevice::dispatch).
-    const auto touches = [&](std::uint64_t begin, std::uint64_t end) {
-        const auto bytes = static_cast<std::size_t>(end - begin);
-        if (resources->WritesOverlap(begin, bytes) || resources->ReadsOverlap(begin, bytes)) return true;
-        if (!gpuIndirect) return false;
-        return (begin < args->arguments + args->RangeBytes() && args->arguments < end) || (args->countIndirect && begin < args->countAddress + 4 && args->countAddress < end);
-    };
     if (recorded && recorder->HasQueuedKeyStores() && (resources->HoldsLease() || recorder->AnyQueuedKeyStore(touches))) recorder->FlushKeyStores();
     // A queued label store over such memory likewise (Recorder::RecordStore).
     if (recorded && recorder->HasQueuedStores() && (resources->HoldsLease() || recorder->AnyQueuedStore(touches))) recorder->FlushStores();
@@ -3137,7 +3140,7 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D
     APS5_LOG_CHARS_OUT_DEBUG("SubmitAndWait");
     timer.phase(PhaseRecord);
     if (recorded) {
-        keepRecordedDraw(*recorder, resources, pipeline, framebuffer, inputs, owners, [&] { std::vector<std::pair<std::uint32_t, std::uint32_t>> slices; for (const auto& binding : targets) slices.emplace_back(binding.color.arraySlice, binding.color.arrayLastSlice != 0 ? binding.color.layers : 1u); return slices; }(), std::move(scratch), std::move(checkRecords), listed, completion, outcome);
+        keepRecordedDraw(*recorder, resources, pipeline, framebuffer, inputs, owners, [&] { std::vector<std::pair<std::uint32_t, std::uint32_t>> slices; for (const auto& binding : targets) slices.emplace_back(binding.color.arraySlice, binding.color.arrayLastSlice != 0 ? binding.color.layers : 1u); return slices; }(), state.scissor, std::move(scratch), std::move(checkRecords), listed, completion, outcome);
         timer.phase(PhaseKeep);
         if (outcome.waited) {
             // The wait the dispatch path makes for such work (source 3, "address-based"): the

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -69,6 +69,7 @@ struct ImageMirror {
     std::uint64_t bytes = 0;
     bool writable = false;
     bool heap = false;
+    bool joinedHeap = false;
     bool sparse = false;
     std::uint64_t reserved = 0;
     std::vector<std::pair<std::uint64_t, std::uint64_t>> committed;
@@ -1492,8 +1493,32 @@ bool evictingRefusalCurrent(const HostImports& state, const EvictingRefusal& ref
     return refusal.epoch == state.epoch && refusal.liveBytes == state.liveBytes && budget <= refusal.budget && refusal.window == importWindow.load(std::memory_order_relaxed) && state.useCounter < refusal.ready && (state.buildFloor <= refusal.floor || state.useCounter < refusal.readyHeld);
 }
 
+void noteRetiredUnwatched(std::vector<std::pair<std::uint64_t, std::uint64_t>>& spans, const HostImport& entry, const GuestAllocations::Lease& lease) {
+    if (!entry.unwatched) return;
+    for (const auto& range : entry.ranges) {
+        if (leasedRangeOwner(lease, range->address) != range) continue;
+        const auto begin = std::max(range->address, entry.base);
+        const auto end = std::min(range->address + range->bytes, entry.base + entry.bytes);
+        if (begin < end) spans.emplace_back(begin, end);
+    }
+}
+
+void rewatchRetired(const HostImports& state, std::vector<std::pair<std::uint64_t, std::uint64_t>> spans) {
+    std::erase_if(spans, [&](const auto& span) {
+        auto it = state.imports.upper_bound(span.first);
+        if (it != state.imports.begin()) --it;
+        for (; it != state.imports.end() && it->first < span.second; ++it) {
+            if (it->second.unwatched && span.first < it->first + it->second.bytes) return true;
+        }
+        const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> one{{span}};
+        return StorageTexture::AnyLiveOverlaps(one);
+    });
+    if (!spans.empty()) GuestMemory::Rewatch(spans);
+}
+
 std::uint64_t retireLeastRecent(const Context& context, HostImports& state, const GuestAllocations::Lease& lease, std::uint64_t wanted) {
     std::uint64_t freed = 0;
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> unwatched;
     while (freed < wanted) {
         auto oldest = state.imports.end();
         for (auto it = state.imports.begin(); it != state.imports.end(); ++it) {
@@ -1501,8 +1526,10 @@ std::uint64_t retireLeastRecent(const Context& context, HostImports& state, cons
         }
         if (oldest == state.imports.end()) break;
         freed += oldest->second.pinnedBytes;
+        noteRetiredUnwatched(unwatched, oldest->second, lease);
         retireImport(context.device, state, oldest, lease);
     }
+    rewatchRetired(state, std::move(unwatched));
     return freed;
 }
 
@@ -1949,7 +1976,7 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
         GuestMemory::ImportWatched(base, bytes, [&] {
             step = createBackingImport(context, entry, backing, result);
             return step == nullptr;
-        }, watchedDmaBuf);
+        }, watchedDmaBuf, false);
     }
     if (step == nullptr && entry.dmaBuf && bytes >= settleFloorBytes && context.settleQueue != VK_NULL_HANDLE && context.settleQueueMutex != nullptr) {
         static const bool traceSettle = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr;
@@ -2087,6 +2114,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
     std::erase_if(state.sparseRunEnds, [&](const auto& entry) { return !starts(entry.first); });
     std::erase_if(state.sparseRunJoins, [&](const auto& entry) { return !starts(entry.first); });
     const auto budget = effectiveBudget(context, state);
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> unwatched;
     for (auto it = state.imports.begin(); it != state.imports.end();) {
         if (sameRange(it->second, lease, budget)) {
             if (it->second.unwatched) GuestMemory::Unwatch(it->first, it->second.bytes);
@@ -2096,9 +2124,11 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
         static const bool trace = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr;
         if (trace && leasedRangeOwner(lease, it->first, it->first + it->second.bytes) != nullptr) std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx retired: the range was mapped again\n", static_cast<unsigned long long>(it->first), static_cast<unsigned long long>(it->second.bytes));
         const auto next = std::next(it);
+        noteRetiredUnwatched(unwatched, it->second, lease);
         retireImport(context.device, state, it, lease);
         it = next;
     }
+    rewatchRetired(state, std::move(unwatched));
     for (auto it = state.failed.begin(); it != state.failed.end();) it = sameRange(it->second, lease, budget) ? std::next(it) : state.failed.erase(it);
     for (auto it = state.failedJoins.begin(); it != state.failedJoins.end();) it = sameRun(it->second, joinedRun(lease, it->second.base, it->second.base + it->second.bytes)) ? std::next(it) : state.failedJoins.erase(it);
     for (auto it = state.failedBlocks.begin(); it != state.failedBlocks.end();) it = sameRun(it->second, exactRun(lease, it->first.first, it->first.second)) ? std::next(it) : state.failedBlocks.erase(it);
@@ -2269,6 +2299,39 @@ void unlistMirror(ImageMirrors& state, ImageMirror& mirror) {
     mirror.listed = false;
 }
 
+struct JoinedHeapRange {
+    std::vector<std::weak_ptr<const GuestAllocations::Range>> members;
+    std::shared_ptr<const GuestAllocations::Range> range;
+};
+
+struct JoinedHeapRanges {
+    std::mutex mutex;
+    std::map<std::uint64_t, JoinedHeapRange> entries;
+};
+
+JoinedHeapRanges& JoinedHeaps() {
+    static JoinedHeapRanges ranges;
+    return ranges;
+}
+
+bool releaseExpiredJoinedHeaps() {
+    std::vector<std::shared_ptr<const GuestAllocations::Range>> released;
+    {
+        auto& joined = JoinedHeaps();
+        std::lock_guard lock(joined.mutex);
+        for (auto it = joined.entries.begin(); it != joined.entries.end();) {
+            const auto& members = it->second.members;
+            if (std::none_of(members.begin(), members.end(), [](const auto& member) { return member.expired(); })) {
+                ++it;
+                continue;
+            }
+            released.push_back(std::move(it->second.range));
+            it = joined.entries.erase(it);
+        }
+    }
+    return !released.empty();
+}
+
 std::uint64_t blockBytes(const GuestRuns& runs) {
     constexpr std::uint64_t block = ResidentBuffer::BlockBytes;
     std::uint64_t blocks = 0;
@@ -2783,7 +2846,7 @@ const char* heapCopyReason(bool overBudget, bool watched) {
 // The mirror for a leased image range: the existing one, or a new one when none exists or the
 // registered Range object changed (a protection change). Null when the range cannot be mirrored. An
 // existing writable mirror's blocks are appended to `blocks` for the caller's one compare of every
-std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::shared_ptr<const GuestAllocations::Range>& range, std::vector<RefreshBlock>& blocks, bool heap, std::uint64_t begin = 0, std::uint64_t bytes = 0) {
+std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::shared_ptr<const GuestAllocations::Range>& range, std::vector<RefreshBlock>& blocks, bool heap, std::uint64_t begin = 0, std::uint64_t bytes = 0, bool joinedHeap = false) {
     auto selected = *range;
     if (bytes != 0) {
         Require(begin >= range->address && bytes <= range->bytes && begin - range->address <= range->bytes - bytes, "heap mirror selection exceeds its registered range");
@@ -2804,7 +2867,7 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
         }
         if (state.failed.contains(selected.address)) return nullptr;
         const auto found = state.entries.find(selected.address);
-        if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->reserved == selected.bytes && found->second->writable == range->writable && found->second->heap == heap) mirror = found->second;
+        if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->reserved == selected.bytes && found->second->writable == range->writable && found->second->heap == heap && found->second->joinedHeap == joinedHeap) mirror = found->second;
     }
     if (mirror != nullptr) {
         if (!mirror->sparse) {
@@ -2827,6 +2890,7 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
     mirror->sparse = sparse;
     mirror->writable = range->writable;
     mirror->heap = heap;
+    mirror->joinedHeap = joinedHeap;
     mirror->range = range;
     GuestRuns backed;
     if (heap) {
@@ -2959,9 +3023,11 @@ std::shared_ptr<ImageMirror> findMirror(const Context& context, std::uint64_t be
 
 // Drops mirrors whose registered range is gone (a later build made a new one for the replacement).
 void sweepMirrors() {
+    const bool released = releaseExpiredJoinedHeaps();
     const auto space = Spaces().current.load();
     auto& state = Mirrors();
     std::lock_guard lock(state.mutex);
+    if (released) ++state.version;
     if (space != nullptr && state.pinnedSpace == space->serial && state.pinnedVersion == state.version) return;
     ++state.sweeps;
     bool pinned = space != nullptr;
@@ -2985,6 +3051,14 @@ void sweepMirrors() {
 
 void releaseImportedMirrors(const Context& context, std::span<const std::pair<std::uint64_t, std::uint64_t>> imported) {
     if (imported.empty()) return;
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> coverage(imported.begin(), imported.end());
+    std::sort(coverage.begin(), coverage.end());
+    std::size_t count = 0;
+    for (const auto& interval : coverage) {
+        if (count != 0 && interval.first <= coverage[count - 1].second) coverage[count - 1].second = std::max(coverage[count - 1].second, interval.second);
+        else coverage[count++] = interval;
+    }
+    coverage.resize(count);
     std::vector<std::shared_ptr<ImageMirror>> released;
     auto& state = Mirrors();
     std::lock_guard lock(state.mutex);
@@ -2998,6 +3072,13 @@ void releaseImportedMirrors(const Context& context, std::span<const std::pair<st
                 ++it;
                 continue;
             }
+            if (mirror->joinedHeap) {
+                const auto covered = std::upper_bound(coverage.begin(), coverage.end(), mirror->base, [](std::uint64_t address, const auto& interval) { return address < interval.first; });
+                if (covered == coverage.begin() || std::prev(covered)->second < mirror->base + mirror->reserved) {
+                    ++it;
+                    continue;
+                }
+            }
             ++state.heapImported;
             state.heapImportedBytes += mirror->accounted;
             unlistMirror(state, *mirror);
@@ -4388,15 +4469,40 @@ bool GuestBufferMemory::sparseHeapCandidate(const Region& region) {
     return region.sparseHeap && region.sparse && region.mirror == nullptr && region.direct == nullptr && (region.hostBacked || region.writable) && mirrorsEnabled() && sparseMirrorsEnabled();
 }
 
+std::shared_ptr<const GuestAllocations::Range> joinedHeapRange(const GuestAllocations::Lease& lease, std::uint64_t begin, std::uint64_t end) {
+    const auto run = crossedRun(lease, begin, end);
+    if (run.ranges.size() < 2 || std::any_of(run.ranges.begin(), run.ranges.end(), [](const auto& range) { return !range->releasable; })) return nullptr;
+    releaseExpiredJoinedHeaps();
+    auto& joined = JoinedHeaps();
+    std::lock_guard lock(joined.mutex);
+    auto& entry = joined.entries[run.base];
+    bool same = entry.range != nullptr && entry.members.size() == run.ranges.size();
+    for (std::size_t index = 0; same && index < run.ranges.size(); ++index) same = sameRange(entry.members[index], run.ranges[index]);
+    if (!same) {
+        const auto& first = *run.ranges.front();
+        entry.members.assign(run.ranges.begin(), run.ranges.end());
+        entry.range = std::make_shared<const GuestAllocations::Range>(GuestAllocations::Range{run.base, static_cast<std::size_t>(run.bytes), true, first.writable, first.allocationAddress, first.allocationBytes, true, first.gpu, first.backingSerial});
+    }
+    return entry.range;
+}
+
 bool GuestBufferMemory::bindSparseMirror(Region& region, const GuestAllocations::Lease& lease) {
     const auto backedEnd = region.backed.empty() ? region.begin : region.backed.back().second;
     const auto neededEnd = std::min(region.end, std::max(backedEnd, std::max(region.lastView, region.begin) + 4u));
-    const auto* range = containingRange(lease, region.begin, neededEnd);
-    if (range == nullptr || !range->releasable) return false;
+    std::shared_ptr<const GuestAllocations::Range> owner;
+    bool joined = false;
+    if (const auto* range = containingRange(lease, region.begin, neededEnd); range != nullptr) {
+        if (!range->releasable) return false;
+        owner = leasedRangeOwner(lease, range->address);
+    } else {
+        joined = true;
+        owner = joinedHeapRange(lease, region.begin, neededEnd);
+        if (owner == nullptr) return false;
+    }
     std::vector<RefreshBlock> blocks;
     std::shared_ptr<ImageMirror> mirror;
     try {
-        mirror = acquireMirror(context, leasedRangeOwner(lease, range->address), blocks, true);
+        mirror = acquireMirror(context, owner, blocks, true, 0, 0, joined);
     } catch (const std::runtime_error& error) {
         static std::atomic<int> reported{0};
         if (reported.fetch_add(1, std::memory_order_relaxed) < 4) std::fprintf(stderr, "[gpu] %s; copying the sparse heap instead\n", error.what());

core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp

@@ -1127,6 +1127,14 @@ struct PipelineStore {
     std::uint64_t evicted = 0;
     PipelineStoreCounters totals;
     std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now();
+    std::size_t capacity = 0;
+    bool fixedCapacity = false;
+    std::list<std::uint64_t> evictedOrder;
+    std::unordered_map<std::uint64_t, std::list<std::uint64_t>::iterator> evictedKeys;
+    std::size_t evictionsSinceCheck = 0;
+    std::size_t returnsSinceCheck = 0;
+    static constexpr std::size_t BaseEntries = 1024;
+    static constexpr std::size_t MaximumEntries = 4096;
 };
 
 // Never destroyed: the pipelines belong to a device that may already be gone when statics die, and
@@ -1155,9 +1163,41 @@ std::list<PipelineStore::Entry>::iterator abandon(PipelineStore& store, std::lis
     return store.entries.erase(it);
 }
 
-std::size_t storeBound() {
-    static const auto bound = static_cast<std::size_t>(std::max<std::uint64_t>(1u, PipelineSwitch("APS5_PIPELINE_STORE_ENTRIES", 1024u)));
-    return bound;
+std::size_t storeBound(PipelineStore& store) {
+    if (store.capacity == 0) {
+        const auto set = PipelineSwitch("APS5_PIPELINE_STORE_ENTRIES", 0u);
+        store.fixedCapacity = set != 0;
+        store.capacity = store.fixedCapacity ? static_cast<std::size_t>(set) : PipelineStore::BaseEntries;
+    }
+    return store.capacity;
+}
+
+void noteStoreEviction(PipelineStore& store, std::uint64_t hash) {
+    if (store.fixedCapacity) return;
+    if (const auto found = store.evictedKeys.find(hash); found != store.evictedKeys.end()) store.evictedOrder.erase(found->second);
+    store.evictedOrder.push_front(hash);
+    store.evictedKeys[hash] = store.evictedOrder.begin();
+    while (store.evictedOrder.size() > store.capacity) {
+        store.evictedKeys.erase(store.evictedOrder.back());
+        store.evictedOrder.pop_back();
+    }
+    if (++store.evictionsSinceCheck < store.capacity / 32u) return;
+    const bool churning = store.returnsSinceCheck * 2u > store.evictionsSinceCheck;
+    store.evictionsSinceCheck = 0;
+    store.returnsSinceCheck = 0;
+    if (!churning || store.capacity >= PipelineStore::MaximumEntries) return;
+    const auto previous = store.capacity;
+    store.capacity = std::min(store.capacity * 2u, PipelineStore::MaximumEntries);
+    std::fprintf(stderr, "[pipecache] bound %zu -> %zu pipelines: pipelines it evicted keep coming back\n", previous, store.capacity);
+}
+
+void noteStoreReturn(PipelineStore& store, std::uint64_t hash) {
+    if (store.fixedCapacity) return;
+    const auto found = store.evictedKeys.find(hash);
+    if (found == store.evictedKeys.end()) return;
+    store.evictedOrder.erase(found->second);
+    store.evictedKeys.erase(found);
+    ++store.returnsSinceCheck;
 }
 
 void reportPipelines(PipelineStore& store) {
@@ -1229,12 +1269,12 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
         }
         ++store.misses;
         ++store.totals.misses;
+        noteStoreReturn(store, hash);
     }
     // Built outside the mutex: a draw that finds its pipeline resident is not held behind another
     // thread's build (the journal replay builds here too), and a device's driver compiles in
     // parallel when several threads build at once.
     auto pipeline = std::make_shared<Pipeline>(context, state, vertexInput, resources, shaders, attachmentLayout);
-    const auto bound = storeBound();
     const bool replaying = PipelineJournalReplaying();
     {
         std::lock_guard lock(store.mutex);
@@ -1252,7 +1292,10 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
         }
         // A replayed pipeline enters at the least recently used end and never displaces a draw's:
         // the title's own pipelines evict it first, and a full store takes none.
-        if (replaying && store.entries.size() >= bound) return pipeline;
+        if (replaying && store.entries.size() >= storeBound(store)) {
+            if (store.fixedCapacity || store.capacity >= PipelineStore::MaximumEntries) return pipeline;
+            store.capacity = std::min(store.capacity * 2u, PipelineStore::MaximumEntries);
+        }
         if (replaying) {
             store.entries.push_front({context.device, context.bufferPool, context.bufferPool != nullptr, hash, std::vector<std::byte>(key), pipeline});
             store.index[hash] = store.entries.begin();
@@ -1260,12 +1303,14 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
             store.entries.push_back({context.device, context.bufferPool, context.bufferPool != nullptr, hash, std::vector<std::byte>(key), pipeline});
             store.index[hash] = std::prev(store.entries.end());
         }
-        while (store.entries.size() > bound) {
+        while (store.entries.size() > storeBound(store)) {
             // Only an entry no recorded draw still holds may go (Kept keeps its shared_ptr until the fence).
             const auto victim = std::find_if(store.entries.begin(), store.entries.end(), [](const PipelineStore::Entry& entry) { return entry.pipeline.use_count() == 1; });
             if (victim == store.entries.end()) break;
+            const auto evictedHash = victim->hash;
             store.index.erase(victim->hash);
             store.entries.erase(victim);
+            noteStoreEviction(store, evictedHash);
             ++store.evicted;
             ++store.totals.evicted;
         }
@@ -1284,10 +1329,16 @@ PipelineStoreCounters PipelineStoreTotals() {
     std::lock_guard lock(store.mutex);
     auto totals = store.totals;
     totals.cached = store.entries.size();
-    totals.bound = storeBound();
+    totals.bound = storeBound(store);
     return totals;
 }
 
+bool PipelineStoreCanGrow() {
+    auto& store = Pipelines();
+    std::lock_guard lock(store.mutex);
+    return storeBound(store) < PipelineStore::MaximumEntries && !store.fixedCapacity;
+}
+
 void ClearCachedPipelines(VkDevice device) {
     auto& store = Pipelines();
     std::lock_guard lock(store.mutex);
@@ -1299,6 +1350,13 @@ void ClearCachedPipelines(VkDevice device) {
         store.index.erase(it->hash);
         it = store.entries.erase(it);
     }
+    if (store.entries.empty() && !store.fixedCapacity) {
+        store.capacity = 0;
+        store.evictedOrder.clear();
+        store.evictedKeys.clear();
+        store.evictionsSinceCheck = 0;
+        store.returnsSinceCheck = 0;
+    }
     ClearPipelineLibraries(device);
     ReleasePixelFunctionLds(device);
 }

core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp

@@ -1352,7 +1352,7 @@ private:
         std::call_once(loadOnce, [&] {
             try {
                 auto journal = Journal::Instance().Load();
-                const auto limit = configured("APS5_PIPELINE_JOURNAL_ENTRIES", 1024u);
+                const auto limit = configured("APS5_PIPELINE_JOURNAL_ENTRIES", 4096u);
                 if (journal.entries.size() > limit) journal.entries.erase(journal.entries.begin(), journal.entries.end() - static_cast<std::ptrdiff_t>(limit));
                 loaded = std::move(journal);
             } catch (const std::exception& error) {
@@ -1438,7 +1438,7 @@ private:
             return;
         }
         const auto store = PipelineStoreTotals();
-        if (store.cached >= store.bound) {
+        if (store.cached >= store.bound && !PipelineStoreCanGrow()) {
             counters.skipped.fetch_add(1, std::memory_order_relaxed);
             return;
         }

core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp

@@ -185,7 +185,8 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
     std::vector<VkPipeline> libraries;
     for (std::size_t part = 0; part < flags.size(); ++part) {
         if (part == 0 && mesh) continue;
-        const auto key = Combined(*partKeys[part], *contextKeys[part]);
+        auto key = Combined(*partKeys[part], *contextKeys[part]);
+        key.push_back(static_cast<std::byte>(optimize));
         auto& cache = device.parts[part];
         if (const auto found = cache.find(key); found != cache.end()) {
             libraries.push_back(found->second);
@@ -195,7 +196,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
         VkGraphicsPipelineLibraryCreateInfoEXT library{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT, flags[part] == VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT ? &rendering : &shaderRendering};
         library.flags = flags[part];
         VkGraphicsPipelineCreateInfo create{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, &library};
-        create.flags = VK_PIPELINE_CREATE_LIBRARY_BIT_KHR | VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT;
+        create.flags = optimize ? VK_PIPELINE_CREATE_LIBRARY_BIT_KHR | VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT : VK_PIPELINE_CREATE_LIBRARY_BIT_KHR;
         create.pDynamicState = info.pDynamicState;
         create.layout = partLayouts[part];
         switch (flags[part]) {

core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp

@@ -8,6 +8,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
 #include "prx/libSceAgcDriver/Graphics/shaders/MeshArguments_spv.h"
 #include "prx/libSceAgcDriver/Graphics/shaders/SampleCounter_spv.h"
 #include "prx/libSceAgcDriver/Graphics/shaders/SampleDumps_spv.h"
@@ -3134,6 +3135,26 @@ std::uint64_t Recorder::PendingWriterSerial(std::uint64_t address, std::size_t b
     return 0;
 }
 
+std::uint64_t Recorder::PendingWriterSerial(std::uint64_t address, std::size_t bytes, std::span<const std::uint64_t> exceptNotes) const {
+    if (exceptNotes.empty()) return PendingWriterSerial(address, bytes);
+    if (bytes == 0) return 0;
+    const auto end = address + bytes;
+    const auto writes = [&](const Batch& batch) {
+        for (std::size_t index = 0; index < batch.writes.size(); ++index) {
+            const auto& [begin, finish] = batch.writes[index];
+            if (address >= finish || begin >= end) continue;
+            if (index < batch.writeNotes.size() && std::find(exceptNotes.begin(), exceptNotes.end(), batch.writeNotes[index]) != exceptNotes.end()) continue;
+            return true;
+        }
+        return readOverlap(batch, address, end, ReadKind::AddressBased).has_value();
+    };
+    if (open != nullptr && writes(*open)) return OpenSerial();
+    for (auto it = inFlight.rbegin(); it != inFlight.rend(); ++it) {
+        if (writes(**it)) return (*it)->serial;
+    }
+    return 0;
+}
+
 bool Recorder::signaled(const Batch& batch) const {
     return batch.submitted && fenceStatus(batch.fence) == VK_SUCCESS;
 }
@@ -3659,6 +3680,7 @@ void Recorder::Submit() {
         submission.pSignalSemaphores = &timeline;
     }
     if (hold != VK_NULL_HANDLE || timeline != VK_NULL_HANDLE) submission.pNext = &timelineInfo;
+    DestroyDeferredAtIdle(context);
     const auto submitStart = std::chrono::steady_clock::now();
     const auto result = function(queueSubmit, "vkQueueSubmit")(context.queue, 1, &submission, batch->fence);
     if (result == VK_ERROR_DEVICE_LOST) {

core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
@@ -381,6 +382,7 @@ void CommandBatch::Submit() {
     submission.commandBufferCount = 1;
     submission.pCommandBuffers = &commands;
     timing.Mark("command_end");
+    DestroyDeferredAtIdle(context);
     Check(context.Function<PFN_vkQueueSubmit>("vkQueueSubmit")(context.queue, 1, &submission, fence), "vkQueueSubmit graphics");
     pending = true;
     submitted = true;

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp

@@ -624,10 +624,7 @@ std::uint64_t storageBudget(const Context& context, std::uint64_t required = 0)
         cached.device = context.physical;
         cached.read = now;
         cached.budget = StorageTextureBudget(context.memory, &reported, SampledTextureMemory(), storageCacheBytes());
-        cached.ceiling = cached.budget;
-        if (const auto heap = largestDeviceLocalHeap(context.memory); heap && reported.heapBudget[*heap] != 0) {
-            cached.ceiling = std::max<std::uint64_t>(512ull << 20u, std::min<std::uint64_t>(TextureCacheBudget(context.memory), reported.heapBudget[*heap] / 4u));
-        }
+        cached.ceiling = StorageTextureCeiling(context.memory, &reported);
     };
     if (cached.device != context.physical || now - cached.read >= std::chrono::seconds(1)) read();
     const auto needed = std::min(required, cached.ceiling);
@@ -752,7 +749,14 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
         normalized.format = unorm16;
         return cachedTexture(context, words, normalized, components, guestBytes, depthCompare);
     }
-    if (const auto window = ViewedArrayWindow(resource); window.baseAddress != resource.baseAddress) {
+    const auto viewedWindow = [&] {
+        const auto window = ViewedArrayWindow(resource, false);
+        if (window.baseAddress != resource.baseAddress || resource.dccAddress == 0) return window;
+        if (ViewedArrayWindow(resource, true).baseAddress == resource.baseAddress) return window;
+        if (TextureClearKeys(resource, guestBytes != 0 ? guestBytes : DescribeSurface(resource).guestBytes) != DccKeys::Uncompressed) return window;
+        return ViewedArrayWindow(resource, true);
+    };
+    if (const auto window = viewedWindow(); window.baseAddress != resource.baseAddress) {
         static std::atomic<std::uint64_t> windows{0};
         const auto count = windows.fetch_add(1) + 1;
         if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] sampled array view 0x%llx (%ux%u format %u, dimension %d, slices %u..%u) served from its viewed slices at 0x%llx; %llu such views\n", static_cast<unsigned long long>(resource.baseAddress), resource.width, resource.height, resource.format, static_cast<int>(resource.dimension), resource.baseArray, resource.depthOrLastArray, static_cast<unsigned long long>(window.baseAddress), static_cast<unsigned long long>(count));
@@ -1185,9 +1189,9 @@ std::shared_ptr<StorageTexture> cachedStorageTexture(const Context& context, std
     const auto viewed = ArrayLayerWindow(requested, context.limits.maxImageArrayLayers);
     if (viewed.baseAddress != requested.baseAddress) guestBytes = 0;
     Require(viewed.dimension != TextureDimension::k2DMsaa, "storage image access to a multisampled texture is not implemented");
-    RequireNoMultisampleTarget(viewed.baseAddress, guestBytes != 0 ? static_cast<std::size_t>(guestBytes) : 1u, "a storage image access");
-    if (auto depth = DepthSurfaceStorage(context, words, viewed, mip)) return depth;
     const auto viewedBytes = guestBytes != 0 ? guestBytes : DescribeSurface(viewed).guestBytes;
+    RequireNoMultisampleTarget(viewed.baseAddress, static_cast<std::size_t>(viewedBytes), "a storage image access");
+    if (auto depth = DepthSurfaceStorage(context, words, viewed, mip)) return depth;
     if (viewedBytes >= (1ull << 30u)) {
         static std::atomic<std::uint64_t> large{0};
         const auto count = large.fetch_add(1) + 1;
@@ -1283,14 +1287,19 @@ std::uint64_t TextureCacheBudget(const VkPhysicalDeviceMemoryProperties& memory)
     return std::max<std::uint64_t>(2048ull << 20u, heap ? memory.memoryHeaps[*heap].size / 4u : 0u);
 }
 
+std::uint64_t StorageTextureCeiling(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported) {
+    const auto heap = largestDeviceLocalHeap(memory);
+    if (reported == nullptr || !heap || reported->heapBudget[*heap] == 0) return TextureCacheBudget(memory);
+    return std::max<std::uint64_t>(512ull << 20u, std::min<std::uint64_t>(memory.memoryHeaps[*heap].size / 2u, reported->heapBudget[*heap] / 2u));
+}
+
 std::uint64_t StorageTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t sampledBytes, std::uint64_t storageBytes) {
     const auto heap = largestDeviceLocalHeap(memory);
     if (reported == nullptr || !heap || reported->heapBudget[*heap] == 0) return TextureCacheBudget(memory);
     const auto budget = reported->heapBudget[*heap];
     const auto usage = reported->heapUsage[*heap];
     const auto reserved = usage - std::min<std::uint64_t>(usage, sampledBytes + storageBytes) + sampledBytes + budget / 8u;
-    const auto cap = std::min<std::uint64_t>(memory.memoryHeaps[*heap].size / 2u, budget / 2u);
-    return std::max<std::uint64_t>(512ull << 20u, std::min<std::uint64_t>(cap, budget > reserved ? budget - reserved : 0u));
+    return std::min<std::uint64_t>(StorageTextureCeiling(memory, reported), std::max<std::uint64_t>(512ull << 20u, budget > reserved ? budget - reserved : 0u));
 }
 
 std::uint64_t SampledTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t textureBytes, std::uint64_t storageBytes) {
@@ -2075,9 +2084,10 @@ void ShaderResources::buildPrepare(std::span<const CompiledShader> shaders, cons
                         const bool classified = element < binding.bufferWritten.size() && element < binding.bufferRead.size() && element < binding.bufferAtomic.size();
                         allocations[index].writeOnly = classified && allocations[index].guest && allocations[index].written && binding.bufferWritten[element] && !read && !atomic;
                         const auto& push = shader.program->pushConstants;
-                        if (!push.empty()) {
+                        const auto pushDataBytes = shader.program->stencilReferences.empty() ? push.size() : shader.program->stencilReferenceDword * sizeof(std::uint32_t);
+                        if (pushDataBytes != 0u) {
                             const auto position = shader.program->memoryOffsetDword * 4u + element;
-                            Require(position < push.size(), "guest buffer offset lies outside the shader's push constants");
+                            Require(position < pushDataBytes, "guest buffer offset lies outside the shader's push constants");
                             allocations[index].pushByte = static_cast<std::int32_t>(shader.pushConstantOffset + position);
                         } else {
                             allocations[index].dataByte = shader.program->memoryOffsetDword * 4u + element;

core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp

@@ -6,6 +6,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libc/include/General.hpp"
 #include <algorithm>
@@ -314,13 +315,33 @@ VkStencilOpState stencilFace(std::uint32_t compare, std::uint32_t ops, std::uint
     auto fixed = compare == 0 || compare == 7 ? 0u : face.compareMask;
     std::uint32_t invertMask = 0;
     std::uint32_t inverted = 0;
+    std::uint32_t logicMask = 0;
+    const auto logic = [](std::uint32_t op) { return op == 10 || op == 11 || op == 12 || op == 15; };
     const auto convert = [&](std::uint32_t op, bool testFailed) {
         if (face.writeMask == 0) return VK_STENCIL_OP_KEEP;
         char detail[112];
-        if (opValExport && ((op == 4 && !opValReference) || op == 5 || op == 6 || op == 8 || op == 9)) {
+        if (opValExport && ((op == 4 && !opValReference) || (op >= 5 && op != 7))) {
             std::snprintf(detail, sizeof(detail), "AGC graphics: stencil operation %u with a pixel shader that exports the stencil operation value is unsupported", op);
             throw std::runtime_error(detail);
         }
+        if (logic(op)) {
+            const auto bits = (op == 11 || op == 12 ? opValue : ~opValue) & face.writeMask;
+            if (bits == 0) return VK_STENCIL_OP_KEEP;
+            if (logicMask != 0 && logicMask != bits) {
+                std::snprintf(detail, sizeof(detail), "AGC graphics: stencil logic operations on write masks 0x%02x and 0x%02x are unsupported", logicMask, bits);
+                throw std::runtime_error(detail);
+            }
+            logicMask = bits;
+            if (op != 11) return op == 10 ? VK_STENCIL_OP_ZERO : VK_STENCIL_OP_INVERT;
+            if (testValExport) throw std::runtime_error("AGC graphics: stencil OR is unsupported with a pixel shader that exports the stencil test value");
+            if ((~reference & fixed & bits) != 0) {
+                std::snprintf(detail, sizeof(detail), "AGC graphics: stencil OR of 0x%02x against test value 0x%02x (masks 0x%02x/0x%02x) is unsupported", opValue, test, face.compareMask, face.writeMask);
+                throw std::runtime_error(detail);
+            }
+            reference |= bits;
+            fixed |= bits;
+            return VK_STENCIL_OP_REPLACE;
+        }
         switch (op) {
             case 0: return VK_STENCIL_OP_KEEP;
             case 1: return VK_STENCIL_OP_ZERO;
@@ -377,6 +398,15 @@ VkStencilOpState stencilFace(std::uint32_t compare, std::uint32_t ops, std::uint
         if (inverted != active) throw std::runtime_error("AGC graphics: a stencil replacement drawn as an inversion beside another stencil operation is unsupported");
         face.writeMask = invertMask;
     }
+    if (logicMask != 0) {
+        const auto other = [&](std::uint32_t op) { return op != 0 && !logic(op); };
+        if ((other(ops & 0xfu) || other((ops >> 4u) & 0xfu) || other((ops >> 8u) & 0xfu)) && logicMask != face.writeMask) {
+            char detail[128];
+            std::snprintf(detail, sizeof(detail), "AGC graphics: stencil operations 0x%03x combine a logic operation on write mask 0x%02x with write mask 0x%02x", ops & 0xfffu, logicMask, face.writeMask);
+            throw std::runtime_error(detail);
+        }
+        face.writeMask = logicMask;
+    }
     face.reference = reference;
     Require(!opValReference || compare == 0 || compare == 7 || face.compareMask == 0, "a stencil compare against the test value is unsupported with a pixel shader that exports the stencil operation value");
     return face;
@@ -1326,6 +1356,27 @@ ReversedBlend TranslateReversedBlend(std::uint32_t blendControl, std::uint8_t co
     return {blend, routing->code};
 }
 
+void ApplyDepthBoundsEmulation(const Context& context, State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel) {
+    if (!state.depthBoundsTest || !state.depth || context.depthBounds) return;
+    ValidateDepthBounds(context, state);
+    Require(!context.depthRangeUnrestricted, "the depth bounds test without the depthBounds feature is emulated only where stored depth stays in [0, 1]");
+    Require(state.depth->format == VK_FORMAT_D32_SFLOAT || state.depth->format == VK_FORMAT_D32_SFLOAT_S8_UINT, "the depth bounds test on a normalized depth target needs the depthBounds feature");
+    Require(!state.depthTest && !state.depthWrite, "the depth bounds test with depth testing needs the depthBounds feature");
+    Require(!pixel.posZ && !pixel.depthExportEnable, "the depth bounds test from a pixel program that reads or exports depth needs the depthBounds feature");
+    const auto keepsOnFail = [](const VkStencilOpState& face) { return (face.writeMask & 0xffu) == 0 || (face.failOp == VK_STENCIL_OP_KEEP && face.depthFailOp == VK_STENCIL_OP_KEEP); };
+    Require(!state.stencilTest || (keepsOnFail(state.stencilFront) && keepsOnFail(state.stencilBack)), "the depth bounds test with a stencil test that writes on a failed test needs the depthBounds feature");
+    const bool upper = state.minDepthBounds <= 0.0f;
+    Require(upper || state.maxDepthBounds >= 1.0f, "the depth bounds test with both bounds inside (0, 1) needs the depthBounds feature");
+    const float bound = upper ? state.maxDepthBounds : state.minDepthBounds;
+    state.depthBoundsTest = false;
+    state.depthTest = true;
+    state.depthCompare = upper ? VK_COMPARE_OP_GREATER_OR_EQUAL : VK_COMPARE_OP_LESS_OR_EQUAL;
+    state.depthBias = false;
+    state.depthBiasPerFace = false;
+    state.viewport.minDepth = bound;
+    state.viewport.maxDepth = bound;
+}
+
 void ApplyDepthRemap(const Context& context, State& state) {
     const auto& viewport = state.viewport;
     const bool inRange = viewport.minDepth >= 0 && viewport.minDepth <= 1 && viewport.maxDepth >= 0 && viewport.maxDepth <= 1;

core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp

@@ -123,6 +123,7 @@ struct StorageTraffic {
     std::map<std::string, std::pair<std::uint64_t, std::uint64_t>> writeBacks;
     std::array<std::pair<std::uint64_t, std::uint64_t>, 4> uploads{};
     std::map<std::string, std::pair<std::uint64_t, std::uint64_t>> uploadReasons;
+    std::map<std::string, std::pair<std::uint64_t, std::uint64_t>> drawnStores;
     std::uint64_t directWriteBacks = 0;
     std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now();
 };
@@ -132,6 +133,14 @@ StorageTraffic& Traffic() {
     return traffic;
 }
 
+struct DrawnPending {
+    std::size_t images = 0;
+    std::uint64_t bytes = 0;
+    double oldestMs = 0;
+};
+
+DrawnPending drawnPending();
+
 void reportStorageTraffic(StorageTraffic& traffic) {
     const auto now = std::chrono::steady_clock::now();
     if (now - traffic.lastReport < std::chrono::seconds(10)) return;
@@ -154,12 +163,19 @@ void reportStorageTraffic(StorageTraffic& traffic) {
         std::snprintf(text, sizeof(text), " %llu/%.1f", static_cast<unsigned long long>(totals.first), totals.second / 1048576.0);
         line += " " + reason + text;
     }
+    std::string drawnLine;
+    for (const auto& [reason, totals] : traffic.drawnStores) {
+        std::snprintf(text, sizeof(text), " %llu/%.1f", static_cast<unsigned long long>(totals.first), totals.second / 1048576.0);
+        drawnLine += " " + reason + text;
+    }
+    const auto drawn = drawnPending();
     const auto take = [](std::atomic<std::uint64_t>& counter) { return static_cast<unsigned long long>(counter.exchange(0, std::memory_order_relaxed)); };
     const auto partialUploadCount = take(partialUploads), partialWriteBackCount = take(partialWriteBacks);
     const auto uploadMiB = take(partialUploadBytes) / 1048576.0, writeBackMiB = take(partialWriteBackBytes) / 1048576.0;
-    AgcDriver::ProfilePrint_nid_no_patch( "[storage] uploads by path (count/MiB, 10 s):%s; %llu write-backs GPU-direct, %llu stored nothing; block units: partial uploads %llu/%.1f, partial write-backs %llu/%.1f, %llu widened to every pending unit, units dropped %llu, superseded %llu, handed over at %llu clear-key refreshes (%llu units superseded), dropped unregistered %llu, kept over a key flip %llu%s, pretest skipped %llu, held for a queued draw read %llu, evicted stale %llu\n", line.c_str(), static_cast<unsigned long long>(traffic.directWriteBacks), take(emptyWriteBacks), partialUploadCount, uploadMiB, partialWriteBackCount, writeBackMiB, take(coalescedWriteBacks), take(unitsDropped), take(unitsSuperseded), take(clearHandOvers), take(handedOverSuperseded), take(unregisteredDropped), take(keyFlipKept), ShadowReport().c_str(), take(pretestSkipped), take(queuedReadHeld), take(staleEvicted));
+    AgcDriver::ProfilePrint_nid_no_patch( "[storage] uploads by path (count/MiB, 10 s):%s; %llu write-backs GPU-direct, %llu stored nothing; block units: partial uploads %llu/%.1f, partial write-backs %llu/%.1f, %llu widened to every pending unit, units dropped %llu, superseded %llu, handed over at %llu clear-key refreshes (%llu units superseded), dropped unregistered %llu, kept over a key flip %llu%s, pretest skipped %llu, held for a queued draw read %llu, evicted stale %llu; drawn linear targets: stores by reason (count/MiB):%s, pending %zu images / %.1f MiB, oldest %.1f ms\n", line.c_str(), static_cast<unsigned long long>(traffic.directWriteBacks), take(emptyWriteBacks), partialUploadCount, uploadMiB, partialWriteBackCount, writeBackMiB, take(coalescedWriteBacks), take(unitsDropped), take(unitsSuperseded), take(clearHandOvers), take(handedOverSuperseded), take(unregisteredDropped), take(keyFlipKept), ShadowReport().c_str(), take(pretestSkipped), take(queuedReadHeld), take(staleEvicted), drawnLine.c_str(), drawn.images, drawn.bytes / 1048576.0, drawn.oldestMs);
     traffic.writeBacks.clear();
     traffic.uploadReasons.clear();
+    traffic.drawnStores.clear();
     traffic.directWriteBacks = 0;
 }
 
@@ -175,7 +191,7 @@ void countStorageUpload(std::size_t path, std::uint64_t bytes) {
     reportStorageTraffic(traffic);
 }
 
-void countStorageWriteBack(std::uint64_t bytes, bool direct) {
+void countStorageWriteBack(std::uint64_t bytes, bool direct, bool drawn = false) {
     if (!LookupOutcomes::Profiled()) return;
     auto& traffic = Traffic();
     std::lock_guard lock(traffic.mutex);
@@ -183,6 +199,11 @@ void countStorageWriteBack(std::uint64_t bytes, bool direct) {
     ++totals.first;
     totals.second += bytes;
     if (direct) ++traffic.directWriteBacks;
+    if (drawn) {
+        auto& stores = traffic.drawnStores[flushReason != nullptr ? flushReason : "other"];
+        ++stores.first;
+        stores.second += bytes;
+    }
     reportStorageTraffic(traffic);
 }
 
@@ -1538,6 +1559,13 @@ struct PendingWrites {
     // Images taken out of `textures` by a FlushPending still storing them (see adjacentPendingUnchanged).
     std::vector<StorageTexture*> flushing;
     std::atomic<std::size_t> flushingCount{0};
+    struct Drawn {
+        StorageTexture* texture;
+        std::uint64_t bytes;
+        std::chrono::steady_clock::time_point since;
+    };
+    std::vector<Drawn> drawn;
+    std::atomic<std::size_t> drawnCount{0};
 };
 
 PendingWrites& Pending() {
@@ -1545,6 +1573,20 @@ PendingWrites& Pending() {
     return pending;
 }
 
+DrawnPending drawnPending() {
+    DrawnPending result;
+    auto& pending = Pending();
+    if (pending.drawnCount.load(std::memory_order_acquire) == 0) return result;
+    const auto now = std::chrono::steady_clock::now();
+    std::lock_guard lock(pending.mutex);
+    result.images = pending.drawn.size();
+    for (const auto& entry : pending.drawn) {
+        result.bytes += entry.bytes;
+        result.oldestMs = std::max(result.oldestMs, std::chrono::duration<double, std::milli>(now - entry.since).count());
+    }
+    return result;
+}
+
 // See StorageTexture::PendingSerial: bumped after every mutation of the registry above.
 std::atomic<std::uint64_t> pendingSerial{0};
 
@@ -3804,7 +3846,13 @@ void StorageTexture::markLayersPending(std::uint32_t first, std::uint32_t count)
         }
     }
     if (anyUnsaved && !GuestMemory::Watched(descriptor.baseAddress, static_cast<std::size_t>(guestBytes))) captureGuestBytes(&unsaved);
-    for (std::uint32_t layer = first; layer < first + count; ++layer) layerPending[layer] = true;
+    for (std::uint32_t layer = first; layer < first + count; ++layer) {
+        if (!layerPending[layer] && !completionKept.empty()) {
+            const auto end = layerBegin(layer) + layerBytes(layer);
+            for (auto at = layerBegin(layer) & ~std::uint64_t{65535}; at < end; at += 65536) completionKept[static_cast<std::size_t>(at / 65536 - descriptor.baseAddress / 65536)] = 0;
+        }
+        layerPending[layer] = true;
+    }
     // Results of this image now cover the alias's results borrowed into these units: this image
     // stores them, the alias no longer has to (its results, unchanged since the borrow, are in
     // this image's content). An alias written since keeps its own newer results pending.
@@ -3827,6 +3875,10 @@ void StorageTexture::markLayersPending(std::uint32_t first, std::uint32_t count)
     }
     auto& pending = Pending();
     std::lock_guard lock(pending.mutex);
+    if (!drawingTarget) {
+        pendingFromOthers = true;
+        if (drawnListed) drawnWhole = true;
+    }
     // The active recorder installs a hook that also waits for recorded work; without one (tests),
     // pending storage results alone are flushed.
     if (Recorder::Active() == nullptr) {
@@ -3851,6 +3903,7 @@ void StorageTexture::reconcilePending() {
     auto& pending = Pending();
     std::lock_guard lock(pending.mutex);
     const bool any = anyLayerPending();
+    if (!any) forgetDrawn();
     if (any == dirty) return;
     dirty = any;
     if (any) pending.textures.push_back(this);
@@ -3864,6 +3917,215 @@ bool StorageTexture::Dirty() const {
     return dirty;
 }
 
+constexpr std::size_t MaxDrawnRects = 16;
+
+bool StorageTexture::MarkDrawn(std::uint32_t slice, std::uint32_t count, VkRect2D scissor) {
+    struct Drawing {
+        bool& flag;
+        bool previous;
+        ~Drawing() { flag = previous; }
+    } drawing{drawingTarget, std::exchange(drawingTarget, true)};
+    MarkSliceDirty(slice, count);
+    const auto clip = [](std::int64_t value, std::uint32_t limit) { return static_cast<std::uint32_t>(std::clamp<std::int64_t>(value, 0, limit)); };
+    const auto left = clip(scissor.offset.x, descriptor.width);
+    const auto top = clip(scissor.offset.y, descriptor.height);
+    const auto right = clip(std::int64_t{scissor.offset.x} + scissor.extent.width, descriptor.width);
+    const auto bottom = clip(std::int64_t{scissor.offset.y} + scissor.extent.height, descriptor.height);
+    const bool rectangle = proxyImage == VK_NULL_HANDLE && descriptor.tileMode == TextureTileMode::kLinear && !blockUnits && trackedLayers == 1 && descriptor.mipCount == 1 && arrayLayers == 1 && geometry.imageLayers == 1 && geometry.imageDepth == 1 && !geometry.thick && descriptor.dccAddress == 0;
+    auto& pending = Pending();
+    std::lock_guard lock(pending.mutex);
+    if (!anyLayerPending()) return true;
+    if (!drawnListed) {
+        drawnListed = true;
+        drawnWhole = false;
+        drawnRects.clear();
+        pending.drawn.push_back({this, guestBytes, std::chrono::steady_clock::now()});
+        pending.drawnCount.store(pending.drawn.size(), std::memory_order_release);
+    }
+    if (!rectangle || pendingFromOthers) drawnWhole = true;
+    if (!drawnWhole && right > left && bottom > top) addDrawnRect({left, top, right, bottom});
+    return !drawnWhole && drawnRects.size() <= MaxDrawnRects;
+}
+
+void StorageTexture::addDrawnRect(DrawnRect rect) {
+    const auto inside = [](const DrawnRect& outer, const DrawnRect& inner) { return outer.left <= inner.left && outer.top <= inner.top && inner.right <= outer.right && inner.bottom <= outer.bottom; };
+    if (std::any_of(drawnRects.begin(), drawnRects.end(), [&](const DrawnRect& kept) { return inside(kept, rect); })) return;
+    for (auto kept = drawnRects.begin(); kept != drawnRects.end();) {
+        const bool columns = kept->left == rect.left && kept->right == rect.right && kept->top <= rect.bottom && rect.top <= kept->bottom;
+        const bool rows = kept->top == rect.top && kept->bottom == rect.bottom && kept->left <= rect.right && rect.left <= kept->right;
+        if (!columns && !rows && !inside(rect, *kept)) {
+            ++kept;
+            continue;
+        }
+        rect = {std::min(kept->left, rect.left), std::min(kept->top, rect.top), std::max(kept->right, rect.right), std::max(kept->bottom, rect.bottom)};
+        drawnRects.erase(kept);
+        kept = drawnRects.begin();
+    }
+    drawnRects.push_back(rect);
+}
+
+void StorageTexture::forgetDrawn() {
+    pendingFromOthers = false;
+    if (!drawnListed) return;
+    drawnListed = false;
+    drawnWhole = false;
+    drawnRects.clear();
+    auto& pending = Pending();
+    std::erase_if(pending.drawn, [this](const PendingWrites::Drawn& entry) { return entry.texture == this; });
+    pending.drawnCount.store(pending.drawn.size(), std::memory_order_release);
+}
+
+void StorageTexture::noteStore(std::uint64_t note) {
+    if (note == 0) return;
+    auto& pending = Pending();
+    std::lock_guard lock(pending.mutex);
+    storeNotes.push_back(note);
+    if (storeNotes.size() > 8) storeNotes.erase(storeNotes.begin());
+}
+
+StorageTexture::DrawnArea StorageTexture::drawnArea() const {
+    auto& pending = Pending();
+    std::lock_guard lock(pending.mutex);
+    if (!drawnListed) return {};
+    DrawnArea area{true, !drawnWhole && !pendingFromOthers};
+    if (!area.rect || drawnRects.empty()) return area;
+    area.left = area.top = std::numeric_limits<std::uint32_t>::max();
+    for (const auto& rect : drawnRects) {
+        area.left = std::min(area.left, rect.left);
+        area.top = std::min(area.top, rect.top);
+        area.right = std::max(area.right, rect.right);
+        area.bottom = std::max(area.bottom, rect.bottom);
+    }
+    area.rects = drawnRects;
+    return area;
+}
+
+std::vector<std::pair<std::uint64_t, std::uint64_t>> StorageTexture::drawnSpans(const DrawnArea& area, std::span<const std::pair<std::uint64_t, std::uint64_t>> keep) const {
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> spans;
+    if (area.rects.empty()) return spans;
+    const auto& mip = mips.front();
+    const auto elementBytes = static_cast<std::uint64_t>(BytesPerElement(descriptor.format));
+    const auto rowBase = descriptor.baseAddress + mip.tiledOffset;
+    std::vector<std::pair<std::uint32_t, std::uint32_t>> columns;
+    std::size_t next = 0;
+    for (auto row = area.top; row < area.bottom; ++row) {
+        columns.clear();
+        for (const auto& rect : area.rects) {
+            if (rect.top <= row && row < rect.bottom) columns.emplace_back(rect.left, rect.right);
+        }
+        std::sort(columns.begin(), columns.end());
+        const auto rowStart = rowBase + std::uint64_t{row} * mip.pitchBytes;
+        for (std::size_t first = 0; first < columns.size();) {
+            auto right = columns[first].second;
+            auto last = first + 1;
+            for (; last < columns.size() && columns[last].first <= right; ++last) right = std::max(right, columns[last].second);
+            const auto rowBegin = rowStart + columns[first].first * elementBytes;
+            const auto rowEnd = rowStart + right * elementBytes;
+            first = last;
+            while (next < keep.size() && keep[next].second <= rowBegin) ++next;
+            for (auto index = next; index < keep.size() && keep[index].first < rowEnd; ++index) {
+                const auto from = std::max(rowBegin, keep[index].first);
+                const auto to = std::min(rowEnd, keep[index].second);
+                if (from < to) spans.emplace_back(from, to);
+            }
+        }
+    }
+    return spans;
+}
+
+std::uint64_t StorageTexture::storeDrawnRect(const DrawnArea& area, std::span<const ImportSegment> segments, std::span<const std::pair<std::uint64_t, std::uint64_t>> stores, const std::vector<std::uint32_t>& saveUnits) {
+    const auto& mip = mips.front();
+    const auto elementBytes = static_cast<std::uint64_t>(BytesPerElement(descriptor.format));
+    const auto rowBase = descriptor.baseAddress + mip.tiledOffset;
+    const auto width = area.right - area.left;
+    const auto height = area.bottom - area.top;
+    const auto rowBytes = std::uint64_t{width} * elementBytes;
+    const auto hullBegin = stores.front().first;
+    const auto hullBytes = static_cast<std::size_t>(stores.back().second - hullBegin);
+    std::uint64_t storedBytes = 0;
+    for (const auto& [from, to] : stores) storedBytes += to - from;
+    auto scratch = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(rowBytes * height), VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
+    auto* recorder = Recorder::Active();
+    std::unique_ptr<CommandBatch> batch;
+    VkCommandBuffer commands = VK_NULL_HANDLE;
+    auto timing = Recorder::NoTiming;
+    if (recorder != nullptr) {
+        recorder->FlushKeyStoresOverlapping(hullBegin, hullBytes);
+        recorder->FlushStoresOverlapping(hullBegin, hullBytes);
+        commands = recorder->Commands();
+        timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack);
+        recorder->Keep(scratch, scratch->Size());
+        if (auto self = weak_from_this().lock()) keepStorageTexture(*recorder, std::move(self));
+        Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 3);
+        if (Recorder::BarrierValidate()) {
+            const std::pair<VkImage, bool> read{image, false};
+            recorder->NoteAccess(Recorder::CommandClass::StorageWriteBack, Recorder::Access{{}, stores, std::span(&read, 1), VK_PIPELINE_STAGE_TRANSFER_BIT});
+        }
+    } else {
+        batch = std::make_unique<CommandBatch>(context);
+        commands = batch->Handle();
+    }
+    VkImageMemoryBarrier toSource{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
+    toSource.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
+    toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
+    toSource.oldLayout = VK_IMAGE_LAYOUT_GENERAL;
+    toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
+    toSource.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+    toSource.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+    toSource.image = image;
+    toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers};
+    const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT};
+    context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &importReady, 0, nullptr, 1, &toSource);
+    VkBufferImageCopy region{};
+    region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
+    region.imageOffset = {static_cast<std::int32_t>(area.left), static_cast<std::int32_t>(area.top), 0};
+    region.imageExtent = {width, height, 1u};
+    context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, scratch->Handle(), 1, &region);
+    const auto scratchRead = WholeBufferBarrier(scratch->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
+    context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &scratchRead, 0, nullptr);
+    const auto scratchOffset = [&](std::uint64_t address) {
+        const auto offset = address - rowBase;
+        const auto row = offset / mip.pitchBytes;
+        return (row - area.top) * rowBytes + offset - row * mip.pitchBytes - area.left * elementBytes;
+    };
+    const auto copyBuffer = context.Function<PFN_vkCmdCopyBuffer>("vkCmdCopyBuffer");
+    for (const auto& segment : segments) {
+        std::vector<VkBufferCopy> copies;
+        for (const auto& [from, to] : stores) {
+            const auto begin = std::max(from, segment.begin);
+            const auto end = std::min(to, segment.end);
+            if (begin < end) copies.push_back({scratchOffset(begin), HostImportOffset(*segment.import, begin), end - begin});
+        }
+        if (!copies.empty()) copyBuffer(commands, scratch->Handle(), segment.import->buffer, static_cast<std::uint32_t>(copies.size()), copies.data());
+    }
+    if (!saveUnits.empty()) {
+        const auto saved = savedBytes->Bytes();
+        for (const auto unit : saveUnits) {
+            const auto offset = static_cast<std::size_t>(layerBegin(unit) - descriptor.baseAddress);
+            std::memcpy(saved.data() + offset, original.data() + offset, static_cast<std::size_t>(layerBytes(unit)));
+        }
+        std::vector<VkBufferCopy> saves;
+        for (const auto& [from, to] : stores) saves.push_back({scratchOffset(from), from - descriptor.baseAddress, to - from});
+        copyBuffer(commands, scratch->Handle(), savedBytes->Handle(), static_cast<std::uint32_t>(saves.size()), saves.data());
+    }
+    VkImageMemoryBarrier backToGeneral = toSource;
+    backToGeneral.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
+    backToGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
+    backToGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
+    backToGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL;
+    const VkMemoryBarrier stored{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT};
+    context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, 1, &backToGeneral);
+    if (batch) {
+        batch->SubmitAndWait();
+        return storedBytes;
+    }
+    recorder->EndGpuTiming(timing, storedBytes);
+    recorder->MarkShaderReadsCovered();
+    recorder->NotePendingWrite(hullBegin, hullBytes);
+    noteStore(recorder->LastWriteNote(hullBegin, hullBytes));
+    return storedBytes;
+}
+
 void StorageTexture::Flush() {
     {
         auto& live = Live();
@@ -3914,13 +4176,38 @@ void waitForIdlePublish(const char* reason) {
 bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except, const char* reason, PublishScope scope, bool* published, bool waited) {
     static_cast<void>(FlushDepthPlanes(address, bytes));
     if (published != nullptr) *published = false;
+    const auto nothingToStore = [&] {
+        auto& pending = Pending();
+        if (pending.flushingCount.load(std::memory_order_acquire) != 0) return false;
+        if (!pending.textures.SurelyClear(address, bytes)) {
+            std::lock_guard lock(pending.mutex);
+            const auto* exempt = refreshing;
+            if (pending.textures.MayOverlap(address, bytes) && std::any_of(pending.textures.begin(), pending.textures.end(), [&](const StorageTexture* texture) { return texture != except && texture != exempt && texture->overlaps(address, bytes) && texture->pendingUnitInside(address, bytes); })) return false;
+            if (pending.flushingCount.load(std::memory_order_acquire) != 0) return false;
+        }
+        return scope == PublishScope::None || !AnyShadowedOverlaps(address, bytes);
+    };
+    if (nothingToStore()) return false;
     if (AgcDriver::DriverDetail::DrawPipeline::HeldReads(address, bytes)) {
         queuedReadHeld.fetch_add(1, std::memory_order_relaxed);
         return false;
     }
+    std::unique_lock<GuestMemory::GpuMutexType> drawnHold;
+    if (auto& drawnPending = Pending(); drawnPending.drawnCount.load(std::memory_order_acquire) != 0 && !GuestMemory::GpuMutex().HeldByThisThread()) {
+        bool overlap = false;
+        {
+            std::lock_guard lock(drawnPending.mutex);
+            overlap = std::any_of(drawnPending.drawn.begin(), drawnPending.drawn.end(), [&](const PendingWrites::Drawn& entry) { return entry.texture->dirty && entry.texture->overlaps(address, bytes); });
+        }
+        if (overlap) {
+            GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush);
+            drawnHold = std::unique_lock(GuestMemory::GpuMutex());
+        }
+    }
     // The images stay alive across the scan: their last owner may be a batch's kept list, which
     // another thread releases outside the GPU mutex once the batch completed.
     std::vector<std::shared_ptr<StorageTexture>> flush;
+    std::vector<std::uint64_t> ownNotes;
     {
         auto& pending = Pending();
         std::lock_guard lock(pending.mutex);
@@ -3948,8 +4235,12 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons
         for (const auto& texture : flush) pending.flushing.push_back(texture.get());
         pending.flushingCount.store(pending.flushing.size(), std::memory_order_release);
         if (!flush.empty()) BumpPendingSerial();
+        for (const auto& texture : flush) {
+            if (texture->drawnListed) ownNotes.insert(ownNotes.end(), texture->storeNotes.begin(), texture->storeNotes.end());
+        }
     }
     if (flush.empty()) {
+        if (drawnHold.owns_lock()) drawnHold.unlock();
         const bool stored = WaitForFlushing(address, bytes, except);
         // Units shadowed over the range (another image's results retiled into the import's unit
         // shadow, or an earlier flush's) still reach the import as the scope asks.
@@ -3973,10 +4264,11 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons
         for (const auto& texture : flush) std::fprintf(stderr, "[flush] image 0x%llx+0x%llx for %s 0x%llx+0x%zx\n", static_cast<unsigned long long>(texture->descriptor.baseAddress), static_cast<unsigned long long>(texture->guestBytes), reason, static_cast<unsigned long long>(address), bytes);
     }
     std::lock_guard gpu(GuestMemory::GpuMutex());
-    if (auto* recorder = Recorder::Active(); recorder != nullptr && !Recorder::InCompletion() && recorder->PendingWriterSerial(address, bytes) != 0) {
+    if (!ownNotes.empty() && (flush.size() != 1 || std::any_of(flush.begin(), flush.end(), [](const std::shared_ptr<StorageTexture>& texture) { return ExistingHostImportFor(texture->context, texture->descriptor.baseAddress, static_cast<std::size_t>(texture->guestBytes)) == nullptr; }))) ownNotes.clear();
+    if (auto* recorder = Recorder::Active(); recorder != nullptr && !Recorder::InCompletion() && recorder->PendingWriterSerial(address, bytes, ownNotes) != 0) {
         try {
             recorder->Reap();
-            if (const auto writer = recorder->PendingWriterSerial(address, bytes); writer == recorder->OpenSerial()) {
+            if (const auto writer = recorder->PendingWriterSerial(address, bytes, ownNotes); writer == recorder->OpenSerial()) {
                 Recorder::CountSync(recorder->OpenWriteOverlaps(address, bytes) ? 1 : 3);
                 recorder->Sync();
             } else if (writer != 0) {
@@ -4086,7 +4378,8 @@ void StorageTexture::saveBeforeUnwatch() {
         const auto end = begin + layerBytes(unit);
         const auto spanBegin = begin & ~(block - 1);
         const auto spanBlocks = static_cast<std::size_t>((end - spanBegin + block - 1) / block);
-        const std::vector<std::uint64_t> generations(spanBlocks, layerGeneration[unit]);
+        std::vector<std::uint64_t> generations(spanBlocks);
+        for (std::size_t k = 0; k < spanBlocks; ++k) generations[k] = unitBlockGeneration(unit, spanBegin + k * block);
         std::vector<std::uint8_t> changedBlocks(spanBlocks);
         if (!GuestMemory::ChangedBlocks(begin, static_cast<std::size_t>(end - begin), generations, changedBlocks)) continue;
         forgetKeptAtUnwatch(unit);
@@ -4094,7 +4387,7 @@ void StorageTexture::saveBeforeUnwatch() {
             const auto from = std::max(at, begin);
             const auto to = std::min(at + block, end);
             const bool edge = from != at || to != at + block;
-            if (changedBlocks[static_cast<std::size_t>((at - spanBegin) / block)] == GuestMemory::BlockWritten && (!edge || GuestMemory::StoredOver(from, static_cast<std::size_t>(to - from), layerGeneration[unit])) && !unchangedSinceBaseline(from, to)) keptAtUnwatch.emplace(from, layerGeneration[unit]);
+            if (changedBlocks[static_cast<std::size_t>((at - spanBegin) / block)] == GuestMemory::BlockWritten && (!edge || GuestMemory::StoredOver(from, static_cast<std::size_t>(to - from), unitBlockGeneration(unit, at))) && !unchangedSinceBaseline(from, to)) keptAtUnwatch.emplace(from, layerGeneration[unit]);
         }
         layerOriginal[unit] = true;
         cancelStoredSave(unit);
@@ -4258,6 +4551,49 @@ void StorageTexture::StoreReadbackBeforeLabel(const char* reason) {
     for (const auto& [address, bytes] : ranges) static_cast<void>(FlushPending(address, bytes, nullptr, reason));
 }
 
+std::size_t StorageTexture::StoreDrawnLinear(const char* reason) {
+    auto& pending = Pending();
+    if (pending.drawnCount.load(std::memory_order_acquire) == 0 || Recorder::InCompletion()) return 0;
+    GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush);
+    std::lock_guard gpu(GuestMemory::GpuMutex());
+    std::vector<std::pair<std::uint64_t, std::size_t>> ranges;
+    {
+        std::lock_guard lock(pending.mutex);
+        for (const auto& entry : pending.drawn) {
+            if (entry.texture->dirty) ranges.emplace_back(entry.texture->descriptor.baseAddress, static_cast<std::size_t>(entry.texture->guestBytes));
+        }
+    }
+    std::size_t stored = 0;
+    for (const auto& [address, bytes] : ranges) {
+        if (FlushPending(address, bytes, nullptr, reason)) ++stored;
+    }
+    return stored;
+}
+
+std::size_t StorageTexture::StoreDrawnLinearTouching(const char* reason, const std::function<bool(std::uint64_t, std::uint64_t)>& touches, bool all) {
+    auto& pending = Pending();
+    if (pending.drawnCount.load(std::memory_order_acquire) == 0 || Recorder::InCompletion()) return 0;
+    std::vector<std::pair<std::uint64_t, std::size_t>> ranges;
+    {
+        std::lock_guard lock(pending.mutex);
+        for (const auto& entry : pending.drawn) {
+            if (!entry.texture->dirty) continue;
+            const auto begin = entry.texture->descriptor.baseAddress;
+            const auto end = begin + entry.texture->guestBytes;
+            if (!all && !touches(begin, end)) continue;
+            ranges.emplace_back(begin, static_cast<std::size_t>(entry.texture->guestBytes));
+        }
+    }
+    if (ranges.empty()) return 0;
+    GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush);
+    std::lock_guard gpu(GuestMemory::GpuMutex());
+    std::size_t stored = 0;
+    for (const auto& [address, bytes] : ranges) {
+        if (FlushPending(address, bytes, nullptr, reason)) ++stored;
+    }
+    return stored;
+}
+
 bool StorageTexture::AnyPending() {
     auto& pending = Pending();
     std::lock_guard lock(pending.mutex);
@@ -4324,37 +4660,96 @@ bool PendingStorageOverlaps(std::uint64_t address, std::size_t bytes, const Stor
 }
 
 std::uint64_t StorageTexture::StampKeepingNewerResults(std::uint64_t address, std::size_t bytes, std::uint64_t recorded, const std::function<std::uint64_t()>& stamp) {
-    std::vector<std::pair<std::shared_ptr<StorageTexture>, std::uint32_t>> kept;
+    struct Candidate {
+        std::shared_ptr<StorageTexture> texture;
+        std::uint32_t unit;
+        bool drawn;
+    };
+    struct Kept {
+        std::shared_ptr<StorageTexture> texture;
+        std::uint32_t unit;
+        std::uint64_t from;
+        std::uint64_t to;
+        std::uint64_t since;
+        bool whole;
+        bool drawn;
+    };
+    std::vector<Candidate> candidates;
     if (recorded != 0 && bytes != 0) {
         auto& pending = Pending();
         std::lock_guard lock(pending.mutex);
-        for (auto* texture : pending.textures) {
-            if (!texture->blockUnits || !texture->overlaps(address, bytes)) continue;
+        const auto consider = [&](StorageTexture* texture) {
+            if ((!texture->blockUnits && !texture->drawnListed) || !texture->overlaps(address, bytes)) return;
             auto alive = texture->weak_from_this().lock();
-            if (alive == nullptr) continue;
+            if (alive == nullptr) return;
             for (std::uint32_t unit = 0; unit < texture->trackedLayers; ++unit) {
                 const auto begin = texture->layerBegin(unit);
                 const auto unitBytes = texture->layerBytes(unit);
                 if (!texture->layerPending[unit] || begin >= address + bytes || address >= begin + unitBytes) continue;
-                if (texture->layerGeneration[unit] >= recorded) kept.emplace_back(alive, unit);
+                if (texture->layerGeneration[unit] >= recorded) candidates.push_back({alive, unit, texture->drawnListed});
             }
+        };
+        for (auto* texture : pending.textures) consider(texture);
+        for (auto* texture : pending.flushing) consider(texture);
+    }
+    constexpr std::uint64_t block = 65536;
+    std::vector<Kept> kept;
+    for (const auto& [texture, unit, drawn] : candidates) {
+        const auto begin = texture->layerBegin(unit);
+        const auto end = begin + texture->layerBytes(unit);
+        GuestMemory::CollectWritesUncached(begin, static_cast<std::size_t>(end - begin));
+        if (GuestMemory::UnchangedSince(begin, static_cast<std::size_t>(end - begin), texture->layerGeneration[unit])) {
+            kept.push_back({texture, unit, begin, end, texture->layerGeneration[unit], true, drawn});
+            continue;
+        }
+        if (!drawn) continue;
+        const auto first = std::max(begin, address) / block;
+        const auto last = (std::min(end, address + bytes) - 1) / block;
+        const auto from = std::max(begin, first * block);
+        const auto to = std::min(end, (last + 1) * block);
+        std::vector<std::uint64_t> generations(static_cast<std::size_t>(last - first + 1));
+        for (std::size_t k = 0; k < generations.size(); ++k) generations[k] = texture->unitBlockGeneration(unit, (first + k) * block);
+        std::vector<std::uint8_t> changed(generations.size());
+        if (!GuestMemory::ChangedBlocks(from, static_cast<std::size_t>(to - from), generations, changed)) continue;
+        for (std::size_t k = 0; k < generations.size(); ++k) {
+            const auto at = (first + k) * block;
+            const auto blockFrom = std::max(at, begin);
+            const auto blockTo = std::min(at + block, end);
+            const bool edge = blockFrom != at || blockTo != at + block;
+            if (changed[k] == GuestMemory::BlockWritten && (!edge || !GuestMemory::UnchangedSince(blockFrom, static_cast<std::size_t>(blockTo - blockFrom), generations[k]))) continue;
+            if (!kept.empty() && !kept.back().whole && kept.back().texture == texture && kept.back().unit == unit && kept.back().since == generations[k] && kept.back().to == blockFrom) kept.back().to = blockTo;
+            else kept.push_back({texture, unit, blockFrom, blockTo, generations[k], false, true});
         }
     }
-    std::erase_if(kept, [](const auto& entry) {
-        const auto& [texture, unit] = entry;
-        const auto begin = texture->layerBegin(unit);
-        const auto unitBytes = static_cast<std::size_t>(texture->layerBytes(unit));
-        GuestMemory::CollectWritesUncached(begin, unitBytes);
-        return !GuestMemory::UnchangedSince(begin, unitBytes, texture->layerGeneration[unit]);
-    });
     const auto generation = stamp();
     if (generation == 0 || kept.empty()) return generation;
+    std::vector<Kept> confirmed;
+    for (const auto& entry : kept) {
+        if ((entry.whole && !entry.drawn) || GuestMemory::UnchangedSinceCollected(entry.from, static_cast<std::size_t>(entry.to - entry.from), entry.since)) {
+            confirmed.push_back(entry);
+            continue;
+        }
+        if (entry.from / block == (entry.to - 1) / block) continue;
+        const auto last = (std::min(entry.to, address + bytes) - 1) / block;
+        for (auto at = std::max(entry.from, address) / block * block; at <= last * block; at += block) {
+            const auto from = std::max(at, entry.from);
+            const auto to = std::min(at + block, entry.to);
+            if (GuestMemory::UnchangedSinceCollected(from, static_cast<std::size_t>(to - from), entry.since)) confirmed.push_back({entry.texture, entry.unit, from, to, entry.since, false, true});
+        }
+    }
     auto& pending = Pending();
     std::lock_guard lock(pending.mutex);
-    for (const auto& [texture, unit] : kept) {
-        if (!texture->layerPending[unit]) continue;
-        texture->layerGeneration[unit] = generation;
-        texture->refreshGeneration();
+    for (const auto& entry : confirmed) {
+        auto& texture = *entry.texture;
+        if (!texture.layerPending[entry.unit]) continue;
+        if (entry.whole) {
+            texture.layerGeneration[entry.unit] = generation;
+            texture.refreshGeneration();
+            continue;
+        }
+        const auto spanBegin = texture.descriptor.baseAddress / block;
+        if (texture.completionKept.empty()) texture.completionKept.assign(static_cast<std::size_t>((texture.descriptor.baseAddress + texture.guestBytes - 1) / block - spanBegin + 1), 0);
+        for (auto at = entry.from / block; at <= (entry.to - 1) / block; ++at) texture.completionKept[static_cast<std::size_t>(at - spanBegin)] = generation;
     }
     return generation;
 }
@@ -4470,6 +4865,7 @@ bool StorageTexture::AccessKeptByCpu(std::uint64_t address, std::size_t bytes, s
     // store there would see, and the stamps only grow.
     GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Hook);
     std::lock_guard gpu(GuestMemory::GpuMutex());
+    if (std::any_of(overlapping.begin(), overlapping.end(), [](const std::shared_ptr<StorageTexture>& texture) { return texture->drawnArea().drawn; })) return false;
     if (!blocksKept(overlapping, address, bytes)) return false;
     const auto site = static_cast<std::uint8_t>(GuestMemory::CurrentReadSite());
     std::size_t dropped = 0;
@@ -4601,6 +4997,7 @@ std::size_t StorageTexture::DiscardPendingInside(std::uint64_t address, std::siz
             texture->layerPending.assign(texture->trackedLayers, false);
             texture->layerOriginal.assign(texture->trackedLayers, false);
             texture->cancelStoredSaves();
+            texture->forgetDrawn();
             it = pending.textures.erase(it);
             ++discarded;
         } else if (texture != exempt && texture->blockUnits && texture->overlaps(address, bytes)) {
@@ -5210,6 +5607,12 @@ void StorageTexture::blockGenerations(std::vector<std::uint64_t>& generations) c
     }
 }
 
+std::uint64_t StorageTexture::unitBlockGeneration(std::uint32_t unit, std::uint64_t at) const {
+    const auto unitGeneration = layerGeneration[unit];
+    if (unitGeneration == 0 || !layerPending[unit] || completionKept.empty()) return unitGeneration;
+    return std::max(unitGeneration, completionKept[static_cast<std::size_t>(at / 65536 - descriptor.baseAddress / 65536)]);
+}
+
 namespace {
 
 std::atomic<bool>& ClearHandOverOn() {
@@ -5506,10 +5909,12 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
     for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) {
         if (!layers[layer]) continue;
         const auto end = layerBegin(layer) + layerBytes(layer);
-        for (auto at = layerBegin(layer) & ~(block - 1); at < end; at += block) generations[static_cast<std::size_t>((at - spanBegin) / block)] = layerGeneration[layer];
+        for (auto at = layerBegin(layer) & ~(block - 1); at < end; at += block) generations[static_cast<std::size_t>((at - spanBegin) / block)] = unitBlockGeneration(layer, at);
     }
+    const auto drawn = drawnArea();
     std::vector<std::uint8_t> changedBlocks(spanBlocks);
-    const bool tracked = GuestMemory::ChangedBlocks(firstStored, static_cast<std::size_t>(lastStored - firstStored), generations, changedBlocks);
+    std::vector<std::uint8_t> cpuBlocks(drawn.drawn ? spanBlocks : 0);
+    const bool tracked = GuestMemory::ChangedBlocks(firstStored, static_cast<std::size_t>(lastStored - firstStored), generations, changedBlocks, cpuBlocks);
     std::vector<bool> compared(trackedLayers, false);
     if (!tracked) {
         awaitStoredSave(&layers);
@@ -5534,7 +5939,8 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
         if (!layers[layer]) continue;
         const auto begin = layerBegin(layer);
         const auto end = begin + layerBytes(layer);
-        if (tracked ? layerGeneration[layer] == 0 : !compared[layer]) {
+        const bool unknown = tracked ? layerGeneration[layer] == 0 : !compared[layer];
+        if (unknown && !drawn.rect) {
             unsavedOutsideWatch = unsavedOutsideWatch || !tracked;
             keep.emplace_back(begin, end);
             continue;
@@ -5544,12 +5950,16 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
             const auto to = std::min(at + block, end);
             if (from >= to) continue;
             const bool edge = from != at || to != at + block;
-            if (keptAtUnwatchHolds(from, layer) || (changedBlocks[static_cast<std::size_t>((at - spanBegin) / block)] == GuestMemory::BlockWritten && (compared[layer] || !edge || GuestMemory::StoredOver(from, static_cast<std::size_t>(to - from), layerGeneration[layer])) && !unchangedSinceBaseline(from, to))) {
+            const auto index = static_cast<std::size_t>((at - spanBegin) / block);
+            const bool kept = keptAtUnwatchHolds(from, layer);
+            const bool written = kept || unknown || (changedBlocks[index] == GuestMemory::BlockWritten && (compared[layer] || !edge || GuestMemory::StoredOver(from, static_cast<std::size_t>(to - from), unitBlockGeneration(layer, at))) && !unchangedSinceBaseline(from, to));
+            if (written && (!drawn.rect || (tracked && !kept && !unknown && cpuBlocks[index] == 0))) {
                 skippedAny = true;
                 skippedLayer[layer] = true;
                 ++skipped;
                 continue;
             }
+            if (written && drawn.rect) skippedLayer[layer] = true;
             if (!keep.empty() && keep.back().second == from) keep.back().second = to;
             else keep.emplace_back(from, to);
         }
@@ -5563,10 +5973,15 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
         const auto first = keep.empty() ? descriptor.baseAddress + guestBytes : keep.front().first;
         std::fprintf(stderr, "[flush] image 0x%llx+0x%llx keeps %zu CPU-written 64 KiB blocks (first stored byte at +0x%llx, %zu ranges stored, generation %llu)\n", static_cast<unsigned long long>(descriptor.baseAddress), static_cast<unsigned long long>(guestBytes), skipped, static_cast<unsigned long long>(first - descriptor.baseAddress), keep.size(), static_cast<unsigned long long>(generation));
     }
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> spans;
+    if (drawn.rect) spans = drawnSpans(drawn, keep);
+    const auto& stores = drawn.rect ? spans : keep;
+    const auto stampBegin = drawn.rect && !stores.empty() ? stores.front().first : firstStored;
+    const auto stampEnd = drawn.rect && !stores.empty() ? stores.back().second : lastStored;
     // Taken before this write-back stamps anything (see advanceAdjacent), with the 64 KiB block span
     // its stamps cover.
-    const auto firstBlock = firstStored / block;
-    const auto lastBlock = (lastStored - 1) / block;
+    const auto firstBlock = stampBegin / block;
+    const auto lastBlock = (stampEnd - 1) / block;
     const auto adjacent = adjacentPendingUnchanged(firstBlock, lastBlock);
     // After the store: the stored layers are in sync at a fresh generation unless blocks were kept
     // for the CPU (then the layer is stale there, keeps its old generation and re-uploads at its
@@ -5603,7 +6018,7 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
         advanceAdjacent(adjacent, now, firstBlock, lastBlock);
         if (!shadowed.empty()) markShadowed(now);
     };
-    if (keep.empty()) {
+    if (stores.empty()) {
         // Every selected block was written by the CPU since the layer's generation: nothing to
         // store, the keys stay as they are, and the image is stale there (re-uploaded at its next
         // use, never from a matching `original`).
@@ -5633,6 +6048,24 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
             ++Profile().storageDirectWriteBacks;
             return;
         }
+        if (drawn.rect) {
+            PublishShadow(stampBegin, static_cast<std::size_t>(stampEnd - stampBegin), PublishScope::PartialUnits, PublishReason::Upload);
+            const auto storedBytes = storeDrawnRect(drawn, segments, stores, saveUnits);
+            countStorageWriteBack(storedBytes, true, true);
+            if (profile) Profile().storageGpu += timer.lap();
+            forgetStored();
+            uploadedKeys = DccKeys::Uncompressed;
+            std::vector<std::pair<std::uint64_t, std::uint64_t>> stamps;
+            for (const auto& [from, to] : stores) {
+                if (!stamps.empty() && from / block <= (stamps.back().second - 1) / block + 1) stamps.back().second = to;
+                else stamps.emplace_back(from, to);
+            }
+            for (const auto& [from, to] : stamps) GuestMemory::MarkWritten(from, static_cast<std::size_t>(to - from));
+            settle(true);
+            commitStoredSave(saveUnits);
+            ++Profile().storageDirectWriteBacks;
+            return;
+        }
         // The whole-layer copies below overwrite the interior units of the span; the edge units
         // keep a neighbour's shadowed results in the rest of their bytes, which reach the import
         // first (this write-back's stamps then make those units stale).
@@ -5735,8 +6168,9 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
             recorder->EndGpuTiming(timing, storedBytes);
             recorder->MarkShaderReadsCovered();
             recorder->NotePendingWrite(firstStored, static_cast<std::size_t>(lastStored - firstStored));
+            if (drawn.drawn) noteStore(recorder->LastWriteNote(firstStored, static_cast<std::size_t>(lastStored - firstStored)));
         }
-        countStorageWriteBack(storedBytes, true);
+        countStorageWriteBack(storedBytes, true, drawn.drawn);
         if (profile) Profile().storageGpu += timer.lap();
         // The guest bytes now differ from `original`; other caches of the range see the write.
         forgetStored();
@@ -5777,10 +6211,10 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
             overhang = covered >= end;
         }
     }
-    const bool registered = std::all_of(keep.begin(), keep.end(), [](const auto& range) { return RegisteredReadableCovers(range.first, static_cast<std::size_t>(range.second - range.first)); });
+    const bool registered = std::all_of(stores.begin(), stores.end(), [](const auto& range) { return RegisteredReadableCovers(range.first, static_cast<std::size_t>(range.second - range.first)); });
     if (unregisteredDrop && !overhang && !registered) {
         static std::atomic<int> reports{0};
-        if (reports.fetch_add(1) < 4) std::fprintf(stderr, "[gpu] storage image 0x%llx+0x%llx: memory no longer registered; %zu pending ranges dropped\n", static_cast<unsigned long long>(descriptor.baseAddress), static_cast<unsigned long long>(guestBytes), keep.size());
+        if (reports.fetch_add(1) < 4) std::fprintf(stderr, "[gpu] storage image 0x%llx+0x%llx: memory no longer registered; %zu pending ranges dropped\n", static_cast<unsigned long long>(descriptor.baseAddress), static_cast<unsigned long long>(guestBytes), stores.size());
         unregisteredDropped.fetch_add(1, std::memory_order_relaxed);
         forgetStored();
         for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) {
@@ -5797,6 +6231,10 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
     std::vector<bool> unsaved = layerOriginal;
     unsaved.flip();
     for (const auto& [begin, end] : unitRuns(unsaved)) {
+        if (drawn.rect) {
+            GuestMemory::ReadCommitted(descriptor.baseAddress + begin, std::span(original).subspan(static_cast<std::size_t>(begin), static_cast<std::size_t>(end - begin)));
+            continue;
+        }
         for (const auto& [from, to] : keep) {
             const auto first = std::max(descriptor.baseAddress + begin, from);
             const auto last = std::min(descriptor.baseAddress + end, to);
@@ -5858,7 +6296,9 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
     APS5_LOG_CHARS_OUT("StorageTexture writeback submit");
     batch.SubmitAndWait();
     APS5_LOG_CHARS_OUT("StorageTexture writeback done");
-    countStorageWriteBack(guestBytes, false);
+    std::uint64_t storedBytes = 0;
+    for (const auto& [from, to] : stores) storedBytes += to - from;
+    countStorageWriteBack(drawn.rect ? storedBytes : guestBytes, false, drawn.drawn);
     if (profile) Profile().storageGpu += timer.lap();
     if (dump) {
         char name[64];
@@ -5873,7 +6313,7 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
     // Store only the kept ranges' blocks that changed, so concurrent CPU writes to untouched texels
     // survive; `original` follows for the stored layers (the others' guest bytes are unchanged).
     const auto current = host.Bytes();
-    for (const auto& [from, to] : keep) {
+    for (const auto& [from, to] : stores) {
         const auto offset = static_cast<std::size_t>(from - descriptor.baseAddress);
         const auto length = static_cast<std::size_t>(to - from);
         GuestMemory::WriteChangedCommitted(from, current.subspan(offset, length), std::span<const std::byte>(original).subspan(offset, length));
@@ -5909,6 +6349,7 @@ StorageTexture::~StorageTexture() {
     {
         auto& pending = Pending();
         std::lock_guard lock(pending.mutex);
+        forgetDrawn();
         if (dirty) {
             dirty = false;
             pending.textures.remove(this);

core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp

@@ -466,23 +466,26 @@ bool LevelsFitAllocation(const GuestTextureResource& surface, std::uint32_t leve
     return true;
 }
 
-GuestTextureResource ViewedArrayWindow(const GuestTextureResource& viewed) {
+GuestTextureResource ViewedArrayWindow(const GuestTextureResource& viewed, bool keysReadAsStored) {
     const bool layered = viewed.dimension == TextureDimension::k2DArray || viewed.dimension == TextureDimension::k1DArray || viewed.dimension == TextureDimension::kCube;
-    if (!layered || viewed.baseArray == 0 || viewed.baseArray > viewed.depthOrLastArray || viewed.dccAddress != 0) return viewed;
+    if (!layered || viewed.baseArray == 0 || viewed.baseArray > viewed.depthOrLastArray || (viewed.dccAddress != 0 && !keysReadAsStored)) return viewed;
+    std::uint32_t sliceMask = 0;
     if (const auto xorMode = XorSwizzleMode(viewed.tileMode); xorMode != 0u) {
         const auto* equation = FindTextureSwizzleEquation(xorMode, BytesPerElement(viewed.format));
         if (equation == nullptr) return viewed;
         std::uint32_t sliceBits = 0;
         for (const auto bit : equation->bits) sliceBits |= bit >> 24u;
-        const auto sliceMask = sliceBits == 0 ? 0u : (1u << std::bit_width(sliceBits)) - 1u;
-        if ((viewed.baseArray & sliceMask) != 0) return viewed;
+        sliceMask = sliceBits == 0 ? 0u : (1u << std::bit_width(sliceBits)) - 1u;
     }
+    const auto first = viewed.baseArray & ~sliceMask;
+    if (first == 0) return viewed;
     const auto geometry = DescribeSurface(viewed);
     if (geometry.thick || geometry.layers != viewed.depthOrLastArray + 1u) return viewed;
     auto window = viewed;
-    window.baseAddress = viewed.baseAddress + geometry.GuestLayerOffset(viewed.baseArray);
-    window.depthOrLastArray = viewed.depthOrLastArray - viewed.baseArray;
-    window.baseArray = 0;
+    window.baseAddress = viewed.baseAddress + geometry.GuestLayerOffset(first);
+    window.depthOrLastArray = viewed.depthOrLastArray - first;
+    window.baseArray = viewed.baseArray - first;
+    window.dccAddress = 0;
     return window;
 }
 

core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp

@@ -2360,6 +2360,64 @@ void importedHeapMirrorTests(const Context& context, const BdaTestAccess& access
     GuestArena::GuestArenaRelease_nid_postfix(block, bytes);
 }
 
+void retiredImportWatchTests(const Context& context) {
+#ifdef _WIN32
+    if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) {
+        std::cout << "guest arena unavailable or not write-watched: the write watch of retired imports not tested\n";
+        return;
+    }
+    constexpr std::size_t piece = 65536;
+    constexpr std::size_t bytes = 2 * piece;
+    void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, bytes);
+    GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes);
+    std::memset(block, 0x5a, bytes);
+    auto* guest = static_cast<std::uint8_t*>(block);
+    const auto address = reinterpret_cast<std::uintptr_t>(block);
+    auto importing = context;
+    importing.hostImportAlignment = piece;
+    SetHostImportWindow(bytes);
+    SetHostImportDeferredSettles(false);
+    ClearHostImports(importing.device);
+    const auto decided = PrepareImportWatch(importing);
+    SetImportWatch(importing, ImportWatch::Unwatch);
+    const auto registry = [&](std::size_t offset, bool add) {
+        auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
+        if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, guest + offset, piece, true, true, true);
+        else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, guest + offset);
+        GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
+    };
+    const auto build = [&] {
+        GuestBufferMemory leased(importing);
+        leased.AcquireRegistered();
+        leased.Upload(true);
+        leased.WriteBack();
+    };
+    registry(0, true);
+    build();
+    Require(HostImportCovers(importing, address, piece), "the first 64 KiB piece was not imported on its own");
+    Require(!AgcDriver::GuestMemory::Watched(address, piece), "an unwatched 64 KiB import kept its piece in the write watch");
+    registry(piece, true);
+    SetHostImportBudget(4096);
+    const auto before = MirrorCounters();
+    build();
+    Require(!HostImportCovers(importing, address, piece), "the piece's import outlived the run that grew past it, or a 128 KiB block fit a 4 KiB budget");
+    Require(AgcDriver::GuestMemory::Watched(address, piece), "a 64 KiB piece whose import retired when its run grew into a block stayed outside the write watch");
+    Require(MirrorCounters().heapMirrors == before.heapMirrors + 2, "a 64 KiB piece whose import retired when its run grew into a block was copied, not heap mirrored, once no import fit");
+    registry(piece, false);
+    registry(0, false);
+    build();
+    SetImportWatch(importing, decided);
+    ClearHostImports(importing.device);
+    SetHostImportBudget(0);
+    SetHostImportWindow(0);
+    SetHostImportDeferredSettles(true);
+    GuestArena::GuestArenaReset_nid_postfix(block, bytes);
+    GuestArena::GuestArenaRelease_nid_postfix(block, bytes);
+#else
+    static_cast<void>(context);
+#endif
+}
+
 void importedFreshTests(const Context& context) {
 #ifndef _WIN32
     if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) {
@@ -3574,6 +3632,200 @@ void hostWriteWatchExclusionTests() {
 #endif
 }
 
+void sparseJoinedHeapMirrorTests(const Context& context, const BdaTestAccess& access) {
+    constexpr std::size_t half = 1u << 20u;
+    constexpr std::size_t reserved = 2 * half;
+    constexpr std::size_t page = 65536;
+#ifdef _WIN32
+    if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) {
+        std::cout << "guest arena unavailable or not write-watched: sparse heap mirrors over adjacent ranges not tested\n";
+        return;
+    }
+    void* block = GuestArena::GuestArenaAllocate_nid_postfix(reserved, page);
+    auto* guest = static_cast<std::uint8_t*>(block);
+    GuestArena::GuestArenaCommit_nid_postfix(guest, page, PAGE_READWRITE, page);
+    GuestArena::GuestArenaCommit_nid_postfix(guest + half, page, PAGE_READWRITE, page);
+#else
+    if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) {
+        std::cout << "write watch unavailable: sparse heap mirrors over adjacent ranges not tested\n";
+        return;
+    }
+    void* mapping = mmap(nullptr, reserved + page, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    Require(mapping != MAP_FAILED, "cannot reserve the sparse heap over adjacent ranges");
+    auto* guest = reinterpret_cast<std::uint8_t*>((reinterpret_cast<std::uintptr_t>(mapping) + page - 1) & ~std::uintptr_t{page - 1});
+    Require(mprotect(guest, page, PROT_READ | PROT_WRITE) == 0 && mprotect(guest + half, page, PROT_READ | PROT_WRITE) == 0, "cannot commit the sparse heap pages");
+    GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(guest, reserved);
+    void* block = guest;
+#endif
+    auto* second = guest + half;
+    std::memset(guest, 0x5a, page);
+    std::memset(second, 0x6b, page);
+    const auto base = reinterpret_cast<std::uintptr_t>(block);
+    const auto registry = [&](bool add) {
+        auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
+        if (add) {
+            GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, guest, half, true, true, true);
+            GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, second, half, true, true, true);
+        } else {
+            GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, guest);
+            GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, second);
+        }
+        GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
+    };
+    const auto sweep = [&] {
+        GuestBufferMemory memory(context);
+        memory.AcquireRegistered();
+        memory.Upload(true);
+        memory.WriteBack();
+    };
+    sweep();
+    registry(true);
+    const auto bind = [&] {
+        GuestBufferMemory memory(context);
+        memory.AddWritable(base, reserved);
+        memory.Upload(false);
+        std::uint32_t adjustment = 0;
+        const auto view = memory.Descriptor(base, reserved, adjustment);
+        Require(adjustment == 0, "a sparse heap view over adjacent ranges is adjusted");
+        const auto bytes = access.bytes(view.buffer);
+        Require(view.range >= half + page && bytes[view.offset + page - 1] == std::byte{0x5a} && bytes[view.offset + page] == std::byte{} && bytes[view.offset + half] == std::byte{0x6b}, "a sparse heap over adjacent ranges is not bound to its committed extent");
+        bytes[view.offset + half + 8] = std::byte{0x77};
+        memory.WriteBack();
+        return view;
+    };
+    const auto before = MirrorCounters();
+    const auto first = bind();
+    const auto made = MirrorCounters();
+    Require(made.sparseBinds == before.sparseBinds + 1 && made.heapMirrors == before.heapMirrors + 1, "a sparse heap whose committed span crosses an adjacent range was not mirrored");
+    Require(second[8] == 0x77, "a store to the adjacent range of a mirrored sparse heap was not written back");
+    second[8] = 0x6b;
+    const auto again = bind();
+    Require(again.buffer == first.buffer && MirrorCounters().rebuilds == made.rebuilds && MirrorCounters().sparseBinds == made.sparseBinds + 1, "a sparse heap over unchanged adjacent ranges did not reuse its mirror");
+    registry(false);
+    sweep();
+    Require(MirrorCounters().heapMirrors == before.heapMirrors && MirrorCounters().heapBytes == before.heapBytes, "the sparse heap mirror over adjacent ranges outlived its ranges");
+#ifdef _WIN32
+    GuestArena::GuestArenaReset_nid_postfix(block, reserved);
+    GuestArena::GuestArenaRelease_nid_postfix(block, reserved);
+#else
+    GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(guest, reserved);
+    munmap(mapping, reserved + page);
+#endif
+}
+
+void joinedHeapMirrorPartialImportTests(const Context& context, const BdaTestAccess& access) {
+    constexpr std::size_t half = 1u << 20u;
+    constexpr std::size_t committed = 2 * half;
+    constexpr std::size_t reserved = 4 * half;
+    constexpr std::size_t page = 65536;
+#ifdef _WIN32
+    if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) {
+        std::cout << "guest arena unavailable or not write-watched: joined heap mirrors under a partial import not tested\n";
+        return;
+    }
+    void* block = GuestArena::GuestArenaAllocate_nid_postfix(reserved, half);
+    auto* guest = static_cast<std::uint8_t*>(block);
+    GuestArena::GuestArenaCommit_nid_postfix(guest, committed, PAGE_READWRITE, page);
+#else
+    if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) {
+        std::cout << "write watch unavailable: joined heap mirrors under a partial import not tested\n";
+        return;
+    }
+    void* mapping = mmap(nullptr, reserved + half, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    Require(mapping != MAP_FAILED, "cannot reserve the joined heap");
+    auto* guest = reinterpret_cast<std::uint8_t*>((reinterpret_cast<std::uintptr_t>(mapping) + half - 1) & ~std::uintptr_t{half - 1});
+    Require(mprotect(guest, committed, PROT_READ | PROT_WRITE) == 0, "cannot commit the joined heap ranges");
+    GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(guest, reserved);
+    void* block = guest;
+#endif
+    std::memset(guest, 0x3c, committed);
+    const auto base = reinterpret_cast<std::uintptr_t>(block);
+    const auto registry = [&](bool add) {
+        auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
+        if (add) {
+            GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, guest, half, true, true, true);
+            GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, guest + half, half, true, true, true);
+        } else {
+            GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, guest);
+            GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, guest + half);
+        }
+        GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
+    };
+    const auto sweep = [&](const Context& with) {
+        GuestBufferMemory memory(with);
+        memory.AcquireRegistered();
+        memory.Upload(true);
+        memory.WriteBack();
+    };
+    const auto bind = [&] {
+        GuestBufferMemory memory(context);
+        memory.AddWritable(base, reserved);
+        memory.Upload(false);
+        std::uint32_t adjustment = 0;
+        const auto view = memory.Descriptor(base, reserved, adjustment);
+        Require(adjustment == 0 && access.bytes(view.buffer)[view.offset + half + 32] == std::byte{0x3c}, "a heap bound across two committed ranges did not read its second range");
+        memory.WriteBack();
+        return view;
+    };
+    sweep(context);
+    registry(true);
+    const auto before = MirrorCounters();
+    const auto first = bind();
+    const auto made = MirrorCounters();
+    Require(made.heapMirrors == before.heapMirrors + 1 && made.sparseBinds == before.sparseBinds + 1, "a heap bound across two committed ranges was not mirrored over both");
+    auto importing = context;
+    importing.hostImportAlignment = page;
+    SetHostImportWindow(half);
+    SetHostImportRangeWindow(half);
+    SetHostImportBudget(HostImportLiveBytes() + half);
+    {
+        GuestBufferMemory memory(importing);
+        memory.AcquireRegistered();
+        Require(ExistingHostImportFor(importing, base, half) != nullptr && ExistingHostImportFor(importing, base + half, half) == nullptr, "the joined heap partial-import fixture did not import exactly its first range");
+        Require(HeapMirrorsWithin(base, base + half).mirrors == 1, "an address build that imported one range of a joined heap mirror released the mirror");
+        memory.Upload(true);
+        memory.WriteBack();
+    }
+    const auto partial = MirrorCounters();
+    const auto again = bind();
+    Require(again.buffer == first.buffer && MirrorCounters().rebuilds == partial.rebuilds, "a joined heap mirror was rebuilt after an import of one of its ranges");
+    {
+        GuestBufferMemory memory(importing);
+        memory.AcquireRegistered();
+        memory.AddWritable(base + half, 32);
+        memory.Upload(true);
+        if (ExistingHostImportFor(importing, base + half, 32) != nullptr) guest[half] = 0x71;
+        else {
+            std::uint32_t adjustment = 0;
+            const auto view = memory.Descriptor(base + half, 32, adjustment);
+            access.bytes(view.buffer)[view.offset + adjustment] = std::byte{0x71};
+        }
+        guest[half + 8] = 0x52;
+        memory.WriteBack();
+    }
+    Require(guest[half] == 0x71 && guest[half + 8] == 0x52, "a joined heap address build lost GPU or genuine CPU stores");
+    const auto refreshed = bind();
+    Require(refreshed.buffer == first.buffer && access.bytes(refreshed.buffer)[refreshed.offset + half] == std::byte{0x71} && access.bytes(refreshed.buffer)[refreshed.offset + half + 8] == std::byte{0x52}, "the retained joined mirror did not refresh stores from an address build");
+    SetHostImportBudget(HostImportLiveBytes() + committed);
+    GuestAllocations::GuestAllocationsEnd_nid_postfix(GuestAllocations::GuestAllocationsBegin_nid_postfix());
+    sweep(importing);
+    Require(ExistingHostImportFor(importing, base, half) != nullptr && ExistingHostImportFor(importing, base + half, half) != nullptr, "the joined heap full-import fixture did not import both ranges");
+    Require(HeapMirrorsWithin(base, base + committed).mirrors == 0, "imports covering every joined heap range left a heap mirror allocated");
+    SetHostImportBudget(0);
+    SetHostImportWindow(0);
+    SetHostImportRangeWindow(std::uint64_t{64} << 20u);
+    ClearHostImports(context.device);
+    registry(false);
+    sweep(context);
+#ifdef _WIN32
+    GuestArena::GuestArenaReset_nid_postfix(block, reserved);
+    GuestArena::GuestArenaRelease_nid_postfix(block, reserved);
+#else
+    GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(guest, reserved);
+    munmap(mapping, reserved + half);
+#endif
+}
+
 void sparseHeapMirrorTests(const Context& context, const BdaTestAccess& access) {
 #ifdef _WIN32
     if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) {
@@ -3974,6 +4226,7 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
     heapMirrorTests(context, access);
     partialHeapImportBudgetTests(context, access);
     importedHeapMirrorTests(context, access);
+    retiredImportWatchTests(context);
     importWatchTests();
     importedFreshTests(context);
     gpuMappingTests(context);
@@ -3982,6 +4235,8 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
     spaceCommittedPagesTests(context, access);
     hostWriteWatchExclusionTests();
     sparseHeapMirrorTests(context, access);
+    sparseJoinedHeapMirrorTests(context, access);
+    joinedHeapMirrorPartialImportTests(context, access);
     Require(AddressCopyOverflow({{0x1000, 0x3000, 0x2000, "uncommitted pages"}}, 0x2000).empty(), "copies within the limit were refused");
     const auto copies = AddressCopyOverflow({{0x1000, 0x2000, 0x1000, "not mirrored"}, {0x10000, 0x30000, 0x18000, "uncommitted pages"}}, 0x2000);
     Require(!copies.empty() && copies.find("0x10000+0x20000 (0.1 MiB committed, uncommitted pages)") < copies.find("0x1000+0x1000"), "the copy limit does not name the largest copy first");

core/libs/prx/libSceAgcDriver/tests/DeferredDestruction.cpp

@@ -0,0 +1,131 @@
+#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
+#include <chrono>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <exception>
+#include <string>
+#include <vector>
+
+using namespace AgcDriver::Graphics;
+
+namespace {
+
+int failures = 0;
+
+void Expect(bool condition, const std::string& what) {
+    if (condition) return;
+    std::fprintf(stderr, "FAIL: %s\n", what.c_str());
+    ++failures;
+}
+
+std::vector<std::string> events;
+VkResult idleResult = VK_SUCCESS;
+
+template<typename THandle>
+THandle Handle(std::uintptr_t value) {
+    return reinterpret_cast<THandle>(value);
+}
+
+std::string Name(const char* kind, const void* handle) {
+    return std::string(kind) + std::to_string(reinterpret_cast<std::uintptr_t>(handle));
+}
+
+VKAPI_ATTR void VKAPI_CALL mockDestroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks*) {
+    events.push_back(Name("buffer ", reinterpret_cast<const void*>(buffer)) + Name(" on ", device));
+}
+
+VKAPI_ATTR void VKAPI_CALL mockFreeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks*) {
+    events.push_back(Name("memory ", reinterpret_cast<const void*>(memory)) + Name(" on ", device));
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL mockQueueWaitIdle(VkQueue queue) {
+    events.push_back(Name("idle ", queue));
+    return idleResult;
+}
+
+VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL mockProc(VkDevice, const char* name) {
+    if (std::strcmp(name, "vkDestroyBuffer") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&mockDestroyBuffer);
+    if (std::strcmp(name, "vkFreeMemory") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&mockFreeMemory);
+    if (std::strcmp(name, "vkQueueWaitIdle") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&mockQueueWaitIdle);
+    return nullptr;
+}
+
+Context ContextFor(VkDevice device, PFN_vkGetDeviceProcAddr proc) {
+    Context context{};
+    context.device = device;
+    context.queue = Handle<VkQueue>(0x900);
+    context.deviceProc = proc;
+    return context;
+}
+
+void DeferredUntilTheQueueIsIdle() {
+    events.clear();
+    const auto device = Handle<VkDevice>(0x100);
+    const auto proc = DeferDestructionToIdle(device, mockProc);
+    const auto context = ContextFor(device, proc);
+    Expect(context.Function<PFN_vkQueueWaitIdle>("vkQueueWaitIdle") == &mockQueueWaitIdle, "a function other than vkDestroyBuffer and vkFreeMemory was not passed through");
+    context.Function<PFN_vkFreeMemory>("vkFreeMemory")(device, Handle<VkDeviceMemory>(0x21), nullptr);
+    context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(device, Handle<VkBuffer>(0x11), nullptr);
+    context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(device, VK_NULL_HANDLE, nullptr);
+    Expect(events.empty() && DeferredObjects(device) == 2, "a destroy or free was not deferred, or a null handle was queued");
+    const auto start = std::chrono::steady_clock::now();
+    DestroyDeferredAtIdle(context, start);
+    Expect(events.empty() && DeferredObjects(device) == 2, "deferred objects younger than the drain age were destroyed, or the queue was waited for");
+    DestroyDeferredAtIdle(context, start + std::chrono::seconds(1));
+    const std::vector<std::string> drained{"idle 2304", "buffer 17 on 256", "memory 33 on 256"};
+    Expect(events == drained && DeferredObjects(device) == 0, "old deferred objects were not destroyed after a queue wait, buffers before memory");
+    events.clear();
+    for (std::uintptr_t index = 0; index < 512; ++index) context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(device, Handle<VkBuffer>(0x1000 + index), nullptr);
+    DestroyDeferredAtIdle(context, std::chrono::steady_clock::now());
+    Expect(events.size() == 513 && events.front() == "idle 2304" && DeferredObjects(device) == 0, "512 young deferred objects were not destroyed after a queue wait");
+    events.clear();
+    idleResult = VK_ERROR_OUT_OF_HOST_MEMORY;
+    context.Function<PFN_vkFreeMemory>("vkFreeMemory")(device, Handle<VkDeviceMemory>(0x22), nullptr);
+    bool threw = false;
+    try {
+        DestroyDeferredAtIdle(context, std::chrono::steady_clock::now() + std::chrono::seconds(1));
+    } catch (const std::exception&) {
+        threw = true;
+    }
+    Expect(threw && DeferredObjects(device) == 1 && events.size() == 1, "a failed queue wait did not throw, or destroyed the deferred objects");
+    idleResult = VK_SUCCESS;
+    events.clear();
+    DestroyDeferredObjects(device);
+    Expect(events == std::vector<std::string>{"memory 34 on 256"} && DeferredObjects(device) == 0, "the deferred objects were not destroyed when the device went, or the queue was waited for");
+    events.clear();
+    context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(device, Handle<VkBuffer>(0x12), nullptr);
+    Expect(events == std::vector<std::string>{"buffer 18 on 256"}, "a destroy after the device went was deferred");
+}
+
+void OtherDevicesDestroyAtOnce() {
+    events.clear();
+    const auto device = Handle<VkDevice>(0x101);
+    const auto proc = DeferDestructionToIdle(device, mockProc);
+    const auto other = Handle<VkDevice>(0x102);
+    reinterpret_cast<PFN_vkFreeMemory>(proc(other, "vkFreeMemory"))(other, Handle<VkDeviceMemory>(0x23), nullptr);
+    Expect(events == std::vector<std::string>{"memory 35 on 258"} && DeferredObjects(other) == 0, "a free on a device without deferred destruction was deferred");
+    bool threw = false;
+    try {
+        DeferDestructionToIdle(device, mockProc);
+    } catch (const std::exception&) {
+        threw = true;
+    }
+    Expect(threw, "deferred destruction was set up twice for one device");
+    DestroyDeferredObjects(device);
+}
+
+}
+
+int main() {
+    try {
+        DeferredUntilTheQueueIsIdle();
+        OtherDevicesDestroyAtOnce();
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "FAIL: %s\n", error.what());
+        return 1;
+    }
+    if (failures != 0) return 1;
+    std::puts("deferred destruction tests passed");
+    return 0;
+}

core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp

@@ -17,6 +17,7 @@
 #include <memory>
 #include <stdexcept>
 #include <string>
+#include <thread>
 #include <vector>
 #ifdef _WIN32
 #include <windows.h>
@@ -100,14 +101,18 @@ std::vector<std::byte> pattern(std::uint8_t seed) {
     return bytes;
 }
 
-void copyFrame(std::uint32_t* words, const Layout& layout, std::byte* target) {
-    CommandBuffer buffer{words, words + CommandBytes / sizeof(std::uint32_t), words, words + CommandBytes / sizeof(std::uint32_t), nullptr, nullptr, 0};
-    sceAgcDcbDmaData(&buffer, 0, DmaMemory, 0, at(target), DmaMemory, 0, at(layout.source), DataBytes, 0, 1, 0);
-    sceAgcDcbWriteData(&buffer, GraphicsMemory, 0, at(layout.label), &Done, 1, 0, 1);
+void copy(std::uint32_t* words, std::size_t commandBytes, const std::byte* source, std::byte* target, std::uint32_t* label) {
+    CommandBuffer buffer{words, words + commandBytes / sizeof(std::uint32_t), words, words + commandBytes / sizeof(std::uint32_t), nullptr, nullptr, 0};
+    sceAgcDcbDmaData(&buffer, 0, DmaMemory, 0, at(target), DmaMemory, 0, at(source), DataBytes, 0, 1, 0);
+    sceAgcDcbWriteData(&buffer, GraphicsMemory, 0, at(label), &Done, 1, 0, 1);
     Packet packet{words, static_cast<std::uint32_t>(buffer.cursor_up - words), 0, {}};
     check(sceAgcDriverSubmitDcb(&packet) == 0, "submission failed");
 }
 
+void copyFrame(std::uint32_t* words, const Layout& layout, std::byte* target) {
+    copy(words, CommandBytes, layout.source, target, layout.label);
+}
+
 void flip() {
     AgcDriverSubmitFlip_nid_postfix(AgcDriver::FlipInfo{Handle, -1, 1, 0});
 }
@@ -187,15 +192,75 @@ void testCaptureAndReplay(const std::filesystem::path& tool) {
     check(readFile(secondDump) == second, "the replay did not reproduce the second copy");
 }
 
+void testConcurrentCopies(const std::filesystem::path& tool) {
+    constexpr std::size_t Copies = 64;
+    constexpr std::size_t Frames = 8;
+    constexpr std::size_t CopyCommandBytes = 1024;
+    const char* directory = std::getenv("APS5_FRAME_CAPTURE");
+    check(directory != nullptr && *directory != '\0', "APS5_FRAME_CAPTURE is not set");
+    const std::filesystem::path capture(directory);
+    const auto layout = allocate();
+    const auto other = allocate();
+    auto* const otherSource = other.base + 128 * 1024;
+    auto* const targets = otherSource + DataBytes;
+    auto* const otherLabel = reinterpret_cast<std::uint32_t*>(other.base + 512 * 1024);
+    const auto copied = pattern(57);
+    std::memcpy(otherSource, copied.data(), copied.size());
+    auto output = std::make_shared<Output>();
+    AgcDriverRegisterVideoOutput_nid_postfix(Handle, output);
+
+    flip();
+    std::thread copier([&] {
+        for (std::size_t index = 0; index < Copies; ++index) {
+            copy(reinterpret_cast<std::uint32_t*>(other.base + index * CopyCommandBytes), CopyCommandBytes, otherSource, targets + index * DataBytes, otherLabel);
+        }
+    });
+    for (std::size_t frame = 0; frame + 1 < Frames; ++frame) {
+        const auto bytes = pattern(static_cast<std::uint8_t>(frame));
+        std::memcpy(layout.source, bytes.data(), bytes.size());
+        copyFrame(layout.firstCommands, layout, layout.firstTarget);
+        flip();
+    }
+    copier.join();
+    flip();
+    AgcDriverWaitIdle_nid_postfix();
+
+    const FrameCaptureReader reader(capture);
+    check(!reader.Truncated(), "the capture is truncated");
+    std::uint64_t base = 0;
+    std::size_t submits = 0, memory = 0;
+    for (const auto& call : reader.Calls()) {
+        if (call.kind == FrameCallKind::FrameStart) base = call.completed;
+        if (call.kind == FrameCallKind::Memory) {
+            ++memory;
+            check(call.completed >= base + submits, "a memory delta was read with " + std::to_string(base + submits - call.completed) + " of the " + std::to_string(submits) + " submissions recorded before it unfinished");
+        }
+        submits += call.kind == FrameCallKind::Submit;
+    }
+    check(submits == Copies + Frames - 1, "the capture holds " + std::to_string(submits) + " submits, not " + std::to_string(Copies + Frames - 1));
+    check(memory != 0, "the capture holds no memory deltas");
+
+    const auto dump = capture / "targets.bin";
+    const auto command = "\"\"" + tool.string() + "\" \"" + capture.string() + "\" --strict --dump " + std::to_string(at(targets)) + ":" + std::to_string(Copies * DataBytes) + ":\"" + dump.string() + "\"\"";
+    const auto code = std::system(command.c_str());
+    check(code == 0, "agc_frame_replay exited with " + std::to_string(code));
+    const auto replayed = readFile(dump);
+    check(replayed.size() == Copies * DataBytes, "the replay dumped " + std::to_string(replayed.size()) + " bytes of the copy targets");
+    for (std::size_t index = 0; index < Copies; ++index) {
+        check(std::memcmp(replayed.data() + index * DataBytes, copied.data(), DataBytes) == 0, "the replay did not reproduce copy " + std::to_string(index) + " of the second thread");
+    }
+}
+
 }
 
 int main(int argc, char** argv) {
-    if (argc != 2) {
-        std::fprintf(stderr, "usage: agc_driver_frame_replay_tests <agc_frame_replay>\n");
+    if (argc != 2 && !(argc == 3 && std::string(argv[2]) == "concurrent")) {
+        std::fprintf(stderr, "usage: agc_driver_frame_replay_tests <agc_frame_replay> [concurrent]\n");
         return 2;
     }
     try {
-        testCaptureAndReplay(argv[1]);
+        if (argc == 3) testConcurrentCopies(argv[1]);
+        else testCaptureAndReplay(argv[1]);
     } catch (const std::exception& error) {
         std::fprintf(stderr, "FAILED: %s\n", error.what());
         return 1;

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -4674,6 +4674,72 @@ void DepthRemapTests() {
     Require(floating.depthRemap.mode == 1 && floating.viewport.minDepth == 0 && floating.viewport.maxDepth == 1 && floating.depthClamp, "a written float depth target refused clipped clamp bounds");
 }
 
+void DepthBoundsEmulationTests() {
+    auto queue = makeState();
+    queue.context[0x204] = 0;
+    queue.context[0x113] = std::bit_cast<std::uint32_t>(0.5f);
+    queue.context[0x114] = std::bit_cast<std::uint32_t>(0.5f);
+    auto base = AgcDriver::Graphics::DecodeState(queue);
+    base.depth = AgcDriver::Graphics::DepthTarget{0x10000, 0, {64, 64}, VK_FORMAT_D32_SFLOAT_S8_UINT, 0.0f, 0};
+    base.depthTest = base.depthWrite = base.stencilTest = false;
+    base.depthBoundsTest = true;
+    base.depthBias = true;
+    base.depthBiasConstant = 4.0f;
+    const ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    const AgcDriver::Graphics::Context emulating{};
+    const auto emulated = [&](float minBound, float maxBound) {
+        auto state = base;
+        state.minDepthBounds = minBound;
+        state.maxDepthBounds = maxBound;
+        AgcDriver::Graphics::ApplyDepthBoundsEmulation(emulating, state, pixel);
+        Require(!state.depthBoundsTest && state.depthTest && !state.depthWrite && !state.depthBias && state.viewport.x == base.viewport.x && state.viewport.width == base.viewport.width && state.viewport.height == base.viewport.height, "an emulated depth bounds test did not become a read-only depth test over the same coverage");
+        return state;
+    };
+    const auto upper = emulated(0.0f, 0.000125f);
+    Require(upper.depthCompare == VK_COMPARE_OP_GREATER_OR_EQUAL && upper.viewport.minDepth == 0.000125f && upper.viewport.maxDepth == 0.000125f, "an upper depth bound was not tested as the bound at or above the stored depth");
+    const auto lower = emulated(0.000125f, 1.0f);
+    Require(lower.depthCompare == VK_COMPARE_OP_LESS_OR_EQUAL && lower.viewport.minDepth == 0.000125f && lower.viewport.maxDepth == 0.000125f, "a lower depth bound was not tested as the bound at or below the stored depth");
+    const auto whole = emulated(0.0f, 1.0f);
+    Require(whole.depthCompare == VK_COMPARE_OP_GREATER_OR_EQUAL && whole.viewport.minDepth == 1.0f && whole.viewport.maxDepth == 1.0f, "bounds covering [0, 1] were not tested against the upper bound");
+    AgcDriver::Graphics::Context native{};
+    native.depthBounds = true;
+    auto kept = base;
+    AgcDriver::Graphics::ApplyDepthBoundsEmulation(native, kept, pixel);
+    Require(kept.depthBoundsTest && !kept.depthTest && kept.depthBias && kept.viewport.minDepth == base.viewport.minDepth, "a device with the depth bounds test had it emulated");
+    auto absent = base;
+    absent.depth.reset();
+    AgcDriver::Graphics::ApplyDepthBoundsEmulation(emulating, absent, pixel);
+    Require(absent.depthBoundsTest && !absent.depthTest, "a draw without a depth target had its depth bounds emulated");
+    const auto refuses = [&](const char* expected, auto change) {
+        auto state = base;
+        state.minDepthBounds = 0.0f;
+        state.maxDepthBounds = 0.5f;
+        auto context = emulating;
+        auto program = pixel;
+        change(state, context, program);
+        expectFailure([&] { AgcDriver::Graphics::ApplyDepthBoundsEmulation(context, state, program); }, expected);
+    };
+    refuses("both bounds inside", [](auto& state, auto&, auto&) { state.minDepthBounds = 0.25f; });
+    refuses("with depth testing", [](auto& state, auto&, auto&) { state.depthTest = true; });
+    refuses("reads or exports depth", [](auto&, auto&, auto& program) { program.posZ = true; });
+    refuses("reads or exports depth", [](auto&, auto&, auto& program) { program.depthExportEnable = true; });
+    refuses("normalized depth target", [](auto& state, auto&, auto&) { state.depth->format = VK_FORMAT_D16_UNORM; });
+    refuses("stays in [0, 1]", [](auto&, auto& context, auto&) { context.depthRangeUnrestricted = true; });
+    refuses("depth bounds outside [0, 1]", [](auto& state, auto&, auto&) { state.maxDepthBounds = 1.5f; });
+    refuses("writes on a failed test", [](auto& state, auto&, auto&) {
+        state.stencilTest = true;
+        state.stencilFront.writeMask = 0xffu;
+        state.stencilFront.failOp = VK_STENCIL_OP_REPLACE;
+    });
+    auto stencil = base;
+    stencil.stencilTest = true;
+    stencil.stencilFront.writeMask = stencil.stencilBack.writeMask = 0xffu;
+    stencil.stencilFront.passOp = stencil.stencilBack.passOp = VK_STENCIL_OP_REPLACE;
+    stencil.maxDepthBounds = 0.5f;
+    AgcDriver::Graphics::ApplyDepthBoundsEmulation(emulating, stencil, pixel);
+    Require(stencil.depthTest && stencil.stencilTest, "a stencil test that writes only on a pass refused the depth bounds emulation");
+}
+
 void UnusedViewportDepthTests() {
     auto queue = makeState();
     queue.context[0x204] = 0;
@@ -7099,6 +7165,27 @@ void storageTextureBudgetTests() {
     memory.memoryHeaps[0].flags = 0;
     memory.memoryHeaps[2].flags = 0;
     Require(StorageTextureBudget(memory, &reported, 0, 0) == 2 * GiB, "a device without a device-local heap does not keep 2 GiB of storage images");
+    using AgcDriver::Graphics::StorageTextureCeiling;
+    memory.memoryHeaps[0] = {256 * MiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT};
+    memory.memoryHeaps[2] = {16 * GiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT};
+    reported = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
+    Require(StorageTextureCeiling(memory, nullptr) == 4 * GiB, "without VK_EXT_memory_budget the storage image cache ceiling is not a quarter of the 16 GiB heap");
+    reported.heapBudget[2] = 6 * GiB;
+    Require(StorageTextureCeiling(memory, &reported) == 3 * GiB, "the storage image cache ceiling on a 6 GiB budget is not half of the budget, the cap the cache may reach");
+    reported.heapBudget[2] = 40 * GiB;
+    Require(StorageTextureCeiling(memory, &reported) == 8 * GiB, "the storage image cache ceiling on a 40 GiB budget is not half of the 16 GiB heap");
+    reported.heapBudget[2] = 768 * MiB;
+    Require(StorageTextureCeiling(memory, &reported) == 512 * MiB, "the storage image cache ceiling on a 768 MiB budget does not keep the 512 MiB floor");
+    for (std::uint64_t budget = 1 * GiB; budget <= 24 * GiB; budget += 512 * MiB) {
+        reported.heapBudget[2] = budget;
+        const auto ceiling = StorageTextureCeiling(memory, &reported);
+        for (std::uint64_t used = 0; used <= budget; used += 256 * MiB) {
+            reported.heapUsage[2] = used;
+            Require(StorageTextureBudget(memory, &reported, 256 * MiB, 512 * MiB) <= ceiling, "the storage image cache budget exceeds the ceiling idle device buffers are reclaimed up to");
+        }
+        reported.heapUsage[2] = 0;
+        Require(StorageTextureBudget(memory, &reported, 0, 0) == ceiling, "an idle heap does not give the storage image cache its full ceiling");
+    }
 }
 
 void sampledBudgetReportTests() {
@@ -9148,7 +9235,7 @@ void stencilRefExportTests() {
             Require(AgcDriver::Graphics::DrawRejection(unused, false).empty(), "the precheck rejected STENCIL_OP_VAL_EXPORT_ENABLE without a stencil test in DB_SHADER_CONTROL " + std::to_string(shaderControl));
             Require(AgcDriver::Graphics::DecodePixelStageInfo(unused.context, IdentityExports).stencilRefExportEnable == ((shaderControl & 2u) != 0), "STENCIL_OP_VAL_EXPORT_ENABLE without a stencil test exported the stencil reference");
             const bool replaces = (shaderControl & 6u) == 4u;
-            for (const auto [ops, writeMask, compare, accepted] : {std::tuple{0x000u, 0xffu, 7u, true}, std::tuple{0x737u, 0xffu, 7u, true}, std::tuple{0x110u, 0xffu, 7u, true}, std::tuple{0x050u, 0x00u, 7u, true}, std::tuple{0x040u, 0xffu, 7u, replaces}, std::tuple{0x40040u, 0xffu, 7u, replaces}, std::tuple{0x404u, 0x0fu, 0u, replaces}, std::tuple{0x040u, 0xffu, 2u, false}, std::tuple{0x340u, 0xffu, 7u, false}, std::tuple{0x042u, 0xffu, 7u, false}, std::tuple{0x050u, 0xffu, 7u, false}, std::tuple{0x600u, 0xffu, 7u, false}, std::tuple{0x008u, 0xffu, 7u, false}, std::tuple{0x900u, 0xffu, 7u, false}, std::tuple{0x5000u, 0xffu, 7u, false}}) {
+            for (const auto [ops, writeMask, compare, accepted] : {std::tuple{0x000u, 0xffu, 7u, true}, std::tuple{0x737u, 0xffu, 7u, true}, std::tuple{0x110u, 0xffu, 7u, true}, std::tuple{0x050u, 0x00u, 7u, true}, std::tuple{0x040u, 0xffu, 7u, replaces}, std::tuple{0x40040u, 0xffu, 7u, replaces}, std::tuple{0x404u, 0x0fu, 0u, replaces}, std::tuple{0x040u, 0xffu, 2u, false}, std::tuple{0x340u, 0xffu, 7u, false}, std::tuple{0x042u, 0xffu, 7u, false}, std::tuple{0x050u, 0xffu, 7u, false}, std::tuple{0x600u, 0xffu, 7u, false}, std::tuple{0x008u, 0xffu, 7u, false}, std::tuple{0x900u, 0xffu, 7u, false}, std::tuple{0x5000u, 0xffu, 7u, false}, std::tuple{0x0c0u, 0xffu, 7u, false}, std::tuple{0x00bu, 0xffu, 7u, false}}) {
                 auto stenciled = stencilRefExportQueue(shaderControl, zFormat);
                 for (const auto [offset, word] : {std::pair{0x010u, 0x80000180u}, std::pair{0x011u, 0x20000181u}, std::pair{0x012u, 0x00001000u}, std::pair{0x013u, 0x00002000u}, std::pair{0x015u, 0x00002000u}, std::pair{0x007u, 0x003f003fu}, std::pair{0x000u, 0u}, std::pair{0x002u, 0u}, std::pair{0x00au, 0u}, std::pair{0x00bu, 0x3f800000u}, std::pair{0x200u, 0x007007b3u}}) stenciled.context[offset] = word;
                 stenciled.context[0x10b] = ops;
@@ -9458,6 +9545,99 @@ void stencilReferenceUnusedTests() {
     std::cout << "Unused stencil reference sample equivalence passed: " << accepted << " accepted states, " << countermodels << " dependent countermodels\n";
 }
 
+void stencilLogicOperationTests() {
+    const auto queueFor = [](std::uint32_t ops, std::uint32_t front, std::uint32_t back, std::uint32_t depthControl) {
+        auto queue = makeState();
+        for (const auto [offset, word] : {std::pair{0x000u, 0u}, std::pair{0x002u, 0u}, std::pair{0x007u, (1u << 16u) | 3u}, std::pair{0x00au, 0u}, std::pair{0x00bu, 0u}, std::pair{0x010u, 0x22900983u}, std::pair{0x011u, 0x20000181u}, std::pair{0x012u, 0x100u}, std::pair{0x014u, 0x100u}, std::pair{0x013u, 0x200u}, std::pair{0x015u, 0x200u}, std::pair{0x005u, 0x300u}, std::pair{0x01eu, 0x1u}}) queue.context[offset] = word;
+        queue.context[0x10b] = ops;
+        queue.context[0x10c] = front;
+        queue.context[0x10d] = back;
+        queue.context[0x200] = depthControl;
+        return queue;
+    };
+    const auto refMaskOf = [](std::uint32_t test, std::uint32_t writeMask, std::uint32_t value) { return test | 0xff00u | (writeMask << 16u) | (value << 24u); };
+    const auto apply = [](std::uint32_t op, std::uint32_t stencil, std::uint32_t value) {
+        switch (op) {
+            case 1: return 0u;
+            case 10: return stencil & value;
+            case 11: return stencil | value;
+            case 12: return stencil ^ value;
+            case 15: return ~(stencil ^ value);
+            default: return stencil;
+        }
+    };
+    const auto matches = [&](const AgcDriver::Graphics::State& state, const VkStencilOpState& face, std::uint32_t ops, std::uint32_t compare, std::uint32_t refMask, const std::string& name) {
+        const auto test = refMask & 0xffu;
+        const auto compareMask = (refMask >> 8u) & 0xffu;
+        const auto writeMask = (refMask >> 16u) & 0xffu;
+        const auto value = refMask >> 24u;
+        for (std::uint32_t stencil = 0; stencil < 256u; ++stencil) {
+            for (const bool depthPass : {false, true}) {
+                const bool stencilPass = stencilComparison(static_cast<VkCompareOp>(compare), test & compareMask, stencil & compareMask);
+                const auto op = (ops >> (!stencilPass ? 0u : !depthPass ? 8u : 4u)) & 0xfu;
+                const auto expected = ((stencil & ~writeMask) | (apply(op, stencil, value) & writeMask)) & 0xffu;
+                const auto [actual, passed] = stencilReferenceOutcome(state, face, face.reference, stencil, depthPass, true);
+                Require(actual == expected && passed == (stencilPass && depthPass), name + ": stencil " + std::to_string(stencil) + (depthPass ? " passing" : " failing") + " the depth test became " + std::to_string(actual) + ", expected " + std::to_string(expected));
+            }
+        }
+    };
+    std::size_t accepted = 0;
+    for (const auto op : {10u, 11u, 12u, 15u}) {
+        for (const auto placement : {0x010u, 0x001u, 0x100u, 0x111u}) {
+            for (const auto compare : {7u, 0u, 2u, 3u}) {
+                for (const auto [test, writeMask, value] : {std::tuple{0x00u, 0xffu, 0x5au}, std::tuple{0xa5u, 0xffu, 0x21u}, std::tuple{0xa5u, 0x3cu, 0xffu}, std::tuple{0xffu, 0xf0u, 0x0fu}, std::tuple{0x5au, 0x81u, 0x00u}}) {
+                    const auto ops = op * placement;
+                    const auto refMask = refMaskOf(test, writeMask, value);
+                    const auto queue = queueFor(ops, refMask, refMask, 0x13u | (compare << 8u));
+                    const auto name = "stencil operation " + std::to_string(op) + " in DB_STENCIL_CONTROL " + std::to_string(ops) + " with compare " + std::to_string(compare) + " and DB_STENCILREFMASK " + std::to_string(refMask);
+                    const auto bits = (op == 11u || op == 12u ? value : ~value) & writeMask;
+                    if (op == 11u && compare != 0u && compare != 7u && (~test & bits) != 0u) {
+                        expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeState(queue)); }, "stencil OR of");
+                        continue;
+                    }
+                    const auto state = AgcDriver::Graphics::DecodeState(queue);
+                    Require(state.stencilTest && state.depthTest && (state.stencilFront.writeMask & 0xffu) == (bits != 0u ? bits : writeMask), "the write mask is not narrowed to the bits changed by " + name);
+                    matches(state, state.stencilFront, ops, compare, refMask, name);
+                    ++accepted;
+                }
+            }
+        }
+    }
+    for (const auto [ops, refMask, name] : {std::tuple{0x0c1u, refMaskOf(0x80u, 0xffu, 0xffu), "XOR of every written bit beside ZERO"}, std::tuple{0x0cbu, refMaskOf(0x8fu, 0xffu, 0x0fu), "OR and XOR of the same bits"}, std::tuple{0x0fau, refMaskOf(0x80u, 0xffu, 0xf0u), "AND and XNOR of the same bits"}, std::tuple{0xb0bu, refMaskOf(0x30u, 0x3cu, 0xf0u), "OR on stencil fail and depth fail"}}) {
+        const auto queue = queueFor(ops, refMask, refMask, 0x13u | (3u << 8u));
+        const auto state = AgcDriver::Graphics::DecodeState(queue);
+        matches(state, state.stencilFront, ops, 3u, refMask, name);
+        ++accepted;
+    }
+    {
+        const auto front = refMaskOf(0x11u, 0xffu, 0x0fu);
+        const auto back = refMaskOf(0x22u, 0x3cu, 0x30u);
+        const auto queue = queueFor(0x0c0u | (0x0a0u << 12u), front, back, 0x93u | (7u << 8u) | (5u << 20u));
+        const auto state = AgcDriver::Graphics::DecodeState(queue);
+        Require(state.stencilFront.passOp == VK_STENCIL_OP_INVERT && state.stencilFront.writeMask == 0x0fu && state.stencilBack.passOp == VK_STENCIL_OP_ZERO && state.stencilBack.writeMask == 0x0cu, "front XOR and back AND did not keep their own write masks");
+        matches(state, state.stencilFront, 0x0c0u, 7u, front, "front XOR beside a back AND");
+        matches(state, state.stencilBack, 0x0a0u, 5u, back, "back AND beside a front XOR");
+        accepted += 2;
+    }
+    for (const auto [ops, refMask, reason] : {std::tuple{0x0d0u, refMaskOf(0u, 0xffu, 0x0fu), "stencil operation 13 is unsupported"}, std::tuple{0x0e0u, refMaskOf(0u, 0xffu, 0x0fu), "stencil operation 14 is unsupported"}, std::tuple{0x0cau, refMaskOf(0u, 0xffu, 0x0fu), "stencil logic operations on write masks"}, std::tuple{0x0fcu, refMaskOf(0u, 0xffu, 0x0fu), "stencil logic operations on write masks"},std::tuple{0x0c1u, refMaskOf(0u, 0xffu, 0x0fu), "combine a logic operation"}, std::tuple{0x5c0u, refMaskOf(0u, 0xffu, 0x01u), "combine a logic operation"}}) {
+        const auto queue = queueFor(ops, refMask, refMask, 0x13u | (7u << 8u));
+        expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeState(queue)); }, reason);
+    }
+    for (const auto ops : {0x0c0u, 0x0b0u}) {
+        auto exported = stencilRefExportQueue(0x12u, 9u);
+        for (const auto [offset, word] : {std::pair{0x010u, 0x80000180u}, std::pair{0x011u, 0x20000181u}, std::pair{0x012u, 0x00001000u}, std::pair{0x013u, 0x00002000u}, std::pair{0x015u, 0x00002000u}, std::pair{0x007u, 0x003f003fu}, std::pair{0x000u, 0u}, std::pair{0x002u, 0u}, std::pair{0x00au, 0u}, std::pair{0x00bu, 0x3f800000u}, std::pair{0x200u, 0x007007b3u}}) exported.context[offset] = word;
+        exported.context[0x10b] = ops;
+        exported.context[0x10c] = exported.context[0x10d] = refMaskOf(0u, 0xffu, 0x0fu);
+        if (ops == 0x0c0u) {
+            const auto state = AgcDriver::Graphics::DecodeState(exported);
+            Require(state.stencilFront.passOp == VK_STENCIL_OP_INVERT && state.stencilFront.writeMask == 0x0fu, "a stencil test value export changed a stencil XOR");
+        } else {
+            expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeState(exported)); }, "stencil OR is unsupported with a pixel shader that exports the stencil test value");
+        }
+    }
+    std::cout << "Stencil logic operations matched AND, OR, XOR and XNOR under the write mask for " << accepted << " register states\n";
+}
+
 std::set<std::string> bufferViews(std::span<const std::uint32_t> words) {
     std::set<std::string> views;
     for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
@@ -11452,6 +11632,117 @@ void pipelineStoreLazyBufferPoolTests() {
     Require(mock.live == 0, "lazy buffer pool pipeline tests leaked Vulkan objects");
 }
 
+// The pipeline store's bound without a GPU: 1536 configurations cycled through the default 1024
+// entries keep coming back after their eviction, so the bound doubles and a later pass finds every
+// pipeline resident; a bound set with APS5_PIPELINE_STORE_ENTRIES never moves.
+void pipelineStoreGrowthTests(bool fixed) {
+    using namespace AgcDriver::Graphics;
+    mock = MockVulkan{};
+    auto context = mockContext();
+    context.device = makeHandle<VkDevice>();
+    context.graphicsPipelineLibrary = true;
+    context.pipelineLinkTimeOptimization = false;
+    context.limits.framebufferNoAttachmentsSampleCounts = VK_SAMPLE_COUNT_1_BIT;
+    ShaderRecompiler::RecompileResult vertex, fragment;
+    vertex.spirv = makeModule({});
+    fragment.spirv = makeModule({.fragment = true});
+    vertex.variantId = 301;
+    fragment.variantId = 302;
+    const std::array<CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, PipelinePushSlotBytes}}};
+    std::array<std::uint32_t, 1> mainKey{41}, fragmentKey{42};
+    const PipelineSetLayouts resources{makeHandle<VkDescriptorSetLayout>(), makeHandle<VkDescriptorSetLayout>(), true, mainKey, fragmentKey};
+    constexpr std::uint32_t configurations = 1536;
+    const auto pass = [&](std::uint32_t count) {
+        std::uint32_t built = 0;
+        for (std::uint32_t configuration = 0; configuration < count; ++configuration) {
+            VertexInputLayout input;
+            input.bindings.push_back({0, 4u * (configuration + 1u), VK_VERTEX_INPUT_RATE_VERTEX});
+            bool created = false;
+            static_cast<void>(CachedPipeline(context, State{}, input, resources, shaders, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, &created));
+            built += created ? 1u : 0u;
+        }
+        return built;
+    };
+    std::uint32_t lastBuilt = 0;
+    for (int round = 0; round < 6; ++round) lastBuilt = pass(configurations);
+    const auto grown = PipelineStoreTotals();
+    if (fixed) {
+        Require(grown.bound == 1024, "a bound set with APS5_PIPELINE_STORE_ENTRIES changed");
+        Require(lastBuilt == configurations, "a fixed 1024-entry bound kept pipelines of a 1536-configuration cycle");
+    } else {
+        Require(grown.bound == 2048, "the bound did not grow to 2048 while evicted pipelines kept coming back");
+        Require(lastBuilt == 0, "a pass over 1536 configurations still rebuilt pipelines after the bound grew");
+    }
+    ClearCachedPipelines(context.device);
+    Require(mock.live == 0, "pipeline store growth tests leaked Vulkan objects");
+}
+
+// A journal longer than the store's starting bound: 1032 recorded pipelines replayed into an empty
+// store all stay resident (the replay grows the bound instead of dropping what does not fit).
+void pipelineStoreReplayGrowthTests() {
+    using namespace AgcDriver::Graphics;
+    const auto root = std::filesystem::temp_directory_path() / ("aps5-pipeline-store-replay-" + std::to_string(std::random_device{}()));
+    std::filesystem::create_directories(root);
+    struct Cleanup {
+        std::filesystem::path directory;
+        ~Cleanup() {
+            ShaderRecompiler::ShaderDiskCache::Shutdown();
+            std::error_code error;
+            std::filesystem::remove_all(directory, error);
+        }
+    } cleanup{root};
+#ifdef _WIN32
+    _putenv_s("ANYPS5_NO_SHADER_CACHE", "0");
+    _wputenv_s(L"ANYPS5_SHADER_CACHE_DIR", root.c_str());
+    _putenv_s("APS5_PIPELINE_JOURNAL_THREADS", "1");
+#else
+    setenv("ANYPS5_NO_SHADER_CACHE", "0", 1);
+    setenv("ANYPS5_SHADER_CACHE_DIR", root.string().c_str(), 1);
+    setenv("APS5_PIPELINE_JOURNAL_THREADS", "1", 1);
+#endif
+    Require(PipelineJournalEnabled(), "the journal is disabled with a shader cache directory set");
+    mock = MockVulkan{};
+    auto context = mockContext();
+    context.device = makeHandle<VkDevice>();
+    context.graphicsPipelineLibrary = true;
+    context.pipelineLinkTimeOptimization = false;
+    context.limits.framebufferNoAttachmentsSampleCounts = VK_SAMPLE_COUNT_1_BIT;
+    ShaderRecompiler::RecompileResult vertex, fragment;
+    vertex.spirv = makeModule({});
+    fragment.spirv = makeModule({.fragment = true});
+    vertex.variantId = 401;
+    fragment.variantId = 402;
+    const std::array<CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, PipelinePushSlotBytes}}};
+    auto cache = std::make_unique<DescriptorCache>(context);
+    context.descriptorCache = cache.get();
+    const auto layouts = DescribeSetLayouts(context, shaders);
+    const auto resources = layouts.View();
+    constexpr std::uint64_t fingerprint = 0x5eed'0000'0000'0002ull;
+    {
+        auto absent = StartPipelineJournalReplay(context, fingerprint, {});
+        Require(absent != nullptr, "the first replay did not start");
+        absent->Wait();
+    }
+    constexpr std::uint32_t configurations = 1032;
+    for (std::uint32_t configuration = 0; configuration < configurations; ++configuration) {
+        VertexInputLayout input;
+        input.bindings.push_back({0, 4u * (configuration + 1u), VK_VERTEX_INPUT_RATE_VERTEX});
+        static_cast<void>(CachedPipeline(context, State{}, input, resources, shaders));
+    }
+    FlushPipelineJournal();
+    ClearCachedPipelines(context.device);
+    Require(PipelineStoreTotals().cached == 0, "clearing the device left pipelines in the store");
+    auto replay = StartPipelineJournalReplay(context, fingerprint, {});
+    Require(replay != nullptr, "the replay did not start");
+    replay->Wait();
+    const auto totals = PipelineStoreTotals();
+    Require(totals.cached == configurations, "a replay of " + std::to_string(configurations) + " recorded pipelines kept " + std::to_string(totals.cached) + " in the store (bound " + std::to_string(totals.bound) + ")");
+    ClearCachedPipelines(context.device);
+    context.descriptorCache = nullptr;
+    cache.reset();
+    Require(mock.live == 0, "pipeline store replay growth tests leaked Vulkan objects");
+}
+
 void pipelineJournalTests(bool longPath = false, bool extended = false) {
     using namespace AgcDriver::Graphics;
     using Stage = ShaderRecompiler::ShaderStage;
@@ -11883,6 +12174,16 @@ int main(int argc, char** argv) {
             std::cout << "pipeline library clear waits for active part builds\n";
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--pipeline-store-replay-growth") {
+            pipelineStoreReplayGrowthTests();
+            std::cout << "a replayed journal longer than the store bound stays resident\n";
+            return 0;
+        }
+        if (argc == 2 && std::string_view(argv[1]) == "--pipeline-store-growth") {
+            pipelineStoreGrowthTests(std::getenv("APS5_PIPELINE_STORE_ENTRIES") != nullptr);
+            std::cout << "pipeline store bound follows the evicted pipelines that come back\n";
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--pipeline-store-lazy-buffer-pool") {
             pipelineStoreLazyBufferPoolTests();
             std::cout << "cached pipelines survive lazy buffer pool creation\n";
@@ -12152,6 +12453,7 @@ int main(int argc, char** argv) {
         srgb8TargetTests();
         DepthClipTests();
         DepthRemapTests();
+        DepthBoundsEmulationTests();
         UnusedViewportDepthTests();
         DepthStencilTests();
         OneDimensionalColorTests();
@@ -12168,6 +12470,7 @@ int main(int argc, char** argv) {
         orderedPixelShaderTests();
         stencilRefExportTests();
         stencilReferenceUnusedTests();
+        stencilLogicOperationTests();
         aliasedBufferViewTests();
         LineStripTests();
         ShaderModuleKeyTests();

core/libs/prx/libSceAgcDriver/tests/GuardedMemoryLoads.cpp

@@ -32,7 +32,7 @@ struct LoadCase {
     std::uint32_t components = 1;
 };
 
-std::vector<std::uint32_t> emitLoad(const LoadCase& test, bool active, std::uint32_t offset, std::uint32_t records, bool robust) {
+std::vector<std::uint32_t> emitLoad(const LoadCase& test, bool active, std::uint32_t offset, std::uint32_t records, bool robust, bool valid = true) {
     IrProgram program;
     program.SetWaveSize(32);
     program.Resources().stage = IrShaderStage::Compute;
@@ -76,7 +76,8 @@ std::vector<std::uint32_t> emitLoad(const LoadCase& test, bool active, std::uint
         program.Metadata().bindings.pushDataStartDword = 0;
         program.Metadata().bindings.memoryOffsetCount = 1;
         auto& size = constant(records);
-        auto& handle = emit(IrOpcode::GetBufferResource, IrType::BufferResource, {&zero, &zero, &size, &zero});
+        auto& base = constant(valid ? 0x100u : 0u);
+        auto& handle = emit(IrOpcode::GetBufferResource, IrType::BufferResource, {&base, &zero, &size, &zero});
         emit(test.opcode, test.type, {&handle, &zero, &address, &zero, &enabled});
     } else {
         if (test.kind == ResourceKind::Gds) program.Metadata().bindings.descriptors.push_back({DescriptorBindingKind::Gds, {0}});
@@ -104,8 +105,8 @@ std::vector<std::uint32_t> emitLoad(const LoadCase& test, bool active, std::uint
 
 class AccessTrace {
 public:
-    AccessTrace(const std::vector<std::uint32_t>& module, std::uint32_t misalignment = 0, bool valid = true)
-        : words(module), misalignment(misalignment), valid(valid) {
+    AccessTrace(const std::vector<std::uint32_t>& module, std::uint32_t misalignment = 0)
+        : words(module), misalignment(misalignment) {
         require(words.size() >= 5, "SPIR-V module header is missing");
         for (std::size_t offset = 5; offset < words.size();) {
             const auto count = words[offset] >> 16;
@@ -206,7 +207,6 @@ private:
             const auto spec = specializations.at(id);
             if (spec == PipelineSpecialization::BufferBase) return 0;
             if (spec == PipelineSpecialization::BufferBase + 1) return 0x30001000;
-            if (spec == PipelineSpecialization::BufferBase + 2) return valid;
             if (spec == PipelineSpecialization::BufferBase + 3) return misalignment;
             if (spec == PipelineSpecialization::PushDataOffset) return 0;
             throw std::runtime_error("unexpected specialization in access fixture");
@@ -250,7 +250,6 @@ private:
 
     const std::vector<std::uint32_t>& words;
     std::uint32_t misalignment;
-    bool valid;
     std::uint32_t entryFunction = 0;
     std::map<std::uint32_t, std::span<const std::uint32_t>> definitions;
     std::map<std::uint32_t, std::string> names;
@@ -281,8 +280,8 @@ std::uint32_t ldsLength(const std::vector<std::uint32_t>& words) {
 }
 
 void expectReads(const LoadCase& test, bool robust, bool active, std::uint32_t offset, std::vector<std::uint32_t> expected, std::uint32_t records = 64, std::uint32_t misalignment = 0, bool valid = true) {
-    const auto module = emitLoad(test, active, offset, records, robust);
-    const auto actual = AccessTrace(module, misalignment, valid).Run();
+    const auto module = emitLoad(test, active, offset, records, robust, valid);
+    const auto actual = AccessTrace(module, misalignment).Run();
     if (actual != expected) {
         std::string detail = std::string(robust ? "robust" : "plain") + " guest reads:";
         for (const auto index : actual) detail += " " + std::to_string(index);

core/libs/prx/libSceAgcDriver/tests/PresentationPass.cpp

@@ -568,6 +568,64 @@ void residentPresentTests(const Context& context) {
     blitLetterboxMatchesClear(context);
 }
 
+PFN_vkGetDeviceProcAddr realDeviceProc = nullptr;
+PFN_vkAllocateMemory realAllocateMemory = nullptr;
+const VkPhysicalDeviceMemoryProperties* refusingMemory = nullptr;
+std::uint32_t refusedAllocations = 0;
+std::uint32_t lastAllocatedType = UINT32_MAX;
+
+VKAPI_ATTR VkResult VKAPI_CALL refuseDeviceLocalAllocation(VkDevice device, const VkMemoryAllocateInfo* info, const VkAllocationCallbacks* callbacks, VkDeviceMemory* memory) {
+    const auto heap = refusingMemory->memoryTypes[info->memoryTypeIndex].heapIndex;
+    if ((refusingMemory->memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0) {
+        ++refusedAllocations;
+        return VK_ERROR_OUT_OF_DEVICE_MEMORY;
+    }
+    lastAllocatedType = info->memoryTypeIndex;
+    return realAllocateMemory(device, info, callbacks, memory);
+}
+
+VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL refusingDeviceProc(VkDevice device, const char* name) {
+    if (std::strcmp(name, "vkAllocateMemory") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&refuseDeviceLocalAllocation);
+    return realDeviceProc(device, name);
+}
+
+void sourceImageOutsideDeviceMemoryTests(const Context& base) {
+    VkImageCreateInfo probeInfo{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
+    probeInfo.imageType = VK_IMAGE_TYPE_2D;
+    probeInfo.format = VK_FORMAT_B8G8R8A8_UNORM;
+    probeInfo.extent = {64u, 64u, 1u};
+    probeInfo.mipLevels = 1;
+    probeInfo.arrayLayers = 1;
+    probeInfo.samples = VK_SAMPLE_COUNT_1_BIT;
+    probeInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
+    probeInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
+    VkImage probe = VK_NULL_HANDLE;
+    Check(base.Function<PFN_vkCreateImage>("vkCreateImage")(base.device, &probeInfo, nullptr, &probe), "vkCreateImage probe");
+    VkMemoryRequirements requirements{};
+    base.Function<PFN_vkGetImageMemoryRequirements>("vkGetImageMemoryRequirements")(base.device, probe, &requirements);
+    base.Function<PFN_vkDestroyImage>("vkDestroyImage")(base.device, probe, nullptr);
+    if (!base.HostMemoryType(requirements.memoryTypeBits)) {
+        std::printf("presentation source in host memory: skipped, the device offers no host memory type for the image\n");
+        return;
+    }
+    auto context = base;
+    context.bufferPool.reset();
+    realDeviceProc = base.deviceProc;
+    realAllocateMemory = base.Function<PFN_vkAllocateMemory>("vkAllocateMemory");
+    refusingMemory = &context.memory;
+    refusedAllocations = 0;
+    lastAllocatedType = UINT32_MAX;
+    context.deviceProc = refusingDeviceProc;
+    {
+        PresentationScaler scaler(context, VK_FORMAT_B8G8R8A8_UNORM, VK_FORMAT_B8G8R8A8_UNORM);
+        scaler.EnsureSourceImage(64u, 64u);
+        Require(refusedAllocations != 0, "the presentation source never asked for device-local memory");
+        Require(lastAllocatedType != UINT32_MAX && (context.memory.memoryHeaps[context.memory.memoryTypes[lastAllocatedType].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0, "the presentation source was not placed in host memory when device-local memory was refused");
+        Require(scaler.SourceWidth() == 64u && scaler.SourceHeight() == 64u, "the presentation source in host memory lost its extent");
+    }
+    context.bufferPool.reset();
+}
+
 }
 
 int main() {
@@ -582,6 +640,7 @@ int main() {
             return VulkanTestSkipped;
         }
         residentPresentTests(device->GetContext());
+        sourceImageOutsideDeviceMemoryTests(device->GetContext());
         std::cout << "resident present format, presentation pass and letterbox tests passed\n";
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/Recorder.cpp

@@ -50,6 +50,7 @@
 #endif
 #include <io.h>
 #include <windows.h>
+#include <psapi.h>
 #else
 #include <fcntl.h>
 #include <sys/mman.h>
@@ -67,6 +68,7 @@
 #include <cstring>
 #include <filesystem>
 #include <fstream>
+#include <iomanip>
 #include <future>
 #include <iostream>
 #include <memory>
@@ -322,6 +324,7 @@ public:
                 }
                 return false;
             };
+            memoryBudget = hasExtension(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME);
             auto bytes = AgcDriver::QueryBdaByteFeatures(context.physical, function<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2"), available);
             auto address = AgcDriver::QueryBdaFeatures(context.physical, function<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2"), available);
             const auto queues = function<PFN_vkGetPhysicalDeviceQueueFamilyProperties>("vkGetPhysicalDeviceQueueFamilyProperties");
@@ -516,6 +519,21 @@ public:
         return properties;
     }
     void WaitQueue() const { Check(context.Function<PFN_vkQueueWaitIdle>("vkQueueWaitIdle")(context.queue), "vkQueueWaitIdle"); }
+    struct HeapUsage {
+        std::uint64_t deviceLocal = 0;
+        std::uint64_t host = 0;
+    };
+    std::optional<HeapUsage> MemoryUsage() const {
+        if (!memoryBudget) return std::nullopt;
+        VkPhysicalDeviceMemoryBudgetPropertiesEXT budget{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
+        VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, &budget};
+        function<PFN_vkGetPhysicalDeviceMemoryProperties2>("vkGetPhysicalDeviceMemoryProperties2")(context.physical, &properties);
+        HeapUsage usage;
+        for (std::uint32_t heap = 0; heap < properties.memoryProperties.memoryHeapCount; ++heap) {
+            ((properties.memoryProperties.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0 ? usage.deviceLocal : usage.host) += budget.heapUsage[heap];
+        }
+        return usage;
+    }
 
 private:
     template<typename TFunction>
@@ -545,6 +563,7 @@ private:
     bool graphics = false;
     std::mutex settleMutex;
     bool nullDescriptors = false;
+    bool memoryBudget = false;
 };
 
 using Kind = Recorder::ReadKind;
@@ -10112,6 +10131,98 @@ bool vertexTargetRangeTests(const Device& device, Recorder& recorder, bool impor
     return true;
 }
 
+std::uint64_t processPrivateBytes() {
+#ifdef _WIN32
+    PROCESS_MEMORY_COUNTERS_EX counters{};
+    counters.cb = sizeof(counters);
+    if (!K32GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PROCESS_MEMORY_COUNTERS*>(&counters), sizeof(counters))) return 0;
+    return counters.PrivateUsage;
+#else
+    std::ifstream status("/proc/self/status");
+    std::string line;
+    while (std::getline(status, line)) {
+        if (line.rfind("RssAnon:", 0) == 0) return std::stoull(line.substr(8)) * 1024u;
+    }
+    return 0;
+#endif
+}
+
+// What one graphics pipeline the store keeps costs: `count` pipelines, each with its own fragment
+// module (so its own fragment shader library), built through CachedPipeline and held, after their
+// optimized relinks; process private bytes and VK_EXT_memory_budget heap usage before and after,
+// and after the store is cleared.
+int pipelineStoreMemoryBench(const Device& device, std::uint32_t count) {
+    if (!device.Graphics() || count == 0) {
+        std::cout << "skipped, the pipeline store memory bench requires a graphics queue and at least one pipeline\n";
+        return 77;
+    }
+    std::lock_guard gpu(GpuMutex());
+    const auto& context = device.GetContext();
+    struct Cleanup {
+        VkDevice device;
+        ~Cleanup() { ClearCachedPipelines(device); }
+    } cleanup{context.device};
+    ClearCachedPipelines(context.device);
+    ShaderRecompiler::RecompileResult vertex;
+    vertex.variantId = 21;
+    vertex.spirv = std::vector<std::uint32_t>(std::begin(TRIANGLE_vert_SPV), std::end(TRIANGLE_vert_SPV));
+    std::vector<ShaderRecompiler::RecompileResult> fragments(count + 1u);
+    for (std::uint32_t index = 0; index <= count; ++index) {
+        fragments[index].variantId = 1000u + index;
+        fragments[index].spirv = std::vector<std::uint32_t>(std::begin(TRIANGLE_frag_SPV), std::end(TRIANGLE_frag_SPV));
+        fragments[index].spirv[2] = 0x50535430u + index;
+    }
+    State state{};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.hasColorTarget = true;
+    state.color.format = VK_FORMAT_R8G8B8A8_UNORM;
+    state.colors.push_back(state.color);
+    state.blend.colorWriteMask = 15;
+    state.blends.push_back(state.blend);
+    const std::array first{CompiledShader{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, CompiledShader{ShaderRecompiler::ShaderStage::Fragment, &fragments[count], 0}};
+    ShaderResources resources(context, first, state.color, static_cast<std::uint32_t>(state.colors.size()), 0, 0);
+    const VertexInputLayout input;
+    std::vector<std::shared_ptr<Pipeline>> held;
+    held.reserve(count + 1u);
+    const auto build = [&](std::uint32_t index) {
+        const std::array shaders{CompiledShader{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, CompiledShader{ShaderRecompiler::ShaderStage::Fragment, &fragments[index], 0}};
+        held.push_back(CachedPipeline(context, state, input, resources, shaders, VK_IMAGE_LAYOUT_GENERAL));
+    };
+    struct Sample {
+        std::uint64_t privateBytes;
+        std::optional<Device::HeapUsage> heaps;
+    };
+    const auto sample = [&] {
+        WaitForOptimizedPipelines(context.device);
+        return Sample{processPrivateBytes(), device.MemoryUsage()};
+    };
+    build(count);
+    const auto before = sample();
+    const auto started = std::chrono::steady_clock::now();
+    for (std::uint32_t index = 0; index < count; ++index) build(index);
+    const auto built = std::chrono::steady_clock::now();
+    const auto after = sample();
+    const auto linked = std::chrono::steady_clock::now();
+    const auto totals = PipelineStoreTotals();
+    held.clear();
+    ClearCachedPipelines(context.device);
+    const auto cleared = sample();
+    const auto mib = [](std::uint64_t later, std::uint64_t earlier) { return (static_cast<double>(later) - static_cast<double>(earlier)) / (1024.0 * 1024.0); };
+    const auto perPipeline = [&](std::uint64_t later, std::uint64_t earlier) { return (static_cast<double>(later) - static_cast<double>(earlier)) / 1024.0 / count; };
+    const char* linking = !context.graphicsPipelineLibrary ? "whole pipelines" : context.pipelineLinkTimeOptimization ? "libraries and optimized relinks" : "libraries, fast links only";
+    std::cout << std::fixed << std::setprecision(1);
+    std::cout << "Pipeline store memory: " << count << " graphics pipelines (one fragment module each, " << linking << "), " << totals.cached << " in the store, built in "
+              << std::chrono::duration<double>(built - started).count() << " s, relinked in " << std::chrono::duration<double>(linked - built).count() << " s\n";
+    std::cout << "  process private bytes: +" << mib(after.privateBytes, before.privateBytes) << " MiB = " << perPipeline(after.privateBytes, before.privateBytes) << " KiB per pipeline; after clearing the store " << mib(cleared.privateBytes, before.privateBytes) << " MiB above the start\n";
+    if (before.heaps && after.heaps && cleared.heaps) {
+        std::cout << "  VK_EXT_memory_budget heap usage: device-local +" << mib(after.heaps->deviceLocal, before.heaps->deviceLocal) << " MiB = " << perPipeline(after.heaps->deviceLocal, before.heaps->deviceLocal) << " KiB per pipeline, host +" << mib(after.heaps->host, before.heaps->host) << " MiB = " << perPipeline(after.heaps->host, before.heaps->host)
+                  << " KiB per pipeline; after clearing the store device-local " << mib(cleared.heaps->deviceLocal, before.heaps->deviceLocal) << " MiB, host " << mib(cleared.heaps->host, before.heaps->host) << " MiB above the start\n";
+    } else {
+        std::cout << "  VK_EXT_memory_budget unavailable: no device heap usage\n";
+    }
+    return 0;
+}
+
 void pipelineCacheTests(const Device& device, bool benchmark) {
     std::thread establishThreadedProcess([] {});
     establishThreadedProcess.join();
@@ -10947,6 +11058,97 @@ void denormalClearTests(const Device& device, Recorder& recorder) {
     recorder.Sync();
 }
 
+class LibraryCreateFlags {
+public:
+    explicit LibraryCreateFlags(const Context& context) : resolver(context.deviceProc), device(context.device) {
+        create = function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines");
+        active = this;
+    }
+    ~LibraryCreateFlags() { active = nullptr; }
+    LibraryCreateFlags(const LibraryCreateFlags&) = delete;
+    LibraryCreateFlags& operator=(const LibraryCreateFlags&) = delete;
+    PFN_vkGetDeviceProcAddr Resolver() const { return resolve; }
+    std::uint32_t Libraries() const { return libraries.load(); }
+    std::uint32_t Retaining() const { return retaining.load(); }
+
+private:
+    template<typename TFunction>
+    TFunction function(const char* name) const { return reinterpret_cast<TFunction>(resolver(device, name)); }
+    static VKAPI_ATTR VkResult VKAPI_CALL createPipelines(VkDevice device, VkPipelineCache cache, std::uint32_t count, const VkGraphicsPipelineCreateInfo* infos, const VkAllocationCallbacks* callbacks, VkPipeline* pipelines) {
+        const auto result = active->create(device, cache, count, infos, callbacks, pipelines);
+        for (std::uint32_t index = 0; index < count; ++index) {
+            if (pipelines[index] == VK_NULL_HANDLE || (infos[index].flags & VK_PIPELINE_CREATE_LIBRARY_BIT_KHR) == 0) continue;
+            active->libraries.fetch_add(1);
+            if ((infos[index].flags & VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT) != 0) active->retaining.fetch_add(1);
+        }
+        return result;
+    }
+    static VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL resolve(VkDevice device, const char* name) {
+        if (std::strcmp(name, "vkCreateGraphicsPipelines") == 0) return reinterpret_cast<PFN_vkVoidFunction>(createPipelines);
+        return active->resolver(device, name);
+    }
+    PFN_vkGetDeviceProcAddr resolver;
+    VkDevice device;
+    PFN_vkCreateGraphicsPipelines create = nullptr;
+    std::atomic<std::uint32_t> libraries{0};
+    std::atomic<std::uint32_t> retaining{0};
+    static inline LibraryCreateFlags* active = nullptr;
+};
+
+void pipelineLibraryLtoInfoTests(const Device& device) {
+    std::lock_guard gpu(GpuMutex());
+    const auto& context = device.GetContext();
+    LibraryCreateFlags created(context);
+    struct Cleanup {
+        VkDevice device;
+        ~Cleanup() { ClearCachedPipelines(device); }
+    } cleanup{context.device};
+    ClearCachedPipelines(context.device);
+    if (!context.graphicsPipelineLibrary) {
+        std::cout << "graphics pipeline libraries unavailable or disabled: no library retains link-time optimization info\n";
+        return;
+    }
+    ShaderRecompiler::RecompileResult vertex, fragment;
+    vertex.variantId = 41;
+    fragment.variantId = 42;
+    vertex.spirv = std::vector<std::uint32_t>(std::begin(TRIANGLE_vert_SPV), std::end(TRIANGLE_vert_SPV));
+    fragment.spirv = std::vector<std::uint32_t>(std::begin(TRIANGLE_frag_SPV), std::end(TRIANGLE_frag_SPV));
+    const std::array shaders{CompiledShader{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, CompiledShader{ShaderRecompiler::ShaderStage::Fragment, &fragment, 0}};
+    State state{};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.hasColorTarget = true;
+    state.color.format = VK_FORMAT_R8G8B8A8_UNORM;
+    state.colors.push_back(state.color);
+    state.blend.colorWriteMask = 15;
+    state.blends.push_back(state.blend);
+    ShaderResources resources(context, shaders, state.color, static_cast<std::uint32_t>(state.colors.size()), 0, 0);
+    VertexInputLayout input;
+    auto fast = context;
+    fast.deviceProc = created.Resolver();
+    fast.pipelineLinkTimeOptimization = false;
+    auto optimizing = fast;
+    optimizing.pipelineLinkTimeOptimization = true;
+    const bool lto = std::getenv("APS5_NO_PIPELINE_LTO") == nullptr;
+    const auto fastPipeline = CachedPipeline(fast, state, input, resources, shaders, VK_IMAGE_LAYOUT_GENERAL);
+    const auto fastLibraries = created.Libraries();
+    Require(fastLibraries != 0, "the pipeline was not linked from libraries created through the traced device");
+    Require(created.Retaining() == 0, std::to_string(created.Retaining()) + " of the " + std::to_string(fastLibraries) + " libraries made for a context without optimized links were created to retain link-time optimization info");
+    WaitForOptimizedPipelines(context.device);
+    Require(!fastPipeline->Optimized(), "a pipeline of a context without optimized links was optimized in the background");
+    const auto optimizedPipeline = CachedPipeline(optimizing, state, input, resources, shaders, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+    Require(optimizedPipeline != fastPipeline, "the two attachment layouts shared one cached pipeline");
+    const auto libraries = created.Libraries() - fastLibraries;
+    if (lto) {
+        Require(libraries == fastLibraries, "a link that will be optimized made " + std::to_string(libraries) + " libraries, expected " + std::to_string(fastLibraries) + ": it reused libraries made without link-time optimization info");
+        Require(created.Retaining() == libraries, std::to_string(created.Retaining()) + " of the " + std::to_string(libraries) + " libraries of a link that will be optimized were created to retain link-time optimization info");
+    } else {
+        Require(libraries == 0 && created.Retaining() == 0, "with APS5_NO_PIPELINE_LTO set, forcing pipelineLinkTimeOptimization changed the libraries or their create flags");
+    }
+    WaitForOptimizedPipelines(context.device);
+    Require(optimizedPipeline->Optimized() == lto, lto ? "the pipeline linked from libraries that retain link-time optimization info was not optimized in the background" : "a pipeline was optimized with APS5_NO_PIPELINE_LTO set");
+    std::cout << "Pipeline library link-time optimization info tests passed: " << fastLibraries << " libraries without the info, " << created.Retaining() << " with it\n";
+}
+
 void pipelineLibraryTests(const Device& device) {
     std::lock_guard gpu(GpuMutex());
     const auto& context = device.GetContext();
@@ -11327,10 +11529,15 @@ int main(int argc, char** argv) {
             }
             return sparseRunImportTests(device, true) ? 0 : 77;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--pipeline-library-lto-info-only") {
+            pipelineLibraryLtoInfoTests(device);
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--pipeline-library-only") {
             pipelineLibraryTests(device);
             return 0;
         }
+        if ((argc == 2 || argc == 3) && std::string_view(argv[1]) == "--benchmark-pipeline-store-memory") return pipelineStoreMemoryBench(device, argc == 3 ? static_cast<std::uint32_t>(std::stoul(argv[2])) : 4096u);
         if (argc == 2 && (std::string_view(argv[1]) == "--benchmark-pipeline-cache" || std::string_view(argv[1]) == "--pipeline-cache-only")) {
             pipelineCacheTests(device, std::string_view(argv[1]) == "--benchmark-pipeline-cache");
             return 0;

core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp

@@ -6833,6 +6833,12 @@ void verifyUnnormalizedSamplers() {
     const auto adjusted = recompile(reservedDword3(0xf0800f09u), userData(normalized));
     require(proven(adjusted, false), "sampler dword 3: an image_sample_a whose S# dword 3 holds only a ballot in its reserved bits was not compiled");
     expectFailure([&] { static_cast<void>(recompile(reservedDword3(0xf0800f08u), userData(normalized))); }, "GetSamplerResource dword 3 is not a valid runtime value", "sampler dword 3: an image_sample whose S# dword 3 holds a ballot was accepted");
+    for (const auto& [adjust, plain, compiled, accepted] : {
+             std::tuple{0xf1200109u, 0xf1200108u, "sampler dword 3: an image_gather4_c_a whose S# dword 3 holds only a ballot in its reserved bits was not compiled", "sampler dword 3: an image_gather4_c with a computed S# dword 3 was accepted"},
+             std::tuple{0xf1600109u, 0xf1600108u, "sampler dword 3: an image_gather4_c_o_a whose S# dword 3 holds only a ballot in its reserved bits was not compiled", "sampler dword 3: an image_gather4_c_o with a computed S# dword 3 was accepted"}}) {
+        require(proven(recompile(reservedDword3(adjust), imageData(normalized, Type2D, Format32Float, 0u)), false), compiled);
+        expectFailure([&] { static_cast<void>(recompile(reservedDword3(plain), imageData(normalized, Type2D, Format32Float, 0u))); }, "GetSamplerResource dword 3 is not a valid runtime value", accepted);
+    }
     reject(0xf09c0f18u, imageData(unnormalized, TypeCube, Format8888UNorm, 5u), "unnormalized guest sampler samples a 1D-array, 2D-array, 3D, cube or multisampled image", "unnormalized samplers: a cube T# was accepted");
     reject(0xf09c0f10u, imageData(unnormalized, Type3D, Format8888UNorm, 3u), "unnormalized guest sampler samples a 1D-array, 2D-array, 3D, cube or multisampled image", "unnormalized samplers: a 3D T# was accepted");
     reject(0xf09c0f28u, imageData(unnormalized, Type2DArray, Format8888UNorm, 5u), "unnormalized guest sampler samples a 1D-array, 2D-array, 3D, cube or multisampled image", "unnormalized samplers: a 2D-array sample was accepted");

core/libs/prx/libSceAgcDriver/tests/Submit.cpp

@@ -952,7 +952,8 @@ int main(int argc, char** argv) {
         });
         for (auto& reader : readers) reader.join();
         for (const auto& snapshot : concurrent) check(snapshot == branched, "concurrent raw compute readers lost snapshot reuse");
-        alignas(256) std::array<std::array<std::uint32_t, 64>, 65> programs{};
+        struct alignas(256) RawProgram : std::array<std::uint32_t, 64> {};
+        std::vector<RawProgram> programs(4097);
         for (auto& program : programs) program[0] = 0xbf810000;
         const auto firstAddress = reinterpret_cast<std::uintptr_t>(programs.front().data());
         readers.clear();
@@ -962,6 +963,9 @@ int main(int argc, char** argv) {
         for (auto& reader : readers) reader.join();
         for (const auto& snapshot : concurrent) check(snapshot == concurrent.front(), "concurrent raw compute misses duplicated snapshots");
         const auto evicted = AgcDriver::DriverDetail::ReadRawShader(firstAddress);
+        for (std::size_t i = 1; i <= 400; ++i)
+            AgcDriver::DriverDetail::ReadRawShader(reinterpret_cast<std::uintptr_t>(programs[i].data()));
+        check(AgcDriver::DriverDetail::ReadRawShader(firstAddress) == evicted, "raw shader cache evicted a program while a title used only 401");
         for (std::size_t i = 1; i < programs.size(); ++i)
             AgcDriver::DriverDetail::ReadRawShader(reinterpret_cast<std::uintptr_t>(programs[i].data()));
         check(AgcDriver::DriverDetail::ReadRawShader(firstAddress) != evicted && evicted->code[0] == 0xbf810000,

core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp

@@ -1,4 +1,6 @@
 #include "GraphicsTests.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
@@ -424,26 +426,35 @@ void RunTextureTilingTests() {
         array.depthOrLastArray = 1536;
         array.baseArray = 1536;
         const auto layerBytes = DescribeSurface(array).layerBytes;
-        const auto window = ViewedArrayWindow(array);
+        const auto window = ViewedArrayWindow(array, false);
         Require(window.baseAddress == array.baseAddress + 1536ull * layerBytes && window.depthOrLastArray == 0 && window.baseArray == 0, "a sampled view of one slice at 1536 of an R_X array must start at that slice");
         Require(DescribeSurface(window).layerBytes == layerBytes && DescribeSurface(window).layers == 1, "a sampled array window must keep the slice layout");
 
         auto unaligned = array;
         unaligned.baseArray = 1530;
-        Require(ViewedArrayWindow(unaligned).baseAddress == array.baseAddress, "an R_X window at a slice the swizzle mixes in must keep the whole array");
+        const auto mixed = ViewedArrayWindow(unaligned, false);
+        Require(mixed.baseAddress == array.baseAddress + 1520ull * layerBytes && mixed.baseArray == 10 && mixed.depthOrLastArray == 16, "an R_X view at a slice the swizzle mixes in must start its window at the last slice before it with those bits clear");
+        auto low = array;
+        low.baseArray = 9;
+        Require(ViewedArrayWindow(low, false).baseAddress == array.baseAddress, "an R_X view below the first slice with those bits clear keeps the whole array");
 
         auto plain = array;
         plain.tileMode = TextureTileMode::kStandard64KB;
         plain.baseArray = 1530;
-        Require(ViewedArrayWindow(plain).baseAddress == array.baseAddress + 1530ull * DescribeSurface(plain).layerBytes && ViewedArrayWindow(plain).depthOrLastArray == 6, "a non-XOR window may start at any slice");
+        Require(ViewedArrayWindow(plain, false).baseAddress == array.baseAddress + 1530ull * DescribeSurface(plain).layerBytes && ViewedArrayWindow(plain, false).depthOrLastArray == 6, "a non-XOR window may start at any slice");
 
         auto fromZero = array;
         fromZero.baseArray = 0;
-        Require(ViewedArrayWindow(fromZero).baseAddress == array.baseAddress && ViewedArrayWindow(fromZero).depthOrLastArray == 1536, "a view from slice 0 keeps its descriptor");
+        Require(ViewedArrayWindow(fromZero, false).baseAddress == array.baseAddress && ViewedArrayWindow(fromZero, false).depthOrLastArray == 1536, "a view from slice 0 keeps its descriptor");
 
         auto compressed = array;
         compressed.dccAddress = 0x20000000ull;
-        Require(ViewedArrayWindow(compressed).baseAddress == array.baseAddress, "a view with DCC metadata keeps its descriptor");
+        Require(ViewedArrayWindow(compressed, false).baseAddress == array.baseAddress, "a view with DCC metadata keeps its descriptor");
+        const auto stored = ViewedArrayWindow(compressed, true);
+        Require(stored.baseAddress == window.baseAddress && stored.depthOrLastArray == 0 && stored.baseArray == 0 && stored.dccAddress == 0, "a view whose DCC keys read as stored is cut to its slices without metadata");
+        auto compressedFromZero = compressed;
+        compressedFromZero.baseArray = 0;
+        Require(ViewedArrayWindow(compressedFromZero, true).baseAddress == array.baseAddress && ViewedArrayWindow(compressedFromZero, true).dccAddress == compressed.dccAddress, "a view with DCC metadata from slice 0 keeps its descriptor");
 
         GuestTextureResource cube{};
         cube.baseAddress = 0x30000000ull;
@@ -455,13 +466,45 @@ void RunTextureTilingTests() {
         cube.tileMode = TextureTileMode::kStandard4KB;
         cube.depthOrLastArray = 7232;
         cube.baseArray = 7227;
-        const auto faces = ViewedArrayWindow(cube);
+        const auto faces = ViewedArrayWindow(cube, false);
         Require(faces.baseAddress == cube.baseAddress + 7227ull * DescribeSurface(cube).layerBytes && faces.depthOrLastArray == 5 && faces.baseArray == 0 && faces.dimension == TextureDimension::kCube, "a cube view of six faces at 7227 must start at its first face");
 
+        auto compressedVolume = cube;
+        compressedVolume.dimension = TextureDimension::k3D;
+        compressedVolume.baseArray = 0;
+        compressedVolume.dccAddress = 0x20000000ull;
+        Require(ViewedArrayWindow(compressedVolume, true).dccAddress == compressedVolume.dccAddress, "a volume with DCC metadata keeps its descriptor");
+
         auto volume = cube;
         volume.dimension = TextureDimension::k3D;
         volume.baseArray = 0;
-        Require(ViewedArrayWindow(volume).baseAddress == cube.baseAddress, "a volume keeps its descriptor");
+        Require(ViewedArrayWindow(volume, false).baseAddress == cube.baseAddress, "a volume keeps its descriptor");
+    }
+
+    {
+        const std::array<std::uint32_t, 8> words{0x027c0000u, 0xc1600000u, 0x00ffc0ffu, 0xd1800924u, 0x027cc27cu, 0xc1400000u, 0x00704018u, 0x91b00204u};
+        const auto view = DecodeTextureResource(words);
+        Require(view.baseAddress == 0x27c000000ull && view.dccAddress == 0x91b002040000ull && view.dimension == TextureDimension::k2DArray && view.baseArray == 636 && view.depthOrLastArray == 636 && view.width == 1024 && view.height == 1024 && view.format == 22, "the title's DCC array view decodes to slice 636 of 637");
+        const auto whole = DescribeSurface(view);
+        Require(whole.guestBytes == 2671771648ull, "the uncut DCC array view covers all 637 slices (" + std::to_string(whole.guestBytes) + " bytes)");
+        Require(TextureClearKeys(view, whole.guestBytes) == DccKeys::Uncompressed, "unreadable DCC keys of the uncut view read the texels as stored");
+        Require(ViewedArrayWindow(view, false).baseAddress == view.baseAddress, "a DCC view whose keys are not known to read as stored keeps its descriptor");
+        const auto window = ViewedArrayWindow(view, true);
+        const auto cut = DescribeSurface(window);
+        Require(window.baseAddress == view.baseAddress + whole.GuestLayerOffset(624) && window.dccAddress == 0 && window.baseArray == 12 && window.depthOrLastArray == 12, "the window of a stored-keys DCC view starts at the slice-swizzle boundary below its slice without metadata");
+        Require(cut.layers == 13 && cut.layerBytes == whole.layerBytes && window.baseAddress + cut.guestBytes == view.baseAddress + whole.guestBytes, "the window covers slices 624..636 (" + std::to_string(cut.guestBytes) + " bytes)");
+        for (std::uint32_t slice = 0; slice < cut.layers; ++slice) Require(window.baseAddress + cut.GuestLayerOffset(slice) == view.baseAddress + whole.GuestLayerOffset(624 + slice), "a window slice lies where the uncut surface keeps it");
+        const auto* equation = FindTextureSwizzleEquation(XorSwizzleMode(view.tileMode), 4);
+        const auto block = ThinBlockLayout(view.tileMode, 4);
+        Require(equation != nullptr && whole.mips.size() == 1 && !whole.mips[0].tail, "the title's DCC array is one untailed Z_X level");
+        const auto texel = [&](const GuestTextureResource& surface, const SurfaceGeometry& geometry, std::uint32_t x, std::uint32_t y, std::uint32_t slice) {
+            const auto& mip = geometry.mips[0];
+            return surface.baseAddress + geometry.GuestLayerOffset(slice) + mip.tiledOffset + (static_cast<std::uint64_t>(y / block[2]) * mip.blocksPerRow + x / block[1]) * block[0] + equationOffset(*equation, x, y, slice);
+        };
+        for (std::uint32_t y = 0; y < 1024; y += 7) {
+            for (std::uint32_t x = 0; x < 1024; x += 5) Require(texel(window, cut, x, y, window.baseArray) == texel(view, whole, x, y, view.baseArray), "a texel of the viewed slice detiles from the same guest byte in the window as in the uncut surface");
+        }
+        Require(TextureClearKeys(window, cut.guestBytes) == DccKeys::Uncompressed, "the window reads its texels as stored, as the uncut view does");
     }
 
     {

core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp

@@ -1506,7 +1506,7 @@ void CheckImportWindow() {
         bytes[0x100] = 0x11;
         bytes[Block + 0x2000] = 0x22;
         return true;
-    }), "the import window import failed");
+    }, true, false), "the import window import failed");
     Require(Watched(base, 3 * Block), "an import window left its range unwatched");
     const std::array<std::uint64_t, 3> generations{synced, synced, synced};
     std::array<std::uint8_t, 3> changed{};
@@ -1718,6 +1718,80 @@ void CheckUnwatchedImport() {
 #endif
 }
 
+struct HookCall {
+    std::uint64_t address;
+    std::size_t bytes;
+    bool watched;
+};
+
+std::vector<HookCall> hookCalls;
+std::uint64_t hookProbe = 0;
+std::size_t hookProbeBytes = 0;
+
+void RecordHook(std::uint64_t address, std::size_t bytes) {
+    hookCalls.push_back({address, bytes, Watched(hookProbe, hookProbeBytes)});
+}
+
+struct RecordedHook {
+    RecordedHook() {
+        hookCalls.clear();
+        SetUnwatchHook(&RecordHook);
+    }
+    ~RecordedHook() { SetUnwatchHook(nullptr); }
+    RecordedHook(const RecordedHook&) = delete;
+    RecordedHook& operator=(const RecordedHook&) = delete;
+};
+
+void RequireHookOver(std::uint64_t address, std::size_t bytes, const std::string& path) {
+    Require(hookCalls.size() == 1, path + " did not run the unwatch hook once");
+    const auto& call = hookCalls.front();
+    Require(call.address <= address && address + bytes <= call.address + call.bytes, path + " gave the unwatch hook a range that misses its own");
+    Require(call.watched, path + " ran the unwatch hook after its range left the write watch");
+}
+
+void CheckUnwatchHooks() {
+    void* memory = AllocateWatched(8 * Block);
+    const auto base = reinterpret_cast<std::uint64_t>(memory);
+    std::memset(memory, 0x11, 8 * Block);
+    Require(CollectWritesUncached(base, 8 * Block) != 0, "the hooked block is not collected");
+    const RecordedHook recorded;
+    hookProbe = base;
+    hookProbeBytes = Block;
+    Unwatch(base, Block);
+    RequireHookOver(base, Block, "Unwatch");
+    Require(!Watched(base, Block), "Unwatch left its range watched");
+
+    hookCalls.clear();
+    hookProbe = base + Block;
+    bool hookedBeforeImport = false;
+    Require(ImportWatched(base + Block, Block, [&] {
+        hookedBeforeImport = hookCalls.size() == 1;
+        return true;
+    }, false), "an import that does not stay watched failed");
+    RequireHookOver(base + Block, Block, "an import that does not stay watched");
+    Require(hookedBeforeImport, "an import that does not stay watched ran the unwatch hook after the import");
+    Require(!Watched(base + Block, Block), "an import that does not stay watched left its range watched");
+
+    hookCalls.clear();
+    Require(ImportWatched(base + 2 * Block, Block, [] { return true; }), "an import that stays watched failed");
+    Require(hookCalls.empty() && Watched(base + 2 * Block, Block), "an import whose page walk vouched for its pages ran the unwatch hook or gave up its range");
+
+#ifndef _WIN32
+    hookCalls.clear();
+    const auto late = base + 4 * Block;
+    hookProbe = late;
+    bool hookedBeforeLoss = false;
+    Require(ImportWatched(late, 3 * Block, [&] {
+        GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast<const void*>(late + 2 * Block), Block);
+        hookedBeforeLoss = !hookCalls.empty();
+        return true;
+    }), "an import whose range lost a block during the import failed");
+    RequireHookOver(late, 3 * Block, "an import whose page walk failed after the import");
+    Require(!hookedBeforeLoss, "an import whose page walk failed after the import ran the unwatch hook before it");
+    Require(!Watched(late, Block), "an import whose page walk failed after the import left its range watched");
+#endif
+}
+
 class PartlyCommitted {
 public:
     static constexpr std::size_t Bytes = 4 * Block;
@@ -1948,6 +2022,7 @@ int main() {
         CheckStoreAcrossImports();
         CheckImportWindowFootprint();
         CheckUnwatchedImport();
+        CheckUnwatchHooks();
         CheckCommittedRanges();
         CheckAdoptLatestEpoch();
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/execution/AddressPasses.cpp

@@ -141,6 +141,7 @@ struct Step {
     const std::uint32_t* pixel;
     std::uint64_t pointer;
     std::array<std::uint32_t, 4> descriptor{};
+    std::uint64_t color = 0;
 };
 
 void Submit(const GuestBlock& vertices, const GuestBlock& target, std::initializer_list<Step> steps, bool linear = false) {
@@ -179,6 +180,10 @@ void Submit(const GuestBlock& vertices, const GuestBlock& target, std::initializ
         Append(commands, 0x76, {0x008, static_cast<std::uint32_t>(pixel >> 8u), static_cast<std::uint32_t>(pixel >> 40u)});
         Append(commands, 0x76, {0x00e, static_cast<std::uint32_t>(step.pointer), static_cast<std::uint32_t>(step.pointer >> 32u)});
         Append(commands, 0x76, {0x010, step.descriptor[0], step.descriptor[1], step.descriptor[2], step.descriptor[3]});
+        if (step.color != 0) {
+            Append(commands, 0x69, {0x318, static_cast<std::uint32_t>(step.color >> 8u)});
+            Append(commands, 0x69, {0x390, static_cast<std::uint32_t>(step.color >> 40u)});
+        }
         Append(commands, 0x2d, {3, 2});
     }
     Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
@@ -285,6 +290,26 @@ void TargetReadAsWhenSplit(const GuestBlock& vertices, const GuestBlock& target,
     if (!linear && !split) Expect(after.passesContinued - before.passesContinued == 1, what + ": the reading draw did not continue the rendering draw's pass, so the case does not test it: " + Counts(before, after));
 }
 
+void TargetReadSynchronously(const GuestBlock& vertices, const GuestBlock& target, const GuestBlock& unmapped) {
+    Submit(vertices, target, {{PixelRead.data(), target.At(ColorOffset)}}, true);
+    std::memset(target.Bytes(ColorOffset), 0, ColorBytes);
+    std::memset(unmapped.Bytes(ColorOffset), 0, ColorBytes);
+    const auto before = AddressDraws();
+    const auto otherBefore = OtherDraws();
+    Submit(vertices, target, {{PixelColorWord.data(), 0}, {PixelRead.data(), target.At(ColorOffset), {}, unmapped.At(ColorOffset)}}, true);
+    const auto after = AddressDraws();
+    const auto otherAfter = OtherDraws();
+    const unsigned red = unmapped.Bytes(ColorOffset)[0];
+    std::printf("a synchronous draw reading through the table the linear target the previous draw rendered read %s (red byte %u); recorded: %s, %s\n", red == 255u ? "that draw's color" : "the bytes before it", red, Counts(before, after).c_str(), Counts(otherBefore, otherAfter, "other draws").c_str());
+    if (otherAfter.draws == otherBefore.draws) {
+        std::puts("a synchronous draw reading through the table the linear target the previous draw rendered: skipped, that draw was not recorded because the linear target is not resident on this device, so the target was stored right after it");
+        return;
+    }
+    Expect(otherAfter.draws - otherBefore.draws == 1, "the draw rendering the linear target was not recorded as one draw: " + Counts(otherBefore, otherAfter, "other draws"));
+    Expect(after.draws == before.draws, "the draw reading through the table into an unregistered target was recorded, so the case does not test a synchronous draw: " + Counts(before, after));
+    Expect(red == 255u, "a synchronous draw reading through the table the linear target the previous draw rendered read the bytes before it (red byte " + std::to_string(red) + "), not that draw's color");
+}
+
 }
 
 int main(int argc, char** argv) {
@@ -306,6 +331,7 @@ int main(int argc, char** argv) {
         GuestBlock readOnlyVertices;
         GuestBlock data;
         GuestBlock target;
+        GuestBlock unmapped;
         FillVertices(readOnlyVertices);
         readOnlyVertices.Map(false);
         data.Map(true);
@@ -320,6 +346,7 @@ int main(int argc, char** argv) {
             TargetReadAsWhenSplit(readOnlyVertices, target, false);
             TargetReadAsWhenSplit(readOnlyVertices, target, true);
         }
+        if (only.empty() || only == "--synchronous-target-read") TargetReadSynchronously(readOnlyVertices, target, unmapped);
         AgcDriverShutdown_nid_postfix();
         std::puts("address passes: draws reading through the BDA table share a render pass, and a store through it still begins one");
         return 0;

core/libs/prx/libSceAgcDriver/tests/execution/DepthBoundsEmulationDraw.cpp

@@ -0,0 +1,173 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "Recompiler.hpp"
+#include "DepthFastClearHarness.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <optional>
+#include <span>
+#include <stdexcept>
+#include <vector>
+
+namespace {
+
+namespace Harness = DepthFastClearHarness;
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+struct Bounds {
+    float min;
+    float max;
+};
+
+struct Case {
+    float stored;
+    Bounds bounds;
+    bool covered;
+    const char* what;
+};
+
+alignas(4096) std::array<std::array<float, Harness::Width * Harness::Height>, 16> Depths{};
+alignas(256) std::array<std::array<std::array<float, 4>, 3>, 64> VertexSets{};
+std::size_t vertexSets = 0;
+
+std::uint32_t Draw(AgcDriver::VulkanDevice& device, std::uint64_t depth, float z, std::optional<Bounds> bounds, bool emulate) {
+    Require(vertexSets < VertexSets.size(), "the test ran out of vertex sets");
+    auto& vertices = VertexSets[vertexSets++];
+    vertices = {{{-1.0f, -1.0f, z, 1.0f}, {3.0f, -1.0f, z, 1.0f}, {-1.0f, 3.0f, z, 1.0f}}};
+    Harness::Pixels.fill(std::byte{0});
+
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 32u, 32u, std::nullopt, std::nullopt};
+    state.color = {reinterpret_cast<std::uintptr_t>(Harness::Pixels.data()), {Harness::Width, Harness::Height}, VK_FORMAT_R8G8B8A8_UNORM, Harness::Pixels.size(), 0xe4u};
+    state.colors = {state.color};
+    state.hasColorTarget = true;
+    state.depth = AgcDriver::Graphics::DepthTarget{depth, 0, {Harness::Width, Harness::Height}, VK_FORMAT_D32_SFLOAT, 1.0f, 0};
+    state.depthTest = !bounds.has_value();
+    state.depthWrite = !bounds.has_value();
+    state.depthCompare = VK_COMPARE_OP_ALWAYS;
+    if (bounds) {
+        state.depthBoundsTest = true;
+        state.minDepthBounds = bounds->min;
+        state.maxDepthBounds = bounds->max;
+    }
+    state.renderExtent = {Harness::Width, Harness::Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Harness::Height), static_cast<float>(Harness::Width), -static_cast<float>(Harness::Height), 0.0f, 1.0f};
+    state.negativeOneToOne = false;
+    state.scissor = {{0, 0}, {Harness::Width, Harness::Height}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blend.colorWriteMask = 15;
+    state.blends = {state.blend};
+    state.blendConstants = {};
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.wave32 = true;
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY);
+    pixel.posX = true;
+    pixel.posY = true;
+    pixel.targetOutputMode[0] = 9;
+    pixel.targetExportMapping.fill(0xe4u);
+    if (bounds && emulate) {
+        auto context = device.graphicsContext();
+        context.depthBounds = false;
+        context.depthRangeUnrestricted = false;
+        AgcDriver::Graphics::ApplyDepthBoundsEmulation(context, state, pixel);
+        Require(!state.depthBoundsTest && state.depthTest && !state.depthWrite, "the depth bounds test was not emulated");
+    }
+
+    auto target = device.Target();
+    target.depthRangeRemap = false;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = Harness::BufferDescriptor(vertices.data(), 16u, static_cast<std::uint32_t>(vertices.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(Harness::VertexCode.data()), std::as_bytes(std::span(Harness::VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(Harness::VertexCode.data()), Harness::VertexCode, 0, {}},
+        {32, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+    std::vector<std::uint32_t> pixelUserData(4, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(Harness::PixelCode.data()), std::as_bytes(std::span(Harness::PixelCode))}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(Harness::PixelCode.data()), Harness::PixelCode, 0, {}},
+        {32, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(vertices.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+    device.WaitIdle();
+    std::uint32_t covered = 0;
+    for (std::size_t index = 0; index < Harness::Pixels.size() / 4u; ++index) {
+        if (std::to_integer<std::uint8_t>(Harness::Pixels[index * 4u]) == 255u) ++covered;
+    }
+    return covered;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        const auto context = device->graphicsContext();
+        const float below = std::nextafter(0.25f, 0.0f);
+        const float above = std::nextafter(0.25f, 1.0f);
+        const std::array<Case, 12> cases{{
+            {0.25f, {0.0f, 0.25f}, true, "stored 0.25, upper bound equal to it"},
+            {0.25f, {0.0f, below}, false, "stored 0.25, upper bound one step below it"},
+            {0.25f, {0.25f, 1.0f}, true, "stored 0.25, lower bound equal to it"},
+            {0.25f, {above, 1.0f}, false, "stored 0.25, lower bound one step above it"},
+            {0.000125f, {0.0f, 0.000125f}, true, "stored 0.000125 inside [0, 0.000125]"},
+            {0.5f, {0.0f, 0.000125f}, false, "stored 0.5 outside [0, 0.000125]"},
+            {0.5f, {0.000125f, 1.0f}, true, "stored 0.5 inside [0.000125, 1]"},
+            {0.0f, {0.000125f, 1.0f}, false, "stored 0 outside [0.000125, 1]"},
+            {0.0f, {0.0f, 1.0f}, true, "stored 0 inside [0, 1]"},
+            {1.0f, {0.0f, 1.0f}, true, "stored 1 inside [0, 1]"},
+            {1.0f, {0.0f, 0.5f}, false, "stored 1 outside [0, 0.5]"},
+            {0.75f, {0.5f, 1.0f}, true, "stored 0.75 inside [0.5, 1]"},
+        }};
+        std::size_t compared = 0;
+        for (std::size_t index = 0; index < cases.size(); ++index) {
+            const auto& test = cases[index];
+            const auto depth = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(Depths[index].data()));
+            Require(Draw(*device, depth, test.stored, std::nullopt, false) == Harness::Covered, "the draw that stores the depth did not cover the target");
+            const auto expected = test.covered ? Harness::Covered : 0u;
+            for (const bool emulate : {true, false}) {
+                if (!emulate && !context.depthBounds) continue;
+                const auto passed = Draw(*device, depth, 0.5f, test.bounds, emulate);
+                if (passed != expected) {
+                    char detail[256];
+                    std::snprintf(detail, sizeof(detail), "%s (%s): %u of %u pixels passed, expected %s", test.what, emulate ? "emulated" : "native", passed, Harness::Covered, test.covered ? "all" : "none");
+                    throw std::runtime_error(detail);
+                }
+                if (!emulate) ++compared;
+            }
+        }
+        std::printf("depth bounds emulation draw tests passed (%zu cases, %zu compared with the native test)\n", cases.size(), compared);
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/DepthPlaneGuestLayers.cpp

@@ -223,6 +223,65 @@ private:
     std::uint8_t* block = nullptr;
 };
 
+class SparseLayers {
+public:
+    SparseLayers() {
+#ifdef _WIN32
+        memory = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocateAtOrAbove_nid_postfix(GuestArena::GuestArenaHighestFree_nid_no_patch(BlockBytes, 65536), BlockBytes, 65536));
+#else
+        constexpr std::uintptr_t alignment = 65536;
+        void* raw = mmap(nullptr, BlockBytes + alignment, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        Require(raw != MAP_FAILED, "cannot reserve sparse depth layers");
+        const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+        const auto aligned = (begin + alignment - 1) & ~(alignment - 1);
+        if (aligned != begin) munmap(raw, aligned - begin);
+        if (aligned + BlockBytes != begin + BlockBytes + alignment) munmap(reinterpret_cast<void*>(aligned + BlockBytes), begin + alignment - aligned);
+        memory = reinterpret_cast<std::uint8_t*>(aligned);
+#endif
+        Require(memory != nullptr, "cannot reserve sparse depth layers");
+    }
+
+    ~SparseLayers() {
+#ifdef _WIN32
+        GuestArena::GuestArenaReset_nid_postfix(memory, BlockBytes);
+        GuestAllocations::GuestAllocationsInvalidate_nid_postfix(Address(), BlockBytes);
+        GuestArena::GuestArenaRelease_nid_postfix(memory, BlockBytes);
+#else
+        GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(memory, BlockBytes);
+        munmap(memory, BlockBytes);
+        GuestAllocations::GuestAllocationsInvalidate_nid_postfix(Address(), BlockBytes);
+#endif
+    }
+
+    void Commit(std::size_t offset, std::size_t bytes) {
+#ifdef _WIN32
+        GuestArena::GuestArenaCommit_nid_postfix(memory + offset, bytes, PAGE_READWRITE, 65536);
+#else
+        Require(mmap(memory + offset, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) != MAP_FAILED, "cannot commit sparse depth layer pages");
+        GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(memory + offset, bytes);
+#endif
+        GuestAllocations::GuestAllocationsInvalidate_nid_postfix(Address() + offset, bytes);
+    }
+
+    void DecommitGuestLayer() {
+#ifdef _WIN32
+        GuestArena::GuestArenaReset_nid_postfix(memory + LayerBytes, LayerBytes);
+#else
+        GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(memory + LayerBytes, LayerBytes);
+        Require(mmap(memory + LayerBytes, LayerBytes, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) != MAP_FAILED, "cannot decommit sparse depth layer pages");
+#endif
+        GuestAllocations::GuestAllocationsInvalidate_nid_postfix(Address() + LayerBytes, LayerBytes);
+    }
+
+    std::uint8_t* Data() const { return memory; }
+    std::uint64_t Address() const { return reinterpret_cast<std::uint64_t>(memory); }
+    SparseLayers(const SparseLayers&) = delete;
+    SparseLayers& operator=(const SparseLayers&) = delete;
+
+private:
+    std::uint8_t* memory = nullptr;
+};
+
 std::atomic<std::uint64_t> watchedBegin{0};
 std::atomic<std::uint64_t> watchedEnd{0};
 std::atomic<std::uint32_t> watchedReads{0};
@@ -308,6 +367,59 @@ void Run(const Context& context, const WatchedBlock& block) {
     AgcDriver::GuestMemory::SetFlushHook(nullptr);
 }
 
+void SparseGuestLayers(const Context& context) {
+    ClearDepthSurfaces(context.device);
+    SparseLayers block;
+    block.Commit(0, LayerBytes);
+    FillLayer(block.Data(), 1.0f);
+    const auto address = block.Address();
+    DepthSurfaceView(context, DepthTarget{address, 0, {Side, Side}, VK_FORMAT_D32_SFLOAT, 1.0f, 0, 0});
+    const auto resource = ArrayOver(address);
+    watchedBegin = address + LayerBytes;
+    watchedEnd = address + BlockBytes;
+    AgcDriver::GuestMemory::SetFlushHook(&CountReads);
+    const auto sample = [&](const char* when, bool changed) {
+        AgcDriver::GuestMemory::BumpCollectEpoch();
+        watchedReads = 0;
+        Require(DepthSurfaceTexture(context, words, resource, components) != nullptr, "sparse depth layers were not sampled");
+        Require((watchedReads.load() != 0) == changed, std::string("sparse depth layer read mismatch ") + when);
+    };
+    try {
+        sample("at first use", true);
+        sample("with the guest layer still uncommitted", false);
+        block.Commit(LayerBytes, 65536);
+        const float value = 0.25f;
+        std::memcpy(block.Data() + LayerBytes, &value, sizeof(value));
+        sample("after committing part of the guest layer", true);
+        block.Data()[LayerBytes + 64] = 1;
+        sample("after a CPU write to the committed part", true);
+        AgcDriver::GuestMemory::MarkWritten(address + LayerBytes + 128, 4);
+        sample("after a driver store to the committed part", true);
+        block.DecommitGuestLayer();
+        sample("after decommitting the guest layer", true);
+        sample("with the guest layer still decommitted", false);
+        block.Commit(LayerBytes, 65536);
+        sample("after committing the same pages again", true);
+        block.DecommitGuestLayer();
+        block.Commit(LayerBytes, 65536);
+        sample("after replacing a mapping with the same committed range", true);
+        block.DecommitGuestLayer();
+        block.Commit(LayerBytes, LayerBytes);
+        FillLayer(block.Data() + LayerBytes, 0.5f);
+        sample("after fully committing the guest layer", true);
+        sample("with the fully committed layer unchanged", false);
+        block.DecommitGuestLayer();
+        sample("after decommitting a fully committed guest layer", true);
+        sample("with the fully committed layer now empty", false);
+    } catch (...) {
+        AgcDriver::GuestMemory::SetFlushHook(nullptr);
+        ClearDepthSurfaces(context.device);
+        throw;
+    }
+    AgcDriver::GuestMemory::SetFlushHook(nullptr);
+    ClearDepthSurfaces(context.device);
+}
+
 void RecordedUploads(const Context& context, const WatchedBlock& block) {
     const auto resource = ArrayOver(block.Address());
     Recorder recorder(context);
@@ -418,6 +530,7 @@ int main() {
             RecordedUploads(context, block);
             RunCubeGather(context, block);
             RunCubeTiledSlice(context, block);
+            SparseGuestLayers(context);
         }
         device.reset();
         std::puts("depth plane guest layer tests passed");

core/libs/prx/libSceAgcDriver/tests/execution/DispatchAheadAdoptCheck.cpp

@@ -0,0 +1,208 @@
+#include "VulkanTestDevice.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Submit/include/Acb.hpp"
+#include <array>
+#include <atomic>
+#include <chrono>
+#include <cstdint>
+#include <initializer_list>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace {
+
+void Require(bool condition, const std::string& message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+alignas(256) std::array<std::uint32_t, 7> CounterCode{0xf4080200u, 0xfa000000u, 0xbf8cc07fu, 0x7e000281u, 0xe0c80000u, 0x80020000u, 0xbf810000u};
+alignas(256) std::array<std::uint32_t, 7> StoreCode{0xf4080200u, 0xfa000000u, 0xbf8cc07fu, 0x7e000202u, 0xe0700000u, 0x80020000u, 0xbf810000u};
+alignas(256) std::array<std::uint32_t, 15> SpinCode{0xf4080200u, 0xfa000000u, 0xbe840381u, 0x9304ff04u, 0x0019660du, 0x8004ff04u, 0x3c6ef35fu, 0x80828102u, 0xbf078002u, 0xbf85fff9u, 0xbf8cc07fu, 0x7e000204u, 0xe0700000u, 0x80020000u, 0xbf810000u};
+
+constexpr std::uint32_t Compute = 0x20u;
+constexpr std::uint32_t OtherCompute = 0x28u;
+constexpr std::uint32_t SpinIterations = 1u << 20u;
+
+alignas(256) std::array<std::uint32_t, 4> Stale{};
+alignas(256) std::array<std::uint32_t, 4> Fresh{};
+alignas(256) std::array<std::uint32_t, 4> Target{};
+alignas(256) std::array<std::uint32_t, 4> TargetDescriptor{};
+alignas(256) std::array<std::uint32_t, 4> Spun{};
+alignas(256) std::array<std::uint32_t, 4> SpunDescriptor{};
+alignas(256) volatile std::uint32_t Gate = 1;
+alignas(256) volatile std::uint32_t Done = 0;
+alignas(256) volatile std::uint32_t OtherDone = 0;
+
+enum class Window { None, CpuStore, QueuedGpuStore };
+
+std::atomic<Window> Armed{Window::None};
+std::uint32_t SpinDispatches = 16;
+std::vector<std::uint32_t> OtherWords;
+bool SawPending = false;
+
+std::uint32_t Low(const volatile void* pointer) {
+    return static_cast<std::uint32_t>(reinterpret_cast<std::uintptr_t>(pointer));
+}
+
+std::uint32_t High(const volatile void* pointer) {
+    return static_cast<std::uint32_t>(static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(pointer)) >> 32u);
+}
+
+std::array<std::uint32_t, 4> Descriptor(const void* pointer, std::uint32_t bytes) {
+    return {Low(pointer), High(pointer) & 0xffffu, bytes, 0x31016facu};
+}
+
+void SetShaderRegister(std::vector<std::uint32_t>& words, std::uint32_t offset, std::uint32_t value) {
+    words.insert(words.end(), {0xc0017600u, offset, value});
+}
+
+void Program(std::vector<std::uint32_t>& words, const void* code, std::initializer_list<std::uint32_t> userData) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(code));
+    SetShaderRegister(words, 0x20c, static_cast<std::uint32_t>(address >> 8u));
+    SetShaderRegister(words, 0x20d, static_cast<std::uint32_t>(address >> 40u) & 0xffu);
+    for (const std::uint32_t offset : {0x207u, 0x208u, 0x209u}) SetShaderRegister(words, offset, 1);
+    SetShaderRegister(words, 0x212, 0);
+    SetShaderRegister(words, 0x213, static_cast<std::uint32_t>(userData.size()) << 1u);
+    std::uint32_t offset = 0x240;
+    for (const auto value : userData) SetShaderRegister(words, offset++, value);
+}
+
+void Dispatch(std::vector<std::uint32_t>& words, const void* code, std::initializer_list<std::uint32_t> userData) {
+    Program(words, code, userData);
+    words.insert(words.end(), {0xc0031500u, 1, 1, 1, 0x8041u});
+}
+
+void WriteData(std::vector<std::uint32_t>& words, const volatile std::uint32_t* target, std::uint32_t value) {
+    words.insert(words.end(), {0xc0033700u, 0x00100200u, Low(target), High(target), value});
+}
+
+void Submit(std::uint32_t queue, std::vector<std::uint32_t>& words) {
+    Packet packet{words.data(), static_cast<std::uint32_t>(words.size()), 0, {}};
+    Require(sceAgcDriverSubmitAcb(queue, &packet) == 0, "compute submit failed");
+}
+
+void QueueStoreAhead() {
+    OtherWords.clear();
+    for (std::uint32_t index = 0; index < SpinDispatches; ++index) Dispatch(OtherWords, SpinCode.data(), {Low(SpunDescriptor.data()), High(SpunDescriptor.data()), SpinIterations});
+    Dispatch(OtherWords, StoreCode.data(), {Low(TargetDescriptor.data()), High(TargetDescriptor.data()), Low(Fresh.data())});
+    WriteData(OtherWords, &OtherDone, 1);
+    Submit(OtherCompute, OtherWords);
+    const auto start = std::chrono::steady_clock::now();
+    while (std::chrono::steady_clock::now() - start < std::chrono::seconds(5) && OtherDone == 0) {
+        if (AgcDriver::DriverDetail::Driver::QueryPendingSpan(reinterpret_cast<std::uintptr_t>(Target.data()), sizeof(Target), {}) != AgcDriver::ShaderMemory::PendingWrite::None) {
+            SawPending = true;
+            return;
+        }
+        std::this_thread::yield();
+    }
+}
+
+void BeforeAdoption() {
+    const auto window = Armed.exchange(Window::None);
+    if (window == Window::CpuStore) reinterpret_cast<volatile std::uint32_t&>(Target[0]) = Low(Fresh.data());
+    else if (window == Window::QueuedGpuStore) QueueStoreAhead();
+}
+
+void WaitIdle() {
+    AgcDriverWaitIdle_nid_postfix();
+    for (const auto* buffer : {&Stale, &Fresh, &Target}) AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast<std::uintptr_t>(buffer->data()), sizeof(*buffer));
+}
+
+void Reset() {
+    WaitIdle();
+    Stale = {};
+    Fresh = {};
+    Target = Descriptor(Stale.data(), sizeof(Stale));
+    Done = 0;
+    OtherDone = 0;
+    SawPending = false;
+}
+
+void Gated(Window window) {
+    Reset();
+    std::vector<std::uint32_t> words;
+    words.insert(words.end(), {0xc0053c00u, 0x13u, Low(&Gate), High(&Gate), 1u, 0xffffffffu, 0x19u});
+    Dispatch(words, CounterCode.data(), {Low(Target.data()), High(Target.data())});
+    WriteData(words, &Done, 1);
+    Armed = window;
+    Submit(Compute, words);
+    WaitIdle();
+    Armed = Window::None;
+    Require(Done == 1, "the gated submission did not finish");
+}
+
+std::string Counts() {
+    return "(stale " + std::to_string(Stale[0]) + ", fresh " + std::to_string(Fresh[0]) + ")";
+}
+
+}
+
+int main() {
+    try {
+        auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        device.reset();
+        Require(High(Stale.data()) == High(Fresh.data()), "the stale and fresh buffers differ in their high address word");
+        TargetDescriptor = Descriptor(Target.data(), sizeof(Target));
+        SpunDescriptor = Descriptor(Spun.data(), sizeof(Spun));
+        AgcDriverSetAheadAdoptHook_nid_postfix(&BeforeAdoption);
+        Reset();
+        std::vector<std::uint32_t> warmUp;
+        Dispatch(warmUp, CounterCode.data(), {Low(Target.data()), High(Target.data())});
+        WriteData(warmUp, &Done, 1);
+        Submit(Compute, warmUp);
+        std::vector<std::uint32_t> otherWarmUp;
+        Dispatch(otherWarmUp, SpinCode.data(), {Low(SpunDescriptor.data()), High(SpunDescriptor.data()), SpinIterations});
+        Dispatch(otherWarmUp, StoreCode.data(), {Low(TargetDescriptor.data()), High(TargetDescriptor.data()), Low(Fresh.data())});
+        Submit(OtherCompute, otherWarmUp);
+        WaitIdle();
+        Require(Done == 1 && Target[0] == Low(Fresh.data()), "the warm-up dispatches did not run");
+
+        Gated(Window::None);
+        Require(Stale[0] == 1 && Fresh[0] == 0, "the gated dispatch without a store in the window did not count into the stale buffer " + Counts());
+        const auto adopted = AgcDriverDispatchesAdoptedAhead_nid_postfix();
+        Require(adopted != 0, "the gated dispatch was not resolved ahead and adopted");
+
+        auto rejected = AgcDriverDispatchesRejectedAhead_nid_postfix();
+        Gated(Window::CpuStore);
+        Require(Fresh[0] == 1 && Stale[0] == 0, "a dispatch resolved ahead whose descriptor the CPU replaced before its adoption used the old descriptor " + Counts());
+        Require(AgcDriverDispatchesRejectedAhead_nid_postfix() == rejected + 1, "the CPU store in the adoption window was not caught by the adoption check");
+        std::cout << "cpu store before adoption: passed\n";
+
+        bool conclusive = false;
+        for (int attempt = 0; attempt < 4 && !conclusive; ++attempt, SpinDispatches *= 4) {
+            rejected = AgcDriverDispatchesRejectedAhead_nid_postfix();
+            Gated(Window::QueuedGpuStore);
+            conclusive = SawPending;
+            if (!conclusive) {
+                std::cout << "queued gpu store before adoption: the store finished before the adoption with " << SpinDispatches << " spin dispatches, retrying\n";
+                continue;
+            }
+            Require(Fresh[0] == 1 && Stale[0] == 0, "a dispatch resolved ahead whose descriptor a GPU store queued on another queue replaces before its adoption used the old descriptor " + Counts());
+            Require(AgcDriverDispatchesRejectedAhead_nid_postfix() == rejected + 1, "the queued GPU store in the adoption window was not caught by the adoption check");
+        }
+        AgcDriverSetAheadAdoptHook_nid_postfix(nullptr);
+        if (!conclusive) {
+            std::cout << "queued gpu store before adoption: never observed the store pending at the adoption\n";
+            AgcDriverShutdown_nid_postfix();
+            return VulkanTestSkipped;
+        }
+        std::cout << "queued gpu store before adoption: passed\n";
+        std::cout << "dispatch ahead adopt check test passed\n";
+        AgcDriverShutdown_nid_postfix();
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        try {
+            AgcDriverShutdown_nid_postfix();
+        } catch (const std::exception& shutdownError) {
+            std::cerr << shutdownError.what() << '\n';
+        }
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/Float16Dot.cpp

@@ -31,6 +31,12 @@ alignas(256) constexpr std::array<std::uint32_t, 37> Code{
     0xe0781020, 0x80011203, 0xe0781030, 0x80011603, 0xbf810000,
 };
 
+alignas(256) constexpr std::array<std::uint32_t, 20> AccumulatorCode{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xcc13600a, 0x1c1a0b04, 0xcc13000b,
+    0x1c1a0b04, 0xcc13440c, 0x1c1a0b04, 0xcc13400d, 0x9c1a0b04, 0xcc13440e, 0x9c1a0b04, 0xe0781000,
+    0x80010a03, 0xe0701010, 0x80010e03, 0xbf810000,
+};
+
 constexpr std::uint32_t Rows[32][4] = {
     {0x0001bc00u, 0x81bfacbau, 0xc541aa2fu, 0x00000000u},
     {0x01f61a79u, 0x3c000381u, 0xc1729fffu, 0x00000000u},
@@ -101,6 +107,43 @@ constexpr std::uint32_t Expected[32][10] = {
     {0xbaf43c38u, 0xbaf43c38u, 0x3af43c38u, 0x3af43c38u, 0x3abe63ecu, 0xbaf43c38u, 0xc021e000u, 0x3fb50000u, 0xba90cd60u, 0x3abe63ecu}
 };
 
+constexpr std::uint32_t AccumulatorExpected[32][5] = {
+    {0xc541a901u, 0xc541a901u, 0x4541ab5du, 0x4541ab5du, 0xc541a901u},
+    {0xc1729fdfu, 0xc1729fdfu, 0x4172a01fu, 0x4172a01fu, 0xc1729fdfu},
+    {0x4187e315u, 0x4187e315u, 0x4187e315u, 0xc1882737u, 0xc1882737u},
+    {0x3f7fdb0fu, 0x3f7fdb0fu, 0x3f7fdb0fu, 0xbf801279u, 0xbf801279u},
+    {0xbe511687u, 0xbe511687u, 0xbe511687u, 0xbe5166f4u, 0xbe5166f4u},
+    {0xc1e56f13u, 0xc1e56f13u, 0x41e55ba5u, 0x41e55ba5u, 0xc1e56f13u},
+    {0x439a1228u, 0x439a1228u, 0x439a1228u, 0xc3a5d2acu, 0xc3a5d2acu},
+    {0x37f5fb14u, 0x37f5fb14u, 0x37f5fc0cu, 0x37f5fc0cu, 0x37f5fb14u},
+    {0x7fd0a000u, 0x7fd0a000u, 0x7fd0a000u, 0x7fd0a000u, 0x7fd0a000u},
+    {0xffc8ab86u, 0xffc8ab86u, 0x7fc8ab86u, 0x7fc8ab86u, 0xffc8ab86u},
+    {0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u},
+    {0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u},
+    {0xb974e480u, 0xb974e480u, 0xb974e480u, 0xb974e480u, 0xb974e480u},
+    {0x7f800000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0xff800000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x80000000u},
+    {0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u},
+    {0x7fc42000u, 0x7fc42000u, 0x7fc42000u, 0x7fc42000u, 0x7fc42000u},
+    {0xbf052ae0u, 0xbf052ae0u, 0xbf052ae0u, 0xbf052ae0u, 0xbf052ae0u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x5f834da7u, 0x5f834da7u, 0x5f834da7u, 0xdf834da7u, 0xdf834da7u},
+    {0x7f7fffffu, 0x7f7fffffu, 0x7f7fffffu, 0xff7fffffu, 0xff7fffffu},
+    {0x42750eeeu, 0x42750eeeu, 0x42750eeeu, 0xc2750eeeu, 0xc2750eeeu},
+    {0x43380748u, 0x43380748u, 0x43380748u, 0xc3380746u, 0xc3380746u},
+    {0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u},
+    {0xf540cddbu, 0xf540cddbu, 0x7540cddbu, 0x7540cddbu, 0xf540cddbu},
+    {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u},
+    {0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0xffe00000u, 0xffe00000u},
+    {0x3cb7c720u, 0x3cb7c720u, 0x3cb7c720u, 0x3cb7c720u, 0x3cb7c720u},
+    {0xbf9bd158u, 0xbf9bd158u, 0xbf9bd158u, 0xbf9bd158u, 0xbf9bd158u},
+    {0x565f49ebu, 0x565f49ebu, 0x565f49ebu, 0xd65f49ebu, 0xd65f49ebu},
+    {0xbaf43c38u, 0xbaf43c38u, 0xbaf43c38u, 0xbaf43c38u, 0xbaf43c38u}
+};
+
+constexpr const char* AccumulatorNames[5] = {"v_dot2 op_sel src2", "v_dot2 op_sel_hi src2 cleared", "v_dot2 neg_hi src2", "v_dot2 neg_lo src2", "v_dot2 neg_lo and neg_hi src2"};
+
 constexpr const char* Names[10] = {"v_dot2c", "v_dot2", "v_dot2 neg_lo src0/src2 neg_hi src1", "v_dot2 neg_lo src1 neg_hi src0", "v_dot2 op_sel src0", "v_dot2 clamp", "v_dot2c inline 1.0", "v_dot2c literal", "v_dot2c dpp quad_perm", "v_dot2 op_sel src1"};
 
 void Fill(std::uint32_t tid, std::uint32_t* words) {
@@ -140,21 +183,9 @@ auto Compile(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> cod
     return ShaderRecompiler::Recompile(request);
 }
 
-void CheckRefused(AgcDriver::VulkanDevice& device, std::uint32_t word0, const std::string& what) {
-    alignas(256) const std::array<std::uint32_t, 3> code{word0, 0x1c1a0b04u, 0xbf810000u};
-    std::string refusal;
-    try {
-        static_cast<void>(Compile(device, code));
-    } catch (const std::exception& error) {
-        refusal = error.what();
-    }
-    Require(refusal.find("v_dot2_f32_f16 accumulator op_sel, op_sel_hi and neg_hi are not implemented") != std::string::npos, what + " was not refused");
-}
-
-void Run(AgcDriver::VulkanDevice& device) {
+void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code) {
     for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
     Output.fill(0xdeadbeefu);
-    const std::span<const std::uint32_t> code(Code);
     const auto result = Compile(device, code);
     device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
     device.WaitIdle();
@@ -167,6 +198,13 @@ void Check() {
     }
 }
 
+void CheckAccumulator() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::uint32_t* out = &Output[tid * Results];
+        for (std::uint32_t i = 0; i < 5; ++i) Expect(tid, out[i], AccumulatorExpected[tid][i], AccumulatorNames[i]);
+    }
+}
+
 }
 
 int main() {
@@ -177,11 +215,10 @@ int main() {
             std::puts("skipped, v_dot2_f32_f16 is computed in f64 and the device has no shaderFloat64");
             return VulkanTestSkipped;
         }
-        Run(*device);
+        Run(*device, Code);
         Check();
-        CheckRefused(*device, 0xcc13600au, "op_sel on the accumulator");
-        CheckRefused(*device, 0xcc13440au, "neg_hi on the accumulator");
-        CheckRefused(*device, 0xcc13000au, "op_sel_hi cleared on the accumulator");
+        Run(*device, AccumulatorCode);
+        CheckAccumulator();
         std::puts("v_dot2_f32_f16 tests passed");
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreLineThroughPlaneArray.cpp

@@ -0,0 +1,185 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Width = 32;
+constexpr std::uint32_t Type1D = 8;
+constexpr std::uint32_t IdentitySwizzle = 0xfacu;
+constexpr std::size_t TexelBytes = 4096;
+constexpr std::size_t BlockBytes = 65536;
+
+alignas(256) std::array<std::uint32_t, Threads * 4> Input{};
+
+alignas(256) constexpr std::array<std::uint32_t, 17> StoreCode{
+    0x34020084, 0xe0381000, 0x80001401, 0xbf8c3f70, 0x7e3c0300, 0x7e3e0281, 0x7e400280, 0xf0201f28, 0x0001141e,
+    0x7e3e0280, 0x7e4002c1, 0xf0201528, 0x0001141e, 0x7e400281, 0xf0201528, 0x0001141e, 0xbf810000,
+};
+
+class GuestBlock {
+public:
+    GuestBlock() {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
+#endif
+        Require(block != nullptr, "image store line through plane array: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+#ifdef _WIN32
+        VirtualFree(block, 0, MEM_RELEASE);
+#else
+        std::free(block);
+#endif
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* Data() { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+struct Format {
+    const char* name;
+    std::uint32_t format;
+    std::uint32_t components;
+};
+
+constexpr std::array<Format, 2> Formats{{
+    {"32_SINT", 21u, 1u},
+    {"32_32_32_32_SINT", 76u, 4u},
+}};
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+std::uint32_t Value(std::uint32_t tid, std::uint32_t index) {
+    return (tid * 4u + index + 1u) * 0x9e3779b1u;
+}
+
+std::uint64_t TexelOffset(const Format& format, std::uint32_t x) {
+    const auto mip = AgcDriver::Graphics::ComputeMipLayout(AgcDriver::Graphics::TextureTileMode::kLinear, format.format, Width, 1u, 1u).at(0);
+    return mip.tiledOffset + static_cast<std::uint64_t>(x) * format.components * 4u;
+}
+
+std::vector<std::uint8_t> Initial() {
+    std::vector<std::uint8_t> bytes(TexelBytes);
+    for (std::size_t index = 0; index < bytes.size(); ++index) bytes[index] = static_cast<std::uint8_t>(0x80u + 37u * index + 11u * (index >> 4u));
+    return bytes;
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+std::array<std::uint32_t, 8> TextureDescriptor(const void* data, std::uint32_t format) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(data));
+    return {
+        static_cast<std::uint32_t>(address >> 8u),
+        static_cast<std::uint32_t>((address >> 40u) & 0xffu) | (format << 20u) | (((Width - 1u) & 3u) << 30u),
+        (Width - 1u) >> 2u,
+        IdentitySwizzle | (Type1D << 28u),
+        0u,
+        0u,
+        0u,
+        0u,
+    };
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span<const std::uint32_t> code, std::uint32_t format) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        for (std::uint32_t index = 0; index < 4u; ++index) Input[tid * 4u + index] = Value(tid, index);
+    }
+    const auto initial = Initial();
+    std::copy(initial.begin(), initial.end(), texels);
+    std::vector<std::uint32_t> userData(16, 0u);
+    const auto buffer = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto texture = TextureDescriptor(texels, format);
+    std::copy(buffer.begin(), buffer.end(), userData.begin());
+    std::copy(texture.begin(), texture.end(), userData.begin() + 4);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+    AgcDriver::Graphics::StorageTexture::FlushPending(reinterpret_cast<std::uintptr_t>(texels), TexelBytes, nullptr, "test");
+    device.WaitIdle();
+}
+
+void CheckTexels(const std::uint8_t* texels, const Format& format) {
+    const auto initial = Initial();
+    for (std::uint32_t x = 0; x < Width; ++x) {
+        const auto offset = TexelOffset(format, x);
+        for (std::uint32_t component = 0; component < format.components; ++component) {
+            std::uint32_t actual = 0;
+            std::memcpy(&actual, texels + offset + component * 4u, 4u);
+            const auto wanted = Value(x, component);
+            Require(actual == wanted, std::string("image_store 2d_array through a 1D ") + format.name + " T#: texel " + std::to_string(x) + " component " + std::to_string(component) + " is " + Hex(actual) + ", expected the y = 1, slice = 0 store " + Hex(wanted) + " and the stores to slices 0xffffffff and 1 dropped");
+        }
+    }
+    const auto end = TexelOffset(format, Width);
+    Require(std::equal(texels + end, texels + TexelBytes, initial.begin() + static_cast<std::ptrdiff_t>(end)), std::string("image_store 2d_array through a 1D ") + format.name + " T#: bytes past the texture changed");
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        GuestBlock block;
+        auto* texels = block.Data();
+        for (const auto& format : Formats) {
+            Dispatch(*device, texels, StoreCode, format.format);
+            CheckTexels(texels, format);
+        }
+        std::puts("image store line through plane array tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetCompletionPoints.cpp

@@ -0,0 +1,395 @@
+#include "prx/libkernel/include/Api.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp"
+#include "prx/libSceAgcDriver/Eq/include/Event.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "prx/libkernel/Equeue/Equeue.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <bit>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <functional>
+#include <initializer_list>
+#include <iostream>
+#include <memory>
+#include <span>
+#include <string>
+#include <string_view>
+#include <thread>
+#include <utility>
+#include <vector>
+
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+
+constexpr std::uint32_t Width = 256;
+constexpr std::uint32_t Height = 128;
+constexpr std::size_t Pitch = std::size_t{Width} * 4u;
+constexpr std::size_t TargetBytes = Pitch * Height;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t LabelOffset = 3 * BlockBytes;
+constexpr std::uint32_t Untouched = 0x5a5a5a5au;
+constexpr std::uint32_t Drawn = 0xff0000ffu;
+constexpr std::uint32_t LabelValue = 0x600dcafeu;
+constexpr std::uint32_t OutputHandle = 9;
+constexpr std::uint32_t Rgba8UnormInfo = 0x28028u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::array<std::uint32_t, 4> Scissor{32, 16, 160, 80};
+
+alignas(256) constexpr std::array<std::uint32_t, 20> VertexCode{
+    0x7e000d05u, 0x080200f4u, 0x10020300u, 0x100202f4u, 0x060202f3u, 0x060400f3u, 0x10040500u, 0x060404f3u,
+    0x7e060208u, 0x7e0802f2u, 0x7e0a02ffu, 0x44800000u, 0xf80000d1u, 0x00000005u, 0xf80008cfu, 0x04030201u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u,
+};
+alignas(256) constexpr std::array<std::uint32_t, 8> PixelCode{0x7e000200u, 0x7e020280u, 0xf800180fu, 0x00010100u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 9> FillCode{
+    0xd7460004, 0x04010c08, 0x7e000204, 0x7e020205, 0x7e040206, 0x7e060207, 0xe01c2000, 0x80000004, 0xbf810000,
+};
+
+alignas(4) std::atomic<std::uint32_t> Gate = 0;
+
+struct Header {
+    Shader shader{};
+    ShaderUserData userData{};
+};
+
+alignas(8) Header VertexHeader{};
+alignas(8) Header PixelHeader{};
+
+void Registered(Header& header, const std::uint32_t* code, std::size_t bytes, std::uint8_t type) {
+    header.shader.file_header = 0x34333231u;
+    header.shader.version = 0x18u;
+    header.shader.header_size = sizeof(Header);
+    header.shader.shader_size = static_cast<std::uint32_t>(bytes);
+    header.shader.code = code;
+    header.shader.type = type;
+    header.shader.user_data = &header.userData;
+    AgcDriverRegisterShader_nid_postfix(&header.shader);
+}
+
+void Append(std::vector<std::uint32_t>& commands, std::uint32_t opcode, std::initializer_list<std::uint32_t> payload) {
+    commands.push_back(0xc0000000u | (static_cast<std::uint32_t>(payload.size() - 1) << 16u) | (opcode << 8u));
+    commands.insert(commands.end(), payload);
+}
+
+void AppendDraw(std::vector<std::uint32_t>& commands, std::uint64_t target) {
+    const auto vertex = reinterpret_cast<std::uintptr_t>(VertexCode.data());
+    const auto pixel = reinterpret_cast<std::uintptr_t>(PixelCode.data());
+    const auto extent = (Height << 16u) | Width;
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> context{
+        {0x2d5, 0x2000},
+        {0x1b6, 0}, {0x207, 0x01200000}, {0x200, 0}, {0x203, 0x800},
+        {0x2dc, 0xaa00}, {0x2f8, 0}, {0x292, 2}, {0x293, 0},
+        {0x80, 0}, {0x8d, 0}, {0x83, 0xffff}, {0x8c, 0xa},
+        {0x2f9, 0x2d}, {0x313, 0x6000}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff},
+        {0x206, 0x43f}, {0x204, 0x80000}, {0x205, 0x240},
+        {0x8e, 0xf}, {0x8f, 0xf}, {0x202, 0xcc0010},
+        {0x1c4, 0}, {0x1c5, 9}, {0x1c3, 4}, {0x31c, Rgba8UnormInfo},
+        {0x31b, 0}, {0x31d, 0}, {0x3b0, ((Width - 1u) << 14u) | (Height - 1u)},
+        {0x3b8, LinearAttrib3}, {0x1e0, 0},
+        {0xc, 0}, {0xd, extent},
+        {0x81, 0x80000000}, {0x82, extent},
+        {0x90, 0x80000000u | (Scissor[1] << 16u) | Scissor[0]}, {0x91, (Scissor[3] << 16u) | Scissor[2]},
+        {0x94, 0x80000000}, {0x95, extent},
+        {0x10f, std::bit_cast<std::uint32_t>(2.0f * Width)}, {0x110, std::bit_cast<std::uint32_t>(2.0f * Width)},
+        {0x111, std::bit_cast<std::uint32_t>(2.0f * Height)}, {0x112, std::bit_cast<std::uint32_t>(2.0f * Height)},
+        {0x113, std::bit_cast<std::uint32_t>(1.0f)}, {0x114, 0},
+        {0xb4, 0}, {0xb5, std::bit_cast<std::uint32_t>(1.0f)},
+        {0x2fe, 0}, {0x302, 0}, {0x306, 0}, {0x30a, 0},
+        {0x318, static_cast<std::uint32_t>(target >> 8u)}, {0x390, static_cast<std::uint32_t>(target >> 40u)},
+    };
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> shader{
+        {0xc8, static_cast<std::uint32_t>(vertex >> 8u)}, {0xc9, static_cast<std::uint32_t>(vertex >> 40u)}, {0x8b, 1u << 1u}, {0x8c, 0},
+        {0x008, static_cast<std::uint32_t>(pixel >> 8u)}, {0x009, static_cast<std::uint32_t>(pixel >> 40u)}, {0x00b, 1u << 1u}, {0x00c, std::bit_cast<std::uint32_t>(1.0f)},
+    };
+    Append(commands, 0x79, {0x242, 4});
+    Append(commands, 0x79, {0x24a, 0});
+    for (const auto& [offset, value] : context) Append(commands, 0x69, {offset, value});
+    for (const auto& [offset, value] : shader) Append(commands, 0x76, {offset, value});
+    Append(commands, 0x2d, {3, 2});
+}
+
+void AppendGate(std::vector<std::uint32_t>& commands) {
+    const auto gate = reinterpret_cast<std::uintptr_t>(&Gate);
+    commands.insert(commands.end(), {0xc0053c00u, 0x13u, static_cast<std::uint32_t>(gate), static_cast<std::uint32_t>(gate >> 32u), 1u, 0xffffffffu, 0x19u});
+}
+
+void Submit(std::vector<std::uint32_t>& commands) {
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    AgcDriver::Submit(&packet, 0);
+}
+
+class Region {
+public:
+    Region(std::size_t bytes, bool watched) : bytes(bytes), arena(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (arena) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            if (data != nullptr) GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (arena) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "linear target completion points: cannot map the guest region");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "linear target completion points: cannot allocate the guest region");
+        auto* words = reinterpret_cast<std::uint32_t*>(data);
+        std::fill(words, words + bytes / 4u, Untouched);
+        if (arena && !watched) {
+            AgcDriver::GuestMemory::Unwatch(Address(), bytes);
+            Require(!AgcDriver::GuestMemory::Watched(Address(), bytes), "linear target completion points: the region is still write-watched");
+        }
+        GuestAllocations::Mutation().Add(data, bytes, true, true, true);
+    }
+    ~Region() {
+        GuestAllocations::Mutation().Remove(data);
+#ifdef _WIN32
+        if (arena) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (arena) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Region(const Region&) = delete;
+    Region& operator=(const Region&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::uint32_t Word(std::size_t offset) const { return std::atomic_ref<std::uint32_t>(*reinterpret_cast<std::uint32_t*>(data + offset)).load(std::memory_order_acquire); }
+    std::size_t bytes;
+
+private:
+    bool arena;
+    std::uint8_t* data = nullptr;
+};
+
+void Import(const Region& region) {
+    const auto address = region.Address();
+    const std::vector<std::uint32_t> userData{
+        static_cast<std::uint32_t>(address),
+        static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (16u << 16u),
+        static_cast<std::uint32_t>(region.bytes / 16u),
+        0x4bfacu,
+        Untouched,
+        Untouched,
+        Untouched,
+        Untouched,
+    };
+    std::vector<std::uint32_t> commands;
+    const auto code = reinterpret_cast<std::uintptr_t>(FillCode.data());
+    Append(commands, 0x76, {0x207, 64, 1, 1});
+    Append(commands, 0x76, {0x20c, static_cast<std::uint32_t>(code >> 8u), static_cast<std::uint32_t>(code >> 40u)});
+    Append(commands, 0x76, {0x213, (static_cast<std::uint32_t>(userData.size()) << 1u) | 0x80u});
+    commands.push_back(0xc0007600u | (static_cast<std::uint32_t>(userData.size()) << 16u));
+    commands.push_back(0x240u);
+    commands.insert(commands.end(), userData.begin(), userData.end());
+    commands.insert(commands.end(), {0xc0031500u, static_cast<std::uint32_t>(region.bytes / 1024u), 1, 1, 0x41u});
+    Submit(commands);
+    AgcDriverWaitIdle_nid_postfix();
+}
+
+bool InsideScissor(std::uint32_t x, std::uint32_t y) {
+    return x >= Scissor[0] && x < Scissor[2] && y >= Scissor[1] && y < Scissor[3];
+}
+
+std::string Mismatches(const Region& region) {
+    std::size_t drawn = 0, outside = 0;
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const auto value = region.Word(y * Pitch + x * 4u);
+            if (InsideScissor(x, y)) drawn += value == Drawn ? 0u : 1u;
+            else outside += value == Untouched ? 0u : 1u;
+        }
+    }
+    if (drawn == 0 && outside == 0) return {};
+    return std::to_string(drawn) + " of the " + std::to_string((Scissor[2] - Scissor[0]) * (Scissor[3] - Scissor[1])) + " pixels inside the scissor do not hold the draw's color and " + std::to_string(outside) + " pixels outside it changed";
+}
+
+bool Await(const std::function<bool()>& done) {
+    const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5);
+    while (!done()) {
+        if (std::chrono::steady_clock::now() >= deadline) return false;
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+    return true;
+}
+
+class Request final : public AgcDriver::IFlipRequest {
+public:
+    explicit Request(std::atomic<int>& ready) : ready(ready) {}
+    void GpuReady(const std::shared_ptr<AgcDriver::FrameTiming>&) override { ++ready; }
+    void Fail(std::exception_ptr) noexcept override {}
+private:
+    std::atomic<int>& ready;
+};
+
+class Output final : public AgcDriver::IVideoOutput {
+public:
+    std::shared_ptr<AgcDriver::IFlipRequest> Reserve(const AgcDriver::FlipInfo&) override { return std::make_shared<Request>(ready); }
+    void Fail(std::exception_ptr) noexcept override {}
+    std::atomic<int> ready = 0;
+};
+
+enum class Point { Flip, Label, Interrupt };
+
+struct Events {
+    KernelEqueue queue = 0;
+    KernelEqueueRef owner;
+    int tag = 0;
+};
+
+std::string Run(Point point, bool watched, const Region& region, Output& output, Events& events) {
+    const char* name = point == Point::Flip ? "flip" : point == Point::Label ? "label" : "end-of-pipe interrupt";
+    const auto target = region.Address();
+    Import(region);
+    Gate.store(0, std::memory_order_release);
+    std::vector<std::uint32_t> commands;
+    AppendDraw(commands, target);
+    if (point == Point::Flip) {
+        commands.insert(commands.end(), {AgcDriver::FlipPacketHeader, OutputHandle, 0xfffffffeu, 1, 0, 0});
+    } else if (point == Point::Label) {
+        const auto label = target + LabelOffset;
+        commands.insert(commands.end(), {0xc0064900u, 0x514u, 1u << 29u, static_cast<std::uint32_t>(label), static_cast<std::uint32_t>(label >> 32u), LabelValue, 0, 0});
+    } else {
+        commands.insert(commands.end(), {0xc0064900u, 0x06000528u, 2u << 24u, 0, 0, 0, 0, 0});
+    }
+    AppendGate(commands);
+    const auto readyBefore = output.ready.load();
+    if (point == Point::Interrupt) {
+        std::array<KernelEvent, 4> stale{};
+        while (events.owner->GetTriggeredEvents(stale.data(), static_cast<int>(stale.size())) > 0) {}
+    }
+    Submit(commands);
+    const bool watchedAtStore = AgcDriver::GuestMemory::Watched(target, TargetBytes);
+    std::string failure;
+    if (point == Point::Flip) {
+        const bool flipped = Await([&] { return output.ready.load() != readyBefore; });
+        if (!flipped) failure = "the flip did not reach the GPU";
+        else if (!Await([&] { return Mismatches(region).empty(); })) failure = "after the flip, while the queue waited: " + Mismatches(region);
+    } else if (point == Point::Label) {
+        if (!Await([&] { return region.Word(LabelOffset) == LabelValue; })) failure = "the label after the draw never landed";
+        else if (const auto mismatches = Mismatches(region); !mismatches.empty()) failure = "when the label after the draw landed: " + mismatches;
+    } else {
+        std::array<KernelEvent, 1> delivered{};
+        if (!Await([&] { return events.owner->GetTriggeredEvents(delivered.data(), 1) == 1; })) failure = "the end-of-pipe interrupt after the draw was not delivered";
+        else if (delivered[0].udata != &events.tag) failure = "an unexpected event was delivered";
+        else if (const auto mismatches = Mismatches(region); !mismatches.empty()) failure = "when the end-of-pipe interrupt after the draw was delivered: " + mismatches;
+    }
+    Gate.store(1, std::memory_order_release);
+    AgcDriverWaitIdle_nid_postfix();
+    AgcDriver::Graphics::StorageTexture::FlushPending(target, region.bytes, nullptr, "test release");
+    std::printf("linear target completion point: %s, %s memory (watched at the store: %d): %s\n", name, watched ? "write-watched" : "unwatched", watchedAtStore ? 1 : 0, failure.empty() ? "stored" : failure.c_str());
+    if (failure.empty()) return {};
+    return std::string(name) + (watched ? " (write-watched memory): " : " (unwatched memory): ") + failure;
+}
+
+}
+
+int main(int argc, char** argv) {
+    try {
+        bool flips = argc == 1, labels = argc == 1, interrupts = argc == 1, watchedFlip = argc == 1;
+        for (int index = 1; index < argc; ++index) {
+            const std::string_view argument(argv[index]);
+            if (argument == "--flip") flips = true;
+            else if (argument == "--label") labels = true;
+            else if (argument == "--interrupt") interrupts = true;
+            else Require(false, "linear target completion points: unknown argument " + std::string(argument));
+        }
+        {
+            const auto device = OpenVulkanTestDevice();
+            if (!device) return VulkanTestSkipped;
+            if (AgcDriver::VulkanDeviceTestAccess::ContextOf(*device).hostImportAlignment == 0) {
+                std::puts("linear target completion point tests skipped: no host import on this device, so the target is never resident");
+                return VulkanTestSkipped;
+            }
+        }
+        std::vector<std::unique_ptr<Region>> regions;
+        struct ShutdownGuard {
+            ~ShutdownGuard() { try { AgcDriverShutdown_nid_postfix(); } catch (...) {} }
+        } shutdown;
+        Registered(VertexHeader, VertexCode.data(), sizeof(VertexCode), 2);
+        Registered(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1);
+        auto output = std::make_shared<Output>();
+        AgcDriverRegisterVideoOutput_nid_postfix(OutputHandle, output);
+        Events events;
+        if (interrupts) {
+            Require(sceKernelCreateEqueue(&events.queue, "AGC linear target") == 0, "linear target completion points: event queue creation failed");
+            events.owner = EqueuePin_nid_postfix(events.queue);
+            Require(sceAgcDriverAddEqEvent(events.queue, 0, &events.tag) == 0, "linear target completion points: graphics event registration failed");
+        }
+        std::vector<std::string> failures;
+        const auto run = [&](Point point, bool watched) {
+            regions.push_back(std::make_unique<Region>(watched ? 16 * BlockBytes : 4 * BlockBytes, watched));
+            if (auto failure = Run(point, watched, *regions.back(), *output, events); !failure.empty()) failures.push_back(std::move(failure));
+        };
+        if (flips) run(Point::Flip, false);
+        if (watchedFlip) run(Point::Flip, true);
+        if (labels) run(Point::Label, false);
+        if (interrupts) run(Point::Interrupt, false);
+        if (interrupts) {
+            Require(sceAgcDriverDeleteEqEvent(events.queue, 0) == 0, "linear target completion points: graphics event deletion failed");
+            events.owner.reset();
+            Require(sceKernelDeleteEqueue(events.queue) == 0, "linear target completion points: event queue deletion failed");
+        }
+        AgcDriverUnregisterVideoOutput_nid_postfix(OutputHandle, output);
+        if (!failures.empty()) {
+            std::string message = "a CPU read after a point where the guest can observe a draw's completion did not see the drawn linear target:";
+            for (const auto& failure : failures) message += "\n  " + failure;
+            Require(false, message);
+        }
+        AgcDriverShutdown_nid_postfix();
+        std::puts("linear target completion point tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetCompletionStores.cpp

@@ -0,0 +1,304 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using AgcDriver::Graphics::StorageTexture;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 16;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t TargetBytes = std::size_t{Width} * Height * 16u;
+constexpr std::size_t LabelOffset = BlockBytes;
+constexpr std::uint32_t Rgba32FloatInfo = 0x8738u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t Float32AbgrExport = 9;
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::array<std::uint32_t, 16>, 2> Programs{};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+std::span<const std::uint32_t> Program(std::size_t index, bool blue) {
+    auto& code = Programs.at(index);
+    std::size_t at = 0;
+    code[at++] = 0x36000000u | (20u << 17u) | 0xffu;
+    code[at++] = 0x0000ffffu;
+    code[at++] = 0x2c000000u | (21u << 17u) | 0x90u;
+    code[at++] = 0x7e000c00u | (20u << 17u) | (256u + 20u);
+    code[at++] = 0x7e000c00u | (21u << 17u) | (256u + 21u);
+    code[at++] = (blue ? 0x7e0002f2u : 0x7e000280u) | (22u << 17u);
+    code[at++] = 0x7e0002f2u | (23u << 17u);
+    code[at++] = 0xf800180fu;
+    code[at++] = 0x17161514u;
+    code[at++] = 0xbf810000u;
+    return std::span<const std::uint32_t>(code.data(), at);
+}
+
+struct Block {
+    explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (watched) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (watched) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map the color target");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "cannot allocate the color target");
+        Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is not write-watched");
+        GuestAllocations::Mutation mutation;
+        mutation.Add(data, bytes, true, true, true);
+    }
+    ~Block() {
+        StorageTexture::FlushPending(Address(), bytes, nullptr, "test release");
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Remove(data);
+        }
+#ifdef _WIN32
+        if (watched) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (watched) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Block(const Block&) = delete;
+    Block& operator=(const Block&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::size_t bytes;
+    bool watched;
+    std::uint8_t* data = nullptr;
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block) {
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(block.Address() >> 8u);
+    cx[0x31b] = 0;
+    cx[0x31c] = Rgba32FloatInfo;
+    cx[0x31d] = 0;
+    cx[0x323] = 0;
+    cx[0x324] = 0;
+    cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u);
+    cx[0x3b8] = LinearAttrib3;
+    cx[0x390] = static_cast<std::uint32_t>(block.Address() >> 40u);
+    auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    color.exportIndex = 0;
+    return color;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::span<const std::uint32_t> pixelCode, VkRect2D scissor) {
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast<std::uint32_t>(Triangle.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionFixedPoint);
+    pixel.targetOutputMode.at(0) = Float32AbgrExport;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::vector<std::uint32_t> pixelUserData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.blendEnable = VK_FALSE;
+    blend.colorWriteMask = 0xf;
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = scissor;
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+}
+
+std::array<float, 4> Texel(const std::uint8_t* target, std::uint32_t x, std::uint32_t y) {
+    std::array<float, 4> texel{};
+    std::memcpy(texel.data(), target + (static_cast<std::size_t>(y) * Width + x) * 16u, sizeof(texel));
+    return texel;
+}
+
+void Label(AgcDriver::VulkanDevice& device, const Block& block, std::uint32_t value, std::uint64_t submissions, bool watched, const char* what) {
+    const auto bytes = std::as_bytes(std::span(&value, 1));
+    const auto reason = device.WriteLabelOnGpu(block.Address() + LabelOffset, bytes, value, 0);
+    Require(reason == 0, std::string(what) + ": the label was not recorded on the GPU (outcome " + std::to_string(reason) + ")");
+    Require(StorageTexture::FindPending(block.Address(), TargetBytes) == nullptr, std::string(what) + ": the drawn linear target was still pending after a label the GPU writes");
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(!watched || (recorder != nullptr && recorder->Submissions() == submissions), std::string(what) + ": storing the drawn linear target before the label waited for recorded work (" + std::to_string(recorder != nullptr ? recorder->Submissions() - submissions : 0) + " submissions)");
+}
+
+bool Expect(AgcDriver::VulkanDevice& device) {
+    Block block(16 * BlockBytes);
+    std::fill(block.data, block.data + block.bytes, std::uint8_t{0xff});
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    if (AgcDriver::Graphics::HostImportFor(context, block.Address(), block.bytes) == nullptr) return false;
+    const auto color = DecodeTarget(block);
+    Require(color.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && color.bytes == TargetBytes, "the target is not a linear R32G32B32A32 target");
+    const auto red = Program(0, false);
+    const auto blue = Program(1, true);
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(recorder != nullptr, "the device has no recorder");
+    const bool watched = AgcDriver::GuestMemory::Watched(block.Address(), TargetBytes);
+    if (!watched) std::puts("linear target completion stores: the target is outside the write watch, so a store may wait for the previous store's saved bytes; the submission checks are skipped");
+    const auto pending = [&](const char* what) {
+        Require(StorageTexture::FindPending(block.Address(), TargetBytes) != nullptr, std::string(what) + ": the drawn linear target was stored before any point where its completion is observable");
+    };
+    Draw(device, color, red, {{0, 0}, {Width, Height}});
+    pending("after the first draw");
+    const auto submissions = recorder->Submissions();
+    Draw(device, color, blue, {{0, 0}, {Width / 2u, Height}});
+    pending("after the second draw");
+    Require(!watched || recorder->Submissions() == submissions, "the second draw into the linear target submitted recorded work");
+    Draw(device, color, red, {{Width / 4u, Height / 4u}, {Width / 4u, Height / 2u}});
+    pending("after the third draw");
+    Require(!watched || recorder->Submissions() == submissions, "the third draw into the linear target submitted recorded work");
+    Label(device, block, 1u, submissions, watched, "the first label");
+    Draw(device, color, blue, {{Width / 2u, 0}, {Width / 2u, Height}});
+    pending("after the fourth draw");
+    Label(device, block, 2u, submissions, watched, "the second label");
+    Draw(device, color, red, {{0, Height / 2u}, {Width, Height / 2u}});
+    pending("after the fifth draw");
+    Require(!watched || recorder->Submissions() == submissions, "the fifth draw into the linear target submitted recorded work");
+    device.WaitIdle();
+    Require(StorageTexture::FindPending(block.Address(), TargetBytes) == nullptr, "the drawn linear target was still pending after the device wait");
+    std::vector<std::uint8_t> stored(TargetBytes);
+    std::memcpy(stored.data(), block.data, TargetBytes);
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const bool lastRed = y >= Height / 2u;
+            const bool lastBlue = !lastRed && x >= Width / 2u;
+            const bool thirdRed = !lastRed && !lastBlue && x >= Width / 4u && x < Width / 2u && y >= Height / 4u;
+            const bool secondBlue = !lastRed && !lastBlue && !thirdRed;
+            const float z = lastRed || thirdRed ? 0.0f : secondBlue || lastBlue ? 1.0f : -1.0f;
+            const std::array<float, 4> expected{static_cast<float>(x), static_cast<float>(y), z, 1.0f};
+            const auto texel = Texel(stored.data(), x, y);
+            Require(texel == expected, "after the device wait, raw guest read: pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") holds (" + std::to_string(texel[0]) + ", " + std::to_string(texel[1]) + ", " + std::to_string(texel[2]) + ", " + std::to_string(texel[3]) + ")");
+        }
+    }
+    std::uint32_t label = 0;
+    std::memcpy(&label, block.data + LabelOffset, sizeof(label));
+    Require(label == 2u, "the second label did not land: it holds " + std::to_string(label));
+    return true;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (!Expect(*device)) {
+            std::puts("linear target completion store test skipped: no host import on this device");
+            return VulkanTestSkipped;
+        }
+        std::puts("linear target completion store tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetCpuWrites.cpp

@@ -0,0 +1,310 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 16;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t TargetBytes = std::size_t{Width} * Height * 16u;
+constexpr std::size_t RowBytes = std::size_t{Width} * 16u;
+constexpr std::uint32_t Drawn = Width / 2u;
+constexpr std::uint32_t Rgba32FloatInfo = 0x8738u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t Float32AbgrExport = 9;
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::array<std::uint32_t, 16>, 2> Programs{};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+std::span<const std::uint32_t> Program(std::size_t index, bool blue) {
+    auto& code = Programs.at(index);
+    std::size_t at = 0;
+    code[at++] = 0x36000000u | (20u << 17u) | 0xffu;
+    code[at++] = 0x0000ffffu;
+    code[at++] = 0x2c000000u | (21u << 17u) | 0x90u;
+    code[at++] = 0x7e000c00u | (20u << 17u) | (256u + 20u);
+    code[at++] = 0x7e000c00u | (21u << 17u) | (256u + 21u);
+    code[at++] = (blue ? 0x7e0002f2u : 0x7e000280u) | (22u << 17u);
+    code[at++] = 0x7e0002f2u | (23u << 17u);
+    code[at++] = 0xf800180fu;
+    code[at++] = 0x17161514u;
+    code[at++] = 0xbf810000u;
+    return std::span<const std::uint32_t>(code.data(), at);
+}
+
+struct Block {
+    Block(std::size_t bytes, bool stayWatched) : bytes(bytes), arena(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (arena) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (arena) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map the color target");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "cannot allocate the color target");
+        Require(!arena || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is not write-watched");
+        if (arena && !stayWatched) {
+            AgcDriver::GuestMemory::Unwatch(Address(), bytes);
+            Require(!AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is still write-watched");
+        }
+        GuestAllocations::Mutation mutation;
+        mutation.Add(data, bytes, true, true, true);
+    }
+    ~Block() {
+        AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release");
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Remove(data);
+        }
+#ifdef _WIN32
+        if (arena) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (arena) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Block(const Block&) = delete;
+    Block& operator=(const Block&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::size_t bytes;
+    bool arena;
+    std::uint8_t* data = nullptr;
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block) {
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(block.Address() >> 8u);
+    cx[0x31b] = 0;
+    cx[0x31c] = Rgba32FloatInfo;
+    cx[0x31d] = 0;
+    cx[0x323] = 0;
+    cx[0x324] = 0;
+    cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u);
+    cx[0x3b8] = LinearAttrib3;
+    cx[0x390] = static_cast<std::uint32_t>(block.Address() >> 40u);
+    auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    color.exportIndex = 0;
+    return color;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::span<const std::uint32_t> pixelCode, std::uint32_t scissorWidth) {
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast<std::uint32_t>(Triangle.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionFixedPoint);
+    pixel.targetOutputMode.at(0) = Float32AbgrExport;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::vector<std::uint32_t> pixelUserData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.blendEnable = VK_FALSE;
+    blend.colorWriteMask = 0xf;
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = {{0, 0}, {scissorWidth, Height}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+}
+
+std::array<float, 4> Texel(const std::uint8_t* target, std::uint32_t x, std::uint32_t y) {
+    std::array<float, 4> texel{};
+    std::memcpy(texel.data(), target + static_cast<std::size_t>(y) * RowBytes + static_cast<std::size_t>(x) * 16u, sizeof(texel));
+    return texel;
+}
+
+enum class Outcome { Kept, Lost, Skipped };
+
+Outcome Run(AgcDriver::VulkanDevice& device, bool watched, bool seeded, bool hookRead) {
+    const auto name = std::string(seeded ? "seeded" : "unseeded") + ", " + (watched ? "write-watched" : "unwatched") + " memory" + (hookRead ? ", driver read before the store" : "");
+    Block block(watched ? std::size_t{16} * BlockBytes : BlockBytes, watched);
+    std::fill(block.data, block.data + block.bytes, std::uint8_t{0xff});
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    if (AgcDriver::Graphics::HostImportFor(context, block.Address(), block.bytes) == nullptr) return Outcome::Skipped;
+    const auto color = DecodeTarget(block);
+    Require(color.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && color.bytes == TargetBytes, "the target is not a linear R32G32B32A32 target");
+    Draw(device, color, Program(0, false), Width);
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(recorder != nullptr && !recorder->Idle(), "the draw into an imported linear target waited for the GPU instead of being recorded");
+    device.WaitIdle();
+    std::vector<std::uint8_t> stored(TargetBytes);
+    std::memcpy(stored.data(), block.data, TargetBytes);
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const std::array<float, 4> expected{static_cast<float>(x), static_cast<float>(y), 0.0f, 1.0f};
+            const auto texel = Texel(stored.data(), x, y);
+            Require(texel == expected, name + ": after the first draw, raw guest read: pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") holds (" + std::to_string(texel[0]) + ", " + std::to_string(texel[1]) + ", " + std::to_string(texel[2]) + ", " + std::to_string(texel[3]) + ")");
+        }
+    }
+    if (seeded) {
+        for (std::uint32_t y = 0; y < Height; ++y) std::memset(block.data + y * RowBytes, 0x5a, Drawn * 16u);
+    }
+    Draw(device, color, Program(1, true), Drawn);
+    for (std::uint32_t y = 0; y < Height; ++y) std::memset(block.data + y * RowBytes + Drawn * 16u, 0x42, RowBytes - Drawn * 16u);
+    const bool watchedAtStore = AgcDriver::GuestMemory::Watched(block.Address(), TargetBytes);
+    if (hookRead) {
+        std::vector<std::uint8_t> row(RowBytes);
+        AgcDriver::GuestMemory::Read(block.Address(), std::as_writable_bytes(std::span(row)), 1);
+        for (std::uint32_t x = 0; x < Drawn; ++x) {
+            const std::array<float, 4> expected{static_cast<float>(x), 0.0f, 1.0f, 1.0f};
+            const auto texel = Texel(row.data(), x, 0);
+            Require(texel == expected, name + ": a driver read of row 0 before the store: pixel (" + std::to_string(x) + ", 0) holds (" + std::to_string(texel[0]) + ", " + std::to_string(texel[1]) + ", " + std::to_string(texel[2]) + ", " + std::to_string(texel[3]) + "), not the second draw's");
+        }
+    }
+    device.WaitIdle();
+    std::size_t wrongGpu = 0;
+    std::size_t wrongCpu = 0;
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Drawn; ++x) {
+            const std::array<float, 4> expected{static_cast<float>(x), static_cast<float>(y), 1.0f, 1.0f};
+            if (Texel(block.data, x, y) != expected) ++wrongGpu;
+        }
+        auto* begin = block.data + y * RowBytes + Drawn * 16u;
+        wrongCpu += static_cast<std::size_t>(std::count_if(begin, block.data + (y + 1u) * RowBytes, [](std::uint8_t byte) { return byte != 0x42u; }));
+    }
+    std::printf("linear target CPU writes, %s (watched at the store: %d): wrong GPU pixels %zu/%u, wrong CPU bytes %zu/%zu\n", name.c_str(), watchedAtStore ? 1 : 0, wrongGpu, Drawn * Height, wrongCpu, (RowBytes - Drawn * 16u) * Height);
+    return wrongGpu == 0 && wrongCpu == 0 ? Outcome::Kept : Outcome::Lost;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        bool kept = true;
+        bool ran = false;
+        for (const bool watched : {true, false}) {
+            for (const bool seeded : {false, true}) {
+                for (const bool hookRead : {false, true}) {
+                    const auto outcome = Run(*device, watched, seeded, hookRead);
+                    ran = ran || outcome != Outcome::Skipped;
+                    kept = kept && outcome != Outcome::Lost;
+                }
+            }
+        }
+        if (!ran) {
+            std::puts("linear target CPU write test skipped: no host import on this device");
+            return VulkanTestSkipped;
+        }
+        Require(kept, "a drawn linear target's store lost the drawn half or the CPU's later stores to the other half");
+        std::puts("linear target CPU write tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetDirectWriteCompletion.cpp

@@ -0,0 +1,452 @@
+#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <bit>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <functional>
+#include <initializer_list>
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+
+constexpr std::uint32_t Width = 256;
+constexpr std::uint32_t Height = 128;
+constexpr std::size_t Pitch = std::size_t{Width} * 4u;
+constexpr std::size_t TargetBytes = Pitch * Height;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t RegionBytes = 16 * BlockBytes;
+constexpr std::size_t LabelOffset = 3 * BlockBytes;
+constexpr std::uint32_t Untouched = 0x5a5a5a5au;
+constexpr std::uint32_t Filled = 0x9abcdef0u;
+constexpr std::uint32_t Drawn = 0xff0000ffu;
+constexpr std::uint32_t LabelValue = 0x600dcafeu;
+constexpr std::uint32_t CpuStored = 0x0badf00du;
+constexpr std::uint32_t Rgba8UnormInfo = 0x28028u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t WaitEqual = 3;
+constexpr std::uint32_t WaitAtLeast = 5;
+constexpr std::array<std::array<std::uint32_t, 4>, 2> Scissors{{{16, 8, 128, 96}, {128, 32, 240, 120}}};
+constexpr std::array<std::uint32_t, 4> BlockZeroScissor{24, 8, 232, 56};
+constexpr std::size_t CpuStoreOffset = BlockBytes + 32 * Pitch + 64 * 4;
+static_assert(BlockZeroScissor[3] * Pitch <= BlockBytes);
+static_assert(CpuStoreOffset >= BlockBytes && CpuStoreOffset < TargetBytes);
+
+alignas(256) constexpr std::array<std::uint32_t, 20> VertexCode{
+    0x7e000d05u, 0x080200f4u, 0x10020300u, 0x100202f4u, 0x060202f3u, 0x060400f3u, 0x10040500u, 0x060404f3u,
+    0x7e060208u, 0x7e0802f2u, 0x7e0a02ffu, 0x44800000u, 0xf80000d1u, 0x00000005u, 0xf80008cfu, 0x04030201u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u,
+};
+alignas(256) constexpr std::array<std::uint32_t, 8> PixelCode{0x7e000200u, 0x7e020280u, 0xf800180fu, 0x00010100u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 9> FillCode{
+    0xd7460004, 0x04010c08, 0x7e000204, 0x7e020205, 0x7e040206, 0x7e060207, 0xe01c2000, 0x80000004, 0xbf810000,
+};
+alignas(256) constexpr std::array<std::uint32_t, 11> PairFillCode{
+    0xd7460004, 0x04010c0c, 0x7e000204, 0x7e020205, 0x7e040206, 0x7e060207, 0xe01c2000, 0x80000004, 0xe01c2000, 0x80020004, 0xbf810000,
+};
+
+alignas(4) std::atomic<std::uint32_t> Gate = 0;
+
+struct Header {
+    Shader shader{};
+    ShaderUserData userData{};
+};
+
+alignas(8) Header VertexHeader{};
+alignas(8) Header PixelHeader{};
+
+void Registered(Header& header, const std::uint32_t* code, std::size_t bytes, std::uint8_t type) {
+    header.shader.file_header = 0x34333231u;
+    header.shader.version = 0x18u;
+    header.shader.header_size = sizeof(Header);
+    header.shader.shader_size = static_cast<std::uint32_t>(bytes);
+    header.shader.code = code;
+    header.shader.type = type;
+    header.shader.user_data = &header.userData;
+    AgcDriverRegisterShader_nid_postfix(&header.shader);
+}
+
+void Append(std::vector<std::uint32_t>& commands, std::uint32_t opcode, std::initializer_list<std::uint32_t> payload) {
+    commands.push_back(0xc0000000u | (static_cast<std::uint32_t>(payload.size() - 1) << 16u) | (opcode << 8u));
+    commands.insert(commands.end(), payload);
+}
+
+void AppendDraw(std::vector<std::uint32_t>& commands, std::uint64_t target, const std::array<std::uint32_t, 4>& scissor) {
+    const auto vertex = reinterpret_cast<std::uintptr_t>(VertexCode.data());
+    const auto pixel = reinterpret_cast<std::uintptr_t>(PixelCode.data());
+    const auto extent = (Height << 16u) | Width;
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> context{
+        {0x2d5, 0x2000},
+        {0x1b6, 0}, {0x207, 0x01200000}, {0x200, 0}, {0x203, 0x800},
+        {0x2dc, 0xaa00}, {0x2f8, 0}, {0x292, 2}, {0x293, 0},
+        {0x80, 0}, {0x8d, 0}, {0x83, 0xffff}, {0x8c, 0xa},
+        {0x2f9, 0x2d}, {0x313, 0x6000}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff},
+        {0x206, 0x43f}, {0x204, 0x80000}, {0x205, 0x240},
+        {0x8e, 0xf}, {0x8f, 0xf}, {0x202, 0xcc0010},
+        {0x1c4, 0}, {0x1c5, 9}, {0x1c3, 4}, {0x31c, Rgba8UnormInfo},
+        {0x31b, 0}, {0x31d, 0}, {0x3b0, ((Width - 1u) << 14u) | (Height - 1u)},
+        {0x3b8, LinearAttrib3}, {0x1e0, 0},
+        {0xc, 0}, {0xd, extent},
+        {0x81, 0x80000000}, {0x82, extent},
+        {0x90, 0x80000000u | (scissor[1] << 16u) | scissor[0]}, {0x91, (scissor[3] << 16u) | scissor[2]},
+        {0x94, 0x80000000}, {0x95, extent},
+        {0x10f, std::bit_cast<std::uint32_t>(2.0f * Width)}, {0x110, std::bit_cast<std::uint32_t>(2.0f * Width)},
+        {0x111, std::bit_cast<std::uint32_t>(2.0f * Height)}, {0x112, std::bit_cast<std::uint32_t>(2.0f * Height)},
+        {0x113, std::bit_cast<std::uint32_t>(1.0f)}, {0x114, 0},
+        {0xb4, 0}, {0xb5, std::bit_cast<std::uint32_t>(1.0f)},
+        {0x2fe, 0}, {0x302, 0}, {0x306, 0}, {0x30a, 0},
+        {0x318, static_cast<std::uint32_t>(target >> 8u)}, {0x390, static_cast<std::uint32_t>(target >> 40u)},
+    };
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> shader{
+        {0xc8, static_cast<std::uint32_t>(vertex >> 8u)}, {0xc9, static_cast<std::uint32_t>(vertex >> 40u)}, {0x8b, 1u << 1u}, {0x8c, 0},
+        {0x008, static_cast<std::uint32_t>(pixel >> 8u)}, {0x009, static_cast<std::uint32_t>(pixel >> 40u)}, {0x00b, 1u << 1u}, {0x00c, std::bit_cast<std::uint32_t>(1.0f)},
+    };
+    Append(commands, 0x79, {0x242, 4});
+    Append(commands, 0x79, {0x24a, 0});
+    for (const auto& [offset, value] : context) Append(commands, 0x69, {offset, value});
+    for (const auto& [offset, value] : shader) Append(commands, 0x76, {offset, value});
+    Append(commands, 0x2d, {3, 2});
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(std::uint64_t address, std::size_t bytes) {
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (16u << 16u), static_cast<std::uint32_t>(bytes / 16u), 0x4bfacu};
+}
+
+void AppendDispatch(std::vector<std::uint32_t>& commands, const std::uint32_t* program, const std::vector<std::uint32_t>& userData, std::uint32_t groups) {
+    const auto code = reinterpret_cast<std::uintptr_t>(program);
+    Append(commands, 0x76, {0x207, 64, 1, 1});
+    Append(commands, 0x76, {0x20c, static_cast<std::uint32_t>(code >> 8u), static_cast<std::uint32_t>(code >> 40u)});
+    Append(commands, 0x76, {0x213, (static_cast<std::uint32_t>(userData.size()) << 1u) | 0x80u});
+    commands.push_back(0xc0007600u | (static_cast<std::uint32_t>(userData.size()) << 16u));
+    commands.push_back(0x240u);
+    commands.insert(commands.end(), userData.begin(), userData.end());
+    commands.insert(commands.end(), {0xc0031500u, groups, 1, 1, 0x41u});
+}
+
+void AppendFill(std::vector<std::uint32_t>& commands, std::uint64_t target, const std::uint32_t* copied, std::size_t bytes) {
+    const auto direct = BufferDescriptor(target, bytes);
+    const auto other = BufferDescriptor(reinterpret_cast<std::uintptr_t>(copied), bytes);
+    std::vector<std::uint32_t> userData(direct.begin(), direct.end());
+    userData.insert(userData.end(), {Filled, Filled, Filled, Filled});
+    userData.insert(userData.end(), other.begin(), other.end());
+    AppendDispatch(commands, PairFillCode.data(), userData, static_cast<std::uint32_t>(bytes / 1024u));
+}
+
+void AppendLabel(std::vector<std::uint32_t>& commands, std::uint64_t label) {
+    commands.insert(commands.end(), {0xc0064900u, 0x514u, 1u << 29u, static_cast<std::uint32_t>(label), static_cast<std::uint32_t>(label >> 32u), LabelValue, 0, 0});
+}
+
+void AppendGate(std::vector<std::uint32_t>& commands, std::uint32_t function, std::uint32_t reference) {
+    const auto gate = reinterpret_cast<std::uintptr_t>(&Gate);
+    commands.insert(commands.end(), {0xc0053c00u, 0x10u | function, static_cast<std::uint32_t>(gate), static_cast<std::uint32_t>(gate >> 32u), reference, 0xffffffffu, 0x19u});
+}
+
+void Submit(std::vector<std::uint32_t>& commands) {
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    AgcDriver::Submit(&packet, 0);
+}
+
+class Region {
+public:
+    explicit Region(std::size_t bytes) : bytes(bytes), arena(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (arena) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            if (data != nullptr) GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (arena) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "linear target direct write completion: cannot map the guest region");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "linear target direct write completion: cannot allocate the guest region");
+        auto* words = reinterpret_cast<std::uint32_t*>(data);
+        std::fill(words, words + bytes / 4u, Untouched);
+        GuestAllocations::Mutation().Add(data, bytes, true, true, true);
+    }
+    ~Region() {
+        GuestAllocations::Mutation().Remove(data);
+#ifdef _WIN32
+        if (arena) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (arena) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Region(const Region&) = delete;
+    Region& operator=(const Region&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::uint32_t Word(std::size_t offset) const { return std::atomic_ref<std::uint32_t>(*reinterpret_cast<std::uint32_t*>(data + offset)).load(std::memory_order_acquire); }
+    void Put(std::size_t offset, std::uint32_t value) const { std::atomic_ref<std::uint32_t>(*reinterpret_cast<std::uint32_t*>(data + offset)).store(value, std::memory_order_release); }
+    std::size_t bytes;
+
+private:
+    bool arena;
+    std::uint8_t* data = nullptr;
+};
+
+void Import(const Region& region) {
+    const auto buffer = BufferDescriptor(region.Address(), region.bytes);
+    std::vector<std::uint32_t> userData(buffer.begin(), buffer.end());
+    userData.insert(userData.end(), {Untouched, Untouched, Untouched, Untouched});
+    std::vector<std::uint32_t> commands;
+    AppendDispatch(commands, FillCode.data(), userData, static_cast<std::uint32_t>(region.bytes / 1024u));
+    Submit(commands);
+    AgcDriverWaitIdle_nid_postfix();
+}
+
+bool Covered(std::uint32_t x, std::uint32_t y) {
+    return std::any_of(Scissors.begin(), Scissors.end(), [&](const std::array<std::uint32_t, 4>& scissor) { return x >= scissor[0] && x < scissor[2] && y >= scissor[1] && y < scissor[3]; });
+}
+
+std::string Mismatches(const Region& region, bool drawsLast) {
+    std::size_t covered = 0, wrongCovered = 0, filledCovered = 0, wrongElsewhere = 0, drawnElsewhere = 0;
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const auto value = region.Word(y * Pitch + x * 4u);
+            if (drawsLast && Covered(x, y)) {
+                ++covered;
+                wrongCovered += value == Drawn ? 0u : 1u;
+                filledCovered += value == Filled ? 1u : 0u;
+            } else {
+                wrongElsewhere += value == Filled ? 0u : 1u;
+                drawnElsewhere += value == Drawn ? 1u : 0u;
+            }
+        }
+    }
+    if (wrongCovered == 0 && wrongElsewhere == 0) return {};
+    const auto elsewhere = std::to_string(std::size_t{Width} * Height - covered);
+    if (!drawsLast) return std::to_string(wrongElsewhere) + " of the " + elsewhere + " pixels do not hold the fill's pattern (" + std::to_string(drawnElsewhere) + " hold the draws' color)";
+    return std::to_string(wrongCovered) + " of the " + std::to_string(covered) + " pixels the draws covered do not hold their color (" + std::to_string(filledCovered) + " hold the fill's pattern) and " + std::to_string(wrongElsewhere) + " of the " + elsewhere + " pixels outside them do not hold the fill's pattern";
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+std::string CpuStoreMismatches(const Region& region) {
+    std::size_t covered = 0, wrongCovered = 0, filledCovered = 0, others = 0, wrongOthers = 0;
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const auto offset = y * Pitch + x * 4u;
+            if (offset == CpuStoreOffset) continue;
+            const auto value = region.Word(offset);
+            if (x >= BlockZeroScissor[0] && x < BlockZeroScissor[2] && y >= BlockZeroScissor[1] && y < BlockZeroScissor[3]) {
+                ++covered;
+                wrongCovered += value == Drawn ? 0u : 1u;
+                filledCovered += value == Filled ? 1u : 0u;
+            } else {
+                ++others;
+                wrongOthers += value == (offset < BlockBytes ? Filled : Untouched) ? 0u : 1u;
+            }
+        }
+    }
+    std::vector<std::string> parts;
+    if (wrongCovered != 0) parts.push_back(std::to_string(wrongCovered) + " of the " + std::to_string(covered) + " pixels the draw covered in block 0 do not hold its color (" + std::to_string(filledCovered) + " hold the fill's pattern)");
+    if (const auto stored = region.Word(CpuStoreOffset); stored != CpuStored) parts.push_back("the CPU store in block 1 holds " + Hex(stored) + ", not " + Hex(CpuStored));
+    if (wrongOthers != 0) parts.push_back(std::to_string(wrongOthers) + " of the " + std::to_string(others) + " other pixels do not hold the fill's pattern in block 0 or the untouched pattern in block 1");
+    std::string text;
+    for (const auto& part : parts) text += (text.empty() ? "" : "; ") + part;
+    return text;
+}
+
+bool Await(const std::function<bool()>& done) {
+    const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5);
+    while (!done()) {
+        if (std::chrono::steady_clock::now() >= deadline) return false;
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+    return true;
+}
+
+enum class Order { FillFirst, DrawsFirst };
+
+bool Run(Order order, const Region& region, std::vector<std::string>& failures) {
+    const bool fillFirst = order == Order::FillFirst;
+    const char* name = fillFirst ? "fill, then draws" : "draws, then fill";
+    const auto target = region.Address();
+    Import(region);
+    if (!AgcDriver::GuestMemory::Watched(target, TargetBytes)) return false;
+    std::vector<std::uint32_t> copied(TargetBytes / 4u + 4u, Untouched);
+    GuestAllocations::Mutation().Add(copied.data(), TargetBytes + 16u, true, true, true);
+    struct Unregister {
+        void* data;
+        ~Unregister() { GuestAllocations::Mutation().Remove(data); }
+    } unregister{copied.data()};
+    Gate.store(0, std::memory_order_release);
+    std::vector<std::uint32_t> commands;
+    if (fillFirst) AppendFill(commands, target, copied.data(), TargetBytes);
+    for (const auto& scissor : Scissors) AppendDraw(commands, target, scissor);
+    if (!fillFirst) AppendFill(commands, target, copied.data(), TargetBytes);
+    AppendLabel(commands, target + LabelOffset);
+    if (fillFirst) AppendGate(commands, WaitEqual, 1);
+    Submit(commands);
+    std::string failure;
+    if (fillFirst) {
+        if (!Await([&] { return region.Word(LabelOffset) == LabelValue; })) failure = "the label after the draws never landed";
+        else if (const auto mismatches = Mismatches(region, true); !mismatches.empty()) failure = "when the label after the draws landed: " + mismatches;
+        Gate.store(1, std::memory_order_release);
+        AgcDriverWaitIdle_nid_postfix();
+    } else {
+        AgcDriverWaitIdle_nid_postfix();
+        if (region.Word(LabelOffset) != LabelValue) failure = "the label after the fill had not landed by the device wait";
+        else if (const auto mismatches = Mismatches(region, false); !mismatches.empty()) failure = "after the label and the device wait: " + mismatches;
+    }
+    AgcDriver::Graphics::StorageTexture::FlushPending(target, region.bytes, nullptr, "test release");
+    std::printf("linear target direct write completion, %s: %s\n", name, failure.empty() ? "the later writer's values" : failure.c_str());
+    if (!failure.empty()) failures.push_back(std::string(name) + ": " + failure);
+    return true;
+}
+
+bool RunCpuStore(bool watched, const Region& region, std::vector<std::string>& failures) {
+    const std::string name = std::string("fill, draw in block 0, CPU store in block 1, ") + (watched ? "write-watched" : "unwatched") + " memory";
+    const auto target = region.Address();
+    Import(region);
+    if (watched && !AgcDriver::GuestMemory::Watched(target, TargetBytes)) return false;
+    if (!watched) {
+        AgcDriver::GuestMemory::Unwatch(target, region.bytes);
+        Require(!AgcDriver::GuestMemory::Watched(target, region.bytes), "linear target direct write completion: the region is still write-watched");
+    }
+    std::vector<std::uint32_t> copied(TargetBytes / 4u + 4u, Untouched);
+    GuestAllocations::Mutation().Add(copied.data(), TargetBytes + 16u, true, true, true);
+    struct Unregister {
+        void* data;
+        ~Unregister() { GuestAllocations::Mutation().Remove(data); }
+    } unregister{copied.data()};
+    Gate.store(0, std::memory_order_release);
+    std::vector<std::uint32_t> commands;
+    AppendFill(commands, target, copied.data(), BlockBytes);
+    AppendDraw(commands, target, BlockZeroScissor);
+    AppendGate(commands, WaitAtLeast, 1);
+    AppendLabel(commands, target + LabelOffset);
+    AppendGate(commands, WaitAtLeast, 2);
+    Submit(commands);
+    const auto completed = [&] { return std::atomic_ref<std::uint32_t>(copied[0]).load(std::memory_order_acquire) == Filled; };
+    bool pendingAtStore = false;
+    const bool recorded = Await([&] {
+        std::unique_lock hold(AgcDriver::GuestMemory::GpuMutex());
+        if (AgcDriver::Graphics::StorageTexture::FindLive(target, TargetBytes) == nullptr) return false;
+        pendingAtStore = !completed();
+        region.Put(CpuStoreOffset, CpuStored);
+        return true;
+    });
+    Gate.store(1, std::memory_order_release);
+    std::string failure;
+    bool completedAtLabel = false;
+    if (!recorded) failure = "the draw into block 0 never made the target's image";
+    else if (!Await([&] { return region.Word(LabelOffset) == LabelValue; })) failure = "the label after the draw never landed";
+    else {
+        completedAtLabel = completed();
+        failure = CpuStoreMismatches(region);
+        if (!failure.empty()) failure = "when the label after the draw landed: " + failure;
+        if (watched && (!pendingAtStore || !completedAtLabel)) failure += std::string(failure.empty() ? "" : "; ") + (pendingAtStore ? "the fill's completion had not run when the label landed" : "the fill's completion had run before the CPU store") + ", so the ordering under test did not occur";
+    }
+    const bool watchedAtLabel = AgcDriver::GuestMemory::Watched(target, TargetBytes);
+    Gate.store(2, std::memory_order_release);
+    AgcDriverWaitIdle_nid_postfix();
+    AgcDriver::Graphics::StorageTexture::FlushPending(target, region.bytes, nullptr, "test release");
+    std::printf("linear target direct write completion, %s (watched at the label: %d, completion pending at the CPU store: %d, run by the label: %d): %s\n", name.c_str(), watchedAtLabel ? 1 : 0, pendingAtStore ? 1 : 0, completedAtLabel ? 1 : 0, failure.empty() ? "the draw's color and the CPU store" : failure.c_str());
+    if (!failure.empty()) failures.push_back(name + ": " + failure);
+    return true;
+}
+
+}
+
+int main() {
+    try {
+        {
+            const auto device = OpenVulkanTestDevice();
+            if (!device) return VulkanTestSkipped;
+            if (AgcDriver::VulkanDeviceTestAccess::ContextOf(*device).hostImportAlignment == 0) {
+                std::puts("linear target direct write completion tests skipped, host imports are unavailable on this device, so the target is never resident");
+                return VulkanTestSkipped;
+            }
+        }
+        std::vector<std::unique_ptr<Region>> regions;
+        struct ShutdownGuard {
+            ~ShutdownGuard() { try { AgcDriverShutdown_nid_postfix(); } catch (...) {} }
+        } shutdown;
+        Registered(VertexHeader, VertexCode.data(), sizeof(VertexCode), 2);
+        Registered(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1);
+        const auto unwatchedImports = [] {
+            std::puts("linear target direct write completion tests skipped, the device does not keep host imports write-watched, so no completion stamp over the target is tracked");
+            return VulkanTestSkipped;
+        };
+        std::vector<std::string> failures;
+        for (const auto order : {Order::FillFirst, Order::DrawsFirst}) {
+            regions.push_back(std::make_unique<Region>(RegionBytes));
+            if (!Run(order, *regions.back(), failures)) return unwatchedImports();
+        }
+        for (const bool watched : {true, false}) {
+            regions.push_back(std::make_unique<Region>(RegionBytes));
+            if (!RunCpuStore(watched, *regions.back(), failures)) return unwatchedImports();
+        }
+        if (!failures.empty()) {
+            std::string message = "linear target direct write completion: at a completion point, the memory of a resident linear target did not hold the later of a direct buffer write with completion work and draws into it, or lost a CPU store made outside the draws:";
+            for (const auto& failure : failures) message += "\n  " + failure;
+            Require(false, message);
+        }
+        AgcDriverShutdown_nid_postfix();
+        std::puts("linear target direct write completion tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetDispatchAccess.cpp

@@ -0,0 +1,300 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using AgcDriver::Graphics::StorageTexture;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 16;
+constexpr std::uint32_t Threads = 64;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t TargetBytes = std::size_t{Width} * Height * 16u;
+constexpr std::uint32_t Rgba32FloatInfo = 0x8738u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t Float32AbgrExport = 9;
+constexpr std::uint32_t ReadX = 10;
+constexpr std::uint32_t ReadY = 6;
+constexpr std::uint32_t Drawn = 0x3f800000u;
+constexpr std::uint32_t Unwritten = 0xdeadbeefu;
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 7> CopyCode{0x7e000280u, 0xdc308000u, 0x01000000u, 0xbf8c3f70u, 0xdc708000u, 0x00020100u, 0xbf810000u};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::array<std::uint32_t, 16>, 2> Programs{};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+std::span<const std::uint32_t> Program(std::size_t index, bool blue) {
+    auto& code = Programs.at(index);
+    std::size_t at = 0;
+    code[at++] = 0x36000000u | (20u << 17u) | 0xffu;
+    code[at++] = 0x0000ffffu;
+    code[at++] = 0x2c000000u | (21u << 17u) | 0x90u;
+    code[at++] = 0x7e000c00u | (20u << 17u) | (256u + 20u);
+    code[at++] = 0x7e000c00u | (21u << 17u) | (256u + 21u);
+    code[at++] = (blue ? 0x7e0002f2u : 0x7e000280u) | (22u << 17u);
+    code[at++] = 0x7e0002f2u | (23u << 17u);
+    code[at++] = 0xf800180fu;
+    code[at++] = 0x17161514u;
+    code[at++] = 0xbf810000u;
+    return std::span<const std::uint32_t>(code.data(), at);
+}
+
+struct Block {
+    explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (watched) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (watched) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map a guest block");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "cannot allocate a guest block");
+        Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "a guest block is not write-watched");
+        GuestAllocations::Mutation mutation;
+        mutation.Add(data, bytes, true, true, true);
+    }
+    ~Block() {
+        StorageTexture::FlushPending(Address(), bytes, nullptr, "test release");
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Remove(data);
+        }
+#ifdef _WIN32
+        if (watched) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (watched) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Block(const Block&) = delete;
+    Block& operator=(const Block&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::size_t bytes;
+    bool watched;
+    std::uint8_t* data = nullptr;
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block) {
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(block.Address() >> 8u);
+    cx[0x31b] = 0;
+    cx[0x31c] = Rgba32FloatInfo;
+    cx[0x31d] = 0;
+    cx[0x323] = 0;
+    cx[0x324] = 0;
+    cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u);
+    cx[0x3b8] = LinearAttrib3;
+    cx[0x390] = static_cast<std::uint32_t>(block.Address() >> 40u);
+    auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    color.exportIndex = 0;
+    return color;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::span<const std::uint32_t> pixelCode, VkRect2D scissor) {
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast<std::uint32_t>(Triangle.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionFixedPoint);
+    pixel.targetOutputMode.at(0) = Float32AbgrExport;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::vector<std::uint32_t> pixelUserData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.blendEnable = VK_FALSE;
+    blend.colorWriteMask = 0xf;
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = scissor;
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, std::uint64_t source, std::uint64_t destination) {
+    constexpr std::uint32_t waveSize = 64;
+    const std::array<std::uint32_t, 4> userData{static_cast<std::uint32_t>(source), static_cast<std::uint32_t>(source >> 32u), static_cast<std::uint32_t>(destination), static_cast<std::uint32_t>(destination >> 32u)};
+    const std::span<const std::uint32_t> code(CopyCode);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+std::uint32_t Dword(const std::uint8_t* bytes) {
+    std::uint32_t value = 0;
+    std::memcpy(&value, bytes, sizeof(value));
+    return value;
+}
+
+bool Expect(AgcDriver::VulkanDevice& device) {
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    if (!context.bufferDeviceAddress) {
+        std::puts("skipped, the device lacks bufferDeviceAddress");
+        return false;
+    }
+    Block block(16 * BlockBytes);
+    std::fill(block.data, block.data + block.bytes, std::uint8_t{0});
+    Block output(BlockBytes);
+    for (std::size_t offset = 0; offset < output.bytes; offset += sizeof(Unwritten)) std::memcpy(output.data + offset, &Unwritten, sizeof(Unwritten));
+    if (AgcDriver::Graphics::HostImportFor(context, block.Address(), block.bytes) == nullptr) {
+        std::puts("skipped, host imports are unavailable");
+        return false;
+    }
+    const auto color = DecodeTarget(block);
+    Require(color.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && color.bytes == TargetBytes, "the target is not a linear R32G32B32A32 target");
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(recorder != nullptr, "the device has no recorder");
+    Draw(device, color, Program(0, false), {{0, 0}, {Width, Height}});
+    device.WaitIdle();
+    Require(StorageTexture::FindPending(block.Address(), TargetBytes) == nullptr, "the linear target was still pending after the device wait");
+    const auto blueOffset = (static_cast<std::size_t>(ReadY) * Width + ReadX) * 16u + 8u;
+    Require(Dword(block.data + blueOffset) == 0u, "the first draw left " + Hex(Dword(block.data + blueOffset)) + " in the blue channel of pixel (" + std::to_string(ReadX) + ", " + std::to_string(ReadY) + ")");
+    Draw(device, color, Program(1, true), {{8, 4}, {16, 8}});
+    Require(recorder->Recording(), "the draw into the linear target was not left in the open batch");
+    Dispatch(device, block.Address() + blueOffset, output.Address());
+    device.WaitIdle();
+    const auto read = Dword(output.data);
+    Require(read == Drawn, "the address-based dispatch recorded after the draw read " + Hex(read) + " from the blue channel of pixel (" + std::to_string(ReadX) + ", " + std::to_string(ReadY) + "), not the drawn " + Hex(Drawn));
+    Require(Dword(block.data + blueOffset) == Drawn, "the second draw left " + Hex(Dword(block.data + blueOffset)) + " in the blue channel of pixel (" + std::to_string(ReadX) + ", " + std::to_string(ReadY) + ")");
+    return true;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (!Expect(*device)) return VulkanTestSkipped;
+        std::puts("linear target dispatch access tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetDrawBindings.cpp

@@ -0,0 +1,334 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using AgcDriver::Graphics::StorageTexture;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 16;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t TargetBytes = std::size_t{Width} * Height * 16u;
+constexpr std::size_t CopyTargetOffset = 0x40000;
+constexpr std::size_t FirstSourceOffset = 0x80000;
+constexpr std::size_t SecondSourceOffset = 0x90000;
+constexpr std::size_t PointedOffset = 0xa0000;
+constexpr std::uint32_t Rgba32FloatInfo = 0x8738u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t Float32AbgrExport = 9;
+constexpr std::uint32_t FirstSource = 0x3e800000u;
+constexpr std::uint32_t SecondSource = 0x3f400000u;
+constexpr std::uint32_t Pointed = 0x3f800000u;
+constexpr std::uint32_t Unwritten = 0xdeadbeefu;
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 7> PixelOpaque{0x7e100280u, 0x7e120280u, 0x7e140280u, 0x7e1602f2u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 11> PixelRead{0x7e1a0280u, 0xdc308000u, 0x0c02000du, 0xbf8c3f70u, 0x7e10030cu, 0x7e120280u, 0x7e140280u, 0x7e160280u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 13> PixelCopy{0x7e040280u, 0xe0301000u, 0x80010102u, 0xbf8c3f70u, 0xe0701000u, 0x80020102u, 0x7e100301u, 0x7e120280u, 0x7e140280u, 0x7e1602f2u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::uint32_t, 4> FirstOutput{};
+alignas(256) std::array<std::uint32_t, 4> SecondOutput{};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+std::array<std::uint32_t, 4> RawBuffer(std::uint64_t address, std::uint32_t bytes) {
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+struct Block {
+    explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (watched) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (watched) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map the color targets");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "cannot allocate the color targets");
+        Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color targets are not write-watched");
+        GuestAllocations::Mutation mutation;
+        mutation.Add(data, bytes, true, true, true);
+    }
+    ~Block() {
+        StorageTexture::FlushPending(Address(), bytes, nullptr, "test release");
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Remove(data);
+        }
+#ifdef _WIN32
+        if (watched) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (watched) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Block(const Block&) = delete;
+    Block& operator=(const Block&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::size_t bytes;
+    bool watched;
+    std::uint8_t* data = nullptr;
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(std::uint64_t address) {
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(address >> 8u);
+    cx[0x31b] = 0;
+    cx[0x31c] = Rgba32FloatInfo;
+    cx[0x31d] = 0;
+    cx[0x323] = 0;
+    cx[0x324] = 0;
+    cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u);
+    cx[0x3b8] = LinearAttrib3;
+    cx[0x390] = static_cast<std::uint32_t>(address >> 40u);
+    auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    color.exportIndex = 0;
+    return color;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::span<const std::uint32_t> pixelCode, std::span<const std::uint32_t> pixelUserData, VkRect2D scissor) {
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast<std::uint32_t>(Triangle.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionFixedPoint);
+    pixel.targetOutputMode.at(0) = Float32AbgrExport;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.blendEnable = VK_FALSE;
+    blend.colorWriteMask = 0xf;
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = scissor;
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+}
+
+std::array<std::uint32_t, 12> CopyUserData(std::uint64_t source, const std::array<std::uint32_t, 4>& output) {
+    std::array<std::uint32_t, 12> userData{};
+    const auto read = RawBuffer(source, 16u);
+    const auto written = RawBuffer(reinterpret_cast<std::uintptr_t>(output.data()), 16u);
+    std::copy(read.begin(), read.end(), userData.begin() + 4);
+    std::copy(written.begin(), written.end(), userData.begin() + 8);
+    return userData;
+}
+
+std::array<float, 4> Texel(const std::uint8_t* target, std::uint32_t x, std::uint32_t y) {
+    std::array<float, 4> texel{};
+    std::memcpy(texel.data(), target + (static_cast<std::size_t>(y) * Width + x) * 16u, sizeof(texel));
+    return texel;
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+std::string Describe(const std::array<float, 4>& texel) {
+    return "(" + std::to_string(texel[0]) + ", " + std::to_string(texel[1]) + ", " + std::to_string(texel[2]) + ", " + std::to_string(texel[3]) + ")";
+}
+
+void ExpectTexels(const std::uint8_t* target, const char* what, std::uint32_t firstX, std::uint32_t lastX, const std::array<float, 4>& expected) {
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = firstX; x < lastX; ++x) {
+            const auto texel = Texel(target, x, y);
+            Require(texel == expected, std::string(what) + ": pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") holds " + Describe(texel) + ", expected " + Describe(expected));
+        }
+    }
+}
+
+bool Expect(AgcDriver::VulkanDevice& device, bool unwatch) {
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    if (!context.bufferDeviceAddress) {
+        std::puts("skipped, the device lacks bufferDeviceAddress");
+        return false;
+    }
+    Block block(16 * BlockBytes);
+    std::fill(block.data, block.data + block.bytes, std::uint8_t{0});
+    const auto put = [&](std::size_t offset, std::uint32_t value, std::size_t dwords) {
+        for (std::size_t dword = 0; dword < dwords; ++dword) std::memcpy(block.data + offset + dword * 4u, &value, sizeof(value));
+    };
+    put(FirstSourceOffset, FirstSource, 4);
+    put(SecondSourceOffset, SecondSource, 4);
+    put(PointedOffset, Pointed, 1);
+    FirstOutput.fill(Unwritten);
+    SecondOutput.fill(Unwritten);
+    if (AgcDriver::Graphics::HostImportFor(context, block.Address(), block.bytes) == nullptr) {
+        std::puts("skipped, host imports are unavailable");
+        return false;
+    }
+    const auto drawn = DecodeTarget(block.Address());
+    const auto copied = DecodeTarget(block.Address() + CopyTargetOffset);
+    Require(drawn.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && drawn.bytes == TargetBytes && copied.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && copied.bytes == TargetBytes, "the targets are not linear R32G32B32A32 targets");
+    auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(recorder != nullptr && context.descriptorCache != nullptr, "the device has no recorder or descriptor cache");
+    const auto pending = [](std::uint64_t address) { return StorageTexture::FindPending(address, TargetBytes) != nullptr; };
+    const VkRect2D whole{{0, 0}, {Width, Height}};
+    Draw(device, drawn, PixelOpaque, {}, whole);
+    Draw(device, copied, PixelOpaque, {}, whole);
+    device.WaitIdle();
+    Require(!pending(drawn.address) && !pending(copied.address), "a linear target was still pending after the device wait");
+    if (unwatch) AgcDriver::GuestMemory::Unwatch(block.Address(), block.bytes);
+    const bool watched = AgcDriver::GuestMemory::Watched(copied.address, TargetBytes);
+    Require(!unwatch || !watched, "the copied target is still write-watched after its block was taken out of the write watch");
+    const auto submitted = recorder->Submissions();
+    const auto pointed = block.Address() + PointedOffset;
+    const std::array<std::uint32_t, 4> pointer{0u, 0u, static_cast<std::uint32_t>(pointed), static_cast<std::uint32_t>(pointed >> 32u)};
+    Draw(device, drawn, PixelRead, pointer, whole);
+    Require(pending(drawn.address), "the address-based draw did not leave its linear target pending");
+    Require(recorder->Recording() && recorder->Submissions() == submitted, "the address-based draw did not stay in the open batch");
+    const auto setsBefore = context.descriptorCache->Counters().sets;
+    const auto firstCopy = CopyUserData(block.Address() + FirstSourceOffset, FirstOutput);
+    Draw(device, copied, PixelCopy, firstCopy, {{0, 0}, {Width / 2u, Height}});
+    Require(!pending(drawn.address), "the draw writing a copied buffer did not store the drawn linear target first");
+    Require(pending(copied.address), "the first copying draw did not leave its linear target pending");
+    Require(recorder->Recording() && recorder->Submissions() == submitted + 1, "storing the drawn linear target did not submit the batch the address-based draw read it in (" + std::to_string(recorder->Submissions() - submitted) + " submissions)");
+    const auto setsBetween = context.descriptorCache->Counters().sets;
+    const auto secondCopy = CopyUserData(block.Address() + SecondSourceOffset, SecondOutput);
+    Draw(device, copied, PixelCopy, secondCopy, {{static_cast<std::int32_t>(Width / 2u), 0}, {Width / 2u, Height}});
+    if (watched) Require(recorder->Recording() && recorder->Submissions() == submitted + 1, "the second copying draw submitted recorded work");
+    else Require(recorder->Submissions() == submitted + 2, "the second copying draw into a stored target outside the write watch did not submit once to capture its bytes (" + std::to_string(recorder->Submissions() - submitted - 1) + " submissions)");
+    const auto setsAfter = context.descriptorCache->Counters().sets;
+    device.WaitIdle();
+    Require(FirstOutput[0] == FirstSource, "the first copying draw's output holds " + Hex(FirstOutput[0]) + ", not its source's " + Hex(FirstSource));
+    Require(SecondOutput[0] == SecondSource, "the second copying draw's output holds " + Hex(SecondOutput[0]) + ", not its source's " + Hex(SecondSource));
+    ExpectTexels(block.data + CopyTargetOffset, "the first copying draw's half", 0, Width / 2u, {0.25f, 0.0f, 0.0f, 1.0f});
+    ExpectTexels(block.data + CopyTargetOffset, "the second copying draw's half", Width / 2u, Width, {0.75f, 0.0f, 0.0f, 1.0f});
+    ExpectTexels(block.data, "the address-based draw's target", 0, Width, {1.0f, 0.0f, 0.0f, 0.0f});
+    Require(setsAfter - setsBetween == setsBetween - setsBefore, "the second copying draw allocated " + std::to_string(setsAfter - setsBetween) + " descriptor sets, the first " + std::to_string(setsBetween - setsBefore));
+    return true;
+}
+
+}
+
+int main() {
+#ifdef _WIN32
+    _putenv_s("APS5_RELEASE_ON_UNLOCK", "1");
+#else
+    setenv("APS5_RELEASE_ON_UNLOCK", "1", 1);
+#endif
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (!Expect(*device, false)) return VulkanTestSkipped;
+        if (!Expect(*device, true)) return VulkanTestSkipped;
+        std::puts("linear target draw binding tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetRectangleBound.cpp

@@ -0,0 +1,344 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t RegionBytes = 8 * BlockBytes;
+constexpr std::uint32_t Rgba32FloatInfo = 0x8738u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t Float32AbgrExport = 9;
+constexpr std::uint8_t Initial = 0x3f;
+constexpr std::uint8_t Sentinel = 0x37;
+constexpr std::uint8_t Gap = 0x29;
+constexpr std::size_t GapBytes = 0x100;
+
+struct Surface {
+    std::size_t offset;
+    std::uint32_t width;
+    std::uint32_t height;
+    std::size_t RowBytes() const { return std::size_t{width} * 16u; }
+    std::size_t Bytes() const { return RowBytes() * height; }
+};
+
+constexpr Surface Tall{0, 64, 256};
+constexpr Surface Shifted{6 * BlockBytes + GapBytes, 64, 16};
+constexpr VkRect2D Seed{{63, 255}, {1, 1}};
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::array<std::uint32_t, 16>, 2> Programs{};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+std::span<const std::uint32_t> Program(std::size_t index, bool blue) {
+    auto& code = Programs.at(index);
+    std::size_t at = 0;
+    code[at++] = 0x36000000u | (20u << 17u) | 0xffu;
+    code[at++] = 0x0000ffffu;
+    code[at++] = 0x2c000000u | (21u << 17u) | 0x90u;
+    code[at++] = 0x7e000c00u | (20u << 17u) | (256u + 20u);
+    code[at++] = 0x7e000c00u | (21u << 17u) | (256u + 21u);
+    code[at++] = (blue ? 0x7e0002f2u : 0x7e000280u) | (22u << 17u);
+    code[at++] = 0x7e0002f2u | (23u << 17u);
+    code[at++] = 0xf800180fu;
+    code[at++] = 0x17161514u;
+    code[at++] = 0xbf810000u;
+    return std::span<const std::uint32_t>(code.data(), at);
+}
+
+struct Block {
+    explicit Block(std::size_t bytes) : bytes(bytes), arena(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (arena) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (arena) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map the color targets");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "cannot allocate the color targets");
+        GuestAllocations::Mutation mutation;
+        mutation.Add(data, bytes, true, true, true);
+    }
+    ~Block() {
+        AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release");
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Remove(data);
+        }
+#ifdef _WIN32
+        if (arena) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (arena) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Block(const Block&) = delete;
+    Block& operator=(const Block&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::size_t bytes;
+    bool arena;
+    std::uint8_t* data = nullptr;
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block, const Surface& surface) {
+    const auto address = block.Address() + surface.offset;
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(address >> 8u);
+    cx[0x31b] = 0;
+    cx[0x31c] = Rgba32FloatInfo;
+    cx[0x31d] = 0;
+    cx[0x323] = 0;
+    cx[0x324] = 0;
+    cx[0x3b0] = ((surface.width - 1u) << 14u) | (surface.height - 1u);
+    cx[0x3b8] = LinearAttrib3;
+    cx[0x390] = static_cast<std::uint32_t>(address >> 40u);
+    auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    color.exportIndex = 0;
+    return color;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, const Surface& surface, std::span<const std::uint32_t> pixelCode, VkRect2D scissor) {
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast<std::uint32_t>(Triangle.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionFixedPoint);
+    pixel.targetOutputMode.at(0) = Float32AbgrExport;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::vector<std::uint32_t> pixelUserData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.blendEnable = VK_FALSE;
+    blend.colorWriteMask = 0xf;
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = {surface.width, surface.height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(surface.height), static_cast<float>(surface.width), -static_cast<float>(surface.height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = scissor;
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+}
+
+bool Inside(const VkRect2D& rect, std::uint32_t x, std::uint32_t y) {
+    return x >= static_cast<std::uint32_t>(rect.offset.x) && x < static_cast<std::uint32_t>(rect.offset.x) + rect.extent.width && y >= static_cast<std::uint32_t>(rect.offset.y) && y < static_cast<std::uint32_t>(rect.offset.y) + rect.extent.height;
+}
+
+std::uint8_t* Pixel(const Block& block, const Surface& surface, std::uint32_t x, std::uint32_t y) {
+    return block.data + surface.offset + static_cast<std::size_t>(y) * surface.RowBytes() + static_cast<std::size_t>(x) * 16u;
+}
+
+bool Holds(const std::uint8_t* pixel, std::uint8_t value) {
+    return std::all_of(pixel, pixel + 16, [&](std::uint8_t byte) { return byte == value; });
+}
+
+bool HoldsOutput(const std::uint8_t* pixel, std::uint32_t x, std::uint32_t y, float z) {
+    std::array<float, 4> texel{};
+    std::memcpy(texel.data(), pixel, sizeof(texel));
+    return texel == std::array<float, 4>{static_cast<float>(x), static_cast<float>(y), z, 1.0f};
+}
+
+bool Expect(AgcDriver::VulkanDevice& device) {
+    Block block(RegionBytes);
+    std::fill(block.data, block.data + block.bytes, Initial);
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    if (AgcDriver::Graphics::HostImportFor(context, block.Address(), block.bytes) == nullptr) return false;
+    const auto tall = DecodeTarget(block, Tall);
+    const auto shifted = DecodeTarget(block, Shifted);
+    Require(tall.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && tall.bytes == Tall.Bytes() && shifted.bytes == Shifted.Bytes(), "the targets are not linear R32G32B32A32 targets");
+    constexpr VkRect2D first{{8, 4}, {16, 16}};
+    constexpr VkRect2D second{{16, 12}, {24, 16}};
+    constexpr VkRect2D whole{{0, 0}, {Shifted.width, Shifted.height}};
+    Draw(device, tall, Tall, Program(0, false), Seed);
+    device.WaitIdle();
+    Draw(device, tall, Tall, Program(0, false), first);
+    Draw(device, tall, Tall, Program(1, true), second);
+    Draw(device, shifted, Shifted, Program(0, false), whole);
+    const std::array<std::array<std::uint32_t, 2>, 3> sentinels{{{50, 10}, {20, 40}, {30, 200}}};
+    const std::array<std::array<std::uint32_t, 2>, 2> between{{{30, 6}, {10, 25}}};
+    for (const auto& [x, y] : sentinels) std::memset(Pixel(block, Tall, x, y), Sentinel, 16);
+    for (const auto& [x, y] : between) std::memset(Pixel(block, Tall, x, y), Sentinel, 16);
+    Require((block.Address() + Shifted.offset) / BlockBytes * BlockBytes == block.Address() + Shifted.offset - GapBytes, "the shifted target does not start past a 64 KiB boundary");
+    std::memset(block.data + Shifted.offset - GapBytes, Gap, GapBytes);
+    device.WaitIdle();
+    std::size_t wrongDrawn = 0, wrongSentinels = 0, wrongBetween = 0, wrongUntouched = 0, wrongShifted = 0;
+    for (std::uint32_t y = 0; y < Tall.height; ++y) {
+        for (std::uint32_t x = 0; x < Tall.width; ++x) {
+            const auto* pixel = Pixel(block, Tall, x, y);
+            const auto at = [&](const auto& point) { return point[0] == x && point[1] == y; };
+            if (std::any_of(sentinels.begin(), sentinels.end(), at)) wrongSentinels += Holds(pixel, Sentinel) ? 0u : 1u;
+            else if (std::any_of(between.begin(), between.end(), at)) wrongBetween += Holds(pixel, Sentinel) ? 0u : 1u;
+            else if (Inside(second, x, y)) wrongDrawn += HoldsOutput(pixel, x, y, 1.0f) ? 0u : 1u;
+            else if (Inside(first, x, y) || Inside(Seed, x, y)) wrongDrawn += HoldsOutput(pixel, x, y, 0.0f) ? 0u : 1u;
+            else wrongUntouched += Holds(pixel, Initial) ? 0u : 1u;
+        }
+    }
+    for (std::uint32_t y = 0; y < Shifted.height; ++y) {
+        for (std::uint32_t x = 0; x < Shifted.width; ++x) wrongShifted += HoldsOutput(Pixel(block, Shifted, x, y), x, y, 0.0f) ? 0u : 1u;
+    }
+    const auto gapLost = static_cast<std::size_t>(std::count_if(block.data + Shifted.offset - GapBytes, block.data + Shifted.offset, [](std::uint8_t byte) { return byte != Gap; }));
+    std::printf("linear target rectangle bound: wrong drawn pixels %zu, lost sentinels %zu/%zu, lost sentinels between the rectangles %zu/%zu, changed untouched pixels %zu, wrong shifted-target pixels %zu, lost gap bytes %zu/%zu\n", wrongDrawn, wrongSentinels, sentinels.size(), wrongBetween, between.size(), wrongUntouched, wrongShifted, gapLost, GapBytes);
+    Require(wrongDrawn == 0 && wrongShifted == 0, "a drawn linear target's store lost pixels its draws wrote");
+    Require(wrongSentinels == 0 && gapLost == 0, "a drawn linear target's store overwrote CPU stores outside the drawn rectangles");
+    Require(wrongBetween == 0, "a drawn linear target's store overwrote CPU stores between two draws' scissor rectangles");
+    Require(wrongUntouched == 0, "a drawn linear target's store changed pixels no draw and no CPU store wrote");
+    return true;
+}
+
+bool ExpectManyRows(AgcDriver::VulkanDevice& device) {
+    Block block(RegionBytes);
+    std::fill(block.data, block.data + block.bytes, Initial);
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    if (AgcDriver::Graphics::HostImportFor(context, block.Address(), block.bytes) == nullptr) return false;
+    const auto tall = DecodeTarget(block, Tall);
+    constexpr std::uint32_t Rows = 17;
+    constexpr std::uint32_t Spacing = 15;
+    Draw(device, tall, Tall, Program(0, false), Seed);
+    device.WaitIdle();
+    for (std::uint32_t row = 0; row < Rows; ++row) Draw(device, tall, Tall, Program(0, false), VkRect2D{{8, static_cast<std::int32_t>(row * Spacing)}, {16, 1}});
+    constexpr std::array<std::uint32_t, 2> between{10, 70};
+    std::memset(Pixel(block, Tall, between[0], between[1]), Sentinel, 16);
+    device.WaitIdle();
+    std::size_t wrongDrawn = 0, wrongUntouched = 0;
+    for (std::uint32_t y = 0; y < Tall.height; ++y) {
+        const bool drawnRow = y % Spacing == 0 && y / Spacing < Rows;
+        for (std::uint32_t x = 0; x < Tall.width; ++x) {
+            if (x == between[0] && y == between[1]) continue;
+            const auto* pixel = Pixel(block, Tall, x, y);
+            if ((drawnRow && x >= 8 && x < 24) || Inside(Seed, x, y)) wrongDrawn += HoldsOutput(pixel, x, y, 0.0f) ? 0u : 1u;
+            else wrongUntouched += Holds(pixel, Initial) ? 0u : 1u;
+        }
+    }
+    const bool kept = Holds(Pixel(block, Tall, between[0], between[1]), Sentinel);
+    std::printf("linear target rectangle bound, %u one-row scissors: wrong drawn pixels %zu, sentinel between the rows %s, changed untouched pixels %zu\n", Rows, wrongDrawn, kept ? "kept" : "lost", wrongUntouched);
+    Require(wrongDrawn == 0, "a drawn linear target's store lost pixels its one-row draws wrote");
+    Require(kept, "a drawn linear target's store overwrote a CPU store between two one-row scissors");
+    Require(wrongUntouched == 0, "a drawn linear target's store changed pixels no one-row draw and no CPU store wrote");
+    return true;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (!Expect(*device) || !ExpectManyRows(*device)) {
+            std::puts("linear target rectangle bound test skipped: no host import on this device");
+            return VulkanTestSkipped;
+        }
+        std::puts("linear target rectangle bound tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/NativeMultisampleDepth.cpp

@@ -704,6 +704,48 @@ void CheckDepthAttachmentAlias() {
     std::puts("native depth attachment alias changes layout while retaining held GPU views");
 }
 
+void CheckStorageAliasRetirement() {
+    const auto device = OpenVulkanTestDevice();
+    Require(device != nullptr, "native depth storage alias requires Vulkan");
+    const auto context = device->graphicsContext();
+    constexpr VkExtent2D extent{5u, 3u};
+    constexpr std::uint32_t samples = 4u;
+    constexpr std::size_t prefix = 65536u;
+    const auto bytes = static_cast<std::size_t>(MultisampledDepthBytes_nid_no_patch(extent, 4u, samples));
+    TargetMemory memory(prefix + bytes), metadata(65536u);
+    std::memset(memory.Data(), 0x3c, prefix + bytes);
+    std::memset(metadata.Data(), 0, 65536u);
+    struct Cleanup {
+        AgcDriver::VulkanDevice& device;
+        ~Cleanup() { device.WaitIdle(); ClearCachedTextures(device.Device()); ClearDepthSurfaces(device.Device()); }
+    } cleanup{*device};
+    GuestTextureResource alias{};
+    alias.baseAddress = memory.Address();
+    alias.width = 256u;
+    alias.height = static_cast<std::uint32_t>((prefix + bytes) / 1024u);
+    alias.mipCount = alias.allocatedMipCount = 1u;
+    alias.dimension = TextureDimension::k2D;
+    alias.tileMode = TextureTileMode::kLinear;
+    alias.format = 22u;
+    alias.dstSelX = 0u;
+    alias.dstSelY = alias.dstSelZ = 4u;
+    alias.dstSelW = 5u;
+    auto cached = CachedStorageSurface(context, alias);
+    Require(cached != nullptr, "storage alias was not cached before native depth creation");
+    DepthTarget depth{memory.Address() + prefix, 0u, extent, VK_FORMAT_D32_SFLOAT, 0.0f, 0u, metadata.Address(), false, samples};
+    depth.depthSwizzle = 24u;
+    depth.iterate256 = depth.htilePipeAligned = depth.htileDepthOnly = true;
+    Require(DepthSurfaceView(context, depth) != VK_NULL_HANDLE && DepthSurfaceAt(depth.address), "storage alias fixture has no native depth owner");
+    auto disjoint = alias;
+    disjoint.height = static_cast<std::uint32_t>(prefix / 1024u);
+    Require(CachedStorageSurface(context, disjoint) != nullptr, "disjoint storage alias was not created");
+    Require(DepthSurfaceAt(depth.address), "storage access ending at native depth retired a disjoint owner");
+    Require(CachedStorageSurface(context, alias) != nullptr, "overlapping cached storage alias was not returned");
+    Require(!DepthSurfaceAt(depth.address), "a cached storage view kept a native depth owner inside its range");
+    for (std::size_t offset = 0u; offset < prefix + bytes; ++offset) Require(memory.Data()[offset] == std::byte{0x3c}, "retiring unwritten native depth changed guest bytes");
+    std::puts("cached storage span retires only intersecting native depth owners");
+}
+
 enum class ClearFault { None, PartialStore, CpuWrite, DepthWrite, BadMetadata, UnprovenDispatch, PendingWrite, Overlap, PossibleStore, CompletedPossibleStore, DriverDispatch, ImportWindowWrite };
 
 void CheckClearDispatch() {
@@ -1337,6 +1379,7 @@ int main(int argc, char** argv) {
         CheckDrawFeedback();
         const auto context = device->GetContext();
         if (argc == 1) CheckDepthAttachmentAlias();
+        if (argc == 1) CheckStorageAliasRetirement();
         std::unique_lock lock(AgcDriver::GuestMemory::GpuMutex());
         Require(DepthSupported(context, VK_FORMAT_D32_SFLOAT, 4u), "the native depth regression requires sampled D32 with four samples");
         if (argc == 2 && std::string_view(argv[1]) == "--clear-refusals") {

core/libs/prx/libSceAgcDriver/tests/execution/PipelineOptimizedReadback.cpp

@@ -64,6 +64,8 @@ struct PipelineTrace {
     std::mutex mutex;
     std::condition_variable released;
     bool allowOptimization = false;
+    std::uint32_t libraries = 0;
+    std::uint32_t retaining = 0;
     std::vector<VkPipeline> fast;
     std::vector<VkPipeline> optimized;
     VkCommandBuffer boundCommands = VK_NULL_HANDLE;
@@ -90,7 +92,12 @@ VKAPI_ATTR VkResult VKAPI_CALL CreatePipelines(VkDevice device, VkPipelineCache
     const auto result = Trace->create(device, cache, count, infos, allocator, pipelines);
     std::lock_guard lock(Trace->mutex);
     for (std::uint32_t index = 0; index < count; ++index) {
-        if (pipelines[index] == VK_NULL_HANDLE || (infos[index].flags & VK_PIPELINE_CREATE_LIBRARY_BIT_KHR) != 0) continue;
+        if (pipelines[index] == VK_NULL_HANDLE) continue;
+        if ((infos[index].flags & VK_PIPELINE_CREATE_LIBRARY_BIT_KHR) != 0) {
+            ++Trace->libraries;
+            if ((infos[index].flags & VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT) != 0) ++Trace->retaining;
+            continue;
+        }
         auto& handles = (infos[index].flags & VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT) != 0 ? Trace->optimized : Trace->fast;
         handles.push_back(pipelines[index]);
     }
@@ -217,12 +224,17 @@ void Run(AgcDriver::VulkanDevice& device, Context context, PipelineTrace& trace,
         CheckPixels(masked, phase);
     };
     render(false, "fast link");
+    {
+        std::lock_guard lock(trace.mutex);
+        Require(trace.libraries != 0, "the readback pipeline's libraries were not created through the traced device");
+        Require(trace.retaining == (optimize ? trace.libraries : 0u), std::to_string(trace.retaining) + " of the " + std::to_string(trace.libraries) + " libraries were created to retain link-time optimization info, expected " + (optimize ? "all of them" : "none, since no optimized link will be made"));
+    }
     trace.ReleaseOptimization();
     WaitForOptimizedPipelines(context.device);
     Require(PipelineLibraryCountersOf(context.device).linked == 1, "the readback pipeline was not linked from graphics libraries");
     Require(trace.optimized.size() == (optimize ? 1u : 0u), "the optimized pipeline creation count differs from the requested mode");
     render(optimize, "completed link");
-    std::printf("pipeline readback passed: fast draws %u, optimized draws %u, optimized creations %zu\n", optimize ? 1u : 2u, optimize ? 1u : 0u, trace.optimized.size());
+    std::printf("pipeline readback passed: fast draws %u, optimized draws %u, optimized creations %zu, libraries %u (%u retaining link-time optimization info)\n", optimize ? 1u : 2u, optimize ? 1u : 0u, trace.optimized.size(), trace.libraries, trace.retaining);
 }
 
 }

core/libs/prx/libSceAgcDriver/tests/execution/PresentKeyWriterWait.cpp

@@ -0,0 +1,202 @@
+#include "prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <exception>
+#include <iostream>
+#include <mutex>
+#include <optional>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Side = 128;
+constexpr std::uint64_t DisplayFormat = 0x8000000000000000ull;
+constexpr std::size_t SurfaceBytes = 65536;
+constexpr std::size_t KeysOffset = SurfaceBytes;
+constexpr std::size_t KeysCapacity = 32768;
+constexpr std::size_t CopiedKeys = 256;
+constexpr std::size_t SourceKeysOffset = KeysOffset + KeysCapacity;
+constexpr std::size_t BlockBytes = 2 * 65536;
+constexpr std::uint8_t Uncompressed = 0xff;
+constexpr std::uint8_t Cleared = 0x00;
+
+alignas(256) constexpr std::array<std::uint32_t, 7> CopyKeysCode{
+    0x34020083, 0xe0341000, 0x80000201, 0xbf8c3f70, 0xe0741000, 0x80010201, 0xbf810000,
+};
+
+std::uint64_t AddressOf(const void* data) {
+    return static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(data));
+}
+
+class GuestBlock {
+public:
+    GuestBlock() {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint8_t*>(std::aligned_alloc(65536, BlockBytes));
+#endif
+        Require(block != nullptr, "present key writer wait: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+#ifdef _WIN32
+        VirtualFree(block, 0, MEM_RELEASE);
+#else
+        std::free(block);
+#endif
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* Data() { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = AddressOf(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData) {
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    return ShaderRecompiler::Recompile(request);
+}
+
+struct Scenario {
+    std::uint8_t* surface;
+    std::uint8_t* keys;
+    std::uint8_t* source;
+    std::size_t keyBytes;
+    AgcDriver::DisplayBuffer buffer;
+    std::vector<std::uint32_t> copyData;
+};
+
+Scenario Prepare(std::uint8_t* block) {
+    Scenario scenario{block, block + KeysOffset, block + SourceKeysOffset, 0, {AddressOf(block), DisplayFormat, Side, Side}, std::vector<std::uint32_t>(16, 0u)};
+    scenario.buffer.dccAddress = AddressOf(scenario.keys);
+    scenario.keyBytes = AgcDriver::DisplayBufferKeyBytes(scenario.buffer);
+    Require(AgcDriver::DisplayBufferSize(scenario.buffer) <= SurfaceBytes && scenario.keyBytes >= CopiedKeys && scenario.keyBytes <= KeysCapacity, "present key writer wait: the display buffer does not fit the guest block");
+    const auto source = BufferDescriptor(scenario.source, CopiedKeys);
+    const auto destination = BufferDescriptor(scenario.keys, CopiedKeys);
+    std::copy(source.begin(), source.end(), scenario.copyData.begin());
+    std::copy(destination.begin(), destination.end(), scenario.copyData.begin() + 4);
+    return scenario;
+}
+
+void Reset(const Scenario& scenario) {
+    std::memset(scenario.surface, 0x55, SurfaceBytes);
+    std::memset(scenario.keys, Uncompressed, scenario.keyBytes);
+    std::memset(scenario.keys, Cleared, CopiedKeys);
+    std::memset(scenario.source, Uncompressed, CopiedKeys);
+    AgcDriver::GuestMemory::BumpCollectEpoch();
+}
+
+std::string PresentFailure(AgcDriver::VulkanDevice& device, const AgcDriver::DisplayBuffer& buffer) {
+    try {
+        device.PresentDisplayBuffer(buffer);
+    } catch (const std::exception& error) {
+        return error.what();
+    }
+    return "the headless present succeeded";
+}
+
+const std::string NoSwapchain = "Vulkan presentation: device has no swapchain";
+
+bool CanCopyKeys(AgcDriver::VulkanDevice& device, const Scenario& scenario) {
+    const std::array<std::uint32_t, 4> pattern{0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu};
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    const bool filled = device.FillBuffer(AddressOf(scenario.keys), CopiedKeys, pattern);
+    device.WaitIdle();
+    return filled;
+}
+
+void RunPendingWriter(AgcDriver::VulkanDevice& device, const Scenario& scenario) {
+    Reset(scenario);
+    const auto copy = Compile(device, CopyKeysCode, scenario.copyData);
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    Require(AgcDriver::GuestMemory::GpuMutex().DepthOnThisThread() == 1, "present key writer wait: the presenter's hold is not the outermost");
+    device.Dispatch(copy, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(CopyKeysCode.data()));
+    Require(AgcDriver::Graphics::DccKeyWritersPending(scenario.buffer.dccAddress, scenario.keyBytes), "a recorded kernel that copies data over the display keys is not seen as a pending key writer");
+    const auto holds = AgcDriver::GuestMemory::GpuHoldSerial();
+    const auto failure = PresentFailure(device, scenario.buffer);
+    const auto acquired = AgcDriver::GuestMemory::GpuHoldSerial() - holds;
+    Require(failure == NoSwapchain, "a present behind a pending writer of the display keys stopped before the swapchain: " + failure);
+    Require(acquired != 0, "the presenter synchronized with the display keys' writer without releasing the GPU mutex");
+    Require(!AgcDriver::Graphics::DccKeyWritersPending(scenario.buffer.dccAddress, scenario.keyBytes), "the display keys' writer is still pending after the present");
+    Require(scenario.keys[0] == Uncompressed && scenario.keys[CopiedKeys - 1] == Uncompressed, "the present read the display keys before their writer landed");
+}
+
+void RunNoWriter(AgcDriver::VulkanDevice& device, const Scenario& scenario) {
+    std::memset(scenario.keys, Uncompressed, scenario.keyBytes);
+    AgcDriver::GuestMemory::BumpCollectEpoch();
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    device.WaitIdle();
+    Require(!AgcDriver::Graphics::DccKeyWritersPending(scenario.buffer.dccAddress, scenario.keyBytes), "the idle device still reports a pending writer of the display keys");
+    const auto holds = AgcDriver::GuestMemory::GpuHoldSerial();
+    const auto failure = PresentFailure(device, scenario.buffer);
+    const auto acquired = AgcDriver::GuestMemory::GpuHoldSerial() - holds;
+    Require(failure == NoSwapchain, "a present with no pending writer of the display keys stopped before the swapchain: " + failure);
+    Require(acquired == 0, "a present with no pending writer of the display keys released the GPU mutex");
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (!device->CanWaitUnlocked()) {
+            std::puts("skipped, the device has no timeline semaphores: the presenter cannot wait without the GPU mutex");
+            return VulkanTestSkipped;
+        }
+        GuestBlock block;
+        const auto scenario = Prepare(block.Data());
+        if (!CanCopyKeys(*device, scenario)) {
+            std::puts("skipped, host imports are unavailable: the key writer of the test is a copied writer");
+            return VulkanTestSkipped;
+        }
+        RunPendingWriter(*device, scenario);
+        RunNoWriter(*device, scenario);
+        std::puts("present key writer wait tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/QueuedIndexReadOrder.cpp

@@ -371,6 +371,11 @@ int main() {
         AgcDriverWaitIdle_nid_postfix();
         Require(AgcDriver::Graphics::StorageTexture::AnyPending(), "queued index read: the image write left no pending result to order against");
         Require(IndicesIntact(), "queued index read: the image's results reached guest memory before any consumer asked");
+        const auto copiedAddress = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(Copied));
+        const auto heldQueries = AgcDriver::DriverDetail::DrawPipeline::HeldReadQueries();
+        Require(!AgcDriver::Graphics::StorageTexture::FlushPending(copiedAddress, PoolBytes), "queued index read: a flush over memory without pending results stored something");
+        Require(AgcDriver::DriverDetail::DrawPipeline::HeldReadQueries() == heldQueries, "queued index read: a flush with nothing to store asked the draw pipeline for held reads");
+        Require(IndicesIntact(), "queued index read: a flush over other memory stored the image's results");
         GateUntilQueued();
         std::fill_n(ColorMemory, ColorBytes, std::byte{0});
         Submit(Draw(Vertices.data(), Vertices.size()), "the indexed draw");

core/libs/prx/libSceAgcDriver/tests/execution/SdwaCvtF32Ubyte.cpp

@@ -17,20 +17,50 @@ using ShaderRecompiler::ShaderStage;
 constexpr std::uint32_t Threads = 64;
 constexpr std::uint32_t Inputs = 4;
 constexpr std::uint32_t Results = 4;
+constexpr std::uint32_t Selectors = 7;
 alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
 alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
 
-alignas(256) constexpr std::array<std::uint32_t, 30> SdwaCode{
+constexpr std::array<std::uint32_t, Results> ModifierWords{14, 16, 18, 20};
+
+alignas(256) std::array<std::uint32_t, 30> SdwaCode{
     0x34020082, 0x34060082, 0xe0302000, 0x80000401,
     0x7e1402ff, 0xabcd1234, 0x7e1602ff, 0xabcd1234, 0x7e1802ff, 0xabcd1234, 0x7e1a02ff, 0xabcd1234,
-    0xbf8c3f70, 0x7e1424f9, 0x00040604, 0x7e1624f9, 0x00050604, 0x7e1826f9, 0x000d0604, 0x7e1a28f9, 0x000b0604,
+    0xbf8c3f70, 0x7e1422f9, 0x00060604, 0x7e1624f9, 0x00060604, 0x7e1826f9, 0x00060604, 0x7e1a28f9, 0x00060604,
     0xe0702000, 0x80010a03, 0xe0702004, 0x80010b03, 0xe0702008, 0x80010c03, 0xe070200c, 0x80010d03,
     0xbf810000,
 };
 
+const std::array<const char*, Selectors> SelectorNames{"BYTE_0", "BYTE_1", "BYTE_2", "BYTE_3", "WORD_0", "WORD_1", "DWORD"};
+
+struct SelectVector {
+    std::uint32_t value;
+    std::array<std::uint32_t, Selectors> zeroExtended;
+    std::array<std::uint32_t, Selectors> signExtended;
+};
+
+constexpr std::array<SelectVector, 16> Vectors{{
+    {0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}},
+    {0xffffffffu, {0x000000ffu, 0x000000ffu, 0x000000ffu, 0x000000ffu, 0x0000ffffu, 0x0000ffffu, 0xffffffffu}, {0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu}},
+    {0x12345678u, {0x00000078u, 0x00000056u, 0x00000034u, 0x00000012u, 0x00005678u, 0x00001234u, 0x12345678u}, {0x00000078u, 0x00000056u, 0x00000034u, 0x00000012u, 0x00005678u, 0x00001234u, 0x12345678u}},
+    {0x80808080u, {0x00000080u, 0x00000080u, 0x00000080u, 0x00000080u, 0x00008080u, 0x00008080u, 0x80808080u}, {0xffffff80u, 0xffffff80u, 0xffffff80u, 0xffffff80u, 0xffff8080u, 0xffff8080u, 0x80808080u}},
+    {0x7f7f7f7fu, {0x0000007fu, 0x0000007fu, 0x0000007fu, 0x0000007fu, 0x00007f7fu, 0x00007f7fu, 0x7f7f7f7fu}, {0x0000007fu, 0x0000007fu, 0x0000007fu, 0x0000007fu, 0x00007f7fu, 0x00007f7fu, 0x7f7f7f7fu}},
+    {0x8182a3f4u, {0x000000f4u, 0x000000a3u, 0x00000082u, 0x00000081u, 0x0000a3f4u, 0x00008182u, 0x8182a3f4u}, {0xfffffff4u, 0xffffffa3u, 0xffffff82u, 0xffffff81u, 0xffffa3f4u, 0xffff8182u, 0x8182a3f4u}},
+    {0x7f00ff80u, {0x00000080u, 0x000000ffu, 0x00000000u, 0x0000007fu, 0x0000ff80u, 0x00007f00u, 0x7f00ff80u}, {0xffffff80u, 0xffffffffu, 0x00000000u, 0x0000007fu, 0xffffff80u, 0x00007f00u, 0x7f00ff80u}},
+    {0x00008000u, {0x00000000u, 0x00000080u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00000000u, 0x00008000u}, {0x00000000u, 0xffffff80u, 0x00000000u, 0x00000000u, 0xffff8000u, 0x00000000u, 0x00008000u}},
+    {0x0000007fu, {0x0000007fu, 0x00000000u, 0x00000000u, 0x00000000u, 0x0000007fu, 0x00000000u, 0x0000007fu}, {0x0000007fu, 0x00000000u, 0x00000000u, 0x00000000u, 0x0000007fu, 0x00000000u, 0x0000007fu}},
+    {0x00000080u, {0x00000080u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000080u, 0x00000000u, 0x00000080u}, {0xffffff80u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000080u, 0x00000000u, 0x00000080u}},
+    {0x80000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000080u, 0x00000000u, 0x00008000u, 0x80000000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0xffffff80u, 0x00000000u, 0xffff8000u, 0x80000000u}},
+    {0x00800000u, {0x00000000u, 0x00000000u, 0x00000080u, 0x00000000u, 0x00000000u, 0x00000080u, 0x00800000u}, {0x00000000u, 0x00000000u, 0xffffff80u, 0x00000000u, 0x00000000u, 0x00000080u, 0x00800000u}},
+    {0x0000ff00u, {0x00000000u, 0x000000ffu, 0x00000000u, 0x00000000u, 0x0000ff00u, 0x00000000u, 0x0000ff00u}, {0x00000000u, 0xffffffffu, 0x00000000u, 0x00000000u, 0xffffff00u, 0x00000000u, 0x0000ff00u}},
+    {0xdeadbeefu, {0x000000efu, 0x000000beu, 0x000000adu, 0x000000deu, 0x0000beefu, 0x0000deadu, 0xdeadbeefu}, {0xffffffefu, 0xffffffbeu, 0xffffffadu, 0xffffffdeu, 0xffffbeefu, 0xffffdeadu, 0xdeadbeefu}},
+    {0x01020304u, {0x00000004u, 0x00000003u, 0x00000002u, 0x00000001u, 0x00000304u, 0x00000102u, 0x01020304u}, {0x00000004u, 0x00000003u, 0x00000002u, 0x00000001u, 0x00000304u, 0x00000102u, 0x01020304u}},
+    {0xf0e0d0c0u, {0x000000c0u, 0x000000d0u, 0x000000e0u, 0x000000f0u, 0x0000d0c0u, 0x0000f0e0u, 0xf0e0d0c0u}, {0xffffffc0u, 0xffffffd0u, 0xffffffe0u, 0xfffffff0u, 0xffffd0c0u, 0xfffff0e0u, 0xf0e0d0c0u}},
+}};
+
 void FillInput() {
     for (std::uint32_t tid = 0; tid < Threads; ++tid) {
-        Input[tid * Inputs] = tid | ((tid + 64u) << 8u) | ((255u - tid) << 16u) | (((tid * 3u + 128u) & 0xffu) << 24u);
+        Input[tid * Inputs] = Vectors[tid % Vectors.size()].value;
     }
 }
 
@@ -49,7 +79,10 @@ std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t co
     return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu};
 }
 
-void Run(AgcDriver::VulkanDevice& device) {
+void Run(AgcDriver::VulkanDevice& device, std::uint32_t selector, bool signExtend) {
+    for (const auto word : ModifierWords) {
+        SdwaCode[word] = 0x00000604u | (selector << 16u) | (signExtend ? 1u << 19u : 0u);
+    }
     Output.fill(0xdeadbeefu);
     std::vector<std::uint32_t> userData(8, 0u);
     const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size()));
@@ -71,18 +104,16 @@ void Run(AgcDriver::VulkanDevice& device) {
     device.WaitIdle();
 }
 
-void Check() {
+void Check(std::uint32_t selector, bool signExtend) {
     for (std::uint32_t tid = 0; tid < Threads; ++tid) {
-        const auto source = Input[tid * Inputs];
+        const auto& vector = Vectors[tid % Vectors.size()];
+        const auto selected = signExtend ? vector.signExtended[selector] : vector.zeroExtended[selector];
         const auto* out = &Output[tid * Results];
-        const auto where = [&](std::uint32_t result) { return "sdwa cvt f32 ubyte: thread " + std::to_string(tid) + " result " + std::to_string(result) + " is " + Hex(out[result]); };
-        const auto byte1 = FloatBits((source >> 8u) & 0xffu);
-        const auto byte3 = FloatBits(source >> 24u);
-        const auto signFill = FloatBits((source & 0x80000000u) != 0u ? 0xffu : 0u);
-        Require(out[0] == byte1, where(0) + ", expected " + Hex(byte1) + ": v_cvt_f32_ubyte1 src0_sel:WORD_0 must convert bits 15:8");
-        Require(out[1] == byte3, where(1) + ", expected " + Hex(byte3) + ": v_cvt_f32_ubyte1 src0_sel:WORD_1 must convert bits 31:24");
-        Require(out[2] == signFill, where(2) + ", expected " + Hex(signFill) + ": v_cvt_f32_ubyte2 src0_sel:WORD_1 src0_sext must convert the sign extension of the high word");
-        Require(out[3] == signFill, where(3) + ", expected " + Hex(signFill) + ": v_cvt_f32_ubyte3 src0_sel:BYTE_3 src0_sext must convert the sign extension of the top byte");
+        for (std::uint32_t byte = 0; byte < Results; ++byte) {
+            const auto expected = FloatBits((selected >> (byte * 8u)) & 0xffu);
+            Require(out[byte] == expected, "sdwa v_cvt_f32_ubyte" + std::to_string(byte) + " src0_sel:" + SelectorNames[selector] + (signExtend ? " src0_sext" : "") + " of " +
+                                               Hex(vector.value) + " is " + Hex(out[byte]) + ", expected " + Hex(expected));
+        }
     }
 }
 
@@ -93,8 +124,12 @@ int main() {
         const auto device = OpenVulkanTestDevice();
         if (!device) return VulkanTestSkipped;
         FillInput();
-        Run(*device);
-        Check();
+        for (const bool signExtend : {false, true}) {
+            for (std::uint32_t selector = 0; selector < Selectors; ++selector) {
+                Run(*device, selector, signExtend);
+                Check(selector, signExtend);
+            }
+        }
         std::puts("sdwa cvt f32 ubyte tests passed");
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/execution/StencilLogicOperations.cpp

@@ -0,0 +1,202 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 16;
+constexpr std::uint32_t WaveSize = 64;
+constexpr std::uint32_t PlainControl = 0x50u;
+constexpr std::byte Kept{0x40};
+alignas(256) std::array<std::byte, Width * Height * 4> Pixels{};
+alignas(256) std::array<std::byte, 128 * 128 * 4> DepthMemory{};
+alignas(256) std::array<std::byte, 256 * 256> StencilMemory{};
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 4> PixelCode{
+    0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::array<float, 4>, 3> Vertices{{
+    {-1.0f, -1.0f, 0.25f, 1.0f}, {3.0f, -1.0f, 0.25f, 1.0f}, {-1.0f, 3.0f, 0.25f, 1.0f},
+}};
+
+struct Shaders {
+    ShaderRecompiler::RecompileResult vertex;
+    ShaderRecompiler::RecompileResult pixel;
+    std::uint32_t pixelPush = 0;
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+ShaderRecompiler::ShaderPixelStageInfo PixelInfo() {
+    const AgcDriver::Registers context{
+        {0x1b3, 0x700u}, {0x1b4, 0x700u}, {0x1b6, 0u}, {0x1c5, 9u}, {0x203, PlainControl},
+    };
+    return AgcDriver::Graphics::DecodePixelStageInfo(context, {0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u});
+}
+
+Shaders Compile(AgcDriver::VulkanDevice& device) {
+    const auto target = device.Target();
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(Vertices.data(), 16u, static_cast<std::uint32_t>(Vertices.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {WaveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    Shaders shaders{ShaderRecompiler::Recompile(vertex), {}, 0};
+    const auto vertexPush = static_cast<std::uint32_t>(shaders.vertex.pushConstants.size());
+    std::vector<std::uint32_t> pixelUserData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(PixelCode.data()), std::as_bytes(std::span(PixelCode))}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(PixelCode.data()), PixelCode, 0, {}},
+        {WaveSize, 0, pixelUserData, std::nullopt, PixelInfo(), std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    shaders.pixel = ShaderRecompiler::Recompile(fragment);
+    shaders.pixelPush = PixelPushOffset(vertexPush, target);
+    return shaders;
+}
+
+AgcDriver::QueueState StencilQueue(std::uint32_t ops, std::uint32_t refMask, std::uint32_t depthControl) {
+    AgcDriver::QueueState queue;
+    queue.userConfig[0x242] = 4;
+    queue.context = {
+        {0x2d5, 0x2000},
+        {0x1b6, 0}, {0x1b8, 0}, {0x207, 0}, {0x203, 0x800},
+        {0x2dc, 0xaa00}, {0x2f8, 0}, {0x292, 2}, {0x293, 0},
+        {0x80, 0}, {0x8d, 0}, {0x83, 0xffff}, {0x8c, 0xa},
+        {0x2f9, 0x2d}, {0x313, 0x6000}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff},
+        {0x206, 0x43f}, {0x204, 0x80000}, {0x205, 0x240},
+        {0x8e, 0xf}, {0x8f, 0xf}, {0x202, 0xcc0010},
+        {0x1c4, 0}, {0x1c5, 9}, {0x1c3, 4}, {0x31c, 0x28028},
+        {0x31b, 0}, {0x31d, 0}, {0x3b0, (63u << 14u) | 3u},
+        {0x3b8, 0x9000000}, {0x1e0, 0},
+        {0xc, 0}, {0xd, 0x40040},
+        {0x81, 0x80000000}, {0x82, 0x40040},
+        {0x90, 0x80000000}, {0x91, 0x40040},
+        {0x94, 0x80000000}, {0x95, 0x40040},
+        {0x000, 0}, {0x002, 0}, {0x007, (1u << 16u) | 3u}, {0x00a, 0}, {0x00b, 0}, {0x010, 0x22900983u}, {0x011, 0x20000181u},
+        {0x012, 0x100}, {0x014, 0x100}, {0x013, 0x200}, {0x015, 0x200}, {0x005, 0x300}, {0x01e, 0x1},
+        {0x10b, ops}, {0x10c, refMask}, {0x10d, refMask}, {0x200, depthControl}
+    };
+    const auto address = reinterpret_cast<std::uintptr_t>(Pixels.data());
+    queue.context[0x318] = static_cast<std::uint32_t>(address >> 8u);
+    queue.context[0x390] = static_cast<std::uint32_t>(address >> 40u);
+    queue.context[0x10f] = std::bit_cast<std::uint32_t>(32.0f);
+    queue.context[0x110] = std::bit_cast<std::uint32_t>(32.0f);
+    queue.context[0x111] = std::bit_cast<std::uint32_t>(-2.0f);
+    queue.context[0x112] = std::bit_cast<std::uint32_t>(2.0f);
+    queue.context[0x113] = std::bit_cast<std::uint32_t>(1.0f);
+    queue.context[0x114] = 0;
+    queue.context[0xb4] = 0;
+    queue.context[0xb5] = std::bit_cast<std::uint32_t>(1.0f);
+    queue.shader[0x008] = 0x100;
+    queue.shader[0x009] = 0;
+    return queue;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const Shaders& shaders, const VkStencilOpState& stencil) {
+    Pixels.fill(Kept);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> stages{{
+        {ShaderStage::Vertex, &shaders.vertex, 0},
+        {ShaderStage::Fragment, &shaders.pixel, shaders.pixelPush}
+    }};
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, WaveSize, WaveSize, std::nullopt, std::nullopt};
+    state.color = {reinterpret_cast<std::uintptr_t>(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u};
+    state.colors = {state.color};
+    state.hasColorTarget = true;
+    state.depth = AgcDriver::Graphics::DepthTarget{reinterpret_cast<std::uintptr_t>(DepthMemory.data()), reinterpret_cast<std::uintptr_t>(StencilMemory.data()), {Width, Height}, VK_FORMAT_D32_SFLOAT_S8_UINT, 1.0f, 0};
+    state.depthTest = false;
+    state.depthCompare = VK_COMPARE_OP_ALWAYS;
+    state.stencilTest = true;
+    state.stencilFront = stencil;
+    state.stencilBack = stencil;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = {{0, 0}, {Width, Height}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blend.colorWriteMask = 15;
+    state.blends = {state.blend};
+    state.blendConstants = {};
+    device.Draw(state, AgcDriver::Pm4::DrawParameters{0, 3u, 0, 1u, 0, false}, stages);
+    device.WaitIdle();
+}
+
+std::uint32_t Expected(std::uint32_t op, std::uint32_t stored, std::uint32_t value, std::uint32_t writeMask) {
+    const auto result = op == 10u ? stored & value : op == 11u ? stored | value : op == 12u ? stored ^ value : ~(stored ^ value);
+    return ((stored & ~writeMask) | (result & writeMask)) & 0xffu;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        const auto shaders = Compile(*device);
+        const auto replace = [](std::uint32_t value) { return VkStencilOpState{VK_STENCIL_OP_REPLACE, VK_STENCIL_OP_REPLACE, VK_STENCIL_OP_REPLACE, VK_COMPARE_OP_ALWAYS, 0xffu, 0xffu, value}; };
+        const auto equal = [](std::uint32_t value) { return VkStencilOpState{VK_STENCIL_OP_KEEP, VK_STENCIL_OP_KEEP, VK_STENCIL_OP_KEEP, VK_COMPARE_OP_EQUAL, 0xffu, 0u, value}; };
+        std::size_t checked = 0;
+        for (const auto stored : {0xa5u, 0x3cu}) {
+            for (const auto op : {10u, 11u, 12u, 15u}) {
+                for (const auto [writeMask, value] : {std::pair{0xffu, 0x3cu}, std::pair{0x0fu, 0x96u}, std::pair{0xf6u, 0xffu}}) {
+                    for (const auto [ops, compare] : {std::pair{op << 4u, 7u}, std::pair{op, 0u}}) {
+                        const auto refMask = 0xff00u | (writeMask << 16u) | (value << 24u);
+                        const auto face = AgcDriver::Graphics::DecodeState(StencilQueue(ops, refMask, 1u | (compare << 8u))).stencilFront;
+                        const auto expected = Expected(op, stored, value, writeMask);
+                        const auto name = "stencil operation " + std::to_string(op) + " in DB_STENCIL_CONTROL " + std::to_string(ops) + " with DB_STENCILREFMASK " + std::to_string(refMask) + " on stored value " + std::to_string(stored);
+                        Draw(*device, shaders, replace(stored));
+                        Draw(*device, shaders, face);
+                        Draw(*device, shaders, equal(expected));
+                        for (std::uint32_t pixel = 0; pixel < Width * Height; ++pixel) {
+                            Require(Pixels[pixel * 4u] == std::byte{255}, name + ": pixel " + std::to_string(pixel) + " does not hold the expected stencil value " + std::to_string(expected));
+                        }
+                        Draw(*device, shaders, equal(expected ^ 0x80u));
+                        Require(std::ranges::all_of(std::span(Pixels).first(4), [](std::byte value) { return value == Kept; }), name + ": the stencil value also equals " + std::to_string(expected ^ 0x80u));
+                        ++checked;
+                    }
+                }
+            }
+        }
+        std::printf("stencil logic operation tests passed: %zu states\n", checked);
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/StencilReplay.cpp

@@ -88,7 +88,7 @@ ShaderRecompiler::ShaderPixelStageInfo PixelInfo(std::uint32_t shaderControl) {
     return AgcDriver::Graphics::DecodePixelStageInfo(context, {0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u});
 }
 
-void Draw(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> pixelCode, std::uint32_t shaderControl, const VkStencilOpState& stencil, std::uint32_t vertices = 3u, std::uint32_t instances = 1u, bool depthFail = false, const std::function<void(AgcDriver::Graphics::State&)>& alter = {}, const std::function<void(AgcDriver::Pm4::DrawParameters&)>& alterDraw = {}, std::span<const std::array<float, 4>> vertexData = Vertices, std::span<const std::uint32_t> vertexCode = VertexCode, std::span<const std::uint32_t> pixelData = {}, std::span<const ShaderRecompiler::MemoryRegion> pixelSrt = {}, const std::function<void(ShaderRecompiler::RecompileResult&)>& alterVertex = {}) {
+void Draw(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> pixelCode, std::uint32_t shaderControl, const VkStencilOpState& stencil, std::uint32_t vertices = 3u, std::uint32_t instances = 1u, bool depthFail = false, const std::function<void(AgcDriver::Graphics::State&)>& alter = {}, const std::function<void(AgcDriver::Pm4::DrawParameters&)>& alterDraw = {}, std::span<const std::array<float, 4>> vertexData = Vertices, std::span<const std::uint32_t> vertexCode = VertexCode, std::span<const std::uint32_t> pixelData = {}, std::span<const ShaderRecompiler::MemoryRegion> pixelSrt = {}, const std::function<void(ShaderRecompiler::RecompileResult&)>& alterVertex = {}, std::uint32_t pixelPushBytes = 0u) {
     Pixels.fill(Kept);
     auto target = device.Target();
     std::vector<std::uint32_t> capabilities(target.supportedCapabilities.begin(), target.supportedCapabilities.end());
@@ -113,11 +113,13 @@ void Draw(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> pixelC
     auto pixelUserData = pixelData.empty() ? std::vector<std::uint32_t>(8, 0u) : std::vector<std::uint32_t>(pixelData.begin(), pixelData.end());
     std::vector<ShaderRecompiler::MemoryRegion> pixelMemory{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}};
     pixelMemory.insert(pixelMemory.end(), pixelSrt.begin(), pixelSrt.end());
+    auto pixelLayout = PixelPushLayout(vertexPush, target);
+    if (pixelPushBytes != 0u) pixelLayout.pushConstantSizeBytes = pixelPushBytes;
     ShaderRecompiler::RecompileRequest fragment{
         {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
         {WaveSize, 0, pixelUserData, std::nullopt, PixelInfo(shaderControl), std::nullopt, pixelMemory},
         target,
-        PixelPushLayout(vertexPush, target)
+        pixelLayout
     };
     fragment.useCache = false;
     const auto pixelResult = ShaderRecompiler::Recompile(fragment);
@@ -350,6 +352,14 @@ int main() {
         const std::array<std::uint32_t, 1> colorData{0x3f800000u};
         Draw(*device, userColor, exported, replace(), 3u, 1u, false, {}, {}, Vertices, VertexCode, colorData);
         Check(all, "dynamic reference preserves the adjacent shader-data push payload");
+        auto bufferColor = std::vector<std::uint32_t>(alternating.begin(), alternating.end());
+        *std::find(bufferColor.begin(), bufferColor.end(), 0x7e0e02f2u) = 0x7e0e0208u;
+        bufferColor.insert(bufferColor.begin(), {0xf4200200u, 0xfa000000u, 0xbf8cc07fu});
+        alignas(256) const std::array<std::uint32_t, 4> colorBuffer{0x3f800000u, 0u, 0u, 0u};
+        const auto colorDescriptor = BufferDescriptor(colorBuffer.data(), 4u, static_cast<std::uint32_t>(colorBuffer.size()));
+        const std::array<ShaderRecompiler::MemoryRegion, 1> colorMemory{{{reinterpret_cast<std::uintptr_t>(colorBuffer.data()), std::as_bytes(std::span(colorBuffer))}}};
+        Draw(*device, bufferColor, exported, replace(), 3u, 1u, false, {}, {}, Vertices, VertexCode, colorDescriptor, colorMemory, {}, sizeof(std::uint32_t));
+        Check(all, "dynamic reference keeps guest buffer offsets in shader data outside the push constants");
         auto guarded = std::vector<std::uint32_t>(alternating.begin(), alternating.end());
         *std::find(guarded.begin(), guarded.end(), 0x7e0e02f2u) = 0x7e0e020cu;
         guarded.insert(guarded.begin(), {0xf4040280u, 0xfa000018u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu});
@@ -369,6 +379,29 @@ int main() {
         catch (const std::exception& error) { faultThrown = std::string(error.what()).find("BDA") != std::string::npos; }
         Require(faultThrown, "poisoned readonly SRT replay did not throw a host fault");
         Require(std::ranges::all_of(Pixels, [](std::byte value) { return value == Kept; }), "poisoned replay published guest color results before its fault");
+        alignas(256) std::array<std::uint32_t, Width + 1u> columnValues{};
+        alignas(256) std::array<float, Width> columnTexels{};
+        columnValues[0] = 0x2fu;
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            columnValues[x + 1u] = 0x30u + x;
+            columnTexels[x] = 1.0f;
+        }
+        constexpr std::array<std::uint32_t, 15> bufferImageCode{
+            0x7e060f00u, 0x7e080280u, 0xf0001108u, 0x00010503u, 0xe0302000u, 0x80000603u, 0xe0302000u, 0x80030803u,
+            0xbf8c3f70u, 0x7e0e02f2u, 0xf8000082u, 0x00000600u, 0xf800180fu, 0x07050805u, 0xbf810000u,
+        };
+        const auto columnBuffer = BufferDescriptor(columnValues.data() + 1, 4u, Width);
+        const auto alignedBuffer = BufferDescriptor(columnValues.data(), 4u, Width + 1u);
+        const auto texelAddress = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(columnTexels.data()));
+        const std::array<std::uint32_t, 8> columnTexture{static_cast<std::uint32_t>(texelAddress >> 8u), static_cast<std::uint32_t>((texelAddress >> 40u) & 0xffu) | (22u << 20u) | (((Width - 1u) & 3u) << 30u), (Width - 1u) >> 2u, 0xfacu | (9u << 28u), 0u, 0u, 0u, 0u};
+        std::vector<std::uint32_t> bufferImageData(columnBuffer.begin(), columnBuffer.end());
+        bufferImageData.insert(bufferImageData.end(), columnTexture.begin(), columnTexture.end());
+        bufferImageData.insert(bufferImageData.end(), alignedBuffer.begin(), alignedBuffer.end());
+        Draw(*device, PlainPixelCode, plain, replace(0u));
+        Draw(*device, bufferImageCode, exported, replace(), 3u, 1u, false, {}, {}, Vertices, VertexCode, bufferImageData);
+        Check(all, "a replayed export beside an image draws the image's texels");
+        Draw(*device, PlainPixelCode, plain, equal(0x35u));
+        Check([](std::uint32_t x, std::uint32_t) { return x == 5u; }, "a replayed export beside an image reads its guest buffer at a 4-byte offset");
         std::puts("stencil replay tests passed");
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAjm.native/CMakeLists.txt

@@ -2,7 +2,7 @@ enable_language(C)
 
 set(LIBATRAC9_DIR ${CMAKE_SOURCE_DIR}/3rdparty/LibAtrac9/C/src)
 set(LIBATRAC9_PATCHED_DIR ${CMAKE_CURRENT_BINARY_DIR}/atrac9)
-set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${LIBATRAC9_DIR}/unpack.c ${LIBATRAC9_DIR}/bit_reader.c ${LIBATRAC9_DIR}/decinit.c)
+set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${LIBATRAC9_DIR}/unpack.c ${LIBATRAC9_DIR}/bit_reader.c ${LIBATRAC9_DIR}/decinit.c ${LIBATRAC9_DIR}/utility.c)
 include(${CMAKE_CURRENT_SOURCE_DIR}/PatchLibAtrac9.cmake)
 add_library(atrac9 STATIC EXCLUDE_FROM_ALL
         ${LIBATRAC9_DIR}/band_extension.c
@@ -17,7 +17,7 @@ add_library(atrac9 STATIC EXCLUDE_FROM_ALL
         ${LIBATRAC9_DIR}/scale_factors.c
         ${LIBATRAC9_DIR}/tables.c
         ${LIBATRAC9_PATCHED_DIR}/unpack.c
-        ${LIBATRAC9_DIR}/utility.c
+        ${LIBATRAC9_PATCHED_DIR}/utility.c
 )
 target_include_directories(atrac9 PRIVATE ${LIBATRAC9_DIR})
 set_target_properties(atrac9 PROPERTIES POSITION_INDEPENDENT_CODE ON)
@@ -36,6 +36,9 @@ if(BUILD_TESTING)
     add_test_executable(atrac9_gradient_tests tests/Atrac9Gradient.c ${LIBATRAC9_DIR}/bit_reader.c)
     target_include_directories(atrac9_gradient_tests PRIVATE ${LIBATRAC9_PATCHED_DIR} ${LIBATRAC9_DIR})
     target_link_libraries(atrac9_gradient_tests PRIVATE atrac9)
+    if(NOT WIN32)
+        target_link_libraries(atrac9_gradient_tests PRIVATE m)
+    endif()
     target_compile_options(atrac9_gradient_tests PRIVATE -ffunction-sections -fdata-sections)
     if(APPLE)
         target_link_options(atrac9_gradient_tests PRIVATE LINKER:-dead_strip)

core/libs/prx/libSceAjm.native/PatchLibAtrac9.cmake

@@ -28,3 +28,21 @@ if(libatrac9ConfigGuardPosition EQUAL -1)
 endif()
 string(REPLACE "${libatrac9OldConfigGuard}" "${libatrac9NewConfigGuard}" libatrac9Decinit "${libatrac9Decinit}")
 file(WRITE ${LIBATRAC9_PATCHED_DIR}/decinit.c "${libatrac9Decinit}")
+
+file(READ ${LIBATRAC9_DIR}/utility.c libatrac9Utility)
+set(libatrac9OldReverse "\treturn value >> (32 - bitCount);")
+set(libatrac9NewReverse "\treturn bitCount ? value >> (32 - bitCount) : 0u;")
+string(FIND "${libatrac9Utility}" "${libatrac9OldReverse}" libatrac9ReversePosition)
+if(libatrac9ReversePosition EQUAL -1)
+    message(FATAL_ERROR "LibAtrac9 BitReverse32 has changed; review the local shift-by-32 guard")
+endif()
+string(REPLACE "${libatrac9OldReverse}" "${libatrac9NewReverse}" libatrac9Utility "${libatrac9Utility}")
+
+set(libatrac9OldSignExtend "\treturn (value << shift) >> shift;")
+set(libatrac9NewSignExtend "\treturn (int)((unsigned int)value << shift) >> shift;")
+string(FIND "${libatrac9Utility}" "${libatrac9OldSignExtend}" libatrac9SignExtendPosition)
+if(libatrac9SignExtendPosition EQUAL -1)
+    message(FATAL_ERROR "LibAtrac9 SignExtend32 has changed; review the local signed-shift guard")
+endif()
+string(REPLACE "${libatrac9OldSignExtend}" "${libatrac9NewSignExtend}" libatrac9Utility "${libatrac9Utility}")
+file(WRITE ${LIBATRAC9_PATCHED_DIR}/utility.c "${libatrac9Utility}")

core/libs/prx/libSceAjm.native/include/Api.hpp

@@ -70,5 +70,9 @@ int APS5_VABI sceAjmBatchStart(uint32_t context, const AjmBatchInfo* info, int p
 int APS5_VABI sceAjmBatchWait(uint32_t context, uint32_t batch, uint32_t timeout, AjmBatchError* error);
 int APS5_VABI sceAjmBatchCancel(uint32_t context, uint32_t batch);
 int APS5_VABI sceAjmBatchErrorDump(const AjmBatchInfo* info, AjmBatchError* error);
+void* APS5_VABI sceAjmBatchJobControlBufferRa(void* position, uint32_t instance, uint64_t flags, const void* sideband_input, size_t sideband_input_size, void* sideband_output, size_t sideband_output_size, void* return_address);
+void* APS5_VABI sceAjmBatchJobRunBufferRa(void* position, uint32_t instance, uint64_t flags, const void* data_input, size_t data_input_size, void* data_output, size_t data_output_size, void* sideband_output, size_t sideband_output_size, void* return_address);
+void* APS5_VABI sceAjmBatchJobRunSplitBufferRa(void* position, uint32_t instance, uint64_t flags, const AjmBuffer* input_buffers, size_t input_buffers_num, const AjmBuffer* output_buffers, size_t output_buffers_num, void* sideband_output, size_t sideband_output_size, void* return_address);
+int APS5_VABI sceAjmBatchStartBuffer(uint32_t context, const uint8_t* batch, uint32_t batch_size, int priority, AjmBatchError* error, uint32_t* batch_id);
 
 }

core/libs/prx/libSceAjm.native/src/Ajm.cpp

@@ -36,6 +36,8 @@ constexpr int SCE_AJM_ERROR_INVALID_INSTANCE = static_cast<int>(0x80930003);
 constexpr int SCE_AJM_ERROR_INVALID_BATCH = static_cast<int>(0x80930004);
 constexpr int SCE_AJM_ERROR_INVALID_PARAMETER = static_cast<int>(0x80930005);
 constexpr int SCE_AJM_ERROR_OUT_OF_RESOURCES = static_cast<int>(0x80930007);
+constexpr int SCE_AJM_ERROR_MALFORMED_BATCH = static_cast<int>(0x80930011);
+constexpr int SCE_AJM_ERROR_INVALID_ADDRESS = static_cast<int>(0x80930016);
 
 constexpr std::int32_t AJM_RESULT_NOT_INITIALIZED = 0x00000001;
 constexpr std::int32_t AJM_RESULT_INVALID_DATA = 0x00000002;
@@ -216,6 +218,7 @@ struct JobHeader {
     std::uint32_t inputCount;
     std::uint32_t outputCount;
     std::uint8_t parameters[16];
+    bool console;
 };
 
 struct PackedJob {
@@ -691,9 +694,17 @@ void EmitFrame(Instance& instance, PcmOutputs& outputs, const std::uint8_t* pcm,
     ResamplerProduce(instance, outputs, channels, encoding, sampleBytes);
 }
 
+// A PS4 reports an ATRAC9 stream's bitrate from its superframe: 448-byte superframes of 4 frames of 256
+// samples at 48 kHz gave 168000 (Knowledge is Power: Decades, every run job's format sideband).
+std::uint32_t At9Bitrate(const Instance& instance) {
+    const auto samples = static_cast<std::uint64_t>(instance.info.framesInSuperframe) * static_cast<std::uint64_t>(instance.info.frameSamples);
+    if (samples == 0) return 0;
+    return static_cast<std::uint32_t>(static_cast<std::uint64_t>(instance.info.superframeSize) * 8u * static_cast<std::uint64_t>(instance.info.samplingRate) / samples);
+}
+
 SidebandFormat CurrentFormat(const Instance& instance) {
     switch (instance.codec) {
-    case CODEC_AT9: return {static_cast<std::uint32_t>(instance.info.channels), ChannelMask(static_cast<std::size_t>(instance.info.channels)), static_cast<std::uint32_t>(instance.info.samplingRate), PcmEncoding(instance), 0, 0};
+    case CODEC_AT9: return {static_cast<std::uint32_t>(instance.info.channels), ChannelMask(static_cast<std::size_t>(instance.info.channels)), static_cast<std::uint32_t>(instance.info.samplingRate), PcmEncoding(instance), At9Bitrate(instance), 0};
     case CODEC_MP3: return {instance.mp3Channels, ChannelMask(instance.mp3Channels), instance.mp3SampleRate, PcmEncoding(instance), instance.mp3Bitrate, 0};
     case CODEC_OPUS: return {instance.opusChannels, ChannelMask(instance.opusChannels), instance.opusSampleRate, PcmEncoding(instance), 0, 0};
     case CODEC_M4AAC: return {instance.aacChannels, ChannelMask(instance.aacChannels), instance.aacSampleRate, PcmEncoding(instance), instance.aacBitrate, 0};
@@ -768,12 +779,22 @@ void RunAt9(Instance& instance, const JobHeader& job, const AjmBuffer* inputs, c
         if (segmentEnded()) break;
         if (instance.superframeRemaining == 0) consumed += RiffDataOffset(input.data() + consumed, input.size() - consumed);
         const std::size_t needed = instance.superframeRemaining == 0 ? superframeSize : instance.superframeRemaining;
+        const bool roomForFrame = Resampling(instance) ? pcmOutputs.Room() >= channels * sampleBytes : pcmOutputs.Room() >= frameBytes;
+        // A PS4 reports which side ran out (probe runs of a title's ATRAC9 stream through libSceAjm's
+        // builders): PARTIAL_INPUT when the input ran out, on a superframe boundary or not, with room
+        // left; NOT_ENOUGH_ROOM when the room ran out with input left, after however many frames (and
+        // with none decoded for a zero room); both (0x18) when the input ended just as the output
+        // filled, which is every Wwise run job. A run without input (a format query) reports 0.
+        // Resampled runs were not measured and keep the earlier rule: a trailing partial superframe is
+        // partial input, a zero-output run is out of room.
+        const bool measured = !Resampling(instance);
         if (input.size() - consumed < needed) {
-            if (input.size() != consumed) result |= AJM_RESULT_PARTIAL_INPUT;
+            if (input.size() != consumed || (measured && !input.empty())) result |= AJM_RESULT_PARTIAL_INPUT;
+            if (measured && !input.empty() && !roomForFrame) result |= AJM_RESULT_NOT_ENOUGH_ROOM;
             break;
         }
-        if (Resampling(instance) ? pcmOutputs.Room() < channels * sampleBytes : pcmOutputs.Room() < frameBytes) {
-            if (frames == 0 && pcmOutputs.produced == 0) result |= AJM_RESULT_NOT_ENOUGH_ROOM;
+        if (!roomForFrame) {
+            if (measured || (frames == 0 && pcmOutputs.produced == 0)) result |= AJM_RESULT_NOT_ENOUGH_ROOM;
             break;
         }
         if (instance.superframeRemaining == 0) {
@@ -797,6 +818,7 @@ void RunAt9(Instance& instance, const JobHeader& job, const AjmBuffer* inputs, c
         ++frames;
 
         EmitFrame(instance, pcmOutputs, pcm.data(), frameSamples, channels, sampleBytes);
+        if (job.console && (job.flags & RUN_MULTIPLE_FRAMES) == 0) break;
     }
     if (segmentEnded()) instance.segmentSamples = 0;
 
@@ -1692,6 +1714,169 @@ void Execute(const JobHeader& job, const AjmBuffer* inputs, const AjmBuffer* out
     }
 }
 
+int RegisterBatch(std::uint32_t context, AjmBatchError* error, std::uint32_t* batch) {
+    if (error) std::memset(error, 0, sizeof(*error));
+    const auto id = g_nextBatch.fetch_add(1, std::memory_order_relaxed);
+    {
+        std::lock_guard lock(g_batchLock);
+        g_batches.emplace(id, context);
+    }
+    *batch = id;
+    return 0;
+}
+
+// The PS4 batch buffer (sceAjmBatchStartBuffer) is not private: the PS4 SDK builds it with inline code
+// in the title, so it has the console's layout, taken from shadPS4's reimplementation of the PS4
+// library. It is a run of chunks, each starting with a 32-bit word that holds the chunk kind in bits
+// 0-5 and a payload in bits 6-25. A job chunk has the instance as payload and, in a second word, the
+// byte count of the chunks after it that describe the job; an inline chunk holds that many bytes of data
+// a job's control buffer may point to. A job's flags chunk has flags bits 32-51 as payload and bits 0-31
+// in its second word; its buffer chunks have a 32-bit size and a 64-bit address.
+enum class ConsoleChunk : std::uint32_t {
+    Job = 0,
+    InputRunBuffer = 1,
+    InputControlBuffer = 2,
+    ControlFlags = 3,
+    RunFlags = 4,
+    ReturnAddress = 6,
+    InlineBuffer = 7,
+    OutputRunBuffer = 17,
+    OutputControlBuffer = 18,
+};
+
+constexpr std::uint32_t CONSOLE_STATISTICS_INSTANCE = 0x80000;
+constexpr std::uint64_t CONSOLE_FLAGS_VERSION_AND_CODEC = 0x7FF;
+
+// The low header word as the PS4's libSceAjm builders write it (a trace of 32 545 batches a Wwise title
+// started on a PS4): bits 0-3 the kind, bit 4 the output direction of a buffer chunk
+// (0x11 output run, 0x12 output control), bits 6-25 the job's instance or the flags' bits 32-51. The
+// builders leave bits 5, 26, 27 and 31 as they were, and a title that reuses its batch memory (Wwise
+// does) carries the previous batch's bytes there: a 6-bit kind read job headers as 16, 32 or 48 and run
+// flags as 20, 36 or 52, and a 26-bit payload read flags bit 52 set.
+ConsoleChunk ConsoleChunkKind(std::uint32_t word) {
+    const std::uint32_t kind = word & 0xFu;
+    return static_cast<ConsoleChunk>(kind == 1 || kind == 2 ? kind | (word & 0x10u) : kind);
+}
+
+std::uint32_t ConsoleChunkPayload(std::uint32_t word) {
+    return (word >> 6) & 0xFFFFFu;
+}
+
+std::uint8_t* PutConsoleChunk(std::uint8_t* at, ConsoleChunk kind, std::uint32_t payload, std::uint32_t word) {
+    const std::uint32_t words[2] = {static_cast<std::uint32_t>(kind) | ((payload & 0xFFFFFu) << 6), word};
+    std::memcpy(at, words, sizeof(words));
+    return at + sizeof(words);
+}
+
+std::uint8_t* PutConsoleFlags(std::uint8_t* at, ConsoleChunk kind, std::uint64_t flags) {
+    return PutConsoleChunk(at, kind, static_cast<std::uint32_t>(flags >> 32), static_cast<std::uint32_t>(flags));
+}
+
+std::uint8_t* PutConsoleBuffer(std::uint8_t* at, ConsoleChunk kind, const void* address, std::size_t size) {
+    at = PutConsoleChunk(at, kind, 0, static_cast<std::uint32_t>(size));
+    std::memcpy(at, &address, sizeof(address));
+    return at + sizeof(address);
+}
+
+std::uint8_t* BeginConsoleJob(std::uint8_t* job, const void* returnAddress) {
+    std::uint8_t* at = job + 2 * sizeof(std::uint32_t);
+    return returnAddress ? PutConsoleBuffer(at, ConsoleChunk::ReturnAddress, returnAddress, 0) : at;
+}
+
+void* EndConsoleJob(std::uint8_t* job, std::uint8_t* end, std::uint32_t instance) {
+    PutConsoleChunk(job, ConsoleChunk::Job, instance, static_cast<std::uint32_t>(end - job) - 2 * sizeof(std::uint32_t));
+    return end;
+}
+
+struct ConsoleJob {
+    JobHeader header;
+    std::vector<AjmBuffer> buffers;
+};
+
+// What sceAjmBatchStartBuffer reports in the AjmBatchError when it refuses a batch: the error, the job
+// chunk, the offending chunk's offset in the job and the job's return address, as a PS4 fills them for a
+// buffer chunk with a null address and a nonzero size (INVALID_ADDRESS, nothing of the batch run). The
+// other refusals here report MALFORMED_BATCH the same way; they were not measured.
+struct ConsoleRefusal {
+    std::int32_t code = SCE_AJM_ERROR_MALFORMED_BATCH;
+    const std::uint8_t* job = nullptr;
+    std::uint32_t offset = 0;
+    const void* returnAddress = nullptr;
+};
+
+// Reads one console job: the chunks between the job header at `job` and `end`. A buffer chunk of size
+// zero is dropped whatever its address, as a PS4 does. Returns false with the refusal filled for a null
+// address with a nonzero size, for chunks that overrun the job and for a job without flags.
+bool ReadConsoleJob(std::uint32_t instance, const std::uint8_t* job, const std::uint8_t* end, ConsoleJob& out, ConsoleRefusal& refusal) {
+    out.header = JobHeader{};
+    out.header.instance = instance;
+    out.header.console = true;
+    std::vector<AjmBuffer> inputs;
+    std::vector<AjmBuffer> outputs;
+    AjmBuffer controlInput{};
+    AjmBuffer controlOutput{};
+    ConsoleChunk flagsKind = ConsoleChunk::Job;
+    refusal = ConsoleRefusal{};
+    refusal.job = job;
+    const std::uint8_t* at = job + 2 * sizeof(std::uint32_t);
+    while (at < end) {
+        refusal.offset = static_cast<std::uint32_t>(at - job);
+        if (end - at < 8) return false;
+        std::uint32_t words[2];
+        std::memcpy(words, at, sizeof(words));
+        const auto kind = ConsoleChunkKind(words[0]);
+        if (kind == ConsoleChunk::ControlFlags || kind == ConsoleChunk::RunFlags) {
+            if (flagsKind != ConsoleChunk::Job) return false;
+            flagsKind = kind;
+            out.header.flags = (std::uint64_t{ConsoleChunkPayload(words[0])} << 32) | words[1];
+            at += sizeof(words);
+            continue;
+        }
+        if (kind != ConsoleChunk::InputRunBuffer && kind != ConsoleChunk::InputControlBuffer && kind != ConsoleChunk::ReturnAddress && kind != ConsoleChunk::OutputRunBuffer && kind != ConsoleChunk::OutputControlBuffer) {
+            NotImplemented_nid_no_patch(("sceAjmBatchStartBuffer (job chunk kind " + std::to_string(static_cast<unsigned>(kind)) + ")").c_str());
+        }
+        if (end - at < 16) return false;
+        AjmBuffer buffer{nullptr, words[1]};
+        std::memcpy(&buffer.ptr, at + sizeof(words), sizeof(buffer.ptr));
+        at += 16;
+        if (kind == ConsoleChunk::ReturnAddress) {
+            refusal.returnAddress = buffer.ptr;
+            continue;
+        }
+        if (buffer.size == 0) continue;
+        if (!buffer.ptr) {
+            refusal.code = SCE_AJM_ERROR_INVALID_ADDRESS;
+            return false;
+        }
+        if (kind == ConsoleChunk::InputRunBuffer) inputs.push_back(buffer);
+        if (kind == ConsoleChunk::OutputRunBuffer) outputs.push_back(buffer);
+        if (kind == ConsoleChunk::InputControlBuffer) controlInput = buffer;
+        if (kind == ConsoleChunk::OutputControlBuffer) controlOutput = buffer;
+    }
+    refusal.offset = static_cast<std::uint32_t>(end - job);
+    if (flagsKind == ConsoleChunk::Job) return false;
+    JobHeader& header = out.header;
+    header.sideband = controlOutput.ptr;
+    header.sidebandSize = controlOutput.size;
+    if (instance == CONSOLE_STATISTICS_INSTANCE) {
+        header.kind = JobKind::GetStatistics;
+    } else if (flagsKind == ConsoleChunk::ControlFlags) {
+        constexpr std::uint64_t implemented = CONSOLE_FLAGS_VERSION_AND_CODEC | CONTROL_RESET | CONTROL_INITIALIZE | SIDEBAND_GAPLESS_DECODE;
+        if ((header.flags & ~implemented) != 0) NotImplemented_nid_no_patch("sceAjmBatchStartBuffer (control job flags other than RESET, INITIALIZE and SIDEBAND_GAPLESS_DECODE)");
+        if (!inputs.empty() || !outputs.empty()) NotImplemented_nid_no_patch("sceAjmBatchStartBuffer (control job with run buffers)");
+        header.kind = JobKind::Control;
+        if (controlInput.size) inputs.push_back(controlInput);
+    } else {
+        if (controlInput.size) NotImplemented_nid_no_patch("sceAjmBatchStartBuffer (run job with an input control buffer)");
+        header.kind = JobKind::Run;
+    }
+    header.inputCount = static_cast<std::uint32_t>(inputs.size());
+    header.outputCount = static_cast<std::uint32_t>(outputs.size());
+    AJM_TRACE("[ajm] console job: instance %u, %s flags 0x%llx, %u input and %u output buffers, control input %llu and output %llu bytes\n", instance, flagsKind == ConsoleChunk::ControlFlags ? "control" : "run", static_cast<unsigned long long>(header.flags), header.inputCount, header.outputCount, static_cast<unsigned long long>(controlInput.size), static_cast<unsigned long long>(controlOutput.size));
+    out.buffers = std::move(inputs);
+    out.buffers.insert(out.buffers.end(), outputs.begin(), outputs.end());
+    return true;
+}
 }
 
 extern "C" {
@@ -1893,14 +2078,19 @@ int APS5_VABI sceAjmBatchJobGetStatistics(AjmBatchInfo* info, float interval, vo
 int APS5_VABI sceAjmBatchStart(uint32_t context, const AjmBatchInfo* info, int priority, AjmBatchError* error, uint32_t* batch) {
     if (context == 0 || context >= g_nextContext.load(std::memory_order_relaxed)) return SCE_AJM_ERROR_INVALID_CONTEXT;
     (void)priority;
-    if (!info || !batch) return SCE_AJM_ERROR_INVALID_PARAMETER;
+    if (!info || !batch || !info->p_buffer || info->offset > info->size) return SCE_AJM_ERROR_INVALID_PARAMETER;
     AJM_TRACE("[ajm] batch start: context %u, priority %d, %llu of %llu buffer bytes used\n", context, priority, static_cast<unsigned long long>(info->offset), static_cast<unsigned long long>(info->size));
     const auto* cursor = static_cast<const std::uint8_t*>(info->p_buffer);
-    const auto* end = cursor + info->offset;
+    auto remaining = info->offset;
     std::vector<AjmBuffer> buffers;
-    while (cursor < end) {
+    while (remaining != 0) {
+        if (remaining < sizeof(PackedJob)) NotImplemented_nid_no_patch("AJM batch contains a truncated private job header");
         PackedJob record;
         std::memcpy(&record, cursor, sizeof(record));
+        const std::size_t bufferCount = static_cast<std::size_t>(record.inputCount) + record.outputCount;
+        const std::size_t parameterBytes = (static_cast<std::size_t>(record.parameterSize) + 7u) & ~std::size_t{7};
+        const std::size_t requiredBytes = sizeof(PackedJob) + bufferCount * sizeof(AjmBuffer) + parameterBytes;
+        if (record.parameterSize > sizeof(JobHeader::parameters) || record.bytes < requiredBytes || record.bytes > remaining) NotImplemented_nid_no_patch("AJM batch contains an invalid private job size");
         JobHeader job{};
         job.kind = static_cast<JobKind>(record.kind);
         job.bytes = record.bytes;
@@ -1915,16 +2105,10 @@ int APS5_VABI sceAjmBatchStart(uint32_t context, const AjmBatchInfo* info, int p
         if (!buffers.empty()) std::memcpy(buffers.data(), cursor + sizeof(record), buffers.size() * sizeof(AjmBuffer));
         if (record.parameterSize) std::memcpy(job.parameters, cursor + sizeof(record) + buffers.size() * sizeof(AjmBuffer), record.parameterSize);
         Execute(job, buffers.data(), buffers.data() + job.inputCount);
-        cursor += record.bytes;
+        remaining -= record.bytes;
+        if (remaining != 0) cursor += record.bytes;
     }
-    if (error) std::memset(error, 0, sizeof(*error));
-    const auto id = g_nextBatch.fetch_add(1, std::memory_order_relaxed);
-    {
-        std::lock_guard lock(g_batchLock);
-        g_batches.emplace(id, context);
-    }
-    *batch = id;
-    return 0;
+    return RegisterBatch(context, error, batch);
 }
 
 int APS5_VABI sceAjmBatchWait(uint32_t context, uint32_t batch, uint32_t timeout, AjmBatchError* error) {
@@ -1953,4 +2137,69 @@ int APS5_VABI sceAjmBatchErrorDump(const AjmBatchInfo* info, AjmBatchError* erro
     return 0;
 }
 
+void* APS5_VABI sceAjmBatchJobControlBufferRa(void* position, uint32_t instance, uint64_t flags, const void* sideband_input, size_t sideband_input_size, void* sideband_output, size_t sideband_output_size, void* return_address) {
+    auto* job = static_cast<std::uint8_t*>(position);
+    auto* at = BeginConsoleJob(job, return_address);
+    at = PutConsoleBuffer(at, ConsoleChunk::InputControlBuffer, sideband_input, sideband_input_size);
+    at = PutConsoleFlags(at, ConsoleChunk::ControlFlags, flags);
+    at = PutConsoleBuffer(at, ConsoleChunk::OutputControlBuffer, sideband_output, sideband_output_size);
+    return EndConsoleJob(job, at, instance);
+}
+
+void* APS5_VABI sceAjmBatchJobRunSplitBufferRa(void* position, uint32_t instance, uint64_t flags, const AjmBuffer* input_buffers, size_t input_buffers_num, const AjmBuffer* output_buffers, size_t output_buffers_num, void* sideband_output, size_t sideband_output_size, void* return_address) {
+    if ((input_buffers_num && !input_buffers) || (output_buffers_num && !output_buffers)) NotImplemented_nid_no_patch("AJM batch job with a null buffer list and a nonzero buffer count");
+    auto* job = static_cast<std::uint8_t*>(position);
+    auto* at = BeginConsoleJob(job, return_address);
+    for (std::size_t index = 0; index < input_buffers_num; ++index) at = PutConsoleBuffer(at, ConsoleChunk::InputRunBuffer, input_buffers[index].ptr, input_buffers[index].size);
+    at = PutConsoleFlags(at, ConsoleChunk::RunFlags, flags);
+    for (std::size_t index = 0; index < output_buffers_num; ++index) at = PutConsoleBuffer(at, ConsoleChunk::OutputRunBuffer, output_buffers[index].ptr, output_buffers[index].size);
+    at = PutConsoleBuffer(at, ConsoleChunk::OutputControlBuffer, sideband_output, sideband_output_size);
+    return EndConsoleJob(job, at, instance);
+}
+
+void* APS5_VABI sceAjmBatchJobRunBufferRa(void* position, uint32_t instance, uint64_t flags, const void* data_input, size_t data_input_size, void* data_output, size_t data_output_size, void* sideband_output, size_t sideband_output_size, void* return_address) {
+    const AjmBuffer input{const_cast<void*>(data_input), data_input_size};
+    const AjmBuffer output{data_output, data_output_size};
+    return sceAjmBatchJobRunSplitBufferRa(position, instance, flags, &input, 1, &output, 1, sideband_output, sideband_output_size, return_address);
+}
+
+int APS5_VABI sceAjmBatchStartBuffer(uint32_t context, const uint8_t* batch, uint32_t batch_size, int priority, AjmBatchError* error, uint32_t* batch_id) {
+    if (context == 0 || context >= g_nextContext.load(std::memory_order_relaxed)) return SCE_AJM_ERROR_INVALID_CONTEXT;
+    if (!batch_id || !batch) return SCE_AJM_ERROR_INVALID_PARAMETER;
+    AJM_TRACE("[ajm] batch start buffer: context %u, priority %d, %u bytes\n", context, priority, batch_size);
+    std::vector<ConsoleJob> jobs;
+    ConsoleRefusal refusal;
+    const std::uint8_t* at = batch;
+    const std::uint8_t* end = batch + batch_size;
+    bool accepted = batch_size % 8 == 0;
+    while (accepted && at < end) {
+        refusal = ConsoleRefusal{};
+        refusal.job = at;
+        if (end - at < 8) {
+            accepted = false;
+            break;
+        }
+        std::uint32_t words[2];
+        std::memcpy(words, at, sizeof(words));
+        const std::uint8_t* body = at + sizeof(words);
+        if (words[1] > static_cast<std::size_t>(end - body)) {
+            accepted = false;
+            break;
+        }
+        const auto kind = ConsoleChunkKind(words[0]);
+        if (kind == ConsoleChunk::Job) {
+            accepted = ReadConsoleJob(ConsoleChunkPayload(words[0]), at, body + words[1], jobs.emplace_back(), refusal);
+        } else if (kind != ConsoleChunk::InlineBuffer) {
+            NotImplemented_nid_no_patch(("sceAjmBatchStartBuffer (batch chunk kind " + std::to_string(static_cast<unsigned>(kind)) + ")").c_str());
+        }
+        at = body + words[1];
+    }
+    if (!accepted) {
+        if (error) *error = AjmBatchError{refusal.code, refusal.job, refusal.offset, refusal.returnAddress};
+        return SCE_AJM_ERROR_MALFORMED_BATCH;
+    }
+    for (const ConsoleJob& job : jobs) Execute(job.header, job.buffers.data(), job.buffers.data() + job.header.inputCount);
+    return RegisterBatch(context, error, batch_id);
+}
+
 }

core/libs/prx/libSceAjm/Export.cpp

@@ -6,19 +6,6 @@
 
 extern "C" {
 
-int APS5_VABI sceAjmBatchJobControlBufferRa() {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
-}
-
-int APS5_VABI sceAjmBatchJobRunSplitBufferRa() {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
-}
-
-int APS5_VABI sceAjmBatchStartBuffer() {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
-}
+APS5_DUMMY_FUN
 
 }

core/libs/prx/libSceAjm/include/Api.hpp

@@ -4,8 +4,4 @@
 
 extern "C" {
 
-int APS5_VABI sceAjmBatchJobControlBufferRa();
-int APS5_VABI sceAjmBatchJobRunSplitBufferRa();
-int APS5_VABI sceAjmBatchStartBuffer();
-
 }