Merge main into feat/npmanager-idmapper-requests

.github/workflows/web-tests.yml

@@ -36,4 +36,4 @@ jobs:
           node --check demo/web/output-directory.mjs
           node --check demo/web/worker.mjs
       - name: Test browser converter helpers
-        run: node --test demo/web/tests/batch.test.mjs demo/web/tests/output-directory.test.mjs
+        run: node --test demo/web/tests/app.test.mjs demo/web/tests/batch.test.mjs demo/web/tests/output-directory.test.mjs

CMakeLists.txt

@@ -136,10 +136,17 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/tests/test_shader_cache_native_paths.py
             ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ShaderCacheVersion.cmake ${CMAKE_COMMAND})
         set_tests_properties(shader_cache_native_paths PROPERTIES TIMEOUT 20)
-        add_test(NAME shader_cache_runtime_sources COMMAND ${Python3_EXECUTABLE}
-            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_shader_cache_runtime_sources.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler ${CMAKE_COMMAND})
-        set_tests_properties(shader_cache_runtime_sources PROPERTIES TIMEOUT 180)
+        foreach(shard RANGE 0 6)
+            if(shard EQUAL 0)
+                set(shader_cache_sources_test_name shader_cache_runtime_sources)
+            else()
+                set(shader_cache_sources_test_name shader_cache_runtime_sources_shard_${shard})
+            endif()
+            add_test(NAME ${shader_cache_sources_test_name} COMMAND ${Python3_EXECUTABLE}
+                ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_shader_cache_runtime_sources.py
+                ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler ${CMAKE_COMMAND} --shard ${shard}/7)
+            set_tests_properties(${shader_cache_sources_test_name} PROPERTIES TIMEOUT 180)
+        endforeach()
         add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py)
         add_test(NAME pr_overlap_dependencies COMMAND ${Python3_EXECUTABLE}
@@ -241,8 +248,19 @@ if(BUILD_TESTING)
         add_test(NAME pad_timeline COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pad_timeline.py)
         set_tests_properties(pad_timeline PROPERTIES TIMEOUT 20)
+        set(check_compile_modes_source "${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile_modes.py")
+        set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${check_compile_modes_source}")
+        file(STRINGS "${check_compile_modes_source}" check_compile_modes_cases REGEX "^    def test_")
+        foreach(test_case IN LISTS check_compile_modes_cases)
+            string(REGEX REPLACE "^    def (test_[A-Za-z0-9_]+).*" "\\1" test_case "${test_case}")
+            add_test(NAME check_compile_modes_${test_case} COMMAND ${Python3_EXECUTABLE}
+                "${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile_modes.py" CheckCompileModeTests.${test_case})
+            set_tests_properties(check_compile_modes_${test_case} PROPERTIES TIMEOUT 30)
+        endforeach()
         add_test(NAME trace_digest COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_trace_digest.py)
+        add_test(NAME symbolize_samples COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_symbolize_samples.py)
         add_test(NAME conventions_bom_shebang COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_bom_shebang.py)
         set_tests_properties(conventions_bom_shebang PROPERTIES TIMEOUT 20)
@@ -268,6 +286,9 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_alignment.py)
         add_test(NAME conventions_pr_template COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_pr_template.py)
+        add_test(NAME check_compile_preprocessor COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile_preprocessor.py)
+        set_tests_properties(check_compile_preprocessor PROPERTIES TIMEOUT 20)
         add_test(NAME check_compile COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile.py)
         add_test(NAME performance_compare COMMAND ${Python3_EXECUTABLE}

core/libs/CMakeLists.txt

@@ -366,12 +366,28 @@ if(BUILD_TESTING)
     target_link_libraries(guest_kernel_sys_pthread_tests PRIVATE libkernel_sys libkernel libc)
     configure_windows_unwind(guest_kernel_sys_pthread_tests)
     add_test(NAME guest_kernel_sys_pthread COMMAND guest_kernel_sys_pthread_tests)
+    add_test_executable(guest_kernel_sys_constants_tests tests/GuestKernelSysConstants.cpp)
+    target_include_directories(guest_kernel_sys_constants_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_constants_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_constants_tests)
+    add_test(NAME guest_kernel_sys_constants COMMAND guest_kernel_sys_constants_tests)
+    set_tests_properties(guest_kernel_sys_constants PROPERTIES TIMEOUT 20)
+    add_test_executable(guest_firmware_constant_returns_tests tests/GuestFirmwareConstantReturns.cpp)
+    target_include_directories(guest_firmware_constant_returns_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_firmware_constant_returns_tests PRIVATE libSceComposite libSceFsInternalForVsh libSceHttpCache libScePlayReady libSceSdma libSceSsl libSceVisionManager libmdbg_syscore libkernel libc)
+    configure_windows_unwind(guest_firmware_constant_returns_tests)
+    add_test(NAME guest_firmware_constant_returns COMMAND guest_firmware_constant_returns_tests)
+    set_tests_properties(guest_firmware_constant_returns PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_firmware_constant_returns PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceComposite>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceFsInternalForVsh>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceHttpCache>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libScePlayReady>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceSdma>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceSsl>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceVisionManager>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libmdbg_syscore>")
+    endif()
     if(Python3_Interpreter_FOUND)
         add_test(NAME nid_verneed COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_verneed.py $<TARGET_FILE:nid_patcher>)
     endif()
     if(WIN32)
-        set_tests_properties(guest_kernel_sys guest_kernel_sys_thread_defaults guest_kernel_sys_barrier guest_kernel_sys_pthread PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(guest_kernel_sys guest_kernel_sys_thread_defaults guest_kernel_sys_barrier guest_kernel_sys_pthread guest_kernel_sys_constants PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
     endif()
     add_test_executable(guest_kernel_sys_threads_tests tests/GuestKernelSysThreads.cpp
@@ -429,6 +445,23 @@ if(BUILD_TESTING)
         endif()
         add_test(NAME windows_write_watch COMMAND windows_write_watch_tests)
         set_tests_properties(windows_write_watch PROPERTIES TIMEOUT 20)
+        add_test_executable(windows_shared_map_cost_tests tests/WindowsSharedMapCost.cpp)
+        target_include_directories(windows_shared_map_cost_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+        target_compile_definitions(windows_shared_map_cost_tests PRIVATE _WIN32_WINNT=0x0A00)
+        if(MINGW)
+            target_compile_options(windows_shared_map_cost_tests PRIVATE -fasynchronous-unwind-tables)
+        endif()
+        add_test(NAME windows_shared_map_cost COMMAND windows_shared_map_cost_tests)
+        set_tests_properties(windows_shared_map_cost PROPERTIES TIMEOUT 120)
+    endif()
+    add_test_executable(guest_kernel_sys_login_tests tests/GuestKernelSysLogin.cpp)
+    target_include_directories(guest_kernel_sys_login_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_login_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_login_tests)
+    add_test(NAME guest_kernel_sys_login COMMAND guest_kernel_sys_login_tests)
+    if(WIN32)
+        set_tests_properties(guest_kernel_sys_login PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
     endif()
 
     if(WIN32)
@@ -511,6 +544,17 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceMove>")
     endif()
 
+    add_test_executable(guest_pthread_create_suspended_tests tests/GuestPthreadCreateSuspended.cpp)
+    target_include_directories(guest_pthread_create_suspended_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_pthread_create_suspended_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_pthread_create_suspended_tests)
+    add_test(NAME guest_pthread_create_suspended COMMAND guest_pthread_create_suspended_tests)
+    set_tests_properties(guest_pthread_create_suspended PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_pthread_create_suspended PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+    endif()
+
     add_test_executable(guest_np_party_tests tests/GuestNpParty.cpp)
     target_include_directories(guest_np_party_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_np_party_tests PRIVATE libSceNpParty libc)
@@ -881,6 +925,18 @@ if(BUILD_TESTING)
     endif()
     add_test(NAME MemoryPoolTest COMMAND MemoryPoolTest)
 
+    add_test_executable(guest_kernel_sys_swapcontext_tests tests/GuestKernelSysSwapcontext.cpp)
+    target_include_directories(guest_kernel_sys_swapcontext_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_compile_options(guest_kernel_sys_swapcontext_tests PRIVATE -fno-omit-frame-pointer)
+    target_link_libraries(guest_kernel_sys_swapcontext_tests PRIVATE libkernel_sys libSceLibcInternal libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_swapcontext_tests)
+    add_test(NAME guest_kernel_sys_swapcontext COMMAND guest_kernel_sys_swapcontext_tests)
+    set_tests_properties(guest_kernel_sys_swapcontext PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_kernel_sys_swapcontext PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+    endif()
+
     add_test_executable(guest_acm_tests tests/GuestAcm.cpp)
     target_include_directories(guest_acm_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_acm_tests PRIVATE libSceAcm libc)

core/libs/SceTypes.hpp

@@ -1502,6 +1502,15 @@ struct SharePlayConnectionInfoA {
     std::int32_t visitorUserId;
 };
 static_assert(sizeof(SharePlayConnectionInfoA) == 72 && offsetof(SharePlayConnectionInfoA, hostAccountId) == 48 && offsetof(SharePlayConnectionInfoA, hostUserId) == 64);
+struct SharePlayConnectionInfo {
+    std::int32_t status;
+    std::int32_t mode;
+    NpOnlineId hostOnlineId;
+    NpOnlineId visitorOnlineId;
+    std::int32_t hostUserId;
+    std::int32_t visitorUserId;
+};
+static_assert(sizeof(SharePlayConnectionInfo) == 56 && offsetof(SharePlayConnectionInfo, hostUserId) == 48);
 struct NpId { NpOnlineId online_id; std::uint8_t opaque[4]; };
 struct NpCreateAsyncRequestParameter { std::uint8_t opaque[64]; };
 struct NpCheckPremiumParameter { std::uint8_t opaque[64]; };

core/libs/nid/include/nid/ElfPatcher.hpp

@@ -22,6 +22,17 @@ struct Elf64_Ehdr {
     std::uint16_t e_shstrndx;
 };
 
+struct Elf64_Phdr {
+    std::uint32_t p_type;
+    std::uint32_t p_flags;
+    std::uint64_t p_offset;
+    std::uint64_t p_vaddr;
+    std::uint64_t p_paddr;
+    std::uint64_t p_filesz;
+    std::uint64_t p_memsz;
+    std::uint64_t p_align;
+};
+
 struct Elf64_Shdr {
     std::uint32_t sh_name;
     std::uint32_t sh_type;
@@ -103,7 +114,7 @@ constexpr std::uint16_t kShnUndef = 0u;
 
 class ElfNidPatcher final : public IBinaryPatcher {
 public:
-    void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const override;
+    void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options = {}) const override;
 };
 
 }

core/libs/nid/include/nid/IBinaryPatcher.hpp

@@ -8,11 +8,16 @@
 
 namespace Nid {
 
+struct PatchOptions {
+    bool executable = false;
+    bool onlySce = false;
+};
+
 class IBinaryPatcher {
 public:
     virtual ~IBinaryPatcher() = default;
 
-    virtual void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const = 0;
+    virtual void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options = {}) const = 0;
 };
 
 }

core/libs/nid/include/nid/MachONidPatcher.hpp

@@ -22,7 +22,7 @@ std::vector<MachOExport> ReadMachOExports(const std::vector<std::uint8_t>& binar
 
 class MachONidPatcher final : public IBinaryPatcher {
 public:
-    void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const override;
+    void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options = {}) const override;
 };
 
 }

core/libs/nid/include/nid/NidResolver.hpp

@@ -1,6 +1,7 @@
 #ifndef NID_NIDRESOLVER_HPP
 #define NID_NIDRESOLVER_HPP
 
+#include <nid/IBinaryPatcher.hpp>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -8,6 +9,10 @@
 
 namespace Nid {
 
+bool IsSceImport(const std::string& name);
+
+std::unordered_map<std::string, std::string> ResolveImportNids(const std::vector<std::string>& names, const std::unordered_set<std::string>& excluded, const PatchOptions& options);
+
 std::string ResolveOneName(const std::string& funcName);
 
 std::unordered_map<std::string, std::string> ResolveNids(const std::vector<std::string>& exportedNames, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports);

core/libs/nid/include/nid/PeNidPatcher.hpp

@@ -39,7 +39,7 @@ struct PeSectionHeader {
 
 class PeNidPatcher final : public IBinaryPatcher {
 public:
-    void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const override;
+    void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options = {}) const override;
 };
 
 }

core/libs/nid/src/ElfNidPatcher.cpp

@@ -354,7 +354,7 @@ std::vector<std::uint32_t> CollectVerneedNameOffsets(const std::vector<std::uint
 
 }
 
-void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const {
+void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options) const {
     using namespace Internal;
 
     if (elf.size() < sizeof(Elf64_Ehdr)) throw std::runtime_error("file too small");
@@ -370,11 +370,13 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
     std::size_t sysvHashOffset = 0u, sysvHashSize = 0u;
     std::uint32_t sysvHashLink = 0u;
     std::size_t versymOffset = 0u;
+    bool hasDynamic = false;
     std::uint32_t dynSymSectionIndex = 0u;
     std::uint32_t gnuHashSymOffset = 0u;
     for (std::uint16_t i = 0u; i < ehdr.e_shnum; ++i) {
         const std::size_t shOffset = static_cast<std::size_t>(ehdr.e_shoff) + i * sizeof(Elf64_Shdr);
         const auto shdr = Read<Elf64_Shdr>(elf, shOffset);
+        if (shdr.sh_type == kShtDynamic) hasDynamic = true;
         if (shdr.sh_type == kShtDynsym) {
             if (shdr.sh_entsize != sizeof(Elf64_Sym) || shdr.sh_size % sizeof(Elf64_Sym) || shdr.sh_link >= ehdr.e_shnum)
                 throw std::runtime_error("invalid ELF dynsym layout");
@@ -398,6 +400,19 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
             throw std::runtime_error("unsupported .gnu.version_d section: symbol version definitions cannot be preserved");
     }
 
+    if (options.executable && dynSymOffset == 0u && !hasDynamic) {
+        if (ehdr.e_phnum) {
+            if (!ehdr.e_phoff || ehdr.e_phentsize != sizeof(Elf64_Phdr) || ehdr.e_phnum == 0xffffu)
+                throw std::runtime_error("unsupported ELF program header layout");
+            RequireElfRange(elf, ehdr.e_phoff, std::uint64_t{ehdr.e_phnum} * sizeof(Elf64_Phdr), "program headers");
+            for (std::uint16_t index = 0u; index < ehdr.e_phnum; ++index) {
+                const auto segment = Read<Elf64_Phdr>(elf, ehdr.e_phoff + index * sizeof(Elf64_Phdr));
+                if (segment.p_type == 2u)
+                    throw std::runtime_error("unsupported ELF PT_DYNAMIC without a .dynsym section");
+            }
+        }
+        return;
+    }
     if (dynSymOffset == 0u) throw std::runtime_error("no .dynsym section");
 
     const std::size_t symCount = dynSymSize / sizeof(Elf64_Sym);
@@ -437,19 +452,26 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
         elf.begin() + static_cast<std::ptrdiff_t>(dynStrOffset + dynStrSize)
     );
 
+    std::vector<std::string> importedNames;
     std::vector<std::string> exportedNames;
     for (std::size_t i = 1u; i < symCount; ++i) {
         const std::size_t symOffset = dynSymOffset + i * sizeof(Elf64_Sym);
         const auto sym = Read<Elf64_Sym>(elf, symOffset);
         if (sym.st_name == 0u) continue;
         const std::uint8_t binding = sym.st_info >> 4u;
-        if (binding == kStbLocal || sym.st_shndx == kShnUndef) continue;
+        if (binding == kStbLocal) continue;
+        if (sym.st_shndx == kShnUndef) {
+            const auto name = ReadCStr(origDynStr, sym.st_name);
+            if (!name.empty()) importedNames.push_back(name);
+            continue;
+        }
         const std::string symName = ReadCStr(origDynStr, sym.st_name);
         if (symName.empty()) continue;
         exportedNames.push_back(symName);
     }
 
-    const auto nidMap = ResolveNids(exportedNames, libraryName, excludedExports);
+    const auto nidMap = options.executable ? std::unordered_map<std::string, std::string>{} : ResolveNids(exportedNames, libraryName, excludedExports);
+    const auto importMap = ResolveImportNids(importedNames, excludedExports, options);
 
     struct SymbolNameUse {
         std::size_t symIndex;
@@ -481,14 +503,11 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
         } else if (binding != kStbLocal && sym.st_shndx == kShnUndef) {
             const bool isUnversioned = versymOffset == 0u ||
                 (Read<std::uint16_t>(elf, versymOffset + i * sizeof(std::uint16_t)) <= 1u);
-            const bool hasNidPostfix = symName.size() >= kNidPostfixLen &&
-                symName.compare(symName.size() - kNidPostfixLen, kNidPostfixLen, kNidPostfix) == 0;
-            const bool hasScePrefix = symName.size() >= 3u &&
-                std::tolower(static_cast<unsigned char>(symName[0])) == 's' &&
-                std::tolower(static_cast<unsigned char>(symName[1])) == 'c' &&
-                std::tolower(static_cast<unsigned char>(symName[2])) == 'e';
-            if (isUnversioned && (hasNidPostfix || hasScePrefix))
-                newValue = ResolveOneName(symName);
+            if (options.executable || isUnversioned) {
+                newValue = importMap.at(symName);
+                if (options.executable && newValue != symName && versymOffset != 0u)
+                    Write(elf, versymOffset + i * sizeof(std::uint16_t), std::uint16_t{1});
+            }
         }
 
         uses.push_back(SymbolNameUse{i, sym.st_name, newValue});
@@ -538,11 +557,69 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
     for (const auto offset : preservedOffsets)
         preservedToNewOffset.emplace_back(offset, valueOffsets.at(ReadCStr(origDynStr, offset)));
 
+    if (newDynStr.size() > dynStrSize && options.executable) {
+        if (ehdr.e_phentsize != sizeof(Elf64_Phdr) || ehdr.e_phnum == 0 || ehdr.e_phnum >= 0xfffdu)
+            throw std::runtime_error("invalid ELF program header table for string growth");
+        RequireElfRange(elf, ehdr.e_phoff, std::uint64_t{ehdr.e_phnum} * sizeof(Elf64_Phdr), "program headers");
+        const auto align = [](std::uint64_t value) {
+            if (value > UINT64_MAX - 4095u) throw std::runtime_error("ELF address overflow");
+            return (value + 4095u) & ~std::uint64_t{4095u};
+        };
+        std::vector<Elf64_Phdr> segments;
+        std::uint64_t virtualEnd = 0;
+        bool hasPhdr = false;
+        for (std::size_t index = 0; index < ehdr.e_phnum; ++index) {
+            const auto segment = Read<Elf64_Phdr>(elf, ehdr.e_phoff + index * sizeof(Elf64_Phdr));
+            segments.push_back(segment);
+            hasPhdr |= segment.p_type == 6;
+            if (segment.p_type != 1) continue;
+            if (segment.p_memsz > UINT64_MAX - segment.p_vaddr) throw std::runtime_error("ELF segment address overflow");
+            virtualEnd = std::max(virtualEnd, segment.p_vaddr + segment.p_memsz);
+        }
+        if (!virtualEnd) throw std::runtime_error("ELF executable has no load segment");
+        if (!hasPhdr) segments.insert(segments.begin(), Elf64_Phdr{6, 4, 0, 0, 0, 0, 0, 8});
+        const auto fileOffset = align(elf.size());
+        const auto address = align(virtualEnd);
+        const auto tableSize = (segments.size() + 1u) * sizeof(Elf64_Phdr);
+        const auto segmentSize = tableSize + newDynStr.size();
+        if (segmentSize > UINT64_MAX - address || segmentSize > SIZE_MAX - fileOffset) throw std::runtime_error("ELF string segment overflow");
+        for (auto& segment : segments) {
+            if (segment.p_type != 6) continue;
+            segment = {6, 4, fileOffset, address, address, tableSize, tableSize, 8};
+        }
+        segments.push_back({1, 4, fileOffset, address, address, segmentSize, segmentSize, 4096});
+        elf.resize(static_cast<std::size_t>(fileOffset + segmentSize), 0);
+        for (std::size_t index = 0; index < segments.size(); ++index)
+            Write(elf, fileOffset + index * sizeof(Elf64_Phdr), segments[index]);
+        Write(elf, offsetof(Elf64_Ehdr, e_phoff), fileOffset);
+        Write(elf, offsetof(Elf64_Ehdr, e_phnum), static_cast<std::uint16_t>(segments.size()));
+        dynStrOffset = static_cast<std::size_t>(fileOffset + tableSize);
+        dynStrSize = newDynStr.size();
+        const auto stringAddress = address + tableSize;
+        const auto sectionOffset = ehdr.e_shoff + dynStrSectionLink * sizeof(Elf64_Shdr);
+        auto strings = Read<Elf64_Shdr>(elf, sectionOffset);
+        strings.sh_offset = dynStrOffset;
+        strings.sh_addr = stringAddress;
+        strings.sh_size = dynStrSize;
+        Write(elf, sectionOffset, strings);
+        bool foundAddress = false;
+        bool foundSize = false;
+        for (std::size_t index = 0; index < ehdr.e_shnum; ++index) {
+            const auto section = Read<Elf64_Shdr>(elf, ehdr.e_shoff + index * sizeof(Elf64_Shdr));
+            if (section.sh_type != kShtDynamic) continue;
+            RequireElfRange(elf, section.sh_offset, section.sh_size, "dynamic");
+            for (std::uint64_t offset = 0; offset + sizeof(Elf64_Dyn) <= section.sh_size; offset += sizeof(Elf64_Dyn)) {
+                auto entry = Read<Elf64_Dyn>(elf, section.sh_offset + offset);
+                if (entry.d_tag == kDtNull) break;
+                if (entry.d_tag == 5) { entry.d_val = stringAddress; foundAddress = true; }
+                if (entry.d_tag == 10) { entry.d_val = dynStrSize; foundSize = true; }
+                Write(elf, section.sh_offset + offset, entry);
+            }
+        }
+        if (!foundAddress || !foundSize) throw std::runtime_error("ELF executable lacks DT_STRTAB or DT_STRSZ");
+    }
     if (newDynStr.size() > dynStrSize) {
-        throw std::runtime_error(
-            "rebuilt .dynstr (" + std::to_string(newDynStr.size()) + " bytes) exceeds original section size (" +
-            std::to_string(dynStrSize) + " bytes) - relinking with a larger section is required"
-        );
+        throw std::runtime_error("rebuilt .dynstr exceeds original section size; relinking with a larger section is required");
     }
     newDynStr.resize(dynStrSize, 0u);
 

core/libs/nid/src/MachONidPatcher.cpp

@@ -341,15 +341,6 @@ std::string StripUnderscore(const std::string& name) {
     return !name.empty() && name[0] == '_' ? name.substr(1) : name;
 }
 
-bool IsGuestFacingImport(const std::string& name) {
-    using namespace Internal;
-    const bool hasNidPostfix = name.size() >= kNidPostfixLen && name.compare(name.size() - kNidPostfixLen, kNidPostfixLen, kNidPostfix) == 0;
-    const bool hasScePrefix = name.size() >= 3u &&
-        std::tolower(static_cast<unsigned char>(name[0])) == 's' &&
-        std::tolower(static_cast<unsigned char>(name[1])) == 'c' &&
-        std::tolower(static_cast<unsigned char>(name[2])) == 'e';
-    return hasNidPostfix || hasScePrefix;
-}
 
 std::vector<bool> DylibOrdinalsArePrx(const std::vector<std::uint8_t>& binary, const Image& image) {
     std::vector<bool> result;
@@ -364,11 +355,48 @@ std::vector<bool> DylibOrdinalsArePrx(const std::vector<std::uint8_t>& binary, c
     return result;
 }
 
-void PatchChainedImports(std::vector<std::uint8_t>& binary, const Image& image) {
+void WriteLinkeditData(std::vector<std::uint8_t>& binary, const Image& image, const TrieLocation& location, const std::vector<std::uint8_t>& trie) {
+    const auto oldOffset = Read<std::uint32_t>(binary, location.OffsetField);
+    const auto oldSize = Read<std::uint32_t>(binary, location.SizeField);
+    if (trie.size() <= oldSize) {
+        std::copy(trie.begin(), trie.end(), binary.begin() + oldOffset);
+        std::fill(binary.begin() + oldOffset + static_cast<std::ptrdiff_t>(trie.size()), binary.begin() + oldOffset + oldSize, 0);
+        Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
+        return;
+    }
+    std::optional<std::size_t> linkedit;
+    for (const auto offset : image.Commands) {
+        const auto cmd = Read<LoadCommand>(binary, offset).Cmd;
+        if (cmd == kLcCodeSignature) throw std::runtime_error("signed Mach-O images cannot grow linkedit data");
+        if (cmd != kLcSegment64) continue;
+        const auto segment = Read<SegmentCommand64>(binary, offset);
+        if (std::strncmp(segment.Name, "__LINKEDIT", sizeof(segment.Name)) == 0) linkedit = offset;
+    }
+    if (!linkedit) throw std::runtime_error("Mach-O image has no __LINKEDIT segment");
+    auto segment = Read<SegmentCommand64>(binary, *linkedit);
+    if (segment.FileOffset + segment.FileSize != binary.size()) throw std::runtime_error("__LINKEDIT does not end the file");
+    const auto newOffset = (binary.size() + 7) & ~std::size_t{7};
+    if (newOffset + trie.size() > 0xffffffffu) throw std::runtime_error("linkedit data offset overflow");
+    binary.resize(newOffset, 0);
+    binary.insert(binary.end(), trie.begin(), trie.end());
+    std::fill(binary.begin() + oldOffset, binary.begin() + oldOffset + oldSize, 0);
+    segment.FileSize = binary.size() - segment.FileOffset;
+    segment.VmSize = std::max(segment.VmSize, (segment.FileSize + kPageSize - 1) & ~(kPageSize - 1));
+    Write(binary, *linkedit, segment);
+    Write(binary, location.OffsetField, static_cast<std::uint32_t>(newOffset));
+    Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
+}
+
+void PatchChainedImports(std::vector<std::uint8_t>& binary, const Image& image, const std::unordered_set<std::string>& excluded, const PatchOptions& options) {
+    std::size_t commandOffset = 0;
     std::optional<LinkeditDataCommand> fixups;
     for (const auto offset : image.Commands) {
         const auto cmd = Read<LoadCommand>(binary, offset).Cmd;
-        if (cmd == kLcDyldChainedFixups) fixups = Read<LinkeditDataCommand>(binary, offset);
+        if (cmd == kLcDyldChainedFixups) {
+            fixups = Read<LinkeditDataCommand>(binary, offset);
+            commandOffset = offset;
+        }
+        if (options.executable && cmd == kLcCodeSignature) throw std::runtime_error("remove the Mach-O code signature before patching and sign again afterwards");
         if ((cmd == kLcDyldInfo || cmd == kLcDyldInfoOnly)) {
             const auto info = Read<DyldInfoCommand>(binary, offset);
             if (info.BindSize != 0 || info.LazyBindSize != 0 || info.WeakBindSize != 0) throw std::runtime_error("Mach-O bind opcodes are not supported; link with chained fixups");
@@ -400,61 +428,72 @@ void PatchChainedImports(std::vector<std::uint8_t>& binary, const Image& image)
             imports.push_back({raw >> 9, raw & 0xff});
         }
     }
-    std::set<std::uint32_t> nameOffsets;
-    for (const auto& item : imports) nameOffsets.insert(item.NameOffset);
-    std::set<std::uint32_t> rewritten;
+    std::vector<std::string> names;
+    std::vector<std::string> originalNames;
+    std::vector<bool> selected;
     for (const auto& item : imports) {
-        const bool special = header.ImportsFormat == kImportFormatAddend64 ? item.Ordinal >= 0xfff0 : item.Ordinal >= 0xf0;
-        if (item.Ordinal == 0 || special || item.Ordinal > prx.size() || !prx[item.Ordinal - 1]) continue;
-        if (rewritten.contains(item.NameOffset)) continue;
         const auto at = symbols + item.NameOffset;
         if (at >= symbolsEnd) throw std::runtime_error("chained import name out of bounds");
         const auto name = Internal::ReadCStr(binary, at);
         if (at + name.size() >= symbolsEnd) throw std::runtime_error("unterminated chained import name");
         const auto bare = StripUnderscore(name);
-        if (bare == name || !IsGuestFacingImport(bare)) continue;
-        const auto patched = "_" + ResolveOneName(bare);
-        if (patched.size() > name.size()) throw std::runtime_error("import NID longer than original name: " + name);
-        const auto inside = nameOffsets.upper_bound(item.NameOffset);
-        if (inside != nameOffsets.end() && *inside <= item.NameOffset + name.size()) throw std::runtime_error("chained import names share storage: " + name);
-        std::memcpy(binary.data() + at, patched.data(), patched.size());
-        std::fill(binary.begin() + static_cast<std::ptrdiff_t>(at + patched.size()), binary.begin() + static_cast<std::ptrdiff_t>(at + name.size() + 1), 0);
-        rewritten.insert(item.NameOffset);
+        const bool special = header.ImportsFormat == kImportFormatAddend64 ? item.Ordinal >= 0xfff0 : item.Ordinal >= 0xf0;
+        const bool eligible = options.executable || (bare != name && item.Ordinal != 0 && !special && item.Ordinal <= prx.size() && prx[item.Ordinal - 1]);
+        originalNames.push_back(name);
+        selected.push_back(eligible);
+        if (eligible) names.push_back(bare);
     }
-}
-
-void WriteTrie(std::vector<std::uint8_t>& binary, const Image& image, const TrieLocation& location, const std::vector<std::uint8_t>& trie) {
-    const auto oldOffset = Read<std::uint32_t>(binary, location.OffsetField);
-    const auto oldSize = Read<std::uint32_t>(binary, location.SizeField);
-    if (trie.size() <= oldSize) {
-        std::copy(trie.begin(), trie.end(), binary.begin() + oldOffset);
-        std::fill(binary.begin() + oldOffset + static_cast<std::ptrdiff_t>(trie.size()), binary.begin() + oldOffset + oldSize, 0);
-        Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
+    const auto mapping = ResolveImportNids(names, excluded, options);
+    if (!options.executable) {
+        std::set<std::uint32_t> nameOffsets;
+        for (const auto& item : imports) nameOffsets.insert(item.NameOffset);
+        std::set<std::uint32_t> rewritten;
+        for (const auto& item : imports) {
+            const bool special = header.ImportsFormat == kImportFormatAddend64 ? item.Ordinal >= 0xfff0 : item.Ordinal >= 0xf0;
+            if (item.Ordinal == 0 || special || item.Ordinal > prx.size() || !prx[item.Ordinal - 1]) continue;
+            if (rewritten.contains(item.NameOffset)) continue;
+            const auto at = symbols + item.NameOffset;
+            if (at >= symbolsEnd) throw std::runtime_error("chained import name out of bounds");
+            const auto name = Internal::ReadCStr(binary, at);
+            if (at + name.size() >= symbolsEnd) throw std::runtime_error("unterminated chained import name");
+            const auto bare = StripUnderscore(name);
+            if (bare == name) continue;
+            const auto patched = "_" + mapping.at(bare);
+            if (patched.size() > name.size()) throw std::runtime_error("import NID longer than original name: " + name);
+            const auto inside = nameOffsets.upper_bound(item.NameOffset);
+            if (inside != nameOffsets.end() && *inside <= item.NameOffset + name.size()) throw std::runtime_error("chained import names share storage: " + name);
+            std::memcpy(binary.data() + at, patched.data(), patched.size());
+            std::fill(binary.begin() + static_cast<std::ptrdiff_t>(at + patched.size()), binary.begin() + static_cast<std::ptrdiff_t>(at + name.size() + 1), 0);
+            rewritten.insert(item.NameOffset);
+        }
         return;
     }
-    std::optional<std::size_t> linkedit;
-    for (const auto offset : image.Commands) {
-        const auto cmd = Read<LoadCommand>(binary, offset).Cmd;
-        if (cmd == kLcCodeSignature) throw std::runtime_error("signed Mach-O images cannot grow their export trie");
-        if (cmd != kLcSegment64) continue;
-        const auto segment = Read<SegmentCommand64>(binary, offset);
-        if (std::strncmp(segment.Name, "__LINKEDIT", sizeof(segment.Name)) == 0) linkedit = offset;
+    if (imports.empty()) return;
+    std::vector<std::uint8_t> data(binary.begin() + base, binary.begin() + symbolsEnd);
+    const auto newSymbols = data.size();
+    if (newSymbols > 0xffffffffu) throw std::runtime_error("chained symbol offset overflow");
+    Write(data, offsetof(ChainedFixupsHeader, SymbolsOffset), static_cast<std::uint32_t>(newSymbols));
+    for (std::size_t index = 0; index < imports.size(); ++index) {
+        const auto& original = originalNames[index];
+        const auto name = selected[index] ? (original.starts_with("_") ? "_" : "") + mapping.at(StripUnderscore(original)) : original;
+        const auto offset = data.size() - newSymbols;
+        const auto at = header.ImportsOffset + index * entrySize;
+        if (header.ImportsFormat == kImportFormatAddend64) {
+            if (offset > 0xffffffffu) throw std::runtime_error("chained symbol offset overflow");
+            const auto raw = Read<std::uint64_t>(data, at);
+            Write(data, at, (raw & 0xffffffffu) | (std::uint64_t{offset} << 32));
+        } else {
+            if (offset > 0x7fffffu) throw std::runtime_error("chained symbol offset overflow");
+            const auto raw = Read<std::uint32_t>(data, at);
+            Write(data, at, (raw & 0x1ffu) | (static_cast<std::uint32_t>(offset) << 9));
+        }
+        data.insert(data.end(), name.begin(), name.end());
+        data.push_back(0);
     }
-    if (!linkedit) throw std::runtime_error("Mach-O image has no __LINKEDIT segment");
-    auto segment = Read<SegmentCommand64>(binary, *linkedit);
-    if (segment.FileOffset + segment.FileSize != binary.size()) throw std::runtime_error("__LINKEDIT does not end the file");
-    const auto newOffset = (binary.size() + 7) & ~std::size_t{7};
-    if (newOffset + trie.size() > 0xffffffffu) throw std::runtime_error("export trie offset overflow");
-    binary.resize(newOffset, 0);
-    binary.insert(binary.end(), trie.begin(), trie.end());
-    std::fill(binary.begin() + oldOffset, binary.begin() + oldOffset + oldSize, 0);
-    segment.FileSize = binary.size() - segment.FileOffset;
-    segment.VmSize = std::max(segment.VmSize, (segment.FileSize + kPageSize - 1) & ~(kPageSize - 1));
-    Write(binary, *linkedit, segment);
-    Write(binary, location.OffsetField, static_cast<std::uint32_t>(newOffset));
-    Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
+    WriteLinkeditData(binary, image, {commandOffset + 8, commandOffset + 12}, data);
 }
 
+
 }
 
 std::vector<MachOExport> ReadMachOExports(const std::vector<std::uint8_t>& binary) {
@@ -464,8 +503,12 @@ std::vector<MachOExport> ReadMachOExports(const std::vector<std::uint8_t>& binar
     return ParseTrie(binary, Read<std::uint32_t>(binary, location->OffsetField), Read<std::uint32_t>(binary, location->SizeField));
 }
 
-void MachONidPatcher::PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const {
+void MachONidPatcher::PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options) const {
     const auto image = ParseImage(binary);
+    if (options.executable) {
+        PatchChainedImports(binary, image, excludedExports, options);
+        return;
+    }
     const auto location = FindTrie(binary, image);
     if (!location) throw std::runtime_error("no export trie");
     auto exports = ParseTrie(binary, Read<std::uint32_t>(binary, location->OffsetField), Read<std::uint32_t>(binary, location->SizeField));
@@ -486,8 +529,8 @@ void MachONidPatcher::PatchNids(std::vector<std::uint8_t>& binary, const std::st
         if (found == nidMap.end()) throw std::runtime_error("symbol not in nid map: " + name);
         exports[index].Name = "_" + found->second;
     }
-    WriteTrie(binary, image, *location, TrieBuilder(exports).Build());
-    PatchChainedImports(binary, ParseImage(binary));
+    WriteLinkeditData(binary, image, *location, TrieBuilder(exports).Build());
+    PatchChainedImports(binary, ParseImage(binary), excludedExports, options);
 }
 
 }

core/libs/nid/src/NidResolver.cpp

@@ -3,21 +3,32 @@
 #include <nid/NidCompute.hpp>
 #include <nid/NidPatcherUtils.hpp>
 #include <cstdint>
+#include <cctype>
 #include <stdexcept>
 #include <unordered_set>
 #include <vector>
 
 namespace Nid {
 
+bool IsSceImport(const std::string& name) {
+    return name.ends_with(Internal::kNidPostfix) || (name.size() >= 3 &&
+        std::tolower(static_cast<unsigned char>(name[0])) == 's' &&
+        std::tolower(static_cast<unsigned char>(name[1])) == 'c' &&
+        std::tolower(static_cast<unsigned char>(name[2])) == 'e');
+}
+
+std::unordered_map<std::string, std::string> ResolveImportNids(const std::vector<std::string>& names, const std::unordered_set<std::string>& excluded, const PatchOptions& options) {
+    const std::unordered_set<std::string> unique(names.begin(), names.end());
+    auto result = ResolveNids(std::vector<std::string>(unique.begin(), unique.end()), "", excluded);
+    for (const auto& name : unique) {
+        if ((!options.executable || options.onlySce) && !IsSceImport(name)) result.at(name) = name;
+        else if (!options.executable) result.at(name) = ResolveOneName(name);
+    }
+    return result;
+}
+
 std::string ResolveOneName(const std::string& funcName) {
-    using namespace Internal;
-
-    if (IsNidNoPatchCut(funcName))
-        return StripNidNoPatchCut(funcName);
-    if (IsNidNoPatch(funcName))
-        return funcName;
-
-    return ComputeNid(StripNidPostfix(funcName), "");
+    return ResolveNids({funcName}, "", {}).at(funcName);
 }
 
 std::unordered_map<std::string, std::string> ResolveNids(const std::vector<std::string>& exportedNames, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) {

core/libs/nid/src/PeNidPatcher.cpp

@@ -36,7 +36,7 @@ std::size_t RvaToOffset(const std::vector<std::uint8_t>& pe, std::uint32_t rva,
 
 }
 
-void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const {
+void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options) const {
     using namespace Internal;
 
     if (pe.size() < 0x40) throw std::runtime_error("file too small");
@@ -54,7 +54,7 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
     const std::size_t sectionTableOffset = coffHeaderOffset + 20u + sizeOfOptionalHeader;
     const auto symbolTable = Read<std::uint32_t>(pe, coffHeaderOffset + 8u);
     const auto symbolCount = Read<std::uint32_t>(pe, coffHeaderOffset + 12u);
-    for (std::uint16_t i = 0; i < numberOfSections; ++i) {
+    for (std::uint16_t i = 0; !options.executable && i < numberOfSections; ++i) {
         const auto offset = sectionTableOffset + i * sizeof(PeSectionHeader);
         auto section = Read<PeSectionHeader>(pe, offset);
         if (pe[offset] != '/' || !symbolTable) continue;
@@ -79,159 +79,162 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
         throw std::runtime_error("unsupported optional header magic");
     }
 
-    const auto exportDir = Read<PeDataDirectory>(pe, dataDirectoryOffset);
-    if (exportDir.VirtualAddress == 0u) throw std::runtime_error("no export directory");
+    if (!options.executable) {
+        const auto exportDir = Read<PeDataDirectory>(pe, dataDirectoryOffset);
+        if (exportDir.VirtualAddress == 0u) throw std::runtime_error("no export directory");
 
-    const std::size_t exportDirOffset = RvaToOffset(pe, exportDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
-    const auto exportTable = Read<PeExportDirectory>(pe, exportDirOffset);
+        const std::size_t exportDirOffset = RvaToOffset(pe, exportDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+        const auto exportTable = Read<PeExportDirectory>(pe, exportDirOffset);
 
-    if (exportTable.NumberOfNames == 0u) throw std::runtime_error("no exported names");
+        if (exportTable.NumberOfNames == 0u) throw std::runtime_error("no exported names");
 
-    const std::size_t namesArrayOffset = RvaToOffset(pe, exportTable.AddressOfNames, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
-    const std::size_t ordinalsArrayOffset = RvaToOffset(pe, exportTable.AddressOfNameOrdinals, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+        const std::size_t namesArrayOffset = RvaToOffset(pe, exportTable.AddressOfNames, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+        const std::size_t ordinalsArrayOffset = RvaToOffset(pe, exportTable.AddressOfNameOrdinals, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
 
-    std::vector<std::uint32_t> nameRvas(exportTable.NumberOfNames);
-    std::vector<std::string> names(exportTable.NumberOfNames);
-    for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
-        const auto nameRva = Read<std::uint32_t>(pe, namesArrayOffset + i * 4u);
-        const std::size_t nameOffset = RvaToOffset(pe, nameRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
-        const std::string name = ReadCStr(pe, nameOffset);
-        if (name.empty()) throw std::runtime_error("empty exported name");
-        nameRvas[i] = nameRva;
-        names[i] = name;
-    }
-
-    const auto nidMap = ResolveNids(names, libraryName, excludedExports);
-
-    const std::size_t edataSectionOffset = FindSectionOffsetByRva(pe, exportDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
-    const auto edataSection = Read<PeSectionHeader>(pe, edataSectionOffset);
-
-    const auto fileAlignment = Read<std::uint32_t>(pe, optionalHeaderOffset + 36u);
-    if (fileAlignment == 0u) throw std::runtime_error("invalid file alignment");
-
-    std::uint32_t stringsRegionStart = std::numeric_limits<std::uint32_t>::max();
-    for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
-        const std::uint32_t rvaFromSection = nameRvas[i] - edataSection.VirtualAddress;
-        if (rvaFromSection < stringsRegionStart) stringsRegionStart = rvaFromSection;
-    }
-    if (stringsRegionStart == std::numeric_limits<std::uint32_t>::max())
-        throw std::runtime_error("could not determine strings region start");
-
-    std::uint32_t rawLimit = ((edataSection.SizeOfRawData + fileAlignment - 1u) / fileAlignment) * fileAlignment;
-    const auto dllName = ReadCStr(pe, RvaToOffset(pe, exportTable.Name, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
-    const bool repackDllName = exportTable.Name >= edataSection.VirtualAddress &&
-        exportTable.Name - edataSection.VirtualAddress < rawLimit;
-    if (repackDllName) stringsRegionStart = std::min(stringsRegionStart, exportTable.Name - edataSection.VirtualAddress);
-    struct Forwarder {
-        std::size_t EntryOffset;
-        std::string Target;
-    };
-    std::vector<Forwarder> forwarders;
-    std::vector<std::uint32_t> directRvas;
-    const auto functionsArrayOffset = RvaToOffset(pe, exportTable.AddressOfFunctions, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
-    for (std::uint32_t index = 0u; index < exportTable.NumberOfFunctions; ++index) {
-        const auto entryOffset = functionsArrayOffset + index * sizeof(std::uint32_t);
-        const auto rva = Read<std::uint32_t>(pe, entryOffset);
-        if (rva < exportDir.VirtualAddress || std::uint64_t{rva} - exportDir.VirtualAddress >= exportDir.Size) {
-            directRvas.push_back(rva);
-            continue;
+        std::vector<std::uint32_t> nameRvas(exportTable.NumberOfNames);
+        std::vector<std::string> names(exportTable.NumberOfNames);
+        for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
+            const auto nameRva = Read<std::uint32_t>(pe, namesArrayOffset + i * 4u);
+            const std::size_t nameOffset = RvaToOffset(pe, nameRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+            const std::string name = ReadCStr(pe, nameOffset);
+            if (name.empty()) throw std::runtime_error("empty exported name");
+            nameRvas[i] = nameRva;
+            names[i] = name;
         }
-        const auto target = ReadCStr(pe, RvaToOffset(pe, rva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
-        if (target.size() + 1u > exportDir.Size - (rva - exportDir.VirtualAddress))
-            throw std::runtime_error("forwarder string outside export directory");
-        forwarders.push_back({entryOffset, target});
-        stringsRegionStart = std::min(stringsRegionStart, rva - edataSection.VirtualAddress);
-    }
-    for (const auto rva : directRvas) {
-        if (rva >= edataSection.VirtualAddress && rva - edataSection.VirtualAddress >= stringsRegionStart)
-            rawLimit = std::min(rawLimit, rva - edataSection.VirtualAddress);
-    }
-    const auto reserveTable = [&](std::uint32_t rva, std::uint64_t size) {
-        if (rva < edataSection.VirtualAddress || rva - edataSection.VirtualAddress >= rawLimit) return;
-        const auto end = std::uint64_t{rva - edataSection.VirtualAddress} + size;
-        if (end > rawLimit) throw std::runtime_error("export table outside edata section");
-        stringsRegionStart = std::max(stringsRegionStart, static_cast<std::uint32_t>(end));
-    };
-    reserveTable(exportDir.VirtualAddress, sizeof(PeExportDirectory));
-    reserveTable(exportTable.AddressOfFunctions, std::uint64_t{exportTable.NumberOfFunctions} * 4u);
-    reserveTable(exportTable.AddressOfNames, std::uint64_t{exportTable.NumberOfNames} * 4u);
-    reserveTable(exportTable.AddressOfNameOrdinals, std::uint64_t{exportTable.NumberOfNames} * 2u);
 
-    std::vector<std::string> finalNames(exportTable.NumberOfNames);
-    for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
-        const auto it = nidMap.find(names[i]);
-        if (it == nidMap.end()) throw std::runtime_error("symbol not in nid map: " + names[i]);
-        finalNames[i] = it->second;
-    }
+        const auto nidMap = ResolveNids(names, libraryName, excludedExports);
 
-    std::vector<std::uint8_t> strings;
-    std::unordered_map<std::string, std::uint32_t> stringRvas;
-    const auto storeString = [&](const std::string& name) {
-        const auto existing = stringRvas.find(name);
-        if (existing != stringRvas.end()) return existing->second;
-        const auto rva = edataSection.VirtualAddress + stringsRegionStart + static_cast<std::uint32_t>(strings.size());
-        stringRvas.emplace(name, rva);
-        strings.insert(strings.end(), name.begin(), name.end());
-        strings.push_back(0u);
-        return rva;
-    };
-    std::vector<std::uint32_t> forwarderRvas;
-    for (const auto& forwarder : forwarders) forwarderRvas.push_back(storeString(forwarder.Target));
-    if (!forwarders.empty() && (std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart < exportDir.VirtualAddress ||
-        std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart + strings.size() > std::uint64_t{exportDir.VirtualAddress} + exportDir.Size))
-        throw std::runtime_error("not enough export directory space to repack forwarders");
-    const auto dllNameRva = repackDllName ? storeString(dllName) : exportTable.Name;
-    std::vector<std::uint32_t> finalNameRvas;
-    for (const auto& name : finalNames) finalNameRvas.push_back(storeString(name));
-    const auto requiredSize = std::uint64_t{stringsRegionStart} + strings.size();
-    if (requiredSize > rawLimit)
-        throw std::runtime_error("not enough raw space in edata section to repack export names");
-    const auto directoryEnd = std::max(std::uint64_t{exportDir.VirtualAddress} + exportDir.Size,
-        std::uint64_t{edataSection.VirtualAddress} + requiredSize);
-    for (const auto rva : directRvas) {
-        if (rva != 0u && rva >= exportDir.VirtualAddress && std::uint64_t{rva} < directoryEnd)
-            throw std::runtime_error("repacked export directory would cover a direct export");
-    }
-    const auto directorySize = directoryEnd - exportDir.VirtualAddress;
-    if (directorySize > std::numeric_limits<std::uint32_t>::max()) throw std::runtime_error("export directory too large");
-    const auto stringFileOffset = static_cast<std::size_t>(edataSection.PointerToRawData) + stringsRegionStart;
-    if (stringFileOffset > pe.size() || strings.size() > pe.size() - stringFileOffset)
-        throw std::runtime_error("repack write out of file bounds");
-    std::memcpy(pe.data() + stringFileOffset, strings.data(), strings.size());
-    for (std::size_t index = 0u; index < finalNameRvas.size(); ++index)
-        Write(pe, namesArrayOffset + index * 4u, finalNameRvas[index]);
-    for (std::size_t index = 0u; index < forwarders.size(); ++index)
-        Write(pe, forwarders[index].EntryOffset, forwarderRvas[index]);
-    Write(pe, exportDirOffset + offsetof(PeExportDirectory, Name), dllNameRva);
-    Write(pe, dataDirectoryOffset + offsetof(PeDataDirectory, Size), static_cast<std::uint32_t>(directorySize));
+        const std::size_t edataSectionOffset = FindSectionOffsetByRva(pe, exportDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+        const auto edataSection = Read<PeSectionHeader>(pe, edataSectionOffset);
 
-    const std::uint32_t newVirtualSize = std::max(edataSection.VirtualSize, static_cast<std::uint32_t>(requiredSize));
-    auto patchedSection = edataSection;
-    patchedSection.VirtualSize = newVirtualSize;
-    Write(pe, edataSectionOffset, patchedSection);
+        const auto fileAlignment = Read<std::uint32_t>(pe, optionalHeaderOffset + 36u);
+        if (fileAlignment == 0u) throw std::runtime_error("invalid file alignment");
 
-    struct ExportName {
-        std::string Name;
-        std::uint32_t Rva;
-        std::uint16_t Ordinal;
-    };
+        std::uint32_t stringsRegionStart = std::numeric_limits<std::uint32_t>::max();
+        for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
+            const std::uint32_t rvaFromSection = nameRvas[i] - edataSection.VirtualAddress;
+            if (rvaFromSection < stringsRegionStart) stringsRegionStart = rvaFromSection;
+        }
+        if (stringsRegionStart == std::numeric_limits<std::uint32_t>::max())
+            throw std::runtime_error("could not determine strings region start");
+
+        std::uint32_t rawLimit = ((edataSection.SizeOfRawData + fileAlignment - 1u) / fileAlignment) * fileAlignment;
+        const auto dllName = ReadCStr(pe, RvaToOffset(pe, exportTable.Name, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
+        const bool repackDllName = exportTable.Name >= edataSection.VirtualAddress &&
+            exportTable.Name - edataSection.VirtualAddress < rawLimit;
+        if (repackDllName) stringsRegionStart = std::min(stringsRegionStart, exportTable.Name - edataSection.VirtualAddress);
+        struct Forwarder {
+            std::size_t EntryOffset;
+            std::string Target;
+        };
+        std::vector<Forwarder> forwarders;
+        std::vector<std::uint32_t> directRvas;
+        const auto functionsArrayOffset = RvaToOffset(pe, exportTable.AddressOfFunctions, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+        for (std::uint32_t index = 0u; index < exportTable.NumberOfFunctions; ++index) {
+            const auto entryOffset = functionsArrayOffset + index * sizeof(std::uint32_t);
+            const auto rva = Read<std::uint32_t>(pe, entryOffset);
+            if (rva < exportDir.VirtualAddress || std::uint64_t{rva} - exportDir.VirtualAddress >= exportDir.Size) {
+                directRvas.push_back(rva);
+                continue;
+            }
+            const auto target = ReadCStr(pe, RvaToOffset(pe, rva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
+            if (target.size() + 1u > exportDir.Size - (rva - exportDir.VirtualAddress))
+                throw std::runtime_error("forwarder string outside export directory");
+            forwarders.push_back({entryOffset, target});
+            stringsRegionStart = std::min(stringsRegionStart, rva - edataSection.VirtualAddress);
+        }
+        for (const auto rva : directRvas) {
+            if (rva >= edataSection.VirtualAddress && rva - edataSection.VirtualAddress >= stringsRegionStart)
+                rawLimit = std::min(rawLimit, rva - edataSection.VirtualAddress);
+        }
+        const auto reserveTable = [&](std::uint32_t rva, std::uint64_t size) {
+            if (rva < edataSection.VirtualAddress || rva - edataSection.VirtualAddress >= rawLimit) return;
+            const auto end = std::uint64_t{rva - edataSection.VirtualAddress} + size;
+            if (end > rawLimit) throw std::runtime_error("export table outside edata section");
+            stringsRegionStart = std::max(stringsRegionStart, static_cast<std::uint32_t>(end));
+        };
+        reserveTable(exportDir.VirtualAddress, sizeof(PeExportDirectory));
+        reserveTable(exportTable.AddressOfFunctions, std::uint64_t{exportTable.NumberOfFunctions} * 4u);
+        reserveTable(exportTable.AddressOfNames, std::uint64_t{exportTable.NumberOfNames} * 4u);
+        reserveTable(exportTable.AddressOfNameOrdinals, std::uint64_t{exportTable.NumberOfNames} * 2u);
+
+        std::vector<std::string> finalNames(exportTable.NumberOfNames);
+        for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
+            const auto it = nidMap.find(names[i]);
+            if (it == nidMap.end()) throw std::runtime_error("symbol not in nid map: " + names[i]);
+            finalNames[i] = it->second;
+        }
+
+        std::vector<std::uint8_t> strings;
+        std::unordered_map<std::string, std::uint32_t> stringRvas;
+        const auto storeString = [&](const std::string& name) {
+            const auto existing = stringRvas.find(name);
+            if (existing != stringRvas.end()) return existing->second;
+            const auto rva = edataSection.VirtualAddress + stringsRegionStart + static_cast<std::uint32_t>(strings.size());
+            stringRvas.emplace(name, rva);
+            strings.insert(strings.end(), name.begin(), name.end());
+            strings.push_back(0u);
+            return rva;
+        };
+        std::vector<std::uint32_t> forwarderRvas;
+        for (const auto& forwarder : forwarders) forwarderRvas.push_back(storeString(forwarder.Target));
+        if (!forwarders.empty() && (std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart < exportDir.VirtualAddress ||
+            std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart + strings.size() > std::uint64_t{exportDir.VirtualAddress} + exportDir.Size))
+            throw std::runtime_error("not enough export directory space to repack forwarders");
+        const auto dllNameRva = repackDllName ? storeString(dllName) : exportTable.Name;
+        std::vector<std::uint32_t> finalNameRvas;
+        for (const auto& name : finalNames) finalNameRvas.push_back(storeString(name));
+        const auto requiredSize = std::uint64_t{stringsRegionStart} + strings.size();
+        if (requiredSize > rawLimit)
+            throw std::runtime_error("not enough raw space in edata section to repack export names");
+        const auto directoryEnd = std::max(std::uint64_t{exportDir.VirtualAddress} + exportDir.Size,
+            std::uint64_t{edataSection.VirtualAddress} + requiredSize);
+        for (const auto rva : directRvas) {
+            if (rva != 0u && rva >= exportDir.VirtualAddress && std::uint64_t{rva} < directoryEnd)
+                throw std::runtime_error("repacked export directory would cover a direct export");
+        }
+        const auto directorySize = directoryEnd - exportDir.VirtualAddress;
+        if (directorySize > std::numeric_limits<std::uint32_t>::max()) throw std::runtime_error("export directory too large");
+        const auto stringFileOffset = static_cast<std::size_t>(edataSection.PointerToRawData) + stringsRegionStart;
+        if (stringFileOffset > pe.size() || strings.size() > pe.size() - stringFileOffset)
+            throw std::runtime_error("repack write out of file bounds");
+        std::memcpy(pe.data() + stringFileOffset, strings.data(), strings.size());
+        for (std::size_t index = 0u; index < finalNameRvas.size(); ++index)
+            Write(pe, namesArrayOffset + index * 4u, finalNameRvas[index]);
+        for (std::size_t index = 0u; index < forwarders.size(); ++index)
+            Write(pe, forwarders[index].EntryOffset, forwarderRvas[index]);
+        Write(pe, exportDirOffset + offsetof(PeExportDirectory, Name), dllNameRva);
+        Write(pe, dataDirectoryOffset + offsetof(PeDataDirectory, Size), static_cast<std::uint32_t>(directorySize));
+
+        const std::uint32_t newVirtualSize = std::max(edataSection.VirtualSize, static_cast<std::uint32_t>(requiredSize));
+        auto patchedSection = edataSection;
+        patchedSection.VirtualSize = newVirtualSize;
+        Write(pe, edataSectionOffset, patchedSection);
+
+        struct ExportName {
+            std::string Name;
+            std::uint32_t Rva;
+            std::uint16_t Ordinal;
+        };
+
+        std::vector<ExportName> exportNames;
+        exportNames.reserve(exportTable.NumberOfNames);
+        for (std::size_t i = 0; i < exportTable.NumberOfNames; ++i) {
+            const auto nameRva = Read<std::uint32_t>(pe, namesArrayOffset + i * sizeof(std::uint32_t));
+            const auto ordinal = Read<std::uint16_t>(pe, ordinalsArrayOffset + i * sizeof(std::uint16_t));
+            if (ordinal >= exportTable.NumberOfFunctions)
+                throw std::runtime_error("export name ordinal out of bounds");
+            const auto nameOffset = RvaToOffset(pe, nameRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+            exportNames.push_back({ReadCStr(pe, nameOffset), nameRva, ordinal});
+        }
+        std::sort(exportNames.begin(), exportNames.end(), [](const ExportName& left, const ExportName& right) { return left.Name < right.Name; });
+        for (std::size_t i = 0; i < exportNames.size(); ++i) {
+            if (i != 0 && exportNames[i - 1].Name == exportNames[i].Name)
+                throw std::runtime_error("duplicate patched export name: " + exportNames[i].Name);
+            Write(pe, namesArrayOffset + i * sizeof(std::uint32_t), exportNames[i].Rva);
+            Write(pe, ordinalsArrayOffset + i * sizeof(std::uint16_t), exportNames[i].Ordinal);
+        }
 
-    std::vector<ExportName> exportNames;
-    exportNames.reserve(exportTable.NumberOfNames);
-    for (std::size_t i = 0; i < exportTable.NumberOfNames; ++i) {
-        const auto nameRva = Read<std::uint32_t>(pe, namesArrayOffset + i * sizeof(std::uint32_t));
-        const auto ordinal = Read<std::uint16_t>(pe, ordinalsArrayOffset + i * sizeof(std::uint16_t));
-        if (ordinal >= exportTable.NumberOfFunctions)
-            throw std::runtime_error("export name ordinal out of bounds");
-        const auto nameOffset = RvaToOffset(pe, nameRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
-        exportNames.push_back({ReadCStr(pe, nameOffset), nameRva, ordinal});
-    }
-    std::sort(exportNames.begin(), exportNames.end(), [](const ExportName& left, const ExportName& right) { return left.Name < right.Name; });
-    for (std::size_t i = 0; i < exportNames.size(); ++i) {
-        if (i != 0 && exportNames[i - 1].Name == exportNames[i].Name)
-            throw std::runtime_error("duplicate patched export name: " + exportNames[i].Name);
-        Write(pe, namesArrayOffset + i * sizeof(std::uint32_t), exportNames[i].Rva);
-        Write(pe, ordinalsArrayOffset + i * sizeof(std::uint16_t), exportNames[i].Ordinal);
     }
 
     constexpr std::size_t kImportDirIndex = 1u;
@@ -250,6 +253,20 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
         std::uint32_t FirstThunk;
     };
 
+    if (options.executable) {
+        for (const auto index : {4u, 11u, 13u}) {
+            const auto directory = Read<PeDataDirectory>(pe, dataDirectoryOffset + index * sizeof(PeDataDirectory));
+            if (directory.VirtualAddress || directory.Size) throw std::runtime_error("signed, bound or delay-loaded PE imports are not supported in --exec mode");
+        }
+    }
+    struct ImportUse {
+        std::size_t NameOffset;
+        std::size_t LookupOffset;
+        std::size_t AddressOffset;
+        std::string Name;
+    };
+    std::vector<ImportUse> imports;
+    std::vector<std::string> importNames;
     std::size_t importDescOffset = RvaToOffset(pe, importDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
 
     while (true) {
@@ -286,19 +303,9 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
 
                 const std::string funcName = ReadCStr(pe, hintNameOffset + 2u);
                 if (!funcName.empty()) {
-                    const bool hasNidPostfix = funcName.size() >= kNidPostfixLen &&
-                        funcName.compare(funcName.size() - kNidPostfixLen, kNidPostfixLen, kNidPostfix) == 0;
-                    const bool hasScePrefix = funcName.size() >= 3u &&
-                        std::tolower(static_cast<unsigned char>(funcName[0])) == 's' &&
-                        std::tolower(static_cast<unsigned char>(funcName[1])) == 'c' &&
-                        std::tolower(static_cast<unsigned char>(funcName[2])) == 'e';
-                    if (hasNidPostfix || hasScePrefix) {
-                        const std::string nid = ResolveOneName(funcName);
-                        if (nid.size() > funcName.size())
-                            throw std::runtime_error("import NID longer than original name: " + funcName);
-                        std::memcpy(pe.data() + hintNameOffset + 2u, nid.data(), nid.size());
-                        pe[hintNameOffset + 2u + nid.size()] = 0u;
-                    }
+                    const auto addressOffset = RvaToOffset(pe, desc.FirstThunk, peHeaderOffset, numberOfSections, sizeOfOptionalHeader) + thunkOffset - RvaToOffset(pe, thunkRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
+                    imports.push_back({hintNameOffset + 2u, thunkOffset, addressOffset, funcName});
+                    importNames.push_back(funcName);
                 }
             }
 
@@ -307,6 +314,72 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
 
         importDescOffset += sizeof(PeImportDescriptor);
     }
+    const auto importMap = ResolveImportNids(importNames, excludedExports, options);
+    if (!options.executable) {
+        for (const auto& item : imports) {
+            const auto& name = importMap.at(item.Name);
+            if (name.size() > item.Name.size()) throw std::runtime_error("import NID longer than original name: " + item.Name);
+            std::memcpy(pe.data() + item.NameOffset, name.c_str(), name.size() + 1);
+        }
+        return;
+    }
+    std::vector<std::uint8_t> strings;
+    std::unordered_map<std::string, std::uint32_t> stringOffsets;
+    for (const auto& item : imports) {
+        const auto& name = importMap.at(item.Name);
+        if (name == item.Name || stringOffsets.contains(name)) continue;
+        if (strings.size() & 1u) strings.push_back(0);
+        if (strings.size() > 0xffffffffu - name.size() - 3u) throw std::runtime_error("PE import string table overflow");
+        stringOffsets.emplace(name, static_cast<std::uint32_t>(strings.size()));
+        strings.insert(strings.end(), {0, 0});
+        strings.insert(strings.end(), name.begin(), name.end());
+        strings.push_back(0);
+    }
+    if (strings.empty()) return;
+    const auto fileAlignment = Read<std::uint32_t>(pe, optionalHeaderOffset + 36u);
+    const auto sectionAlignment = Read<std::uint32_t>(pe, optionalHeaderOffset + 32u);
+    const auto align = [](std::uint64_t value, std::uint32_t alignment) {
+        if (!alignment || (alignment & (alignment - 1u))) throw std::runtime_error("invalid PE alignment");
+        const auto result = (value + alignment - 1u) & ~std::uint64_t{alignment - 1u};
+        if (result > 0xffffffffu) throw std::runtime_error("PE section overflow");
+        return static_cast<std::uint32_t>(result);
+    };
+    const auto newHeader = sectionTableOffset + numberOfSections * sizeof(PeSectionHeader);
+    auto headerLimit = std::size_t{Read<std::uint32_t>(pe, optionalHeaderOffset + 60u)};
+    std::uint64_t virtualEnd = headerLimit;
+    for (std::uint16_t index = 0; index < numberOfSections; ++index) {
+        const auto section = Read<PeSectionHeader>(pe, sectionTableOffset + index * sizeof(PeSectionHeader));
+        if (section.PointerToRawData) headerLimit = std::min(headerLimit, std::size_t{section.PointerToRawData});
+        virtualEnd = std::max(virtualEnd, std::uint64_t{section.VirtualAddress} + std::max(section.VirtualSize, section.SizeOfRawData));
+    }
+    if (numberOfSections == 0xffffu || newHeader + sizeof(PeSectionHeader) > headerLimit || headerLimit > pe.size()) throw std::runtime_error("no room for PE import section header");
+    PeSectionHeader section{};
+    const char sectionName[] = ".nidimp";
+    std::memcpy(section.Name, sectionName, sizeof(sectionName));
+    section.VirtualSize = static_cast<std::uint32_t>(strings.size());
+    section.VirtualAddress = align(virtualEnd, sectionAlignment);
+    section.PointerToRawData = align(pe.size(), fileAlignment);
+    section.SizeOfRawData = align(strings.size(), fileAlignment);
+    section.Characteristics = 0x40000040u;
+    const auto imageSize = align(std::uint64_t{section.VirtualAddress} + section.VirtualSize, sectionAlignment);
+    const auto rawEnd = std::uint64_t{section.PointerToRawData} + section.SizeOfRawData;
+    if (rawEnd > 0xffffffffu) throw std::runtime_error("PE file size overflow");
+    pe.resize(static_cast<std::size_t>(rawEnd), 0);
+    std::copy(strings.begin(), strings.end(), pe.begin() + section.PointerToRawData);
+    Write(pe, newHeader, section);
+    Write(pe, coffHeaderOffset + 2u, static_cast<std::uint16_t>(numberOfSections + 1u));
+    Write(pe, optionalHeaderOffset + 56u, imageSize);
+    Write(pe, optionalHeaderOffset + 64u, std::uint32_t{0});
+    for (const auto& item : imports) {
+        const auto& name = importMap.at(item.Name);
+        if (name == item.Name) continue;
+        const auto rva = section.VirtualAddress + stringOffsets.at(name);
+        for (const auto offset : {item.LookupOffset, item.AddressOffset}) {
+            if (magic == 0x20bu) Write(pe, offset, std::uint64_t{rva});
+            else Write(pe, offset, rva);
+        }
+    }
+
 }
 
 }

core/libs/nid/src/main.cpp

@@ -28,6 +28,7 @@ void _writeFile(const std::string& path, const std::vector<std::uint8_t>& data)
 int Run(int argc, char* argv[]) {
     if (argc < 3) {
         std::cerr << "Usage: nid_patcher <library_name> [--preserve-exports <unpatched-library> | --preserve-export-list <export-list>] <file> [file2 ...]\n"
+                     "       nid_patcher --exec [--only-sce] [--preserve-exports <unpatched-library> | --preserve-export-list <export-list>] <file> [file2 ...]\n"
                      "       nid_patcher --write-export-list <unpatched-library> <export-list>\n";
         return 1;
     }
@@ -46,14 +47,20 @@ int Run(int argc, char* argv[]) {
         return 0;
     }
 
-    const std::string libraryName = argv[1];
+    Nid::PatchOptions options;
+    options.executable = std::string(argv[1]) == "--exec";
+    const std::string libraryName = options.executable ? "" : argv[1];
     std::unordered_set<std::string> excludedExports;
     std::vector<std::string> paths;
     try {
+        if (!options.executable && libraryName.starts_with("--")) throw std::runtime_error("unknown option: " + libraryName);
         bool hasReference = false;
         for (int i = 2; i < argc; ++i) {
             const std::string argument = argv[i];
-            if (argument == "--preserve-exports" || argument == "--preserve-export-list") {
+            if (argument == "--only-sce") {
+                if (!options.executable || options.onlySce) throw std::runtime_error("--only-sce requires --exec and may appear once");
+                options.onlySce = true;
+            } else if (argument == "--preserve-exports" || argument == "--preserve-export-list") {
                 if (hasReference || i + 1 == argc) throw std::runtime_error(argument + " requires exactly one reference");
                 hasReference = true;
                 excludedExports = argument == "--preserve-exports" ? Nid::ReadExportExclusions(argv[++i]) : Nid::ReadExportList(argv[++i]);
@@ -72,7 +79,7 @@ int Run(int argc, char* argv[]) {
         try {
             auto binary = _readFile(path);
             const auto patcher = Nid::MakePatcher(binary);
-            patcher->PatchNids(binary, libraryName, excludedExports);
+            patcher->PatchNids(binary, libraryName, excludedExports, options);
             _writeFile(path, binary);
             std::cout << "OK: " << path << "\n";
         } catch (const std::exception& e) {

core/libs/nid/tests/test_elf_duplicates.py

@@ -4,7 +4,7 @@ import sys
 import unittest
 from pathlib import Path
 
-from test_symbol_versions import build_image, patch, read_dynsym, IMPORTS, NID_NAME, SHN_UNDEF, STT_FUNC
+from test_symbol_versions import build_image, patch, read_dynsym, IMPORTS, NID_NAME, SHN_UNDEF, STT_FUNC, PATCHER
 
 
 def nid(name):
@@ -21,6 +21,7 @@ def image(names, addresses=None):
     return bytes(build_image(entries, False))
 
 
+@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
 class ElfDuplicateTests(unittest.TestCase):
     def test_exports_patched_to_one_name_are_rejected(self):
         cases = (
@@ -65,5 +66,4 @@ class ElfDuplicateTests(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    PATCHER = Path(sys.argv.pop(1)).resolve()
     unittest.main()

core/libs/nid/tests/test_elf_layout.py

@@ -3,9 +3,10 @@ import sys
 import unittest
 from pathlib import Path
 
-from test_symbol_versions import build_image, symbols, patch, SHOFF
+from test_symbol_versions import build_image, symbols, patch, SHOFF, PATCHER
 
 
+@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
 class ElfLayoutTests(unittest.TestCase):
     def test_declared_layout_must_match_supported_tables(self):
         changes = ((5, "B", 2), (0x3a, "H", 32),
@@ -44,5 +45,4 @@ class ElfLayoutTests(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    PATCHER = Path(sys.argv.pop(1)).resolve()
     unittest.main()

core/libs/nid/tests/test_gnu_hash.py

@@ -3,7 +3,7 @@ import sys
 import unittest
 from pathlib import Path
 
-from test_symbol_versions import build_image, symbols, patch, HASH_OFFSET, SHOFF, read_dynsym, gnu_hash, sections
+from test_symbol_versions import build_image, symbols, patch, HASH_OFFSET, SHOFF, read_dynsym, gnu_hash, sections, PATCHER
 
 
 def sysv_hash(name):
@@ -31,6 +31,7 @@ def sysv_lookup(image, name):
     return None
 
 
+@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
 class GnuHashTests(unittest.TestCase):
     def image(self, buckets=4, bloom=1, shift=6, symoffset=3):
         image = build_image(symbols(False), False)
@@ -95,6 +96,7 @@ class GnuHashTests(unittest.TestCase):
         self.assertEqual(struct.unpack_from("<I", after, HASH_OFFSET)[0], 0)
 
 
+@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
 class SysvHashTests(unittest.TestCase):
     def image(self, both, buckets):
         entries = symbols(False)
@@ -153,5 +155,4 @@ class SysvHashTests(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    PATCHER = Path(sys.argv.pop(1)).resolve()
     unittest.main()

core/libs/nid/tests/test_macho_nids.py

@@ -7,6 +7,7 @@ import subprocess
 import sys
 import tempfile
 from pathlib import Path
+from test_symbol_versions import find_patcher, PATCHER
 
 SUFFIX = bytes([0x51, 0x8D, 0x64, 0xA6, 0x35, 0xDE, 0xD8, 0xC1, 0xE6, 0xB0, 0x39, 0xB1, 0xC3, 0xE5, 0x52, 0x30])
 BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-"
@@ -262,7 +263,7 @@ def check_export_list(patcher, directory):
 def check_refusals(patcher, directory):
     growing = [("_ab", 0, 0x100), ("_cd", 0, 0x108)]
     result, _ = patch(patcher, directory, "signed.dylib", build_dylib(growing, [], signed=True))
-    assert result.returncode != 0 and "signed Mach-O images cannot grow their export trie" in result.stdout + result.stderr, (result.stdout, result.stderr)
+    assert result.returncode != 0 and "signed Mach-O images cannot grow linkedit data" in result.stdout + result.stderr, (result.stdout, result.stderr)
     result, _ = patch(patcher, directory, "opcodes.dylib", build_dylib([("_puts_nid_postfix", 0, 0x100)], [], bind_opcodes=True))
     assert result.returncode != 0 and "bind opcodes are not supported" in result.stdout + result.stderr, (result.stdout, result.stderr)
     result, _ = patch(patcher, directory, "long.dylib", build_dylib([("_puts_nid_postfix", 0, 0x100)], [("_sceXy", 1)]))
@@ -270,7 +271,13 @@ def check_refusals(patcher, directory):
 
 
 def main():
-    patcher = Path(sys.argv[1]).resolve()
+    if len(sys.argv) > 1:
+        patcher = Path(sys.argv[1]).resolve()
+    else:
+        patcher = PATCHER or find_patcher()
+        if patcher is None:
+            print("nid_patcher executable required")
+            sys.exit(0)
     with tempfile.TemporaryDirectory(prefix="anyps5-macho-nid-a\u00e7\u00e3o-\u65e5\u672c-") as directory:
         check_shrinking(patcher, directory)
         check_growing(patcher, directory)

core/libs/nid/tests/test_native_paths.py

@@ -5,16 +5,14 @@ import tempfile
 import unittest
 from pathlib import Path
 
-from test_macho_nids import build_dylib, exports_of, nid
+from test_macho_nids import build_dylib, exports_of, nid, PATCHER
 
 try:
     import resource
 except ImportError:
     resource = None
 
-PATCHER = Path(sys.argv.pop(1)).resolve()
-
-
+@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
 class NativePathTests(unittest.TestCase):
     def invoke(self, *arguments):
         return subprocess.run([str(PATCHER), "libtest", *map(str, arguments)],

core/libs/nid/tests/test_pe_references.py

@@ -5,7 +5,7 @@ import tempfile
 import unittest
 from pathlib import Path
 
-from test_macho_nids import build_dylib, exports_of, nid
+from test_macho_nids import build_dylib, exports_of, nid, PATCHER
 
 
 def reference(*, headers=False, address=0x2000, ordinal=0):
@@ -100,6 +100,7 @@ def forwarding_exports(image, bits):
     return string(table[4]), names, targets, addresses, (export_rva, export_size)
 
 
+@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
 class ReferenceTests(unittest.TestCase):
     def patch_with_reference(self, data):
         with tempfile.TemporaryDirectory() as tmp:
@@ -135,6 +136,7 @@ class ReferenceTests(unittest.TestCase):
         self.assertEqual(image, original)
 
 
+@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
 class ForwardingExportTests(unittest.TestCase):
     def patch(self, image):
         with tempfile.TemporaryDirectory() as temporary:
@@ -241,6 +243,203 @@ class ForwardingExportTests(unittest.TestCase):
         self.assertEqual(after, image)
 
 
+class ExecutableImportTests(unittest.TestCase):
+    names = ["foo", "foo_nid_postfix", "z", "sceExample", "helper_nid_no_patch", "plain_nid_no_patch_cut", "SDL_Init", "scePreserved", "hostFunctionLongName", "alias_nid_disambig12_nid_postfix"]
+
+    def pe(self, bits=64):
+        image = bytearray(2048)
+        image[:2] = b"MZ"
+        struct.pack_into("<I", image, 0x3c, 0x40)
+        optional_size = 240 if bits == 64 else 224
+        struct.pack_into("<IHHIIIHH", image, 0x40, 0x4550, 0x8664 if bits == 64 else 0x14c, 1, 0, 0, 0, optional_size, 0x22)
+        struct.pack_into("<H", image, 0x58, 0x20b if bits == 64 else 0x10b)
+        struct.pack_into("<II", image, 0x58 + 32, 4096, 512)
+        struct.pack_into("<II", image, 0x58 + 56, 8192, 512)
+        directory = 0x58 + (112 if bits == 64 else 96)
+        struct.pack_into("<I", image, directory - 4, 16)
+        struct.pack_into("<II", image, directory + 8, 0x1000, 40)
+        struct.pack_into("<8sIIIIIIHHI", image, 0x58 + optional_size, b".idata", 1536, 0x1000, 1536, 512, 0, 0, 0, 0, 0xc0000040)
+        struct.pack_into("<IIIII", image, 512, 0x1040, 0, 0, 0x1028, 0x10c0)
+        image[552:561] = b"test.prx\0"
+        offset = 0x380
+        for index, name in enumerate(self.names):
+            entry = name.encode() + b"\0"
+            struct.pack_into("<H", image, offset, 0)
+            image[offset + 2:offset + 2 + len(entry)] = entry
+            for table in (576, 704):
+                struct.pack_into("<Q" if bits == 64 else "<I", image, table + index * (bits // 8), offset - 512 + 0x1000)
+            offset += len(entry) + 2
+        return image
+
+    def elf(self):
+        from test_symbol_versions import build_image, SHOFF
+        entries = [(None, 1, 0, 0, 0)] + [(name, 1, 0, 2, 0) for name in self.names] + [("application_export", 1, 9, 2, 0x400100)]
+        import test_symbol_versions
+        from unittest.mock import patch
+        with patch.object(test_symbol_versions, "DYNSYM_OFFSET", 0xa00):
+            image = build_image(entries, False)
+        struct.pack_into("<H", image, 16, 2)
+        struct.pack_into("<Q", image, 32, 64)
+        struct.pack_into("<HH", image, 54, 56, 2)
+        struct.pack_into("<IIQQQQQQ", image, 64, 1, 5, 0, 0x400000, 0x400000, 4096, 8192, 4096)
+        struct.pack_into("<IIQQQQQQ", image, 120, 2, 6, 0x700, 0x400700, 0x400700, 48, 48, 8)
+        struct.pack_into("<H", image, 60, 5)
+        struct.pack_into("<IIQQQQIIQQ", image, SHOFF + 4 * 64, 0, 6, 3, 0x400700, 0x700, 48, 2, 0, 8, 16)
+        size = struct.unpack_from("<Q", image, SHOFF + 2 * 64 + 32)[0]
+        struct.pack_into("<qQqQqQ", image, 0x700, 5, 0x400300, 10, size, 0, 0)
+        return image
+
+    def macho(self):
+        image = bytearray(build_dylib([("_application_export", 0, 0x100)], [("_" + name, 1 if index % 2 else 2) for index, name in enumerate(self.names)]))
+        struct.pack_into("<I", image, 12, 2)
+        return image
+
+    def read_names(self, image, kind):
+        if kind == "elf":
+            from test_symbol_versions import read_dynsym
+            return [name for name, section, _ in read_dynsym(image) if name and section == 0]
+        if kind == "macho":
+            from test_macho_nids import chained_imports
+            return [name[1:] for name, _ in chained_imports(image)]
+        bits = 64 if kind == "pe64" else 32
+        optional_size = 240 if bits == 64 else 224
+        count = struct.unpack_from("<H", image, 0x46)[0]
+        sections = [struct.unpack_from("<8sIIIIIIHHI", image, 0x58 + optional_size + index * 40) for index in range(count)]
+        result = []
+        for index in range(len(self.names)):
+            rva = struct.unpack_from("<Q" if bits == 64 else "<I", image, 576 + index * (bits // 8))[0]
+            self.assertEqual(rva, struct.unpack_from("<Q" if bits == 64 else "<I", image, 704 + index * (bits // 8))[0])
+            section = next(s for s in sections if s[2] <= rva < s[2] + s[3])
+            offset = section[4] + rva - section[2] + 2
+            result.append(image[offset:image.index(0, offset)].decode())
+        return result
+
+    def test_shared_import_rules_and_filter_in_all_formats(self):
+        for kind, build in (("pe64", self.pe), ("pe32", lambda: self.pe(32)), ("elf", self.elf), ("macho", self.macho)):
+            for only_sce in (False, True):
+                with self.subTest(kind=kind, only_sce=only_sce), tempfile.TemporaryDirectory() as directory:
+                    path = Path(directory) / "arbitrary.extension"
+                    image = build()
+                    path.write_bytes(image)
+                    excluded = Path(directory) / "preserved.txt"
+                    excluded.write_bytes(b"scePreserved\n")
+                    args = [str(PATCHER), "--exec", "--preserve-export-list", str(excluded)]
+                    if only_sce:
+                        args.append("--only-sce")
+                    result = subprocess.run([*args, str(path)], capture_output=True, text=True, timeout=20)
+                    self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+                    expected = ["foo", nid("foo"), nid("z"), nid("sceExample"), "helper_nid_no_patch", "plain", "SDL_Init", "scePreserved", nid("hostFunctionLongName"), nid("alias")]
+                    if only_sce:
+                        expected[2] = "z"
+                        expected[5] = "plain_nid_no_patch_cut"
+                        expected[8] = "hostFunctionLongName"
+                    self.assertCountEqual(self.read_names(path.read_bytes(), kind), expected)
+                    if kind == "macho":
+                        self.assertEqual(exports_of(path.read_bytes()), exports_of(image))
+                    if kind == "elf":
+                        from test_symbol_versions import read_dynsym
+                        self.assertIn(("application_export", 9, 0x400100), read_dynsym(path.read_bytes()))
+
+    def test_exec_accepts_macho_without_exports(self):
+        from test_macho_nids import LC_DYLD_EXPORTS_TRIE, command_fields
+        image = self.macho()
+        command = command_fields(image)[LC_DYLD_EXPORTS_TRIE][0]
+        struct.pack_into("<II", image, command + 8, 0, 0)
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "program"
+            path.write_bytes(image)
+            result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
+            self.assertEqual(result.returncode, 0, result.stderr)
+
+    def test_elf_without_sections_rejects_dynamic_segment_without_writing(self):
+        image = self.elf()
+        struct.pack_into("<Q", image, 40, 0)
+        struct.pack_into("<HH", image, 60, 0, 0)
+        for flags in ([], ["--only-sce"]):
+            with self.subTest(flags=flags), tempfile.TemporaryDirectory() as directory:
+                path = Path(directory) / "program"
+                path.write_bytes(image)
+                result = subprocess.run([str(PATCHER), "--exec", *flags, str(path)], capture_output=True, text=True, timeout=20)
+                self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
+                self.assertIn("PT_DYNAMIC without a .dynsym section", result.stderr)
+                self.assertEqual(path.read_bytes(), image)
+
+    def test_elf_without_sections_or_dynamic_segment_is_unchanged(self):
+        image = self.elf()
+        struct.pack_into("<Q", image, 40, 0)
+        struct.pack_into("<HH", image, 60, 0, 0)
+        struct.pack_into("<H", image, 56, 1)
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "program"
+            path.write_bytes(image)
+            result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
+            self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+            self.assertEqual(path.read_bytes(), image)
+
+    def test_elf_short_imports_grow_without_overwriting_bss(self):
+        from test_symbol_versions import sections
+        self.names = ["a", "b", "c", "d"]
+        image = self.elf()
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "program"
+            path.write_bytes(image)
+            result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            after = path.read_bytes()
+            self.assertCountEqual(self.read_names(after, "elf"), [nid(name) for name in self.names])
+            offset = struct.unpack_from("<Q", after, 32)[0]
+            count = struct.unpack_from("<H", after, 56)[0]
+            segments = [struct.unpack_from("<IIQQQQQQ", after, offset + index * 56) for index in range(count)]
+            original = next(segment for segment in segments if segment[0] == 1 and segment[3] == 0x400000)
+            self.assertEqual(original[5:7], (4096, 8192))
+            phdr = next(segment for segment in segments if segment[0] == 6)
+            load = next(segment for segment in segments if segment[0] == 1 and segment[2] == offset)
+            self.assertEqual(phdr[2:4], load[2:4])
+            strings = next(section for section in sections(after) if section[1] == 3)
+            self.assertEqual(struct.unpack_from("<Q", after, 0x708)[0], strings[3])
+            self.assertEqual(struct.unpack_from("<Q", after, 0x718)[0], strings[5])
+            self.assertGreater(strings[4], len(image))
+
+    def test_pe_own_exports_remain_unchanged(self):
+        image = self.pe()
+        struct.pack_into("<II", image, 0x58 + 112, 0x1500, 96)
+        struct.pack_into("<IIHHIIIIIII", image, 0x700, 0, 0, 0, 0, 0x1540, 1, 1, 1, 0x1528, 0x152c, 0x1530)
+        struct.pack_into("<IIH", image, 0x728, 0x1100, 0x1550, 0)
+        image[0x740:0x748] = b"app.exe\0"
+        image[0x750:0x75b] = b"own_export\0"
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "program"
+            path.write_bytes(image)
+            result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            self.assertEqual(path.read_bytes()[0x700:0x760], image[0x700:0x760])
+
+    def test_unsupported_exec_layouts_fail_without_writing(self):
+        signed = bytearray(build_dylib([], [("_sceExample", 1)], signed=True))
+        struct.pack_into("<I", signed, 12, 2)
+        pe = self.pe()
+        struct.pack_into("<II", pe, 0x58 + 112 + 13 * 8, 0x1400, 32)
+        full_headers = self.pe()
+        struct.pack_into("<I", full_headers, 0x58 + 60, 0x58 + 240 + 40)
+        for image, message in ((signed, "signature"), (pe, "delay-loaded"), (full_headers, "section header"), (b"invalid", "unrecognized")):
+            with self.subTest(message=message), tempfile.TemporaryDirectory() as directory:
+                path = Path(directory) / "program"
+                path.write_bytes(image)
+                result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
+                self.assertEqual(result.returncode, 2)
+                self.assertIn(message, result.stderr)
+                self.assertEqual(path.read_bytes(), image)
+
+    def test_only_sce_requires_exec_and_failure_does_not_write(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "program"
+            image = self.pe()
+            path.write_bytes(image)
+            for arguments in (["libtest", "--only-sce"], ["--only-sce"], ["--exec", "--only-sce", "--only-sce"]):
+                result = subprocess.run([str(PATCHER), *arguments, str(path)], capture_output=True, timeout=20)
+                self.assertNotEqual(result.returncode, 0)
+                self.assertEqual(path.read_bytes(), image)
+
+
 if __name__ == "__main__":
-    PATCHER = Path(sys.argv.pop(1)).resolve()
     unittest.main()

core/libs/nid/tests/test_shared_strings.py

@@ -3,9 +3,10 @@ import sys
 import unittest
 from pathlib import Path
 
-from test_symbol_versions import build_image, symbols, patch, SHOFF, DYNSYM_OFFSET, read_dynsym, sections, DEFINITIONS
+from test_symbol_versions import build_image, symbols, patch, SHOFF, DYNSYM_OFFSET, read_dynsym, sections, DEFINITIONS, PATCHER
 
 
+@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
 class SharedStringTests(unittest.TestCase):
     def test_dynamic_names_keep_original_string_when_export_shares_offset(self):
         for tag in (1, 14, 15, 29):
@@ -54,5 +55,4 @@ class SharedStringTests(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    PATCHER = Path(sys.argv.pop(1)).resolve()
     unittest.main()

core/libs/nid/tests/test_symbol_versions.py

@@ -1,3 +1,4 @@
+import os
 import re
 import struct
 import subprocess
@@ -198,8 +199,36 @@ def test_unversioned_library_is_still_patched(patcher):
     assert len({entry[0] for entry in after if entry[0]}) == len([entry for entry in after if entry[0]])
 
 
+def find_patcher():
+    for index, argument in enumerate(sys.argv[1:], 1):
+        candidate = Path(argument)
+        if candidate.stem == "nid_patcher" and candidate.is_file():
+            sys.argv.pop(index)
+            return candidate.resolve()
+    env = os.environ.get("ANYPS5_NID_PATCHER") or os.environ.get("NID_PATCHER")
+    if env and Path(env).is_file():
+        return Path(env).resolve()
+    repo_root = Path(__file__).resolve().parents[4]
+    for candidate in (
+        repo_root / "build" / "core" / "libs" / "nid_patcher.exe",
+        repo_root / "build" / "core" / "libs" / "nid_patcher",
+    ):
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+PATCHER = find_patcher()
+
+
 def main():
-    patcher = Path(sys.argv[1]).resolve()
+    if len(sys.argv) > 1:
+        patcher = Path(sys.argv[1]).resolve()
+    else:
+        patcher = PATCHER or find_patcher()
+        if patcher is None:
+            print("nid_patcher executable required")
+            sys.exit(0)
     test_versioned_library_is_rejected(patcher)
     test_unversioned_library_is_still_patched(patcher)
     print("NID patcher symbol version tests passed")

core/libs/nid/tests/test_verneed.py

@@ -3,7 +3,7 @@ import sys
 import unittest
 from pathlib import Path
 
-from test_symbol_versions import build_image, symbols, patch, SHOFF, DYNSTR_OFFSET, sections
+from test_symbol_versions import build_image, symbols, patch, SHOFF, DYNSTR_OFFSET, sections, PATCHER
 
 VERNEED = 0x580
 
@@ -22,6 +22,7 @@ def image(*, auxiliary=16, count=1, size=32, version=1, auxnext=0, nextrecord=0)
     return data
 
 
+@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
 class VersionRequirementsTests(unittest.TestCase):
     def test_auxiliary_record_cannot_leave_declared_section(self):
         self.reject(image(auxiliary=32))
@@ -72,5 +73,4 @@ class VersionRequirementsTests(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    PATCHER = Path(sys.argv.pop(1)).resolve()
     unittest.main()

core/libs/prx/libSceAgc/Command/src/Packet.cpp

@@ -115,17 +115,23 @@ std::uint32_t* WriteRegisters(CommandBuffer* buffer, std::uint32_t opcode, const
         Require(buffer != nullptr, function, "null command buffer");
         return buffer->cursor_up;
     }
-    CheckAddress(reinterpret_cast<std::uintptr_t>(registers), 4, function);
+    CheckAddress(reinterpret_cast<std::uintptr_t>(registers), 1, function);
+    const auto* bytes = reinterpret_cast<const std::byte*>(const_cast<const ShaderRegister*>(registers));
+    const auto entryAt = [&](std::uint32_t index) {
+        ShaderRegister entry;
+        std::memcpy(&entry, bytes + static_cast<std::size_t>(index) * sizeof(ShaderRegister), sizeof(entry));
+        return entry;
+    };
     std::vector<ShaderRegister> snapshot;
     if (snapshotAll) {
         snapshot.reserve(count);
         for (std::uint32_t i = 0; i < count; ++i) {
-            snapshot.push_back({registers[i].offset, registers[i].value});
+            snapshot.push_back(entryAt(i));
             CheckBits(snapshot.back().offset, 0xffffu, function);
         }
     }
-    const auto offsetAt = [&](std::uint32_t index) { return snapshotAll ? snapshot[index].offset : registers[index].offset; };
-    const auto valueAt = [&](std::uint32_t index) { return snapshotAll ? snapshot[index].value : registers[index].value; };
+    const auto offsetAt = [&](std::uint32_t index) { return snapshotAll ? snapshot[index].offset : entryAt(index).offset; };
+    const auto valueAt = [&](std::uint32_t index) { return snapshotAll ? snapshot[index].value : entryAt(index).value; };
     std::uint32_t* first = nullptr;
     std::uint32_t index = 0;
     while (index < count) {

core/libs/prx/libSceAgc/tests/RegisterCommands.cpp

@@ -6,6 +6,7 @@
 #include <algorithm>
 #include <array>
 #include <cstdio>
+#include <cstring>
 #include <stdexcept>
 
 namespace {
@@ -99,6 +100,26 @@ void testRangeFromAnyByte() {
     }
 }
 
+void testListFromAnyByte() {
+    const std::array writers{sceAgcCbSetShRegistersDirect, sceAgcCbSetUcRegistersDirect};
+    const std::array opcodes{0x7600u, 0x7900u};
+    const std::array<ShaderRegister, 3> registers{{{0x10u, 0x11223344u}, {0x11u, 0x55667788u}, {0x30u, 0x99aabbccu}}};
+    for (std::size_t i = 0; i < writers.size(); ++i) {
+        for (const std::size_t offset : {1u, 2u, 3u, 5u}) {
+            alignas(16) std::array<unsigned char, 48> bytes{};
+            std::memcpy(bytes.data() + offset, registers.data(), sizeof(registers));
+            Storage storage;
+            storage.words.fill(0xabcdef01u);
+            const auto* list = reinterpret_cast<const volatile ShaderRegister*>(bytes.data() + offset);
+            auto* packet = writers[i](&storage.buffer, list, static_cast<std::uint32_t>(registers.size()));
+            const std::array<std::uint32_t, 7> expected{0xc0020000u | opcodes[i], 0x10u, 0x11223344u, 0x55667788u, 0xc0010000u | opcodes[i], 0x30u, 0x99aabbccu};
+            check(packet == storage.words.data() && storage.buffer.cursor_up == packet + expected.size(), "unaligned register list allocation mismatch");
+            check(std::equal(expected.begin(), expected.end(), packet), "unaligned register list was not read byte for byte");
+            check(packet[expected.size()] == 0xabcdef01u, "unaligned register list overwrote the following word");
+        }
+    }
+}
+
 void testIndirect() {
     const std::array writers{sceAgcDcbSetCxRegistersIndirect, sceAgcDcbSetShRegistersIndirect, sceAgcDcbSetUcRegistersIndirect};
     const std::array sizes{sceAgcDcbSetCxRegistersIndirectGetSize, sceAgcDcbSetShRegistersIndirectGetSize, sceAgcDcbSetUcRegistersIndirectGetSize};
@@ -213,6 +234,7 @@ int main() {
         testDirect();
         testConfig();
         testRangeFromAnyByte();
+        testListFromAnyByte();
         testIndirect();
         testIndirectPlaceholder();
         testDirectList();

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -805,7 +805,9 @@ if(BUILD_TESTING)
         set_tests_properties(agc_import_alias_shared PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 20)
         add_test(NAME agc_compared_shared_generations COMMAND agc_write_tracking_tests --compared-shared-generations)
         set_tests_properties(agc_compared_shared_generations PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 20)
-        set_tests_properties(agc_write_tracking agc_write_tracking_metadata_commit agc_resident_direct_remap agc_rewatched_direct_shared agc_import_alias_shared agc_compared_shared_generations PROPERTIES ENVIRONMENT_MODIFICATION
+        add_test(NAME agc_write_tracking_rewatch_import_pages COMMAND agc_write_tracking_tests --rewatch-import-pages)
+        set_tests_properties(agc_write_tracking_rewatch_import_pages PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 20)
+        set_tests_properties(agc_write_tracking agc_write_tracking_metadata_commit agc_resident_direct_remap agc_rewatched_direct_shared agc_import_alias_shared agc_compared_shared_generations agc_write_tracking_rewatch_import_pages PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 endif()
@@ -860,6 +862,15 @@ if(BUILD_TESTING)
     add_test(NAME agc_fragment_barycentric COMMAND agc_fragment_barycentric_tests)
 endif()
 
+add_test_executable(agc_tessellation_pipeline_cache_tests EXCLUDE_FROM_ALL tests/TessellationPipelineCache.cpp)
+target_include_directories(agc_tessellation_pipeline_cache_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
+set_target_properties(agc_tessellation_pipeline_cache_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_tessellation_pipeline_cache_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_tessellation_pipeline_cache_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_tessellation_pipeline_cache COMMAND agc_tessellation_pipeline_cache_tests)
+endif()
+
 add_test_executable(agc_float32_flush_control_tests EXCLUDE_FROM_ALL tests/Float32FlushControl.cpp)
 target_include_directories(agc_float32_flush_control_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
 set_target_properties(agc_float32_flush_control_tests PROPERTIES CXX_EXTENSIONS OFF)
@@ -1572,6 +1583,8 @@ if(BUILD_TESTING)
     endif()
     add_test(NAME agc_import_watch COMMAND agc_driver_recorder_tests --import-watch-only)
     set_tests_properties(agc_import_watch PROPERTIES SKIP_RETURN_CODE 77)
+    add_test(NAME agc_import_settle_initial_dcc COMMAND agc_driver_recorder_tests --settle-initial-dcc-only)
+    set_tests_properties(agc_import_settle_initial_dcc PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
     if(WIN32)
         add_test(NAME agc_import_watch_other_family COMMAND agc_driver_recorder_tests --import-watch-other-family-only)
         set_tests_properties(agc_import_watch_other_family PROPERTIES SKIP_RETURN_CODE 77)
@@ -1589,7 +1602,7 @@ if(BUILD_TESTING)
         add_test(NAME agc_import_settle_retire COMMAND agc_driver_recorder_tests --settle-retire-only)
         add_test(NAME agc_import_settle_storage_image COMMAND agc_driver_recorder_tests --settle-storage-image-only)
         set_tests_properties(agc_import_settle_deferred agc_import_settle_deferred_other_family agc_import_settle_first_use agc_import_settle_earlier_import agc_import_settle_retire agc_import_settle_storage_image PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 60)
-        set_tests_properties(agc_import_watch agc_import_watch_other_family agc_import_settle_deferred agc_import_settle_deferred_other_family agc_import_settle_first_use agc_import_settle_earlier_import agc_import_settle_retire agc_import_settle_storage_image PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_import_watch agc_import_watch_other_family agc_import_settle_deferred agc_import_settle_deferred_other_family agc_import_settle_first_use agc_import_settle_earlier_import agc_import_settle_retire agc_import_settle_storage_image agc_import_settle_initial_dcc PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     if(WIN32)
@@ -2168,6 +2181,12 @@ if(BUILD_TESTING)
         set_tests_properties(agc_driver_address_passes_targets_split PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_driver_address_passes_draw_transitions COMMAND agc_driver_address_passes_tests --targets)
+    set_tests_properties(agc_driver_address_passes_draw_transitions PROPERTIES ENVIRONMENT "APS5_NO_ADDRESS_READ_PASSES=1;APS5_DRAW_TRANSITIONS=1" SKIP_RETURN_CODE 77 TIMEOUT 60)
+    if(WIN32)
+        set_tests_properties(agc_driver_address_passes_draw_transitions PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     set_target_properties(agc_driver_rcp_hardware_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
     add_test(NAME agc_driver_rcp_hardware_exact COMMAND agc_driver_rcp_hardware_tests)
     set_tests_properties(agc_driver_rcp_hardware_exact PROPERTIES ENVIRONMENT "APS5_EXACT_RCP=1;ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 60)
@@ -2212,6 +2231,7 @@ endif()
 
 target_link_libraries(agc_driver_rect_list_corner_tests PRIVATE shader_recompiler_agc_driver)
 target_link_libraries(agc_driver_queue_zero_late_interrupt_tests PRIVATE libkernel)
+target_link_libraries(agc_driver_linear_target_completion_points_tests PRIVATE libkernel)
 
 target_link_libraries(agc_driver_storage_file_write_tests PRIVATE libkernel)
 
@@ -2226,6 +2246,16 @@ target_include_directories(agc_driver_image_table_cache_bounds_tests PRIVATE $<T
 target_include_directories(agc_driver_draw_cache_relocation_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
 if(BUILD_TESTING)
+    add_test(NAME agc_driver_linear_target_completion_points_sync_flip COMMAND agc_driver_linear_target_completion_points_tests --flip)
+    set_tests_properties(agc_driver_linear_target_completion_points_sync_flip PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_SYNC_FLIP=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_driver_linear_target_completion_points_unpipelined COMMAND agc_driver_linear_target_completion_points_tests --label --interrupt)
+    set_tests_properties(agc_driver_linear_target_completion_points_unpipelined PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=0" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_linear_target_completion_points_sync_flip agc_driver_linear_target_completion_points_unpipelined PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_submit_progress COMMAND agc_driver_submit_progress_tests)
     set_tests_properties(agc_submit_progress PROPERTIES
             ENVIRONMENT "APS5_SUBMIT_PROGRESS=1;ANYPS5_REQUIRE_VULKAN=1" TIMEOUT 30 SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2474,12 +2504,14 @@ if(BUILD_TESTING)
     set_tests_properties(agc_shader_srt_read_patching PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_shader_table_snapshot_reuse COMMAND agc_shader_memory_tests --table-snapshot-reuse)
     set_tests_properties(agc_shader_table_snapshot_reuse PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_shader_table_snapshot_reuse_unmasked COMMAND agc_shader_memory_tests --table-snapshot-reuse)
+    set_tests_properties(agc_shader_table_snapshot_reuse_unmasked PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_MASKED_RESULT_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_driver_image_table_vouched_lookups COMMAND agc_driver_image_table_cache_bounds_tests --vouched-lookups)
     set_tests_properties(agc_driver_image_table_vouched_lookups PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_WRITE_WATCH_IMPORTS=watch" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60)
     add_test(NAME agc_driver_image_table_vouch_queued_draws COMMAND agc_driver_image_table_cache_bounds_tests --queued-draws)
     set_tests_properties(agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60)
     if(WIN32)
-        set_tests_properties(agc_shader_table_snapshot_reuse agc_driver_image_table_vouched_lookups agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_shader_table_snapshot_reuse agc_shader_table_snapshot_reuse_unmasked agc_driver_image_table_vouched_lookups agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_shader_type_zero_image_materialization COMMAND agc_shader_memory_tests --type-zero-image-materialization)
     set_tests_properties(agc_shader_type_zero_image_materialization PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2721,6 +2753,11 @@ if(BUILD_TESTING)
     set_tests_properties(agc_pipeline_journal_long_paths PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_pipeline_library_clear_waits_for_build COMMAND agc_driver_graphics_tests --pipeline-library-clear-waits-for-build)
     add_test(NAME agc_pipeline_store_lazy_buffer_pool COMMAND agc_driver_graphics_tests --pipeline-store-lazy-buffer-pool)
+    add_test(NAME agc_pipeline_store_growth COMMAND agc_driver_graphics_tests --pipeline-store-growth)
+    add_test(NAME agc_pipeline_store_fixed_bound COMMAND agc_driver_graphics_tests --pipeline-store-growth)
+    add_test(NAME agc_pipeline_store_replay_growth COMMAND agc_driver_graphics_tests --pipeline-store-replay-growth)
+    set_tests_properties(agc_pipeline_store_growth agc_pipeline_store_fixed_bound agc_pipeline_store_replay_growth PROPERTIES TIMEOUT 60 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    set_tests_properties(agc_pipeline_store_fixed_bound PROPERTIES ENVIRONMENT "APS5_PIPELINE_STORE_ENTRIES=1024")
     set_tests_properties(agc_pipeline_journal PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     set_tests_properties(agc_pipeline_library_clear_waits_for_build PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     set_tests_properties(agc_pipeline_store_lazy_buffer_pool PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2728,7 +2765,7 @@ if(BUILD_TESTING)
         add_test(NAME agc_pipeline_journal_extended_paths COMMAND agc_driver_graphics_tests --pipeline-journal-extended-paths)
         set_tests_properties(agc_pipeline_journal_extended_paths PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
         set_tests_properties(agc_pipeline_journal_long_paths agc_pipeline_journal_extended_paths PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
-        set_tests_properties(agc_pipeline_journal agc_pipeline_library_clear_waits_for_build agc_pipeline_store_lazy_buffer_pool PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_pipeline_journal agc_pipeline_library_clear_waits_for_build agc_pipeline_store_lazy_buffer_pool agc_pipeline_store_growth agc_pipeline_store_fixed_bound agc_pipeline_store_replay_growth PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     set_tests_properties(agc_driver_sparse_buffer_holes PROPERTIES TIMEOUT 30)
     if(WIN32)
@@ -2756,6 +2793,12 @@ if(BUILD_TESTING)
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 
+    add_test(NAME agc_address_copy_cap COMMAND agc_driver_graphics_tests --address-copy-cap)
+    add_test(NAME agc_address_copy_cap_remembered COMMAND agc_driver_graphics_tests --address-copy-cap-remembered)
+    set_tests_properties(agc_address_copy_cap agc_address_copy_cap_remembered PROPERTIES TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_address_copy_cap agc_address_copy_cap_remembered PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_emulated_color_compare_srgb_decode COMMAND agc_driver_emulated_color_compare_tests)
     set_tests_properties(agc_driver_emulated_color_compare_srgb_decode PROPERTIES
             ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_SRGB_SHADER_DECODE=1"
@@ -3118,7 +3161,7 @@ if(BUILD_TESTING)
         set_tests_properties(agc_driver_array_layer_target_cpu PROPERTIES ENVIRONMENT_MODIFICATION
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
-    foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable registered_workgroup registered_workgroup_sync abi_deferred_compute abi_deferred_pixel abi_deferred_stages abi_deferred_rectangle abi_deferred_undecodable_state abi_deferred_registration_failure abi_deferred_vertex_inputs abi_deferred_rectangle_pairs lazy lazy_sync)
+    foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable registered_workgroup registered_workgroup_sync dispatch_ahead_prewarm abi_deferred_compute abi_deferred_pixel abi_deferred_stages abi_deferred_rectangle abi_deferred_undecodable_state abi_deferred_registration_failure abi_deferred_vertex_inputs abi_deferred_rectangle_pairs lazy lazy_sync)
         set(preparedTest "agc_driver_prepared_shaders_${preparedMode}")
         if(preparedMode MATCHES "^abi_deferred_")
             string(REPLACE "_" "-" preparedArgument ${preparedMode})
@@ -3133,6 +3176,8 @@ if(BUILD_TESTING)
             add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --lazy)
         elseif(preparedMode STREQUAL "registered_workgroup" OR preparedMode STREQUAL "registered_workgroup_sync")
             add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --registered-workgroup)
+        elseif(preparedMode STREQUAL "dispatch_ahead_prewarm")
+            add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --dispatch-ahead-prewarm)
         elseif(preparedMode STREQUAL "debug_failure")
             add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --fail-before-registration)
             set_tests_properties(${preparedTest} PROPERTIES WILL_FAIL TRUE TIMEOUT 30)
@@ -3957,6 +4002,6 @@ endif()
 
 if(BUILD_TESTING)
     get_directory_property(agcRegisteredTests TESTS)
-    list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal(_long_paths|_extended_paths)?|agc_driver_pipeline_journal)$")
+    list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal(_long_paths|_extended_paths)?|agc_driver_pipeline_journal|agc_pipeline_store_replay_growth)$")
     set_property(TEST ${agcRegisteredTests} APPEND PROPERTY ENVIRONMENT "APS5_NO_PIPELINE_JOURNAL=1")
 endif()

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp

@@ -10,7 +10,7 @@ namespace AgcDriver::DriverDetail {
 
 double TraceMs();
 
-std::uint64_t SkippedShader();
+bool SkippedShader(std::uint64_t address);
 
 class RefusedDraw final : public std::runtime_error {
 public:

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp

@@ -55,6 +55,7 @@ public:
     std::uint64_t DecodeDrains() const { return decodeDrains.load(std::memory_order_relaxed); }
     std::optional<std::uint64_t> PendingLabel(std::uint64_t address, std::size_t bytes);
     static bool HeldReads(std::uint64_t address, std::size_t bytes);
+    static std::uint64_t HeldReadQueries();
     void ReleaseRunningReads();
 
     static bool MetadataPassesPipelined();

core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp

@@ -63,7 +63,7 @@ void WriteChanged(std::uint64_t address, std::span<const std::byte> current, std
 bool WriteWatched();
 void Unwatch(std::uint64_t address, std::size_t bytes);
 void Rewatch(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges);
-bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched = true);
+bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched = true, bool importMayWriteGuest = true);
 void SetUnwatchHook(void (*hook)(std::uint64_t address, std::size_t bytes));
 #ifdef _WIN32
 using FreshCollectHook = bool (*)(std::uintptr_t address, std::size_t bytes, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context);

core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp

@@ -451,18 +451,30 @@ public:
 
     static void AppendString(std::string& output, std::string_view text) {
         output += '"';
-        for (const char character : text) {
+        std::size_t index = 0;
+        while (index < text.size()) {
+            const char character = text[index];
             const auto byte = static_cast<unsigned char>(character);
             if (character == '"' || character == '\\') {
                 output += '\\';
                 output += character;
-            } else if (byte < 0x20 || byte >= 0x7f) {
+            } else if (byte >= 0x80) {
+                if (const auto length = utf8SequenceLength(text, index); length != 0) {
+                    output.append(text.substr(index, length));
+                    index += length;
+                    continue;
+                }
+                char escaped[8];
+                std::snprintf(escaped, sizeof escaped, "\\u%04x", static_cast<unsigned>(byte));
+                output += escaped;
+            } else if (byte < 0x20 || byte == 0x7f) {
                 char escaped[8];
                 std::snprintf(escaped, sizeof escaped, "\\u%04x", static_cast<unsigned>(byte));
                 output += escaped;
             } else {
                 output += character;
             }
+            ++index;
         }
         output += '"';
     }
@@ -697,6 +709,34 @@ public:
     }
 
 private:
+    static std::size_t utf8SequenceLength(std::string_view text, std::size_t index) {
+        const auto lead = static_cast<unsigned char>(text[index]);
+        std::size_t length = 0;
+        unsigned char low = 0x80;
+        unsigned char high = 0xbf;
+        if (lead >= 0xc2 && lead <= 0xdf) {
+            length = 2;
+        } else if (lead >= 0xe0 && lead <= 0xef) {
+            length = 3;
+            if (lead == 0xe0) low = 0xa0;
+            if (lead == 0xed) high = 0x9f;
+        } else if (lead >= 0xf0 && lead <= 0xf4) {
+            length = 4;
+            if (lead == 0xf0) low = 0x90;
+            if (lead == 0xf4) high = 0x8f;
+        } else {
+            return 0;
+        }
+        if (text.size() - index < length) return 0;
+        for (std::size_t offset = 1; offset < length; ++offset) {
+            const auto next = static_cast<unsigned char>(text[index + offset]);
+            if (next < low || next > high) return 0;
+            low = 0x80;
+            high = 0xbf;
+        }
+        return length;
+    }
+
     static void AppendMachine(std::string& output, const TraceSummary& summary) {
         if (summary.host) {
             output += ",\"cpu\":";

core/libs/prx/libSceAgcDriver/Execution/include/PresentMode.hpp

@@ -3,6 +3,8 @@
 
 #include <vulkan/vulkan.h>
 #include <algorithm>
+#include <cstdint>
+#include <optional>
 #include <span>
 #include <stdexcept>
 #include <string_view>
@@ -22,6 +24,11 @@ inline VkPresentModeKHR SelectPresentMode(const char* requested, std::span<const
     return mode;
 }
 
+inline std::optional<std::uint32_t> SelectPresentQueue(std::uint32_t familyQueues, std::uint32_t queuesTaken) {
+    if (queuesTaken >= familyQueues) return std::nullopt;
+    return queuesTaken;
+}
+
 }
 
 #endif

core/libs/prx/libSceAgcDriver/Execution/include/SwapchainImages.hpp

@@ -0,0 +1,17 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SWAPCHAINIMAGES_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SWAPCHAINIMAGES_HPP
+
+#include <vulkan/vulkan.h>
+#include <algorithm>
+#include <cstdint>
+
+namespace AgcDriver {
+
+inline std::uint32_t SwapchainImageCount(const VkSurfaceCapabilitiesKHR& surface) {
+    const std::uint32_t count = std::max(surface.minImageCount, 3u);
+    return surface.maxImageCount != 0 ? std::min(count, surface.maxImageCount) : count;
+}
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Execution/include/TessellationPipelineCache.hpp

@@ -0,0 +1,14 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_TESSELLATIONPIPELINECACHE_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_TESSELLATIONPIPELINECACHE_HPP
+
+#include <vulkan/vulkan.h>
+
+namespace AgcDriver {
+
+inline bool TessellationPipelineCacheUsable(VkDriverId driver) {
+    return driver != VK_DRIVER_ID_MOLTENVK;
+}
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp

@@ -45,9 +45,11 @@ struct DeviceQueuePlan {
     bool sparseBinding = false;
     std::uint32_t sparseFamily = VK_QUEUE_FAMILY_IGNORED;
     std::uint32_t sparseIndex = 0;
+    bool present = false;
+    std::uint32_t presentIndex = 0;
 };
 
-DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> families, std::uint32_t family, bool settles, bool sparseBinding);
+DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> families, std::uint32_t family, bool settles, bool sparseBinding, bool presents);
 
 class VulkanDevice {
 public:
@@ -58,6 +60,8 @@ public:
     // Builds the pipeline of a journaled compute result ahead of its first dispatch (the journal
     // replay's hook, on its threads): false when the device already holds a pipeline for the module.
     bool ReplayComputePipeline(const ShaderRecompiler::RecompileResult& shader, bool pushConstants);
+    bool HasComputePipeline(const ShaderRecompiler::RecompileResult& shader) const;
+    void PrebuildComputePipeline(const ShaderRecompiler::RecompileResult& shader, std::uint64_t programAddress = 0);
     // Starts the journal replay if the device has not (a headless device starts it at its first
     // draw or dispatch) and joins it (tests; the teardown joins it anyway).
     void WaitForPipelineJournalReplay();
@@ -226,7 +230,7 @@ public:
     bool ProvokingVertexLast() const;
     bool GraphicsPipelineLibraries() const;
     // A presentation is a few steps so the presenter holds GuestMemory::GpuMutex only while it
-    // touches the queue. Presentations are slots (FlipInFlight() + 1, each with its own command
+    // touches the main queue. Presentations are slots (FlipInFlight() + 1, each with its own command
     // buffer, fence, kept resident image and dump buffer): RetirePresents(keep) (no mutex) retires
     // the slots whose fence signaled, oldest first, and waits for the oldest ones until at most
     // `keep` are in flight (the only CPU wait of the flip path; it returns the time spent waiting);
@@ -235,7 +239,9 @@ public:
     // PresentClear/PresentDisplayBuffer record and submit the frame into the next slot (under the
     // mutex; the blit follows the frame's batches in queue order, so the displayed image is always
     // complete) and return whether one was submitted; QueuePresent hands the image to the swapchain
-    // (under the mutex). FinishPresent = RetirePresents(0): the render fence wait of the
+    // (on the device's presentation queue under that queue's mutex, with the GPU mutex released,
+    // when the presentation family had a free queue; otherwise on the main queue under the GPU
+    // mutex). FinishPresent = RetirePresents(0): the render fence wait of the
     // synchronous paths (APS5_FLIP_INFLIGHT=0, APS5_SYNC_FLIP, PresentPixels). AcquireImage returns
     // false when the swapchain is out of date (the frame is dropped; the next Resize recreates it);
     // a present without a prior AcquireImage acquires itself, one whose slot is in flight waits for
@@ -258,6 +264,7 @@ public:
     bool PresentDisplayBuffer(const DisplayBuffer& buffer);
     double FinishPresent();
     void QueuePresent();
+    bool PresentsOnOwnQueue() const;
     // At the flip packet (under the mutex): the recorder's submissions so far and, under
     // APS5_PROFILE_DRAW, how many in flight have not signaled (the frame record's batches_at_flip).
     void FlipBatches(std::uint64_t& submissions, std::uint64_t& unsignaled) const;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Diagnostics.cpp

@@ -1,11 +1,19 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
+#include <array>
+#include <atomic>
+#include <charconv>
+#include <chrono>
 #include <cstdio>
 #include <cstdlib>
+#include <filesystem>
+#include <fstream>
 #include <mutex>
+#include <optional>
 #include <set>
 #include <stdexcept>
 #include <string_view>
+#include <system_error>
 
 namespace AgcDriver::DriverDetail {
 
@@ -14,9 +22,65 @@ double TraceMs() {
     return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - origin).count();
 }
 
-std::uint64_t SkippedShader() {
-    static const std::uint64_t skipped = [] { const char* text = std::getenv("APS5_SKIP_SHADER"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }();
-    return skipped;
+namespace {
+
+constexpr std::size_t MaxSkippedShaders = 256;
+std::array<std::atomic<std::uint64_t>, MaxSkippedShaders> skippedShaders{};
+std::atomic<std::size_t> skippedShaderCount{0};
+
+std::optional<std::uint64_t> programAddress(std::string_view word) {
+    if (word.starts_with("0x") || word.starts_with("0X")) word.remove_prefix(2);
+    std::uint64_t address = 0;
+    const auto [end, error] = std::from_chars(word.data(), word.data() + word.size(), address, 16);
+    if (word.empty() || error != std::errc{} || end != word.data() + word.size()) return std::nullopt;
+    return address;
+}
+
+void ReloadSkippedShaders(const char* path) {
+    static std::mutex reload;
+    static std::filesystem::file_time_type seen{};
+    const std::unique_lock lock(reload, std::try_to_lock);
+    if (!lock.owns_lock()) return;
+    std::error_code error;
+    const auto stamp = std::filesystem::last_write_time(path, error);
+    if (error || stamp == seen) return;
+    seen = stamp;
+    std::ifstream file(path);
+    std::size_t count = 0;
+    skippedShaderCount.store(0, std::memory_order_release);
+    for (std::string word; count < MaxSkippedShaders && file >> word;) {
+        if (word.starts_with('#')) {
+            std::getline(file, word);
+            continue;
+        }
+        const auto address = programAddress(word);
+        if (!address) {
+            std::fprintf(stderr, "[skip-shader] ignoring '%s' in %s: not a hexadecimal program address\n", word.c_str(), path);
+            continue;
+        }
+        if (*address != 0) skippedShaders[count++].store(*address, std::memory_order_relaxed);
+    }
+    skippedShaderCount.store(count, std::memory_order_release);
+    std::fprintf(stderr, "[skip-shader] %zu programs listed in %s\n", count, path);
+}
+
+}
+
+bool SkippedShader(std::uint64_t address) {
+    static const std::uint64_t single = [] { const char* text = std::getenv("APS5_SKIP_SHADER"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }();
+    static const char* const file = std::getenv("APS5_SKIP_SHADER_FILE");
+    if (address == 0) return false;
+    if (address == single) return true;
+    if (file == nullptr) return false;
+    static std::atomic<std::chrono::steady_clock::rep> nextCheck{0};
+    const auto now = std::chrono::steady_clock::now().time_since_epoch();
+    auto due = nextCheck.load(std::memory_order_relaxed);
+    if (now.count() >= due && nextCheck.compare_exchange_strong(due, std::chrono::duration_cast<std::chrono::steady_clock::duration>(now + std::chrono::milliseconds(500)).count(), std::memory_order_relaxed)) ReloadSkippedShaders(file);
+    const auto count = skippedShaderCount.load(std::memory_order_acquire);
+    for (std::size_t index = 0; index < count; ++index) {
+        if (skippedShaders[index].load(std::memory_order_relaxed) == address) return true;
+    }
+    return false;
 }
 
 bool SkipRefusedDraws() {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp

@@ -66,7 +66,7 @@ bool Driver::pipelineDispatch() {
 void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission, std::uint64_t indirectArguments) {
     PerformanceTimer timing("Driver.Dispatch");
     const auto address = (static_cast<std::uint64_t>(readRegister(queue.shader, 0x20c)) << 8u) | (static_cast<std::uint64_t>(readRegister(queue.shader, 0x20d) & 0xffu) << 40u);
-    if (address == SkippedShader() && address != 0) return;
+    if (SkippedShader(address)) return;
     auto registeredShader = RegisteredProgram(*submission.shaders, address, {0});
     if (!registeredShader) registeredShader = ReadRawShader(address);
     const auto& snapshot = *registeredShader;
@@ -414,6 +414,11 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
         parked.resolvedAt = std::chrono::steady_clock::now();
         ++groupStats.resolved;
         if (cached) ++groupStats.resolvedHits;
+        if (!cached && compiledResult != nullptr && !localDevice->HasComputePipeline(*compiledResult)) {
+            SubmitLibraryPrebuild([localDevice, compiledResult, address] {
+                try { localDevice->PrebuildComputePipeline(*compiledResult, address); } catch (const std::exception&) {}
+            });
+        }
         return;
     }
     if (verifyDataHits() && dataHit) verifyDataHit(snapshot, codeOffset, request, memory, address, *keepVariant, liveWords, *compiledResult);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp

@@ -2,9 +2,11 @@
 #include "ThreadOwned.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp"
+#include "Optimization/ResourceProgram.hpp"
 #include <algorithm>
 #include <cstdlib>
 #include <cstring>
@@ -40,7 +42,17 @@ std::uint64_t Driver::DispatchesAdoptedAhead() const {
     return dispatchesAdoptedAhead.load(std::memory_order_relaxed);
 }
 
+namespace {
+
+std::size_t& PreviewedUntil() {
+    static thread_local std::size_t until = 0;
+    return until;
+}
+
+}
+
 void Driver::clearResolvedAhead() {
+    PreviewedUntil() = 0;
     auto& resolved = resolvedAhead();
     if (resolved.empty()) return;
     groupCaptureStats().stale += resolved.size();
@@ -107,7 +119,8 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l
     static const bool allQueues = std::getenv("APS5_GROUP_CAPTURE_ALL_QUEUES") != nullptr;
     if (!allQueues && submission.queue == 0) return;
     auto& resolved = resolvedAhead();
-    if (resolved.contains(from) || device.Load() == nullptr) return;
+    const auto localDevice = device.Load();
+    if (resolved.contains(from) || localDevice == nullptr) return;
     if (submission.queue == 0 && DrawPipeline::Depth() != 0 && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, (submission.commands[from] >> 8u) & 0xffu);
     auto& stats = groupCaptureStats();
     const auto begin = std::chrono::steady_clock::now();
@@ -126,6 +139,110 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l
         }
     } resolving;
     const auto& commands = submission.commands;
+    if (submission.shaders != nullptr && from >= PreviewedUntil()) {
+        bool hasFurtherDispatch = false;
+        std::size_t limit = from < commands.size() ? from + Pm4::PacketWords(commands[from]) : commands.size();
+        while (limit < commands.size()) {
+            const auto header = commands[limit];
+            if (Pm4::FillerPacket(header)) {
+                ++limit;
+                continue;
+            }
+            const auto count = Pm4::PacketWords(header);
+            if (limit + count > commands.size()) break;
+            const auto opcode = (header >> 8u) & 0xffu;
+            if (header == FlipPacketHeader || header == RenderingWaitPacketHeader || opcode == 0x22 || opcode == 0x3f || opcode == 0x3c || opcode == 0x93) break;
+            if (opcode == 0x15 || opcode == 0x16) hasFurtherDispatch = true;
+            limit += count;
+        }
+        PreviewedUntil() = limit;
+        if (hasFurtherDispatch) {
+            static thread_local QueueState* previewSlot = nullptr;
+            auto& preview = ShaderRecompiler::ThreadOwned(previewSlot);
+            preview.shader = live.shader;
+            preview.predication = live.predication;
+            std::vector<PreparedShaders*> pendingShaders;
+            std::vector<PreparedShaders*> submittedShaders;
+            std::size_t scan = from;
+            while (scan < limit) {
+                const auto header = commands[scan];
+                if (Pm4::FillerPacket(header)) {
+                    ++scan;
+                    continue;
+                }
+                const auto count = Pm4::PacketWords(header);
+                if (scan + count > limit) break;
+                const auto packet = std::span(commands).subspan(scan, count);
+                const auto opcode = (header >> 8u) & 0xffu;
+                const auto next = scan + count;
+                if (opcode == 0x15 || opcode == 0x16) {
+                    if (count < (opcode == 0x15 ? 5u : 3u)) break;
+                    if (scan > from) {
+                        const auto low = preview.shader.find(0x20c);
+                        const auto high = preview.shader.find(0x20d);
+                        if (low != preview.shader.end() && high != preview.shader.end()) {
+                            const auto address = (static_cast<std::uint64_t>(low->second) << 8u) | (static_cast<std::uint64_t>(high->second & 0xffu) << 40u);
+                            if (auto registered = RegisteredProgram(*submission.shaders, address, {0}); registered != nullptr && registered->type == 0) {
+                                bool isPending = false;
+                                bool isEmpty = false;
+                                {
+                                    std::lock_guard lock(registered->prepared->mutex);
+                                    isPending = registered->prepared->pending;
+                                    isEmpty = registered->prepared->entries.empty();
+                                }
+                                if (isPending) {
+                                    if (std::ranges::find(pendingShaders, registered->prepared.get()) == pendingShaders.end()) {
+                                        pendingShaders.push_back(registered->prepared.get());
+                                    }
+                                } else if (isEmpty && std::ranges::find(submittedShaders, registered->prepared.get()) == submittedShaders.end()) {
+                                    const auto codeOffset = static_cast<std::size_t>((address - registered->codeAddress) / sizeof(std::uint32_t));
+                                    if (codeOffset < registered->code.size()) {
+                                        try {
+                                            auto compute = Graphics::DecodeComputeStageInfo(preview.shader, registered->header);
+                                            submittedShaders.push_back(registered->prepared.get());
+                                            const auto rsrc2 = preview.shader.find(0x213);
+                                            const auto userCount = rsrc2 != preview.shader.end() ? ((rsrc2->second >> 1u) & 0x1fu) : 0u;
+                                            std::vector<std::uint32_t> userData(userCount, 0u);
+                                            for (std::uint32_t i = 0; i < userCount; ++i) {
+                                                if (const auto reg = preview.shader.find(0x240u + i); reg != preview.shader.end()) userData[i] = reg->second;
+                                            }
+                                            const auto wave = (packet.back() & 0x8000u) != 0 ? 32u : 64u;
+                                            SubmitRegistrationPreparation([registered, localDevice, address, codeOffset, wave, compute, userData = std::move(userData), state = preview.shader]() mutable {
+                                                std::vector<ShaderRecompiler::MemoryRegion> memory{{registered->codeAddress, std::as_bytes(std::span(registered->code))}};
+                                                if (!registered->header.empty()) memory.push_back({registered->headerAddress, registered->header});
+                                                ShaderRecompiler::RecompileRequest request{
+                                                    {ShaderRecompiler::ShaderStage::Compute, address, std::span(registered->code).subspan(codeOffset), registered->headerAddress, registered->header},
+                                                    {wave, 0, userData, compute, std::nullopt, std::nullopt, memory, DispatchFloatMode(*registered, state)},
+                                                    localDevice->ComputeTarget(wave),
+                                                    {0, 0, 0, 128}
+                                                };
+                                                try { static_cast<void>(InvocationFor(*registered, codeOffset, request)); } catch (const std::exception&) {}
+                                            }, registered->prepared.get());
+                                        } catch (const std::exception&) {}
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    Pm4::NoteDispatchRegisters(packet, preview);
+                } else if (Pm4::IndirectRegisterOpcode(opcode)) {
+                    const auto found = submission.registerLists.find(scan);
+                    if (count != 5 || found == submission.registerLists.end() || found->second.size() != static_cast<std::size_t>(packet[4]) * 2 || !Pm4::IndirectRegistersValid(found->second)) break;
+                    Pm4::ExecuteIndirectRegisters(packet, found->second, preview);
+                } else if (!Pm4::AccessesMemory(header) && opcode != 0x3c && opcode != 0x93 && opcode != 0x37 && opcode != 0x42 && opcode != 0x46 && opcode != 0x49 && opcode != 0x58) {
+                    try {
+                        Pm4::Execute(packet, preview);
+                    } catch (const std::exception&) {
+                        break;
+                    }
+                }
+                scan = next;
+            }
+            for (auto it = pendingShaders.rbegin(); it != pendingShaders.rend(); ++it) {
+                PrioritizeRegistrationPreparation(**it);
+            }
+        }
+    }
     for (std::size_t cursor = from; cursor < commands.size();) {
         const auto header = commands[cursor];
         if (Pm4::FillerPacket(header)) {
@@ -141,7 +258,7 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l
         if (opcode == 0x3c || opcode == 0x93) {
             if (!Pm4::WaitSatisfiedUnchecked(packet) && !WrittenByRun(packet, labels)) break;
         } else if (opcode == 0x15 || opcode == 0x16) {
-            if (resolved.contains(cursor)) break;
+            if (count < (opcode == 0x15 ? 5u : 3u) || resolved.contains(cursor)) break;
             if (opcode == 0x16) {
                 std::uint64_t arguments = 0;
                 try {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp

@@ -518,7 +518,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     for (const auto& patch : userPatches) writablePrograms[patch.program].userData[patch.index] = patch.value;
     tail.programs = writesPrograms ? &writablePrograms : &decode->programs;
     const auto& programs = *tail.programs;
-    if (const auto skipped = SkippedShader(); skipped != 0 && std::any_of(programs.begin(), programs.end(), [skipped](const DrawProgram& program) { return program.binary.codeAddress == skipped; })) return drawn();
+    if (std::any_of(programs.begin(), programs.end(), [](const DrawProgram& program) { return SkippedShader(program.binary.codeAddress); })) return drawn();
     const bool emptyIndirectSkip = EmptyIndirectSkip();
     auto& cpuIndirectPrograms = CpuIndirectPrograms();
     const auto recordDraws = [&](const Pm4::DrawArguments& arguments) { return arguments.count != 0 && arguments.instances != 0 && (!drawParameters.indexed || !drawParameters.indirect->indexLimitKnown || arguments.firstVertexOrIndex < drawParameters.indexCount); };

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp

@@ -482,6 +482,10 @@ bool DrawPipeline::HeldReads(std::uint64_t address, std::size_t bytes) {
     return false;
 }
 
+std::uint64_t DrawPipeline::HeldReadQueries() {
+    return heldReadQueries.load(std::memory_order_relaxed);
+}
+
 void DrawPipeline::ReleaseRunningReads() {
     std::lock_guard lock(mutex);
     if (!items.empty()) items.front().reads.clear();

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp

@@ -11,6 +11,7 @@
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "prx/libSceAgcDriver/Eq/include/Event.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
 #include <atomic>
 #include <cstdio>
 #include <cstdlib>
@@ -29,11 +30,6 @@ bool orderedFills() {
     return ordered;
 }
 
-bool pipelineDma() {
-    static const bool enabled = std::getenv("APS5_PIPELINE_DMA") != nullptr;
-    return enabled;
-}
-
 bool keepsDrawFinisher(std::uint32_t header, std::uint32_t opcode) {
     if (header == FlipPacketHeader || header == RenderingWaitPacketHeader) return false;
     if (Pm4::DrawOpcode(opcode)) return true;
@@ -65,7 +61,6 @@ bool keepsDrawPipeline(std::span<const std::uint32_t> packet, std::uint32_t head
             if (packet.size() < 7) return false;
             const auto destination = Pm4::DmaDestination(packet);
             if (destination == Pm4::DmaDestinationNowhere) return true;
-            if (!pipelineDma()) return false;
             if (destination == Pm4::DmaSelectGds) return true;
             const auto bytes = Pm4::DmaBytes(packet);
             if (bytes == 0) return true;
@@ -400,6 +395,7 @@ void Driver::execute(const Submission& submission) {
                 GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush);
                 std::lock_guard gpuLock(GuestMemory::GpuMutex());
                 const auto localDevice = device.Load();
+                if (localDevice != nullptr) static_cast<void>(Graphics::StorageTexture::StoreDrawnLinear("flip"));
 
                 if (!recordLabelsForPacket(localDevice.get(), submission.queue) && localDevice != nullptr) localDevice->SubmitRecorded(submission.queue == 0);
                 if (localDevice != nullptr) {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp

@@ -38,11 +38,12 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
             std::lock_guard gpuLock(GuestMemory::GpuMutex());
             const auto localDevice = device.Load();
             recordDeferredLabels(localDevice.get(), submission.queue);
-            const bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0;
+            bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0;
             int reason = localDevice != nullptr ? 0 : 1;
             if (label.has_value()) {
                 const auto bytes = label->Bytes();
                 const auto stamp = ++eventSerial;
+                if (localDevice != nullptr && submission.queue == 0 && Graphics::StorageTexture::StoreDrawnLinear("label") != 0) workOpen = true;
                 reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, stamp, submission.queue) : 4;
                 if (reason == 1) GuestMemory::Write(label->address, bytes, 4);
                 if (reason == 0 || reason == 1 || reason == 5) {
@@ -52,6 +53,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
                 countLabelOutcome(reason);
                 ++immediateLabels;
             } else {
+                if (localDevice != nullptr && submission.queue == 0 && Graphics::StorageTexture::StoreDrawnLinear("interrupt") != 0) workOpen = true;
                 if (localDevice != nullptr) Graphics::StorageTexture::StoreReadbackBeforeLabel("readback before interrupt");
                 ++noOpLabels;
             }
@@ -249,6 +251,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
 
             recordDeferredLabels(draining.get(), submission.queue);
             if (draining != nullptr) {
+                static_cast<void>(Graphics::StorageTexture::StoreDrawnLinear("drain"));
                 if (unlockedDrain && draining->CanWaitUnlocked()) epoch = draining->SubmitAndEpoch();
                 else draining->WaitIdle();
             }
@@ -292,8 +295,7 @@ void Driver::dumpSampleCounters(std::uint64_t address) {
 }
 
 bool Driver::enqueueDmaPacket(std::span<const std::uint32_t> packet, std::uint32_t opcode, std::uint32_t queue, const QueueState& state) {
-    static const bool enabled = std::getenv("APS5_PIPELINE_DMA") != nullptr;
-    if (!enabled || opcode != 0x50 || queue != 0 || packet.size() < 7 || !DrawPipeline::Active() || !deferredLabels().labels.empty()) return false;
+    if (opcode != 0x50 || queue != 0 || packet.size() < 7 || !DrawPipeline::Active() || !deferredLabels().labels.empty()) return false;
     const auto source = ((packet[1] >> 29u) & 3u) | ((packet[6] >> 24u) & 4u) | ((packet[6] >> 25u) & 8u);
     constexpr std::size_t immediateLimit = std::size_t{16} << 20u;
     constexpr std::size_t gpuStoreLimit = 65536;
@@ -438,10 +440,11 @@ void Driver::commitLabel(std::uint32_t queue, std::uint64_t address, std::span<c
     std::lock_guard gpuLock(GuestMemory::GpuMutex());
     const auto localDevice = device.Load();
     recordDeferredLabels(localDevice.get(), queue);
-    const bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0;
+    bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0;
     int reason = localDevice != nullptr ? 0 : 1;
     if (!bytes.empty()) {
         const auto stamp = ++eventSerial;
+        if (localDevice != nullptr && queue == 0 && Graphics::StorageTexture::StoreDrawnLinear("label") != 0) workOpen = true;
         reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(address, bytes, stamp, queue) : 4;
         if (reason != 0 && reason != 5 && reason != 6) {
             if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle();
@@ -456,6 +459,7 @@ void Driver::commitLabel(std::uint32_t queue, std::uint64_t address, std::span<c
     }
     if (endOfPipeInterrupt) {
         bool deferred = false;
+        if (bytes.empty() && localDevice != nullptr && queue == 0 && Graphics::StorageTexture::StoreDrawnLinear("interrupt") != 0) workOpen = true;
         if (localDevice != nullptr && (reason == 0 || reason == 5 || reason == 6)) {
             deferred = localDevice->AfterRecordedWork([this, queue] { deliverGuestEopInterrupt(queue); }, queue == 0);
             if (deferred && workOpen) localDevice->SubmitRecorded(queue == 0);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Presentation.cpp

@@ -93,6 +93,7 @@ void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buff
         bool submitted = false;
         bool presentable = false;
         bool trailing = false;
+        bool queueUnlocked = false;
         double waitedMs = 0;
         {
             GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present);
@@ -141,12 +142,20 @@ void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buff
             }
             if (submitted && (syncFlip || inFlight != 0)) {
 
-                presenting->QueuePresent();
-                timing.Mark("queue_present");
+                if (presenting->PresentsOnOwnQueue()) {
+                    queueUnlocked = true;
+                } else {
+                    presenting->QueuePresent();
+                    timing.Mark("queue_present");
+                }
                 trailing = !syncFlip;
                 submitted = false;
             }
         }
+        if (queueUnlocked) {
+            presenting->QueuePresent();
+            timing.Mark("queue_present");
+        }
         if (trailing) {
             waitedMs += presenting->FinishPresent();
             timing.Mark("render_fence_wait");
@@ -155,9 +164,13 @@ void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buff
             waitedMs = presenting->FinishPresent();
             timing.Mark("render_fence_wait");
 
-            GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present);
-            std::lock_guard lock(GuestMemory::GpuMutex());
-            presenting->QueuePresent();
+            if (presenting->PresentsOnOwnQueue()) {
+                presenting->QueuePresent();
+            } else {
+                GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present);
+                std::lock_guard lock(GuestMemory::GpuMutex());
+                presenting->QueuePresent();
+            }
             timing.Mark("queue_present");
         }
         gpuReady(context);

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -24,6 +24,7 @@
 #include <map>
 #include <memory>
 #include <new>
+#include <optional>
 #include <stdexcept>
 #include <string>
 #include <tuple>
@@ -1751,7 +1752,7 @@ void Rewatch(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges) {
     if (!tracker.watched) return;
     bool restored = false;
     for (const auto& [begin, end] : ranges) {
-        if (end <= begin || begin < tracker.base || end - tracker.base > tracker.size || (begin - tracker.base) % WriteBlockBytes != 0 || (end - tracker.base) % WriteBlockBytes != 0) continue;
+        if (end <= begin || begin < tracker.base || end - tracker.base > tracker.size || (begin - tracker.base) % WritePageBytes != 0 || (end - tracker.base) % WritePageBytes != 0) continue;
         const auto bytes = static_cast<std::size_t>(end - begin);
         if (tracker.coverage.Covers(begin, bytes)) continue;
         GuestArena::GuestArenaCompareWhenShared_nid_no_patch(static_cast<std::uintptr_t>(begin), bytes);
@@ -1770,7 +1771,7 @@ void Rewatch(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges) {
 #endif
 }
 
-bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched) {
+bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched, bool importMayWriteGuest) {
     if (bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return import();
     if (!staysWatched) {
         constexpr std::uint64_t page = 4096;
@@ -1790,7 +1791,11 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function
     const bool covered = tracker.covers(first, static_cast<std::size_t>(stop - first));
     const auto runImport = [&] {
 #ifdef __APPLE__
-        const GuestArena::HostWrite open(reinterpret_cast<void*>(first), static_cast<std::size_t>(stop - first));
+        std::optional<GuestArena::HostWrite> open;
+        if (importMayWriteGuest) open.emplace(reinterpret_cast<void*>(first), static_cast<std::size_t>(stop - first));
+#endif
+#ifndef __APPLE__
+        static_cast<void>(importMayWriteGuest);
 #endif
         return import();
     };

core/libs/prx/libSceAgcDriver/Execution/src/ThreadPriority.cpp

@@ -12,6 +12,12 @@
 
 #ifdef _WIN32
 #include <windows.h>
+#elif defined(__APPLE__)
+#include <mach/mach.h>
+#include <mach/mach_time.h>
+#include <mach/thread_policy.h>
+#include <pthread.h>
+#include <sys/qos.h>
 #else
 #include <dlfcn.h>
 #include <sched.h>
@@ -42,7 +48,30 @@ void LogOnce(const char* role, const std::string& text) {
     std::fprintf(stderr, "[thread-priority] %s: %s\n", role, text.c_str());
 }
 
-#ifndef _WIN32
+#if defined(__APPLE__)
+
+void Raise(const char* role) {
+    if (RequestedMode() == ThreadPriorityMode::Realtime) {
+        mach_timebase_info_data_t timebase{};
+        mach_timebase_info(&timebase);
+        const auto ticks = [&](std::uint64_t nanos) {
+            return static_cast<std::uint32_t>(nanos * timebase.denom / timebase.numer);
+        };
+        thread_time_constraint_policy_data_t policy{ticks(1000000), ticks(500000), ticks(1000000), TRUE};
+        const auto status = thread_policy_set(pthread_mach_thread_np(pthread_self()), THREAD_TIME_CONSTRAINT_POLICY,
+                                              reinterpret_cast<thread_policy_t>(&policy), THREAD_TIME_CONSTRAINT_POLICY_COUNT);
+        if (status != KERN_SUCCESS)
+            throw std::runtime_error(std::string("APS5_THREAD_PRIORITY=rt: ") + role + " time-constraint policy refused (kern_return " + std::to_string(status) + ")");
+        LogOnce(role, "Mach time-constraint policy");
+        return;
+    }
+    const int error = pthread_set_qos_class_self_np(QOS_CLASS_USER_INTERACTIVE, 0);
+    if (error != 0)
+        throw std::runtime_error(std::string("APS5_THREAD_PRIORITY=high: ") + role + " user-interactive QoS refused: " + std::strerror(error));
+    LogOnce(role, "user-interactive QoS");
+}
+
+#elif !defined(_WIN32)
 
 struct DBusConnection;
 struct DBusMessage;

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -20,9 +20,11 @@
 #include "prx/libSceAgcDriver/Execution/include/PipelineLinkOptimization.hpp"
 #include "prx/libSceAgcDriver/Execution/include/DrawInputsInPlace.hpp"
 #include "prx/libSceAgcDriver/Execution/include/SubgroupBarrier.hpp"
+#include "prx/libSceAgcDriver/Execution/include/TessellationPipelineCache.hpp"
 #include "prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PresentMode.hpp"
+#include "prx/libSceAgcDriver/Execution/include/SwapchainImages.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceOverlay.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PresentationPass.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp"
@@ -83,7 +85,7 @@
 
 namespace AgcDriver {
 
-DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> families, std::uint32_t family, bool settles, bool sparseBinding) {
+DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> families, std::uint32_t family, bool settles, bool sparseBinding, bool presents) {
     DeviceQueuePlan plan;
     const auto count = static_cast<std::uint32_t>(families.size());
     if (settles) {
@@ -102,6 +104,13 @@ DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> famili
             }
         }
     }
+    if (presents && family < count) {
+        if (const auto index = SelectPresentQueue(families[family].queueCount, plan.mainCount)) {
+            plan.present = true;
+            plan.presentIndex = *index;
+            plan.mainCount = *index + 1;
+        }
+    }
     if (!sparseBinding) return plan;
     const auto taken = [&](std::uint32_t index) { return plan.settle && index == plan.settleFamily ? plan.settleIndex + 1 : 0u; };
     const auto binds = [&](std::uint32_t index) { return index != family && families[index].queueCount > taken(index) && (families[index].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) != 0; };
@@ -351,6 +360,7 @@ struct VulkanDevice::State {
     VkDriverId driverId = VK_DRIVER_ID_MAX_ENUM;
     std::uint32_t pixelFunctionStorageDwords = 4096u;
     bool individuallyFreeDescriptorSets = false;
+    bool tessellationPipelineCache = true;
     PFN_vkGetCalibratedTimestampsKHR calibratedTimestamps = nullptr;
     PFN_vkCmdSetCheckpointNV checkpointMarker = nullptr;
     PFN_vkCmdWriteBufferMarkerAMD bufferMarker = nullptr;
@@ -439,6 +449,8 @@ struct VulkanDevice::State {
     std::uint32_t settleFamily = 0;
     VkCommandPool settlePool = VK_NULL_HANDLE;
     std::mutex settleQueueMutex;
+    VkQueue presentQueue = VK_NULL_HANDLE;
+    std::mutex presentQueueMutex;
     bool memoryBudget = false;
     bool depthRangeUnrestricted = false;
     bool samplerAnisotropy = false;
@@ -806,7 +818,7 @@ struct VulkanDevice::State {
             for (std::uint32_t x = 0; x < width; ++x) std::memcpy(pixels.data() + (static_cast<std::size_t>(y) * width + x) * 4, full.data() + (static_cast<std::size_t>(y) * scale * slot.dumpWidth + static_cast<std::size_t>(x) * scale) * 4, 4);
         }
         static const auto start = std::chrono::steady_clock::now();
-        std::fprintf(stderr, "[gpu] frame_%03d.bmp: %ux%u display read back on the GPU at %.1f s\n", index, slot.dumpWidth, slot.dumpHeight, std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count());
+        AgcDriver::ProfilePrint_nid_no_patch("[gpu] frame_%03d.bmp: %ux%u display read back on the GPU at %.1f s\n", index, slot.dumpWidth, slot.dumpHeight, std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count());
         dumpWriter.Enqueue(index, width, height, 1, false, std::move(pixels));
     }
 
@@ -1275,6 +1287,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         state->driverId = driver.driverID;
         state->individuallyFreeDescriptorSets = driver.driverID == VK_DRIVER_ID_MESA_LLVMPIPE;
         state->fusedExactArithmetic = driver.driverID == VK_DRIVER_ID_MOLTENVK;
+        state->tessellationPipelineCache = TessellationPipelineCacheUsable(driver.driverID);
         state->subgroupControlBarriers = SubgroupControlBarrierUsable(driver.driverID);
         state->workgroupScopeFences = WorkgroupScopeFenceUsable(driver.driverID);
     }
@@ -1682,7 +1695,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     queueInfo.queueFamilyIndex = family;
     queueInfo.queueCount = 1;
     queueInfo.pQueuePriorities = &priority;
-    const float priorities[2]{1.0f, 1.0f};
+    const float priorities[3]{1.0f, 1.0f, 1.0f};
 #ifdef _WIN32
     const bool settles = true;
 #else
@@ -1729,13 +1742,13 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     // Guest shaders routinely read past descriptor ranges; robust access turns that into zeros
     // instead of a GPU fault that loses the device.
     enabled.robustBufferAccess = available.robustBufferAccess;
-    const auto plan = PlanDeviceQueues(queues, family, settles, available.sparseBinding == VK_TRUE);
+    const auto plan = PlanDeviceQueues(queues, family, settles, available.sparseBinding == VK_TRUE, window != nullptr);
     const bool settleQueue = plan.settle;
     const auto settleFamily = plan.settleFamily;
     const auto settleIndex = plan.settleIndex;
     VkDeviceQueueCreateInfo queueInfos[3]{queueInfo, {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}, {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}};
     queueInfos[0].queueCount = plan.mainCount;
-    queueInfos[0].pQueuePriorities = plan.mainCount == 2 ? priorities : &priority;
+    queueInfos[0].pQueuePriorities = plan.mainCount > 1 ? priorities : &priority;
     deviceInfo.queueCreateInfoCount = 1;
     const auto addQueue = [&](std::uint32_t queueFamily, std::uint32_t index) {
         for (std::uint32_t info = 1; info < deviceInfo.queueCreateInfoCount; ++info) {
@@ -2103,6 +2116,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     }
     state->DeviceFunction<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(state->device, family, 0, &state->queue);
     state->queueFamily = family;
+    if (plan.present) {
+        state->DeviceFunction<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(state->device, family, plan.presentIndex, &state->presentQueue);
+        APS5_LOG_OUT("Presentation queue family=%u index=%u", family, plan.presentIndex);
+    }
     if (settleQueue) {
         state->settleFamily = settleFamily;
         state->DeviceFunction<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(state->device, settleFamily, settleIndex, &state->settleQueue);
@@ -2193,7 +2210,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         require((surface.supportedUsageFlags & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) != 0, "surface does not support color attachment images");
         VkSwapchainCreateInfoKHR swapchain{VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR};
         swapchain.surface = state->surface;
-        swapchain.minImageCount = surface.minImageCount;
+        swapchain.minImageCount = AgcDriver::SwapchainImageCount(surface);
         swapchain.imageFormat = state->surfaceFormat;
         swapchain.imageColorSpace = state->surfaceColorSpace;
         swapchain.imageExtent = state->extent;
@@ -2297,6 +2314,26 @@ bool VulkanDevice::ReplayComputePipeline(const ShaderRecompiler::RecompileResult
     return state->computePipelines.try_emplace(key, std::move(objects)).second;
 }
 
+bool VulkanDevice::HasComputePipeline(const ShaderRecompiler::RecompileResult& shader) const {
+    const std::array<Graphics::CompiledShader, 1> shaders{{{ShaderRecompiler::ShaderStage::Compute, &shader, 0}}};
+    const auto key = ComputePipelineKey::Of(shader, Graphics::PushConstantStages(shaders) != 0);
+    std::lock_guard pipelines(state->computePipelinesMutex);
+    return !key.identity.Empty() && (state->computePipelines.contains(key) || state->computePipelinesBuilding.contains(key));
+}
+
+void VulkanDevice::PrebuildComputePipeline(const ShaderRecompiler::RecompileResult& shader, std::uint64_t programAddress) {
+    if (shader.spirv.size() < 5 || shader.spirv[0] != 0x07230203u) return;
+    ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion);
+    const auto context = graphicsContext();
+    if (context.descriptorCache == nullptr) return;
+    const std::array<Graphics::CompiledShader, 1> shaders{{{ShaderRecompiler::ShaderStage::Compute, &shader, 0}}};
+    const auto pushStages = Graphics::PushConstantStages(shaders);
+    if (pushStages != 0 && state->properties.limits.maxPushConstantsSize < Graphics::ComputePushConstantBytes) return;
+    if (HasComputePipeline(shader)) return;
+    const auto layouts = Graphics::DescribeSetLayouts(context, shaders);
+    static_cast<void>(computePipeline(context, shader, layouts.layouts.main, layouts.mainKey, pushStages, programAddress));
+}
+
 bool VulkanDevice::ImportGuestMemory(const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice) {
     if (adoptDevice) GuestMemory::AssertGpuLockHeld("VulkanDevice::ImportGuestMemory");
     return Graphics::ImportMappedRanges(graphicsContext(), ranges, generation, adoptDevice);
@@ -2325,6 +2362,7 @@ void VulkanDevice::PrepareForReplacement() {
 
 void VulkanDevice::WaitIdle() {
     APS5_LOG_CHARS_OUT_DEBUG("VulkanDevice::WaitIdle begin");
+    static_cast<void>(Graphics::StorageTexture::StoreDrawnLinear("wait idle"));
     RetirePresents(0);
     if (state->recorder) {
         // Announced with this call's return address: the [recorder] site table then names the
@@ -2332,6 +2370,8 @@ void VulkanDevice::WaitIdle() {
         if (!state->recorder->Idle()) Graphics::Recorder::CountSync(0, __builtin_return_address(0));
         state->recorder->Sync();
     }
+    std::unique_lock presentQueueLock(state->presentQueueMutex, std::defer_lock);
+    if (state->presentQueue != VK_NULL_HANDLE) presentQueueLock.lock();
     check(state->DeviceFunction<PFN_vkDeviceWaitIdle>("vkDeviceWaitIdle")(state->device), "vkDeviceWaitIdle");
     Graphics::CheckPixelFunctionLds(graphicsContext());
     APS5_LOG_CHARS_OUT_DEBUG("VulkanDevice::WaitIdle complete");
@@ -2429,6 +2469,7 @@ bool VulkanDevice::StoresPendingOver(std::uint64_t address, std::size_t bytes) c
 int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst) {
     static const bool storeAtLabel = Graphics::StorageTexture::StoreAtLabelRequested(std::getenv("APS5_STORE_AT_LABEL"));
     if (storeAtLabel) Graphics::StorageTexture::FlushAllPending("label");
+    if (queue == 0) static_cast<void>(Graphics::StorageTexture::StoreDrawnLinear("label"));
     Graphics::StorageTexture::StoreReadbackBeforeLabel("readback before label");
     if (!state->recorder || bytes.empty() || bytes.size() > 65536 || address % 4 != 0 || bytes.size() % 4 != 0) return 4;
     auto& recorder = *state->recorder;
@@ -3044,7 +3085,7 @@ void VulkanDevice::recreateSwapchain(std::uint32_t width, std::uint32_t height,
     require((surface.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT) != 0 && (surface.supportedCompositeAlpha & VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR) != 0 && (surface.supportedTransforms & VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR) != 0, "resized surface capabilities are unsupported");
     VkSwapchainCreateInfoKHR create{VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR};
     create.surface = state->surface;
-    create.minImageCount = surface.minImageCount;
+    create.minImageCount = AgcDriver::SwapchainImageCount(surface);
     create.imageFormat = state->surfaceFormat;
     create.imageColorSpace = state->surfaceColorSpace;
     create.imageExtent = {width, height};
@@ -3780,13 +3821,19 @@ void VulkanDevice::QueuePresent() {
     present.swapchainCount = 1;
     present.pSwapchains = &state->swapchain;
     present.pImageIndices = &index;
-    const auto presented = state->DeviceFunction<PFN_vkQueuePresentKHR>("vkQueuePresentKHR")(state->queue, &present);
+    std::unique_lock presentQueueLock(state->presentQueueMutex, std::defer_lock);
+    if (state->presentQueue != VK_NULL_HANDLE) presentQueueLock.lock();
+    const auto presented = state->DeviceFunction<PFN_vkQueuePresentKHR>("vkQueuePresentKHR")(state->presentQueue != VK_NULL_HANDLE ? state->presentQueue : state->queue, &present);
     if (presented == VK_ERROR_OUT_OF_DATE_KHR || presented == VK_SUBOPTIMAL_KHR) state->extent = {0, 0};
     else check(presented, "vkQueuePresentKHR");
     timing.Mark("queue_present");
     APS5_LOG_OUT_DEBUG("vkQueuePresentKHR queued imageIndex=%u", index);
 }
 
+bool VulkanDevice::PresentsOnOwnQueue() const {
+    return state->presentQueue != VK_NULL_HANDLE;
+}
+
 ShaderRecompiler::SpirvTarget VulkanDevice::ComputeTarget(std::uint32_t waveSize) const {
     auto target = Target();
     if (waveSize == 32u && state->computeWave32) target.subgroupSize = 32u;
@@ -3955,6 +4002,7 @@ Graphics::Context VulkanDevice::buildContext() const {
     };
     context.pipelineCacheSynchronization = state->pipelineCache ? &state->pipelineCache->Synchronization() : nullptr;
     context.individuallyFreeDescriptorSets = state->individuallyFreeDescriptorSets;
+    context.tessellationPipelineCache = state->tessellationPipelineCache;
     context.dmaBufImport = state->dmaBufImport;
     context.sparseBinding = state->sparseBinding;
     context.sparseResidencyBuffer = state->sparseResidencyBuffer;
@@ -4332,7 +4380,7 @@ struct DispatchCounters {
     std::array<double, DispatchRows> phaseTotals{};
     std::uint64_t profiledDispatches = 0;
     IndirectHold indirectHold;
-    std::uint64_t cacheHits = 0, cacheMisses = 0, cacheInvalidated = 0, templateRefreshed = 0, templateSameWords = 0;
+    std::uint64_t cacheHits = 0, cacheMisses = 0, cacheInvalidated = 0, templateRefreshed = 0, templateSameWords = 0, templateByNumber = 0;
     double templateRevalidateMs = 0;
     std::uint64_t procLookups = 0, preBarriersRecorded = 0, preBarriersSkipped = 0;
     std::chrono::steady_clock::time_point cacheReport = std::chrono::steady_clock::now();
@@ -4975,6 +5023,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
     };
     if (recorder.HasQueuedKeyStores() && (resources.HoldsLease() || recorder.AnyQueuedKeyStore(touches))) recorder.FlushKeyStores();
     if (recorder.HasQueuedStores() && (resources.HoldsLease() || recorder.AnyQueuedStore(touches))) recorder.FlushStores();
+    static_cast<void>(Graphics::StorageTexture::StoreDrawnLinearTouching("dispatch access", touches, resources.HoldsLease()));
     VkAccessFlags covered = 0;
     const auto commands = recorder.Commands(&covered);
     recordStep(PhaseRecordCommands);
@@ -4993,6 +5042,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
         // after an earlier dispatch's reads of the buffers by that dispatch's trailing barrier. A
         // recipe compares the two 64-bit hashes first: equal hashes mean the buffers hold the words.
         const bool differs = record.dataRefresh == RecordedDispatch::DataRefresh::Words || resources.DataWordsHash() != record.dataWordsHash;
+        if (timer.profile && record.shader->program != nullptr && record.shader->program->bindings.size() != resources.BindingCount()) ++d.templateByNumber;
         if (differs && (record.resourceKeyMatched ? resources.refreshData(commands, *record.shader, &recorder) : resources.RefreshData(commands, *record.shader, &recorder))) {
             covered = 0;
             ++d.templateRefreshed;
@@ -5506,7 +5556,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
     }
     if (profile && std::chrono::steady_clock::now() - d.cacheReport > std::chrono::seconds(10)) {
         d.cacheReport = std::chrono::steady_clock::now();
-        AgcDriver::ProfilePrint_nid_no_patch("[rescache] (cumulative): %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw); template hits: %llu refreshed the data buffers (revalidate %.1f ms), %llu had the same words; Find calls %llu, Touch calls %llu\n", static_cast<unsigned long long>(d.cacheHits), static_cast<unsigned long long>(d.cacheMisses), static_cast<unsigned long long>(d.cacheInvalidated), state->resourceCache.Size(), static_cast<unsigned long long>(d.templateRefreshed), d.templateRevalidateMs, static_cast<unsigned long long>(d.templateSameWords), static_cast<unsigned long long>(ResourceCache::Finds()), static_cast<unsigned long long>(ResourceCache::Touches()));
+        AgcDriver::ProfilePrint_nid_no_patch("[rescache] (cumulative): %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw); template hits: %llu refreshed the data buffers (revalidate %.1f ms), %llu had the same words, %llu paired their data bindings by number (no image table binding of their own); Find calls %llu, Touch calls %llu\n", static_cast<unsigned long long>(d.cacheHits), static_cast<unsigned long long>(d.cacheMisses), static_cast<unsigned long long>(d.cacheInvalidated), state->resourceCache.Size(), static_cast<unsigned long long>(d.templateRefreshed), d.templateRevalidateMs, static_cast<unsigned long long>(d.templateSameWords), static_cast<unsigned long long>(d.templateByNumber), static_cast<unsigned long long>(ResourceCache::Finds()), static_cast<unsigned long long>(ResourceCache::Touches()));
         AgcDriver::ProfilePrint_nid_no_patch("[vk] deviceProc lookups inside the device call (cumulative): %llu (%.2f per dispatch); pre-dispatch barriers recorded %llu, skipped %llu; compute pipelines created %llu (%llu under the GPU mutex)\n", static_cast<unsigned long long>(d.procLookups), d.profiledDispatches != 0 ? static_cast<double>(d.procLookups) / static_cast<double>(d.profiledDispatches) : 0.0, static_cast<unsigned long long>(d.preBarriersRecorded), static_cast<unsigned long long>(d.preBarriersSkipped), static_cast<unsigned long long>(computePipelinesCreated.load(std::memory_order_relaxed)), static_cast<unsigned long long>(computePipelinesCreatedLocked.load(std::memory_order_relaxed)));
         reportRecipes();
         reportDispatchMove();

core/libs/prx/libSceAgcDriver/Execution/src/WorkerSampler.cpp

@@ -163,18 +163,29 @@ struct Sampler {
         return frame;
     }
 
+    static void writeModules(std::FILE* file, const std::map<std::uint64_t, std::string>& loaded) {
+        for (const auto& [base, name] : loaded) {
+            const auto* dos = reinterpret_cast<const IMAGE_DOS_HEADER*>(base);
+            const auto* nt = reinterpret_cast<const IMAGE_NT_HEADERS*>(base + dos->e_lfanew);
+            std::fprintf(file, "# module %s 0x%lx 0x%lx\n", name.c_str(), static_cast<unsigned long>(nt->OptionalHeader.SizeOfImage), static_cast<unsigned long>(nt->FileHeader.TimeDateStamp));
+        }
+    }
+
     void write() {
         std::lock_guard lock(mutex);
         std::FILE* file = std::fopen(path.c_str(), "w");
         if (file == nullptr) return;
         std::fprintf(file, "# %llu samples\n", static_cast<unsigned long long>(samples));
+        std::map<std::uint64_t, std::string> loaded;
         for (const auto& [address, count] : counts) {
             HMODULE module = nullptr;
             char name[MAX_PATH] = "?";
             if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast<LPCSTR>(address), &module)) GetModuleFileNameA(module, name, sizeof(name));
             const char* base = std::strrchr(name, '\\');
+            if (module != nullptr) loaded.emplace(reinterpret_cast<std::uint64_t>(module), base ? base + 1 : name);
             std::fprintf(file, "%s 0x%llx %llu %llu\n", base ? base + 1 : name, static_cast<unsigned long long>(address - reinterpret_cast<std::uint64_t>(module)), static_cast<unsigned long long>(count.first), static_cast<unsigned long long>(count.second));
         }
+        writeModules(file, loaded);
         std::fclose(file);
         if (keepStacks) writeStacks();
     }
@@ -195,11 +206,7 @@ struct Sampler {
         const double tickUs = 1e6 / static_cast<double>(frequency.QuadPart);
         std::fprintf(file, "# samples=%llu depth=%d truncated=%llu suspended_us_mean=%.1f suspended_us_max=%.1f\n", static_cast<unsigned long long>(samples), depth, static_cast<unsigned long long>(stacks.Truncated()),
             samples != 0 ? static_cast<double>(suspendedTicks) * tickUs / static_cast<double>(samples) : 0.0, static_cast<double>(longestSuspendedTicks) * tickUs);
-        for (const auto& [base, name] : loaded) {
-            const auto* dos = reinterpret_cast<const IMAGE_DOS_HEADER*>(base);
-            const auto* nt = reinterpret_cast<const IMAGE_NT_HEADERS*>(base + dos->e_lfanew);
-            std::fprintf(file, "# module %s 0x%lx 0x%lx\n", name.c_str(), static_cast<unsigned long>(nt->OptionalHeader.SizeOfImage), static_cast<unsigned long>(nt->FileHeader.TimeDateStamp));
-        }
+        writeModules(file, loaded);
         const auto text = stacks.Format([&](std::uint64_t address) { return names.at(address); });
         const bool written = std::fwrite(text.data(), 1, text.size(), file) == text.size();
         if (std::fclose(file) != 0 || !written) return;
@@ -246,6 +253,7 @@ struct ProcessSampler {
         std::FILE* file = std::fopen(path.c_str(), "w");
         if (file == nullptr) return;
         std::fprintf(file, "# %llu rounds\n", static_cast<unsigned long long>(rounds));
+        std::map<std::uint64_t, std::string> loaded;
         for (auto& [id, thread] : threads) {
             std::lock_guard lock(thread.mutex);
             std::vector<std::pair<std::uint64_t, std::pair<std::uint64_t, std::uint64_t>>> hot(thread.counts.begin(), thread.counts.end());
@@ -260,9 +268,11 @@ struct ProcessSampler {
                 char name[MAX_PATH] = "?";
                 if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast<LPCSTR>(address), &module)) GetModuleFileNameA(module, name, sizeof(name));
                 const char* base = std::strrchr(name, '\\');
+                if (module != nullptr) loaded.emplace(reinterpret_cast<std::uint64_t>(module), base ? base + 1 : name);
                 std::fprintf(file, "  %s 0x%llx %llu %llu\n", base ? base + 1 : name, static_cast<unsigned long long>(address - reinterpret_cast<std::uint64_t>(module)), static_cast<unsigned long long>(hot[i].second.first), static_cast<unsigned long long>(hot[i].second.second));
             }
         }
+        Sampler::writeModules(file, loaded);
         std::fclose(file);
     }
 };

core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp

@@ -240,6 +240,7 @@ struct Context {
     std::uint32_t imageTableCapacity = 0;
     std::uint32_t samplerTableCapacity = 0;
     bool individuallyFreeDescriptorSets = false;
+    bool tessellationPipelineCache = true;
     bool imageTableUpdateAfterBind = false;
     bool samplerTableUpdateAfterBind = false;
     bool geometryShader = false;

core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp

@@ -38,6 +38,7 @@ struct HostImport {
     // Identity for the life of this import (see HostImportSerial); 0 until first asked for.
     std::uint64_t serial = 0;
     bool unwatched = false;
+    bool settleOnDemand = false;
     bool dmaBuf = false;
     bool joined = false;
     std::shared_ptr<void> chunk;
@@ -102,6 +103,7 @@ void ReportHostImports();
 // address)` in the import's buffer.
 const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes, ImportRequester requester = ImportRequester::Access);
 const HostImport* ExistingHostImportFor(const Context& context, std::uint64_t address, std::size_t bytes);
+bool WatchedAfterSettle(const Context& context, std::uint64_t address, std::size_t bytes);
 void SetHostImportWindow(std::uint64_t bytes);
 // Tests: the window of one large registered range (64 MiB) and the import budget (0: the configured one).
 void SetHostImportRangeWindow(std::uint64_t bytes);
@@ -264,6 +266,8 @@ struct AddressCopy {
     std::uint64_t end;
     std::uint64_t committed;
     const char* reason;
+    bool heap = false;
+    bool overBudget = false;
 };
 std::string AddressCopyOverflow(std::vector<AddressCopy> copies, std::uint64_t limit);
 

core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp

@@ -109,8 +109,10 @@ private:
 // stages and every pipeline state field. A configuration whose stage has no module gets a private
 // pipeline. A miss builds the pipeline outside the store's mutex (a concurrent build of the same
 // key keeps the first inserted) and records its recipe in the pipeline journal. Entries are
-// LRU-bounded (APS5_PIPELINE_STORE_ENTRIES, 1024) and only evicted once no recorded draw holds
-// them. Debug aid: APS5_NO_PIPELINE_CACHE=1 builds one per draw as before.
+// LRU-bounded and only evicted once no recorded draw holds them. The bound starts at 1024 and
+// doubles up to 4096 for a journal replay that does not fit and while pipelines it evicted keep
+// coming back; it never shrinks during a run. APS5_PIPELINE_STORE_ENTRIES fixes it. Debug aid:
+// APS5_NO_PIPELINE_CACHE=1 builds one per draw as before.
 std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
 // `created` reports whether this call built the pipeline (false: served from the store or private).
 std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, bool* created = nullptr);
@@ -124,6 +126,8 @@ struct PipelineStoreCounters {
     std::uint64_t bound = 0;
 };
 PipelineStoreCounters PipelineStoreTotals();
+// Whether the store may still raise its bound (not fixed by APS5_PIPELINE_STORE_ENTRIES, below 4096).
+bool PipelineStoreCanGrow();
 // The pipeline store's key of a configuration (empty when the configuration gets a private pipeline).
 void BuildPipelineStoreKey(std::vector<std::byte>& key, const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout);
 struct DescribedSetLayouts {

core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp

@@ -183,6 +183,7 @@ public:
     // Whether the OPEN (unsubmitted) batch writes the range: a wait on such a range must submit it.
     bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const;
     std::uint64_t PendingWriterSerial(std::uint64_t address, std::size_t bytes) const;
+    std::uint64_t PendingWriterSerial(std::uint64_t address, std::size_t bytes, std::span<const std::uint64_t> exceptNotes) const;
     // What SyncThrough(address, bytes) would wait for, for the [hooksync] attribution: the newest
     // batch whose noted range overlaps the access (the open batch counts first, with the serial it
     // will get), whether that batch's fence has already signaled (its GPU work is done, so the

core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp

@@ -32,6 +32,7 @@ void FlushCachedTextures(VkDevice device);
 void ClearCachedTextures(VkDevice device);
 std::uint64_t ReleaseTextureCaches(std::uint64_t bytes, bool newImage = false);
 std::uint64_t TextureCacheBudget(const VkPhysicalDeviceMemoryProperties& memory);
+std::uint64_t StorageTextureCeiling(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported);
 std::uint64_t StorageTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t sampledBytes, std::uint64_t storageBytes);
 std::uint64_t SampledTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t textureBytes, std::uint64_t storageBytes = 0);
 bool SampledBudgetReportDue(std::uint64_t reported, std::uint64_t budget, std::chrono::steady_clock::duration sinceReport);
@@ -315,6 +316,7 @@ public:
     }
     // Whether RefreshData would record anything for `shader` (the per-word compare; verification).
     bool DataWordsDiffer(const CompiledShader& shader) const;
+    std::size_t BindingCount() const { return bindings.size(); }
     // Why the fast proof of a Revalidate left the object to the full walk (the [rescache]
     // revalidate line's reasons); Count: it did not.
     enum class FastFail : std::size_t { NoRecord, Collect, Pending, Evicted, Changed, Keys, ClearedView, StorageKeys, Depth, Replaced, Audit, Count };

core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp

@@ -45,6 +45,7 @@ struct ColorTarget {
     ShaderRecompiler::ColorExportPacking packing = ShaderRecompiler::ColorExportPacking::None;
     // DCC metadata of a compressed target (CB_COLOR_INFO DCC_ENABLE), or 0 (see DccMetadata.hpp).
     std::uint64_t dccAddress = 0;
+    std::uint64_t chainDccAddress = 0;
     bool dccAlphaOnMsb = false;
     bool dccPipeAligned = false;
     std::uint64_t cmaskAddress = 0;

core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp

@@ -16,6 +16,7 @@
 #include <memory>
 #include <mutex>
 #include <optional>
+#include <functional>
 #include <span>
 #include <tuple>
 #include <utility>
@@ -204,6 +205,7 @@ public:
     bool BlockUnits() const { return blockUnits; }
     std::uint32_t TrackedUnits() const { return trackedLayers; }
     void MarkSliceDirty(std::uint32_t slice, std::uint32_t count = 1);
+    bool MarkDrawn(std::uint32_t slice, std::uint32_t count, VkRect2D scissor);
     void Flush();
     bool Dirty() const;
     // Stores every pending image overlapping the range, except `except`; returns whether any was.
@@ -220,6 +222,8 @@ public:
     static constexpr std::uint64_t SmallStoreBytes = 64 * 1024;
     static std::size_t StoreSmallPending(const char* reason);
     static void StoreReadbackBeforeLabel(const char* reason);
+    static std::size_t StoreDrawnLinear(const char* reason);
+    static std::size_t StoreDrawnLinearTouching(const char* reason, const std::function<bool(std::uint64_t, std::uint64_t)>& touches, bool all);
     static bool AnyPending();
     static void FlushDumpSelected(const char* reason);
     static bool StoreAtFlipRequested(const char* value);
@@ -482,6 +486,27 @@ private:
     // (`shadowed` receives those guest ranges, `imported` the pieces written to the import).
     std::uint64_t uploadWindows(const HostImport& import, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, bool discard = false);
     std::uint64_t writeBackWindows(std::span<const ImportSegment> segments, std::span<const std::pair<std::uint64_t, std::uint64_t>> keep, std::uint64_t firstStored, std::uint64_t lastStored, std::vector<ShadowedRange>& shadowed, std::vector<std::pair<std::uint64_t, std::uint64_t>>& imported, Buffer* save = nullptr);
+    struct DrawnRect {
+        std::uint32_t left = 0;
+        std::uint32_t top = 0;
+        std::uint32_t right = 0;
+        std::uint32_t bottom = 0;
+    };
+    struct DrawnArea {
+        bool drawn = false;
+        bool rect = false;
+        std::uint32_t left = 0;
+        std::uint32_t top = 0;
+        std::uint32_t right = 0;
+        std::uint32_t bottom = 0;
+        std::vector<DrawnRect> rects;
+    };
+    DrawnArea drawnArea() const;
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> drawnSpans(const DrawnArea& area, std::span<const std::pair<std::uint64_t, std::uint64_t>> keep) const;
+    std::uint64_t storeDrawnRect(const DrawnArea& area, std::span<const ImportSegment> segments, std::span<const std::pair<std::uint64_t, std::uint64_t>> stores, const std::vector<std::uint32_t>& saveUnits);
+    void addDrawnRect(DrawnRect rect);
+    void forgetDrawn();
+    void noteStore(std::uint64_t note);
     // Per-layer validity (thin array surfaces whose layers are 64 KiB-aligned guest slices; every
     // other surface is one tracked layer): a layer's generation is the write generation its guest
     // bytes were last known to match the image at, and a pending layer holds results (a clear, a
@@ -518,6 +543,7 @@ private:
     // Per 64 KiB tracker block of the surface (block 0 holds the base address): the generation of
     // the tracked layer it belongs to, for GuestMemory::ChangedBlocks.
     void blockGenerations(std::vector<std::uint64_t>& generations) const;
+    std::uint64_t unitBlockGeneration(std::uint32_t unit, std::uint64_t at) const;
     // Another image of exactly this surface under another storage format with results pending
     // (block units, uncompressed, still cached), if any: Refresh takes its pending units from its
     // image on the device (borrowUnits) instead of having them stored and re-read, and the alias
@@ -602,6 +628,7 @@ private:
     std::uint32_t trackedLayers = 1;
     std::uint64_t trackedLayerBytes = 0;
     std::vector<std::uint64_t> layerGeneration;
+    std::vector<std::uint64_t> completionKept;
     std::vector<bool> layerPending;
     std::vector<bool> layerBehind;
     bool blockUnits = false;
@@ -641,6 +668,12 @@ private:
     VkFormat storageFormat = VK_FORMAT_UNDEFINED;
     // Results are on the GPU only (guarded by the pending-write registry lock).
     bool dirty = false;
+    bool drawnListed = false;
+    bool drawnWhole = false;
+    bool drawingTarget = false;
+    bool pendingFromOthers = false;
+    std::vector<DrawnRect> drawnRects;
+    std::vector<std::uint64_t> storeNotes;
     std::uint64_t version = 0;
     std::uint64_t writes = 0;
     // `original` holds the guest bytes; false after a GPU-side clear, which never read them.

core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp

@@ -75,7 +75,7 @@ struct SurfaceGeometry {
 };
 SurfaceGeometry DescribeSurface(const GuestTextureResource& descriptor);
 bool LevelsFitAllocation(const GuestTextureResource& surface, std::uint32_t levels);
-GuestTextureResource ViewedArrayWindow(const GuestTextureResource& viewed);
+GuestTextureResource ViewedArrayWindow(const GuestTextureResource& viewed, bool keysReadAsStored);
 GuestTextureResource ArrayLayerWindow(const GuestTextureResource& viewed, std::uint32_t maxArrayLayers);
 
 }

core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp

@@ -1227,8 +1227,10 @@ public:
                 continue;
             }
             surface = {};
-            const GuestLayer current{resource.baseAddress + geometry.GuestLayerOffset(layer), geometry.layerBytes, resource.tileMode, geometry.thick, GuestMemory::CollectWrites(resource.baseAddress + geometry.GuestLayerOffset(layer), static_cast<std::size_t>(geometry.layerBytes))};
-            if (held.generation != 0 && held.address == current.address && held.bytes == current.bytes && held.tileMode == current.tileMode && held.thick == current.thick && !Recorder::SnapshotWriteOverlaps(current.address, static_cast<std::size_t>(current.bytes)) && GuestMemory::UnchangedSince(current.address, static_cast<std::size_t>(current.bytes), held.generation)) {
+            GuestLayer current{resource.baseAddress + geometry.GuestLayerOffset(layer), geometry.layerBytes, resource.tileMode, geometry.thick, GuestMemory::CollectWrites(resource.baseAddress + geometry.GuestLayerOffset(layer), static_cast<std::size_t>(geometry.layerBytes))};
+            current.uncommitted = current.generation == 0 && GuestMemory::CommittedRanges(current.address, static_cast<std::size_t>(current.bytes)).empty();
+            const bool unchanged = current.uncommitted ? held.uncommitted : held.generation != 0 && GuestMemory::UnchangedSince(current.address, static_cast<std::size_t>(current.bytes), held.generation);
+            if (held.address == current.address && held.bytes == current.bytes && held.tileMode == current.tileMode && held.thick == current.thick && !Recorder::SnapshotWriteOverlaps(current.address, static_cast<std::size_t>(current.bytes)) && unchanged) {
                 held.generation = current.generation;
                 continue;
             }
@@ -1266,6 +1268,7 @@ private:
         TextureTileMode tileMode = TextureTileMode::kLinear;
         bool thick = false;
         std::uint64_t generation = 0;
+        bool uncommitted = false;
     };
 
     VkImage image = VK_NULL_HANDLE;
@@ -1970,20 +1973,15 @@ VkImageView DepthSurfaceView(const Context& context, const DepthTarget& target,
     const bool created = surface == nullptr;
     GuestAllocations::Lease creationLease;
     if (created) {
-        for (const auto& surface : surfaces()) {
-            if (!surface->active->load(std::memory_order_acquire) || !surface->samplingActive || surface->context.device != context.device) continue;
-            const auto bytes = target.samples == 1u ? DepthSliceBytes(target.extent, target.format == VK_FORMAT_D16_UNORM ? 2u : 4u) : MultisampledDepthBytes_nid_no_patch(target.extent, target.format == VK_FORMAT_D16_UNORM ? 2u : 4u, target.samples);
-            Require(target.address <= std::numeric_limits<std::uint64_t>::max() - bytes && target.htileAddress <= std::numeric_limits<std::uint64_t>::max() - HtileMetadataBytes(target.extent), "native depth attachment range overflows");
-            for (const auto& range : surface->sampling.initial) {
-                Require(target.address >= range.address + range.bytes || range.address >= target.address + bytes, "native multisample depth attachment layout or metadata changed");
-                if (target.stencilAddress != 0u) {
-                    const auto stencilBytes = target.samples == 1u ? DepthSliceBytes(target.extent, 1u) : MultisampledDepthBytes_nid_no_patch(target.extent, 1u, target.samples);
-                    Require(target.stencilAddress <= std::numeric_limits<std::uint64_t>::max() - stencilBytes && (target.stencilAddress >= range.address + range.bytes || range.address >= target.stencilAddress + stencilBytes), "native multisample stencil attachment aliases a resident depth surface");
-                }
-                if (target.htileAddress != 0u) Require(target.htileAddress >= range.address + range.bytes || range.address >= target.htileAddress + HtileMetadataBytes(target.extent), "native multisample depth attachment layout or metadata changed");
-            }
-        }
+        const bool d16 = target.format == VK_FORMAT_D16_UNORM || target.format == VK_FORMAT_D16_UNORM_S8_UINT;
+        const auto depthBytes = target.samples == 1u ? DepthSliceBytes(target.extent, d16 ? 2u : 4u) : MultisampledDepthBytes_nid_no_patch(target.extent, d16 ? 2u : 4u, target.samples);
+        const auto stencilBytes = target.stencilAddress == 0u ? 0u : target.samples == 1u ? DepthSliceBytes(target.extent, 1u) : MultisampledDepthBytes_nid_no_patch(target.extent, 1u, target.samples);
+        const auto metadataBytes = target.htileAddress == 0u ? 0u : HtileMetadataBytes(target.extent);
+        Require(target.address <= std::numeric_limits<std::uint64_t>::max() - depthBytes && target.stencilAddress <= std::numeric_limits<std::uint64_t>::max() - stencilBytes && target.htileAddress <= std::numeric_limits<std::uint64_t>::max() - metadataBytes, "native depth attachment range overflows");
         lock.unlock();
+        RequireNoResidentDepthRead_nid_no_patch(target.address, static_cast<std::size_t>(depthBytes));
+        if (stencilBytes != 0u) RequireNoResidentDepthRead_nid_no_patch(target.stencilAddress, static_cast<std::size_t>(stencilBytes));
+        if (metadataBytes != 0u) RequireNoResidentDepthRead_nid_no_patch(target.htileAddress, static_cast<std::size_t>(metadataBytes));
         auto sampling = target.samples == 1u ? DepthSamplingState{} : prepareDepthSampling(context, target);
         creationLease = target.samples == 1u || !sampling.refusal.empty() ? GuestAllocations::Lease{} : GuestAllocations::GuestAllocationsAcquireAll_nid_postfix();
         lock.lock();

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -1921,7 +1921,7 @@ struct Kept {
 // The completion side of a recorded draw: the kept objects, the record check, the lease outcome,
 // the GPU write notes, the copied-buffer write-back (listed in DrawCopiedWriters or run as a
 // completion action) and the targets marked dirty.
-void keepRecordedDraw(Recorder& recorder, const std::shared_ptr<ShaderResources>& resources, std::shared_ptr<Pipeline> pipeline, std::shared_ptr<Framebuffer> framebuffer, DrawInputs& inputs, std::vector<std::shared_ptr<StorageTexture>> targets, std::vector<std::pair<std::uint32_t, std::uint32_t>> slices, std::unique_ptr<DeviceBuffer> scratch, std::function<void()> checkRecords, bool listed, bool completion, const DrawOutcome& outcome) {
+void keepRecordedDraw(Recorder& recorder, const std::shared_ptr<ShaderResources>& resources, std::shared_ptr<Pipeline> pipeline, std::shared_ptr<Framebuffer> framebuffer, DrawInputs& inputs, std::vector<std::shared_ptr<StorageTexture>> targets, std::vector<std::pair<std::uint32_t, std::uint32_t>> slices, VkRect2D scissor, std::unique_ptr<DeviceBuffer> scratch, std::function<void()> checkRecords, bool listed, bool completion, const DrawOutcome& outcome) {
     auto kept = std::make_shared<Kept>();
     kept->resources = resources;
     kept->pipeline = std::move(pipeline);
@@ -1961,14 +1961,15 @@ void keepRecordedDraw(Recorder& recorder, const std::shared_ptr<ShaderResources>
         if (resources->Reusable()) recorder.OnComplete([resources] { resources->CheckFaults(); });
         else recorder.OnComplete([resources] { resources->WriteBackBuffers(); });
     }
+    std::vector<bool> storeNow(residents.size(), false);
     for (std::size_t index = 0; index < residents.size(); ++index) {
         if (residents[index] == nullptr) continue;
-        if (index < slices.size()) residents[index]->MarkSliceDirty(slices[index].first, slices[index].second);
+        if (residents[index]->Descriptor().tileMode == TextureTileMode::kLinear) storeNow[index] = !residents[index]->MarkDrawn(index < slices.size() ? slices[index].first : 0u, index < slices.size() ? slices[index].second : 0u, scissor) || !residents[index]->Cached();
+        else if (index < slices.size()) residents[index]->MarkSliceDirty(slices[index].first, slices[index].second);
         else residents[index]->MarkDirty();
     }
-    for (const auto& resident : residents) {
-        if (resident == nullptr || resident->Descriptor().tileMode != TextureTileMode::kLinear) continue;
-        StorageTexture::FlushPending(resident->Descriptor().baseAddress, static_cast<std::size_t>(resident->GuestBytes()), nullptr, "linear target");
+    for (std::size_t index = 0; index < residents.size(); ++index) {
+        if (storeNow[index]) StorageTexture::FlushPending(residents[index]->Descriptor().baseAddress, static_cast<std::size_t>(residents[index]->GuestBytes()), nullptr, "linear target");
     }
 }
 
@@ -2195,7 +2196,6 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
         recorder->FlushStores();
         outcome.storesFlushed = true;
     }
-    const auto drawBindings = resources.PrepareDrawBindings(*recorder, record.moved);
     const bool capture = CaptureInputsEnabled();
     const bool meshIndirect = state.stages.mesh && args != nullptr;
     const bool addressBased = resources.HoldsLease();
@@ -2217,6 +2217,8 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
     }
     for (const auto image : record.multisampleImages) passAttachments.emplace_back(image, true);
     const bool unknownMemory = addressBased || resources.UsesAddressTable() || resources.HasCopiedWrites();
+    static_cast<void>(StorageTexture::StoreDrawnLinearTouching("draw access", [&](std::uint64_t begin, std::uint64_t end) { return std::any_of(passReads.begin(), passReads.end(), [&](const auto& read) { return read.first < end && begin < read.second; }) || resources.WritesOverlap(begin, static_cast<std::size_t>(end - begin)); }, unknownMemory));
+    const auto drawBindings = resources.PrepareDrawBindings(*recorder, record.moved);
     static const bool addressReadPasses = std::getenv("APS5_NO_ADDRESS_READ_PASSES") == nullptr;
     const bool readOnlyBda = addressReadPasses && resources.BdaReadOnly();
     outcome.addressDraw = addressBased;
@@ -2361,7 +2363,7 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
     }
     recorder->LeaveRenderPassOpen(passKey, drawTiming, fenced, passAccess, std::move(storeProxies));
     timer.phase(PhaseRecord);
-    keepRecordedDraw(*recorder, record.resources, std::move(record.pipeline), std::move(record.framebuffer), inputs, std::move(record.targets), std::move(record.targetSlices), std::move(scratch), std::move(checkRecords), record.listed, record.completion, outcome);
+    keepRecordedDraw(*recorder, record.resources, std::move(record.pipeline), std::move(record.framebuffer), inputs, std::move(record.targets), std::move(record.targetSlices), state.scissor, std::move(scratch), std::move(checkRecords), record.listed, record.completion, outcome);
     timer.phase(PhaseKeep);
     if (record.waited) {
         // The wait the dispatch path makes for such work (source 3, "address-based"): the
@@ -2952,6 +2954,14 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D
         report(outcome.waited ? (outcome.reason == SyncLease ? " recorded then waited (lease)" : " recorded then waited (copied writes)") : completion ? " recorded with completion" : " recorded");
         return;
     }
+    const bool gpuIndirect = args != nullptr && indirect.path == IndirectDrawPath::Gpu;
+    const auto touches = [&](std::uint64_t begin, std::uint64_t end) {
+        const auto bytes = static_cast<std::size_t>(end - begin);
+        if (resources->WritesOverlap(begin, bytes) || resources->ReadsOverlap(begin, bytes)) return true;
+        if (!gpuIndirect) return false;
+        return (begin < args->arguments + args->RangeBytes() && args->arguments < end) || (args->countIndirect && begin < args->countAddress + 4 && args->countAddress < end);
+    };
+    static_cast<void>(StorageTexture::StoreDrawnLinearTouching("draw access", [&](std::uint64_t begin, std::uint64_t end) { return touches(begin, end) || std::any_of(inputs.inPlaceReads.begin(), inputs.inPlaceReads.end(), [&](const auto& read) { return read.first < end && begin < read.second; }); }, resources->HoldsLease() || resources->UsesAddressTable() || resources->HasCopiedWrites()));
     APS5_LOG_CHARS_OUT_DEBUG("Creating CommandBatch");
     std::optional<CommandBatch> batch;
     if (!recorded) {
@@ -2961,16 +2971,9 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D
     }
     using CommandClass = Recorder::CommandClass;
     const auto countBarrier = [&](std::uint32_t count = 1) { if (recorded) Recorder::CountBarriers(CommandClass::Draw, count); };
-    const bool gpuIndirect = args != nullptr && indirect.path == IndirectDrawPath::Gpu;
     // A queued DCC key store over memory the draw writes or reads in place (unknown for an
     // address-based build), or over its GPU-side records, lands before it, as before a dispatch
     // (VulkanDevice::dispatch).
-    const auto touches = [&](std::uint64_t begin, std::uint64_t end) {
-        const auto bytes = static_cast<std::size_t>(end - begin);
-        if (resources->WritesOverlap(begin, bytes) || resources->ReadsOverlap(begin, bytes)) return true;
-        if (!gpuIndirect) return false;
-        return (begin < args->arguments + args->RangeBytes() && args->arguments < end) || (args->countIndirect && begin < args->countAddress + 4 && args->countAddress < end);
-    };
     if (recorded && recorder->HasQueuedKeyStores() && (resources->HoldsLease() || recorder->AnyQueuedKeyStore(touches))) recorder->FlushKeyStores();
     // A queued label store over such memory likewise (Recorder::RecordStore).
     if (recorded && recorder->HasQueuedStores() && (resources->HoldsLease() || recorder->AnyQueuedStore(touches))) recorder->FlushStores();
@@ -3137,7 +3140,7 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D
     APS5_LOG_CHARS_OUT_DEBUG("SubmitAndWait");
     timer.phase(PhaseRecord);
     if (recorded) {
-        keepRecordedDraw(*recorder, resources, pipeline, framebuffer, inputs, owners, [&] { std::vector<std::pair<std::uint32_t, std::uint32_t>> slices; for (const auto& binding : targets) slices.emplace_back(binding.color.arraySlice, binding.color.arrayLastSlice != 0 ? binding.color.layers : 1u); return slices; }(), std::move(scratch), std::move(checkRecords), listed, completion, outcome);
+        keepRecordedDraw(*recorder, resources, pipeline, framebuffer, inputs, owners, [&] { std::vector<std::pair<std::uint32_t, std::uint32_t>> slices; for (const auto& binding : targets) slices.emplace_back(binding.color.arraySlice, binding.color.arrayLastSlice != 0 ? binding.color.layers : 1u); return slices; }(), state.scissor, std::move(scratch), std::move(checkRecords), listed, completion, outcome);
         timer.phase(PhaseKeep);
         if (outcome.waited) {
             // The wait the dispatch path makes for such work (source 3, "address-based"): the
@@ -3279,7 +3282,7 @@ DrawRecipeOutcome DrawWithRecipe(const Context& context, const State& state, con
         timer.phase(PhaseSetup);
         if (state.colors[index].fmaskAddress != 0) GuardOneFragmentFmask(context, state.colors[index]);
         auto stored = recipe.targets[index].lock();
-        if (stored == nullptr || !StorageImageCached(context, stored.get()) || !StorageImageServesKeys(*stored, state.colors[index].dccAddress)) return miss(DrawRecipeMiss::TargetGone);
+        if (stored == nullptr || !StorageImageCached(context, stored.get()) || !StorageImageServesKeys(*stored, SurfaceForTarget(state.colors[index]).dccAddress)) return miss(DrawRecipeMiss::TargetGone);
         auto resident = refreshResidentTarget(context, state, state.colors[index], outcome, profile, [&] {
             if (state.colors[index].arrayLastSlice != 0) stored->RefreshLayers(state.colors[index].arraySlice, 1);
             else stored->Refresh();

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -69,6 +69,7 @@ struct ImageMirror {
     std::uint64_t bytes = 0;
     bool writable = false;
     bool heap = false;
+    bool joinedHeap = false;
     bool sparse = false;
     std::uint64_t reserved = 0;
     std::vector<std::pair<std::uint64_t, std::uint64_t>> committed;
@@ -285,6 +286,7 @@ struct HostImports {
     bool allocationFailed = false;
     std::uint8_t reportNoted = 0;
     std::map<std::uint64_t, std::uint64_t> spaceUses;
+    std::uint64_t budgetRefusals = 0;
 };
 
 HostImports& Imports() {
@@ -868,6 +870,7 @@ bool importsStale(const Context& context, const HostImports& state);
 constexpr std::uint64_t settleFloorBytes = 0x100000;
 #ifdef _WIN32
 constexpr std::uint64_t settleBlockBytes = 0x10000;
+constexpr std::uint64_t settlePageBytes = 0x1000;
 constexpr auto settlePollPeriod = std::chrono::microseconds(100);
 std::atomic<bool> deferredSettles{std::getenv("APS5_NO_DEFERRED_SETTLES") == nullptr};
 #endif
@@ -1094,8 +1097,22 @@ void reapSettleBatches(const Context& context, HostImports& state) {
     }
 }
 
+bool comparedImportShares(const HostImports& state, std::uint64_t own, std::uint64_t begin, std::uint64_t end) {
+    const auto shares = [&](const HostImport& entry) { return entry.unwatched && entry.memoryBase < end && begin < entry.memoryBase + entry.pinnedBytes; };
+    for (const auto& [base, entry] : state.imports) {
+        if (base != own && shares(entry)) return true;
+    }
+    for (const auto& [device, list] : state.parked) {
+        for (const auto& retired : list) {
+            if (shares(retired->node.mapped())) return true;
+        }
+    }
+    return false;
+}
+
 void rewatchSettledImports(HostImports& state) {
     if (state.settleReady.empty()) return;
+    std::vector<std::uint64_t> bases;
     std::vector<std::pair<std::uint64_t, std::uint64_t>> ranges;
     for (const auto base : state.settleReady) {
         const auto settle = state.pendingSettles.find(base);
@@ -1103,19 +1120,23 @@ void rewatchSettledImports(HostImports& state) {
         state.pendingSettles.erase(settle);
         const auto import = state.imports.find(base);
         if (import == state.imports.end() || !import->second.unwatched) continue;
-        const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> span{{{base, base + import->second.bytes}}};
+        const auto first = base & ~(settleBlockBytes - 1);
+        const auto stop = (base + import->second.bytes + settleBlockBytes - 1) & ~(settleBlockBytes - 1);
+        if ((first != base || stop != base + import->second.bytes) && comparedImportShares(state, base, first, stop)) continue;
+        const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> span{{{first, stop}}};
         if (StorageTexture::AnyLiveOverlaps(span)) {
             ++state.settlesKeptCompared;
             continue;
         }
-        ranges.push_back(span.front());
+        bases.push_back(base);
+        ranges.emplace_back(base & ~(settlePageBytes - 1), (base + import->second.bytes + settlePageBytes - 1) & ~(settlePageBytes - 1));
     }
     state.settleReady.clear();
     if (ranges.empty()) return;
     GuestMemory::Rewatch(ranges);
-    for (const auto& [begin, end] : ranges) {
-        const auto import = state.imports.find(begin);
-        if (import == state.imports.end() || !GuestMemory::Watched(begin, static_cast<std::size_t>(end - begin))) continue;
+    for (const auto base : bases) {
+        const auto import = state.imports.find(base);
+        if (import == state.imports.end() || !GuestMemory::Watched(base, static_cast<std::size_t>(import->second.bytes))) continue;
         import->second.unwatched = false;
         ++state.settlesRewatched;
     }
@@ -1130,6 +1151,49 @@ void settleUnwatchedImports(const Context& context, HostImports& state) {
     rewatchSettledImports(state);
     submitSettleBatch(context, state);
 }
+
+void settleImportsNow(const Context& context, HostImports& state, std::uint64_t begin, std::uint64_t end) {
+    if (context.settleQueue == VK_NULL_HANDLE || context.settleQueueMutex == nullptr || !GuestMemory::GpuMutex().HeldByThisThread()) return;
+    std::vector<std::uint64_t> needed;
+    auto it = state.imports.lower_bound(begin);
+    if (it != state.imports.begin()) --it;
+    for (; it != state.imports.end() && it->first < end; ++it) {
+        auto& entry = it->second;
+        if (entry.base + entry.bytes <= begin || !entry.unwatched) continue;
+        auto settle = state.pendingSettles.find(it->first);
+        if (settle == state.pendingSettles.end()) {
+            if (!entry.settleOnDemand || !deferredSettles.load(std::memory_order_relaxed)) continue;
+            entry.settleOnDemand = false;
+            settle = state.pendingSettles.insert_or_assign(it->first, PendingSettle{}).first;
+        }
+        if (settle->second.phase == SettlePhase::Wanted) {
+            std::erase(state.settleWanted, it->first);
+            state.settleWanted.insert(state.settleWanted.begin(), it->first);
+        }
+        needed.push_back(it->first);
+    }
+    if (needed.empty()) return;
+    auto& deviceSettle = settleFor(context, state);
+    const auto wanted = [&] {
+        return std::any_of(needed.begin(), needed.end(), [&](std::uint64_t base) {
+            const auto settle = state.pendingSettles.find(base);
+            return settle != state.pendingSettles.end() && settle->second.phase == SettlePhase::Wanted;
+        });
+    };
+    while (wanted()) {
+        if (std::all_of(deviceSettle.batches.begin(), deviceSettle.batches.end(), [](const SettleBatch& batch) { return batch.busy; })) {
+            for (const auto& batch : deviceSettle.batches) waitSettleBatch(deviceSettle, context.device, batch);
+            reapSettleBatches(context, state);
+        }
+        submitSettleBatch(context, state);
+    }
+    for (const auto base : needed) {
+        const auto settle = state.pendingSettles.find(base);
+        if (settle != state.pendingSettles.end() && settle->second.phase == SettlePhase::Submitted && settle->second.device == context.device) waitSettleBatch(deviceSettle, context.device, deviceSettle.batches[settle->second.batch]);
+    }
+    reapSettleBatches(context, state);
+    rewatchSettledImports(state);
+}
 #endif
 
 void decideImportWatch(const Context& context, HostImports& state) {
@@ -1631,6 +1695,7 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
         ++counts.refused;
         counts.refusedBytes += lease[index]->bytes;
         ++counts.remembered;
+        ++state.budgetRefusals;
         return nullptr;
     }
     const auto bounds = importRunBounds(lease, address, largest, budget);
@@ -1796,6 +1861,7 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
         if (evict) state.evictingRefusals.insert_or_assign(std::pair{base, bytes}, EvictingRefusal{base, bytes, GuestAllocations::Mapped(run.ranges.begin(), run.ranges.end()), state.epoch, state.liveBytes, budget, room, importWindow.load(std::memory_order_relaxed), state.buildFloor, ready, readyHeld, joined});
         if (stepDown && containing != run.ranges.end() && room >= (*containing)->bytes) return importAllocation(context, state, address, lease, requester, end, evict, std::min(bytes / 2, room));
         if (containing != run.ranges.end() && run.ranges.size() > 1) remember((*containing)->address, (*containing)->bytes, GuestAllocations::Mapped{*containing});
+        ++state.budgetRefusals;
         return nullptr;
     }
     for (auto it = first; it != state.imports.end() && it->first < base + bytes;) {
@@ -1884,7 +1950,7 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
         GuestMemory::ImportWatched(base, bytes, [&] {
             step = createBackingImport(context, entry, backing, result);
             return step == nullptr;
-        }, watchedDmaBuf);
+        }, watchedDmaBuf, false);
     }
     if (step == nullptr && entry.dmaBuf && bytes >= settleFloorBytes && context.settleQueue != VK_NULL_HANDLE && context.settleQueueMutex != nullptr) {
         static const bool traceSettle = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr;
@@ -1948,8 +2014,10 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
     entry.joined = joined;
     entry.lastUse = ++state.useCounter;
 #ifdef _WIN32
-    if (deferredSettles.load(std::memory_order_relaxed) && entry.alias == nullptr && state.watchPrivateImports && writable && base % settleBlockBytes == 0 && bytes % settleBlockBytes == 0) {
-        if (entry.unwatched) {
+    if (deferredSettles.load(std::memory_order_relaxed) && entry.alias == nullptr && state.watchPrivateImports && writable) {
+        if (base % settleBlockBytes != 0 || bytes % settleBlockBytes != 0) {
+            entry.settleOnDemand = entry.unwatched;
+        } else if (entry.unwatched) {
             state.pendingSettles.insert_or_assign(base, PendingSettle{});
             state.settleWanted.push_back(base);
         } else if (settled && !GuestMemory::Watched(base, static_cast<std::size_t>(bytes))) {
@@ -2202,6 +2270,39 @@ void unlistMirror(ImageMirrors& state, ImageMirror& mirror) {
     mirror.listed = false;
 }
 
+struct JoinedHeapRange {
+    std::vector<std::weak_ptr<const GuestAllocations::Range>> members;
+    std::shared_ptr<const GuestAllocations::Range> range;
+};
+
+struct JoinedHeapRanges {
+    std::mutex mutex;
+    std::map<std::uint64_t, JoinedHeapRange> entries;
+};
+
+JoinedHeapRanges& JoinedHeaps() {
+    static JoinedHeapRanges ranges;
+    return ranges;
+}
+
+bool releaseExpiredJoinedHeaps() {
+    std::vector<std::shared_ptr<const GuestAllocations::Range>> released;
+    {
+        auto& joined = JoinedHeaps();
+        std::lock_guard lock(joined.mutex);
+        for (auto it = joined.entries.begin(); it != joined.entries.end();) {
+            const auto& members = it->second.members;
+            if (std::none_of(members.begin(), members.end(), [](const auto& member) { return member.expired(); })) {
+                ++it;
+                continue;
+            }
+            released.push_back(std::move(it->second.range));
+            it = joined.entries.erase(it);
+        }
+    }
+    return !released.empty();
+}
+
 std::uint64_t blockBytes(const GuestRuns& runs) {
     constexpr std::uint64_t block = ResidentBuffer::BlockBytes;
     std::uint64_t blocks = 0;
@@ -2708,10 +2809,15 @@ bool importsNow(const Context& context, HostImports& state, const GuestAllocatio
     return context.hostImportAlignment != 0 && (importsStale(context, state) || importAllocation(context, state, address, lease, ImportRequester::Build, 0, false) != nullptr);
 }
 
+const char* heapCopyReason(bool overBudget, bool watched) {
+    if (overBudget) return watched ? "import over the host import budget, heap mirror refused" : "import over the host import budget, heap not fully write-watched";
+    return watched ? "heap mirror refused" : "heap not fully write-watched";
+}
+
 // The mirror for a leased image range: the existing one, or a new one when none exists or the
 // registered Range object changed (a protection change). Null when the range cannot be mirrored. An
 // existing writable mirror's blocks are appended to `blocks` for the caller's one compare of every
-std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::shared_ptr<const GuestAllocations::Range>& range, std::vector<RefreshBlock>& blocks, bool heap, std::uint64_t begin = 0, std::uint64_t bytes = 0) {
+std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::shared_ptr<const GuestAllocations::Range>& range, std::vector<RefreshBlock>& blocks, bool heap, std::uint64_t begin = 0, std::uint64_t bytes = 0, bool joinedHeap = false) {
     auto selected = *range;
     if (bytes != 0) {
         Require(begin >= range->address && bytes <= range->bytes && begin - range->address <= range->bytes - bytes, "heap mirror selection exceeds its registered range");
@@ -2732,7 +2838,7 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
         }
         if (state.failed.contains(selected.address)) return nullptr;
         const auto found = state.entries.find(selected.address);
-        if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->reserved == selected.bytes && found->second->writable == range->writable && found->second->heap == heap) mirror = found->second;
+        if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->reserved == selected.bytes && found->second->writable == range->writable && found->second->heap == heap && found->second->joinedHeap == joinedHeap) mirror = found->second;
     }
     if (mirror != nullptr) {
         if (!mirror->sparse) {
@@ -2755,6 +2861,7 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
     mirror->sparse = sparse;
     mirror->writable = range->writable;
     mirror->heap = heap;
+    mirror->joinedHeap = joinedHeap;
     mirror->range = range;
     GuestRuns backed;
     if (heap) {
@@ -2887,9 +2994,11 @@ std::shared_ptr<ImageMirror> findMirror(const Context& context, std::uint64_t be
 
 // Drops mirrors whose registered range is gone (a later build made a new one for the replacement).
 void sweepMirrors() {
+    const bool released = releaseExpiredJoinedHeaps();
     const auto space = Spaces().current.load();
     auto& state = Mirrors();
     std::lock_guard lock(state.mutex);
+    if (released) ++state.version;
     if (space != nullptr && state.pinnedSpace == space->serial && state.pinnedVersion == state.version) return;
     ++state.sweeps;
     bool pinned = space != nullptr;
@@ -2913,6 +3022,14 @@ void sweepMirrors() {
 
 void releaseImportedMirrors(const Context& context, std::span<const std::pair<std::uint64_t, std::uint64_t>> imported) {
     if (imported.empty()) return;
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> coverage(imported.begin(), imported.end());
+    std::sort(coverage.begin(), coverage.end());
+    std::size_t count = 0;
+    for (const auto& interval : coverage) {
+        if (count != 0 && interval.first <= coverage[count - 1].second) coverage[count - 1].second = std::max(coverage[count - 1].second, interval.second);
+        else coverage[count++] = interval;
+    }
+    coverage.resize(count);
     std::vector<std::shared_ptr<ImageMirror>> released;
     auto& state = Mirrors();
     std::lock_guard lock(state.mutex);
@@ -2926,6 +3043,13 @@ void releaseImportedMirrors(const Context& context, std::span<const std::pair<st
                 ++it;
                 continue;
             }
+            if (mirror->joinedHeap) {
+                const auto covered = std::upper_bound(coverage.begin(), coverage.end(), mirror->base, [](std::uint64_t address, const auto& interval) { return address < interval.first; });
+                if (covered == coverage.begin() || std::prev(covered)->second < mirror->base + mirror->reserved) {
+                    ++it;
+                    continue;
+                }
+            }
             ++state.heapImported;
             state.heapImportedBytes += mirror->accounted;
             unlistMirror(state, *mirror);
@@ -3235,7 +3359,8 @@ std::string AddressCopyOverflow(std::vector<AddressCopy> copies, std::uint64_t l
     std::string message = text;
     for (std::size_t index = 0; index < copies.size() && index < 8; ++index) {
         const auto& copy = copies[index];
-        std::snprintf(text, sizeof(text), " 0x%llx+0x%llx (%.1f MiB committed, %s)", static_cast<unsigned long long>(copy.begin), static_cast<unsigned long long>(copy.end - copy.begin), copy.committed / 1048576.0, copy.reason);
+        const char* reason = copy.heap ? heapCopyReason(copy.overBudget, GuestMemory::Watched(copy.begin, copy.end - copy.begin)) : copy.reason;
+        std::snprintf(text, sizeof(text), " 0x%llx+0x%llx (%.1f MiB committed, %s)", static_cast<unsigned long long>(copy.begin), static_cast<unsigned long long>(copy.end - copy.begin), copy.committed / 1048576.0, reason);
         message += text;
     }
     return message;
@@ -3716,6 +3841,22 @@ const HostImport* ExistingHostImportFor(const Context& context, std::uint64_t ad
     return findImport(state, address, address + bytes);
 }
 
+bool WatchedAfterSettle(const Context& context, std::uint64_t address, std::size_t bytes) {
+#ifdef _WIN32
+    if (context.hostImportAlignment != 0 && bytes != 0 && bytes <= std::numeric_limits<std::uint64_t>::max() - address && GuestMemory::GpuMutex().HeldByThisThread()) {
+        ensurePinWaiter();
+        auto& state = Imports();
+        std::lock_guard lock(state.mutex);
+        if (importsStale(context, state)) {
+            const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
+            refreshImports(context, state, lease);
+        }
+        settleImportsNow(context, state, address, address + bytes);
+    }
+#endif
+    return GuestMemory::Watched(address, bytes);
+}
+
 bool ImportMappedRanges(const Context& context, const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice) {
     if (context.hostImportAlignment == 0) return true;
     ensurePinWaiter();
@@ -3925,16 +4066,24 @@ void GuestBufferMemory::AcquireRegistered() {
         // The import lock is taken per range: a mirror's preparation may wait for recorded work, which
         // must not happen under it.
         regions.reserve(lease.size());
+        static const std::uint64_t copyLimit = [] {
+            const char* value = std::getenv("APS5_ADDRESS_COPY_MAX_MIB");
+            return (value ? std::strtoull(value, nullptr, 10) : 64ull) << 20u;
+        }();
+        const auto copyOverride = addressCopyLimitOverride.load(std::memory_order_relaxed);
+        const auto cap = copyOverride != 0 ? copyOverride : copyLimit;
         std::vector<AddressCopy> copies;
         std::vector<std::pair<std::uint64_t, std::uint64_t>> imported;
         const HostImport* covering = nullptr;
         std::uint64_t coveredBegin = 0;
         std::uint64_t coveredEnd = 0;
+        std::uint64_t copiedBytes = 0;
         for (const auto& range : lease) {
             if (!range->readable || !range->gpu) continue;
             validate(range->address, range->bytes);
             Region region{range->address, range->address + range->bytes, range->writable, {}, nullptr};
             const HostImport* entry = nullptr;
+            bool overBudget = false;
             std::vector<ImportPiece> pieces;
             const auto limit = importMaxBytes.load(std::memory_order_relaxed);
             if (importable && covering != nullptr && range->address >= coveredBegin && range->address + range->bytes <= coveredEnd) {
@@ -3946,7 +4095,9 @@ void GuestBufferMemory::AcquireRegistered() {
             } else if (importable) {
                 auto& state = Imports();
                 std::lock_guard lock(state.mutex);
+                const auto refusals = state.budgetRefusals;
                 entry = importAllocation(context, state, range->address, lease, ImportRequester::Build, 0, false);
+                overBudget = state.budgetRefusals != refusals;
                 covering = entry;
                 if (entry != nullptr) {
                     coveredBegin = entry->base;
@@ -4012,17 +4163,36 @@ void GuestBufferMemory::AcquireRegistered() {
                 committed = 0;
                 for (const auto& [first, last] : copied.backed) committed += last - first;
             }
-            const char* reason = !mirrorsEnabled() ? "mirrors disabled by APS5_NO_LEASE_MIRROR" : copied.sparse ? "unreadable pages" : !range->releasable ? "image mirror refused" : "outside the write-watched arena";
-            copies.push_back({copied.begin, copied.end, committed, reason});
+            if (overBudget && copiedBytes + committed > cap) {
+                auto& state = Imports();
+                std::lock_guard lock(state.mutex);
+                entry = importAllocation(context, state, range->address, lease, ImportRequester::Build, 0, true);
+                if (entry != nullptr) {
+                    covering = entry;
+                    coveredBegin = entry->base;
+                    coveredEnd = entry->base + entry->bytes;
+                }
+                static auto lastReport = std::chrono::steady_clock::now() - std::chrono::seconds(60);
+                if (entry != nullptr && std::chrono::steady_clock::now() - lastReport > std::chrono::seconds(10)) {
+                    lastReport = std::chrono::steady_clock::now();
+                    std::fprintf(stderr, "[gpu] host import budget: 0x%llx+0x%llx imported by evicting idle imports, as copying it would pass APS5_ADDRESS_COPY_MAX_MIB\n", static_cast<unsigned long long>(range->address), static_cast<unsigned long long>(range->bytes));
+                }
+            }
+            if (entry != nullptr) {
+                regions.pop_back();
+                region.hostBacked = true;
+                region.direct = entry;
+                regions.push_back(std::move(region));
+                imported.emplace_back(range->address, range->address + range->bytes);
+                continue;
+            }
+            copiedBytes += committed;
+            const char* reason = !mirrorsEnabled() ? "mirrors disabled by APS5_NO_LEASE_MIRROR" : copied.sparse ? "unreadable pages" : !range->releasable ? "image mirror refused" : nullptr;
+            copies.push_back({copied.begin, copied.end, committed, reason, reason == nullptr, overBudget});
         }
         releaseImportedMirrors(context, imported);
         regionsSorted = sortedLookup;
-        static const std::uint64_t copyLimit = [] {
-            const char* value = std::getenv("APS5_ADDRESS_COPY_MAX_MIB");
-            return (value ? std::strtoull(value, nullptr, 10) : 64ull) << 20u;
-        }();
-        const auto limit = addressCopyLimitOverride.load(std::memory_order_relaxed);
-        if (const auto overflow = AddressCopyOverflow(std::move(copies), limit != 0 ? limit : copyLimit); !overflow.empty()) {
+        if (const auto overflow = AddressCopyOverflow(std::move(copies), cap); !overflow.empty()) {
             std::fprintf(stderr, "FATAL: %s\n", overflow.c_str());
             std::fflush(stderr);
             std::abort();
@@ -4270,15 +4440,40 @@ bool GuestBufferMemory::sparseHeapCandidate(const Region& region) {
     return region.sparseHeap && region.sparse && region.mirror == nullptr && region.direct == nullptr && (region.hostBacked || region.writable) && mirrorsEnabled() && sparseMirrorsEnabled();
 }
 
+std::shared_ptr<const GuestAllocations::Range> joinedHeapRange(const GuestAllocations::Lease& lease, std::uint64_t begin, std::uint64_t end) {
+    const auto run = crossedRun(lease, begin, end);
+    if (run.ranges.size() < 2 || std::any_of(run.ranges.begin(), run.ranges.end(), [](const auto& range) { return !range->releasable; })) return nullptr;
+    releaseExpiredJoinedHeaps();
+    auto& joined = JoinedHeaps();
+    std::lock_guard lock(joined.mutex);
+    auto& entry = joined.entries[run.base];
+    bool same = entry.range != nullptr && entry.members.size() == run.ranges.size();
+    for (std::size_t index = 0; same && index < run.ranges.size(); ++index) same = sameRange(entry.members[index], run.ranges[index]);
+    if (!same) {
+        const auto& first = *run.ranges.front();
+        entry.members.assign(run.ranges.begin(), run.ranges.end());
+        entry.range = std::make_shared<const GuestAllocations::Range>(GuestAllocations::Range{run.base, static_cast<std::size_t>(run.bytes), true, first.writable, first.allocationAddress, first.allocationBytes, true, first.gpu, first.backingSerial});
+    }
+    return entry.range;
+}
+
 bool GuestBufferMemory::bindSparseMirror(Region& region, const GuestAllocations::Lease& lease) {
     const auto backedEnd = region.backed.empty() ? region.begin : region.backed.back().second;
     const auto neededEnd = std::min(region.end, std::max(backedEnd, std::max(region.lastView, region.begin) + 4u));
-    const auto* range = containingRange(lease, region.begin, neededEnd);
-    if (range == nullptr || !range->releasable) return false;
+    std::shared_ptr<const GuestAllocations::Range> owner;
+    bool joined = false;
+    if (const auto* range = containingRange(lease, region.begin, neededEnd); range != nullptr) {
+        if (!range->releasable) return false;
+        owner = leasedRangeOwner(lease, range->address);
+    } else {
+        joined = true;
+        owner = joinedHeapRange(lease, region.begin, neededEnd);
+        if (owner == nullptr) return false;
+    }
     std::vector<RefreshBlock> blocks;
     std::shared_ptr<ImageMirror> mirror;
     try {
-        mirror = acquireMirror(context, leasedRangeOwner(lease, range->address), blocks, true);
+        mirror = acquireMirror(context, owner, blocks, true, 0, 0, joined);
     } catch (const std::runtime_error& error) {
         static std::atomic<int> reported{0};
         if (reported.fetch_add(1, std::memory_order_relaxed) < 4) std::fprintf(stderr, "[gpu] %s; copying the sparse heap instead\n", error.what());
@@ -4462,6 +4657,7 @@ VkBufferUsageFlags gpuCopyUsage(bool addressable) {
 // window takes those elements there. APS5_ATOMIC_STAGE_MAX_KIB (1024) bounds the atomic one.
 constexpr std::uint64_t SweptShadowMax = std::uint64_t{16} << 20u;
 constexpr std::uint64_t DiscreteWrittenShadowMax = std::uint64_t{8} << 20u;
+constexpr std::uint64_t DiscreteSweptShadowMax = std::uint64_t{256} << 20u;
 
 bool writtenShadowEnabled() {
     static const bool disabled = std::getenv("APS5_NO_WRITTEN_SHADOW") != nullptr;
@@ -4604,7 +4800,7 @@ bool GuestBufferMemory::stagingEligible(const Region& region, bool addressable)
     const auto bytes = region.end - region.begin;
     if (!WritesOverlap(region.begin, static_cast<std::size_t>(bytes))) return false;
     if (region.atomic && atomicStagingEnabled() && bytes <= atomicStageMax()) return true;
-    return writtenShadowEnabled() && bytes >= writtenShadowMin() && bytes <= (region.swept ? std::max(writtenShadowMax(context.discreteGpu), SweptShadowMax) : writtenShadowMax(context.discreteGpu));
+    return writtenShadowEnabled() && bytes >= writtenShadowMin() && bytes <= (region.swept ? std::max(writtenShadowMax(context.discreteGpu), context.discreteGpu ? DiscreteSweptShadowMax : SweptShadowMax) : writtenShadowMax(context.discreteGpu));
 }
 
 bool GuestBufferMemory::overlayCandidate(const Region& inside, bool readOnly) const {

core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp

@@ -4,6 +4,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/FirstSampleCopy.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
@@ -500,6 +501,7 @@ public:
 
     Context context;
     std::shared_ptr<std::atomic<bool>> active = std::make_shared<std::atomic<bool>>(true);
+    std::shared_ptr<std::atomic<bool>> hostOnly = std::make_shared<std::atomic<bool>>(true);
     std::shared_ptr<std::atomic<bool>> metadataValid;
     ColorTarget target;
     VkImage image = VK_NULL_HANDLE;
@@ -574,7 +576,7 @@ private:
         if (use.cmaskAddress != 0) dccRanges.push_back({use.cmaskAddress, use.cmaskBytes, 0});
         for (auto& range : dccRanges) {
             GuestMemory::CheckRange(reinterpret_cast<const void*>(range.address), range.bytes, 256);
-            Require(GuestMemory::Watched(range.address, range.bytes), "initial multisampled DCC contents require watched guest memory");
+            Require(WatchedAfterSettle(context, range.address, range.bytes), "initial multisampled DCC contents require watched guest memory");
             range.generation = collectBlocks(range.address, range.bytes);
             Require(range.generation != 0, "initial multisampled DCC contents have no write generation");
         }
@@ -591,7 +593,7 @@ private:
             fmaskRange = GuestMemory::UnchangedQuery{use.fmaskAddress, static_cast<std::size_t>(DescribeSurface(resource).guestBytes), 0};
             auto& range = *fmaskRange;
             GuestMemory::CheckRange(reinterpret_cast<const void*>(range.address), range.bytes, 65536u);
-            Require(GuestMemory::Watched(range.address, range.bytes), "native FMASK contents require watched guest memory");
+            Require(WatchedAfterSettle(context, range.address, range.bytes), "native FMASK contents require watched guest memory");
             range.generation = GuestMemory::CollectWritesUncached(range.address, range.bytes);
             Require(range.generation != 0, "native FMASK contents have no write generation");
             for (const auto& other : dccRanges) Require(range.address >= other.address + other.bytes || other.address >= range.address + range.bytes, "native FMASK overlaps its color or metadata planes");
@@ -631,7 +633,7 @@ private:
         if (fmaskRange) {
             Require(use.fmaskAddress == target.fmaskAddress && use.fmaskPipeBankXor == target.fmaskPipeBankXor, "native FMASK target layout changed");
             const auto& range = *fmaskRange;
-            requireDccRange(GuestMemory::Watched(range.address, range.bytes), range, "native FMASK sampling requires watched guest memory");
+            requireDccRange(WatchedAfterSettle(context, range.address, range.bytes), range, "native FMASK sampling requires watched guest memory");
             GuestMemory::CollectWritesUncached(range.address, range.bytes);
             requireDccRange(GuestMemory::UnchangedSince(range.address, range.bytes, range.generation) || !GuestMemory::StoredOver(range.address, range.bytes, range.generation), range, "writes to a resident native FMASK are not implemented");
             if (auto* recorder = Recorder::Active(); recorder != nullptr) requireDccRange(!recorder->PendingWriteOverlaps(range.address, range.bytes), range, "pending writes to a resident native FMASK are not implemented");
@@ -660,6 +662,7 @@ struct GuardedRange {
     std::uint32_t samples;
     std::shared_ptr<std::atomic<bool>> active;
     std::shared_ptr<std::atomic<bool>> metadataValid;
+    std::shared_ptr<std::atomic<bool>> hostOnly;
     bool fmask = false;
 };
 
@@ -734,7 +737,7 @@ void retireUnmapped(std::uintptr_t address, std::size_t bytes) {
     anyGuarded().store(!guarded().empty(), std::memory_order_release);
 }
 
-void guard(const Context& context, const ColorTarget& target, const std::shared_ptr<std::atomic<bool>>& active, const std::shared_ptr<std::atomic<bool>>& metadataValid) {
+void guard(const Context& context, const ColorTarget& target, const std::shared_ptr<std::atomic<bool>>& active, const std::shared_ptr<std::atomic<bool>>& metadataValid, const std::shared_ptr<std::atomic<bool>>& hostOnly) {
     auto bytes = std::max<std::uint64_t>(target.bytes * target.samples, static_cast<std::uint64_t>(target.extent.width) * target.extent.height * target.elementBytes * target.samples);
     if (MultisampledBlockLog2(target.tileMode) == 16u && target.tileMode != ColorTileMode::Standard64KB && std::has_single_bit(target.elementBytes) && std::has_single_bit(target.samples)) {
         const auto log2Samples = std::countr_zero(target.samples);
@@ -746,11 +749,11 @@ void guard(const Context& context, const ColorTarget& target, const std::shared_
         bytes = align(target.extent.width, blockWidth) * align(target.extent.height, blockHeight) * target.elementBytes * target.samples;
     }
     std::unique_lock lock(guardedMutex());
-    guarded().push_back({context.device, target.address, bytes, target.samples, active});
+    guarded().push_back({context.device, target.address, bytes, target.samples, active, nullptr, hostOnly});
     if (target.dccAddress != 0) {
-        guarded().push_back({context.device, target.dccAddress, MultisampledDccBytes(target), target.samples, active, metadataValid});
-        if (target.cmaskAddress != 0) guarded().push_back({context.device, target.cmaskAddress, target.cmaskBytes, target.samples, active, metadataValid});
-        if (target.samples == 8u && target.fmaskAddress != 0u && target.cmaskAddress != 0u) guarded().push_back({context.device, target.fmaskAddress, DescribeSurface(fmaskResource(target)).guestBytes, target.samples, active, metadataValid});
+        guarded().push_back({context.device, target.dccAddress, MultisampledDccBytes(target), target.samples, active, metadataValid, hostOnly});
+        if (target.cmaskAddress != 0) guarded().push_back({context.device, target.cmaskAddress, target.cmaskBytes, target.samples, active, metadataValid, hostOnly});
+        if (target.samples == 8u && target.fmaskAddress != 0u && target.cmaskAddress != 0u) guarded().push_back({context.device, target.fmaskAddress, DescribeSurface(fmaskResource(target)).guestBytes, target.samples, active, metadataValid, hostOnly});
     }
     anyGuarded().store(true, std::memory_order_release);
     InstallResidentImageReadGuard_nid_no_patch();
@@ -770,6 +773,13 @@ void discardRetiredTargets() {
     std::erase_if(targets(), [](const auto& target) { return !target->active->load(std::memory_order_acquire); });
 }
 
+bool holdHostOnly(const MultisampleTarget& target) {
+    std::unique_lock lock(guardedMutex());
+    if (!target.active->load(std::memory_order_acquire)) return false;
+    target.hostOnly->store(true, std::memory_order_release);
+    return true;
+}
+
 MultisampleTarget& acquire(const Context& context, const ColorTarget& requested) {
     discardRetiredTargets();
     for (const auto& target : targets()) {
@@ -780,14 +790,21 @@ MultisampleTarget& acquire(const Context& context, const ColorTarget& requested)
         if (recreated(target->target, use)) {
             {
                 std::unique_lock lock(guardedMutex());
+                if (!target->active->load(std::memory_order_acquire)) continue;
                 std::erase_if(guarded(), [&](const auto& range) { return range.active == target->active; });
                 anyGuarded().store(!guarded().empty(), std::memory_order_release);
             }
             target->Recreate(use);
-            guard(context, use, target->active, target->metadataValid);
+            target->hostOnly->store(true, std::memory_order_release);
+            guard(context, use, target->active, target->metadataValid, target->hostOnly);
             target->ApplyFastClear(use);
             return *target;
         }
+        {
+            std::unique_lock lock(guardedMutex());
+            if (!target->active->load(std::memory_order_acquire)) continue;
+            target->hostOnly->store(true, std::memory_order_release);
+        }
         if (target->fromTexture && use.cmaskAddress != 0) {
             target->fromTexture = false;
             target->cmask = MultisampledCmask(use.cmaskAddress, use.cmaskBytes);
@@ -803,14 +820,14 @@ MultisampleTarget& acquire(const Context& context, const ColorTarget& requested)
     target->Keep();
     targets().push_back(std::move(target));
     auto& created = *targets().back();
-    guard(context, use, created.active, created.metadataValid);
+    guard(context, use, created.active, created.metadataValid, created.hostOnly);
     if (use.cmaskFastClear && use.dccAddress == 0) created.cmask.SeedCode(CurrentUniformMetadataByte(use.cmaskAddress, static_cast<std::size_t>(use.cmaskBytes)));
     created.ApplyFastClear(use);
     created.Keep();
     return created;
 }
 
-MultisampleTarget& createFilled(const Context& context, const GuestTextureResource& resource, VkFormat format, const char* access) {
+MultisampleTarget& createFilled(const Context& context, const GuestTextureResource& resource, VkFormat format, const char* access, bool written) {
     const auto refuse = [&](const std::string& reason) {
         char text[192];
         std::snprintf(text, sizeof(text), "AGC graphics: %s the %ux%u multisampled texture 0x%llx of %u samples, which no draw rendered to and ", access, resource.width, resource.height, static_cast<unsigned long long>(resource.baseAddress), resource.samples);
@@ -877,7 +894,8 @@ MultisampleTarget& createFilled(const Context& context, const GuestTextureResour
     target->Keep();
     targets().push_back(std::move(target));
     auto& created = *targets().back();
-    guard(context, use, created.active, created.metadataValid);
+    created.hostOnly->store(written, std::memory_order_release);
+    guard(context, use, created.active, created.metadataValid, created.hostOnly);
     created.colorFill.NoteFill(use.address, static_cast<std::size_t>(bytes), value);
     created.fromTexture = true;
     created.htileAddress = resource.dccAddress;
@@ -1177,7 +1195,7 @@ std::shared_ptr<Texture> MultisampleTargetTexture(const Context& context, const
     }
     if (other == nullptr) {
         Require(Recorder::Active() != nullptr, "a multisampled texture is only sampled by recorded draws and dispatches");
-        return createFilled(context, resource, format, "sampling").Sampled(components, viewType);
+        return createFilled(context, resource, format, "sampling", false).Sampled(components, viewType);
     }
     char text[320];
     std::snprintf(text, sizeof(text), "AGC graphics: sampling multisampled color target 0x%llx (%ux%u, %u samples, vk format %d) as a %ux%u texture of %u samples and guest format %u (vk %d) is not implemented", static_cast<unsigned long long>(resource.baseAddress), other->target.extent.width, other->target.extent.height, other->target.samples, static_cast<int>(other->target.format), resource.width, resource.height, resource.samples, resource.format, static_cast<int>(format));
@@ -1200,7 +1218,7 @@ std::shared_ptr<Texture> MultisampleTargetFirstSample(const Context& context, co
     }
     if (other == nullptr) {
         Require(Recorder::Active() != nullptr, "a multisampled texture is only sampled by recorded draws and dispatches");
-        return createFilled(context, resource, format, "sampling").FirstSample(components);
+        return createFilled(context, resource, format, "sampling", false).FirstSample(components);
     }
     char text[320];
     std::snprintf(text, sizeof(text), "AGC graphics: reading sample 0 of multisampled color target 0x%llx (%ux%u, %u samples, vk format %d) as a %ux%u texture of %u samples and guest format %u (vk %d) is not implemented", static_cast<unsigned long long>(resource.baseAddress), other->target.extent.width, other->target.extent.height, other->target.samples, static_cast<int>(other->target.format), resource.width, resource.height, resource.samples, resource.format, static_cast<int>(format));
@@ -1221,12 +1239,15 @@ VkImageView MultisampleTargetStorageView(const Context& context, const GuestText
         if (!target->active->load(std::memory_order_acquire)) continue;
         if (target->context.device != context.device || target->target.surfaceAddress != resource.baseAddress) continue;
         const auto& rendered = target->target;
-        if (sameStorageForm(context, rendered.format, format) && rendered.extent.width == resource.width && rendered.extent.height == resource.height && rendered.samples == resource.samples && rendered.mip == 0) return target->Storage(StorageFormatForGuest(context, resource.format));
+        if (sameStorageForm(context, rendered.format, format) && rendered.extent.width == resource.width && rendered.extent.height == resource.height && rendered.samples == resource.samples && rendered.mip == 0) {
+            if (!holdHostOnly(*target)) continue;
+            return target->Storage(StorageFormatForGuest(context, resource.format));
+        }
         other = target.get();
     }
     if (other == nullptr) {
         Require(Recorder::Active() != nullptr, "a multisampled color target is only written by recorded dispatches");
-        return createFilled(context, resource, format, "writing").Storage(StorageFormatForGuest(context, resource.format));
+        return createFilled(context, resource, format, "writing", true).Storage(StorageFormatForGuest(context, resource.format));
     }
     char text[320];
     std::snprintf(text, sizeof(text), "AGC graphics: storage image access to multisampled color target 0x%llx (%ux%u, %u samples, vk format %d) as a %ux%u texture of %u samples and guest format %u (vk %d) is not implemented", static_cast<unsigned long long>(resource.baseAddress), other->target.extent.width, other->target.extent.height, other->target.samples, static_cast<int>(other->target.format), resource.width, resource.height, resource.samples, resource.format, static_cast<int>(format));
@@ -1266,10 +1287,13 @@ void ResolveMultisampleTarget(const Context& context, const ColorTarget& source,
 bool MultisampledColorFillAbsorbs(std::uint64_t address, std::size_t bytes, std::uint32_t pattern) {
     if (bytes == 0) return false;
     std::lock_guard lock(targetsMutex());
-    return std::any_of(targets().begin(), targets().end(), [&](const auto& target) {
+    bool absorbed = false;
+    for (const auto& target : targets()) {
         VkClearColorValue clear{};
-        return target->active->load(std::memory_order_acquire) && target->target.address == address && MultisampledColorBytes(target->target) == bytes && MultisampledFillClearColor(target->target, pattern, clear);
-    });
+        if (!target->active->load(std::memory_order_acquire) || target->target.address != address || MultisampledColorBytes(target->target) != bytes || !MultisampledFillClearColor(target->target, pattern, clear)) continue;
+        absorbed = holdHostOnly(*target) || absorbed;
+    }
+    return absorbed;
 }
 
 void NoteColorMetadataFill(std::uint64_t address, std::size_t bytes, std::uint32_t pattern) {
@@ -1327,7 +1351,7 @@ void GuardOneFragmentFmask(const Context& context, const ColorTarget& target) {
     const auto bytes = std::max<std::uint64_t>((bits / 8u + 0xffffu) & ~0xffffull, 0x10000u);
     std::unique_lock lock(guardedMutex());
     const bool known = std::ranges::any_of(guarded(), [&](const auto& range) { return range.fmask && range.device == context.device && range.address == target.fmaskAddress && range.bytes == bytes; });
-    if (!known) guarded().push_back({context.device, target.fmaskAddress, bytes, target.coverageSamples, std::make_shared<std::atomic<bool>>(true), nullptr, true});
+    if (!known) guarded().push_back({context.device, target.fmaskAddress, bytes, target.coverageSamples, std::make_shared<std::atomic<bool>>(true), nullptr, std::make_shared<std::atomic<bool>>(true), true});
     anyGuarded().store(true, std::memory_order_release);
     lock.unlock();
     InstallResidentImageReadGuard_nid_no_patch();
@@ -1340,9 +1364,8 @@ void RequireNoMultisampleTarget(std::uint64_t address, std::size_t bytes, const
 
 void RequireNoMultisampleColorTarget(std::uint64_t address, std::size_t bytes, const char* access) {
     if (bytes == 0 || !anyGuarded().load(std::memory_order_acquire)) return;
-    std::shared_lock lock(guardedMutex());
-    for (const auto& range : guarded()) {
-        if (address >= range.address + range.bytes || range.address >= address + bytes) continue;
+    const auto overlaps = [&](const GuardedRange& range) { return address < range.address + range.bytes && range.address < address + bytes; };
+    const auto refuse = [&](const GuardedRange& range) {
         char text[256];
         if (range.fmask) {
             std::snprintf(text, sizeof(text), "AGC graphics: %s of 0x%llx+0x%zx overlaps the FMASK 0x%llx+0x%llx of a one-fragment color target of %u samples drawn on the host, which does not write it: guest access to it is not implemented", access, static_cast<unsigned long long>(address), bytes, static_cast<unsigned long long>(range.address), static_cast<unsigned long long>(range.bytes), range.samples);
@@ -1350,7 +1373,27 @@ void RequireNoMultisampleColorTarget(std::uint64_t address, std::size_t bytes, c
         }
         std::snprintf(text, sizeof(text), "AGC graphics: %s of 0x%llx+0x%zx overlaps multisampled color target 0x%llx+0x%llx (%u samples), whose samples are only on the host: guest access to it is not implemented", access, static_cast<unsigned long long>(address), bytes, static_cast<unsigned long long>(range.address), static_cast<unsigned long long>(range.bytes), range.samples);
         throw std::runtime_error(text);
+    };
+    {
+        std::shared_lock lock(guardedMutex());
+        bool retirable = false;
+        for (const auto& range : guarded()) {
+            if (!overlaps(range)) continue;
+            if (range.hostOnly->load(std::memory_order_acquire)) refuse(range);
+            retirable = true;
+        }
+        if (!retirable) return;
     }
+    std::unique_lock lock(guardedMutex());
+    std::vector<std::shared_ptr<std::atomic<bool>>> retired;
+    for (const auto& range : guarded()) {
+        if (!overlaps(range)) continue;
+        if (range.hostOnly->load(std::memory_order_acquire)) refuse(range);
+        retired.push_back(range.active);
+    }
+    for (const auto& active : retired) active->store(false, std::memory_order_release);
+    std::erase_if(guarded(), [&](const auto& range) { return std::find(retired.begin(), retired.end(), range.active) != retired.end(); });
+    anyGuarded().store(!guarded().empty(), std::memory_order_release);
 }
 
 }

core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp

@@ -321,7 +321,7 @@ void appendKey(std::vector<std::byte>& key, const TValue& value) {
 
 bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& state, const VertexInputLayout& input, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders) {
     using Stage = ShaderRecompiler::ShaderStage;
-    if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || state.depthBiasPerFace || state.rectList || BarycentricPatches(state, shaders) || SplitsAlternateAlpha(state) || std::ranges::any_of(shaders, [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->functionLdsBuffer; }) || FragmentInterlock(shaders)) return false;
+    if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || (state.stages.tessellation && !context.tessellationPipelineCache) || state.depthBiasPerFace || state.rectList || BarycentricPatches(state, shaders) || SplitsAlternateAlpha(state) || std::ranges::any_of(shaders, [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->functionLdsBuffer; }) || FragmentInterlock(shaders)) return false;
     keys = {};
     for (const auto& shader : shaders) {
         const auto identity = shader.program->Identity();
@@ -723,20 +723,21 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
             prebuilt = true;
             return;
         }
+        const VkPipelineCache pipelineCache = state.stages.tessellation && !context.tessellationPipelineCache ? VK_NULL_HANDLE : context.pipelineCache;
         if (libraries) pipeline = LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, &optimized, false, _modules);
         else {
             auto pipelineCacheLock = context.LockPipelineCacheForUse();
-            Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines");
+            Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines");
         }
         if (alternateAlpha) {
             blend.pAttachments = secondChannels.data();
             auto pipelineCacheLock = context.LockPipelineCacheForUse();
-            Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &secondChannel), "vkCreateGraphicsPipelines");
+            Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, pipelineCache, 1, &pipelineInfo, nullptr, &secondChannel), "vkCreateGraphicsPipelines");
         }
         if (state.depthBiasPerFace) {
             raster.cullMode = VK_CULL_MODE_FRONT_BIT;
             auto pipelineCacheLock = context.LockPipelineCacheForUse();
-            Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &backFaces), "vkCreateGraphicsPipelines");
+            Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, pipelineCache, 1, &pipelineInfo, nullptr, &backFaces), "vkCreateGraphicsPipelines");
         }
         for (const auto module : _modules) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, module, nullptr);
         _modules.clear();
@@ -1126,6 +1127,14 @@ struct PipelineStore {
     std::uint64_t evicted = 0;
     PipelineStoreCounters totals;
     std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now();
+    std::size_t capacity = 0;
+    bool fixedCapacity = false;
+    std::list<std::uint64_t> evictedOrder;
+    std::unordered_map<std::uint64_t, std::list<std::uint64_t>::iterator> evictedKeys;
+    std::size_t evictionsSinceCheck = 0;
+    std::size_t returnsSinceCheck = 0;
+    static constexpr std::size_t BaseEntries = 1024;
+    static constexpr std::size_t MaximumEntries = 4096;
 };
 
 // Never destroyed: the pipelines belong to a device that may already be gone when statics die, and
@@ -1154,9 +1163,41 @@ std::list<PipelineStore::Entry>::iterator abandon(PipelineStore& store, std::lis
     return store.entries.erase(it);
 }
 
-std::size_t storeBound() {
-    static const auto bound = static_cast<std::size_t>(std::max<std::uint64_t>(1u, PipelineSwitch("APS5_PIPELINE_STORE_ENTRIES", 1024u)));
-    return bound;
+std::size_t storeBound(PipelineStore& store) {
+    if (store.capacity == 0) {
+        const auto set = PipelineSwitch("APS5_PIPELINE_STORE_ENTRIES", 0u);
+        store.fixedCapacity = set != 0;
+        store.capacity = store.fixedCapacity ? static_cast<std::size_t>(set) : PipelineStore::BaseEntries;
+    }
+    return store.capacity;
+}
+
+void noteStoreEviction(PipelineStore& store, std::uint64_t hash) {
+    if (store.fixedCapacity) return;
+    if (const auto found = store.evictedKeys.find(hash); found != store.evictedKeys.end()) store.evictedOrder.erase(found->second);
+    store.evictedOrder.push_front(hash);
+    store.evictedKeys[hash] = store.evictedOrder.begin();
+    while (store.evictedOrder.size() > store.capacity) {
+        store.evictedKeys.erase(store.evictedOrder.back());
+        store.evictedOrder.pop_back();
+    }
+    if (++store.evictionsSinceCheck < store.capacity / 32u) return;
+    const bool churning = store.returnsSinceCheck * 2u > store.evictionsSinceCheck;
+    store.evictionsSinceCheck = 0;
+    store.returnsSinceCheck = 0;
+    if (!churning || store.capacity >= PipelineStore::MaximumEntries) return;
+    const auto previous = store.capacity;
+    store.capacity = std::min(store.capacity * 2u, PipelineStore::MaximumEntries);
+    std::fprintf(stderr, "[pipecache] bound %zu -> %zu pipelines: pipelines it evicted keep coming back\n", previous, store.capacity);
+}
+
+void noteStoreReturn(PipelineStore& store, std::uint64_t hash) {
+    if (store.fixedCapacity) return;
+    const auto found = store.evictedKeys.find(hash);
+    if (found == store.evictedKeys.end()) return;
+    store.evictedOrder.erase(found->second);
+    store.evictedKeys.erase(found);
+    ++store.returnsSinceCheck;
 }
 
 void reportPipelines(PipelineStore& store) {
@@ -1228,12 +1269,12 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
         }
         ++store.misses;
         ++store.totals.misses;
+        noteStoreReturn(store, hash);
     }
     // Built outside the mutex: a draw that finds its pipeline resident is not held behind another
     // thread's build (the journal replay builds here too), and a device's driver compiles in
     // parallel when several threads build at once.
     auto pipeline = std::make_shared<Pipeline>(context, state, vertexInput, resources, shaders, attachmentLayout);
-    const auto bound = storeBound();
     const bool replaying = PipelineJournalReplaying();
     {
         std::lock_guard lock(store.mutex);
@@ -1251,7 +1292,10 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
         }
         // A replayed pipeline enters at the least recently used end and never displaces a draw's:
         // the title's own pipelines evict it first, and a full store takes none.
-        if (replaying && store.entries.size() >= bound) return pipeline;
+        if (replaying && store.entries.size() >= storeBound(store)) {
+            if (store.fixedCapacity || store.capacity >= PipelineStore::MaximumEntries) return pipeline;
+            store.capacity = std::min(store.capacity * 2u, PipelineStore::MaximumEntries);
+        }
         if (replaying) {
             store.entries.push_front({context.device, context.bufferPool, context.bufferPool != nullptr, hash, std::vector<std::byte>(key), pipeline});
             store.index[hash] = store.entries.begin();
@@ -1259,12 +1303,14 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
             store.entries.push_back({context.device, context.bufferPool, context.bufferPool != nullptr, hash, std::vector<std::byte>(key), pipeline});
             store.index[hash] = std::prev(store.entries.end());
         }
-        while (store.entries.size() > bound) {
+        while (store.entries.size() > storeBound(store)) {
             // Only an entry no recorded draw still holds may go (Kept keeps its shared_ptr until the fence).
             const auto victim = std::find_if(store.entries.begin(), store.entries.end(), [](const PipelineStore::Entry& entry) { return entry.pipeline.use_count() == 1; });
             if (victim == store.entries.end()) break;
+            const auto evictedHash = victim->hash;
             store.index.erase(victim->hash);
             store.entries.erase(victim);
+            noteStoreEviction(store, evictedHash);
             ++store.evicted;
             ++store.totals.evicted;
         }
@@ -1283,10 +1329,16 @@ PipelineStoreCounters PipelineStoreTotals() {
     std::lock_guard lock(store.mutex);
     auto totals = store.totals;
     totals.cached = store.entries.size();
-    totals.bound = storeBound();
+    totals.bound = storeBound(store);
     return totals;
 }
 
+bool PipelineStoreCanGrow() {
+    auto& store = Pipelines();
+    std::lock_guard lock(store.mutex);
+    return storeBound(store) < PipelineStore::MaximumEntries && !store.fixedCapacity;
+}
+
 void ClearCachedPipelines(VkDevice device) {
     auto& store = Pipelines();
     std::lock_guard lock(store.mutex);
@@ -1298,6 +1350,13 @@ void ClearCachedPipelines(VkDevice device) {
         store.index.erase(it->hash);
         it = store.entries.erase(it);
     }
+    if (store.entries.empty() && !store.fixedCapacity) {
+        store.capacity = 0;
+        store.evictedOrder.clear();
+        store.evictedKeys.clear();
+        store.evictionsSinceCheck = 0;
+        store.returnsSinceCheck = 0;
+    }
     ClearPipelineLibraries(device);
     ReleasePixelFunctionLds(device);
 }

core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp

@@ -38,7 +38,7 @@ static_assert(sizeof(VkStencilOpState) == 28, "VkStencilOpState changed: update
 static_assert(sizeof(VkVertexInputBindingDescription) == 12, "VkVertexInputBindingDescription changed: update the recipe codec");
 static_assert(sizeof(VkVertexInputAttributeDescription) == 16, "VkVertexInputAttributeDescription changed: update the recipe codec");
 #if defined(__linux__) && defined(__x86_64__) && defined(__GLIBCXX__)
-static_assert(sizeof(State) == 832, "State changed: update CanonicalPipelineState and the recipe codec");
+static_assert(sizeof(State) == 840, "State changed: update CanonicalPipelineState and the recipe codec");
 static_assert(sizeof(ShaderStages) == 112, "ShaderStages changed: update the recipe codec");
 static_assert(sizeof(ShaderRecompiler::MeshConfiguration) == 40, "MeshConfiguration changed: update the recipe codec");
 static_assert(sizeof(ShaderRecompiler::TessellationConfiguration) == 40, "TessellationConfiguration changed: update the recipe codec");
@@ -1346,10 +1346,13 @@ public:
 
 private:
     void run() {
+#ifdef _WIN32
+        if (!SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL)) std::fprintf(stderr, "[pipeline-journal] replay thread priority not lowered (error %lu)\n", GetLastError());
+#endif
         std::call_once(loadOnce, [&] {
             try {
                 auto journal = Journal::Instance().Load();
-                const auto limit = configured("APS5_PIPELINE_JOURNAL_ENTRIES", 1024u);
+                const auto limit = configured("APS5_PIPELINE_JOURNAL_ENTRIES", 4096u);
                 if (journal.entries.size() > limit) journal.entries.erase(journal.entries.begin(), journal.entries.end() - static_cast<std::ptrdiff_t>(limit));
                 loaded = std::move(journal);
             } catch (const std::exception& error) {
@@ -1435,7 +1438,7 @@ private:
             return;
         }
         const auto store = PipelineStoreTotals();
-        if (store.cached >= store.bound) {
+        if (store.cached >= store.bound && !PipelineStoreCanGrow()) {
             counters.skipped.fetch_add(1, std::memory_order_relaxed);
             return;
         }

core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp

@@ -3134,6 +3134,26 @@ std::uint64_t Recorder::PendingWriterSerial(std::uint64_t address, std::size_t b
     return 0;
 }
 
+std::uint64_t Recorder::PendingWriterSerial(std::uint64_t address, std::size_t bytes, std::span<const std::uint64_t> exceptNotes) const {
+    if (exceptNotes.empty()) return PendingWriterSerial(address, bytes);
+    if (bytes == 0) return 0;
+    const auto end = address + bytes;
+    const auto writes = [&](const Batch& batch) {
+        for (std::size_t index = 0; index < batch.writes.size(); ++index) {
+            const auto& [begin, finish] = batch.writes[index];
+            if (address >= finish || begin >= end) continue;
+            if (index < batch.writeNotes.size() && std::find(exceptNotes.begin(), exceptNotes.end(), batch.writeNotes[index]) != exceptNotes.end()) continue;
+            return true;
+        }
+        return readOverlap(batch, address, end, ReadKind::AddressBased).has_value();
+    };
+    if (open != nullptr && writes(*open)) return OpenSerial();
+    for (auto it = inFlight.rbegin(); it != inFlight.rend(); ++it) {
+        if (writes(**it)) return (*it)->serial;
+    }
+    return 0;
+}
+
 bool Recorder::signaled(const Batch& batch) const {
     return batch.submitted && fenceStatus(batch.fence) == VK_SUCCESS;
 }

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp

@@ -624,10 +624,7 @@ std::uint64_t storageBudget(const Context& context, std::uint64_t required = 0)
         cached.device = context.physical;
         cached.read = now;
         cached.budget = StorageTextureBudget(context.memory, &reported, SampledTextureMemory(), storageCacheBytes());
-        cached.ceiling = cached.budget;
-        if (const auto heap = largestDeviceLocalHeap(context.memory); heap && reported.heapBudget[*heap] != 0) {
-            cached.ceiling = std::max<std::uint64_t>(512ull << 20u, std::min<std::uint64_t>(TextureCacheBudget(context.memory), reported.heapBudget[*heap] / 4u));
-        }
+        cached.ceiling = StorageTextureCeiling(context.memory, &reported);
     };
     if (cached.device != context.physical || now - cached.read >= std::chrono::seconds(1)) read();
     const auto needed = std::min(required, cached.ceiling);
@@ -752,7 +749,14 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
         normalized.format = unorm16;
         return cachedTexture(context, words, normalized, components, guestBytes, depthCompare);
     }
-    if (const auto window = ViewedArrayWindow(resource); window.baseAddress != resource.baseAddress) {
+    const auto viewedWindow = [&] {
+        const auto window = ViewedArrayWindow(resource, false);
+        if (window.baseAddress != resource.baseAddress || resource.dccAddress == 0) return window;
+        if (ViewedArrayWindow(resource, true).baseAddress == resource.baseAddress) return window;
+        if (TextureClearKeys(resource, guestBytes != 0 ? guestBytes : DescribeSurface(resource).guestBytes) != DccKeys::Uncompressed) return window;
+        return ViewedArrayWindow(resource, true);
+    };
+    if (const auto window = viewedWindow(); window.baseAddress != resource.baseAddress) {
         static std::atomic<std::uint64_t> windows{0};
         const auto count = windows.fetch_add(1) + 1;
         if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] sampled array view 0x%llx (%ux%u format %u, dimension %d, slices %u..%u) served from its viewed slices at 0x%llx; %llu such views\n", static_cast<unsigned long long>(resource.baseAddress), resource.width, resource.height, resource.format, static_cast<int>(resource.dimension), resource.baseArray, resource.depthOrLastArray, static_cast<unsigned long long>(window.baseAddress), static_cast<unsigned long long>(count));
@@ -1283,14 +1287,19 @@ std::uint64_t TextureCacheBudget(const VkPhysicalDeviceMemoryProperties& memory)
     return std::max<std::uint64_t>(2048ull << 20u, heap ? memory.memoryHeaps[*heap].size / 4u : 0u);
 }
 
+std::uint64_t StorageTextureCeiling(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported) {
+    const auto heap = largestDeviceLocalHeap(memory);
+    if (reported == nullptr || !heap || reported->heapBudget[*heap] == 0) return TextureCacheBudget(memory);
+    return std::max<std::uint64_t>(512ull << 20u, std::min<std::uint64_t>(memory.memoryHeaps[*heap].size / 2u, reported->heapBudget[*heap] / 2u));
+}
+
 std::uint64_t StorageTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t sampledBytes, std::uint64_t storageBytes) {
     const auto heap = largestDeviceLocalHeap(memory);
     if (reported == nullptr || !heap || reported->heapBudget[*heap] == 0) return TextureCacheBudget(memory);
     const auto budget = reported->heapBudget[*heap];
     const auto usage = reported->heapUsage[*heap];
     const auto reserved = usage - std::min<std::uint64_t>(usage, sampledBytes + storageBytes) + sampledBytes + budget / 8u;
-    const auto cap = std::min<std::uint64_t>(memory.memoryHeaps[*heap].size / 2u, budget / 2u);
-    return std::max<std::uint64_t>(512ull << 20u, std::min<std::uint64_t>(cap, budget > reserved ? budget - reserved : 0u));
+    return std::min<std::uint64_t>(StorageTextureCeiling(memory, reported), std::max<std::uint64_t>(512ull << 20u, budget > reserved ? budget - reserved : 0u));
 }
 
 std::uint64_t SampledTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t textureBytes, std::uint64_t storageBytes) {
@@ -2445,7 +2454,7 @@ void ShaderResources::buildComplete() {
                         for (auto element = first; element < last; ++element) {
                             const auto index = binding.imageAllocations[element];
                             if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) images.push_back({samplers[index]->Handle(), VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED});
-                            else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) images.push_back({VK_NULL_HANDLE, storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL});
+                            else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) images.push_back({VK_NULL_HANDLE, storageAtomic64[index] ? storageTextures[index]->Atomic64View(storageMips[index], storageFirstLayer[index]) : storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL});
                             else images.push_back({VK_NULL_HANDLE, textureFirstLayer[index] ? textures[index]->FirstLayerView() : textures[index]->View(), textures[index]->Layout()});
                         }
                         if (recorded) runs.emplace_back(write.dstArrayElement, write.descriptorCount);
@@ -4384,7 +4393,8 @@ std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words
     Require(!written || !overlap(address, size, indexAddress, indexBytes), "writable shader buffer aliases the index buffer");
     written = written || allWritten;
     const std::uint64_t recordBytes = descriptor.Stride() != 0 ? descriptor.Stride() : extent;
-    const bool swept = read && dispatchThreads != 0 && recordBytes != 0 && dispatchThreads * recordBytes * 2u >= byteSize;
+    constexpr std::uint64_t WidestBufferStoreBytes = 16;
+    const bool swept = (read || (context.discreteGpu && descriptor.Stride() > WidestBufferStoreBytes)) && dispatchThreads != 0 && recordBytes != 0 && dispatchThreads * recordBytes * 2u >= byteSize;
     if (written) guestMemory.AddWritable(address, size, atomic, swept);
     else {
         guestMemory.AddReadable(address, size);
@@ -4492,12 +4502,11 @@ std::uint64_t ShaderResources::DataWordsHash(const CompiledShader& shader) {
 bool ShaderResources::DataWordsDiffer(const CompiledShader& shader) const {
     Require(shader.program != nullptr, "missing compiled shader");
     const auto& program = *shader.program;
-    if (program.bindings.size() != bindings.size()) return true;
-    for (std::size_t index = 0; index < program.bindings.size(); ++index) {
-        const auto& binding = program.bindings[index];
+    for (const auto& binding : program.bindings) {
         if (!DataRole(binding.role)) continue;
-        if (bindings[index].allocations.size() != 1) return true;
-        const auto& allocation = allocations[bindings[index].allocations.front()];
+        const auto held = std::ranges::find_if(bindings, [&](const Binding& candidate) { return candidate.layout.binding == binding.binding; });
+        if (held == bindings.end() || held->allocations.size() != 1) return true;
+        const auto& allocation = allocations[held->allocations.front()];
         if (allocation.dataWords.size() != binding.guestDescriptor.size() || !std::equal(allocation.dataWords.begin(), allocation.dataWords.end(), binding.guestDescriptor.begin())) return true;
     }
     return false;
@@ -4512,23 +4521,23 @@ bool ShaderResources::RefreshData(VkCommandBuffer commands, const CompiledShader
 bool ShaderResources::refreshData(VkCommandBuffer commands, const CompiledShader& shader, Recorder* recorder) {
     Require(shader.program != nullptr, "missing compiled shader");
     const auto& program = *shader.program;
-    Require(program.bindings.size() == bindings.size(), "template bindings disagree with the shader");
     bool recorded = false;
     // One [gputime] range of class TemplateDataRefresh around the updates, begun at the first one
     // (a refresh that finds every word equal records nothing and times nothing).
     auto timing = Recorder::NoTiming;
     std::uint64_t refreshedBytes = 0;
-    for (std::size_t index = 0; index < program.bindings.size(); ++index) {
-        const auto& binding = program.bindings[index];
+    for (const auto& binding : program.bindings) {
         if (!DataRole(binding.role)) continue;
-        Require(bindings[index].allocations.size() == 1, "data binding without its buffer");
-        auto& allocation = allocations[bindings[index].allocations.front()];
+        const auto held = std::ranges::find_if(bindings, [&](const Binding& candidate) { return candidate.layout.binding == binding.binding; });
+        Require(held != bindings.end(), "template bindings disagree with the shader");
+        Require(held->allocations.size() == 1, "data binding without its buffer");
+        auto& allocation = allocations[held->allocations.front()];
         const auto size = binding.guestDescriptor.size() * sizeof(std::uint32_t);
         Require(allocation.buffer != nullptr && !allocation.guest && allocation.size == size && size <= MaxRefreshBytes, "template data buffer cannot take the dispatch's words");
         if (allocation.dataWords.size() == binding.guestDescriptor.size() && std::equal(allocation.dataWords.begin(), allocation.dataWords.end(), binding.guestDescriptor.begin())) continue;
         if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) ValidateRuntimeResources(shader, binding.guestDescriptor);
         if (recorder != nullptr && timing == Recorder::NoTiming) timing = recorder->BeginGpuTiming(Recorder::CommandClass::TemplateDataRefresh);
-        writeDataWords(commands, bindings[index].allocations.front(), binding.guestDescriptor);
+        writeDataWords(commands, held->allocations.front(), binding.guestDescriptor);
         allocation.dataWords.assign(binding.guestDescriptor.begin(), binding.guestDescriptor.end());
         refreshedBytes += size;
         recorded = true;
@@ -5227,7 +5236,7 @@ void ShaderResources::resolveImageTables() {
                 storageFirstLayer.push_back(firstLayer);
                 storageWritten.push_back(true);
                 storageAtomic.push_back(element < binding.imageAtomic.size() && binding.imageAtomic[element]);
-                storageAtomic64.push_back(false);
+                storageAtomic64.push_back(element < binding.imageAtomic64.size() && binding.imageAtomic64[element]);
                 describedRanges.push_back({"storage", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast<int>(resource.tileMode), resource.dccAddress, true});
                 bindings[storageBinding].imageAllocations[absolute] = storageTextures.size() - 1u;
                 continue;

core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp

@@ -1912,7 +1912,7 @@ GuestTextureResource SurfaceForTarget(const ColorTarget& color) {
     surface.dstSelY = 5;
     surface.dstSelZ = 6;
     surface.dstSelW = 7;
-    surface.dccAddress = color.dccAddress;
+    surface.dccAddress = color.dccAddress != 0 ? color.dccAddress : color.chainDccAddress;
     surface.dccAlphaOnMsb = color.dccAlphaOnMsb;
     surface.dccPipeAligned = color.dccPipeAligned;
     return surface;
@@ -2081,9 +2081,10 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot, bool allo
         Require(color.fmaskAddress != 0, "a compressed one-fragment color target without an FMASK address is unsupported");
     }
     if ((info & 0x10000000u) != 0) {
+        const auto dccHigh = find(cx, 0x3a8 + slot);
+        const auto dccBase = ((dccHigh == cx.end() ? 0ull : static_cast<std::uint64_t>(dccHigh->second & 0xffu)) << 40u) | (static_cast<std::uint64_t>(read(cx, 0x325 + stride)) << 8u);
         if (maxMip == 0) {
-            const auto dccHigh = find(cx, 0x3a8 + slot);
-            color.dccAddress = ((dccHigh == cx.end() ? 0ull : static_cast<std::uint64_t>(dccHigh->second & 0xffu)) << 40u) | (static_cast<std::uint64_t>(read(cx, 0x325 + stride)) << 8u);
+            color.dccAddress = dccBase;
             color.dccAlphaOnMsb = DccAlphaOnMsb(color.format, swap);
             color.dccPipeAligned = ((attrib3 >> 30u) & 1u) != 0;
             if (color.samples > 1) {
@@ -2091,11 +2092,15 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot, bool allo
                 color.dccControl = read(cx, 0x31e + stride);
                 if (!allowMultisampleDcc) Require((color.dccControl & 0x3ff80u) == 0, "multisampled DCC color transforms or lossy modes are not implemented");
             }
+        } else if (!volume && color.arrayLastSlice == 0 && slice == 0) {
+            color.chainDccAddress = dccBase;
+            color.dccAlphaOnMsb = DccAlphaOnMsb(color.format, swap);
+            color.dccPipeAligned = ((attrib3 >> 30u) & 1u) != 0;
         } else {
             static bool reported = false;
             if (!reported) {
                 reported = true;
-                std::fprintf(stderr, "[gpu] DCC keys of mipmapped color targets are ignored\n");
+                std::fprintf(stderr, "[gpu] DCC keys of mipmapped array and 3D color targets are ignored\n");
             }
         }
     }

core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp

@@ -123,6 +123,7 @@ struct StorageTraffic {
     std::map<std::string, std::pair<std::uint64_t, std::uint64_t>> writeBacks;
     std::array<std::pair<std::uint64_t, std::uint64_t>, 4> uploads{};
     std::map<std::string, std::pair<std::uint64_t, std::uint64_t>> uploadReasons;
+    std::map<std::string, std::pair<std::uint64_t, std::uint64_t>> drawnStores;
     std::uint64_t directWriteBacks = 0;
     std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now();
 };
@@ -132,6 +133,14 @@ StorageTraffic& Traffic() {
     return traffic;
 }
 
+struct DrawnPending {
+    std::size_t images = 0;
+    std::uint64_t bytes = 0;
+    double oldestMs = 0;
+};
+
+DrawnPending drawnPending();
+
 void reportStorageTraffic(StorageTraffic& traffic) {
     const auto now = std::chrono::steady_clock::now();
     if (now - traffic.lastReport < std::chrono::seconds(10)) return;
@@ -154,12 +163,19 @@ void reportStorageTraffic(StorageTraffic& traffic) {
         std::snprintf(text, sizeof(text), " %llu/%.1f", static_cast<unsigned long long>(totals.first), totals.second / 1048576.0);
         line += " " + reason + text;
     }
+    std::string drawnLine;
+    for (const auto& [reason, totals] : traffic.drawnStores) {
+        std::snprintf(text, sizeof(text), " %llu/%.1f", static_cast<unsigned long long>(totals.first), totals.second / 1048576.0);
+        drawnLine += " " + reason + text;
+    }
+    const auto drawn = drawnPending();
     const auto take = [](std::atomic<std::uint64_t>& counter) { return static_cast<unsigned long long>(counter.exchange(0, std::memory_order_relaxed)); };
     const auto partialUploadCount = take(partialUploads), partialWriteBackCount = take(partialWriteBacks);
     const auto uploadMiB = take(partialUploadBytes) / 1048576.0, writeBackMiB = take(partialWriteBackBytes) / 1048576.0;
-    AgcDriver::ProfilePrint_nid_no_patch( "[storage] uploads by path (count/MiB, 10 s):%s; %llu write-backs GPU-direct, %llu stored nothing; block units: partial uploads %llu/%.1f, partial write-backs %llu/%.1f, %llu widened to every pending unit, units dropped %llu, superseded %llu, handed over at %llu clear-key refreshes (%llu units superseded), dropped unregistered %llu, kept over a key flip %llu%s, pretest skipped %llu, held for a queued draw read %llu, evicted stale %llu\n", line.c_str(), static_cast<unsigned long long>(traffic.directWriteBacks), take(emptyWriteBacks), partialUploadCount, uploadMiB, partialWriteBackCount, writeBackMiB, take(coalescedWriteBacks), take(unitsDropped), take(unitsSuperseded), take(clearHandOvers), take(handedOverSuperseded), take(unregisteredDropped), take(keyFlipKept), ShadowReport().c_str(), take(pretestSkipped), take(queuedReadHeld), take(staleEvicted));
+    AgcDriver::ProfilePrint_nid_no_patch( "[storage] uploads by path (count/MiB, 10 s):%s; %llu write-backs GPU-direct, %llu stored nothing; block units: partial uploads %llu/%.1f, partial write-backs %llu/%.1f, %llu widened to every pending unit, units dropped %llu, superseded %llu, handed over at %llu clear-key refreshes (%llu units superseded), dropped unregistered %llu, kept over a key flip %llu%s, pretest skipped %llu, held for a queued draw read %llu, evicted stale %llu; drawn linear targets: stores by reason (count/MiB):%s, pending %zu images / %.1f MiB, oldest %.1f ms\n", line.c_str(), static_cast<unsigned long long>(traffic.directWriteBacks), take(emptyWriteBacks), partialUploadCount, uploadMiB, partialWriteBackCount, writeBackMiB, take(coalescedWriteBacks), take(unitsDropped), take(unitsSuperseded), take(clearHandOvers), take(handedOverSuperseded), take(unregisteredDropped), take(keyFlipKept), ShadowReport().c_str(), take(pretestSkipped), take(queuedReadHeld), take(staleEvicted), drawnLine.c_str(), drawn.images, drawn.bytes / 1048576.0, drawn.oldestMs);
     traffic.writeBacks.clear();
     traffic.uploadReasons.clear();
+    traffic.drawnStores.clear();
     traffic.directWriteBacks = 0;
 }
 
@@ -175,7 +191,7 @@ void countStorageUpload(std::size_t path, std::uint64_t bytes) {
     reportStorageTraffic(traffic);
 }
 
-void countStorageWriteBack(std::uint64_t bytes, bool direct) {
+void countStorageWriteBack(std::uint64_t bytes, bool direct, bool drawn = false) {
     if (!LookupOutcomes::Profiled()) return;
     auto& traffic = Traffic();
     std::lock_guard lock(traffic.mutex);
@@ -183,6 +199,11 @@ void countStorageWriteBack(std::uint64_t bytes, bool direct) {
     ++totals.first;
     totals.second += bytes;
     if (direct) ++traffic.directWriteBacks;
+    if (drawn) {
+        auto& stores = traffic.drawnStores[flushReason != nullptr ? flushReason : "other"];
+        ++stores.first;
+        stores.second += bytes;
+    }
     reportStorageTraffic(traffic);
 }
 
@@ -1538,6 +1559,13 @@ struct PendingWrites {
     // Images taken out of `textures` by a FlushPending still storing them (see adjacentPendingUnchanged).
     std::vector<StorageTexture*> flushing;
     std::atomic<std::size_t> flushingCount{0};
+    struct Drawn {
+        StorageTexture* texture;
+        std::uint64_t bytes;
+        std::chrono::steady_clock::time_point since;
+    };
+    std::vector<Drawn> drawn;
+    std::atomic<std::size_t> drawnCount{0};
 };
 
 PendingWrites& Pending() {
@@ -1545,6 +1573,20 @@ PendingWrites& Pending() {
     return pending;
 }
 
+DrawnPending drawnPending() {
+    DrawnPending result;
+    auto& pending = Pending();
+    if (pending.drawnCount.load(std::memory_order_acquire) == 0) return result;
+    const auto now = std::chrono::steady_clock::now();
+    std::lock_guard lock(pending.mutex);
+    result.images = pending.drawn.size();
+    for (const auto& entry : pending.drawn) {
+        result.bytes += entry.bytes;
+        result.oldestMs = std::max(result.oldestMs, std::chrono::duration<double, std::milli>(now - entry.since).count());
+    }
+    return result;
+}
+
 // See StorageTexture::PendingSerial: bumped after every mutation of the registry above.
 std::atomic<std::uint64_t> pendingSerial{0};
 
@@ -3804,7 +3846,13 @@ void StorageTexture::markLayersPending(std::uint32_t first, std::uint32_t count)
         }
     }
     if (anyUnsaved && !GuestMemory::Watched(descriptor.baseAddress, static_cast<std::size_t>(guestBytes))) captureGuestBytes(&unsaved);
-    for (std::uint32_t layer = first; layer < first + count; ++layer) layerPending[layer] = true;
+    for (std::uint32_t layer = first; layer < first + count; ++layer) {
+        if (!layerPending[layer] && !completionKept.empty()) {
+            const auto end = layerBegin(layer) + layerBytes(layer);
+            for (auto at = layerBegin(layer) & ~std::uint64_t{65535}; at < end; at += 65536) completionKept[static_cast<std::size_t>(at / 65536 - descriptor.baseAddress / 65536)] = 0;
+        }
+        layerPending[layer] = true;
+    }
     // Results of this image now cover the alias's results borrowed into these units: this image
     // stores them, the alias no longer has to (its results, unchanged since the borrow, are in
     // this image's content). An alias written since keeps its own newer results pending.
@@ -3827,6 +3875,10 @@ void StorageTexture::markLayersPending(std::uint32_t first, std::uint32_t count)
     }
     auto& pending = Pending();
     std::lock_guard lock(pending.mutex);
+    if (!drawingTarget) {
+        pendingFromOthers = true;
+        if (drawnListed) drawnWhole = true;
+    }
     // The active recorder installs a hook that also waits for recorded work; without one (tests),
     // pending storage results alone are flushed.
     if (Recorder::Active() == nullptr) {
@@ -3851,6 +3903,7 @@ void StorageTexture::reconcilePending() {
     auto& pending = Pending();
     std::lock_guard lock(pending.mutex);
     const bool any = anyLayerPending();
+    if (!any) forgetDrawn();
     if (any == dirty) return;
     dirty = any;
     if (any) pending.textures.push_back(this);
@@ -3864,6 +3917,215 @@ bool StorageTexture::Dirty() const {
     return dirty;
 }
 
+constexpr std::size_t MaxDrawnRects = 16;
+
+bool StorageTexture::MarkDrawn(std::uint32_t slice, std::uint32_t count, VkRect2D scissor) {
+    struct Drawing {
+        bool& flag;
+        bool previous;
+        ~Drawing() { flag = previous; }
+    } drawing{drawingTarget, std::exchange(drawingTarget, true)};
+    MarkSliceDirty(slice, count);
+    const auto clip = [](std::int64_t value, std::uint32_t limit) { return static_cast<std::uint32_t>(std::clamp<std::int64_t>(value, 0, limit)); };
+    const auto left = clip(scissor.offset.x, descriptor.width);
+    const auto top = clip(scissor.offset.y, descriptor.height);
+    const auto right = clip(std::int64_t{scissor.offset.x} + scissor.extent.width, descriptor.width);
+    const auto bottom = clip(std::int64_t{scissor.offset.y} + scissor.extent.height, descriptor.height);
+    const bool rectangle = proxyImage == VK_NULL_HANDLE && descriptor.tileMode == TextureTileMode::kLinear && !blockUnits && trackedLayers == 1 && descriptor.mipCount == 1 && arrayLayers == 1 && geometry.imageLayers == 1 && geometry.imageDepth == 1 && !geometry.thick && descriptor.dccAddress == 0;
+    auto& pending = Pending();
+    std::lock_guard lock(pending.mutex);
+    if (!anyLayerPending()) return true;
+    if (!drawnListed) {
+        drawnListed = true;
+        drawnWhole = false;
+        drawnRects.clear();
+        pending.drawn.push_back({this, guestBytes, std::chrono::steady_clock::now()});
+        pending.drawnCount.store(pending.drawn.size(), std::memory_order_release);
+    }
+    if (!rectangle || pendingFromOthers) drawnWhole = true;
+    if (!drawnWhole && right > left && bottom > top) addDrawnRect({left, top, right, bottom});
+    return !drawnWhole && drawnRects.size() <= MaxDrawnRects;
+}
+
+void StorageTexture::addDrawnRect(DrawnRect rect) {
+    const auto inside = [](const DrawnRect& outer, const DrawnRect& inner) { return outer.left <= inner.left && outer.top <= inner.top && inner.right <= outer.right && inner.bottom <= outer.bottom; };
+    if (std::any_of(drawnRects.begin(), drawnRects.end(), [&](const DrawnRect& kept) { return inside(kept, rect); })) return;
+    for (auto kept = drawnRects.begin(); kept != drawnRects.end();) {
+        const bool columns = kept->left == rect.left && kept->right == rect.right && kept->top <= rect.bottom && rect.top <= kept->bottom;
+        const bool rows = kept->top == rect.top && kept->bottom == rect.bottom && kept->left <= rect.right && rect.left <= kept->right;
+        if (!columns && !rows && !inside(rect, *kept)) {
+            ++kept;
+            continue;
+        }
+        rect = {std::min(kept->left, rect.left), std::min(kept->top, rect.top), std::max(kept->right, rect.right), std::max(kept->bottom, rect.bottom)};
+        drawnRects.erase(kept);
+        kept = drawnRects.begin();
+    }
+    drawnRects.push_back(rect);
+}
+
+void StorageTexture::forgetDrawn() {
+    pendingFromOthers = false;
+    if (!drawnListed) return;
+    drawnListed = false;
+    drawnWhole = false;
+    drawnRects.clear();
+    auto& pending = Pending();
+    std::erase_if(pending.drawn, [this](const PendingWrites::Drawn& entry) { return entry.texture == this; });
+    pending.drawnCount.store(pending.drawn.size(), std::memory_order_release);
+}
+
+void StorageTexture::noteStore(std::uint64_t note) {
+    if (note == 0) return;
+    auto& pending = Pending();
+    std::lock_guard lock(pending.mutex);
+    storeNotes.push_back(note);
+    if (storeNotes.size() > 8) storeNotes.erase(storeNotes.begin());
+}
+
+StorageTexture::DrawnArea StorageTexture::drawnArea() const {
+    auto& pending = Pending();
+    std::lock_guard lock(pending.mutex);
+    if (!drawnListed) return {};
+    DrawnArea area{true, !drawnWhole && !pendingFromOthers};
+    if (!area.rect || drawnRects.empty()) return area;
+    area.left = area.top = std::numeric_limits<std::uint32_t>::max();
+    for (const auto& rect : drawnRects) {
+        area.left = std::min(area.left, rect.left);
+        area.top = std::min(area.top, rect.top);
+        area.right = std::max(area.right, rect.right);
+        area.bottom = std::max(area.bottom, rect.bottom);
+    }
+    area.rects = drawnRects;
+    return area;
+}
+
+std::vector<std::pair<std::uint64_t, std::uint64_t>> StorageTexture::drawnSpans(const DrawnArea& area, std::span<const std::pair<std::uint64_t, std::uint64_t>> keep) const {
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> spans;
+    if (area.rects.empty()) return spans;
+    const auto& mip = mips.front();
+    const auto elementBytes = static_cast<std::uint64_t>(BytesPerElement(descriptor.format));
+    const auto rowBase = descriptor.baseAddress + mip.tiledOffset;
+    std::vector<std::pair<std::uint32_t, std::uint32_t>> columns;
+    std::size_t next = 0;
+    for (auto row = area.top; row < area.bottom; ++row) {
+        columns.clear();
+        for (const auto& rect : area.rects) {
+            if (rect.top <= row && row < rect.bottom) columns.emplace_back(rect.left, rect.right);
+        }
+        std::sort(columns.begin(), columns.end());
+        const auto rowStart = rowBase + std::uint64_t{row} * mip.pitchBytes;
+        for (std::size_t first = 0; first < columns.size();) {
+            auto right = columns[first].second;
+            auto last = first + 1;
+            for (; last < columns.size() && columns[last].first <= right; ++last) right = std::max(right, columns[last].second);
+            const auto rowBegin = rowStart + columns[first].first * elementBytes;
+            const auto rowEnd = rowStart + right * elementBytes;
+            first = last;
+            while (next < keep.size() && keep[next].second <= rowBegin) ++next;
+            for (auto index = next; index < keep.size() && keep[index].first < rowEnd; ++index) {
+                const auto from = std::max(rowBegin, keep[index].first);
+                const auto to = std::min(rowEnd, keep[index].second);
+                if (from < to) spans.emplace_back(from, to);
+            }
+        }
+    }
+    return spans;
+}
+
+std::uint64_t StorageTexture::storeDrawnRect(const DrawnArea& area, std::span<const ImportSegment> segments, std::span<const std::pair<std::uint64_t, std::uint64_t>> stores, const std::vector<std::uint32_t>& saveUnits) {
+    const auto& mip = mips.front();
+    const auto elementBytes = static_cast<std::uint64_t>(BytesPerElement(descriptor.format));
+    const auto rowBase = descriptor.baseAddress + mip.tiledOffset;
+    const auto width = area.right - area.left;
+    const auto height = area.bottom - area.top;
+    const auto rowBytes = std::uint64_t{width} * elementBytes;
+    const auto hullBegin = stores.front().first;
+    const auto hullBytes = static_cast<std::size_t>(stores.back().second - hullBegin);
+    std::uint64_t storedBytes = 0;
+    for (const auto& [from, to] : stores) storedBytes += to - from;
+    auto scratch = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(rowBytes * height), VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
+    auto* recorder = Recorder::Active();
+    std::unique_ptr<CommandBatch> batch;
+    VkCommandBuffer commands = VK_NULL_HANDLE;
+    auto timing = Recorder::NoTiming;
+    if (recorder != nullptr) {
+        recorder->FlushKeyStoresOverlapping(hullBegin, hullBytes);
+        recorder->FlushStoresOverlapping(hullBegin, hullBytes);
+        commands = recorder->Commands();
+        timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack);
+        recorder->Keep(scratch, scratch->Size());
+        if (auto self = weak_from_this().lock()) keepStorageTexture(*recorder, std::move(self));
+        Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 3);
+        if (Recorder::BarrierValidate()) {
+            const std::pair<VkImage, bool> read{image, false};
+            recorder->NoteAccess(Recorder::CommandClass::StorageWriteBack, Recorder::Access{{}, stores, std::span(&read, 1), VK_PIPELINE_STAGE_TRANSFER_BIT});
+        }
+    } else {
+        batch = std::make_unique<CommandBatch>(context);
+        commands = batch->Handle();
+    }
+    VkImageMemoryBarrier toSource{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
+    toSource.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
+    toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
+    toSource.oldLayout = VK_IMAGE_LAYOUT_GENERAL;
+    toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
+    toSource.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+    toSource.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+    toSource.image = image;
+    toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers};
+    const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT};
+    context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &importReady, 0, nullptr, 1, &toSource);
+    VkBufferImageCopy region{};
+    region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
+    region.imageOffset = {static_cast<std::int32_t>(area.left), static_cast<std::int32_t>(area.top), 0};
+    region.imageExtent = {width, height, 1u};
+    context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, scratch->Handle(), 1, &region);
+    const auto scratchRead = WholeBufferBarrier(scratch->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
+    context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &scratchRead, 0, nullptr);
+    const auto scratchOffset = [&](std::uint64_t address) {
+        const auto offset = address - rowBase;
+        const auto row = offset / mip.pitchBytes;
+        return (row - area.top) * rowBytes + offset - row * mip.pitchBytes - area.left * elementBytes;
+    };
+    const auto copyBuffer = context.Function<PFN_vkCmdCopyBuffer>("vkCmdCopyBuffer");
+    for (const auto& segment : segments) {
+        std::vector<VkBufferCopy> copies;
+        for (const auto& [from, to] : stores) {
+            const auto begin = std::max(from, segment.begin);
+            const auto end = std::min(to, segment.end);
+            if (begin < end) copies.push_back({scratchOffset(begin), HostImportOffset(*segment.import, begin), end - begin});
+        }
+        if (!copies.empty()) copyBuffer(commands, scratch->Handle(), segment.import->buffer, static_cast<std::uint32_t>(copies.size()), copies.data());
+    }
+    if (!saveUnits.empty()) {
+        const auto saved = savedBytes->Bytes();
+        for (const auto unit : saveUnits) {
+            const auto offset = static_cast<std::size_t>(layerBegin(unit) - descriptor.baseAddress);
+            std::memcpy(saved.data() + offset, original.data() + offset, static_cast<std::size_t>(layerBytes(unit)));
+        }
+        std::vector<VkBufferCopy> saves;
+        for (const auto& [from, to] : stores) saves.push_back({scratchOffset(from), from - descriptor.baseAddress, to - from});
+        copyBuffer(commands, scratch->Handle(), savedBytes->Handle(), static_cast<std::uint32_t>(saves.size()), saves.data());
+    }
+    VkImageMemoryBarrier backToGeneral = toSource;
+    backToGeneral.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
+    backToGeneral.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
+    backToGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
+    backToGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL;
+    const VkMemoryBarrier stored{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT};
+    context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, 1, &backToGeneral);
+    if (batch) {
+        batch->SubmitAndWait();
+        return storedBytes;
+    }
+    recorder->EndGpuTiming(timing, storedBytes);
+    recorder->MarkShaderReadsCovered();
+    recorder->NotePendingWrite(hullBegin, hullBytes);
+    noteStore(recorder->LastWriteNote(hullBegin, hullBytes));
+    return storedBytes;
+}
+
 void StorageTexture::Flush() {
     {
         auto& live = Live();
@@ -3914,13 +4176,38 @@ void waitForIdlePublish(const char* reason) {
 bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except, const char* reason, PublishScope scope, bool* published, bool waited) {
     static_cast<void>(FlushDepthPlanes(address, bytes));
     if (published != nullptr) *published = false;
+    const auto nothingToStore = [&] {
+        auto& pending = Pending();
+        if (pending.flushingCount.load(std::memory_order_acquire) != 0) return false;
+        if (!pending.textures.SurelyClear(address, bytes)) {
+            std::lock_guard lock(pending.mutex);
+            const auto* exempt = refreshing;
+            if (pending.textures.MayOverlap(address, bytes) && std::any_of(pending.textures.begin(), pending.textures.end(), [&](const StorageTexture* texture) { return texture != except && texture != exempt && texture->overlaps(address, bytes) && texture->pendingUnitInside(address, bytes); })) return false;
+            if (pending.flushingCount.load(std::memory_order_acquire) != 0) return false;
+        }
+        return scope == PublishScope::None || !AnyShadowedOverlaps(address, bytes);
+    };
+    if (nothingToStore()) return false;
     if (AgcDriver::DriverDetail::DrawPipeline::HeldReads(address, bytes)) {
         queuedReadHeld.fetch_add(1, std::memory_order_relaxed);
         return false;
     }
+    std::unique_lock<GuestMemory::GpuMutexType> drawnHold;
+    if (auto& drawnPending = Pending(); drawnPending.drawnCount.load(std::memory_order_acquire) != 0 && !GuestMemory::GpuMutex().HeldByThisThread()) {
+        bool overlap = false;
+        {
+            std::lock_guard lock(drawnPending.mutex);
+            overlap = std::any_of(drawnPending.drawn.begin(), drawnPending.drawn.end(), [&](const PendingWrites::Drawn& entry) { return entry.texture->dirty && entry.texture->overlaps(address, bytes); });
+        }
+        if (overlap) {
+            GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush);
+            drawnHold = std::unique_lock(GuestMemory::GpuMutex());
+        }
+    }
     // The images stay alive across the scan: their last owner may be a batch's kept list, which
     // another thread releases outside the GPU mutex once the batch completed.
     std::vector<std::shared_ptr<StorageTexture>> flush;
+    std::vector<std::uint64_t> ownNotes;
     {
         auto& pending = Pending();
         std::lock_guard lock(pending.mutex);
@@ -3948,8 +4235,12 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons
         for (const auto& texture : flush) pending.flushing.push_back(texture.get());
         pending.flushingCount.store(pending.flushing.size(), std::memory_order_release);
         if (!flush.empty()) BumpPendingSerial();
+        for (const auto& texture : flush) {
+            if (texture->drawnListed) ownNotes.insert(ownNotes.end(), texture->storeNotes.begin(), texture->storeNotes.end());
+        }
     }
     if (flush.empty()) {
+        if (drawnHold.owns_lock()) drawnHold.unlock();
         const bool stored = WaitForFlushing(address, bytes, except);
         // Units shadowed over the range (another image's results retiled into the import's unit
         // shadow, or an earlier flush's) still reach the import as the scope asks.
@@ -3973,10 +4264,11 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons
         for (const auto& texture : flush) std::fprintf(stderr, "[flush] image 0x%llx+0x%llx for %s 0x%llx+0x%zx\n", static_cast<unsigned long long>(texture->descriptor.baseAddress), static_cast<unsigned long long>(texture->guestBytes), reason, static_cast<unsigned long long>(address), bytes);
     }
     std::lock_guard gpu(GuestMemory::GpuMutex());
-    if (auto* recorder = Recorder::Active(); recorder != nullptr && !Recorder::InCompletion() && recorder->PendingWriterSerial(address, bytes) != 0) {
+    if (!ownNotes.empty() && (flush.size() != 1 || std::any_of(flush.begin(), flush.end(), [](const std::shared_ptr<StorageTexture>& texture) { return ExistingHostImportFor(texture->context, texture->descriptor.baseAddress, static_cast<std::size_t>(texture->guestBytes)) == nullptr; }))) ownNotes.clear();
+    if (auto* recorder = Recorder::Active(); recorder != nullptr && !Recorder::InCompletion() && recorder->PendingWriterSerial(address, bytes, ownNotes) != 0) {
         try {
             recorder->Reap();
-            if (const auto writer = recorder->PendingWriterSerial(address, bytes); writer == recorder->OpenSerial()) {
+            if (const auto writer = recorder->PendingWriterSerial(address, bytes, ownNotes); writer == recorder->OpenSerial()) {
                 Recorder::CountSync(recorder->OpenWriteOverlaps(address, bytes) ? 1 : 3);
                 recorder->Sync();
             } else if (writer != 0) {
@@ -4086,7 +4378,8 @@ void StorageTexture::saveBeforeUnwatch() {
         const auto end = begin + layerBytes(unit);
         const auto spanBegin = begin & ~(block - 1);
         const auto spanBlocks = static_cast<std::size_t>((end - spanBegin + block - 1) / block);
-        const std::vector<std::uint64_t> generations(spanBlocks, layerGeneration[unit]);
+        std::vector<std::uint64_t> generations(spanBlocks);
+        for (std::size_t k = 0; k < spanBlocks; ++k) generations[k] = unitBlockGeneration(unit, spanBegin + k * block);
         std::vector<std::uint8_t> changedBlocks(spanBlocks);
         if (!GuestMemory::ChangedBlocks(begin, static_cast<std::size_t>(end - begin), generations, changedBlocks)) continue;
         forgetKeptAtUnwatch(unit);
@@ -4094,7 +4387,7 @@ void StorageTexture::saveBeforeUnwatch() {
             const auto from = std::max(at, begin);
             const auto to = std::min(at + block, end);
             const bool edge = from != at || to != at + block;
-            if (changedBlocks[static_cast<std::size_t>((at - spanBegin) / block)] == GuestMemory::BlockWritten && (!edge || GuestMemory::StoredOver(from, static_cast<std::size_t>(to - from), layerGeneration[unit])) && !unchangedSinceBaseline(from, to)) keptAtUnwatch.emplace(from, layerGeneration[unit]);
+            if (changedBlocks[static_cast<std::size_t>((at - spanBegin) / block)] == GuestMemory::BlockWritten && (!edge || GuestMemory::StoredOver(from, static_cast<std::size_t>(to - from), unitBlockGeneration(unit, at))) && !unchangedSinceBaseline(from, to)) keptAtUnwatch.emplace(from, layerGeneration[unit]);
         }
         layerOriginal[unit] = true;
         cancelStoredSave(unit);
@@ -4258,6 +4551,49 @@ void StorageTexture::StoreReadbackBeforeLabel(const char* reason) {
     for (const auto& [address, bytes] : ranges) static_cast<void>(FlushPending(address, bytes, nullptr, reason));
 }
 
+std::size_t StorageTexture::StoreDrawnLinear(const char* reason) {
+    auto& pending = Pending();
+    if (pending.drawnCount.load(std::memory_order_acquire) == 0 || Recorder::InCompletion()) return 0;
+    GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush);
+    std::lock_guard gpu(GuestMemory::GpuMutex());
+    std::vector<std::pair<std::uint64_t, std::size_t>> ranges;
+    {
+        std::lock_guard lock(pending.mutex);
+        for (const auto& entry : pending.drawn) {
+            if (entry.texture->dirty) ranges.emplace_back(entry.texture->descriptor.baseAddress, static_cast<std::size_t>(entry.texture->guestBytes));
+        }
+    }
+    std::size_t stored = 0;
+    for (const auto& [address, bytes] : ranges) {
+        if (FlushPending(address, bytes, nullptr, reason)) ++stored;
+    }
+    return stored;
+}
+
+std::size_t StorageTexture::StoreDrawnLinearTouching(const char* reason, const std::function<bool(std::uint64_t, std::uint64_t)>& touches, bool all) {
+    auto& pending = Pending();
+    if (pending.drawnCount.load(std::memory_order_acquire) == 0 || Recorder::InCompletion()) return 0;
+    std::vector<std::pair<std::uint64_t, std::size_t>> ranges;
+    {
+        std::lock_guard lock(pending.mutex);
+        for (const auto& entry : pending.drawn) {
+            if (!entry.texture->dirty) continue;
+            const auto begin = entry.texture->descriptor.baseAddress;
+            const auto end = begin + entry.texture->guestBytes;
+            if (!all && !touches(begin, end)) continue;
+            ranges.emplace_back(begin, static_cast<std::size_t>(entry.texture->guestBytes));
+        }
+    }
+    if (ranges.empty()) return 0;
+    GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush);
+    std::lock_guard gpu(GuestMemory::GpuMutex());
+    std::size_t stored = 0;
+    for (const auto& [address, bytes] : ranges) {
+        if (FlushPending(address, bytes, nullptr, reason)) ++stored;
+    }
+    return stored;
+}
+
 bool StorageTexture::AnyPending() {
     auto& pending = Pending();
     std::lock_guard lock(pending.mutex);
@@ -4324,37 +4660,96 @@ bool PendingStorageOverlaps(std::uint64_t address, std::size_t bytes, const Stor
 }
 
 std::uint64_t StorageTexture::StampKeepingNewerResults(std::uint64_t address, std::size_t bytes, std::uint64_t recorded, const std::function<std::uint64_t()>& stamp) {
-    std::vector<std::pair<std::shared_ptr<StorageTexture>, std::uint32_t>> kept;
+    struct Candidate {
+        std::shared_ptr<StorageTexture> texture;
+        std::uint32_t unit;
+        bool drawn;
+    };
+    struct Kept {
+        std::shared_ptr<StorageTexture> texture;
+        std::uint32_t unit;
+        std::uint64_t from;
+        std::uint64_t to;
+        std::uint64_t since;
+        bool whole;
+        bool drawn;
+    };
+    std::vector<Candidate> candidates;
     if (recorded != 0 && bytes != 0) {
         auto& pending = Pending();
         std::lock_guard lock(pending.mutex);
-        for (auto* texture : pending.textures) {
-            if (!texture->blockUnits || !texture->overlaps(address, bytes)) continue;
+        const auto consider = [&](StorageTexture* texture) {
+            if ((!texture->blockUnits && !texture->drawnListed) || !texture->overlaps(address, bytes)) return;
             auto alive = texture->weak_from_this().lock();
-            if (alive == nullptr) continue;
+            if (alive == nullptr) return;
             for (std::uint32_t unit = 0; unit < texture->trackedLayers; ++unit) {
                 const auto begin = texture->layerBegin(unit);
                 const auto unitBytes = texture->layerBytes(unit);
                 if (!texture->layerPending[unit] || begin >= address + bytes || address >= begin + unitBytes) continue;
-                if (texture->layerGeneration[unit] >= recorded) kept.emplace_back(alive, unit);
+                if (texture->layerGeneration[unit] >= recorded) candidates.push_back({alive, unit, texture->drawnListed});
             }
+        };
+        for (auto* texture : pending.textures) consider(texture);
+        for (auto* texture : pending.flushing) consider(texture);
+    }
+    constexpr std::uint64_t block = 65536;
+    std::vector<Kept> kept;
+    for (const auto& [texture, unit, drawn] : candidates) {
+        const auto begin = texture->layerBegin(unit);
+        const auto end = begin + texture->layerBytes(unit);
+        GuestMemory::CollectWritesUncached(begin, static_cast<std::size_t>(end - begin));
+        if (GuestMemory::UnchangedSince(begin, static_cast<std::size_t>(end - begin), texture->layerGeneration[unit])) {
+            kept.push_back({texture, unit, begin, end, texture->layerGeneration[unit], true, drawn});
+            continue;
+        }
+        if (!drawn) continue;
+        const auto first = std::max(begin, address) / block;
+        const auto last = (std::min(end, address + bytes) - 1) / block;
+        const auto from = std::max(begin, first * block);
+        const auto to = std::min(end, (last + 1) * block);
+        std::vector<std::uint64_t> generations(static_cast<std::size_t>(last - first + 1));
+        for (std::size_t k = 0; k < generations.size(); ++k) generations[k] = texture->unitBlockGeneration(unit, (first + k) * block);
+        std::vector<std::uint8_t> changed(generations.size());
+        if (!GuestMemory::ChangedBlocks(from, static_cast<std::size_t>(to - from), generations, changed)) continue;
+        for (std::size_t k = 0; k < generations.size(); ++k) {
+            const auto at = (first + k) * block;
+            const auto blockFrom = std::max(at, begin);
+            const auto blockTo = std::min(at + block, end);
+            const bool edge = blockFrom != at || blockTo != at + block;
+            if (changed[k] == GuestMemory::BlockWritten && (!edge || !GuestMemory::UnchangedSince(blockFrom, static_cast<std::size_t>(blockTo - blockFrom), generations[k]))) continue;
+            if (!kept.empty() && !kept.back().whole && kept.back().texture == texture && kept.back().unit == unit && kept.back().since == generations[k] && kept.back().to == blockFrom) kept.back().to = blockTo;
+            else kept.push_back({texture, unit, blockFrom, blockTo, generations[k], false, true});
         }
     }
-    std::erase_if(kept, [](const auto& entry) {
-        const auto& [texture, unit] = entry;
-        const auto begin = texture->layerBegin(unit);
-        const auto unitBytes = static_cast<std::size_t>(texture->layerBytes(unit));
-        GuestMemory::CollectWritesUncached(begin, unitBytes);
-        return !GuestMemory::UnchangedSince(begin, unitBytes, texture->layerGeneration[unit]);
-    });
     const auto generation = stamp();
     if (generation == 0 || kept.empty()) return generation;
+    std::vector<Kept> confirmed;
+    for (const auto& entry : kept) {
+        if ((entry.whole && !entry.drawn) || GuestMemory::UnchangedSinceCollected(entry.from, static_cast<std::size_t>(entry.to - entry.from), entry.since)) {
+            confirmed.push_back(entry);
+            continue;
+        }
+        if (entry.from / block == (entry.to - 1) / block) continue;
+        const auto last = (std::min(entry.to, address + bytes) - 1) / block;
+        for (auto at = std::max(entry.from, address) / block * block; at <= last * block; at += block) {
+            const auto from = std::max(at, entry.from);
+            const auto to = std::min(at + block, entry.to);
+            if (GuestMemory::UnchangedSinceCollected(from, static_cast<std::size_t>(to - from), entry.since)) confirmed.push_back({entry.texture, entry.unit, from, to, entry.since, false, true});
+        }
+    }
     auto& pending = Pending();
     std::lock_guard lock(pending.mutex);
-    for (const auto& [texture, unit] : kept) {
-        if (!texture->layerPending[unit]) continue;
-        texture->layerGeneration[unit] = generation;
-        texture->refreshGeneration();
+    for (const auto& entry : confirmed) {
+        auto& texture = *entry.texture;
+        if (!texture.layerPending[entry.unit]) continue;
+        if (entry.whole) {
+            texture.layerGeneration[entry.unit] = generation;
+            texture.refreshGeneration();
+            continue;
+        }
+        const auto spanBegin = texture.descriptor.baseAddress / block;
+        if (texture.completionKept.empty()) texture.completionKept.assign(static_cast<std::size_t>((texture.descriptor.baseAddress + texture.guestBytes - 1) / block - spanBegin + 1), 0);
+        for (auto at = entry.from / block; at <= (entry.to - 1) / block; ++at) texture.completionKept[static_cast<std::size_t>(at - spanBegin)] = generation;
     }
     return generation;
 }
@@ -4470,6 +4865,7 @@ bool StorageTexture::AccessKeptByCpu(std::uint64_t address, std::size_t bytes, s
     // store there would see, and the stamps only grow.
     GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Hook);
     std::lock_guard gpu(GuestMemory::GpuMutex());
+    if (std::any_of(overlapping.begin(), overlapping.end(), [](const std::shared_ptr<StorageTexture>& texture) { return texture->drawnArea().drawn; })) return false;
     if (!blocksKept(overlapping, address, bytes)) return false;
     const auto site = static_cast<std::uint8_t>(GuestMemory::CurrentReadSite());
     std::size_t dropped = 0;
@@ -4601,6 +4997,7 @@ std::size_t StorageTexture::DiscardPendingInside(std::uint64_t address, std::siz
             texture->layerPending.assign(texture->trackedLayers, false);
             texture->layerOriginal.assign(texture->trackedLayers, false);
             texture->cancelStoredSaves();
+            texture->forgetDrawn();
             it = pending.textures.erase(it);
             ++discarded;
         } else if (texture != exempt && texture->blockUnits && texture->overlaps(address, bytes)) {
@@ -5210,6 +5607,12 @@ void StorageTexture::blockGenerations(std::vector<std::uint64_t>& generations) c
     }
 }
 
+std::uint64_t StorageTexture::unitBlockGeneration(std::uint32_t unit, std::uint64_t at) const {
+    const auto unitGeneration = layerGeneration[unit];
+    if (unitGeneration == 0 || !layerPending[unit] || completionKept.empty()) return unitGeneration;
+    return std::max(unitGeneration, completionKept[static_cast<std::size_t>(at / 65536 - descriptor.baseAddress / 65536)]);
+}
+
 namespace {
 
 std::atomic<bool>& ClearHandOverOn() {
@@ -5506,10 +5909,12 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
     for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) {
         if (!layers[layer]) continue;
         const auto end = layerBegin(layer) + layerBytes(layer);
-        for (auto at = layerBegin(layer) & ~(block - 1); at < end; at += block) generations[static_cast<std::size_t>((at - spanBegin) / block)] = layerGeneration[layer];
+        for (auto at = layerBegin(layer) & ~(block - 1); at < end; at += block) generations[static_cast<std::size_t>((at - spanBegin) / block)] = unitBlockGeneration(layer, at);
     }
+    const auto drawn = drawnArea();
     std::vector<std::uint8_t> changedBlocks(spanBlocks);
-    const bool tracked = GuestMemory::ChangedBlocks(firstStored, static_cast<std::size_t>(lastStored - firstStored), generations, changedBlocks);
+    std::vector<std::uint8_t> cpuBlocks(drawn.drawn ? spanBlocks : 0);
+    const bool tracked = GuestMemory::ChangedBlocks(firstStored, static_cast<std::size_t>(lastStored - firstStored), generations, changedBlocks, cpuBlocks);
     std::vector<bool> compared(trackedLayers, false);
     if (!tracked) {
         awaitStoredSave(&layers);
@@ -5534,7 +5939,8 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
         if (!layers[layer]) continue;
         const auto begin = layerBegin(layer);
         const auto end = begin + layerBytes(layer);
-        if (tracked ? layerGeneration[layer] == 0 : !compared[layer]) {
+        const bool unknown = tracked ? layerGeneration[layer] == 0 : !compared[layer];
+        if (unknown && !drawn.rect) {
             unsavedOutsideWatch = unsavedOutsideWatch || !tracked;
             keep.emplace_back(begin, end);
             continue;
@@ -5544,12 +5950,16 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
             const auto to = std::min(at + block, end);
             if (from >= to) continue;
             const bool edge = from != at || to != at + block;
-            if (keptAtUnwatchHolds(from, layer) || (changedBlocks[static_cast<std::size_t>((at - spanBegin) / block)] == GuestMemory::BlockWritten && (compared[layer] || !edge || GuestMemory::StoredOver(from, static_cast<std::size_t>(to - from), layerGeneration[layer])) && !unchangedSinceBaseline(from, to))) {
+            const auto index = static_cast<std::size_t>((at - spanBegin) / block);
+            const bool kept = keptAtUnwatchHolds(from, layer);
+            const bool written = kept || unknown || (changedBlocks[index] == GuestMemory::BlockWritten && (compared[layer] || !edge || GuestMemory::StoredOver(from, static_cast<std::size_t>(to - from), unitBlockGeneration(layer, at))) && !unchangedSinceBaseline(from, to));
+            if (written && (!drawn.rect || (tracked && !kept && !unknown && cpuBlocks[index] == 0))) {
                 skippedAny = true;
                 skippedLayer[layer] = true;
                 ++skipped;
                 continue;
             }
+            if (written && drawn.rect) skippedLayer[layer] = true;
             if (!keep.empty() && keep.back().second == from) keep.back().second = to;
             else keep.emplace_back(from, to);
         }
@@ -5563,10 +5973,15 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
         const auto first = keep.empty() ? descriptor.baseAddress + guestBytes : keep.front().first;
         std::fprintf(stderr, "[flush] image 0x%llx+0x%llx keeps %zu CPU-written 64 KiB blocks (first stored byte at +0x%llx, %zu ranges stored, generation %llu)\n", static_cast<unsigned long long>(descriptor.baseAddress), static_cast<unsigned long long>(guestBytes), skipped, static_cast<unsigned long long>(first - descriptor.baseAddress), keep.size(), static_cast<unsigned long long>(generation));
     }
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> spans;
+    if (drawn.rect) spans = drawnSpans(drawn, keep);
+    const auto& stores = drawn.rect ? spans : keep;
+    const auto stampBegin = drawn.rect && !stores.empty() ? stores.front().first : firstStored;
+    const auto stampEnd = drawn.rect && !stores.empty() ? stores.back().second : lastStored;
     // Taken before this write-back stamps anything (see advanceAdjacent), with the 64 KiB block span
     // its stamps cover.
-    const auto firstBlock = firstStored / block;
-    const auto lastBlock = (lastStored - 1) / block;
+    const auto firstBlock = stampBegin / block;
+    const auto lastBlock = (stampEnd - 1) / block;
     const auto adjacent = adjacentPendingUnchanged(firstBlock, lastBlock);
     // After the store: the stored layers are in sync at a fresh generation unless blocks were kept
     // for the CPU (then the layer is stale there, keeps its old generation and re-uploads at its
@@ -5603,7 +6018,7 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
         advanceAdjacent(adjacent, now, firstBlock, lastBlock);
         if (!shadowed.empty()) markShadowed(now);
     };
-    if (keep.empty()) {
+    if (stores.empty()) {
         // Every selected block was written by the CPU since the layer's generation: nothing to
         // store, the keys stay as they are, and the image is stale there (re-uploaded at its next
         // use, never from a matching `original`).
@@ -5633,6 +6048,24 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
             ++Profile().storageDirectWriteBacks;
             return;
         }
+        if (drawn.rect) {
+            PublishShadow(stampBegin, static_cast<std::size_t>(stampEnd - stampBegin), PublishScope::PartialUnits, PublishReason::Upload);
+            const auto storedBytes = storeDrawnRect(drawn, segments, stores, saveUnits);
+            countStorageWriteBack(storedBytes, true, true);
+            if (profile) Profile().storageGpu += timer.lap();
+            forgetStored();
+            uploadedKeys = DccKeys::Uncompressed;
+            std::vector<std::pair<std::uint64_t, std::uint64_t>> stamps;
+            for (const auto& [from, to] : stores) {
+                if (!stamps.empty() && from / block <= (stamps.back().second - 1) / block + 1) stamps.back().second = to;
+                else stamps.emplace_back(from, to);
+            }
+            for (const auto& [from, to] : stamps) GuestMemory::MarkWritten(from, static_cast<std::size_t>(to - from));
+            settle(true);
+            commitStoredSave(saveUnits);
+            ++Profile().storageDirectWriteBacks;
+            return;
+        }
         // The whole-layer copies below overwrite the interior units of the span; the edge units
         // keep a neighbour's shadowed results in the rest of their bytes, which reach the import
         // first (this write-back's stamps then make those units stale).
@@ -5735,8 +6168,9 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
             recorder->EndGpuTiming(timing, storedBytes);
             recorder->MarkShaderReadsCovered();
             recorder->NotePendingWrite(firstStored, static_cast<std::size_t>(lastStored - firstStored));
+            if (drawn.drawn) noteStore(recorder->LastWriteNote(firstStored, static_cast<std::size_t>(lastStored - firstStored)));
         }
-        countStorageWriteBack(storedBytes, true);
+        countStorageWriteBack(storedBytes, true, drawn.drawn);
         if (profile) Profile().storageGpu += timer.lap();
         // The guest bytes now differ from `original`; other caches of the range see the write.
         forgetStored();
@@ -5777,10 +6211,10 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
             overhang = covered >= end;
         }
     }
-    const bool registered = std::all_of(keep.begin(), keep.end(), [](const auto& range) { return RegisteredReadableCovers(range.first, static_cast<std::size_t>(range.second - range.first)); });
+    const bool registered = std::all_of(stores.begin(), stores.end(), [](const auto& range) { return RegisteredReadableCovers(range.first, static_cast<std::size_t>(range.second - range.first)); });
     if (unregisteredDrop && !overhang && !registered) {
         static std::atomic<int> reports{0};
-        if (reports.fetch_add(1) < 4) std::fprintf(stderr, "[gpu] storage image 0x%llx+0x%llx: memory no longer registered; %zu pending ranges dropped\n", static_cast<unsigned long long>(descriptor.baseAddress), static_cast<unsigned long long>(guestBytes), keep.size());
+        if (reports.fetch_add(1) < 4) std::fprintf(stderr, "[gpu] storage image 0x%llx+0x%llx: memory no longer registered; %zu pending ranges dropped\n", static_cast<unsigned long long>(descriptor.baseAddress), static_cast<unsigned long long>(guestBytes), stores.size());
         unregisteredDropped.fetch_add(1, std::memory_order_relaxed);
         forgetStored();
         for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) {
@@ -5797,6 +6231,10 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
     std::vector<bool> unsaved = layerOriginal;
     unsaved.flip();
     for (const auto& [begin, end] : unitRuns(unsaved)) {
+        if (drawn.rect) {
+            GuestMemory::ReadCommitted(descriptor.baseAddress + begin, std::span(original).subspan(static_cast<std::size_t>(begin), static_cast<std::size_t>(end - begin)));
+            continue;
+        }
         for (const auto& [from, to] : keep) {
             const auto first = std::max(descriptor.baseAddress + begin, from);
             const auto last = std::min(descriptor.baseAddress + end, to);
@@ -5858,7 +6296,9 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
     APS5_LOG_CHARS_OUT("StorageTexture writeback submit");
     batch.SubmitAndWait();
     APS5_LOG_CHARS_OUT("StorageTexture writeback done");
-    countStorageWriteBack(guestBytes, false);
+    std::uint64_t storedBytes = 0;
+    for (const auto& [from, to] : stores) storedBytes += to - from;
+    countStorageWriteBack(drawn.rect ? storedBytes : guestBytes, false, drawn.drawn);
     if (profile) Profile().storageGpu += timer.lap();
     if (dump) {
         char name[64];
@@ -5873,7 +6313,7 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
     // Store only the kept ranges' blocks that changed, so concurrent CPU writes to untouched texels
     // survive; `original` follows for the stored layers (the others' guest bytes are unchanged).
     const auto current = host.Bytes();
-    for (const auto& [from, to] : keep) {
+    for (const auto& [from, to] : stores) {
         const auto offset = static_cast<std::size_t>(from - descriptor.baseAddress);
         const auto length = static_cast<std::size_t>(to - from);
         GuestMemory::WriteChangedCommitted(from, current.subspan(offset, length), std::span<const std::byte>(original).subspan(offset, length));
@@ -5909,6 +6349,7 @@ StorageTexture::~StorageTexture() {
     {
         auto& pending = Pending();
         std::lock_guard lock(pending.mutex);
+        forgetDrawn();
         if (dirty) {
             dirty = false;
             pending.textures.remove(this);

core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp

@@ -466,23 +466,26 @@ bool LevelsFitAllocation(const GuestTextureResource& surface, std::uint32_t leve
     return true;
 }
 
-GuestTextureResource ViewedArrayWindow(const GuestTextureResource& viewed) {
+GuestTextureResource ViewedArrayWindow(const GuestTextureResource& viewed, bool keysReadAsStored) {
     const bool layered = viewed.dimension == TextureDimension::k2DArray || viewed.dimension == TextureDimension::k1DArray || viewed.dimension == TextureDimension::kCube;
-    if (!layered || viewed.baseArray == 0 || viewed.baseArray > viewed.depthOrLastArray || viewed.dccAddress != 0) return viewed;
+    if (!layered || viewed.baseArray == 0 || viewed.baseArray > viewed.depthOrLastArray || (viewed.dccAddress != 0 && !keysReadAsStored)) return viewed;
+    std::uint32_t sliceMask = 0;
     if (const auto xorMode = XorSwizzleMode(viewed.tileMode); xorMode != 0u) {
         const auto* equation = FindTextureSwizzleEquation(xorMode, BytesPerElement(viewed.format));
         if (equation == nullptr) return viewed;
         std::uint32_t sliceBits = 0;
         for (const auto bit : equation->bits) sliceBits |= bit >> 24u;
-        const auto sliceMask = sliceBits == 0 ? 0u : (1u << std::bit_width(sliceBits)) - 1u;
-        if ((viewed.baseArray & sliceMask) != 0) return viewed;
+        sliceMask = sliceBits == 0 ? 0u : (1u << std::bit_width(sliceBits)) - 1u;
     }
+    const auto first = viewed.baseArray & ~sliceMask;
+    if (first == 0) return viewed;
     const auto geometry = DescribeSurface(viewed);
     if (geometry.thick || geometry.layers != viewed.depthOrLastArray + 1u) return viewed;
     auto window = viewed;
-    window.baseAddress = viewed.baseAddress + geometry.GuestLayerOffset(viewed.baseArray);
-    window.depthOrLastArray = viewed.depthOrLastArray - viewed.baseArray;
-    window.baseArray = 0;
+    window.baseAddress = viewed.baseAddress + geometry.GuestLayerOffset(first);
+    window.depthOrLastArray = viewed.depthOrLastArray - first;
+    window.baseArray = viewed.baseArray - first;
+    window.dccAddress = 0;
     return window;
 }
 

core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp

@@ -3430,6 +3430,137 @@ void RunHeapMirrorRunUnwatchTests(const Context& context, const BdaTestAccess& a
     std::cout << "an address-based build refreshes the unwatched heap of a run in full and its watched neighbour by its writes\n";
 }
 
+void RunAddressCopyCapTests(const Context& context, const BdaTestAccess& access, bool remembered) {
+    constexpr std::size_t block = 65536;
+    constexpr std::size_t heapBytes = 32 * block;
+    constexpr std::size_t idleBytes = 16 * block;
+    constexpr std::size_t busyBytes = block;
+#ifdef _WIN32
+    if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) {
+        std::cout << "guest arena unavailable or not write-watched: the address copy cap not tested\n";
+        return;
+    }
+#else
+    if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) {
+        std::cout << "write watch unavailable: the address copy cap not tested\n";
+        return;
+    }
+#endif
+    struct Piece {
+        void* pointer = nullptr;
+        void* mapping = nullptr;
+        std::size_t allocated = 0;
+        std::size_t bytes = 0;
+    };
+    std::array<Piece, 4> pieces{};
+    const std::array<std::pair<std::size_t, std::size_t>, 4> sizes{{{heapBytes + block, heapBytes}, {heapBytes + block, idleBytes}, {busyBytes + block, busyBytes}, {busyBytes + block, busyBytes}}};
+    for (std::size_t index = 0; index < pieces.size(); ++index) {
+        auto& piece = pieces[index];
+        piece.allocated = sizes[index].first;
+        piece.bytes = sizes[index].second;
+#ifdef _WIN32
+        piece.pointer = GuestArena::GuestArenaAllocate_nid_postfix(piece.allocated, block);
+        Require(piece.pointer != nullptr, "cannot allocate the address copy cap ranges in the guest arena");
+        GuestArena::GuestArenaCommit_nid_postfix(piece.pointer, piece.allocated, PAGE_READWRITE, piece.allocated);
+        piece.mapping = piece.pointer;
+#else
+        piece.mapping = mmap(nullptr, piece.allocated + block, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        Require(piece.mapping != MAP_FAILED, "cannot map the address copy cap ranges");
+        piece.pointer = reinterpret_cast<void*>((reinterpret_cast<std::uintptr_t>(piece.mapping) + block - 1) & ~std::uintptr_t{block - 1});
+#endif
+        std::memset(piece.pointer, 0x5a, piece.allocated);
+    }
+    if (pieces[1].pointer < pieces[0].pointer) {
+        std::swap(pieces[0].pointer, pieces[1].pointer);
+        std::swap(pieces[0].mapping, pieces[1].mapping);
+    }
+    const auto heap = reinterpret_cast<std::uintptr_t>(pieces[0].pointer);
+    const auto idle = reinterpret_cast<std::uintptr_t>(pieces[1].pointer);
+    const auto busy = reinterpret_cast<std::uintptr_t>(pieces[2].pointer);
+    Require(heap < idle, "the idle range does not follow the heap in the registry");
+#ifndef _WIN32
+    GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(pieces[0].pointer, heapBytes);
+#endif
+    const auto registry = [&](bool add) {
+        auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
+        for (std::size_t index = 0; index < 3; ++index) {
+            if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, pieces[index].pointer, pieces[index].bytes, true, false, true);
+            else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, pieces[index].pointer);
+        }
+        GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
+    };
+    const auto unrelated = [&](bool add) {
+        auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
+        if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, pieces[3].pointer, pieces[3].bytes, true, false, false);
+        else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, pieces[3].pointer);
+        GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
+    };
+    registry(true);
+#ifdef _WIN32
+    AgcDriver::GuestMemory::Unwatch(idle, idleBytes);
+    AgcDriver::GuestMemory::Unwatch(busy, busyBytes);
+#endif
+    auto importing = context;
+    importing.hostImportAlignment = block;
+    SetHostImportBudget(heapBytes + idleBytes + busyBytes - block);
+    Require(HostImportFor(importing, idle, 64) != nullptr && HostImportFor(importing, busy, 64) != nullptr, "the idle and busy ranges were not imported");
+    Require(HostImportLiveBytes() == idleBytes + busyBytes && !HostImportCovers(importing, heap, heapBytes), "the imports before the build are not the idle and busy ranges");
+    Require(AgcDriver::GuestMemory::Watched(heap, heapBytes), "the heap is not write-watched");
+    const auto before = MirrorCounters();
+    const auto builds = [] { return HostImportCounters().requesters[static_cast<std::size_t>(ImportRequester::Build)]; };
+    const auto unwatch = [&] {
+        AgcDriver::GuestMemory::Unwatch(heap + heapBytes / 2, block);
+        Require(AgcDriver::GuestMemory::Watched(heap, heapBytes / 2) && !AgcDriver::GuestMemory::Watched(heap, heapBytes), "the heap is not partly outside the write watch");
+    };
+    if (remembered) {
+        unwatch();
+        SetAddressCopyLimit(2 * heapBytes);
+        {
+            GuestBufferMemory leased(importing);
+            leased.AcquireRegistered();
+            leased.Upload(true);
+            Require(MirrorCounters().heapMirrors == before.heapMirrors && !HostImportCovers(importing, heap, heapBytes), "the first build did not copy the heap whose import the budget refused");
+            leased.WriteBack();
+        }
+        SetAddressCopyLimit(0);
+        unrelated(true);
+    }
+    for (int use = 0; use < 300; ++use) Require(HostImportFor(importing, busy, 64) != nullptr, "the busy range lost its import");
+    if (!remembered) unwatch();
+    const auto refused = builds();
+    {
+        GuestBufferMemory leased(importing);
+        leased.AcquireRegistered();
+        leased.Upload(true);
+        const bool imported = HostImportCovers(importing, heap, heapBytes);
+        Require(imported || (!remembered && MirrorCounters().heapMirrors == before.heapMirrors + 1), "the heap is neither imported nor mirrored");
+        const auto ranges = leased.AddressRanges();
+        const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == heap; });
+        Require(found != ranges.end() && found->deviceAddress != 0u, "the heap is missing from the BDA table");
+        Require(imported || access.addressBytes(found->deviceAddress)[heapBytes / 2 + 3] == std::byte{0x5a}, "the heap mirror does not hold the guest bytes");
+        leased.WriteBack();
+    }
+    Require(!remembered || builds().remembered == refused.remembered + 1, "the build under the copy cap was not answered from the refusal the first build remembered");
+    SetHostImportBudget(0);
+    if (remembered) unrelated(false);
+    registry(false);
+    ClearImageMirrors(importing.device);
+    ClearHostImports(importing.device);
+    ClearSnapshotCopies(importing.device);
+#ifndef _WIN32
+    GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(pieces[0].pointer, heapBytes);
+#endif
+    for (const auto& piece : pieces) {
+#ifdef _WIN32
+        GuestArena::GuestArenaReset_nid_postfix(piece.pointer, piece.allocated);
+        GuestArena::GuestArenaRelease_nid_postfix(piece.pointer, piece.allocated);
+#else
+        munmap(piece.mapping, piece.allocated + block);
+#endif
+    }
+    std::cout << (remembered ? "an address-based build answered from a remembered budget refusal evicts instead of copying a partly unwatched heap past the cap\n" : "an address-based build serves a partly unwatched heap whose import is over the budget\n");
+}
+
 void hostWriteWatchExclusionTests() {
 #ifdef _WIN32
     std::vector<std::uint8_t> host(64, 0x31);
@@ -3443,6 +3574,200 @@ void hostWriteWatchExclusionTests() {
 #endif
 }
 
+void sparseJoinedHeapMirrorTests(const Context& context, const BdaTestAccess& access) {
+    constexpr std::size_t half = 1u << 20u;
+    constexpr std::size_t reserved = 2 * half;
+    constexpr std::size_t page = 65536;
+#ifdef _WIN32
+    if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) {
+        std::cout << "guest arena unavailable or not write-watched: sparse heap mirrors over adjacent ranges not tested\n";
+        return;
+    }
+    void* block = GuestArena::GuestArenaAllocate_nid_postfix(reserved, page);
+    auto* guest = static_cast<std::uint8_t*>(block);
+    GuestArena::GuestArenaCommit_nid_postfix(guest, page, PAGE_READWRITE, page);
+    GuestArena::GuestArenaCommit_nid_postfix(guest + half, page, PAGE_READWRITE, page);
+#else
+    if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) {
+        std::cout << "write watch unavailable: sparse heap mirrors over adjacent ranges not tested\n";
+        return;
+    }
+    void* mapping = mmap(nullptr, reserved + page, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    Require(mapping != MAP_FAILED, "cannot reserve the sparse heap over adjacent ranges");
+    auto* guest = reinterpret_cast<std::uint8_t*>((reinterpret_cast<std::uintptr_t>(mapping) + page - 1) & ~std::uintptr_t{page - 1});
+    Require(mprotect(guest, page, PROT_READ | PROT_WRITE) == 0 && mprotect(guest + half, page, PROT_READ | PROT_WRITE) == 0, "cannot commit the sparse heap pages");
+    GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(guest, reserved);
+    void* block = guest;
+#endif
+    auto* second = guest + half;
+    std::memset(guest, 0x5a, page);
+    std::memset(second, 0x6b, page);
+    const auto base = reinterpret_cast<std::uintptr_t>(block);
+    const auto registry = [&](bool add) {
+        auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
+        if (add) {
+            GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, guest, half, true, true, true);
+            GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, second, half, true, true, true);
+        } else {
+            GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, guest);
+            GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, second);
+        }
+        GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
+    };
+    const auto sweep = [&] {
+        GuestBufferMemory memory(context);
+        memory.AcquireRegistered();
+        memory.Upload(true);
+        memory.WriteBack();
+    };
+    sweep();
+    registry(true);
+    const auto bind = [&] {
+        GuestBufferMemory memory(context);
+        memory.AddWritable(base, reserved);
+        memory.Upload(false);
+        std::uint32_t adjustment = 0;
+        const auto view = memory.Descriptor(base, reserved, adjustment);
+        Require(adjustment == 0, "a sparse heap view over adjacent ranges is adjusted");
+        const auto bytes = access.bytes(view.buffer);
+        Require(view.range >= half + page && bytes[view.offset + page - 1] == std::byte{0x5a} && bytes[view.offset + page] == std::byte{} && bytes[view.offset + half] == std::byte{0x6b}, "a sparse heap over adjacent ranges is not bound to its committed extent");
+        bytes[view.offset + half + 8] = std::byte{0x77};
+        memory.WriteBack();
+        return view;
+    };
+    const auto before = MirrorCounters();
+    const auto first = bind();
+    const auto made = MirrorCounters();
+    Require(made.sparseBinds == before.sparseBinds + 1 && made.heapMirrors == before.heapMirrors + 1, "a sparse heap whose committed span crosses an adjacent range was not mirrored");
+    Require(second[8] == 0x77, "a store to the adjacent range of a mirrored sparse heap was not written back");
+    second[8] = 0x6b;
+    const auto again = bind();
+    Require(again.buffer == first.buffer && MirrorCounters().rebuilds == made.rebuilds && MirrorCounters().sparseBinds == made.sparseBinds + 1, "a sparse heap over unchanged adjacent ranges did not reuse its mirror");
+    registry(false);
+    sweep();
+    Require(MirrorCounters().heapMirrors == before.heapMirrors && MirrorCounters().heapBytes == before.heapBytes, "the sparse heap mirror over adjacent ranges outlived its ranges");
+#ifdef _WIN32
+    GuestArena::GuestArenaReset_nid_postfix(block, reserved);
+    GuestArena::GuestArenaRelease_nid_postfix(block, reserved);
+#else
+    GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(guest, reserved);
+    munmap(mapping, reserved + page);
+#endif
+}
+
+void joinedHeapMirrorPartialImportTests(const Context& context, const BdaTestAccess& access) {
+    constexpr std::size_t half = 1u << 20u;
+    constexpr std::size_t committed = 2 * half;
+    constexpr std::size_t reserved = 4 * half;
+    constexpr std::size_t page = 65536;
+#ifdef _WIN32
+    if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) {
+        std::cout << "guest arena unavailable or not write-watched: joined heap mirrors under a partial import not tested\n";
+        return;
+    }
+    void* block = GuestArena::GuestArenaAllocate_nid_postfix(reserved, half);
+    auto* guest = static_cast<std::uint8_t*>(block);
+    GuestArena::GuestArenaCommit_nid_postfix(guest, committed, PAGE_READWRITE, page);
+#else
+    if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) {
+        std::cout << "write watch unavailable: joined heap mirrors under a partial import not tested\n";
+        return;
+    }
+    void* mapping = mmap(nullptr, reserved + half, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    Require(mapping != MAP_FAILED, "cannot reserve the joined heap");
+    auto* guest = reinterpret_cast<std::uint8_t*>((reinterpret_cast<std::uintptr_t>(mapping) + half - 1) & ~std::uintptr_t{half - 1});
+    Require(mprotect(guest, committed, PROT_READ | PROT_WRITE) == 0, "cannot commit the joined heap ranges");
+    GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(guest, reserved);
+    void* block = guest;
+#endif
+    std::memset(guest, 0x3c, committed);
+    const auto base = reinterpret_cast<std::uintptr_t>(block);
+    const auto registry = [&](bool add) {
+        auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
+        if (add) {
+            GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, guest, half, true, true, true);
+            GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, guest + half, half, true, true, true);
+        } else {
+            GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, guest);
+            GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, guest + half);
+        }
+        GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
+    };
+    const auto sweep = [&](const Context& with) {
+        GuestBufferMemory memory(with);
+        memory.AcquireRegistered();
+        memory.Upload(true);
+        memory.WriteBack();
+    };
+    const auto bind = [&] {
+        GuestBufferMemory memory(context);
+        memory.AddWritable(base, reserved);
+        memory.Upload(false);
+        std::uint32_t adjustment = 0;
+        const auto view = memory.Descriptor(base, reserved, adjustment);
+        Require(adjustment == 0 && access.bytes(view.buffer)[view.offset + half + 32] == std::byte{0x3c}, "a heap bound across two committed ranges did not read its second range");
+        memory.WriteBack();
+        return view;
+    };
+    sweep(context);
+    registry(true);
+    const auto before = MirrorCounters();
+    const auto first = bind();
+    const auto made = MirrorCounters();
+    Require(made.heapMirrors == before.heapMirrors + 1 && made.sparseBinds == before.sparseBinds + 1, "a heap bound across two committed ranges was not mirrored over both");
+    auto importing = context;
+    importing.hostImportAlignment = page;
+    SetHostImportWindow(half);
+    SetHostImportRangeWindow(half);
+    SetHostImportBudget(HostImportLiveBytes() + half);
+    {
+        GuestBufferMemory memory(importing);
+        memory.AcquireRegistered();
+        Require(ExistingHostImportFor(importing, base, half) != nullptr && ExistingHostImportFor(importing, base + half, half) == nullptr, "the joined heap partial-import fixture did not import exactly its first range");
+        Require(HeapMirrorsWithin(base, base + half).mirrors == 1, "an address build that imported one range of a joined heap mirror released the mirror");
+        memory.Upload(true);
+        memory.WriteBack();
+    }
+    const auto partial = MirrorCounters();
+    const auto again = bind();
+    Require(again.buffer == first.buffer && MirrorCounters().rebuilds == partial.rebuilds, "a joined heap mirror was rebuilt after an import of one of its ranges");
+    {
+        GuestBufferMemory memory(importing);
+        memory.AcquireRegistered();
+        memory.AddWritable(base + half, 32);
+        memory.Upload(true);
+        if (ExistingHostImportFor(importing, base + half, 32) != nullptr) guest[half] = 0x71;
+        else {
+            std::uint32_t adjustment = 0;
+            const auto view = memory.Descriptor(base + half, 32, adjustment);
+            access.bytes(view.buffer)[view.offset + adjustment] = std::byte{0x71};
+        }
+        guest[half + 8] = 0x52;
+        memory.WriteBack();
+    }
+    Require(guest[half] == 0x71 && guest[half + 8] == 0x52, "a joined heap address build lost GPU or genuine CPU stores");
+    const auto refreshed = bind();
+    Require(refreshed.buffer == first.buffer && access.bytes(refreshed.buffer)[refreshed.offset + half] == std::byte{0x71} && access.bytes(refreshed.buffer)[refreshed.offset + half + 8] == std::byte{0x52}, "the retained joined mirror did not refresh stores from an address build");
+    SetHostImportBudget(HostImportLiveBytes() + committed);
+    GuestAllocations::GuestAllocationsEnd_nid_postfix(GuestAllocations::GuestAllocationsBegin_nid_postfix());
+    sweep(importing);
+    Require(ExistingHostImportFor(importing, base, half) != nullptr && ExistingHostImportFor(importing, base + half, half) != nullptr, "the joined heap full-import fixture did not import both ranges");
+    Require(HeapMirrorsWithin(base, base + committed).mirrors == 0, "imports covering every joined heap range left a heap mirror allocated");
+    SetHostImportBudget(0);
+    SetHostImportWindow(0);
+    SetHostImportRangeWindow(std::uint64_t{64} << 20u);
+    ClearHostImports(context.device);
+    registry(false);
+    sweep(context);
+#ifdef _WIN32
+    GuestArena::GuestArenaReset_nid_postfix(block, reserved);
+    GuestArena::GuestArenaRelease_nid_postfix(block, reserved);
+#else
+    GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(guest, reserved);
+    munmap(mapping, reserved + half);
+#endif
+}
+
 void sparseHeapMirrorTests(const Context& context, const BdaTestAccess& access) {
 #ifdef _WIN32
     if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) {
@@ -3851,8 +4176,15 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
     spaceCommittedPagesTests(context, access);
     hostWriteWatchExclusionTests();
     sparseHeapMirrorTests(context, access);
+    sparseJoinedHeapMirrorTests(context, access);
+    joinedHeapMirrorPartialImportTests(context, access);
     Require(AddressCopyOverflow({{0x1000, 0x3000, 0x2000, "uncommitted pages"}}, 0x2000).empty(), "copies within the limit were refused");
     const auto copies = AddressCopyOverflow({{0x1000, 0x2000, 0x1000, "not mirrored"}, {0x10000, 0x30000, 0x18000, "uncommitted pages"}}, 0x2000);
     Require(!copies.empty() && copies.find("0x10000+0x20000 (0.1 MiB committed, uncommitted pages)") < copies.find("0x1000+0x1000"), "the copy limit does not name the largest copy first");
+    AgcDriver::GuestMemory::Unwatch(0x10000, 0x20000);
+    AgcDriver::GuestMemory::Unwatch(0x40000, 0x10000);
+    Require(!AgcDriver::GuestMemory::Watched(0x10000, 0x20000), "the unregistered heap copy range is write-watched");
+    const auto heapCopies = AddressCopyOverflow({{0x10000, 0x30000, 0x18000, nullptr, true, true}, {0x40000, 0x50000, 0x10000, nullptr, true, false}}, 0x2000);
+    Require(heapCopies.find("0x10000+0x20000 (0.1 MiB committed, import over the host import budget, heap not fully write-watched)") != std::string::npos && heapCopies.find("0x40000+0x10000 (0.1 MiB committed, heap not fully write-watched)") != std::string::npos, "the copy limit does not name a copied heap's reason");
     ClearSnapshotCopies(context.device);
 }

core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp

@@ -21,6 +21,7 @@ void RunHeapMirrorUnwatchTests(const AgcDriver::Graphics::Context& context, cons
 void RunSparseHeapMirrorUnwatchTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access);
 void RunSparseHeapMirrorBudgetTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access);
 void RunHeapMirrorRunUnwatchTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access);
+void RunAddressCopyCapTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access, bool remembered);
 void RunGuestAllocationTests();
 void RunGuestLeaseWaitTests();
 void RunUnmappedGapTests();

core/libs/prx/libSceAgcDriver/tests/DenormalFlushElimination.cpp

@@ -110,7 +110,8 @@ std::vector<std::uint32_t> Recompiled(std::span<const std::uint32_t> code) {
     request.target.maxWorkgroupSharedMemoryBytes = 49152;
     request.layout = {0, 0, 0, 128};
     request.useCache = false;
-    const auto& words = Recompile(request).spirv.Words();
+    const auto result = Recompile(request);
+    const auto& words = result.spirv.Words();
     return {words.begin(), words.end()};
 }
 

core/libs/prx/libSceAgcDriver/tests/Flip.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PresentMode.hpp"
+#include "prx/libSceAgcDriver/Execution/include/SwapchainImages.hpp"
 #include "prx/libc/include/Shutdown.hpp"
 #include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp"
 #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
@@ -406,6 +407,19 @@ void testReset(bool compute) {
 
 void testPresentMode() {
     using AgcDriver::SelectPresentMode;
+    VkSurfaceCapabilitiesKHR surface{};
+    surface.minImageCount = 2;
+    surface.maxImageCount = 8;
+    check(AgcDriver::SwapchainImageCount(surface) == 3, "the swapchain has no image beyond the surface minimum");
+    surface.minImageCount = 3;
+    check(AgcDriver::SwapchainImageCount(surface) == 3, "a three-image surface minimum gained an image");
+    surface.minImageCount = 5;
+    check(AgcDriver::SwapchainImageCount(surface) == 5, "the swapchain does not follow a larger surface minimum");
+    surface.minImageCount = 2;
+    surface.maxImageCount = 2;
+    check(AgcDriver::SwapchainImageCount(surface) == 2, "the swapchain exceeds the surface maximum");
+    surface.maxImageCount = 0;
+    check(AgcDriver::SwapchainImageCount(surface) == 3, "an unlimited surface maximum capped the swapchain");
     const std::array wayland{VK_PRESENT_MODE_FIFO_KHR, VK_PRESENT_MODE_IMMEDIATE_KHR};
     const std::array all{VK_PRESENT_MODE_IMMEDIATE_KHR, VK_PRESENT_MODE_MAILBOX_KHR, VK_PRESENT_MODE_FIFO_KHR};
     check(SelectPresentMode(nullptr, wayland) == VK_PRESENT_MODE_FIFO_KHR, "the default present mode is not FIFO");
@@ -421,6 +435,11 @@ void testPresentMode() {
     check(expectFailure([&] { SelectPresentMode("", all); }).find("must be fifo, fifo_latest_ready, mailbox or immediate") != std::string::npos, "an empty present mode name was accepted");
     const std::array<VkPresentModeKHR, 0> none{};
     check(expectFailure([&] { SelectPresentMode(nullptr, none); }).find("does not support") != std::string::npos, "FIFO was selected on a surface without it");
+    using AgcDriver::SelectPresentQueue;
+    check(SelectPresentQueue(16, 2) == 2u, "the presentation queue does not follow the main and settle queues");
+    check(SelectPresentQueue(16, 1) == 1u, "the presentation queue does not follow the main queue alone");
+    check(!SelectPresentQueue(2, 2).has_value(), "a presentation queue was taken from a family without a free queue");
+    check(!SelectPresentQueue(1, 1).has_value(), "a presentation queue was taken from a family with one queue");
 }
 
 int main(int argc, char** argv) {

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -3134,6 +3134,13 @@ void metadataPassTests() {
     auto mipmapped = queue;
     mipmapped.context[0x3b0] |= 1u << 28u;
     expectFailure([&] { DecodeColorMetadataPass(mipmapped); }, "mipmapped DCC");
+    const auto chain = AgcDriver::Graphics::DecodeColorBuffer(mipmapped.context, 0);
+    const auto chainSurface = AgcDriver::Graphics::SurfaceForTarget(chain);
+    Require(chain.dccAddress == 0 && chainSurface.mipCount == 2 && chainSurface.dccAddress == keysAddress, "the surface of a mipmapped DCC color target lost its keys");
+    auto mipmappedArray = mipmapped;
+    mipmappedArray.context[0x3b8] |= 1u;
+    mipmappedArray.context[0x31b] = 1u << 13u;
+    Require(AgcDriver::Graphics::SurfaceForTarget(AgcDriver::Graphics::DecodeColorBuffer(mipmappedArray.context, 0)).dccAddress == 0, "a mipmapped DCC array color target took keys whose layers are not modeled");
     const AgcDriver::Graphics::Context context{};
     const auto texels = [&](std::uint32_t value) {
         for (std::size_t offset = 0; offset < colorMemory.size(); offset += 4) {
@@ -7092,6 +7099,27 @@ void storageTextureBudgetTests() {
     memory.memoryHeaps[0].flags = 0;
     memory.memoryHeaps[2].flags = 0;
     Require(StorageTextureBudget(memory, &reported, 0, 0) == 2 * GiB, "a device without a device-local heap does not keep 2 GiB of storage images");
+    using AgcDriver::Graphics::StorageTextureCeiling;
+    memory.memoryHeaps[0] = {256 * MiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT};
+    memory.memoryHeaps[2] = {16 * GiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT};
+    reported = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
+    Require(StorageTextureCeiling(memory, nullptr) == 4 * GiB, "without VK_EXT_memory_budget the storage image cache ceiling is not a quarter of the 16 GiB heap");
+    reported.heapBudget[2] = 6 * GiB;
+    Require(StorageTextureCeiling(memory, &reported) == 3 * GiB, "the storage image cache ceiling on a 6 GiB budget is not half of the budget, the cap the cache may reach");
+    reported.heapBudget[2] = 40 * GiB;
+    Require(StorageTextureCeiling(memory, &reported) == 8 * GiB, "the storage image cache ceiling on a 40 GiB budget is not half of the 16 GiB heap");
+    reported.heapBudget[2] = 768 * MiB;
+    Require(StorageTextureCeiling(memory, &reported) == 512 * MiB, "the storage image cache ceiling on a 768 MiB budget does not keep the 512 MiB floor");
+    for (std::uint64_t budget = 1 * GiB; budget <= 24 * GiB; budget += 512 * MiB) {
+        reported.heapBudget[2] = budget;
+        const auto ceiling = StorageTextureCeiling(memory, &reported);
+        for (std::uint64_t used = 0; used <= budget; used += 256 * MiB) {
+            reported.heapUsage[2] = used;
+            Require(StorageTextureBudget(memory, &reported, 256 * MiB, 512 * MiB) <= ceiling, "the storage image cache budget exceeds the ceiling idle device buffers are reclaimed up to");
+        }
+        reported.heapUsage[2] = 0;
+        Require(StorageTextureBudget(memory, &reported, 0, 0) == ceiling, "an idle heap does not give the storage image cache its full ceiling");
+    }
 }
 
 void sampledBudgetReportTests() {
@@ -7138,6 +7166,41 @@ void misalignedShaderDataTests() {
     Require(mock.live == 0, "misaligned shader data resources leaked Vulkan objects");
 }
 
+void imageTableTemplateRefreshTests() {
+    namespace Abi = ShaderRecompiler::ImageTableAbi;
+    mock = MockVulkan{};
+    auto context = mockContext();
+    context.imageTableCapacity = 2;
+    context.limits.maxDescriptorSetSampledImages = 2;
+    context.limits.maxPerStageDescriptorSampledImages = 2;
+    const auto commands = reinterpret_cast<VkCommandBuffer>(std::uintptr_t{1});
+    {
+        ShaderRecompiler::RecompileResult compute;
+        auto map = makeBinding(Role::ImageTableMap, static_cast<std::uint32_t>(ShaderRecompiler::RuntimeAbi::Binding::ImageTableMap), 1, std::vector<std::uint32_t>(Abi::HeaderWords));
+        map.guestDescriptor[Abi::HeaderVersion] = Abi::Version;
+        map.guestDescriptor[Abi::HeaderElementFaults] = Abi::HeaderWords;
+        map.guestDescriptor[Abi::HeaderSamplerFaults] = Abi::HeaderWords;
+        compute.bindings.push_back(map);
+        auto images = makeBinding(Role::ImageTable, ShaderRecompiler::RuntimeAbi::ImageTableBinding, 0, {});
+        images.kind = Kind::SampledImage;
+        compute.bindings.push_back(images);
+        compute.bindings.push_back(makeBinding(Role::ShaderData, 1, 1, ShaderDataWords({0x11, 0x22})));
+        compute.shaderDataDwords = ShaderRecompiler::RuntimeAbi::ShaderDataDwords;
+        auto live = compute;
+        live.bindings[2].guestDescriptor = ShaderDataWords({0x33, 0x44});
+        const AgcDriver::Graphics::CompiledShader shader{ShaderRecompiler::ShaderStage::Compute, &compute, 0};
+        const AgcDriver::Graphics::CompiledShader liveShader{ShaderRecompiler::ShaderStage::Compute, &live, 0};
+        AgcDriver::Graphics::ShaderResources resources(context, shader);
+        const auto data = findWrite(1).buffers.at(0).buffer;
+        Require(!resources.DataWordsDiffer(shader), "a template's own words differ from the shader that built it");
+        Require(resources.DataWordsDiffer(liveShader), "a template with an image table missed the dispatch's new words");
+        Require(resources.RefreshData(commands, liveShader), "a refresh of a template with an image table recorded nothing");
+        Require(sameBytes(bufferBytes(data), live.bindings[2].guestDescriptor.data(), 3u * sizeof(std::uint32_t)), "a refresh of a template with an image table did not write the dispatch's words");
+        Require(!resources.DataWordsDiffer(liveShader), "the refreshed template's words are not the dispatch's");
+    }
+    Require(mock.live == 0, "image table template refresh resources leaked Vulkan objects");
+}
+
 struct ModuleShape {
     bool fragment = false;
     std::optional<std::uint32_t> fragmentMode;
@@ -8076,6 +8139,38 @@ void pipelineLibraryClearWaitsForBuildTests() {
     Require(mock.live == 0, "clearing during a library build leaked Vulkan objects");
 }
 
+void tessellationPipelineLibraryCacheTests() {
+    using namespace AgcDriver::Graphics;
+    for (const auto shared : {true, false}) {
+        mock = MockVulkan{};
+        auto context = mockContext();
+        context.graphicsPipelineLibrary = true;
+        context.tessellationShader = true;
+        context.tessellationPipelineCache = shared;
+        context.limits.maxTessellationPatchSize = 32;
+        context.limits.maxPushConstantsSize = 256;
+        context.limits.framebufferNoAttachmentsSampleCounts = VK_SAMPLE_COUNT_1_BIT;
+        State state{};
+        state.stages.path = ShaderPath::Tessellation;
+        state.stages.tessellation = ShaderRecompiler::TessellationConfiguration{3, 3, 2, 1, 4, 128, 16, false};
+        ShaderRecompiler::RecompileResult vertex, control, evaluation, fragment;
+        vertex.spirv = makeModule({});
+        control.spirv = makeModule({});
+        evaluation.spirv = makeModule({});
+        fragment.spirv = makeModule({.fragment = true});
+        vertex.variantId = 201;
+        control.variantId = 202;
+        evaluation.variantId = 203;
+        fragment.variantId = 204;
+        const std::array<CompiledShader, 4> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::TessellationControl, &control, 0}, {ShaderRecompiler::ShaderStage::TessellationEvaluation, &evaluation, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, PipelinePushSlotBytes}}};
+        std::array<std::uint32_t, 1> mainKey{31};
+        const PipelineSetLayouts resources{makeHandle<VkDescriptorSetLayout>(), VK_NULL_HANDLE, false, mainKey, {}};
+        Require(Pipeline::Prebuild(context, state, VertexInputLayout{}, resources, shaders) == shared, shared ? "tessellation pipelines skipped graphics pipeline libraries with a shared pipeline cache" : "tessellation pipelines without the pipeline cache were built from graphics pipeline libraries");
+        ClearPipelineLibraries(context.device);
+        Require(mock.live == 0, "tessellation pipeline library tests leaked Vulkan objects");
+    }
+}
+
 void framebufferOwnerTests() {
     using namespace AgcDriver::Graphics;
     mock = MockVulkan{};
@@ -11378,6 +11473,117 @@ void pipelineStoreLazyBufferPoolTests() {
     Require(mock.live == 0, "lazy buffer pool pipeline tests leaked Vulkan objects");
 }
 
+// The pipeline store's bound without a GPU: 1536 configurations cycled through the default 1024
+// entries keep coming back after their eviction, so the bound doubles and a later pass finds every
+// pipeline resident; a bound set with APS5_PIPELINE_STORE_ENTRIES never moves.
+void pipelineStoreGrowthTests(bool fixed) {
+    using namespace AgcDriver::Graphics;
+    mock = MockVulkan{};
+    auto context = mockContext();
+    context.device = makeHandle<VkDevice>();
+    context.graphicsPipelineLibrary = true;
+    context.pipelineLinkTimeOptimization = false;
+    context.limits.framebufferNoAttachmentsSampleCounts = VK_SAMPLE_COUNT_1_BIT;
+    ShaderRecompiler::RecompileResult vertex, fragment;
+    vertex.spirv = makeModule({});
+    fragment.spirv = makeModule({.fragment = true});
+    vertex.variantId = 301;
+    fragment.variantId = 302;
+    const std::array<CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, PipelinePushSlotBytes}}};
+    std::array<std::uint32_t, 1> mainKey{41}, fragmentKey{42};
+    const PipelineSetLayouts resources{makeHandle<VkDescriptorSetLayout>(), makeHandle<VkDescriptorSetLayout>(), true, mainKey, fragmentKey};
+    constexpr std::uint32_t configurations = 1536;
+    const auto pass = [&](std::uint32_t count) {
+        std::uint32_t built = 0;
+        for (std::uint32_t configuration = 0; configuration < count; ++configuration) {
+            VertexInputLayout input;
+            input.bindings.push_back({0, 4u * (configuration + 1u), VK_VERTEX_INPUT_RATE_VERTEX});
+            bool created = false;
+            static_cast<void>(CachedPipeline(context, State{}, input, resources, shaders, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, &created));
+            built += created ? 1u : 0u;
+        }
+        return built;
+    };
+    std::uint32_t lastBuilt = 0;
+    for (int round = 0; round < 6; ++round) lastBuilt = pass(configurations);
+    const auto grown = PipelineStoreTotals();
+    if (fixed) {
+        Require(grown.bound == 1024, "a bound set with APS5_PIPELINE_STORE_ENTRIES changed");
+        Require(lastBuilt == configurations, "a fixed 1024-entry bound kept pipelines of a 1536-configuration cycle");
+    } else {
+        Require(grown.bound == 2048, "the bound did not grow to 2048 while evicted pipelines kept coming back");
+        Require(lastBuilt == 0, "a pass over 1536 configurations still rebuilt pipelines after the bound grew");
+    }
+    ClearCachedPipelines(context.device);
+    Require(mock.live == 0, "pipeline store growth tests leaked Vulkan objects");
+}
+
+// A journal longer than the store's starting bound: 1032 recorded pipelines replayed into an empty
+// store all stay resident (the replay grows the bound instead of dropping what does not fit).
+void pipelineStoreReplayGrowthTests() {
+    using namespace AgcDriver::Graphics;
+    const auto root = std::filesystem::temp_directory_path() / ("aps5-pipeline-store-replay-" + std::to_string(std::random_device{}()));
+    std::filesystem::create_directories(root);
+    struct Cleanup {
+        std::filesystem::path directory;
+        ~Cleanup() {
+            ShaderRecompiler::ShaderDiskCache::Shutdown();
+            std::error_code error;
+            std::filesystem::remove_all(directory, error);
+        }
+    } cleanup{root};
+#ifdef _WIN32
+    _putenv_s("ANYPS5_NO_SHADER_CACHE", "0");
+    _wputenv_s(L"ANYPS5_SHADER_CACHE_DIR", root.c_str());
+    _putenv_s("APS5_PIPELINE_JOURNAL_THREADS", "1");
+#else
+    setenv("ANYPS5_NO_SHADER_CACHE", "0", 1);
+    setenv("ANYPS5_SHADER_CACHE_DIR", root.string().c_str(), 1);
+    setenv("APS5_PIPELINE_JOURNAL_THREADS", "1", 1);
+#endif
+    Require(PipelineJournalEnabled(), "the journal is disabled with a shader cache directory set");
+    mock = MockVulkan{};
+    auto context = mockContext();
+    context.device = makeHandle<VkDevice>();
+    context.graphicsPipelineLibrary = true;
+    context.pipelineLinkTimeOptimization = false;
+    context.limits.framebufferNoAttachmentsSampleCounts = VK_SAMPLE_COUNT_1_BIT;
+    ShaderRecompiler::RecompileResult vertex, fragment;
+    vertex.spirv = makeModule({});
+    fragment.spirv = makeModule({.fragment = true});
+    vertex.variantId = 401;
+    fragment.variantId = 402;
+    const std::array<CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, PipelinePushSlotBytes}}};
+    auto cache = std::make_unique<DescriptorCache>(context);
+    context.descriptorCache = cache.get();
+    const auto layouts = DescribeSetLayouts(context, shaders);
+    const auto resources = layouts.View();
+    constexpr std::uint64_t fingerprint = 0x5eed'0000'0000'0002ull;
+    {
+        auto absent = StartPipelineJournalReplay(context, fingerprint, {});
+        Require(absent != nullptr, "the first replay did not start");
+        absent->Wait();
+    }
+    constexpr std::uint32_t configurations = 1032;
+    for (std::uint32_t configuration = 0; configuration < configurations; ++configuration) {
+        VertexInputLayout input;
+        input.bindings.push_back({0, 4u * (configuration + 1u), VK_VERTEX_INPUT_RATE_VERTEX});
+        static_cast<void>(CachedPipeline(context, State{}, input, resources, shaders));
+    }
+    FlushPipelineJournal();
+    ClearCachedPipelines(context.device);
+    Require(PipelineStoreTotals().cached == 0, "clearing the device left pipelines in the store");
+    auto replay = StartPipelineJournalReplay(context, fingerprint, {});
+    Require(replay != nullptr, "the replay did not start");
+    replay->Wait();
+    const auto totals = PipelineStoreTotals();
+    Require(totals.cached == configurations, "a replay of " + std::to_string(configurations) + " recorded pipelines kept " + std::to_string(totals.cached) + " in the store (bound " + std::to_string(totals.bound) + ")");
+    ClearCachedPipelines(context.device);
+    context.descriptorCache = nullptr;
+    cache.reset();
+    Require(mock.live == 0, "pipeline store replay growth tests leaked Vulkan objects");
+}
+
 void pipelineJournalTests(bool longPath = false, bool extended = false) {
     using namespace AgcDriver::Graphics;
     using Stage = ShaderRecompiler::ShaderStage;
@@ -11809,6 +12015,16 @@ int main(int argc, char** argv) {
             std::cout << "pipeline library clear waits for active part builds\n";
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--pipeline-store-replay-growth") {
+            pipelineStoreReplayGrowthTests();
+            std::cout << "a replayed journal longer than the store bound stays resident\n";
+            return 0;
+        }
+        if (argc == 2 && std::string_view(argv[1]) == "--pipeline-store-growth") {
+            pipelineStoreGrowthTests(std::getenv("APS5_PIPELINE_STORE_ENTRIES") != nullptr);
+            std::cout << "pipeline store bound follows the evicted pipelines that come back\n";
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--pipeline-store-lazy-buffer-pool") {
             pipelineStoreLazyBufferPoolTests();
             std::cout << "cached pipelines survive lazy buffer pool creation\n";
@@ -11917,6 +12133,32 @@ int main(int argc, char** argv) {
             Require(mock.live == 0, "the heap mirror run unwatch test leaked Vulkan objects");
             return 0;
         }
+        if (argc == 2 && (std::string_view(argv[1]) == "--address-copy-cap" || std::string_view(argv[1]) == "--address-copy-cap-remembered")) {
+#ifdef _WIN32
+            _putenv_s("APS5_ADDRESS_COPY_MAX_MIB", "1");
+#else
+            setenv("APS5_ADDRESS_COPY_MAX_MIB", "1", 1);
+#endif
+            {
+                mock = MockVulkan{};
+                auto context = mockContext();
+                context.bufferDeviceAddress = true;
+                context.limits.maxStorageBufferRange = 1u << 27;
+                RunAddressCopyCapTests(context, {
+                    [](VkBuffer buffer) -> std::span<std::byte> { return mockBufferMemory(buffer); },
+                    {},
+                    [](VkDeviceAddress address) {
+                        const auto offset = address - 0x100000000000ULL;
+                        const auto buffer = reinterpret_cast<VkBuffer>(offset / 0x10000);
+                        return mockBufferMemory(buffer).subspan(offset % 0x10000);
+                    },
+                    {},
+                    {}
+                }, std::string_view(argv[1]) == "--address-copy-cap-remembered");
+            }
+            Require(mock.live == 0, "the address copy cap test leaked Vulkan objects");
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--address-space-teardown") {
             addressSpaceTeardownTests();
             std::cout << "address-space device teardown releases cached guest allocations\n";
@@ -12044,6 +12286,7 @@ int main(int argc, char** argv) {
         retainedPixelStageTests();
         pipelineLibrarySetTests();
         pipelineLibraryClearWaitsForBuildTests();
+        tessellationPipelineLibraryCacheTests();
         framebufferOwnerTests();
         hdrComponentSwapTests();
         depthMaintenanceTests();
@@ -12136,6 +12379,7 @@ int main(int argc, char** argv) {
         samplerTableDeviceLimitTests();
         tableUpdateAfterBindTests();
         misalignedShaderDataTests();
+        imageTableTemplateRefreshTests();
         debugBranchTests();
         hostImportBudgetTests();
         textureCacheBudgetTests();

core/libs/prx/libSceAgcDriver/tests/NopDataWritePacket.cpp

@@ -33,5 +33,18 @@ int main() {
     std::memcpy(&address, storage.data() + 8, sizeof(address));
     if (zeroBytesWords != 2 || header != 0xc0001000U || address != 0x01020304U) return 7;
     if (storage[12] != std::byte{0xaa}) return 8;
+
+    for (std::uint32_t len = 1; len <= 4; ++len) {
+        storage.fill(std::byte{0xaa});
+        const auto lenWords = sceAgcDriverNopDataWritePacket(storage.data() + 4, 0xaabbccddU, data.data(), len);
+        std::memcpy(&header, storage.data() + 4, sizeof(header));
+        std::memcpy(&address, storage.data() + 8, sizeof(address));
+        if (lenWords != 3 || header != 0xc0011000U || address != 0xaabbccddU) return 9 + static_cast<int>(len);
+        if (!std::equal(data.begin(), data.begin() + len, storage.begin() + 12)) return 14 + static_cast<int>(len);
+        for (std::size_t pad = 12 + len; pad < 16; ++pad) {
+            if (storage[pad] != std::byte{0}) return 19 + static_cast<int>(len);
+        }
+        if (storage[3] != std::byte{0xaa} || storage[16] != std::byte{0xaa}) return 24 + static_cast<int>(len);
+    }
     return 0;
 }

core/libs/prx/libSceAgcDriver/tests/PerformanceTrace.cpp

@@ -134,6 +134,20 @@ void EscapeStrings() {
     Check(output == "\"a\\\"b\\\\c\\u000a\\u00e9\"", "string escaping is not valid JSON");
 }
 
+void Utf8Strings() {
+    std::string output;
+    TraceFormat::AppendString(output, "Ghost of Y\xc5\x8dtei \xe2\x84\xa2 \xf0\x9f\x8e\xae");
+    Check(output == "\"Ghost of Y\xc5\x8dtei \xe2\x84\xa2 \xf0\x9f\x8e\xae\"", "a UTF-8 title was not written as its own characters");
+    output.clear();
+    TraceFormat::AppendString(output, "\xc0\xaf\xed\xa0\x80\xf4\x90\x80\x80\xe2\x84\x7f\xc5");
+    Check(output == "\"\\u00c0\\u00af\\u00ed\\u00a0\\u0080\\u00f4\\u0090\\u0080\\u0080\\u00e2\\u0084\\u007f\\u00c5\"", "a byte outside a valid UTF-8 sequence was not escaped");
+    AgcDriver::TraceSummary summary;
+    summary.title = AgcDriver::TraceTitle{"PPSA26344", "Ghost of Y\xc5\x8dtei", "01.000.000"};
+    const auto origin = Clock::now();
+    const std::vector<Clock::time_point> presents{origin, origin + milliseconds(16)};
+    Check(TraceFormat::Summary(summary, {}, presents).find("\"title\":\"Ghost of Y\xc5\x8dtei\"") != std::string::npos, "the summary line did not keep a UTF-8 title");
+}
+
 void EventsShape() {
     const auto origin = Clock::now();
     const std::vector<TraceThread::Slice> slices{
@@ -576,6 +590,7 @@ int main() {
         FullBufferEvents();
         FoldShortSlices();
         EscapeStrings();
+        Utf8Strings();
         EventsShape();
         BatchIds();
         SummaryFields();

core/libs/prx/libSceAgcDriver/tests/Recorder.cpp

@@ -50,6 +50,7 @@
 #endif
 #include <io.h>
 #include <windows.h>
+#include <psapi.h>
 #else
 #include <fcntl.h>
 #include <sys/mman.h>
@@ -67,6 +68,7 @@
 #include <cstring>
 #include <filesystem>
 #include <fstream>
+#include <iomanip>
 #include <future>
 #include <iostream>
 #include <memory>
@@ -322,6 +324,7 @@ public:
                 }
                 return false;
             };
+            memoryBudget = hasExtension(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME);
             auto bytes = AgcDriver::QueryBdaByteFeatures(context.physical, function<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2"), available);
             auto address = AgcDriver::QueryBdaFeatures(context.physical, function<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2"), available);
             const auto queues = function<PFN_vkGetPhysicalDeviceQueueFamilyProperties>("vkGetPhysicalDeviceQueueFamilyProperties");
@@ -516,6 +519,21 @@ public:
         return properties;
     }
     void WaitQueue() const { Check(context.Function<PFN_vkQueueWaitIdle>("vkQueueWaitIdle")(context.queue), "vkQueueWaitIdle"); }
+    struct HeapUsage {
+        std::uint64_t deviceLocal = 0;
+        std::uint64_t host = 0;
+    };
+    std::optional<HeapUsage> MemoryUsage() const {
+        if (!memoryBudget) return std::nullopt;
+        VkPhysicalDeviceMemoryBudgetPropertiesEXT budget{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
+        VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, &budget};
+        function<PFN_vkGetPhysicalDeviceMemoryProperties2>("vkGetPhysicalDeviceMemoryProperties2")(context.physical, &properties);
+        HeapUsage usage;
+        for (std::uint32_t heap = 0; heap < properties.memoryProperties.memoryHeapCount; ++heap) {
+            ((properties.memoryProperties.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0 ? usage.deviceLocal : usage.host) += budget.heapUsage[heap];
+        }
+        return usage;
+    }
 
 private:
     template<typename TFunction>
@@ -545,6 +563,7 @@ private:
     bool graphics = false;
     std::mutex settleMutex;
     bool nullDescriptors = false;
+    bool memoryBudget = false;
 };
 
 using Kind = Recorder::ReadKind;
@@ -2511,6 +2530,297 @@ bool settleStorageImageTests(const Device& device) {
     std::cout << "a settled import under a storage image tests passed\n";
     return true;
 }
+
+bool settleInitialDccTests(const Device& device) {
+    using namespace AgcDriver::GuestMemory;
+    const auto& base = device.GetContext();
+    constexpr std::size_t unit = 65536;
+    constexpr std::size_t tail = 0x4000;
+    if ((base.limits.framebufferColorSampleCounts & VK_SAMPLE_COUNT_4_BIT) == 0 || !device.Graphics() || base.hostImportAlignment == 0 || base.hostImportAlignment > tail || base.settleQueue == VK_NULL_HANDLE || !WriteWatched()) {
+        std::cout << "4x color targets, graphics, host imports within 16 KiB, a settle queue or write watching unavailable: an initial multisampled DCC import over a pending settle not tested\n";
+        return false;
+    }
+    {
+        std::lock_guard gpu(GpuMutex());
+        PrepareImportWatch(base);
+        if (!settledImportsWatchPrivate(base)) {
+            std::cout << "this device compares every Windows import: an initial multisampled DCC import over a pending settle not tested\n";
+            return false;
+        }
+    }
+    std::cout << "write watch: Windows private imports stay watched after a settle read; imports under 1 MiB wait for a deferred settle\n";
+    for (const std::size_t count : {256u, 257u, 1025u}) {
+        const auto bytes = count * unit;
+        auto* block = static_cast<std::uint8_t*>(AllocateWatched(bytes, unit));
+        Require(block != nullptr, "(m) cannot allocate adjacent pending imports");
+        const auto address = reinterpret_cast<std::uint64_t>(block);
+        struct Release {
+            const Context& context;
+            std::uint8_t* block;
+            std::size_t bytes;
+            std::size_t registered = 0;
+            ~Release() {
+                {
+                    GuestAllocations::Mutation mutation;
+                    for (std::size_t index = 0; index < registered; ++index) mutation.Remove(block + index * unit);
+                }
+                HostImportFor(context, reinterpret_cast<std::uint64_t>(block), unit);
+                SetHostImportWindow(0);
+                ReleaseWatched(block, bytes);
+            }
+        } release{base, block, bytes};
+        std::memset(block, 0x11, bytes);
+        SetHostImportWindow(unit);
+        {
+            GuestAllocations::Mutation mutation;
+            for (std::size_t index = 0; index < count; ++index) {
+                mutation.Add(block + index * unit, unit, true, true, true);
+                ++release.registered;
+            }
+        }
+        for (std::size_t index = 0; index < count; ++index) {
+            const auto at = address + index * unit;
+            const auto* import = HostImportFor(base, at, unit);
+            Require(import != nullptr && import->base == at && import->bytes == unit && import->unwatched,
+                    "(m) an adjacent 64 KiB import did not start with its settle pending");
+        }
+        {
+            std::lock_guard gpu(GpuMutex());
+            Require(!Watched(address, bytes), "(m) adjacent pending imports were already watched");
+            const auto before = HostImportSettles();
+            Require(WatchedAfterSettle(base, address, bytes),
+                    "(m) one synchronous settle left part of " + std::to_string(count) + " adjacent imports unwatched");
+            const auto after = HostImportSettles();
+            BumpCollectEpoch();
+            const auto collected = CollectWrites(address, bytes);
+            Require(collected != 0, "(m) adjacent settled imports were not collected");
+            static_cast<volatile std::uint8_t*>(block)[bytes - 1] = 0x22;
+            CollectWritesUncached(address, bytes);
+            Require(!UnchangedSince(address, bytes, collected), "(m) a CPU write after the final import settled was missed");
+            std::cout << "(m) " << count << " adjacent pending imports settled: " << after.submitted - before.submitted << " reads submitted\n";
+        }
+    }
+    TextureDetiler detiler(base);
+    auto context = base;
+    context.detiler = &detiler;
+    Recorder recorder(context);
+    recorder.Activate();
+    enum class Case { Pending, PartialBlock, ComparedNeighbour, DeferredOff, UnwatchedKeys, PendingFmask, PendingFmaskAlone };
+    const auto run = [&](Case kind, std::uint32_t height) {
+        const bool fmaskAlone = kind == Case::PendingFmaskAlone;
+        const bool eightSamples = kind == Case::PendingFmask || fmaskAlone;
+        if (eightSamples && (!base.storageImageMultisample || (base.limits.framebufferColorSampleCounts & VK_SAMPLE_COUNT_8_BIT) == 0)) {
+            std::cout << "(m) 8x color or multisampled storage images unavailable: an initial FMASK import over a pending settle not tested\n";
+            return;
+        }
+        const bool partial = kind == Case::PartialBlock || kind == Case::ComparedNeighbour;
+        ColorTarget target{};
+        target.extent = {128, height};
+        target.surfaceExtent = target.extent;
+        target.format = VK_FORMAT_R8G8B8A8_UNORM;
+        target.elementBytes = 4;
+        target.samples = eightSamples ? 8 : 4;
+        target.tileMode = ColorTileMode::RenderTarget;
+        target.dccPipeAligned = true;
+        const auto colorBytes = static_cast<std::size_t>(MultisampledColorBytes(target));
+        const auto dccBytes = MultisampledDccBytes(target);
+        const auto importBytes = colorBytes + (partial ? tail : unit);
+        const auto reserved = (importBytes + unit - 1) & ~(unit - 1);
+        const auto fmaskBytes = static_cast<std::size_t>(target.extent.width) * target.extent.height * 4u;
+        const auto fmaskReserved = (fmaskBytes + unit - 1) & ~(unit - 1);
+        const auto regionBytes = unit + reserved + unit + (kind == Case::UnwatchedKeys ? unit : eightSamples ? fmaskReserved + unit : 0);
+        auto* region = static_cast<std::uint8_t*>(AllocateWatched(regionBytes, unit));
+        void* block = region != nullptr ? region + unit : nullptr;
+        void* keys = region != nullptr && kind == Case::UnwatchedKeys ? region + unit + reserved + unit : nullptr;
+        void* fmask = region != nullptr && eightSamples ? region + unit + reserved + unit : nullptr;
+        const auto destBytes = static_cast<std::size_t>(target.extent.width) * target.extent.height * 4u;
+        void* dest = VirtualAlloc(nullptr, destBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+        Require(block != nullptr && dest != nullptr && (kind != Case::UnwatchedKeys || keys != nullptr) && (!eightSamples || fmask != nullptr), "(m) cannot allocate the target");
+        const auto address = reinterpret_cast<std::uint64_t>(block);
+        const auto neighbour = address + importBytes + tail;
+        const auto describe = [](const HostImport* found, std::uint64_t at, std::size_t bytes) {
+            char text[256];
+            if (found == nullptr) std::snprintf(text, sizeof(text), ": no import of 0x%llx+0x%zx, watched %d", static_cast<unsigned long long>(at), bytes, Watched(at, bytes) ? 1 : 0);
+            else std::snprintf(text, sizeof(text), ": import 0x%llx+0x%llx, unwatched %d, settled on demand %d, for 0x%llx+0x%zx, watched %d", static_cast<unsigned long long>(found->base), static_cast<unsigned long long>(found->bytes), found->unwatched ? 1 : 0, found->settleOnDemand ? 1 : 0, static_cast<unsigned long long>(at), bytes, Watched(at, bytes) ? 1 : 0);
+            return std::string(text);
+        };
+        struct Release {
+            const Context& context;
+            Recorder& recorder;
+            void* region;
+            std::size_t regionBytes;
+            void* block;
+            void* keys;
+            void* fmask;
+            void* dest;
+            std::size_t reserved;
+            std::uint64_t neighbour;
+            ~Release() {
+                {
+                    std::lock_guard gpu(GpuMutex());
+                    recorder.Sync();
+                    ClearMultisampleTargets(context.device);
+                    ClearCachedTextures(context.device);
+                    recorder.FlushDeferredReleases();
+                }
+                {
+                    GuestAllocations::Mutation mutation;
+                    mutation.Remove(block);
+                    mutation.Remove(dest);
+                    if (keys != nullptr) mutation.Remove(keys);
+                    if (fmask != nullptr) mutation.Remove(fmask);
+                    if (neighbour < reinterpret_cast<std::uint64_t>(block) + reserved) mutation.Remove(reinterpret_cast<void*>(neighbour));
+                }
+                HostImportFor(context, reinterpret_cast<std::uint64_t>(block), 16);
+                SetHostImportDeferredSettles(true);
+                ReleaseWatched(region, regionBytes);
+                VirtualFree(dest, 0, MEM_RELEASE);
+            }
+        } release{context, recorder, region, regionBytes, block, keys, fmask, dest, reserved, kind == Case::ComparedNeighbour ? neighbour : std::numeric_limits<std::uint64_t>::max()};
+        target.address = address;
+        target.surfaceAddress = address;
+        target.bytes = colorBytes;
+        target.dccAddress = kind == Case::UnwatchedKeys ? reinterpret_cast<std::uint64_t>(keys) : address + colorBytes;
+        std::memset(block, 0, importBytes);
+        std::memset(reinterpret_cast<void*>(target.dccAddress), 0xc0, dccBytes);
+        std::memset(dest, 0x5a, destBytes);
+        if (fmask != nullptr) {
+            target.cmaskAddress = target.dccAddress + ((dccBytes + 4095) & ~std::size_t{4095});
+            target.cmaskBytes = CmaskBytes(target.extent.width, target.extent.height);
+            target.cmaskFastClear = true;
+            target.fmaskAddress = reinterpret_cast<std::uint64_t>(fmask);
+            Require(target.cmaskAddress + target.cmaskBytes <= address + importBytes, "(m) the CMASK does not fit the target's import");
+            std::memset(reinterpret_cast<void*>(target.cmaskAddress), 0xcc, static_cast<std::size_t>(target.cmaskBytes));
+            std::memset(fmask, 0x11, fmaskBytes);
+        }
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Add(block, importBytes, true, true, true);
+            mutation.Add(dest, destBytes, true, true, true);
+            if (keys != nullptr) mutation.Add(keys, unit, true, true, true);
+            if (fmask != nullptr) mutation.Add(fmask, fmaskBytes, true, true, true);
+            if (kind == Case::ComparedNeighbour) mutation.Add(reinterpret_cast<void*>(neighbour), reserved - (importBytes + tail), true, true, true);
+        }
+        if (kind == Case::UnwatchedKeys) Unwatch(target.dccAddress, dccBytes);
+        if (kind == Case::DeferredOff) SetHostImportDeferredSettles(false);
+        if (kind == Case::ComparedNeighbour) {
+            const auto* other = HostImportFor(context, neighbour, reserved - (importBytes + tail));
+            Require(other != nullptr && other->unwatched && !Watched(neighbour, reserved - (importBytes + tail)), "(m) the import sharing the target's last 64 KiB block was not compared" + describe(other, neighbour, reserved - (importBytes + tail)));
+        }
+        if (fmask != nullptr && !fmaskAlone) {
+            const auto* fmaskImport = HostImportFor(context, target.fmaskAddress, fmaskBytes);
+            Require(fmaskImport != nullptr && fmaskImport->base == target.fmaskAddress && fmaskImport->bytes == fmaskBytes && fmaskImport->unwatched && !Watched(target.fmaskAddress, fmaskBytes), "(m) the FMASK's import was not compared at first" + describe(fmaskImport, target.fmaskAddress, fmaskBytes));
+        }
+        const auto* import = HostImportFor(context, address, importBytes);
+        Require(import != nullptr && import->base == address && import->bytes == importBytes && import->unwatched && !Watched(address, importBytes), "(m) the target's import of " + std::to_string(importBytes) + " bytes was not compared at first" + describe(import, address, importBytes));
+        HostImportSettleCounts quiet{};
+        if (fmaskAlone) {
+            {
+                std::lock_guard gpu(GpuMutex());
+                const auto start = HostImportSettles();
+                Require(WatchedAfterSettle(context, address, importBytes), "(m) the target's import did not settle before the FMASK's import" + describe(ExistingHostImportFor(context, address, importBytes), address, importBytes));
+                const auto settledTarget = HostImportSettles();
+                Require(settledTarget.submitted > start.submitted && settledTarget.rewatched > start.rewatched, "(m) the target's import returned to the write watch without a settle read");
+            }
+            const std::array<std::pair<std::uint64_t, std::size_t>, 3> planes{{{address, colorBytes}, {target.dccAddress, dccBytes}, {target.cmaskAddress, static_cast<std::size_t>(target.cmaskBytes)}}};
+            for (const auto& [at, bytes] : planes) {
+                const auto* covering = ExistingHostImportFor(context, at, bytes);
+                Require(Watched(at, bytes) && (covering == nullptr || !covering->unwatched), "(m) a color, DCC or CMASK plane is not watched before the FMASK's import" + describe(covering, at, bytes));
+            }
+            quiet = HostImportSettles();
+            const auto* fmaskImport = HostImportFor(context, target.fmaskAddress, fmaskBytes);
+            Require(fmaskImport != nullptr && fmaskImport->base == target.fmaskAddress && fmaskImport->bytes == fmaskBytes && fmaskImport->unwatched && !fmaskImport->settleOnDemand && !Watched(target.fmaskAddress, fmaskBytes), "(m) the FMASK's import is not compared with its deferred settle wanted" + describe(fmaskImport, target.fmaskAddress, fmaskBytes));
+        }
+        std::unique_lock gpu(GpuMutex());
+        if (partial) {
+            for (int round = 0; round < 20; ++round) {
+                gpu.unlock();
+                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+                gpu.lock();
+                HostImportFor(context, address, importBytes);
+            }
+            Require(!Watched(address, importBytes), "(m) a deferred settle returned an import that is not a whole number of 64 KiB blocks to the write watch");
+        }
+        const auto before = HostImportSettles();
+        Require(!fmaskAlone || before.submitted == quiet.submitted, "(m) a settle read was submitted between the FMASK's import and the target's first use");
+        std::string refusal;
+        try {
+            Require(MultisampleTargetView(context, target) != VK_NULL_HANDLE, "(m) the multisampled DCC target has no view");
+        } catch (const std::exception& error) {
+            refusal = error.what();
+        }
+        const auto after = HostImportSettles();
+        const auto* settled = ExistingHostImportFor(context, address, importBytes);
+        const char* name = kind == Case::Pending ? "a pending settle" : kind == Case::PartialBlock ? "an import of whole and partial 64 KiB blocks" : kind == Case::ComparedNeighbour ? "a compared import in its last 64 KiB block" : kind == Case::DeferredOff ? "deferred settles off" : kind == Case::UnwatchedKeys ? "explicitly unwatched DCC keys" : kind == Case::PendingFmask ? "an eight-sample target whose color and FMASK imports both have pending settles" : "an eight-sample target whose FMASK alone has a pending settle";
+        std::cout << "(m) initial DCC over " << name << " (import " << importBytes << " bytes): " << (refusal.empty() ? "accepted" : refusal) << "; " << after.submitted - before.submitted << " settle reads submitted, " << after.rewatched - before.rewatched << " back in the write watch\n";
+        if (kind == Case::ComparedNeighbour || kind == Case::DeferredOff || kind == Case::UnwatchedKeys) {
+            Require(refusal.find("initial multisampled DCC contents require watched guest memory") != std::string::npos, std::string("(m) an initial multisampled DCC import over ") + name + " was not refused: " + refusal);
+            Require(kind == Case::UnwatchedKeys || !Watched(address, importBytes), std::string("(m) ") + name + " returned the target's import to the write watch");
+            Require(kind != Case::UnwatchedKeys || (Watched(address, importBytes) && !Watched(target.dccAddress, dccBytes)), "(m) the refusal over explicitly unwatched DCC keys did not come from the keys" + describe(settled, address, importBytes));
+            return;
+        }
+        Require(refusal.empty(), std::string("(m) an initial multisampled DCC import over ") + name + " was refused: " + refusal);
+        Require(!fmaskAlone || (after.submitted - before.submitted == 1 && after.rewatched - before.rewatched == 1), "(m) the first use over a pending FMASK settle alone submitted " + std::to_string(after.submitted - before.submitted) + " settle reads, not the FMASK's alone");
+        Require(after.submitted > before.submitted && after.rewatched > before.rewatched && Watched(address, importBytes) && settled != nullptr && !settled->unwatched, std::string("(m) the initial DCC import over ") + name + " did not settle the import and return it to the write watch");
+        if (fmask != nullptr) {
+            const auto* settledFmask = ExistingHostImportFor(context, target.fmaskAddress, fmaskBytes);
+            Require(Watched(target.fmaskAddress, fmaskBytes) && settledFmask != nullptr && !settledFmask->unwatched, "(m) the FMASK's import did not settle back into the write watch" + describe(settledFmask, target.fmaskAddress, fmaskBytes));
+        }
+        ColorTarget destination{};
+        destination.address = reinterpret_cast<std::uint64_t>(dest);
+        destination.surfaceAddress = destination.address;
+        destination.extent = target.extent;
+        destination.surfaceExtent = target.extent;
+        destination.format = VK_FORMAT_R8G8B8A8_UNORM;
+        destination.elementBytes = 4;
+        destination.bytes = destBytes;
+        destination.samples = 1;
+        destination.tileMode = ColorTileMode::RenderTarget;
+        RunColorMetadataPass(context, {ColorMetadataPass::Mode::Resolve, {destination}, target});
+        recorder.Submit();
+        device.WaitQueue();
+        recorder.Sync();
+        std::vector<std::uint32_t> resolved(destBytes / 4);
+        AgcDriver::GuestMemory::Read(destination.address, std::as_writable_bytes(std::span(resolved)));
+        Require(std::ranges::all_of(resolved, [](std::uint32_t word) { return word == 0xffffffffu; }), std::string("(m) the target made over ") + name + " did not start from its DCC fast clear");
+        BumpCollectEpoch();
+        if (fmask != nullptr) {
+            const auto collectedFmask = CollectWrites(target.fmaskAddress, fmaskBytes);
+            Require(collectedFmask != 0, "(m) the settled FMASK import is not collected");
+            static_cast<volatile std::uint8_t*>(fmask)[0] = 0x22;
+            std::string fmaskReuse;
+            try {
+                MultisampleTargetView(context, target);
+            } catch (const std::exception& error) {
+                fmaskReuse = error.what();
+            }
+            Require(fmaskReuse.find("writes to a resident native FMASK are not implemented") != std::string::npos, "(m) the resident target missed the CPU store to its FMASK: " + fmaskReuse);
+            return;
+        }
+        const auto collected = CollectWrites(address, importBytes);
+        Require(collected != 0, "(m) the settled import is not collected");
+        static_cast<volatile std::uint8_t*>(block)[importBytes - 1] = 0x77;
+        static_cast<volatile std::uint8_t*>(block)[colorBytes] = 0x00;
+        CollectWritesUncached(address, importBytes);
+        Require(!UnchangedSince(address + importBytes - 1, 1, collected) && !UnchangedSince(address + colorBytes, 1, collected), "(m) a CPU store after the initial DCC import was missed");
+        std::string reuse;
+        try {
+            MultisampleTargetView(context, target);
+        } catch (const std::exception& error) {
+            reuse = error.what();
+        }
+        Require(reuse.find("writes to a resident multisampled DCC target or its metadata are not implemented") != std::string::npos, "(m) the resident target missed the CPU store to its DCC keys: " + reuse);
+    };
+    run(Case::Pending, 128);
+    run(Case::PartialBlock, 384);
+    run(Case::ComparedNeighbour, 384);
+    run(Case::DeferredOff, 128);
+    run(Case::UnwatchedKeys, 128);
+    run(Case::PendingFmask, 128);
+    run(Case::PendingFmaskAlone, 128);
+    std::cout << "initial multisampled DCC over a pending settle tests passed\n";
+    return true;
+}
 #endif
 
 void remappedImportTests(const Device& device) {
@@ -3423,7 +3733,7 @@ bool readWrittenStagingTests(const Device& device, Recorder& recorder) {
         std::cout << "host imports unavailable: read and written staging not tested\n";
         return false;
     }
-    constexpr std::size_t bytes = 16u << 20u;
+    constexpr std::size_t bytes = 40u << 20u;
 #ifdef _WIN32
     void* block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
 #else
@@ -3456,9 +3766,8 @@ bool readWrittenStagingTests(const Device& device, Recorder& recorder) {
     Require(boundInPlace(address + 4096, 4u << 20u, false) != context.discreteGpu, context.discreteGpu ? "a written-only 4 MiB element was bound in place on a discrete GPU" : "a written-only 4 MiB element was staged past the written window");
     Require(boundInPlace(address + 4096, 12u << 20u, false), "a written-only 12 MiB element was staged past the discrete written window");
     Require(!boundInPlace(address + 4096, 4u << 20u, true), "a read and written 4 MiB element was bound in place");
-    const auto buildInPlace = [&](std::uint64_t dispatchThreads, bool read, std::uint32_t stride, std::uint32_t extent) {
+    const auto buildInPlace = [&](std::uint64_t dispatchThreads, bool read, std::uint32_t stride, std::uint32_t extent, std::uint32_t elementBytes = 12u << 20u) {
         const auto element = address + 4096;
-        constexpr std::uint32_t elementBytes = 12u << 20u;
         const std::uint32_t records = stride != 0 ? elementBytes / stride : elementBytes;
         ShaderRecompiler::RecompileResult program;
         ShaderRecompiler::DescriptorBinding binding;
@@ -3483,7 +3792,12 @@ bool readWrittenStagingTests(const Device& device, Recorder& recorder) {
     Require(!buildInPlace((12u << 20u) / 16u, true, 16, 0), "a dispatch sweeping a read and written 12 MiB element bound it in place");
     Require(buildInPlace(1024, true, 16, 0), "a dispatch touching 16 KiB of a read and written 12 MiB element staged all of it");
     Require(buildInPlace(0, true, 16, 0), "a dispatch of unknown size staged a read and written 12 MiB element");
-    Require(buildInPlace((12u << 20u) / 16u, false, 16, 0), "a dispatch sweeping a written-only 12 MiB element staged it");
+    Require(buildInPlace((12u << 20u) / 32u, false, 32, 0) != context.discreteGpu, context.discreteGpu ? "a dispatch sweeping a written-only 12 MiB element of 32-byte records bound it in place on a discrete GPU" : "a dispatch sweeping a written-only 12 MiB element of 32-byte records staged it on a GPU without separate VRAM");
+    Require(buildInPlace((12u << 20u) / 16u, false, 16, 0), "a dispatch sweeping a written-only 12 MiB element of 16-byte records, which its stores write contiguously, staged it");
+    Require(buildInPlace(1024, false, 32, 0), "a dispatch touching 32 KiB of a written-only 12 MiB element staged all of it");
+    Require(buildInPlace((12u << 20u) / 32u, false, 0, 32), "a dispatch writing a written-only 12 MiB raw element staged it");
+    Require(buildInPlace(204800, false, 160, 0, 204800u * 160u) != context.discreteGpu, context.discreteGpu ? "a dispatch sweeping a written-only 31.25 MiB element of 160-byte records bound it in place on a discrete GPU" : "a dispatch sweeping a written-only 31.25 MiB element of 160-byte records staged it on a GPU without separate VRAM");
+    Require(buildInPlace(204800, true, 160, 0, 204800u * 160u) != context.discreteGpu, context.discreteGpu ? "a dispatch sweeping a read and written 31.25 MiB element bound it in place on a discrete GPU" : "a dispatch sweeping a read and written 31.25 MiB element staged it past the 16 MiB cap on a GPU without separate VRAM");
     Require(!buildInPlace((12u << 20u) / 32u, true, 0, 32), "a dispatch sweeping a read and written 12 MiB raw element bound it in place");
     Require(buildInPlace(1024, true, 0, 32), "a dispatch touching 32 KiB of a read and written 12 MiB raw element staged all of it");
     Require(buildInPlace((12u << 20u) / 32u, true, 0, 0), "a raw element without an access extent was staged");
@@ -9817,6 +10131,98 @@ bool vertexTargetRangeTests(const Device& device, Recorder& recorder, bool impor
     return true;
 }
 
+std::uint64_t processPrivateBytes() {
+#ifdef _WIN32
+    PROCESS_MEMORY_COUNTERS_EX counters{};
+    counters.cb = sizeof(counters);
+    if (!K32GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PROCESS_MEMORY_COUNTERS*>(&counters), sizeof(counters))) return 0;
+    return counters.PrivateUsage;
+#else
+    std::ifstream status("/proc/self/status");
+    std::string line;
+    while (std::getline(status, line)) {
+        if (line.rfind("RssAnon:", 0) == 0) return std::stoull(line.substr(8)) * 1024u;
+    }
+    return 0;
+#endif
+}
+
+// What one graphics pipeline the store keeps costs: `count` pipelines, each with its own fragment
+// module (so its own fragment shader library), built through CachedPipeline and held, after their
+// optimized relinks; process private bytes and VK_EXT_memory_budget heap usage before and after,
+// and after the store is cleared.
+int pipelineStoreMemoryBench(const Device& device, std::uint32_t count) {
+    if (!device.Graphics() || count == 0) {
+        std::cout << "skipped, the pipeline store memory bench requires a graphics queue and at least one pipeline\n";
+        return 77;
+    }
+    std::lock_guard gpu(GpuMutex());
+    const auto& context = device.GetContext();
+    struct Cleanup {
+        VkDevice device;
+        ~Cleanup() { ClearCachedPipelines(device); }
+    } cleanup{context.device};
+    ClearCachedPipelines(context.device);
+    ShaderRecompiler::RecompileResult vertex;
+    vertex.variantId = 21;
+    vertex.spirv = std::vector<std::uint32_t>(std::begin(TRIANGLE_vert_SPV), std::end(TRIANGLE_vert_SPV));
+    std::vector<ShaderRecompiler::RecompileResult> fragments(count + 1u);
+    for (std::uint32_t index = 0; index <= count; ++index) {
+        fragments[index].variantId = 1000u + index;
+        fragments[index].spirv = std::vector<std::uint32_t>(std::begin(TRIANGLE_frag_SPV), std::end(TRIANGLE_frag_SPV));
+        fragments[index].spirv[2] = 0x50535430u + index;
+    }
+    State state{};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.hasColorTarget = true;
+    state.color.format = VK_FORMAT_R8G8B8A8_UNORM;
+    state.colors.push_back(state.color);
+    state.blend.colorWriteMask = 15;
+    state.blends.push_back(state.blend);
+    const std::array first{CompiledShader{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, CompiledShader{ShaderRecompiler::ShaderStage::Fragment, &fragments[count], 0}};
+    ShaderResources resources(context, first, state.color, static_cast<std::uint32_t>(state.colors.size()), 0, 0);
+    const VertexInputLayout input;
+    std::vector<std::shared_ptr<Pipeline>> held;
+    held.reserve(count + 1u);
+    const auto build = [&](std::uint32_t index) {
+        const std::array shaders{CompiledShader{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, CompiledShader{ShaderRecompiler::ShaderStage::Fragment, &fragments[index], 0}};
+        held.push_back(CachedPipeline(context, state, input, resources, shaders, VK_IMAGE_LAYOUT_GENERAL));
+    };
+    struct Sample {
+        std::uint64_t privateBytes;
+        std::optional<Device::HeapUsage> heaps;
+    };
+    const auto sample = [&] {
+        WaitForOptimizedPipelines(context.device);
+        return Sample{processPrivateBytes(), device.MemoryUsage()};
+    };
+    build(count);
+    const auto before = sample();
+    const auto started = std::chrono::steady_clock::now();
+    for (std::uint32_t index = 0; index < count; ++index) build(index);
+    const auto built = std::chrono::steady_clock::now();
+    const auto after = sample();
+    const auto linked = std::chrono::steady_clock::now();
+    const auto totals = PipelineStoreTotals();
+    held.clear();
+    ClearCachedPipelines(context.device);
+    const auto cleared = sample();
+    const auto mib = [](std::uint64_t later, std::uint64_t earlier) { return (static_cast<double>(later) - static_cast<double>(earlier)) / (1024.0 * 1024.0); };
+    const auto perPipeline = [&](std::uint64_t later, std::uint64_t earlier) { return (static_cast<double>(later) - static_cast<double>(earlier)) / 1024.0 / count; };
+    const char* linking = !context.graphicsPipelineLibrary ? "whole pipelines" : context.pipelineLinkTimeOptimization ? "libraries and optimized relinks" : "libraries, fast links only";
+    std::cout << std::fixed << std::setprecision(1);
+    std::cout << "Pipeline store memory: " << count << " graphics pipelines (one fragment module each, " << linking << "), " << totals.cached << " in the store, built in "
+              << std::chrono::duration<double>(built - started).count() << " s, relinked in " << std::chrono::duration<double>(linked - built).count() << " s\n";
+    std::cout << "  process private bytes: +" << mib(after.privateBytes, before.privateBytes) << " MiB = " << perPipeline(after.privateBytes, before.privateBytes) << " KiB per pipeline; after clearing the store " << mib(cleared.privateBytes, before.privateBytes) << " MiB above the start\n";
+    if (before.heaps && after.heaps && cleared.heaps) {
+        std::cout << "  VK_EXT_memory_budget heap usage: device-local +" << mib(after.heaps->deviceLocal, before.heaps->deviceLocal) << " MiB = " << perPipeline(after.heaps->deviceLocal, before.heaps->deviceLocal) << " KiB per pipeline, host +" << mib(after.heaps->host, before.heaps->host) << " MiB = " << perPipeline(after.heaps->host, before.heaps->host)
+                  << " KiB per pipeline; after clearing the store device-local " << mib(cleared.heaps->deviceLocal, before.heaps->deviceLocal) << " MiB, host " << mib(cleared.heaps->host, before.heaps->host) << " MiB above the start\n";
+    } else {
+        std::cout << "  VK_EXT_memory_budget unavailable: no device heap usage\n";
+    }
+    return 0;
+}
+
 void pipelineCacheTests(const Device& device, bool benchmark) {
     std::thread establishThreadedProcess([] {});
     establishThreadedProcess.join();
@@ -10971,6 +11377,14 @@ int main(int argc, char** argv) {
             importWatchTests(device);
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--settle-initial-dcc-only") {
+#ifdef _WIN32
+            return settleInitialDccTests(device) ? 0 : 77;
+#else
+            std::cout << "skipped, the device has no deferred settle of host imports: imports under 1 MiB wait for one only on Windows\n";
+            return 77;
+#endif
+        }
 #ifdef _WIN32
         if (argc == 2 && std::string_view(argv[1]) == "--host-import-lifetime-other-family-only") {
             if (device.GetContext().settleFamily == device.GetContext().queueFamily) {
@@ -11028,6 +11442,7 @@ int main(int argc, char** argv) {
             pipelineLibraryTests(device);
             return 0;
         }
+        if ((argc == 2 || argc == 3) && std::string_view(argv[1]) == "--benchmark-pipeline-store-memory") return pipelineStoreMemoryBench(device, argc == 3 ? static_cast<std::uint32_t>(std::stoul(argv[2])) : 4096u);
         if (argc == 2 && (std::string_view(argv[1]) == "--benchmark-pipeline-cache" || std::string_view(argv[1]) == "--pipeline-cache-only")) {
             pipelineCacheTests(device, std::string_view(argv[1]) == "--benchmark-pipeline-cache");
             return 0;

core/libs/prx/libSceAgcDriver/tests/RedundantValueElimination.cpp

@@ -209,7 +209,8 @@ std::vector<std::uint32_t> Recompiled(std::span<const std::uint32_t> code) {
     request.target.maxWorkgroupSharedMemoryBytes = 49152;
     request.layout = {0, 0, 0, 128};
     request.useCache = false;
-    const auto& words = Recompile(request).spirv.Words();
+    const auto result = Recompile(request);
+    const auto& words = result.spirv.Words();
     return {words.begin(), words.end()};
 }
 

core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp

@@ -406,6 +406,36 @@ void verifyFrontendPair() {
     }
 }
 
+void verifySpecializedModuleRelease() {
+    using namespace ShaderRecompiler;
+    const std::array<std::uint32_t, 1> color{0x3f800000u};
+    const std::vector<std::uint32_t> code{0xbf800003u, 0x7e000200u, 0xf800180fu, 0u, 0xbf810000u};
+    auto pixel = FrontendRequest(code, color);
+    pixel.shader.stage = ShaderStage::Fragment;
+    pixel.context.compute.reset();
+    pixel.context.pixel = ShaderPixelStageInfo{};
+    pixel.context.pixel->wave32 = true;
+    pixel.context.pixel->targetOutputMode[0] = 9u;
+    pixel.context.waveSize = 32u;
+    pixel.useCache = true;
+    SharedSpirv::Weak released;
+    std::uint64_t specializationId = 0;
+    std::vector<std::uint32_t> words;
+    {
+        const auto first = Recompile(pixel);
+        require(first.specializationId != 0u, "specialized module release: the pixel fixture was not specialized");
+        released = first.spirv.Weaken();
+        specializationId = first.specializationId;
+        words = first.spirv.Copy();
+    }
+    require(released.Expired(), "specialized module release: the module caches kept the words of a module no result uses");
+    const auto again = Recompile(pixel);
+    require(again.specializationId == specializationId, "specialized module release: a module built again from its key got another specialization ID");
+    require(std::ranges::equal(again.spirv.Words(), words), "specialized module release: a module built again from its key has other words");
+    const auto shared = Recompile(pixel);
+    require(shared.spirv.data() == again.spirv.data(), "specialized module release: a second result of a live module copied its words");
+}
+
 ShaderRecompiler::RecompileRequest LayoutPairRequest(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, bool registration, std::uint32_t wordLimit) {
     using namespace ShaderRecompiler;
     auto request = FrontendRequest(code, userData);
@@ -2339,7 +2369,10 @@ void verifyTableSnapshotReuse() {
     const auto otherBuffer = CompileTable(request);
     std::copy(outputV.begin(), outputV.end(), guest.srt.begin() + 12);
     const auto resolvedShared = ResourceMaterializer::TableReuse();
-    require(otherBuffer.result != compiled.result && otherBuffer.result->variantId == compiled.result->variantId && resolvedShared.resolutionsShared - resolvedBefore.resolutionsShared == 1u && resolvedShared.resolutions == resolvedBefore.resolutions && TableMap(*otherBuffer.result) == map && TableBinding(*otherBuffer.result, DescriptorRole::ImageTable)->guestDescriptor == images->guestDescriptor, "table snapshot reuse: a result for another buffer did not reuse the unchanged table's resolution");
+    const bool resolves = std::getenv("APS5_NO_MASKED_RESULT_MEMO") != nullptr || std::getenv("APS5_VERIFY_MASKED_RESULT_MEMO") != nullptr;
+    require(otherBuffer.result != compiled.result && otherBuffer.result->variantId == compiled.result->variantId && resolvedShared.resolutionsShared - resolvedBefore.resolutionsShared == (resolves ? 1u : 0u) && resolvedShared.resolutions == resolvedBefore.resolutions && TableMap(*otherBuffer.result) == map && TableBinding(*otherBuffer.result, DescriptorRole::ImageTable)->guestDescriptor == images->guestDescriptor, "table snapshot reuse: a result for another buffer did not reuse the unchanged table's resolution");
+    const auto* otherBuffers = TableBinding(*otherBuffer.result, DescriptorRole::GuestBuffers);
+    require(otherBuffers != nullptr && std::search(otherBuffers->guestDescriptor.begin(), otherBuffers->guestDescriptor.end(), otherOutputV.begin(), otherOutputV.end()) != otherBuffers->guestDescriptor.end(), "table snapshot reuse: a result for another buffer does not hold that buffer's descriptor");
     const auto storedWord = heap[5u * 8u];
     heap[5u * 8u] = heap[6u * 8u];
     const auto restoredMap = map;
@@ -3364,6 +3397,19 @@ void verifyImageTableRegistration() {
     const auto* atomicTable = TableBinding(*atomic.result, DescriptorRole::ImageStorageTable);
     require((atomicFault & Abi::PoisonFlag) == 0u && atomicTable != nullptr && atomicTable->count != 0u && atomicTable->imageAtomic.size() == atomicTable->count && std::all_of(atomicTable->imageWritten.begin(), atomicTable->imageWritten.end(), [](bool written) { return written; }) && std::all_of(atomicTable->imageAtomic.begin(), atomicTable->imageAtomic.end(), [](bool atomic) { return atomic; }), "image table registration: the image_atomic_add table was not bound as atomic storage elements");
     require(TableBinding(*atomic.result, DescriptorRole::ImageTable) == nullptr || TableBinding(*atomic.result, DescriptorRole::ImageTable)->count == 0u, "image table registration: an atomic-only table bound sampled elements");
+    auto atomic64Code = storeCode;
+    atomic64Code[8] = 0xf0440308u;
+    guest.heap[0] = TableT(texture.bytes.data(), TableFormat32_32UInt, TableType2D);
+    static constexpr std::array<std::uint32_t, 5> int64ImageCapabilities{29u, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, spv::CapabilityInt64ImageEXT};
+    auto atomic64Request = TableRequest(atomic64Code, userData);
+    atomic64Request.target.supportedCapabilities = int64ImageCapabilities;
+    const auto atomic64 = CompileTable(atomic64Request);
+    const auto& atomic64Plan = *atomic64.capture->plan;
+    require(atomic64Plan.info.images.size() == 1u && atomic64Plan.info.images[0].table == 0u && atomic64Plan.info.images[0].tableOperation == TableOperation::Supported, "image table registration: a table T# feeding image_atomic64_add was not a supported table operation");
+    const auto& atomic64Map = TableMap(*atomic64.result);
+    const auto atomic64Fault = atomic64Map[Abi::TableHeader(0) + Abi::TableFault];
+    const auto* atomic64Table = TableBinding(*atomic64.result, DescriptorRole::ImageStorageTable);
+    require((atomic64Fault & Abi::PoisonFlag) == 0u && atomic64Table != nullptr && atomic64Table->count != 0u && atomic64Table->imageAtomic64.size() == atomic64Table->count && std::all_of(atomic64Table->imageAtomic64.begin(), atomic64Table->imageAtomic64.end(), [](bool atomic) { return atomic; }), "image table registration: the image_atomic64_add table was not bound as 64-bit atomic storage elements");
     const std::array<std::pair<std::uint32_t, const char*>, 2> unsupportedStores{{{0xf0240f08u, "image_store_mip"}, {0xf0280f08u, "image_store_pck"}}};
     for (const auto& [instruction, name] : unsupportedStores) {
         auto code = storeCode;
@@ -3794,7 +3840,7 @@ void verifySelectedRecordDirectOperations() {
         require(word0 == 0xf0440308u || (TableBinding(*compiled.result, DescriptorRole::ImageTable) == nullptr && compiled.result->imageTableFaults == 0u && TableCovered(compiled.regions, &heap[2], sizeof(heap[2]))), (what + " through a T# whose record the CPU evaluates was not bound from its record").c_str());
         const auto wholeCode = WithImageOperation(TableWholeCode, word0, false);
         const auto whole = GetResourcePlan(TableRequest(wholeCode, userData));
-        require(whole->info.images.size() == 1u && whole->info.images[0].table == 0u && whole->info.images[0].tableOperation == (word0 == 0xf0200f08u || word0 == 0xf0440108u ? TableOperation::Supported : TableOperation::Unsupported), (what + " through a T# keyed per lane left the image table or has the wrong table operation").c_str());
+        require(whole->info.images.size() == 1u && whole->info.images[0].table == 0u && whole->info.images[0].tableOperation == (word0 == 0xf0200f08u || word0 == 0xf0440108u || word0 == 0xf0440308u ? TableOperation::Supported : TableOperation::Unsupported), (what + " through a T# keyed per lane left the image table or has the wrong table operation").c_str());
     }
     const auto mixedCode = WithImageOperation(TableSrtKeyCode, 0xf0200f08u, true);
     const auto mixed = GetResourcePlan(TableRequest(mixedCode, userData));
@@ -6787,6 +6833,12 @@ void verifyUnnormalizedSamplers() {
     const auto adjusted = recompile(reservedDword3(0xf0800f09u), userData(normalized));
     require(proven(adjusted, false), "sampler dword 3: an image_sample_a whose S# dword 3 holds only a ballot in its reserved bits was not compiled");
     expectFailure([&] { static_cast<void>(recompile(reservedDword3(0xf0800f08u), userData(normalized))); }, "GetSamplerResource dword 3 is not a valid runtime value", "sampler dword 3: an image_sample whose S# dword 3 holds a ballot was accepted");
+    for (const auto& [adjust, plain, compiled, accepted] : {
+             std::tuple{0xf1200109u, 0xf1200108u, "sampler dword 3: an image_gather4_c_a whose S# dword 3 holds only a ballot in its reserved bits was not compiled", "sampler dword 3: an image_gather4_c with a computed S# dword 3 was accepted"},
+             std::tuple{0xf1600109u, 0xf1600108u, "sampler dword 3: an image_gather4_c_o_a whose S# dword 3 holds only a ballot in its reserved bits was not compiled", "sampler dword 3: an image_gather4_c_o with a computed S# dword 3 was accepted"}}) {
+        require(proven(recompile(reservedDword3(adjust), imageData(normalized, Type2D, Format32Float, 0u)), false), compiled);
+        expectFailure([&] { static_cast<void>(recompile(reservedDword3(plain), imageData(normalized, Type2D, Format32Float, 0u))); }, "GetSamplerResource dword 3 is not a valid runtime value", accepted);
+    }
     reject(0xf09c0f18u, imageData(unnormalized, TypeCube, Format8888UNorm, 5u), "unnormalized guest sampler samples a 1D-array, 2D-array, 3D, cube or multisampled image", "unnormalized samplers: a cube T# was accepted");
     reject(0xf09c0f10u, imageData(unnormalized, Type3D, Format8888UNorm, 3u), "unnormalized guest sampler samples a 1D-array, 2D-array, 3D, cube or multisampled image", "unnormalized samplers: a 3D T# was accepted");
     reject(0xf09c0f28u, imageData(unnormalized, Type2DArray, Format8888UNorm, 5u), "unnormalized guest sampler samples a 1D-array, 2D-array, 3D, cube or multisampled image", "unnormalized samplers: a 2D-array sample was accepted");
@@ -8530,6 +8582,7 @@ int main(int argc, char** argv) {
         verifyTargetFeatureKey();
         verifyImageTableKeys();
         verifyImageTables();
+        verifySpecializedModuleRelease();
         verifyImageTableRegistration();
         verifyScalarAddressRule();
         verifyPointerImageTable();

core/libs/prx/libSceAgcDriver/tests/SpirvModule.cpp

@@ -460,6 +460,20 @@ void testInternedSpirvSharesEqualWords() {
     check(edited[6] == spv::CapabilityFloat64 && first[6] == spv::CapabilityShader && again.data() == first.data(), "a write to one holder of interned SPIR-V changed the others");
 }
 
+void testInternedSpirvReleasesUnusedWords() {
+    const std::vector<std::uint32_t> words{spv::MagicNumber, 0x00010400u, 0u, 7u, 0u, (2u << 16u) | spv::OpCapability, spv::CapabilityGeometry};
+    SharedSpirv::Weak released;
+    {
+        const auto first = InternSpirv(words);
+        const auto second = InternSpirv(words);
+        check(first.data() == second.data(), "two equal specialized modules kept separate copies of their words");
+        released = first.Weaken();
+    }
+    check(released.Expired(), "the SPIR-V interning table kept words no holder uses");
+    const auto again = InternSpirv(words);
+    check(std::ranges::equal(again.Words(), words) && released.Lock().empty(), "SPIR-V interned again after its release changed its words");
+}
+
 int main() {
     testFoldingLeavesNoDeadBlocksOrConstants();
     testIdsBeyondTheBoundAreRejected();
@@ -477,6 +491,7 @@ int main() {
     testEmittedWordsAreStable();
     testBlockBallotsOfOnePredicateShareOneBallot();
     testInternedSpirvSharesEqualWords();
+    testInternedSpirvReleasesUnusedWords();
     if (failures != 0) {
         std::fprintf(stderr, "%d check(s) failed\n", failures);
         return 1;

core/libs/prx/libSceAgcDriver/tests/Submit.cpp

@@ -591,8 +591,7 @@ void testSmallCopyBehindPendingSource() {
     waitFor(&copied, 1, "the label after a small copy behind a pending source never landed");
     AgcDriverWaitIdle_nid_postfix();
     check(std::equal(destination, destination + smallWords, produced), "a small copy did not see the GPU copy into its source before it");
-    if (std::getenv("APS5_COPY_SYNC") != nullptr) check(AgcDriver::DriverDetail::copiesBehindPendingSources.load() == before, "a small copy that has to wait for a completion store over its source went through the GPU copy");
-    else check(AgcDriver::DriverDetail::copiesBehindPendingSources.load() == before + 1, "a small copy whose source the GPU still stores into was not recorded on the GPU");
+    check(AgcDriver::DriverDetail::copiesBehindPendingSources.load() == before, "a small copy queued behind the pipeline write into its source was recorded inline");
     std::fill(destination, destination + smallWords, 0u);
     const auto settled = AgcDriver::DriverDetail::copiesBehindPendingSources.load();
     submit(0, commands(memoryCopy(destination, produced, smallBytes), writeData(&settledCopied, 1)));

core/libs/prx/libSceAgcDriver/tests/TessellationPipelineCache.cpp

@@ -0,0 +1,31 @@
+#include "prx/libSceAgcDriver/Execution/include/TessellationPipelineCache.hpp"
+#include <cstdio>
+
+namespace {
+
+struct Case {
+    VkDriverId driver;
+    bool usable;
+};
+
+constexpr Case Cases[]{
+    {VK_DRIVER_ID_MOLTENVK, false},
+    {VK_DRIVER_ID_MESA_RADV, true},
+    {VK_DRIVER_ID_MESA_LLVMPIPE, true},
+    {VK_DRIVER_ID_NVIDIA_PROPRIETARY, true},
+    {VK_DRIVER_ID_AMD_PROPRIETARY, true},
+    {VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS, true},
+};
+
+}
+
+int main() {
+    int failures = 0;
+    for (const auto& item : Cases) {
+        if (AgcDriver::TessellationPipelineCacheUsable(item.driver) != item.usable) {
+            std::fprintf(stderr, "driver %d: expected tessellation pipelines %s the pipeline cache\n", static_cast<int>(item.driver), item.usable ? "to use" : "not to use");
+            ++failures;
+        }
+    }
+    return failures == 0 ? 0 : 1;
+}

core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp

@@ -1,4 +1,6 @@
 #include "GraphicsTests.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
@@ -424,26 +426,35 @@ void RunTextureTilingTests() {
         array.depthOrLastArray = 1536;
         array.baseArray = 1536;
         const auto layerBytes = DescribeSurface(array).layerBytes;
-        const auto window = ViewedArrayWindow(array);
+        const auto window = ViewedArrayWindow(array, false);
         Require(window.baseAddress == array.baseAddress + 1536ull * layerBytes && window.depthOrLastArray == 0 && window.baseArray == 0, "a sampled view of one slice at 1536 of an R_X array must start at that slice");
         Require(DescribeSurface(window).layerBytes == layerBytes && DescribeSurface(window).layers == 1, "a sampled array window must keep the slice layout");
 
         auto unaligned = array;
         unaligned.baseArray = 1530;
-        Require(ViewedArrayWindow(unaligned).baseAddress == array.baseAddress, "an R_X window at a slice the swizzle mixes in must keep the whole array");
+        const auto mixed = ViewedArrayWindow(unaligned, false);
+        Require(mixed.baseAddress == array.baseAddress + 1520ull * layerBytes && mixed.baseArray == 10 && mixed.depthOrLastArray == 16, "an R_X view at a slice the swizzle mixes in must start its window at the last slice before it with those bits clear");
+        auto low = array;
+        low.baseArray = 9;
+        Require(ViewedArrayWindow(low, false).baseAddress == array.baseAddress, "an R_X view below the first slice with those bits clear keeps the whole array");
 
         auto plain = array;
         plain.tileMode = TextureTileMode::kStandard64KB;
         plain.baseArray = 1530;
-        Require(ViewedArrayWindow(plain).baseAddress == array.baseAddress + 1530ull * DescribeSurface(plain).layerBytes && ViewedArrayWindow(plain).depthOrLastArray == 6, "a non-XOR window may start at any slice");
+        Require(ViewedArrayWindow(plain, false).baseAddress == array.baseAddress + 1530ull * DescribeSurface(plain).layerBytes && ViewedArrayWindow(plain, false).depthOrLastArray == 6, "a non-XOR window may start at any slice");
 
         auto fromZero = array;
         fromZero.baseArray = 0;
-        Require(ViewedArrayWindow(fromZero).baseAddress == array.baseAddress && ViewedArrayWindow(fromZero).depthOrLastArray == 1536, "a view from slice 0 keeps its descriptor");
+        Require(ViewedArrayWindow(fromZero, false).baseAddress == array.baseAddress && ViewedArrayWindow(fromZero, false).depthOrLastArray == 1536, "a view from slice 0 keeps its descriptor");
 
         auto compressed = array;
         compressed.dccAddress = 0x20000000ull;
-        Require(ViewedArrayWindow(compressed).baseAddress == array.baseAddress, "a view with DCC metadata keeps its descriptor");
+        Require(ViewedArrayWindow(compressed, false).baseAddress == array.baseAddress, "a view with DCC metadata keeps its descriptor");
+        const auto stored = ViewedArrayWindow(compressed, true);
+        Require(stored.baseAddress == window.baseAddress && stored.depthOrLastArray == 0 && stored.baseArray == 0 && stored.dccAddress == 0, "a view whose DCC keys read as stored is cut to its slices without metadata");
+        auto compressedFromZero = compressed;
+        compressedFromZero.baseArray = 0;
+        Require(ViewedArrayWindow(compressedFromZero, true).baseAddress == array.baseAddress && ViewedArrayWindow(compressedFromZero, true).dccAddress == compressed.dccAddress, "a view with DCC metadata from slice 0 keeps its descriptor");
 
         GuestTextureResource cube{};
         cube.baseAddress = 0x30000000ull;
@@ -455,13 +466,45 @@ void RunTextureTilingTests() {
         cube.tileMode = TextureTileMode::kStandard4KB;
         cube.depthOrLastArray = 7232;
         cube.baseArray = 7227;
-        const auto faces = ViewedArrayWindow(cube);
+        const auto faces = ViewedArrayWindow(cube, false);
         Require(faces.baseAddress == cube.baseAddress + 7227ull * DescribeSurface(cube).layerBytes && faces.depthOrLastArray == 5 && faces.baseArray == 0 && faces.dimension == TextureDimension::kCube, "a cube view of six faces at 7227 must start at its first face");
 
+        auto compressedVolume = cube;
+        compressedVolume.dimension = TextureDimension::k3D;
+        compressedVolume.baseArray = 0;
+        compressedVolume.dccAddress = 0x20000000ull;
+        Require(ViewedArrayWindow(compressedVolume, true).dccAddress == compressedVolume.dccAddress, "a volume with DCC metadata keeps its descriptor");
+
         auto volume = cube;
         volume.dimension = TextureDimension::k3D;
         volume.baseArray = 0;
-        Require(ViewedArrayWindow(volume).baseAddress == cube.baseAddress, "a volume keeps its descriptor");
+        Require(ViewedArrayWindow(volume, false).baseAddress == cube.baseAddress, "a volume keeps its descriptor");
+    }
+
+    {
+        const std::array<std::uint32_t, 8> words{0x027c0000u, 0xc1600000u, 0x00ffc0ffu, 0xd1800924u, 0x027cc27cu, 0xc1400000u, 0x00704018u, 0x91b00204u};
+        const auto view = DecodeTextureResource(words);
+        Require(view.baseAddress == 0x27c000000ull && view.dccAddress == 0x91b002040000ull && view.dimension == TextureDimension::k2DArray && view.baseArray == 636 && view.depthOrLastArray == 636 && view.width == 1024 && view.height == 1024 && view.format == 22, "the title's DCC array view decodes to slice 636 of 637");
+        const auto whole = DescribeSurface(view);
+        Require(whole.guestBytes == 2671771648ull, "the uncut DCC array view covers all 637 slices (" + std::to_string(whole.guestBytes) + " bytes)");
+        Require(TextureClearKeys(view, whole.guestBytes) == DccKeys::Uncompressed, "unreadable DCC keys of the uncut view read the texels as stored");
+        Require(ViewedArrayWindow(view, false).baseAddress == view.baseAddress, "a DCC view whose keys are not known to read as stored keeps its descriptor");
+        const auto window = ViewedArrayWindow(view, true);
+        const auto cut = DescribeSurface(window);
+        Require(window.baseAddress == view.baseAddress + whole.GuestLayerOffset(624) && window.dccAddress == 0 && window.baseArray == 12 && window.depthOrLastArray == 12, "the window of a stored-keys DCC view starts at the slice-swizzle boundary below its slice without metadata");
+        Require(cut.layers == 13 && cut.layerBytes == whole.layerBytes && window.baseAddress + cut.guestBytes == view.baseAddress + whole.guestBytes, "the window covers slices 624..636 (" + std::to_string(cut.guestBytes) + " bytes)");
+        for (std::uint32_t slice = 0; slice < cut.layers; ++slice) Require(window.baseAddress + cut.GuestLayerOffset(slice) == view.baseAddress + whole.GuestLayerOffset(624 + slice), "a window slice lies where the uncut surface keeps it");
+        const auto* equation = FindTextureSwizzleEquation(XorSwizzleMode(view.tileMode), 4);
+        const auto block = ThinBlockLayout(view.tileMode, 4);
+        Require(equation != nullptr && whole.mips.size() == 1 && !whole.mips[0].tail, "the title's DCC array is one untailed Z_X level");
+        const auto texel = [&](const GuestTextureResource& surface, const SurfaceGeometry& geometry, std::uint32_t x, std::uint32_t y, std::uint32_t slice) {
+            const auto& mip = geometry.mips[0];
+            return surface.baseAddress + geometry.GuestLayerOffset(slice) + mip.tiledOffset + (static_cast<std::uint64_t>(y / block[2]) * mip.blocksPerRow + x / block[1]) * block[0] + equationOffset(*equation, x, y, slice);
+        };
+        for (std::uint32_t y = 0; y < 1024; y += 7) {
+            for (std::uint32_t x = 0; x < 1024; x += 5) Require(texel(window, cut, x, y, window.baseArray) == texel(view, whole, x, y, view.baseArray), "a texel of the viewed slice detiles from the same guest byte in the window as in the uncut surface");
+        }
+        Require(TextureClearKeys(window, cut.guestBytes) == DccKeys::Uncompressed, "the window reads its texels as stored, as the uncut view does");
     }
 
     {

core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp

@@ -36,6 +36,7 @@ constexpr std::size_t Block = 65536;
 #ifdef _WIN32
 constexpr wchar_t HostHoleMode[] = L"--host-hole";
 constexpr wchar_t ResidentRemapMode[] = L"--resident-direct-remap";
+constexpr wchar_t RewatchImportPagesMode[] = L"--rewatch-import-pages";
 constexpr wchar_t RewatchedSharedMode[] = L"--rewatched-direct-shared";
 constexpr wchar_t ImportAliasSharedMode[] = L"--import-alias-shared";
 constexpr wchar_t SharedGenerationsMode[] = L"--compared-shared-generations";
@@ -712,13 +713,18 @@ void CheckRewatch() {
     const auto unwatched = UnwatchSerial();
     Require(!Watched(imported, Block) && UntrackedHint(imported, Block) != 0, "the unwatched block does not read as untracked");
     bytes[Block + 8] = 0x22;
+    const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> unaligned{{{imported + 1, imported + Block}}};
+    Rewatch(unaligned);
+    Require(!Watched(imported + 4096, Block - 4096) && UnwatchSerial() == unwatched, "a range that is not whole pages returned to the write watch");
     const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> partial{{{imported + 4096, imported + Block}}};
     Rewatch(partial);
-    Require(!Watched(imported, Block) && UnwatchSerial() == unwatched, "part of a block returned to the write watch");
+    Require(Watched(imported + 4096, Block - 4096) && !Watched(imported, 4096) && !Watched(imported, Block), "rewatching part of a block did not return exactly its pages to the write watch");
+    const auto partly = UnwatchSerial();
+    Require(partly != unwatched, "rewatching part of a block did not invalidate what was cached while it was unwatched");
     const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> first{{{imported, imported + Block}}};
     Rewatch(first);
     Require(Watched(imported, Block) && !Watched(imported + Block, Block), "rewatching did not return exactly its block to the write watch");
-    Require(UnwatchSerial() != unwatched, "rewatching did not invalidate what was cached while the block was unwatched");
+    Require(UnwatchSerial() != partly, "rewatching did not invalidate what was cached while the block was unwatched");
     Require(UntrackedHint(imported, Block) == 0, "a rewatched block still reads as untracked");
     Require(!UnchangedSince(imported, Block, before) && StoredOver(imported, Block, before), "a block returned to the write watch reads as unchanged since before it left");
     Require(!UnchangedSinceCollected(imported, Block, before), "a CPU store made while the block was unwatched reads as no CPU store");
@@ -740,6 +746,125 @@ void CheckRewatch() {
     GuestArena::GuestArenaRelease_nid_postfix(memory, 4 * Block);
 }
 
+using RewatchSpan = std::array<std::pair<std::uint64_t, std::uint64_t>, 1>;
+
+RewatchSpan BlockSpan(std::uint64_t address, std::size_t bytes) {
+    return {{{address & ~std::uint64_t{Block - 1}, (address + bytes + Block - 1) & ~std::uint64_t{Block - 1}}}};
+}
+
+RewatchSpan PageSpan(std::uint64_t address, std::size_t bytes) {
+    return {{{address & ~std::uint64_t{4095}, (address + bytes + 4095) & ~std::uint64_t{4095}}}};
+}
+
+void PrintPageState(const char* page, const void* address) {
+    MEMORY_BASIC_INFORMATION info{};
+    Require(VirtualQuery(address, &info, sizeof(info)) == sizeof(info), "cannot query a page next to an import");
+    std::cout << page << ": State 0x" << std::hex << info.State << " Type 0x" << info.Type << " Protect 0x" << info.Protect << std::dec << '\n';
+}
+
+struct ImportFixture {
+    std::uint8_t* memory;
+    std::size_t reserved;
+    ImportFixture(std::size_t reservedBytes, std::size_t committed) : memory(static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(reservedBytes, Block))), reserved(reservedBytes) {
+        Require(memory != nullptr, "cannot reserve an import fixture");
+        GuestArena::GuestArenaCommit_nid_postfix(memory, committed, PAGE_READWRITE, committed);
+        std::memset(memory, 0x11, committed);
+        BumpCollectEpoch();
+        Require(CollectWrites(Base(), committed) != 0, "an import fixture was not collected");
+    }
+    ImportFixture(const ImportFixture&) = delete;
+    ImportFixture& operator=(const ImportFixture&) = delete;
+    ~ImportFixture() {
+        GuestArena::GuestArenaReset_nid_postfix(memory, reserved);
+        GuestArena::GuestArenaRelease_nid_postfix(memory, reserved);
+    }
+    std::uint64_t Base() const {
+        return reinterpret_cast<std::uint64_t>(memory);
+    }
+};
+
+void UnwatchImport(std::uint64_t address, std::size_t bytes) {
+    Require(ImportWatched(address, bytes, [] { return true; }, false) && !Watched(address, bytes), "an import that leaves the watch stayed watched");
+}
+
+void RequireRewatchedStores(std::uint64_t address, std::size_t bytes) {
+    BumpCollectEpoch();
+    const auto collected = CollectWrites(address, bytes);
+    Require(collected != 0 && UnchangedSince(address, bytes, collected), "a rewatched import is not collected");
+    *reinterpret_cast<volatile std::uint8_t*>(address + bytes - 1) = 0x22;
+    CollectWritesUncached(address, bytes);
+    Require(!UnchangedSince(address + bytes - 1, 1, collected), "a CPU store into a rewatched import's last page was missed");
+}
+
+void CheckRewatchWholeBlockImport() {
+    constexpr std::size_t bytes = 4 * Block;
+    ImportFixture fixture(6 * Block, 6 * Block);
+    const auto import = fixture.Base() + Block;
+    const auto before = CollectWrites(fixture.Base(), 6 * Block);
+    UnwatchImport(import, bytes);
+    Require(PageSpan(import, bytes) == BlockSpan(import, bytes), "a whole-block import has a page span unlike its block span");
+    Rewatch(PageSpan(import, bytes));
+    Require(Watched(import, bytes), "a settled whole-block import did not return to the write watch");
+    Require(!UnchangedSince(import, bytes, before) && UnchangedSince(fixture.Base(), Block, before) && UnchangedSince(import + bytes, Block, before), "rewatching a whole-block import did not stamp exactly its blocks");
+    RequireRewatchedStores(import, bytes);
+}
+
+void CheckRewatchImportTail() {
+    constexpr std::size_t bytes = 0xc4000;
+    ImportFixture fixture(13 * Block, bytes);
+    const auto import = fixture.Base();
+    UnwatchImport(import, bytes);
+    PrintPageState("page past a 0xc4000-byte import", fixture.memory + bytes);
+    const auto unwatched = UnwatchSerial();
+    Rewatch(BlockSpan(import, bytes));
+    Require(!Watched(import, bytes) && UnwatchSerial() == unwatched, "the 64 KiB span past the import's committed pages returned to the write watch");
+    Rewatch(PageSpan(import, bytes));
+    Require(Watched(import, bytes), "a settled import whose size is not a whole block did not return to the write watch");
+    Require(Watched(import + 12 * Block, bytes - 12 * Block) && !Watched(import, bytes + 1) && !Watched(import + bytes - 1, 2) && !Watched(import + bytes, 4096), "rewatching an import ending inside a block did not return exactly its pages");
+    Require(UnwatchSerial() != unwatched && UntrackedHint(import, bytes) == 0, "a rewatched import still reads as unwatched");
+    RequireRewatchedStores(import, bytes);
+}
+
+void CheckRewatchUnalignedImport() {
+    constexpr std::size_t offset = 0x1000;
+    constexpr std::size_t bytes = 0xc4000;
+    constexpr std::size_t committed = 0xc8000;
+    ImportFixture fixture(13 * Block, committed);
+    const auto import = fixture.Base() + offset;
+    UnwatchImport(import, bytes);
+    PrintPageState("page past an import at block offset 0x1000", fixture.memory + committed);
+    Rewatch(BlockSpan(import, bytes));
+    Require(!Watched(import, bytes), "the 64 KiB span past the committed pages around an import returned to the write watch");
+    Rewatch(PageSpan(import, bytes));
+    Require(Watched(import, bytes), "a settled import that does not start on a block did not return to the write watch");
+    Require(!Watched(fixture.Base(), offset) && !Watched(import + bytes, 4096), "rewatching an import returned pages outside it");
+    RequireRewatchedStores(import, bytes);
+}
+
+void CheckRewatchImportBesideUnwatchedImport() {
+    constexpr std::size_t bytes = 0xc4000;
+    constexpr std::size_t otherBytes = 0x1c000;
+    ImportFixture fixture(14 * Block, 14 * Block);
+    const auto import = fixture.Base();
+    const auto other = import + bytes;
+    UnwatchImport(import, bytes);
+    UnwatchImport(other, otherBytes);
+    Rewatch(PageSpan(import, bytes));
+    Require(Watched(import, bytes), "an import sharing its last block with another unwatched import did not return to the write watch");
+    Require(!Watched(other, 4096) && !Watched(import + 13 * Block - 4096, 4096) && !Watched(other + otherBytes - 4096, 4096), "rewatching an import returned another unwatched import's pages in its last block");
+    Require(UntrackedHint(other, otherBytes) != 0, "an unwatched import beside a rewatched one does not read as untracked");
+    RequireRewatchedStores(import, bytes);
+    Rewatch(PageSpan(other, otherBytes));
+    Require(Watched(import, bytes + otherBytes), "rewatching the second import did not return both to the write watch");
+}
+
+void CheckRewatchImportPages() {
+    CheckRewatchWholeBlockImport();
+    CheckRewatchImportTail();
+    CheckRewatchUnalignedImport();
+    CheckRewatchImportBesideUnwatchedImport();
+}
+
 void CheckRewatchedDirectMemoryComparedWhenShared() {
     constexpr std::size_t bytes = 2 * Block;
     auto* memory = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(2 * bytes, Block));
@@ -1381,7 +1506,7 @@ void CheckImportWindow() {
         bytes[0x100] = 0x11;
         bytes[Block + 0x2000] = 0x22;
         return true;
-    }), "the import window import failed");
+    }, true, false), "the import window import failed");
     Require(Watched(base, 3 * Block), "an import window left its range unwatched");
     const std::array<std::uint64_t, 3> generations{synced, synced, synced};
     std::array<std::uint8_t, 3> changed{};
@@ -1773,6 +1898,10 @@ int main() {
             CheckResidentDirectMappingReuse();
             return 0;
         }
+        if (std::wstring_view(GetCommandLineW()).ends_with(RewatchImportPagesMode)) {
+            CheckRewatchImportPages();
+            return 0;
+        }
         if (std::wstring_view(GetCommandLineW()).ends_with(RewatchedSharedMode)) {
             CheckRewatchedDirectMemoryComparedWhenShared();
             CheckImportedDirectMemoryComparedWhenShared();

core/libs/prx/libSceAgcDriver/tests/WriteWatchCoverage.cpp

@@ -99,6 +99,17 @@ int main() {
         coverage.Exclude(base + block, block, 43);
         require(coverage.Restore(base + block - page, 2 * page, 42), "a newer boundary page was not restored");
         require(coverage.Covers(base + block - page, page) && !coverage.Covers(base + block, page), "a stale boundary mapping erased a newer neighbouring exclusion");
+        constexpr std::size_t tail = 0xc4000;
+        coverage.Initialize(base, 14 * block);
+        coverage.Exclude(base, tail, 50);
+        require(coverage.Restore(base, tail, 51), "an import ending inside a block did not restore its pages");
+        require(coverage.Covers(base, tail) && coverage.Covers(base + 12 * block, tail - 12 * block), "an import ending inside a block is not watched");
+        require(!coverage.Covers(base, tail + 1) && !coverage.Covers(base + tail - 1, 2) && !coverage.Covers(base + tail, page) && !coverage.Covers(base, 13 * block), "a range past an import ending inside a block is watched");
+        require(coverage.Covers(base + tail - page, page) && !coverage.Covers(base + 13 * block - page, page) && coverage.Covers(base + 13 * block, block), "an import ending inside a block changed the pages after it");
+        coverage.Initialize(base, 14 * block);
+        coverage.Exclude(base + page, tail, 60);
+        require(coverage.Restore(base + page, tail, 61) && coverage.Covers(base + page, tail), "an import starting and ending inside blocks is not watched");
+        require(!coverage.Covers(base, page) && !coverage.Covers(base, tail) && !coverage.Covers(base + page + tail, page) && !coverage.Covers(base + page, tail + 1), "an import starting and ending inside blocks watched pages outside it");
         std::cout << "write watch coverage passed\n";
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/execution/AddressPasses.cpp

@@ -141,6 +141,7 @@ struct Step {
     const std::uint32_t* pixel;
     std::uint64_t pointer;
     std::array<std::uint32_t, 4> descriptor{};
+    std::uint64_t color = 0;
 };
 
 void Submit(const GuestBlock& vertices, const GuestBlock& target, std::initializer_list<Step> steps, bool linear = false) {
@@ -179,6 +180,10 @@ void Submit(const GuestBlock& vertices, const GuestBlock& target, std::initializ
         Append(commands, 0x76, {0x008, static_cast<std::uint32_t>(pixel >> 8u), static_cast<std::uint32_t>(pixel >> 40u)});
         Append(commands, 0x76, {0x00e, static_cast<std::uint32_t>(step.pointer), static_cast<std::uint32_t>(step.pointer >> 32u)});
         Append(commands, 0x76, {0x010, step.descriptor[0], step.descriptor[1], step.descriptor[2], step.descriptor[3]});
+        if (step.color != 0) {
+            Append(commands, 0x69, {0x318, static_cast<std::uint32_t>(step.color >> 8u)});
+            Append(commands, 0x69, {0x390, static_cast<std::uint32_t>(step.color >> 40u)});
+        }
         Append(commands, 0x2d, {3, 2});
     }
     Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
@@ -285,6 +290,26 @@ void TargetReadAsWhenSplit(const GuestBlock& vertices, const GuestBlock& target,
     if (!linear && !split) Expect(after.passesContinued - before.passesContinued == 1, what + ": the reading draw did not continue the rendering draw's pass, so the case does not test it: " + Counts(before, after));
 }
 
+void TargetReadSynchronously(const GuestBlock& vertices, const GuestBlock& target, const GuestBlock& unmapped) {
+    Submit(vertices, target, {{PixelRead.data(), target.At(ColorOffset)}}, true);
+    std::memset(target.Bytes(ColorOffset), 0, ColorBytes);
+    std::memset(unmapped.Bytes(ColorOffset), 0, ColorBytes);
+    const auto before = AddressDraws();
+    const auto otherBefore = OtherDraws();
+    Submit(vertices, target, {{PixelColorWord.data(), 0}, {PixelRead.data(), target.At(ColorOffset), {}, unmapped.At(ColorOffset)}}, true);
+    const auto after = AddressDraws();
+    const auto otherAfter = OtherDraws();
+    const unsigned red = unmapped.Bytes(ColorOffset)[0];
+    std::printf("a synchronous draw reading through the table the linear target the previous draw rendered read %s (red byte %u); recorded: %s, %s\n", red == 255u ? "that draw's color" : "the bytes before it", red, Counts(before, after).c_str(), Counts(otherBefore, otherAfter, "other draws").c_str());
+    if (otherAfter.draws == otherBefore.draws) {
+        std::puts("a synchronous draw reading through the table the linear target the previous draw rendered: skipped, that draw was not recorded because the linear target is not resident on this device, so the target was stored right after it");
+        return;
+    }
+    Expect(otherAfter.draws - otherBefore.draws == 1, "the draw rendering the linear target was not recorded as one draw: " + Counts(otherBefore, otherAfter, "other draws"));
+    Expect(after.draws == before.draws, "the draw reading through the table into an unregistered target was recorded, so the case does not test a synchronous draw: " + Counts(before, after));
+    Expect(red == 255u, "a synchronous draw reading through the table the linear target the previous draw rendered read the bytes before it (red byte " + std::to_string(red) + "), not that draw's color");
+}
+
 }
 
 int main(int argc, char** argv) {
@@ -306,6 +331,7 @@ int main(int argc, char** argv) {
         GuestBlock readOnlyVertices;
         GuestBlock data;
         GuestBlock target;
+        GuestBlock unmapped;
         FillVertices(readOnlyVertices);
         readOnlyVertices.Map(false);
         data.Map(true);
@@ -320,6 +346,7 @@ int main(int argc, char** argv) {
             TargetReadAsWhenSplit(readOnlyVertices, target, false);
             TargetReadAsWhenSplit(readOnlyVertices, target, true);
         }
+        if (only.empty() || only == "--synchronous-target-read") TargetReadSynchronously(readOnlyVertices, target, unmapped);
         AgcDriverShutdown_nid_postfix();
         std::puts("address passes: draws reading through the BDA table share a render pass, and a store through it still begins one");
         return 0;

core/libs/prx/libSceAgcDriver/tests/execution/DccKeysFirstWrite.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/QueueState.hpp"
 #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
@@ -7,7 +8,9 @@
 #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
 #include "prx/libc/include/GuestArena.hpp"
 #include "prx/libc/include/GuestWriteWatch.hpp"
@@ -826,6 +829,45 @@ void RunSharedImport(AgcDriver::VulkanDevice& device, std::uint8_t* block) {
 
 }
 
+void RunMipmappedTarget(AgcDriver::VulkanDevice& device, std::uint8_t* block) {
+#ifndef _WIN32
+    constexpr std::uint32_t ChainSide = 64;
+    constexpr std::uint32_t ChainLevels = 7;
+    auto* texels = block;
+    auto* keys = block + KeysOffset;
+    const auto surface = AddressOf(texels);
+    const auto meta = AddressOf(keys);
+    std::memset(texels, 0x55, SurfaceBytes);
+    std::memset(keys, 0x00, KeyBytes);
+    AgcDriver::GuestMemory::BumpCollectEpoch();
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(surface >> 8u);
+    cx[0x390] = static_cast<std::uint32_t>(surface >> 40u);
+    cx[0x31b] = 0;
+    cx[0x31c] = 0x10028028u;
+    cx[0x31d] = 0;
+    cx[0x3b0] = ((ChainLevels - 1u) << 28u) | ((ChainSide - 1u) << 14u) | (ChainSide - 1u);
+    cx[0x3b8] = 0x9000000u | (TileRenderTarget64KB << 14u) | (1u << 30u);
+    cx[0x325] = static_cast<std::uint32_t>(meta >> 8u);
+    cx[0x3a8] = static_cast<std::uint32_t>(meta >> 40u);
+    const auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    const auto resource = AgcDriver::Graphics::SurfaceForTarget(color);
+    Require(color.mipCount == ChainLevels && color.dccAddress == 0 && resource.dccAddress == meta, "the resident surface of a mipmapped DCC color target has no keys");
+    const auto bytes = AgcDriver::Graphics::DescribeSurface(resource).guestBytes;
+    Require(bytes <= SurfaceBytes, "the mip chain does not fit the test surface");
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    const auto image = AgcDriver::Graphics::CachedStorageSurface(context, resource);
+    Require(image->UploadedKeys() == DccKeys::Clear0000, "the resident image of a 0000-cleared mipmapped target was not cleared from its keys");
+    image->MarkDirty();
+    Require(AgcDriver::Graphics::CurrentDccKeys(meta, bytes) == DccKeys::Uncompressed, "the first write into a mipmapped DCC target left its keys cleared");
+    Require(AgcDriver::Graphics::TextureClearKeys(resource, bytes) == DccKeys::Uncompressed, "a sampled view of a rendered mipmapped DCC target still reads its clear");
+#else
+    static_cast<void>(device);
+    static_cast<void>(block);
+#endif
+}
+
 int main() {
     try {
 #ifdef _WIN32
@@ -866,6 +908,9 @@ int main() {
         std::optional<GuestBlock> uniform;
         uniform.emplace(false);
         RunUniformKeysBehindKernel(*device, uniform->Data());
+        std::optional<GuestBlock> chain;
+        chain.emplace(false);
+        RunMipmappedTarget(*device, chain->Data());
 #ifdef _WIN32
         shared.emplace(true, true);
         RunSharedImport(*device, shared->Data());

core/libs/prx/libSceAgcDriver/tests/execution/DepthPlaneGuestLayers.cpp

@@ -223,6 +223,65 @@ private:
     std::uint8_t* block = nullptr;
 };
 
+class SparseLayers {
+public:
+    SparseLayers() {
+#ifdef _WIN32
+        memory = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocateAtOrAbove_nid_postfix(GuestArena::GuestArenaHighestFree_nid_no_patch(BlockBytes, 65536), BlockBytes, 65536));
+#else
+        constexpr std::uintptr_t alignment = 65536;
+        void* raw = mmap(nullptr, BlockBytes + alignment, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        Require(raw != MAP_FAILED, "cannot reserve sparse depth layers");
+        const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+        const auto aligned = (begin + alignment - 1) & ~(alignment - 1);
+        if (aligned != begin) munmap(raw, aligned - begin);
+        if (aligned + BlockBytes != begin + BlockBytes + alignment) munmap(reinterpret_cast<void*>(aligned + BlockBytes), begin + alignment - aligned);
+        memory = reinterpret_cast<std::uint8_t*>(aligned);
+#endif
+        Require(memory != nullptr, "cannot reserve sparse depth layers");
+    }
+
+    ~SparseLayers() {
+#ifdef _WIN32
+        GuestArena::GuestArenaReset_nid_postfix(memory, BlockBytes);
+        GuestAllocations::GuestAllocationsInvalidate_nid_postfix(Address(), BlockBytes);
+        GuestArena::GuestArenaRelease_nid_postfix(memory, BlockBytes);
+#else
+        GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(memory, BlockBytes);
+        munmap(memory, BlockBytes);
+        GuestAllocations::GuestAllocationsInvalidate_nid_postfix(Address(), BlockBytes);
+#endif
+    }
+
+    void Commit(std::size_t offset, std::size_t bytes) {
+#ifdef _WIN32
+        GuestArena::GuestArenaCommit_nid_postfix(memory + offset, bytes, PAGE_READWRITE, 65536);
+#else
+        Require(mmap(memory + offset, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) != MAP_FAILED, "cannot commit sparse depth layer pages");
+        GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(memory + offset, bytes);
+#endif
+        GuestAllocations::GuestAllocationsInvalidate_nid_postfix(Address() + offset, bytes);
+    }
+
+    void DecommitGuestLayer() {
+#ifdef _WIN32
+        GuestArena::GuestArenaReset_nid_postfix(memory + LayerBytes, LayerBytes);
+#else
+        GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(memory + LayerBytes, LayerBytes);
+        Require(mmap(memory + LayerBytes, LayerBytes, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) != MAP_FAILED, "cannot decommit sparse depth layer pages");
+#endif
+        GuestAllocations::GuestAllocationsInvalidate_nid_postfix(Address() + LayerBytes, LayerBytes);
+    }
+
+    std::uint8_t* Data() const { return memory; }
+    std::uint64_t Address() const { return reinterpret_cast<std::uint64_t>(memory); }
+    SparseLayers(const SparseLayers&) = delete;
+    SparseLayers& operator=(const SparseLayers&) = delete;
+
+private:
+    std::uint8_t* memory = nullptr;
+};
+
 std::atomic<std::uint64_t> watchedBegin{0};
 std::atomic<std::uint64_t> watchedEnd{0};
 std::atomic<std::uint32_t> watchedReads{0};
@@ -308,6 +367,59 @@ void Run(const Context& context, const WatchedBlock& block) {
     AgcDriver::GuestMemory::SetFlushHook(nullptr);
 }
 
+void SparseGuestLayers(const Context& context) {
+    ClearDepthSurfaces(context.device);
+    SparseLayers block;
+    block.Commit(0, LayerBytes);
+    FillLayer(block.Data(), 1.0f);
+    const auto address = block.Address();
+    DepthSurfaceView(context, DepthTarget{address, 0, {Side, Side}, VK_FORMAT_D32_SFLOAT, 1.0f, 0, 0});
+    const auto resource = ArrayOver(address);
+    watchedBegin = address + LayerBytes;
+    watchedEnd = address + BlockBytes;
+    AgcDriver::GuestMemory::SetFlushHook(&CountReads);
+    const auto sample = [&](const char* when, bool changed) {
+        AgcDriver::GuestMemory::BumpCollectEpoch();
+        watchedReads = 0;
+        Require(DepthSurfaceTexture(context, words, resource, components) != nullptr, "sparse depth layers were not sampled");
+        Require((watchedReads.load() != 0) == changed, std::string("sparse depth layer read mismatch ") + when);
+    };
+    try {
+        sample("at first use", true);
+        sample("with the guest layer still uncommitted", false);
+        block.Commit(LayerBytes, 65536);
+        const float value = 0.25f;
+        std::memcpy(block.Data() + LayerBytes, &value, sizeof(value));
+        sample("after committing part of the guest layer", true);
+        block.Data()[LayerBytes + 64] = 1;
+        sample("after a CPU write to the committed part", true);
+        AgcDriver::GuestMemory::MarkWritten(address + LayerBytes + 128, 4);
+        sample("after a driver store to the committed part", true);
+        block.DecommitGuestLayer();
+        sample("after decommitting the guest layer", true);
+        sample("with the guest layer still decommitted", false);
+        block.Commit(LayerBytes, 65536);
+        sample("after committing the same pages again", true);
+        block.DecommitGuestLayer();
+        block.Commit(LayerBytes, 65536);
+        sample("after replacing a mapping with the same committed range", true);
+        block.DecommitGuestLayer();
+        block.Commit(LayerBytes, LayerBytes);
+        FillLayer(block.Data() + LayerBytes, 0.5f);
+        sample("after fully committing the guest layer", true);
+        sample("with the fully committed layer unchanged", false);
+        block.DecommitGuestLayer();
+        sample("after decommitting a fully committed guest layer", true);
+        sample("with the fully committed layer now empty", false);
+    } catch (...) {
+        AgcDriver::GuestMemory::SetFlushHook(nullptr);
+        ClearDepthSurfaces(context.device);
+        throw;
+    }
+    AgcDriver::GuestMemory::SetFlushHook(nullptr);
+    ClearDepthSurfaces(context.device);
+}
+
 void RecordedUploads(const Context& context, const WatchedBlock& block) {
     const auto resource = ArrayOver(block.Address());
     Recorder recorder(context);
@@ -418,6 +530,7 @@ int main() {
             RecordedUploads(context, block);
             RunCubeGather(context, block);
             RunCubeTiledSlice(context, block);
+            SparseGuestLayers(context);
         }
         device.reset();
         std::puts("depth plane guest layer tests passed");

core/libs/prx/libSceAgcDriver/tests/execution/DeviceQueuePlan.cpp

@@ -34,16 +34,40 @@ VkQueueFamilyProperties family(VkQueueFlags flags, std::uint32_t queues) {
 }
 
 std::string describe(const AgcDriver::DeviceQueuePlan& plan) {
-    return "main " + std::to_string(plan.mainCount) + " queues, settle " + (plan.settle ? std::to_string(plan.settleFamily) + ":" + std::to_string(plan.settleIndex) : std::string("none")) + ", sparse " + (!plan.sparseBinding ? std::string("none") : plan.sparseFamily == Own ? std::string("on the main queue") : std::to_string(plan.sparseFamily) + ":" + std::to_string(plan.sparseIndex));
+    return "main " + std::to_string(plan.mainCount) + " queues, present " + (plan.present ? std::to_string(plan.presentIndex) : std::string("none")) + ", settle " + (plan.settle ? std::to_string(plan.settleFamily) + ":" + std::to_string(plan.settleIndex) : std::string("none")) + ", sparse " + (!plan.sparseBinding ? std::string("none") : plan.sparseFamily == Own ? std::string("on the main queue") : std::to_string(plan.sparseFamily) + ":" + std::to_string(plan.sparseIndex));
 }
 
 void check(const Case& test) {
-    const auto plan = AgcDriver::PlanDeviceQueues(test.families, 0, test.settles, true);
-    const auto dense = AgcDriver::PlanDeviceQueues(test.families, 0, test.settles, false);
+    const auto plan = AgcDriver::PlanDeviceQueues(test.families, 0, test.settles, true, false);
+    const auto dense = AgcDriver::PlanDeviceQueues(test.families, 0, test.settles, false, false);
     Require(plan.mainCount == dense.mainCount && plan.settle == dense.settle && plan.settleFamily == dense.settleFamily && plan.settleIndex == dense.settleIndex, std::string("device queue plan, ") + test.name + ": sparse binding changed the queues: " + describe(dense) + " without it, " + describe(plan) + " with it");
     Require(!dense.sparseBinding && dense.sparseFamily == Own, std::string("device queue plan, ") + test.name + ": a device without sparse binding got a sparse queue: " + describe(dense));
     const bool expected = plan.mainCount == test.mainCount && plan.settle == test.settle && (!test.settle || (plan.settleFamily == test.settleFamily && plan.settleIndex == test.settleIndex)) && plan.sparseBinding == test.sparseBinding && plan.sparseFamily == test.sparseFamily && (test.sparseFamily == Own || plan.sparseIndex == test.sparseIndex);
-    Require(expected, std::string("device queue plan, ") + test.name + ": got " + describe(plan));
+    Require(expected && !plan.present, std::string("device queue plan, ") + test.name + ": got " + describe(plan));
+    const auto windowed = AgcDriver::PlanDeviceQueues(test.families, 0, test.settles, true, true);
+    const bool kept = windowed.settle == plan.settle && windowed.settleFamily == plan.settleFamily && windowed.settleIndex == plan.settleIndex && windowed.sparseBinding == plan.sparseBinding && windowed.sparseFamily == plan.sparseFamily && windowed.sparseIndex == plan.sparseIndex;
+    const bool counted = windowed.present ? windowed.presentIndex == plan.mainCount && windowed.mainCount == plan.mainCount + 1 : windowed.mainCount == plan.mainCount && plan.mainCount >= test.families[0].queueCount;
+    Require(kept && counted, std::string("device queue plan, ") + test.name + ": a window changed more than the presentation queue: " + describe(plan) + " without it, " + describe(windowed) + " with it");
+}
+
+struct WindowCase {
+    const char* name;
+    std::vector<VkQueueFamilyProperties> families;
+    bool settles;
+    std::uint32_t mainCount;
+    bool present;
+    std::uint32_t presentIndex;
+    bool settle;
+    std::uint32_t settleFamily;
+    std::uint32_t settleIndex;
+    std::uint32_t sparseFamily;
+    std::uint32_t sparseIndex;
+};
+
+void check(const WindowCase& test) {
+    const auto plan = AgcDriver::PlanDeviceQueues(test.families, 0, test.settles, true, true);
+    const bool expected = plan.mainCount == test.mainCount && plan.present == test.present && (!test.present || plan.presentIndex == test.presentIndex) && plan.settle == test.settle && (!test.settle || (plan.settleFamily == test.settleFamily && plan.settleIndex == test.settleIndex)) && plan.sparseFamily == test.sparseFamily && (test.sparseFamily == Own || plan.sparseIndex == test.sparseIndex);
+    Require(expected, std::string("device queue plan with a window, ") + test.name + ": got " + describe(plan));
 }
 
 }
@@ -62,6 +86,15 @@ int main() {
             {"only the settle queue binds", {family(Graphics, 1), family(VK_QUEUE_TRANSFER_BIT | Sparse, 1)}, true, 1, true, 1, 0, false, Own, 0},
         };
         for (const auto& test : cases) check(test);
+        const std::vector<WindowCase> windowCases{
+            {"one queue with a window", {family(Graphics | Sparse, 1)}, true, 1, false, 0, false, 0, 0, Own, 0},
+            {"two queues, no settles, with a window", {family(Graphics, 2)}, false, 2, true, 1, false, 0, 0, Own, 0},
+            {"two queues taken by the main and settle queues, with a window", {family(Graphics, 2)}, true, 2, false, 0, true, 0, 1, Own, 0},
+            {"settle on the main family, three queues, with a window", {family(Graphics | Sparse, 3), family(VK_QUEUE_TRANSFER_BIT | Sparse, 1)}, true, 3, true, 2, true, 0, 1, 1, 0},
+            {"one main queue and settle on another family, with a window", {family(Graphics, 1), family(VK_QUEUE_TRANSFER_BIT, 1)}, true, 1, false, 0, true, 1, 0, Own, 0},
+            {"sparse-only family preferred, with a window", {family(Graphics | Sparse, 16), family(VK_QUEUE_TRANSFER_BIT | Sparse, 2), family(Sparse, 1)}, true, 3, true, 2, true, 0, 1, 2, 0},
+        };
+        for (const auto& test : windowCases) check(test);
         std::puts("device queue plan tests passed");
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/execution/F32ConvertDenormalModes.cpp

@@ -0,0 +1,212 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 64;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 16;
+constexpr std::uint32_t Columns = 12;
+constexpr std::uint32_t Float64Low = 5;
+constexpr std::uint32_t Float64High = 6;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 25> Code{
+    0x34020084u, 0x34040086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e141104u, 0x7e160f04u, 0x7e181904u,
+    0x7e1a1b04u, 0x7e1c7f04u, 0xd7680011u, 0x00020b04u, 0xd7690012u, 0x00020b04u, 0xd55e0013u, 0x02010104u,
+    0x5e280b04u, 0x7e2a1504u, 0xe0781000u, 0x80010a02u, 0xe0781010u, 0x80010e02u, 0xe0781020u, 0x80011202u,
+    0xbf810000u,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 13> Float64Code{
+    0x34020084u, 0x34040086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e1e2104u, 0xe0781000u, 0x80010a02u,
+    0xe0781010u, 0x80010e02u, 0xe0781020u, 0x80011202u, 0xbf810000u,
+};
+
+constexpr const char* Names[Columns] = {
+    "v_cvt_i32_f32 v10, v4",
+    "v_cvt_u32_f32 v11, v4",
+    "v_cvt_rpi_i32_f32 v12, v4",
+    "v_cvt_flr_i32_f32 v13, v4",
+    "v_frexp_exp_i32_f32 v14, v4",
+    "v_cvt_f64_f32 v[15:16], v4 (low)",
+    "v_cvt_f64_f32 v[15:16], v4 (high)",
+    "v_cvt_pknorm_i16_f32 v17, v4, v5",
+    "v_cvt_pknorm_u16_f32 v18, v4, v5",
+    "v_cvt_pk_u8_f32 v19, v4, 0, 0",
+    "v_cvt_pkrtz_f16_f32 v20, v4, v5",
+    "v_cvt_f16_f32 v21, v4",
+};
+
+struct ConvertVector {
+    std::uint32_t operands[2];
+    std::uint32_t denormalsFlushed[Columns];
+    std::uint32_t denormalsKept[Columns];
+};
+
+constexpr std::array<ConvertVector, 32> Vectors{{
+    {{0x00000001u, 0x00400000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffff6cu, 0x00000000u, 0x36a00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}},
+    {{0x007fffffu, 0x80000001u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffff82u, 0xc0000000u, 0x380fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u}},
+    {{0x00400000u, 0x3f800000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fff0000u, 0xffff0000u, 0x00000000u, 0x3c000000u, 0x00000000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffff82u, 0x00000000u, 0x38000000u, 0x7fff0000u, 0xffff0000u, 0x00000000u, 0x3c000000u, 0x00000000u}},
+    {{0x80000001u, 0x007fffffu}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0xffffff6cu, 0x00000000u, 0xb6a00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u}},
+    {{0x807fffffu, 0x00000001u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0xffffff82u, 0xc0000000u, 0xb80fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u}},
+    {{0x80400000u, 0xbf000000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0xc0000000u, 0x00000000u, 0x00000000u, 0xb8008000u, 0x00008000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0xffffff82u, 0x00000000u, 0xb8000000u, 0xc0000000u, 0x00000000u, 0x00000000u, 0xb8008000u, 0x00008000u}},
+    {{0x00800000u, 0x80400000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffff83u, 0x00000000u, 0x38100000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffff83u, 0x00000000u, 0x38100000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u}},
+    {{0x80800000u, 0x00000000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0xffffff83u, 0x00000000u, 0xb8100000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0xffffff83u, 0x00000000u, 0xb8100000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u}},
+    {{0x3f000000u, 0x00000001u}, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3fe00000u, 0x00004000u, 0x00008000u, 0x00000000u, 0x00003800u, 0x00003800u}, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3fe00000u, 0x00004000u, 0x00008000u, 0x00000000u, 0x00003800u, 0x00003800u}},
+    {{0xbfc00000u, 0x807fffffu}, {0xffffffffu, 0x00000000u, 0xffffffffu, 0xfffffffeu, 0x00000001u, 0x00000000u, 0xbff80000u, 0x00008001u, 0x00000000u, 0x00000000u, 0x8000be00u, 0x0000be00u}, {0xffffffffu, 0x00000000u, 0xffffffffu, 0xfffffffeu, 0x00000001u, 0x00000000u, 0xbff80000u, 0x00008001u, 0x00000000u, 0x00000000u, 0x8000be00u, 0x0000be00u}},
+    {{0x3f800000u, 0x80000002u}, {0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000000u, 0x3ff00000u, 0x00007fffu, 0x0000ffffu, 0x00000001u, 0x80003c00u, 0x00003c00u}, {0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000000u, 0x3ff00000u, 0x00007fffu, 0x0000ffffu, 0x00000001u, 0x80003c00u, 0x00003c00u}},
+    {{0x00000000u, 0x00000002u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}},
+    {{0x80000000u, 0x3f800000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x7fff0000u, 0xffff0000u, 0x00000000u, 0x3c008000u, 0x00008000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x7fff0000u, 0xffff0000u, 0x00000000u, 0x3c008000u, 0x00008000u}},
+    {{0x33800000u, 0x00000001u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffffe9u, 0x00000000u, 0x3e700000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffffe9u, 0x00000000u, 0x3e700000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000001u}},
+    {{0x477fe000u, 0x80000001u}, {0x0000ffe0u, 0x0000ffe0u, 0x0000ffe0u, 0x0000ffe0u, 0x00000010u, 0x00000000u, 0x40effc00u, 0x00007fffu, 0x0000ffffu, 0x000000ffu, 0x80007bffu, 0x00007bffu}, {0x0000ffe0u, 0x0000ffe0u, 0x0000ffe0u, 0x0000ffe0u, 0x00000010u, 0x00000000u, 0x40effc00u, 0x00007fffu, 0x0000ffffu, 0x000000ffu, 0x80007bffu, 0x00007bffu}},
+    {{0x501502f9u, 0x00800000u}, {0x7fffffffu, 0xffffffffu, 0x7fffffffu, 0x7fffffffu, 0x00000022u, 0x20000000u, 0x4202a05fu, 0x00007fffu, 0x0000ffffu, 0x000000ffu, 0x00007bffu, 0x00007c00u}, {0x7fffffffu, 0xffffffffu, 0x7fffffffu, 0x7fffffffu, 0x00000022u, 0x20000000u, 0x4202a05fu, 0x00007fffu, 0x0000ffffu, 0x000000ffu, 0x00007bffu, 0x00007c00u}},
+    {{0xd01502f9u, 0x007fffffu}, {0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000022u, 0x20000000u, 0xc202a05fu, 0x00008001u, 0x00000000u, 0x00000000u, 0x0000fbffu, 0x0000fc00u}, {0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000022u, 0x20000000u, 0xc202a05fu, 0x00008001u, 0x00000000u, 0x00000000u, 0x0000fbffu, 0x0000fc00u}},
+    {{0x7f800000u, 0xbfc00000u}, {0x7fffffffu, 0xffffffffu, 0x7fffffffu, 0x7fffffffu, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x80017fffu, 0x0000ffffu, 0x000000ffu, 0xbe007c00u, 0x00007c00u}, {0x7fffffffu, 0xffffffffu, 0x7fffffffu, 0x7fffffffu, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x80017fffu, 0x0000ffffu, 0x000000ffu, 0xbe007c00u, 0x00007c00u}},
+    {{0xff800000u, 0x00000001u}, {0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0xfff00000u, 0x00008001u, 0x00000000u, 0x00000000u, 0x0000fc00u, 0x0000fc00u}, {0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0xfff00000u, 0x00008001u, 0x00000000u, 0x00000000u, 0x0000fc00u, 0x0000fc00u}},
+    {{0x7fc00000u, 0x80400000u}, {0x00000000u, 0x00000000u, 0x7fffffffu, 0x7fffffffu, 0x00000000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80007e00u, 0x00007e00u}, {0x00000000u, 0x00000000u, 0x7fffffffu, 0x7fffffffu, 0x00000000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80007e00u, 0x00007e00u}},
+    {{0xffc00000u, 0x00400000u}, {0x00000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x0000fe00u, 0x0000fe00u}, {0x00000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x0000fe00u, 0x0000fe00u}},
+    {{0x4f800000u, 0x80000000u}, {0x7fffffffu, 0xffffffffu, 0x7fffffffu, 0x7fffffffu, 0x00000021u, 0x00000000u, 0x41f00000u, 0x00007fffu, 0x0000ffffu, 0x000000ffu, 0x80007bffu, 0x00007c00u}, {0x7fffffffu, 0xffffffffu, 0x7fffffffu, 0x7fffffffu, 0x00000021u, 0x00000000u, 0x41f00000u, 0x00007fffu, 0x0000ffffu, 0x000000ffu, 0x80007bffu, 0x00007c00u}},
+    {{0xcf000000u, 0x007fffffu}, {0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000020u, 0x00000000u, 0xc1e00000u, 0x00008001u, 0x00000000u, 0x00000000u, 0x0000fbffu, 0x0000fc00u}, {0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000020u, 0x00000000u, 0xc1e00000u, 0x00008001u, 0x00000000u, 0x00000000u, 0x0000fbffu, 0x0000fc00u}},
+    {{0x3f7fffffu, 0x80000001u}, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0xe0000000u, 0x3fefffffu, 0x00007fffu, 0x0000ffffu, 0x00000001u, 0x80003bffu, 0x00003c00u}, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0xe0000000u, 0x3fefffffu, 0x00007fffu, 0x0000ffffu, 0x00000001u, 0x80003bffu, 0x00003c00u}},
+    {{0x00000002u, 0x3f800000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fff0000u, 0xffff0000u, 0x00000000u, 0x3c000000u, 0x00000000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffff6du, 0x00000000u, 0x36b00000u, 0x7fff0000u, 0xffff0000u, 0x00000000u, 0x3c000000u, 0x00000000u}},
+    {{0x80000002u, 0x00000003u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0xffffff6du, 0x00000000u, 0xb6b00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u}},
+    {{0x00000001u, 0x00000001u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffff6cu, 0x00000000u, 0x36a00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}},
+    {{0x007fffffu, 0x807fffffu}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffff82u, 0xc0000000u, 0x380fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u}},
+    {{0x3f000000u, 0x80000001u}, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3fe00000u, 0x00004000u, 0x00008000u, 0x00000000u, 0x80003800u, 0x00003800u}, {0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3fe00000u, 0x00004000u, 0x00008000u, 0x00000000u, 0x80003800u, 0x00003800u}},
+    {{0x80000001u, 0x00400000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0xffffff6cu, 0x00000000u, 0xb6a00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u}},
+    {{0x3e800000u, 0x80800000u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0x00000000u, 0x3fd00000u, 0x00002000u, 0x00004000u, 0x00000000u, 0x80003400u, 0x00003400u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffffffffu, 0x00000000u, 0x3fd00000u, 0x00002000u, 0x00004000u, 0x00000000u, 0x80003400u, 0x00003400u}},
+    {{0xbf800000u, 0x00000001u}, {0xffffffffu, 0x00000000u, 0xffffffffu, 0xffffffffu, 0x00000001u, 0x00000000u, 0xbff00000u, 0x00008001u, 0x00000000u, 0x00000000u, 0x0000bc00u, 0x0000bc00u}, {0xffffffffu, 0x00000000u, 0xffffffffu, 0xffffffffu, 0x00000001u, 0x00000000u, 0xbff00000u, 0x00008001u, 0x00000000u, 0x00000000u, 0x0000bc00u, 0x0000bc00u}},
+}};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+bool InPass(std::uint32_t column, bool float64) {
+    return (column == Float64Low || column == Float64High) == float64;
+}
+
+std::vector<std::uint32_t> Run(AgcDriver::VulkanDevice& device, bool float64, ShaderRecompiler::ShaderFloatMode floatMode) {
+    Input.fill(0u);
+    for (std::uint32_t lane = 0; lane < Vectors.size(); ++lane) {
+        Input[lane * Inputs] = Vectors[lane].operands[0];
+        Input[lane * Inputs + 1u] = Vectors[lane].operands[1];
+    }
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const auto code = float64 ? std::span<const std::uint32_t>(Float64Code) : std::span<const std::uint32_t>(Code);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    request.context.floatMode = floatMode;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+    return {Output.begin(), Output.end()};
+}
+
+std::string Where(std::uint32_t lane, std::uint32_t column, std::uint32_t denormals, bool ieee) {
+    return std::string(Names[column]) + " of " + Hex(Vectors[lane].operands[0]) + ", " + Hex(Vectors[lane].operands[1]) + " in f32 denormal mode " +
+           std::to_string(denormals) + (ieee ? ", IEEE=1" : ", IEEE=0");
+}
+
+void CheckFlushed(const std::vector<std::uint32_t>& output, bool float64, std::uint32_t denormals, bool ieee) {
+    for (std::uint32_t lane = 0; lane < Vectors.size(); ++lane) {
+        for (std::uint32_t column = 0; column < Columns; ++column) {
+            if (!InPass(column, float64)) continue;
+            const auto value = output[lane * Results + column];
+            const auto expected = Vectors[lane].denormalsFlushed[column];
+            Require(value == expected, Where(lane, column, denormals, ieee) + " is " + Hex(value) + ", expected " + Hex(expected));
+        }
+    }
+}
+
+void CheckKept(const std::vector<std::uint32_t>& output, bool float64, bool ieee) {
+    for (std::uint32_t lane = 0; lane < Vectors.size(); ++lane) {
+        for (std::uint32_t column = 0; column < Columns; ++column) {
+            if (!InPass(column, float64)) continue;
+            const auto value = output[lane * Results + column];
+            const auto& vector = Vectors[lane];
+            Require(value == vector.denormalsKept[column] || value == vector.denormalsFlushed[column],
+                    Where(lane, column, 3u, ieee) + " is " + Hex(value) + ", expected " + Hex(vector.denormalsKept[column]) + " or, on a driver that flushes f32 denormals, " +
+                        Hex(vector.denormalsFlushed[column]));
+        }
+    }
+}
+
+void CheckSame(const std::vector<std::uint32_t>& output, const std::vector<std::uint32_t>& reference, bool float64, std::uint32_t denormals, std::uint32_t like, bool ieee) {
+    for (std::uint32_t lane = 0; lane < Vectors.size(); ++lane) {
+        for (std::uint32_t column = 0; column < Columns; ++column) {
+            if (!InPass(column, float64)) continue;
+            const auto value = output[lane * Results + column];
+            const auto expected = reference[lane * Results + column];
+            Require(value == expected, Where(lane, column, denormals, ieee) + " is " + Hex(value) + ", but mode " + std::to_string(like) + " gave " + Hex(expected));
+        }
+    }
+}
+
+ShaderRecompiler::ShaderFloatMode Mode(std::uint32_t denormals, bool ieee) {
+    return ShaderRecompiler::ShaderFloatMode{0xc0u | denormals << 4u, true, ieee, false};
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        const bool hasFloat64 = TargetHasCapability(device->Target(), spv::CapabilityFloat64);
+        if (!hasFloat64) std::puts("v_cvt_f64_f32 skipped, the device has no shaderFloat64");
+        for (const bool float64 : {false, true}) {
+            if (float64 && !hasFloat64) continue;
+            for (const bool ieee : {false, true}) {
+                const auto flushed = Run(*device, float64, Mode(0u, ieee));
+                CheckFlushed(flushed, float64, 0u, ieee);
+                const auto inputsFlushed = Run(*device, float64, Mode(2u, ieee));
+                CheckFlushed(inputsFlushed, float64, 2u, ieee);
+                CheckSame(inputsFlushed, flushed, float64, 2u, 0u, ieee);
+                const auto kept = Run(*device, float64, Mode(3u, ieee));
+                CheckKept(kept, float64, ieee);
+                const auto outputsFlushed = Run(*device, float64, Mode(1u, ieee));
+                CheckSame(outputsFlushed, kept, float64, 1u, 3u, ieee);
+            }
+        }
+        std::puts("f32 conversion denormal mode tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/Float16Dot.cpp

@@ -31,6 +31,12 @@ alignas(256) constexpr std::array<std::uint32_t, 37> Code{
     0xe0781020, 0x80011203, 0xe0781030, 0x80011603, 0xbf810000,
 };
 
+alignas(256) constexpr std::array<std::uint32_t, 20> AccumulatorCode{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xcc13600a, 0x1c1a0b04, 0xcc13000b,
+    0x1c1a0b04, 0xcc13440c, 0x1c1a0b04, 0xcc13400d, 0x9c1a0b04, 0xcc13440e, 0x9c1a0b04, 0xe0781000,
+    0x80010a03, 0xe0701010, 0x80010e03, 0xbf810000,
+};
+
 constexpr std::uint32_t Rows[32][4] = {
     {0x0001bc00u, 0x81bfacbau, 0xc541aa2fu, 0x00000000u},
     {0x01f61a79u, 0x3c000381u, 0xc1729fffu, 0x00000000u},
@@ -101,6 +107,43 @@ constexpr std::uint32_t Expected[32][10] = {
     {0xbaf43c38u, 0xbaf43c38u, 0x3af43c38u, 0x3af43c38u, 0x3abe63ecu, 0xbaf43c38u, 0xc021e000u, 0x3fb50000u, 0xba90cd60u, 0x3abe63ecu}
 };
 
+constexpr std::uint32_t AccumulatorExpected[32][5] = {
+    {0xc541a901u, 0xc541a901u, 0x4541ab5du, 0x4541ab5du, 0xc541a901u},
+    {0xc1729fdfu, 0xc1729fdfu, 0x4172a01fu, 0x4172a01fu, 0xc1729fdfu},
+    {0x4187e315u, 0x4187e315u, 0x4187e315u, 0xc1882737u, 0xc1882737u},
+    {0x3f7fdb0fu, 0x3f7fdb0fu, 0x3f7fdb0fu, 0xbf801279u, 0xbf801279u},
+    {0xbe511687u, 0xbe511687u, 0xbe511687u, 0xbe5166f4u, 0xbe5166f4u},
+    {0xc1e56f13u, 0xc1e56f13u, 0x41e55ba5u, 0x41e55ba5u, 0xc1e56f13u},
+    {0x439a1228u, 0x439a1228u, 0x439a1228u, 0xc3a5d2acu, 0xc3a5d2acu},
+    {0x37f5fb14u, 0x37f5fb14u, 0x37f5fc0cu, 0x37f5fc0cu, 0x37f5fb14u},
+    {0x7fd0a000u, 0x7fd0a000u, 0x7fd0a000u, 0x7fd0a000u, 0x7fd0a000u},
+    {0xffc8ab86u, 0xffc8ab86u, 0x7fc8ab86u, 0x7fc8ab86u, 0xffc8ab86u},
+    {0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u},
+    {0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u},
+    {0xb974e480u, 0xb974e480u, 0xb974e480u, 0xb974e480u, 0xb974e480u},
+    {0x7f800000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0xff800000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x80000000u},
+    {0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u},
+    {0x7fc42000u, 0x7fc42000u, 0x7fc42000u, 0x7fc42000u, 0x7fc42000u},
+    {0xbf052ae0u, 0xbf052ae0u, 0xbf052ae0u, 0xbf052ae0u, 0xbf052ae0u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x5f834da7u, 0x5f834da7u, 0x5f834da7u, 0xdf834da7u, 0xdf834da7u},
+    {0x7f7fffffu, 0x7f7fffffu, 0x7f7fffffu, 0xff7fffffu, 0xff7fffffu},
+    {0x42750eeeu, 0x42750eeeu, 0x42750eeeu, 0xc2750eeeu, 0xc2750eeeu},
+    {0x43380748u, 0x43380748u, 0x43380748u, 0xc3380746u, 0xc3380746u},
+    {0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u},
+    {0xf540cddbu, 0xf540cddbu, 0x7540cddbu, 0x7540cddbu, 0xf540cddbu},
+    {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u},
+    {0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0xffe00000u, 0xffe00000u},
+    {0x3cb7c720u, 0x3cb7c720u, 0x3cb7c720u, 0x3cb7c720u, 0x3cb7c720u},
+    {0xbf9bd158u, 0xbf9bd158u, 0xbf9bd158u, 0xbf9bd158u, 0xbf9bd158u},
+    {0x565f49ebu, 0x565f49ebu, 0x565f49ebu, 0xd65f49ebu, 0xd65f49ebu},
+    {0xbaf43c38u, 0xbaf43c38u, 0xbaf43c38u, 0xbaf43c38u, 0xbaf43c38u}
+};
+
+constexpr const char* AccumulatorNames[5] = {"v_dot2 op_sel src2", "v_dot2 op_sel_hi src2 cleared", "v_dot2 neg_hi src2", "v_dot2 neg_lo src2", "v_dot2 neg_lo and neg_hi src2"};
+
 constexpr const char* Names[10] = {"v_dot2c", "v_dot2", "v_dot2 neg_lo src0/src2 neg_hi src1", "v_dot2 neg_lo src1 neg_hi src0", "v_dot2 op_sel src0", "v_dot2 clamp", "v_dot2c inline 1.0", "v_dot2c literal", "v_dot2c dpp quad_perm", "v_dot2 op_sel src1"};
 
 void Fill(std::uint32_t tid, std::uint32_t* words) {
@@ -140,21 +183,9 @@ auto Compile(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> cod
     return ShaderRecompiler::Recompile(request);
 }
 
-void CheckRefused(AgcDriver::VulkanDevice& device, std::uint32_t word0, const std::string& what) {
-    alignas(256) const std::array<std::uint32_t, 3> code{word0, 0x1c1a0b04u, 0xbf810000u};
-    std::string refusal;
-    try {
-        static_cast<void>(Compile(device, code));
-    } catch (const std::exception& error) {
-        refusal = error.what();
-    }
-    Require(refusal.find("v_dot2_f32_f16 accumulator op_sel, op_sel_hi and neg_hi are not implemented") != std::string::npos, what + " was not refused");
-}
-
-void Run(AgcDriver::VulkanDevice& device) {
+void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code) {
     for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
     Output.fill(0xdeadbeefu);
-    const std::span<const std::uint32_t> code(Code);
     const auto result = Compile(device, code);
     device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
     device.WaitIdle();
@@ -167,6 +198,13 @@ void Check() {
     }
 }
 
+void CheckAccumulator() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::uint32_t* out = &Output[tid * Results];
+        for (std::uint32_t i = 0; i < 5; ++i) Expect(tid, out[i], AccumulatorExpected[tid][i], AccumulatorNames[i]);
+    }
+}
+
 }
 
 int main() {
@@ -177,11 +215,10 @@ int main() {
             std::puts("skipped, v_dot2_f32_f16 is computed in f64 and the device has no shaderFloat64");
             return VulkanTestSkipped;
         }
-        Run(*device);
+        Run(*device, Code);
         Check();
-        CheckRefused(*device, 0xcc13600au, "op_sel on the accumulator");
-        CheckRefused(*device, 0xcc13440au, "neg_hi on the accumulator");
-        CheckRefused(*device, 0xcc13000au, "op_sel_hi cleared on the accumulator");
+        Run(*device, AccumulatorCode);
+        CheckAccumulator();
         std::puts("v_dot2_f32_f16 tests passed");
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp

@@ -47,6 +47,7 @@ constexpr std::uint32_t MaxGroups = 8;
 constexpr std::uint32_t Format32UInt = 20;
 constexpr std::uint32_t Format32SInt = 21;
 constexpr std::uint32_t Format32Float = 22;
+constexpr std::uint32_t Format32_32UInt = 62;
 constexpr std::uint32_t Format11_11_10UInt = 34;
 constexpr std::uint32_t Format8_8Srgb = 129;
 constexpr std::uint32_t Type2D = 9;
@@ -56,6 +57,7 @@ constexpr std::uint32_t DepthSwizzleMode = 0x18;
 alignas(256) std::array<float, Width * Height> FloatTexels{};
 alignas(256) std::array<float, Width * Height> OtherFloatTexels{};
 alignas(256) std::array<std::uint32_t, Width * Height> UintTexels{};
+alignas(256) std::array<std::uint64_t, Width * Height> Uint64Texels{};
 alignas(256) std::array<std::int32_t, Width * Height> SintTexels{};
 alignas(4096) std::array<std::uint32_t, 49152> DepthStorage{};
 alignas(256) std::array<std::uint32_t, Records * Stride / 4> Table{};
@@ -208,6 +210,12 @@ alignas(256) constexpr std::array<std::uint32_t, 16> AtomicAddLaneTableCode{
     0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e040280u, 0x7e080301u, 0xf0448108u, 0x00090401u, 0xbf810000u,
 };
 
+alignas(256) constexpr std::array<std::uint32_t, 18> AtomicAdd64LaneTableCode{
+    0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0x8f108202u, 0xf4200444u, 0x20000000u, 0x9312b011u,
+    0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e040280u, 0x7e080301u, 0x7e0a02ffu, 0x00000000u, 0xf0448308u,
+    0x00090401u, 0xbf810000u,
+};
+
 alignas(256) constexpr std::array<std::uint32_t, 16> AtomicSwapLaneTableCode{
     0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0x8f108202u, 0xf4200444u, 0x20000000u, 0x9312b011u,
     0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e040280u, 0x7e080301u, 0xf03c8108u, 0x00090401u, 0xbf810000u,
@@ -1168,6 +1176,32 @@ void RunAtomicTests(AgcDriver::VulkanDevice& device) {
     for (std::uint32_t x = 0; x < Width; ++x) Require(UintTexels[x] == (x ^ 1u) && std::bit_cast<std::uint32_t>(FloatTexels[x]) == 0x3f800000u + x, "image table atomic: an add through a null entry reached an image:\n" + null.log);
 }
 
+void RunAtomic64Tests(AgcDriver::VulkanDevice& device) {
+    if (!TargetHasCapability(device.Target(), spv::CapabilityInt64ImageEXT)) {
+        std::puts("skipped the 64-bit table atomic rows, the device has no shaderImageInt64Atomics");
+        return;
+    }
+    struct Registration {
+        void* data;
+        Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, true); }
+        ~Registration() { GuestAllocations::Mutation().Remove(data); }
+    } registered(Uint64Texels.data(), sizeof(Uint64Texels));
+    SetClassRecords();
+    for (std::uint32_t x = 0; x < Width; ++x) Uint64Texels[x] = 0x5000000000000000ull + x;
+    SetRecord(9, PointClamp, Image(Uint64Texels.data(), Format32_32UInt));
+    Keys = {};
+    Keys[0] = 9;
+    const auto added = Run(device, AtomicAdd64LaneTableCode, 1, Lanes, 64);
+    AgcDriver::Graphics::StorageTexture::FlushAllPending("image table atomic64 test");
+    Require(!Faulted(added), "image table atomic64: an add through a table faulted:\n" + added.log);
+    Require(added.imageTableCount != 0u, "image table atomic64: the add bound no storage table elements");
+    for (std::uint32_t x = 0; x < Width; ++x) Require(Uint64Texels[x] == 0x5000000000000000ull + x + x, "image table atomic64: texel " + std::to_string(x) + " is wrong");
+    Keys[0] = 6;
+    const auto null = Run(device, AtomicAdd64LaneTableCode, 1, Lanes, 64);
+    AgcDriver::Graphics::StorageTexture::FlushAllPending("image table atomic64 test");
+    for (std::uint32_t x = 0; x < Width; ++x) Require(Uint64Texels[x] == 0x5000000000000000ull + x + x, "image table atomic64: an add through a null entry reached an image:\n" + null.log);
+}
+
 void RunSelectedStoreTests(AgcDriver::VulkanDevice& device) {
     alignas(256) static std::array<std::uint32_t, Width * Height> storeTexels{};
     struct Registration {
@@ -1561,6 +1595,7 @@ int main() {
         RunModeTests(*device);
         RunStoreTests(*device);
         RunAtomicTests(*device);
+        RunAtomic64Tests(*device);
         RunSelectedStoreTests(*device);
         RunPointerTests(*device);
         RunPointerRecordTests(*device);

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetCompletionPoints.cpp

@@ -0,0 +1,395 @@
+#include "prx/libkernel/include/Api.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp"
+#include "prx/libSceAgcDriver/Eq/include/Event.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "prx/libkernel/Equeue/Equeue.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <bit>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <functional>
+#include <initializer_list>
+#include <iostream>
+#include <memory>
+#include <span>
+#include <string>
+#include <string_view>
+#include <thread>
+#include <utility>
+#include <vector>
+
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+
+constexpr std::uint32_t Width = 256;
+constexpr std::uint32_t Height = 128;
+constexpr std::size_t Pitch = std::size_t{Width} * 4u;
+constexpr std::size_t TargetBytes = Pitch * Height;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t LabelOffset = 3 * BlockBytes;
+constexpr std::uint32_t Untouched = 0x5a5a5a5au;
+constexpr std::uint32_t Drawn = 0xff0000ffu;
+constexpr std::uint32_t LabelValue = 0x600dcafeu;
+constexpr std::uint32_t OutputHandle = 9;
+constexpr std::uint32_t Rgba8UnormInfo = 0x28028u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::array<std::uint32_t, 4> Scissor{32, 16, 160, 80};
+
+alignas(256) constexpr std::array<std::uint32_t, 20> VertexCode{
+    0x7e000d05u, 0x080200f4u, 0x10020300u, 0x100202f4u, 0x060202f3u, 0x060400f3u, 0x10040500u, 0x060404f3u,
+    0x7e060208u, 0x7e0802f2u, 0x7e0a02ffu, 0x44800000u, 0xf80000d1u, 0x00000005u, 0xf80008cfu, 0x04030201u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u,
+};
+alignas(256) constexpr std::array<std::uint32_t, 8> PixelCode{0x7e000200u, 0x7e020280u, 0xf800180fu, 0x00010100u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 9> FillCode{
+    0xd7460004, 0x04010c08, 0x7e000204, 0x7e020205, 0x7e040206, 0x7e060207, 0xe01c2000, 0x80000004, 0xbf810000,
+};
+
+alignas(4) std::atomic<std::uint32_t> Gate = 0;
+
+struct Header {
+    Shader shader{};
+    ShaderUserData userData{};
+};
+
+alignas(8) Header VertexHeader{};
+alignas(8) Header PixelHeader{};
+
+void Registered(Header& header, const std::uint32_t* code, std::size_t bytes, std::uint8_t type) {
+    header.shader.file_header = 0x34333231u;
+    header.shader.version = 0x18u;
+    header.shader.header_size = sizeof(Header);
+    header.shader.shader_size = static_cast<std::uint32_t>(bytes);
+    header.shader.code = code;
+    header.shader.type = type;
+    header.shader.user_data = &header.userData;
+    AgcDriverRegisterShader_nid_postfix(&header.shader);
+}
+
+void Append(std::vector<std::uint32_t>& commands, std::uint32_t opcode, std::initializer_list<std::uint32_t> payload) {
+    commands.push_back(0xc0000000u | (static_cast<std::uint32_t>(payload.size() - 1) << 16u) | (opcode << 8u));
+    commands.insert(commands.end(), payload);
+}
+
+void AppendDraw(std::vector<std::uint32_t>& commands, std::uint64_t target) {
+    const auto vertex = reinterpret_cast<std::uintptr_t>(VertexCode.data());
+    const auto pixel = reinterpret_cast<std::uintptr_t>(PixelCode.data());
+    const auto extent = (Height << 16u) | Width;
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> context{
+        {0x2d5, 0x2000},
+        {0x1b6, 0}, {0x207, 0x01200000}, {0x200, 0}, {0x203, 0x800},
+        {0x2dc, 0xaa00}, {0x2f8, 0}, {0x292, 2}, {0x293, 0},
+        {0x80, 0}, {0x8d, 0}, {0x83, 0xffff}, {0x8c, 0xa},
+        {0x2f9, 0x2d}, {0x313, 0x6000}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff},
+        {0x206, 0x43f}, {0x204, 0x80000}, {0x205, 0x240},
+        {0x8e, 0xf}, {0x8f, 0xf}, {0x202, 0xcc0010},
+        {0x1c4, 0}, {0x1c5, 9}, {0x1c3, 4}, {0x31c, Rgba8UnormInfo},
+        {0x31b, 0}, {0x31d, 0}, {0x3b0, ((Width - 1u) << 14u) | (Height - 1u)},
+        {0x3b8, LinearAttrib3}, {0x1e0, 0},
+        {0xc, 0}, {0xd, extent},
+        {0x81, 0x80000000}, {0x82, extent},
+        {0x90, 0x80000000u | (Scissor[1] << 16u) | Scissor[0]}, {0x91, (Scissor[3] << 16u) | Scissor[2]},
+        {0x94, 0x80000000}, {0x95, extent},
+        {0x10f, std::bit_cast<std::uint32_t>(2.0f * Width)}, {0x110, std::bit_cast<std::uint32_t>(2.0f * Width)},
+        {0x111, std::bit_cast<std::uint32_t>(2.0f * Height)}, {0x112, std::bit_cast<std::uint32_t>(2.0f * Height)},
+        {0x113, std::bit_cast<std::uint32_t>(1.0f)}, {0x114, 0},
+        {0xb4, 0}, {0xb5, std::bit_cast<std::uint32_t>(1.0f)},
+        {0x2fe, 0}, {0x302, 0}, {0x306, 0}, {0x30a, 0},
+        {0x318, static_cast<std::uint32_t>(target >> 8u)}, {0x390, static_cast<std::uint32_t>(target >> 40u)},
+    };
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> shader{
+        {0xc8, static_cast<std::uint32_t>(vertex >> 8u)}, {0xc9, static_cast<std::uint32_t>(vertex >> 40u)}, {0x8b, 1u << 1u}, {0x8c, 0},
+        {0x008, static_cast<std::uint32_t>(pixel >> 8u)}, {0x009, static_cast<std::uint32_t>(pixel >> 40u)}, {0x00b, 1u << 1u}, {0x00c, std::bit_cast<std::uint32_t>(1.0f)},
+    };
+    Append(commands, 0x79, {0x242, 4});
+    Append(commands, 0x79, {0x24a, 0});
+    for (const auto& [offset, value] : context) Append(commands, 0x69, {offset, value});
+    for (const auto& [offset, value] : shader) Append(commands, 0x76, {offset, value});
+    Append(commands, 0x2d, {3, 2});
+}
+
+void AppendGate(std::vector<std::uint32_t>& commands) {
+    const auto gate = reinterpret_cast<std::uintptr_t>(&Gate);
+    commands.insert(commands.end(), {0xc0053c00u, 0x13u, static_cast<std::uint32_t>(gate), static_cast<std::uint32_t>(gate >> 32u), 1u, 0xffffffffu, 0x19u});
+}
+
+void Submit(std::vector<std::uint32_t>& commands) {
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    AgcDriver::Submit(&packet, 0);
+}
+
+class Region {
+public:
+    Region(std::size_t bytes, bool watched) : bytes(bytes), arena(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (arena) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            if (data != nullptr) GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (arena) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "linear target completion points: cannot map the guest region");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "linear target completion points: cannot allocate the guest region");
+        auto* words = reinterpret_cast<std::uint32_t*>(data);
+        std::fill(words, words + bytes / 4u, Untouched);
+        if (arena && !watched) {
+            AgcDriver::GuestMemory::Unwatch(Address(), bytes);
+            Require(!AgcDriver::GuestMemory::Watched(Address(), bytes), "linear target completion points: the region is still write-watched");
+        }
+        GuestAllocations::Mutation().Add(data, bytes, true, true, true);
+    }
+    ~Region() {
+        GuestAllocations::Mutation().Remove(data);
+#ifdef _WIN32
+        if (arena) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (arena) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Region(const Region&) = delete;
+    Region& operator=(const Region&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::uint32_t Word(std::size_t offset) const { return std::atomic_ref<std::uint32_t>(*reinterpret_cast<std::uint32_t*>(data + offset)).load(std::memory_order_acquire); }
+    std::size_t bytes;
+
+private:
+    bool arena;
+    std::uint8_t* data = nullptr;
+};
+
+void Import(const Region& region) {
+    const auto address = region.Address();
+    const std::vector<std::uint32_t> userData{
+        static_cast<std::uint32_t>(address),
+        static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (16u << 16u),
+        static_cast<std::uint32_t>(region.bytes / 16u),
+        0x4bfacu,
+        Untouched,
+        Untouched,
+        Untouched,
+        Untouched,
+    };
+    std::vector<std::uint32_t> commands;
+    const auto code = reinterpret_cast<std::uintptr_t>(FillCode.data());
+    Append(commands, 0x76, {0x207, 64, 1, 1});
+    Append(commands, 0x76, {0x20c, static_cast<std::uint32_t>(code >> 8u), static_cast<std::uint32_t>(code >> 40u)});
+    Append(commands, 0x76, {0x213, (static_cast<std::uint32_t>(userData.size()) << 1u) | 0x80u});
+    commands.push_back(0xc0007600u | (static_cast<std::uint32_t>(userData.size()) << 16u));
+    commands.push_back(0x240u);
+    commands.insert(commands.end(), userData.begin(), userData.end());
+    commands.insert(commands.end(), {0xc0031500u, static_cast<std::uint32_t>(region.bytes / 1024u), 1, 1, 0x41u});
+    Submit(commands);
+    AgcDriverWaitIdle_nid_postfix();
+}
+
+bool InsideScissor(std::uint32_t x, std::uint32_t y) {
+    return x >= Scissor[0] && x < Scissor[2] && y >= Scissor[1] && y < Scissor[3];
+}
+
+std::string Mismatches(const Region& region) {
+    std::size_t drawn = 0, outside = 0;
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const auto value = region.Word(y * Pitch + x * 4u);
+            if (InsideScissor(x, y)) drawn += value == Drawn ? 0u : 1u;
+            else outside += value == Untouched ? 0u : 1u;
+        }
+    }
+    if (drawn == 0 && outside == 0) return {};
+    return std::to_string(drawn) + " of the " + std::to_string((Scissor[2] - Scissor[0]) * (Scissor[3] - Scissor[1])) + " pixels inside the scissor do not hold the draw's color and " + std::to_string(outside) + " pixels outside it changed";
+}
+
+bool Await(const std::function<bool()>& done) {
+    const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5);
+    while (!done()) {
+        if (std::chrono::steady_clock::now() >= deadline) return false;
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+    return true;
+}
+
+class Request final : public AgcDriver::IFlipRequest {
+public:
+    explicit Request(std::atomic<int>& ready) : ready(ready) {}
+    void GpuReady(const std::shared_ptr<AgcDriver::FrameTiming>&) override { ++ready; }
+    void Fail(std::exception_ptr) noexcept override {}
+private:
+    std::atomic<int>& ready;
+};
+
+class Output final : public AgcDriver::IVideoOutput {
+public:
+    std::shared_ptr<AgcDriver::IFlipRequest> Reserve(const AgcDriver::FlipInfo&) override { return std::make_shared<Request>(ready); }
+    void Fail(std::exception_ptr) noexcept override {}
+    std::atomic<int> ready = 0;
+};
+
+enum class Point { Flip, Label, Interrupt };
+
+struct Events {
+    KernelEqueue queue = 0;
+    KernelEqueueRef owner;
+    int tag = 0;
+};
+
+std::string Run(Point point, bool watched, const Region& region, Output& output, Events& events) {
+    const char* name = point == Point::Flip ? "flip" : point == Point::Label ? "label" : "end-of-pipe interrupt";
+    const auto target = region.Address();
+    Import(region);
+    Gate.store(0, std::memory_order_release);
+    std::vector<std::uint32_t> commands;
+    AppendDraw(commands, target);
+    if (point == Point::Flip) {
+        commands.insert(commands.end(), {AgcDriver::FlipPacketHeader, OutputHandle, 0xfffffffeu, 1, 0, 0});
+    } else if (point == Point::Label) {
+        const auto label = target + LabelOffset;
+        commands.insert(commands.end(), {0xc0064900u, 0x514u, 1u << 29u, static_cast<std::uint32_t>(label), static_cast<std::uint32_t>(label >> 32u), LabelValue, 0, 0});
+    } else {
+        commands.insert(commands.end(), {0xc0064900u, 0x06000528u, 2u << 24u, 0, 0, 0, 0, 0});
+    }
+    AppendGate(commands);
+    const auto readyBefore = output.ready.load();
+    if (point == Point::Interrupt) {
+        std::array<KernelEvent, 4> stale{};
+        while (events.owner->GetTriggeredEvents(stale.data(), static_cast<int>(stale.size())) > 0) {}
+    }
+    Submit(commands);
+    const bool watchedAtStore = AgcDriver::GuestMemory::Watched(target, TargetBytes);
+    std::string failure;
+    if (point == Point::Flip) {
+        const bool flipped = Await([&] { return output.ready.load() != readyBefore; });
+        if (!flipped) failure = "the flip did not reach the GPU";
+        else if (!Await([&] { return Mismatches(region).empty(); })) failure = "after the flip, while the queue waited: " + Mismatches(region);
+    } else if (point == Point::Label) {
+        if (!Await([&] { return region.Word(LabelOffset) == LabelValue; })) failure = "the label after the draw never landed";
+        else if (const auto mismatches = Mismatches(region); !mismatches.empty()) failure = "when the label after the draw landed: " + mismatches;
+    } else {
+        std::array<KernelEvent, 1> delivered{};
+        if (!Await([&] { return events.owner->GetTriggeredEvents(delivered.data(), 1) == 1; })) failure = "the end-of-pipe interrupt after the draw was not delivered";
+        else if (delivered[0].udata != &events.tag) failure = "an unexpected event was delivered";
+        else if (const auto mismatches = Mismatches(region); !mismatches.empty()) failure = "when the end-of-pipe interrupt after the draw was delivered: " + mismatches;
+    }
+    Gate.store(1, std::memory_order_release);
+    AgcDriverWaitIdle_nid_postfix();
+    AgcDriver::Graphics::StorageTexture::FlushPending(target, region.bytes, nullptr, "test release");
+    std::printf("linear target completion point: %s, %s memory (watched at the store: %d): %s\n", name, watched ? "write-watched" : "unwatched", watchedAtStore ? 1 : 0, failure.empty() ? "stored" : failure.c_str());
+    if (failure.empty()) return {};
+    return std::string(name) + (watched ? " (write-watched memory): " : " (unwatched memory): ") + failure;
+}
+
+}
+
+int main(int argc, char** argv) {
+    try {
+        bool flips = argc == 1, labels = argc == 1, interrupts = argc == 1, watchedFlip = argc == 1;
+        for (int index = 1; index < argc; ++index) {
+            const std::string_view argument(argv[index]);
+            if (argument == "--flip") flips = true;
+            else if (argument == "--label") labels = true;
+            else if (argument == "--interrupt") interrupts = true;
+            else Require(false, "linear target completion points: unknown argument " + std::string(argument));
+        }
+        {
+            const auto device = OpenVulkanTestDevice();
+            if (!device) return VulkanTestSkipped;
+            if (AgcDriver::VulkanDeviceTestAccess::ContextOf(*device).hostImportAlignment == 0) {
+                std::puts("linear target completion point tests skipped: no host import on this device, so the target is never resident");
+                return VulkanTestSkipped;
+            }
+        }
+        std::vector<std::unique_ptr<Region>> regions;
+        struct ShutdownGuard {
+            ~ShutdownGuard() { try { AgcDriverShutdown_nid_postfix(); } catch (...) {} }
+        } shutdown;
+        Registered(VertexHeader, VertexCode.data(), sizeof(VertexCode), 2);
+        Registered(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1);
+        auto output = std::make_shared<Output>();
+        AgcDriverRegisterVideoOutput_nid_postfix(OutputHandle, output);
+        Events events;
+        if (interrupts) {
+            Require(sceKernelCreateEqueue(&events.queue, "AGC linear target") == 0, "linear target completion points: event queue creation failed");
+            events.owner = EqueuePin_nid_postfix(events.queue);
+            Require(sceAgcDriverAddEqEvent(events.queue, 0, &events.tag) == 0, "linear target completion points: graphics event registration failed");
+        }
+        std::vector<std::string> failures;
+        const auto run = [&](Point point, bool watched) {
+            regions.push_back(std::make_unique<Region>(watched ? 16 * BlockBytes : 4 * BlockBytes, watched));
+            if (auto failure = Run(point, watched, *regions.back(), *output, events); !failure.empty()) failures.push_back(std::move(failure));
+        };
+        if (flips) run(Point::Flip, false);
+        if (watchedFlip) run(Point::Flip, true);
+        if (labels) run(Point::Label, false);
+        if (interrupts) run(Point::Interrupt, false);
+        if (interrupts) {
+            Require(sceAgcDriverDeleteEqEvent(events.queue, 0) == 0, "linear target completion points: graphics event deletion failed");
+            events.owner.reset();
+            Require(sceKernelDeleteEqueue(events.queue) == 0, "linear target completion points: event queue deletion failed");
+        }
+        AgcDriverUnregisterVideoOutput_nid_postfix(OutputHandle, output);
+        if (!failures.empty()) {
+            std::string message = "a CPU read after a point where the guest can observe a draw's completion did not see the drawn linear target:";
+            for (const auto& failure : failures) message += "\n  " + failure;
+            Require(false, message);
+        }
+        AgcDriverShutdown_nid_postfix();
+        std::puts("linear target completion point tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetCompletionStores.cpp

@@ -0,0 +1,304 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using AgcDriver::Graphics::StorageTexture;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 16;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t TargetBytes = std::size_t{Width} * Height * 16u;
+constexpr std::size_t LabelOffset = BlockBytes;
+constexpr std::uint32_t Rgba32FloatInfo = 0x8738u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t Float32AbgrExport = 9;
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::array<std::uint32_t, 16>, 2> Programs{};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+std::span<const std::uint32_t> Program(std::size_t index, bool blue) {
+    auto& code = Programs.at(index);
+    std::size_t at = 0;
+    code[at++] = 0x36000000u | (20u << 17u) | 0xffu;
+    code[at++] = 0x0000ffffu;
+    code[at++] = 0x2c000000u | (21u << 17u) | 0x90u;
+    code[at++] = 0x7e000c00u | (20u << 17u) | (256u + 20u);
+    code[at++] = 0x7e000c00u | (21u << 17u) | (256u + 21u);
+    code[at++] = (blue ? 0x7e0002f2u : 0x7e000280u) | (22u << 17u);
+    code[at++] = 0x7e0002f2u | (23u << 17u);
+    code[at++] = 0xf800180fu;
+    code[at++] = 0x17161514u;
+    code[at++] = 0xbf810000u;
+    return std::span<const std::uint32_t>(code.data(), at);
+}
+
+struct Block {
+    explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (watched) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (watched) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map the color target");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "cannot allocate the color target");
+        Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is not write-watched");
+        GuestAllocations::Mutation mutation;
+        mutation.Add(data, bytes, true, true, true);
+    }
+    ~Block() {
+        StorageTexture::FlushPending(Address(), bytes, nullptr, "test release");
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Remove(data);
+        }
+#ifdef _WIN32
+        if (watched) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (watched) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Block(const Block&) = delete;
+    Block& operator=(const Block&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::size_t bytes;
+    bool watched;
+    std::uint8_t* data = nullptr;
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block) {
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(block.Address() >> 8u);
+    cx[0x31b] = 0;
+    cx[0x31c] = Rgba32FloatInfo;
+    cx[0x31d] = 0;
+    cx[0x323] = 0;
+    cx[0x324] = 0;
+    cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u);
+    cx[0x3b8] = LinearAttrib3;
+    cx[0x390] = static_cast<std::uint32_t>(block.Address() >> 40u);
+    auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    color.exportIndex = 0;
+    return color;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::span<const std::uint32_t> pixelCode, VkRect2D scissor) {
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast<std::uint32_t>(Triangle.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionFixedPoint);
+    pixel.targetOutputMode.at(0) = Float32AbgrExport;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::vector<std::uint32_t> pixelUserData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.blendEnable = VK_FALSE;
+    blend.colorWriteMask = 0xf;
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = scissor;
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+}
+
+std::array<float, 4> Texel(const std::uint8_t* target, std::uint32_t x, std::uint32_t y) {
+    std::array<float, 4> texel{};
+    std::memcpy(texel.data(), target + (static_cast<std::size_t>(y) * Width + x) * 16u, sizeof(texel));
+    return texel;
+}
+
+void Label(AgcDriver::VulkanDevice& device, const Block& block, std::uint32_t value, std::uint64_t submissions, bool watched, const char* what) {
+    const auto bytes = std::as_bytes(std::span(&value, 1));
+    const auto reason = device.WriteLabelOnGpu(block.Address() + LabelOffset, bytes, value, 0);
+    Require(reason == 0, std::string(what) + ": the label was not recorded on the GPU (outcome " + std::to_string(reason) + ")");
+    Require(StorageTexture::FindPending(block.Address(), TargetBytes) == nullptr, std::string(what) + ": the drawn linear target was still pending after a label the GPU writes");
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(!watched || (recorder != nullptr && recorder->Submissions() == submissions), std::string(what) + ": storing the drawn linear target before the label waited for recorded work (" + std::to_string(recorder != nullptr ? recorder->Submissions() - submissions : 0) + " submissions)");
+}
+
+bool Expect(AgcDriver::VulkanDevice& device) {
+    Block block(16 * BlockBytes);
+    std::fill(block.data, block.data + block.bytes, std::uint8_t{0xff});
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    if (AgcDriver::Graphics::HostImportFor(context, block.Address(), block.bytes) == nullptr) return false;
+    const auto color = DecodeTarget(block);
+    Require(color.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && color.bytes == TargetBytes, "the target is not a linear R32G32B32A32 target");
+    const auto red = Program(0, false);
+    const auto blue = Program(1, true);
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(recorder != nullptr, "the device has no recorder");
+    const bool watched = AgcDriver::GuestMemory::Watched(block.Address(), TargetBytes);
+    if (!watched) std::puts("linear target completion stores: the target is outside the write watch, so a store may wait for the previous store's saved bytes; the submission checks are skipped");
+    const auto pending = [&](const char* what) {
+        Require(StorageTexture::FindPending(block.Address(), TargetBytes) != nullptr, std::string(what) + ": the drawn linear target was stored before any point where its completion is observable");
+    };
+    Draw(device, color, red, {{0, 0}, {Width, Height}});
+    pending("after the first draw");
+    const auto submissions = recorder->Submissions();
+    Draw(device, color, blue, {{0, 0}, {Width / 2u, Height}});
+    pending("after the second draw");
+    Require(!watched || recorder->Submissions() == submissions, "the second draw into the linear target submitted recorded work");
+    Draw(device, color, red, {{Width / 4u, Height / 4u}, {Width / 4u, Height / 2u}});
+    pending("after the third draw");
+    Require(!watched || recorder->Submissions() == submissions, "the third draw into the linear target submitted recorded work");
+    Label(device, block, 1u, submissions, watched, "the first label");
+    Draw(device, color, blue, {{Width / 2u, 0}, {Width / 2u, Height}});
+    pending("after the fourth draw");
+    Label(device, block, 2u, submissions, watched, "the second label");
+    Draw(device, color, red, {{0, Height / 2u}, {Width, Height / 2u}});
+    pending("after the fifth draw");
+    Require(!watched || recorder->Submissions() == submissions, "the fifth draw into the linear target submitted recorded work");
+    device.WaitIdle();
+    Require(StorageTexture::FindPending(block.Address(), TargetBytes) == nullptr, "the drawn linear target was still pending after the device wait");
+    std::vector<std::uint8_t> stored(TargetBytes);
+    std::memcpy(stored.data(), block.data, TargetBytes);
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const bool lastRed = y >= Height / 2u;
+            const bool lastBlue = !lastRed && x >= Width / 2u;
+            const bool thirdRed = !lastRed && !lastBlue && x >= Width / 4u && x < Width / 2u && y >= Height / 4u;
+            const bool secondBlue = !lastRed && !lastBlue && !thirdRed;
+            const float z = lastRed || thirdRed ? 0.0f : secondBlue || lastBlue ? 1.0f : -1.0f;
+            const std::array<float, 4> expected{static_cast<float>(x), static_cast<float>(y), z, 1.0f};
+            const auto texel = Texel(stored.data(), x, y);
+            Require(texel == expected, "after the device wait, raw guest read: pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") holds (" + std::to_string(texel[0]) + ", " + std::to_string(texel[1]) + ", " + std::to_string(texel[2]) + ", " + std::to_string(texel[3]) + ")");
+        }
+    }
+    std::uint32_t label = 0;
+    std::memcpy(&label, block.data + LabelOffset, sizeof(label));
+    Require(label == 2u, "the second label did not land: it holds " + std::to_string(label));
+    return true;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (!Expect(*device)) {
+            std::puts("linear target completion store test skipped: no host import on this device");
+            return VulkanTestSkipped;
+        }
+        std::puts("linear target completion store tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetCpuWrites.cpp

@@ -0,0 +1,310 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 16;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t TargetBytes = std::size_t{Width} * Height * 16u;
+constexpr std::size_t RowBytes = std::size_t{Width} * 16u;
+constexpr std::uint32_t Drawn = Width / 2u;
+constexpr std::uint32_t Rgba32FloatInfo = 0x8738u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t Float32AbgrExport = 9;
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::array<std::uint32_t, 16>, 2> Programs{};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+std::span<const std::uint32_t> Program(std::size_t index, bool blue) {
+    auto& code = Programs.at(index);
+    std::size_t at = 0;
+    code[at++] = 0x36000000u | (20u << 17u) | 0xffu;
+    code[at++] = 0x0000ffffu;
+    code[at++] = 0x2c000000u | (21u << 17u) | 0x90u;
+    code[at++] = 0x7e000c00u | (20u << 17u) | (256u + 20u);
+    code[at++] = 0x7e000c00u | (21u << 17u) | (256u + 21u);
+    code[at++] = (blue ? 0x7e0002f2u : 0x7e000280u) | (22u << 17u);
+    code[at++] = 0x7e0002f2u | (23u << 17u);
+    code[at++] = 0xf800180fu;
+    code[at++] = 0x17161514u;
+    code[at++] = 0xbf810000u;
+    return std::span<const std::uint32_t>(code.data(), at);
+}
+
+struct Block {
+    Block(std::size_t bytes, bool stayWatched) : bytes(bytes), arena(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (arena) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (arena) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map the color target");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "cannot allocate the color target");
+        Require(!arena || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is not write-watched");
+        if (arena && !stayWatched) {
+            AgcDriver::GuestMemory::Unwatch(Address(), bytes);
+            Require(!AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is still write-watched");
+        }
+        GuestAllocations::Mutation mutation;
+        mutation.Add(data, bytes, true, true, true);
+    }
+    ~Block() {
+        AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release");
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Remove(data);
+        }
+#ifdef _WIN32
+        if (arena) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (arena) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Block(const Block&) = delete;
+    Block& operator=(const Block&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::size_t bytes;
+    bool arena;
+    std::uint8_t* data = nullptr;
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block) {
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(block.Address() >> 8u);
+    cx[0x31b] = 0;
+    cx[0x31c] = Rgba32FloatInfo;
+    cx[0x31d] = 0;
+    cx[0x323] = 0;
+    cx[0x324] = 0;
+    cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u);
+    cx[0x3b8] = LinearAttrib3;
+    cx[0x390] = static_cast<std::uint32_t>(block.Address() >> 40u);
+    auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    color.exportIndex = 0;
+    return color;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::span<const std::uint32_t> pixelCode, std::uint32_t scissorWidth) {
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast<std::uint32_t>(Triangle.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionFixedPoint);
+    pixel.targetOutputMode.at(0) = Float32AbgrExport;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::vector<std::uint32_t> pixelUserData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.blendEnable = VK_FALSE;
+    blend.colorWriteMask = 0xf;
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = {{0, 0}, {scissorWidth, Height}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+}
+
+std::array<float, 4> Texel(const std::uint8_t* target, std::uint32_t x, std::uint32_t y) {
+    std::array<float, 4> texel{};
+    std::memcpy(texel.data(), target + static_cast<std::size_t>(y) * RowBytes + static_cast<std::size_t>(x) * 16u, sizeof(texel));
+    return texel;
+}
+
+enum class Outcome { Kept, Lost, Skipped };
+
+Outcome Run(AgcDriver::VulkanDevice& device, bool watched, bool seeded, bool hookRead) {
+    const auto name = std::string(seeded ? "seeded" : "unseeded") + ", " + (watched ? "write-watched" : "unwatched") + " memory" + (hookRead ? ", driver read before the store" : "");
+    Block block(watched ? std::size_t{16} * BlockBytes : BlockBytes, watched);
+    std::fill(block.data, block.data + block.bytes, std::uint8_t{0xff});
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    if (AgcDriver::Graphics::HostImportFor(context, block.Address(), block.bytes) == nullptr) return Outcome::Skipped;
+    const auto color = DecodeTarget(block);
+    Require(color.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && color.bytes == TargetBytes, "the target is not a linear R32G32B32A32 target");
+    Draw(device, color, Program(0, false), Width);
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(recorder != nullptr && !recorder->Idle(), "the draw into an imported linear target waited for the GPU instead of being recorded");
+    device.WaitIdle();
+    std::vector<std::uint8_t> stored(TargetBytes);
+    std::memcpy(stored.data(), block.data, TargetBytes);
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const std::array<float, 4> expected{static_cast<float>(x), static_cast<float>(y), 0.0f, 1.0f};
+            const auto texel = Texel(stored.data(), x, y);
+            Require(texel == expected, name + ": after the first draw, raw guest read: pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") holds (" + std::to_string(texel[0]) + ", " + std::to_string(texel[1]) + ", " + std::to_string(texel[2]) + ", " + std::to_string(texel[3]) + ")");
+        }
+    }
+    if (seeded) {
+        for (std::uint32_t y = 0; y < Height; ++y) std::memset(block.data + y * RowBytes, 0x5a, Drawn * 16u);
+    }
+    Draw(device, color, Program(1, true), Drawn);
+    for (std::uint32_t y = 0; y < Height; ++y) std::memset(block.data + y * RowBytes + Drawn * 16u, 0x42, RowBytes - Drawn * 16u);
+    const bool watchedAtStore = AgcDriver::GuestMemory::Watched(block.Address(), TargetBytes);
+    if (hookRead) {
+        std::vector<std::uint8_t> row(RowBytes);
+        AgcDriver::GuestMemory::Read(block.Address(), std::as_writable_bytes(std::span(row)), 1);
+        for (std::uint32_t x = 0; x < Drawn; ++x) {
+            const std::array<float, 4> expected{static_cast<float>(x), 0.0f, 1.0f, 1.0f};
+            const auto texel = Texel(row.data(), x, 0);
+            Require(texel == expected, name + ": a driver read of row 0 before the store: pixel (" + std::to_string(x) + ", 0) holds (" + std::to_string(texel[0]) + ", " + std::to_string(texel[1]) + ", " + std::to_string(texel[2]) + ", " + std::to_string(texel[3]) + "), not the second draw's");
+        }
+    }
+    device.WaitIdle();
+    std::size_t wrongGpu = 0;
+    std::size_t wrongCpu = 0;
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Drawn; ++x) {
+            const std::array<float, 4> expected{static_cast<float>(x), static_cast<float>(y), 1.0f, 1.0f};
+            if (Texel(block.data, x, y) != expected) ++wrongGpu;
+        }
+        auto* begin = block.data + y * RowBytes + Drawn * 16u;
+        wrongCpu += static_cast<std::size_t>(std::count_if(begin, block.data + (y + 1u) * RowBytes, [](std::uint8_t byte) { return byte != 0x42u; }));
+    }
+    std::printf("linear target CPU writes, %s (watched at the store: %d): wrong GPU pixels %zu/%u, wrong CPU bytes %zu/%zu\n", name.c_str(), watchedAtStore ? 1 : 0, wrongGpu, Drawn * Height, wrongCpu, (RowBytes - Drawn * 16u) * Height);
+    return wrongGpu == 0 && wrongCpu == 0 ? Outcome::Kept : Outcome::Lost;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        bool kept = true;
+        bool ran = false;
+        for (const bool watched : {true, false}) {
+            for (const bool seeded : {false, true}) {
+                for (const bool hookRead : {false, true}) {
+                    const auto outcome = Run(*device, watched, seeded, hookRead);
+                    ran = ran || outcome != Outcome::Skipped;
+                    kept = kept && outcome != Outcome::Lost;
+                }
+            }
+        }
+        if (!ran) {
+            std::puts("linear target CPU write test skipped: no host import on this device");
+            return VulkanTestSkipped;
+        }
+        Require(kept, "a drawn linear target's store lost the drawn half or the CPU's later stores to the other half");
+        std::puts("linear target CPU write tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetDirectWriteCompletion.cpp

@@ -0,0 +1,452 @@
+#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <bit>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <functional>
+#include <initializer_list>
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+
+constexpr std::uint32_t Width = 256;
+constexpr std::uint32_t Height = 128;
+constexpr std::size_t Pitch = std::size_t{Width} * 4u;
+constexpr std::size_t TargetBytes = Pitch * Height;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t RegionBytes = 16 * BlockBytes;
+constexpr std::size_t LabelOffset = 3 * BlockBytes;
+constexpr std::uint32_t Untouched = 0x5a5a5a5au;
+constexpr std::uint32_t Filled = 0x9abcdef0u;
+constexpr std::uint32_t Drawn = 0xff0000ffu;
+constexpr std::uint32_t LabelValue = 0x600dcafeu;
+constexpr std::uint32_t CpuStored = 0x0badf00du;
+constexpr std::uint32_t Rgba8UnormInfo = 0x28028u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t WaitEqual = 3;
+constexpr std::uint32_t WaitAtLeast = 5;
+constexpr std::array<std::array<std::uint32_t, 4>, 2> Scissors{{{16, 8, 128, 96}, {128, 32, 240, 120}}};
+constexpr std::array<std::uint32_t, 4> BlockZeroScissor{24, 8, 232, 56};
+constexpr std::size_t CpuStoreOffset = BlockBytes + 32 * Pitch + 64 * 4;
+static_assert(BlockZeroScissor[3] * Pitch <= BlockBytes);
+static_assert(CpuStoreOffset >= BlockBytes && CpuStoreOffset < TargetBytes);
+
+alignas(256) constexpr std::array<std::uint32_t, 20> VertexCode{
+    0x7e000d05u, 0x080200f4u, 0x10020300u, 0x100202f4u, 0x060202f3u, 0x060400f3u, 0x10040500u, 0x060404f3u,
+    0x7e060208u, 0x7e0802f2u, 0x7e0a02ffu, 0x44800000u, 0xf80000d1u, 0x00000005u, 0xf80008cfu, 0x04030201u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u,
+};
+alignas(256) constexpr std::array<std::uint32_t, 8> PixelCode{0x7e000200u, 0x7e020280u, 0xf800180fu, 0x00010100u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 9> FillCode{
+    0xd7460004, 0x04010c08, 0x7e000204, 0x7e020205, 0x7e040206, 0x7e060207, 0xe01c2000, 0x80000004, 0xbf810000,
+};
+alignas(256) constexpr std::array<std::uint32_t, 11> PairFillCode{
+    0xd7460004, 0x04010c0c, 0x7e000204, 0x7e020205, 0x7e040206, 0x7e060207, 0xe01c2000, 0x80000004, 0xe01c2000, 0x80020004, 0xbf810000,
+};
+
+alignas(4) std::atomic<std::uint32_t> Gate = 0;
+
+struct Header {
+    Shader shader{};
+    ShaderUserData userData{};
+};
+
+alignas(8) Header VertexHeader{};
+alignas(8) Header PixelHeader{};
+
+void Registered(Header& header, const std::uint32_t* code, std::size_t bytes, std::uint8_t type) {
+    header.shader.file_header = 0x34333231u;
+    header.shader.version = 0x18u;
+    header.shader.header_size = sizeof(Header);
+    header.shader.shader_size = static_cast<std::uint32_t>(bytes);
+    header.shader.code = code;
+    header.shader.type = type;
+    header.shader.user_data = &header.userData;
+    AgcDriverRegisterShader_nid_postfix(&header.shader);
+}
+
+void Append(std::vector<std::uint32_t>& commands, std::uint32_t opcode, std::initializer_list<std::uint32_t> payload) {
+    commands.push_back(0xc0000000u | (static_cast<std::uint32_t>(payload.size() - 1) << 16u) | (opcode << 8u));
+    commands.insert(commands.end(), payload);
+}
+
+void AppendDraw(std::vector<std::uint32_t>& commands, std::uint64_t target, const std::array<std::uint32_t, 4>& scissor) {
+    const auto vertex = reinterpret_cast<std::uintptr_t>(VertexCode.data());
+    const auto pixel = reinterpret_cast<std::uintptr_t>(PixelCode.data());
+    const auto extent = (Height << 16u) | Width;
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> context{
+        {0x2d5, 0x2000},
+        {0x1b6, 0}, {0x207, 0x01200000}, {0x200, 0}, {0x203, 0x800},
+        {0x2dc, 0xaa00}, {0x2f8, 0}, {0x292, 2}, {0x293, 0},
+        {0x80, 0}, {0x8d, 0}, {0x83, 0xffff}, {0x8c, 0xa},
+        {0x2f9, 0x2d}, {0x313, 0x6000}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff},
+        {0x206, 0x43f}, {0x204, 0x80000}, {0x205, 0x240},
+        {0x8e, 0xf}, {0x8f, 0xf}, {0x202, 0xcc0010},
+        {0x1c4, 0}, {0x1c5, 9}, {0x1c3, 4}, {0x31c, Rgba8UnormInfo},
+        {0x31b, 0}, {0x31d, 0}, {0x3b0, ((Width - 1u) << 14u) | (Height - 1u)},
+        {0x3b8, LinearAttrib3}, {0x1e0, 0},
+        {0xc, 0}, {0xd, extent},
+        {0x81, 0x80000000}, {0x82, extent},
+        {0x90, 0x80000000u | (scissor[1] << 16u) | scissor[0]}, {0x91, (scissor[3] << 16u) | scissor[2]},
+        {0x94, 0x80000000}, {0x95, extent},
+        {0x10f, std::bit_cast<std::uint32_t>(2.0f * Width)}, {0x110, std::bit_cast<std::uint32_t>(2.0f * Width)},
+        {0x111, std::bit_cast<std::uint32_t>(2.0f * Height)}, {0x112, std::bit_cast<std::uint32_t>(2.0f * Height)},
+        {0x113, std::bit_cast<std::uint32_t>(1.0f)}, {0x114, 0},
+        {0xb4, 0}, {0xb5, std::bit_cast<std::uint32_t>(1.0f)},
+        {0x2fe, 0}, {0x302, 0}, {0x306, 0}, {0x30a, 0},
+        {0x318, static_cast<std::uint32_t>(target >> 8u)}, {0x390, static_cast<std::uint32_t>(target >> 40u)},
+    };
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> shader{
+        {0xc8, static_cast<std::uint32_t>(vertex >> 8u)}, {0xc9, static_cast<std::uint32_t>(vertex >> 40u)}, {0x8b, 1u << 1u}, {0x8c, 0},
+        {0x008, static_cast<std::uint32_t>(pixel >> 8u)}, {0x009, static_cast<std::uint32_t>(pixel >> 40u)}, {0x00b, 1u << 1u}, {0x00c, std::bit_cast<std::uint32_t>(1.0f)},
+    };
+    Append(commands, 0x79, {0x242, 4});
+    Append(commands, 0x79, {0x24a, 0});
+    for (const auto& [offset, value] : context) Append(commands, 0x69, {offset, value});
+    for (const auto& [offset, value] : shader) Append(commands, 0x76, {offset, value});
+    Append(commands, 0x2d, {3, 2});
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(std::uint64_t address, std::size_t bytes) {
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (16u << 16u), static_cast<std::uint32_t>(bytes / 16u), 0x4bfacu};
+}
+
+void AppendDispatch(std::vector<std::uint32_t>& commands, const std::uint32_t* program, const std::vector<std::uint32_t>& userData, std::uint32_t groups) {
+    const auto code = reinterpret_cast<std::uintptr_t>(program);
+    Append(commands, 0x76, {0x207, 64, 1, 1});
+    Append(commands, 0x76, {0x20c, static_cast<std::uint32_t>(code >> 8u), static_cast<std::uint32_t>(code >> 40u)});
+    Append(commands, 0x76, {0x213, (static_cast<std::uint32_t>(userData.size()) << 1u) | 0x80u});
+    commands.push_back(0xc0007600u | (static_cast<std::uint32_t>(userData.size()) << 16u));
+    commands.push_back(0x240u);
+    commands.insert(commands.end(), userData.begin(), userData.end());
+    commands.insert(commands.end(), {0xc0031500u, groups, 1, 1, 0x41u});
+}
+
+void AppendFill(std::vector<std::uint32_t>& commands, std::uint64_t target, const std::uint32_t* copied, std::size_t bytes) {
+    const auto direct = BufferDescriptor(target, bytes);
+    const auto other = BufferDescriptor(reinterpret_cast<std::uintptr_t>(copied), bytes);
+    std::vector<std::uint32_t> userData(direct.begin(), direct.end());
+    userData.insert(userData.end(), {Filled, Filled, Filled, Filled});
+    userData.insert(userData.end(), other.begin(), other.end());
+    AppendDispatch(commands, PairFillCode.data(), userData, static_cast<std::uint32_t>(bytes / 1024u));
+}
+
+void AppendLabel(std::vector<std::uint32_t>& commands, std::uint64_t label) {
+    commands.insert(commands.end(), {0xc0064900u, 0x514u, 1u << 29u, static_cast<std::uint32_t>(label), static_cast<std::uint32_t>(label >> 32u), LabelValue, 0, 0});
+}
+
+void AppendGate(std::vector<std::uint32_t>& commands, std::uint32_t function, std::uint32_t reference) {
+    const auto gate = reinterpret_cast<std::uintptr_t>(&Gate);
+    commands.insert(commands.end(), {0xc0053c00u, 0x10u | function, static_cast<std::uint32_t>(gate), static_cast<std::uint32_t>(gate >> 32u), reference, 0xffffffffu, 0x19u});
+}
+
+void Submit(std::vector<std::uint32_t>& commands) {
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    AgcDriver::Submit(&packet, 0);
+}
+
+class Region {
+public:
+    explicit Region(std::size_t bytes) : bytes(bytes), arena(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (arena) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            if (data != nullptr) GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (arena) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "linear target direct write completion: cannot map the guest region");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "linear target direct write completion: cannot allocate the guest region");
+        auto* words = reinterpret_cast<std::uint32_t*>(data);
+        std::fill(words, words + bytes / 4u, Untouched);
+        GuestAllocations::Mutation().Add(data, bytes, true, true, true);
+    }
+    ~Region() {
+        GuestAllocations::Mutation().Remove(data);
+#ifdef _WIN32
+        if (arena) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (arena) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Region(const Region&) = delete;
+    Region& operator=(const Region&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::uint32_t Word(std::size_t offset) const { return std::atomic_ref<std::uint32_t>(*reinterpret_cast<std::uint32_t*>(data + offset)).load(std::memory_order_acquire); }
+    void Put(std::size_t offset, std::uint32_t value) const { std::atomic_ref<std::uint32_t>(*reinterpret_cast<std::uint32_t*>(data + offset)).store(value, std::memory_order_release); }
+    std::size_t bytes;
+
+private:
+    bool arena;
+    std::uint8_t* data = nullptr;
+};
+
+void Import(const Region& region) {
+    const auto buffer = BufferDescriptor(region.Address(), region.bytes);
+    std::vector<std::uint32_t> userData(buffer.begin(), buffer.end());
+    userData.insert(userData.end(), {Untouched, Untouched, Untouched, Untouched});
+    std::vector<std::uint32_t> commands;
+    AppendDispatch(commands, FillCode.data(), userData, static_cast<std::uint32_t>(region.bytes / 1024u));
+    Submit(commands);
+    AgcDriverWaitIdle_nid_postfix();
+}
+
+bool Covered(std::uint32_t x, std::uint32_t y) {
+    return std::any_of(Scissors.begin(), Scissors.end(), [&](const std::array<std::uint32_t, 4>& scissor) { return x >= scissor[0] && x < scissor[2] && y >= scissor[1] && y < scissor[3]; });
+}
+
+std::string Mismatches(const Region& region, bool drawsLast) {
+    std::size_t covered = 0, wrongCovered = 0, filledCovered = 0, wrongElsewhere = 0, drawnElsewhere = 0;
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const auto value = region.Word(y * Pitch + x * 4u);
+            if (drawsLast && Covered(x, y)) {
+                ++covered;
+                wrongCovered += value == Drawn ? 0u : 1u;
+                filledCovered += value == Filled ? 1u : 0u;
+            } else {
+                wrongElsewhere += value == Filled ? 0u : 1u;
+                drawnElsewhere += value == Drawn ? 1u : 0u;
+            }
+        }
+    }
+    if (wrongCovered == 0 && wrongElsewhere == 0) return {};
+    const auto elsewhere = std::to_string(std::size_t{Width} * Height - covered);
+    if (!drawsLast) return std::to_string(wrongElsewhere) + " of the " + elsewhere + " pixels do not hold the fill's pattern (" + std::to_string(drawnElsewhere) + " hold the draws' color)";
+    return std::to_string(wrongCovered) + " of the " + std::to_string(covered) + " pixels the draws covered do not hold their color (" + std::to_string(filledCovered) + " hold the fill's pattern) and " + std::to_string(wrongElsewhere) + " of the " + elsewhere + " pixels outside them do not hold the fill's pattern";
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+std::string CpuStoreMismatches(const Region& region) {
+    std::size_t covered = 0, wrongCovered = 0, filledCovered = 0, others = 0, wrongOthers = 0;
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const auto offset = y * Pitch + x * 4u;
+            if (offset == CpuStoreOffset) continue;
+            const auto value = region.Word(offset);
+            if (x >= BlockZeroScissor[0] && x < BlockZeroScissor[2] && y >= BlockZeroScissor[1] && y < BlockZeroScissor[3]) {
+                ++covered;
+                wrongCovered += value == Drawn ? 0u : 1u;
+                filledCovered += value == Filled ? 1u : 0u;
+            } else {
+                ++others;
+                wrongOthers += value == (offset < BlockBytes ? Filled : Untouched) ? 0u : 1u;
+            }
+        }
+    }
+    std::vector<std::string> parts;
+    if (wrongCovered != 0) parts.push_back(std::to_string(wrongCovered) + " of the " + std::to_string(covered) + " pixels the draw covered in block 0 do not hold its color (" + std::to_string(filledCovered) + " hold the fill's pattern)");
+    if (const auto stored = region.Word(CpuStoreOffset); stored != CpuStored) parts.push_back("the CPU store in block 1 holds " + Hex(stored) + ", not " + Hex(CpuStored));
+    if (wrongOthers != 0) parts.push_back(std::to_string(wrongOthers) + " of the " + std::to_string(others) + " other pixels do not hold the fill's pattern in block 0 or the untouched pattern in block 1");
+    std::string text;
+    for (const auto& part : parts) text += (text.empty() ? "" : "; ") + part;
+    return text;
+}
+
+bool Await(const std::function<bool()>& done) {
+    const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5);
+    while (!done()) {
+        if (std::chrono::steady_clock::now() >= deadline) return false;
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+    return true;
+}
+
+enum class Order { FillFirst, DrawsFirst };
+
+bool Run(Order order, const Region& region, std::vector<std::string>& failures) {
+    const bool fillFirst = order == Order::FillFirst;
+    const char* name = fillFirst ? "fill, then draws" : "draws, then fill";
+    const auto target = region.Address();
+    Import(region);
+    if (!AgcDriver::GuestMemory::Watched(target, TargetBytes)) return false;
+    std::vector<std::uint32_t> copied(TargetBytes / 4u + 4u, Untouched);
+    GuestAllocations::Mutation().Add(copied.data(), TargetBytes + 16u, true, true, true);
+    struct Unregister {
+        void* data;
+        ~Unregister() { GuestAllocations::Mutation().Remove(data); }
+    } unregister{copied.data()};
+    Gate.store(0, std::memory_order_release);
+    std::vector<std::uint32_t> commands;
+    if (fillFirst) AppendFill(commands, target, copied.data(), TargetBytes);
+    for (const auto& scissor : Scissors) AppendDraw(commands, target, scissor);
+    if (!fillFirst) AppendFill(commands, target, copied.data(), TargetBytes);
+    AppendLabel(commands, target + LabelOffset);
+    if (fillFirst) AppendGate(commands, WaitEqual, 1);
+    Submit(commands);
+    std::string failure;
+    if (fillFirst) {
+        if (!Await([&] { return region.Word(LabelOffset) == LabelValue; })) failure = "the label after the draws never landed";
+        else if (const auto mismatches = Mismatches(region, true); !mismatches.empty()) failure = "when the label after the draws landed: " + mismatches;
+        Gate.store(1, std::memory_order_release);
+        AgcDriverWaitIdle_nid_postfix();
+    } else {
+        AgcDriverWaitIdle_nid_postfix();
+        if (region.Word(LabelOffset) != LabelValue) failure = "the label after the fill had not landed by the device wait";
+        else if (const auto mismatches = Mismatches(region, false); !mismatches.empty()) failure = "after the label and the device wait: " + mismatches;
+    }
+    AgcDriver::Graphics::StorageTexture::FlushPending(target, region.bytes, nullptr, "test release");
+    std::printf("linear target direct write completion, %s: %s\n", name, failure.empty() ? "the later writer's values" : failure.c_str());
+    if (!failure.empty()) failures.push_back(std::string(name) + ": " + failure);
+    return true;
+}
+
+bool RunCpuStore(bool watched, const Region& region, std::vector<std::string>& failures) {
+    const std::string name = std::string("fill, draw in block 0, CPU store in block 1, ") + (watched ? "write-watched" : "unwatched") + " memory";
+    const auto target = region.Address();
+    Import(region);
+    if (watched && !AgcDriver::GuestMemory::Watched(target, TargetBytes)) return false;
+    if (!watched) {
+        AgcDriver::GuestMemory::Unwatch(target, region.bytes);
+        Require(!AgcDriver::GuestMemory::Watched(target, region.bytes), "linear target direct write completion: the region is still write-watched");
+    }
+    std::vector<std::uint32_t> copied(TargetBytes / 4u + 4u, Untouched);
+    GuestAllocations::Mutation().Add(copied.data(), TargetBytes + 16u, true, true, true);
+    struct Unregister {
+        void* data;
+        ~Unregister() { GuestAllocations::Mutation().Remove(data); }
+    } unregister{copied.data()};
+    Gate.store(0, std::memory_order_release);
+    std::vector<std::uint32_t> commands;
+    AppendFill(commands, target, copied.data(), BlockBytes);
+    AppendDraw(commands, target, BlockZeroScissor);
+    AppendGate(commands, WaitAtLeast, 1);
+    AppendLabel(commands, target + LabelOffset);
+    AppendGate(commands, WaitAtLeast, 2);
+    Submit(commands);
+    const auto completed = [&] { return std::atomic_ref<std::uint32_t>(copied[0]).load(std::memory_order_acquire) == Filled; };
+    bool pendingAtStore = false;
+    const bool recorded = Await([&] {
+        std::unique_lock hold(AgcDriver::GuestMemory::GpuMutex());
+        if (AgcDriver::Graphics::StorageTexture::FindLive(target, TargetBytes) == nullptr) return false;
+        pendingAtStore = !completed();
+        region.Put(CpuStoreOffset, CpuStored);
+        return true;
+    });
+    Gate.store(1, std::memory_order_release);
+    std::string failure;
+    bool completedAtLabel = false;
+    if (!recorded) failure = "the draw into block 0 never made the target's image";
+    else if (!Await([&] { return region.Word(LabelOffset) == LabelValue; })) failure = "the label after the draw never landed";
+    else {
+        completedAtLabel = completed();
+        failure = CpuStoreMismatches(region);
+        if (!failure.empty()) failure = "when the label after the draw landed: " + failure;
+        if (watched && (!pendingAtStore || !completedAtLabel)) failure += std::string(failure.empty() ? "" : "; ") + (pendingAtStore ? "the fill's completion had not run when the label landed" : "the fill's completion had run before the CPU store") + ", so the ordering under test did not occur";
+    }
+    const bool watchedAtLabel = AgcDriver::GuestMemory::Watched(target, TargetBytes);
+    Gate.store(2, std::memory_order_release);
+    AgcDriverWaitIdle_nid_postfix();
+    AgcDriver::Graphics::StorageTexture::FlushPending(target, region.bytes, nullptr, "test release");
+    std::printf("linear target direct write completion, %s (watched at the label: %d, completion pending at the CPU store: %d, run by the label: %d): %s\n", name.c_str(), watchedAtLabel ? 1 : 0, pendingAtStore ? 1 : 0, completedAtLabel ? 1 : 0, failure.empty() ? "the draw's color and the CPU store" : failure.c_str());
+    if (!failure.empty()) failures.push_back(name + ": " + failure);
+    return true;
+}
+
+}
+
+int main() {
+    try {
+        {
+            const auto device = OpenVulkanTestDevice();
+            if (!device) return VulkanTestSkipped;
+            if (AgcDriver::VulkanDeviceTestAccess::ContextOf(*device).hostImportAlignment == 0) {
+                std::puts("linear target direct write completion tests skipped, host imports are unavailable on this device, so the target is never resident");
+                return VulkanTestSkipped;
+            }
+        }
+        std::vector<std::unique_ptr<Region>> regions;
+        struct ShutdownGuard {
+            ~ShutdownGuard() { try { AgcDriverShutdown_nid_postfix(); } catch (...) {} }
+        } shutdown;
+        Registered(VertexHeader, VertexCode.data(), sizeof(VertexCode), 2);
+        Registered(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1);
+        const auto unwatchedImports = [] {
+            std::puts("linear target direct write completion tests skipped, the device does not keep host imports write-watched, so no completion stamp over the target is tracked");
+            return VulkanTestSkipped;
+        };
+        std::vector<std::string> failures;
+        for (const auto order : {Order::FillFirst, Order::DrawsFirst}) {
+            regions.push_back(std::make_unique<Region>(RegionBytes));
+            if (!Run(order, *regions.back(), failures)) return unwatchedImports();
+        }
+        for (const bool watched : {true, false}) {
+            regions.push_back(std::make_unique<Region>(RegionBytes));
+            if (!RunCpuStore(watched, *regions.back(), failures)) return unwatchedImports();
+        }
+        if (!failures.empty()) {
+            std::string message = "linear target direct write completion: at a completion point, the memory of a resident linear target did not hold the later of a direct buffer write with completion work and draws into it, or lost a CPU store made outside the draws:";
+            for (const auto& failure : failures) message += "\n  " + failure;
+            Require(false, message);
+        }
+        AgcDriverShutdown_nid_postfix();
+        std::puts("linear target direct write completion tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetDispatchAccess.cpp

@@ -0,0 +1,300 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using AgcDriver::Graphics::StorageTexture;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 16;
+constexpr std::uint32_t Threads = 64;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t TargetBytes = std::size_t{Width} * Height * 16u;
+constexpr std::uint32_t Rgba32FloatInfo = 0x8738u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t Float32AbgrExport = 9;
+constexpr std::uint32_t ReadX = 10;
+constexpr std::uint32_t ReadY = 6;
+constexpr std::uint32_t Drawn = 0x3f800000u;
+constexpr std::uint32_t Unwritten = 0xdeadbeefu;
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 7> CopyCode{0x7e000280u, 0xdc308000u, 0x01000000u, 0xbf8c3f70u, 0xdc708000u, 0x00020100u, 0xbf810000u};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::array<std::uint32_t, 16>, 2> Programs{};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+std::span<const std::uint32_t> Program(std::size_t index, bool blue) {
+    auto& code = Programs.at(index);
+    std::size_t at = 0;
+    code[at++] = 0x36000000u | (20u << 17u) | 0xffu;
+    code[at++] = 0x0000ffffu;
+    code[at++] = 0x2c000000u | (21u << 17u) | 0x90u;
+    code[at++] = 0x7e000c00u | (20u << 17u) | (256u + 20u);
+    code[at++] = 0x7e000c00u | (21u << 17u) | (256u + 21u);
+    code[at++] = (blue ? 0x7e0002f2u : 0x7e000280u) | (22u << 17u);
+    code[at++] = 0x7e0002f2u | (23u << 17u);
+    code[at++] = 0xf800180fu;
+    code[at++] = 0x17161514u;
+    code[at++] = 0xbf810000u;
+    return std::span<const std::uint32_t>(code.data(), at);
+}
+
+struct Block {
+    explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (watched) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (watched) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map a guest block");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "cannot allocate a guest block");
+        Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "a guest block is not write-watched");
+        GuestAllocations::Mutation mutation;
+        mutation.Add(data, bytes, true, true, true);
+    }
+    ~Block() {
+        StorageTexture::FlushPending(Address(), bytes, nullptr, "test release");
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Remove(data);
+        }
+#ifdef _WIN32
+        if (watched) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (watched) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Block(const Block&) = delete;
+    Block& operator=(const Block&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::size_t bytes;
+    bool watched;
+    std::uint8_t* data = nullptr;
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block) {
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(block.Address() >> 8u);
+    cx[0x31b] = 0;
+    cx[0x31c] = Rgba32FloatInfo;
+    cx[0x31d] = 0;
+    cx[0x323] = 0;
+    cx[0x324] = 0;
+    cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u);
+    cx[0x3b8] = LinearAttrib3;
+    cx[0x390] = static_cast<std::uint32_t>(block.Address() >> 40u);
+    auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    color.exportIndex = 0;
+    return color;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::span<const std::uint32_t> pixelCode, VkRect2D scissor) {
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast<std::uint32_t>(Triangle.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionFixedPoint);
+    pixel.targetOutputMode.at(0) = Float32AbgrExport;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::vector<std::uint32_t> pixelUserData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.blendEnable = VK_FALSE;
+    blend.colorWriteMask = 0xf;
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = scissor;
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, std::uint64_t source, std::uint64_t destination) {
+    constexpr std::uint32_t waveSize = 64;
+    const std::array<std::uint32_t, 4> userData{static_cast<std::uint32_t>(source), static_cast<std::uint32_t>(source >> 32u), static_cast<std::uint32_t>(destination), static_cast<std::uint32_t>(destination >> 32u)};
+    const std::span<const std::uint32_t> code(CopyCode);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+std::uint32_t Dword(const std::uint8_t* bytes) {
+    std::uint32_t value = 0;
+    std::memcpy(&value, bytes, sizeof(value));
+    return value;
+}
+
+bool Expect(AgcDriver::VulkanDevice& device) {
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    if (!context.bufferDeviceAddress) {
+        std::puts("skipped, the device lacks bufferDeviceAddress");
+        return false;
+    }
+    Block block(16 * BlockBytes);
+    std::fill(block.data, block.data + block.bytes, std::uint8_t{0});
+    Block output(BlockBytes);
+    for (std::size_t offset = 0; offset < output.bytes; offset += sizeof(Unwritten)) std::memcpy(output.data + offset, &Unwritten, sizeof(Unwritten));
+    if (AgcDriver::Graphics::HostImportFor(context, block.Address(), block.bytes) == nullptr) {
+        std::puts("skipped, host imports are unavailable");
+        return false;
+    }
+    const auto color = DecodeTarget(block);
+    Require(color.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && color.bytes == TargetBytes, "the target is not a linear R32G32B32A32 target");
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(recorder != nullptr, "the device has no recorder");
+    Draw(device, color, Program(0, false), {{0, 0}, {Width, Height}});
+    device.WaitIdle();
+    Require(StorageTexture::FindPending(block.Address(), TargetBytes) == nullptr, "the linear target was still pending after the device wait");
+    const auto blueOffset = (static_cast<std::size_t>(ReadY) * Width + ReadX) * 16u + 8u;
+    Require(Dword(block.data + blueOffset) == 0u, "the first draw left " + Hex(Dword(block.data + blueOffset)) + " in the blue channel of pixel (" + std::to_string(ReadX) + ", " + std::to_string(ReadY) + ")");
+    Draw(device, color, Program(1, true), {{8, 4}, {16, 8}});
+    Require(recorder->Recording(), "the draw into the linear target was not left in the open batch");
+    Dispatch(device, block.Address() + blueOffset, output.Address());
+    device.WaitIdle();
+    const auto read = Dword(output.data);
+    Require(read == Drawn, "the address-based dispatch recorded after the draw read " + Hex(read) + " from the blue channel of pixel (" + std::to_string(ReadX) + ", " + std::to_string(ReadY) + "), not the drawn " + Hex(Drawn));
+    Require(Dword(block.data + blueOffset) == Drawn, "the second draw left " + Hex(Dword(block.data + blueOffset)) + " in the blue channel of pixel (" + std::to_string(ReadX) + ", " + std::to_string(ReadY) + ")");
+    return true;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (!Expect(*device)) return VulkanTestSkipped;
+        std::puts("linear target dispatch access tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetDrawBindings.cpp

@@ -0,0 +1,334 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using AgcDriver::Graphics::StorageTexture;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 16;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t TargetBytes = std::size_t{Width} * Height * 16u;
+constexpr std::size_t CopyTargetOffset = 0x40000;
+constexpr std::size_t FirstSourceOffset = 0x80000;
+constexpr std::size_t SecondSourceOffset = 0x90000;
+constexpr std::size_t PointedOffset = 0xa0000;
+constexpr std::uint32_t Rgba32FloatInfo = 0x8738u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t Float32AbgrExport = 9;
+constexpr std::uint32_t FirstSource = 0x3e800000u;
+constexpr std::uint32_t SecondSource = 0x3f400000u;
+constexpr std::uint32_t Pointed = 0x3f800000u;
+constexpr std::uint32_t Unwritten = 0xdeadbeefu;
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 7> PixelOpaque{0x7e100280u, 0x7e120280u, 0x7e140280u, 0x7e1602f2u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 11> PixelRead{0x7e1a0280u, 0xdc308000u, 0x0c02000du, 0xbf8c3f70u, 0x7e10030cu, 0x7e120280u, 0x7e140280u, 0x7e160280u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 13> PixelCopy{0x7e040280u, 0xe0301000u, 0x80010102u, 0xbf8c3f70u, 0xe0701000u, 0x80020102u, 0x7e100301u, 0x7e120280u, 0x7e140280u, 0x7e1602f2u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::uint32_t, 4> FirstOutput{};
+alignas(256) std::array<std::uint32_t, 4> SecondOutput{};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+std::array<std::uint32_t, 4> RawBuffer(std::uint64_t address, std::uint32_t bytes) {
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+struct Block {
+    explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (watched) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (watched) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map the color targets");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "cannot allocate the color targets");
+        Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color targets are not write-watched");
+        GuestAllocations::Mutation mutation;
+        mutation.Add(data, bytes, true, true, true);
+    }
+    ~Block() {
+        StorageTexture::FlushPending(Address(), bytes, nullptr, "test release");
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Remove(data);
+        }
+#ifdef _WIN32
+        if (watched) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (watched) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Block(const Block&) = delete;
+    Block& operator=(const Block&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::size_t bytes;
+    bool watched;
+    std::uint8_t* data = nullptr;
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(std::uint64_t address) {
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(address >> 8u);
+    cx[0x31b] = 0;
+    cx[0x31c] = Rgba32FloatInfo;
+    cx[0x31d] = 0;
+    cx[0x323] = 0;
+    cx[0x324] = 0;
+    cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u);
+    cx[0x3b8] = LinearAttrib3;
+    cx[0x390] = static_cast<std::uint32_t>(address >> 40u);
+    auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    color.exportIndex = 0;
+    return color;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::span<const std::uint32_t> pixelCode, std::span<const std::uint32_t> pixelUserData, VkRect2D scissor) {
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast<std::uint32_t>(Triangle.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionFixedPoint);
+    pixel.targetOutputMode.at(0) = Float32AbgrExport;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.blendEnable = VK_FALSE;
+    blend.colorWriteMask = 0xf;
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = scissor;
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+}
+
+std::array<std::uint32_t, 12> CopyUserData(std::uint64_t source, const std::array<std::uint32_t, 4>& output) {
+    std::array<std::uint32_t, 12> userData{};
+    const auto read = RawBuffer(source, 16u);
+    const auto written = RawBuffer(reinterpret_cast<std::uintptr_t>(output.data()), 16u);
+    std::copy(read.begin(), read.end(), userData.begin() + 4);
+    std::copy(written.begin(), written.end(), userData.begin() + 8);
+    return userData;
+}
+
+std::array<float, 4> Texel(const std::uint8_t* target, std::uint32_t x, std::uint32_t y) {
+    std::array<float, 4> texel{};
+    std::memcpy(texel.data(), target + (static_cast<std::size_t>(y) * Width + x) * 16u, sizeof(texel));
+    return texel;
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+std::string Describe(const std::array<float, 4>& texel) {
+    return "(" + std::to_string(texel[0]) + ", " + std::to_string(texel[1]) + ", " + std::to_string(texel[2]) + ", " + std::to_string(texel[3]) + ")";
+}
+
+void ExpectTexels(const std::uint8_t* target, const char* what, std::uint32_t firstX, std::uint32_t lastX, const std::array<float, 4>& expected) {
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = firstX; x < lastX; ++x) {
+            const auto texel = Texel(target, x, y);
+            Require(texel == expected, std::string(what) + ": pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") holds " + Describe(texel) + ", expected " + Describe(expected));
+        }
+    }
+}
+
+bool Expect(AgcDriver::VulkanDevice& device, bool unwatch) {
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    if (!context.bufferDeviceAddress) {
+        std::puts("skipped, the device lacks bufferDeviceAddress");
+        return false;
+    }
+    Block block(16 * BlockBytes);
+    std::fill(block.data, block.data + block.bytes, std::uint8_t{0});
+    const auto put = [&](std::size_t offset, std::uint32_t value, std::size_t dwords) {
+        for (std::size_t dword = 0; dword < dwords; ++dword) std::memcpy(block.data + offset + dword * 4u, &value, sizeof(value));
+    };
+    put(FirstSourceOffset, FirstSource, 4);
+    put(SecondSourceOffset, SecondSource, 4);
+    put(PointedOffset, Pointed, 1);
+    FirstOutput.fill(Unwritten);
+    SecondOutput.fill(Unwritten);
+    if (AgcDriver::Graphics::HostImportFor(context, block.Address(), block.bytes) == nullptr) {
+        std::puts("skipped, host imports are unavailable");
+        return false;
+    }
+    const auto drawn = DecodeTarget(block.Address());
+    const auto copied = DecodeTarget(block.Address() + CopyTargetOffset);
+    Require(drawn.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && drawn.bytes == TargetBytes && copied.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && copied.bytes == TargetBytes, "the targets are not linear R32G32B32A32 targets");
+    auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(recorder != nullptr && context.descriptorCache != nullptr, "the device has no recorder or descriptor cache");
+    const auto pending = [](std::uint64_t address) { return StorageTexture::FindPending(address, TargetBytes) != nullptr; };
+    const VkRect2D whole{{0, 0}, {Width, Height}};
+    Draw(device, drawn, PixelOpaque, {}, whole);
+    Draw(device, copied, PixelOpaque, {}, whole);
+    device.WaitIdle();
+    Require(!pending(drawn.address) && !pending(copied.address), "a linear target was still pending after the device wait");
+    if (unwatch) AgcDriver::GuestMemory::Unwatch(block.Address(), block.bytes);
+    const bool watched = AgcDriver::GuestMemory::Watched(copied.address, TargetBytes);
+    Require(!unwatch || !watched, "the copied target is still write-watched after its block was taken out of the write watch");
+    const auto submitted = recorder->Submissions();
+    const auto pointed = block.Address() + PointedOffset;
+    const std::array<std::uint32_t, 4> pointer{0u, 0u, static_cast<std::uint32_t>(pointed), static_cast<std::uint32_t>(pointed >> 32u)};
+    Draw(device, drawn, PixelRead, pointer, whole);
+    Require(pending(drawn.address), "the address-based draw did not leave its linear target pending");
+    Require(recorder->Recording() && recorder->Submissions() == submitted, "the address-based draw did not stay in the open batch");
+    const auto setsBefore = context.descriptorCache->Counters().sets;
+    const auto firstCopy = CopyUserData(block.Address() + FirstSourceOffset, FirstOutput);
+    Draw(device, copied, PixelCopy, firstCopy, {{0, 0}, {Width / 2u, Height}});
+    Require(!pending(drawn.address), "the draw writing a copied buffer did not store the drawn linear target first");
+    Require(pending(copied.address), "the first copying draw did not leave its linear target pending");
+    Require(recorder->Recording() && recorder->Submissions() == submitted + 1, "storing the drawn linear target did not submit the batch the address-based draw read it in (" + std::to_string(recorder->Submissions() - submitted) + " submissions)");
+    const auto setsBetween = context.descriptorCache->Counters().sets;
+    const auto secondCopy = CopyUserData(block.Address() + SecondSourceOffset, SecondOutput);
+    Draw(device, copied, PixelCopy, secondCopy, {{static_cast<std::int32_t>(Width / 2u), 0}, {Width / 2u, Height}});
+    if (watched) Require(recorder->Recording() && recorder->Submissions() == submitted + 1, "the second copying draw submitted recorded work");
+    else Require(recorder->Submissions() == submitted + 2, "the second copying draw into a stored target outside the write watch did not submit once to capture its bytes (" + std::to_string(recorder->Submissions() - submitted - 1) + " submissions)");
+    const auto setsAfter = context.descriptorCache->Counters().sets;
+    device.WaitIdle();
+    Require(FirstOutput[0] == FirstSource, "the first copying draw's output holds " + Hex(FirstOutput[0]) + ", not its source's " + Hex(FirstSource));
+    Require(SecondOutput[0] == SecondSource, "the second copying draw's output holds " + Hex(SecondOutput[0]) + ", not its source's " + Hex(SecondSource));
+    ExpectTexels(block.data + CopyTargetOffset, "the first copying draw's half", 0, Width / 2u, {0.25f, 0.0f, 0.0f, 1.0f});
+    ExpectTexels(block.data + CopyTargetOffset, "the second copying draw's half", Width / 2u, Width, {0.75f, 0.0f, 0.0f, 1.0f});
+    ExpectTexels(block.data, "the address-based draw's target", 0, Width, {1.0f, 0.0f, 0.0f, 0.0f});
+    Require(setsAfter - setsBetween == setsBetween - setsBefore, "the second copying draw allocated " + std::to_string(setsAfter - setsBetween) + " descriptor sets, the first " + std::to_string(setsBetween - setsBefore));
+    return true;
+}
+
+}
+
+int main() {
+#ifdef _WIN32
+    _putenv_s("APS5_RELEASE_ON_UNLOCK", "1");
+#else
+    setenv("APS5_RELEASE_ON_UNLOCK", "1", 1);
+#endif
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (!Expect(*device, false)) return VulkanTestSkipped;
+        if (!Expect(*device, true)) return VulkanTestSkipped;
+        std::puts("linear target draw binding tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetRectangleBound.cpp

@@ -0,0 +1,344 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t RegionBytes = 8 * BlockBytes;
+constexpr std::uint32_t Rgba32FloatInfo = 0x8738u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t Float32AbgrExport = 9;
+constexpr std::uint8_t Initial = 0x3f;
+constexpr std::uint8_t Sentinel = 0x37;
+constexpr std::uint8_t Gap = 0x29;
+constexpr std::size_t GapBytes = 0x100;
+
+struct Surface {
+    std::size_t offset;
+    std::uint32_t width;
+    std::uint32_t height;
+    std::size_t RowBytes() const { return std::size_t{width} * 16u; }
+    std::size_t Bytes() const { return RowBytes() * height; }
+};
+
+constexpr Surface Tall{0, 64, 256};
+constexpr Surface Shifted{6 * BlockBytes + GapBytes, 64, 16};
+constexpr VkRect2D Seed{{63, 255}, {1, 1}};
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::array<std::uint32_t, 16>, 2> Programs{};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+std::span<const std::uint32_t> Program(std::size_t index, bool blue) {
+    auto& code = Programs.at(index);
+    std::size_t at = 0;
+    code[at++] = 0x36000000u | (20u << 17u) | 0xffu;
+    code[at++] = 0x0000ffffu;
+    code[at++] = 0x2c000000u | (21u << 17u) | 0x90u;
+    code[at++] = 0x7e000c00u | (20u << 17u) | (256u + 20u);
+    code[at++] = 0x7e000c00u | (21u << 17u) | (256u + 21u);
+    code[at++] = (blue ? 0x7e0002f2u : 0x7e000280u) | (22u << 17u);
+    code[at++] = 0x7e0002f2u | (23u << 17u);
+    code[at++] = 0xf800180fu;
+    code[at++] = 0x17161514u;
+    code[at++] = 0xbf810000u;
+    return std::span<const std::uint32_t>(code.data(), at);
+}
+
+struct Block {
+    explicit Block(std::size_t bytes) : bytes(bytes), arena(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (arena) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (arena) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map the color targets");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "cannot allocate the color targets");
+        GuestAllocations::Mutation mutation;
+        mutation.Add(data, bytes, true, true, true);
+    }
+    ~Block() {
+        AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release");
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Remove(data);
+        }
+#ifdef _WIN32
+        if (arena) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (arena) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Block(const Block&) = delete;
+    Block& operator=(const Block&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::size_t bytes;
+    bool arena;
+    std::uint8_t* data = nullptr;
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block, const Surface& surface) {
+    const auto address = block.Address() + surface.offset;
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(address >> 8u);
+    cx[0x31b] = 0;
+    cx[0x31c] = Rgba32FloatInfo;
+    cx[0x31d] = 0;
+    cx[0x323] = 0;
+    cx[0x324] = 0;
+    cx[0x3b0] = ((surface.width - 1u) << 14u) | (surface.height - 1u);
+    cx[0x3b8] = LinearAttrib3;
+    cx[0x390] = static_cast<std::uint32_t>(address >> 40u);
+    auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    color.exportIndex = 0;
+    return color;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, const Surface& surface, std::span<const std::uint32_t> pixelCode, VkRect2D scissor) {
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast<std::uint32_t>(Triangle.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionFixedPoint);
+    pixel.targetOutputMode.at(0) = Float32AbgrExport;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::vector<std::uint32_t> pixelUserData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.blendEnable = VK_FALSE;
+    blend.colorWriteMask = 0xf;
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = {surface.width, surface.height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(surface.height), static_cast<float>(surface.width), -static_cast<float>(surface.height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = scissor;
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+}
+
+bool Inside(const VkRect2D& rect, std::uint32_t x, std::uint32_t y) {
+    return x >= static_cast<std::uint32_t>(rect.offset.x) && x < static_cast<std::uint32_t>(rect.offset.x) + rect.extent.width && y >= static_cast<std::uint32_t>(rect.offset.y) && y < static_cast<std::uint32_t>(rect.offset.y) + rect.extent.height;
+}
+
+std::uint8_t* Pixel(const Block& block, const Surface& surface, std::uint32_t x, std::uint32_t y) {
+    return block.data + surface.offset + static_cast<std::size_t>(y) * surface.RowBytes() + static_cast<std::size_t>(x) * 16u;
+}
+
+bool Holds(const std::uint8_t* pixel, std::uint8_t value) {
+    return std::all_of(pixel, pixel + 16, [&](std::uint8_t byte) { return byte == value; });
+}
+
+bool HoldsOutput(const std::uint8_t* pixel, std::uint32_t x, std::uint32_t y, float z) {
+    std::array<float, 4> texel{};
+    std::memcpy(texel.data(), pixel, sizeof(texel));
+    return texel == std::array<float, 4>{static_cast<float>(x), static_cast<float>(y), z, 1.0f};
+}
+
+bool Expect(AgcDriver::VulkanDevice& device) {
+    Block block(RegionBytes);
+    std::fill(block.data, block.data + block.bytes, Initial);
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    if (AgcDriver::Graphics::HostImportFor(context, block.Address(), block.bytes) == nullptr) return false;
+    const auto tall = DecodeTarget(block, Tall);
+    const auto shifted = DecodeTarget(block, Shifted);
+    Require(tall.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && tall.bytes == Tall.Bytes() && shifted.bytes == Shifted.Bytes(), "the targets are not linear R32G32B32A32 targets");
+    constexpr VkRect2D first{{8, 4}, {16, 16}};
+    constexpr VkRect2D second{{16, 12}, {24, 16}};
+    constexpr VkRect2D whole{{0, 0}, {Shifted.width, Shifted.height}};
+    Draw(device, tall, Tall, Program(0, false), Seed);
+    device.WaitIdle();
+    Draw(device, tall, Tall, Program(0, false), first);
+    Draw(device, tall, Tall, Program(1, true), second);
+    Draw(device, shifted, Shifted, Program(0, false), whole);
+    const std::array<std::array<std::uint32_t, 2>, 3> sentinels{{{50, 10}, {20, 40}, {30, 200}}};
+    const std::array<std::array<std::uint32_t, 2>, 2> between{{{30, 6}, {10, 25}}};
+    for (const auto& [x, y] : sentinels) std::memset(Pixel(block, Tall, x, y), Sentinel, 16);
+    for (const auto& [x, y] : between) std::memset(Pixel(block, Tall, x, y), Sentinel, 16);
+    Require((block.Address() + Shifted.offset) / BlockBytes * BlockBytes == block.Address() + Shifted.offset - GapBytes, "the shifted target does not start past a 64 KiB boundary");
+    std::memset(block.data + Shifted.offset - GapBytes, Gap, GapBytes);
+    device.WaitIdle();
+    std::size_t wrongDrawn = 0, wrongSentinels = 0, wrongBetween = 0, wrongUntouched = 0, wrongShifted = 0;
+    for (std::uint32_t y = 0; y < Tall.height; ++y) {
+        for (std::uint32_t x = 0; x < Tall.width; ++x) {
+            const auto* pixel = Pixel(block, Tall, x, y);
+            const auto at = [&](const auto& point) { return point[0] == x && point[1] == y; };
+            if (std::any_of(sentinels.begin(), sentinels.end(), at)) wrongSentinels += Holds(pixel, Sentinel) ? 0u : 1u;
+            else if (std::any_of(between.begin(), between.end(), at)) wrongBetween += Holds(pixel, Sentinel) ? 0u : 1u;
+            else if (Inside(second, x, y)) wrongDrawn += HoldsOutput(pixel, x, y, 1.0f) ? 0u : 1u;
+            else if (Inside(first, x, y) || Inside(Seed, x, y)) wrongDrawn += HoldsOutput(pixel, x, y, 0.0f) ? 0u : 1u;
+            else wrongUntouched += Holds(pixel, Initial) ? 0u : 1u;
+        }
+    }
+    for (std::uint32_t y = 0; y < Shifted.height; ++y) {
+        for (std::uint32_t x = 0; x < Shifted.width; ++x) wrongShifted += HoldsOutput(Pixel(block, Shifted, x, y), x, y, 0.0f) ? 0u : 1u;
+    }
+    const auto gapLost = static_cast<std::size_t>(std::count_if(block.data + Shifted.offset - GapBytes, block.data + Shifted.offset, [](std::uint8_t byte) { return byte != Gap; }));
+    std::printf("linear target rectangle bound: wrong drawn pixels %zu, lost sentinels %zu/%zu, lost sentinels between the rectangles %zu/%zu, changed untouched pixels %zu, wrong shifted-target pixels %zu, lost gap bytes %zu/%zu\n", wrongDrawn, wrongSentinels, sentinels.size(), wrongBetween, between.size(), wrongUntouched, wrongShifted, gapLost, GapBytes);
+    Require(wrongDrawn == 0 && wrongShifted == 0, "a drawn linear target's store lost pixels its draws wrote");
+    Require(wrongSentinels == 0 && gapLost == 0, "a drawn linear target's store overwrote CPU stores outside the drawn rectangles");
+    Require(wrongBetween == 0, "a drawn linear target's store overwrote CPU stores between two draws' scissor rectangles");
+    Require(wrongUntouched == 0, "a drawn linear target's store changed pixels no draw and no CPU store wrote");
+    return true;
+}
+
+bool ExpectManyRows(AgcDriver::VulkanDevice& device) {
+    Block block(RegionBytes);
+    std::fill(block.data, block.data + block.bytes, Initial);
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    if (AgcDriver::Graphics::HostImportFor(context, block.Address(), block.bytes) == nullptr) return false;
+    const auto tall = DecodeTarget(block, Tall);
+    constexpr std::uint32_t Rows = 17;
+    constexpr std::uint32_t Spacing = 15;
+    Draw(device, tall, Tall, Program(0, false), Seed);
+    device.WaitIdle();
+    for (std::uint32_t row = 0; row < Rows; ++row) Draw(device, tall, Tall, Program(0, false), VkRect2D{{8, static_cast<std::int32_t>(row * Spacing)}, {16, 1}});
+    constexpr std::array<std::uint32_t, 2> between{10, 70};
+    std::memset(Pixel(block, Tall, between[0], between[1]), Sentinel, 16);
+    device.WaitIdle();
+    std::size_t wrongDrawn = 0, wrongUntouched = 0;
+    for (std::uint32_t y = 0; y < Tall.height; ++y) {
+        const bool drawnRow = y % Spacing == 0 && y / Spacing < Rows;
+        for (std::uint32_t x = 0; x < Tall.width; ++x) {
+            if (x == between[0] && y == between[1]) continue;
+            const auto* pixel = Pixel(block, Tall, x, y);
+            if ((drawnRow && x >= 8 && x < 24) || Inside(Seed, x, y)) wrongDrawn += HoldsOutput(pixel, x, y, 0.0f) ? 0u : 1u;
+            else wrongUntouched += Holds(pixel, Initial) ? 0u : 1u;
+        }
+    }
+    const bool kept = Holds(Pixel(block, Tall, between[0], between[1]), Sentinel);
+    std::printf("linear target rectangle bound, %u one-row scissors: wrong drawn pixels %zu, sentinel between the rows %s, changed untouched pixels %zu\n", Rows, wrongDrawn, kept ? "kept" : "lost", wrongUntouched);
+    Require(wrongDrawn == 0, "a drawn linear target's store lost pixels its one-row draws wrote");
+    Require(kept, "a drawn linear target's store overwrote a CPU store between two one-row scissors");
+    Require(wrongUntouched == 0, "a drawn linear target's store changed pixels no one-row draw and no CPU store wrote");
+    return true;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (!Expect(*device) || !ExpectManyRows(*device)) {
+            std::puts("linear target rectangle bound test skipped: no host import on this device");
+            return VulkanTestSkipped;
+        }
+        std::puts("linear target rectangle bound tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/NativeMultisampleDepth.cpp

@@ -671,6 +671,39 @@ void CheckDrawFeedback() {
     std::puts("Native depth feedback refused through the draw resource path; a written surface refused and a depth-tested one retired on guest read");
 }
 
+void CheckDepthAttachmentAlias() {
+    const auto device = OpenVulkanTestDevice();
+    Require(device != nullptr, "native depth attachment alias requires Vulkan");
+    const auto context = device->graphicsContext();
+    constexpr VkExtent2D extent{5u, 3u};
+    constexpr std::uint32_t samples = 4u;
+    const auto bytes = static_cast<std::size_t>(MultisampledDepthBytes_nid_no_patch(extent, 4u, samples));
+    TargetMemory memory(bytes), firstMetadata(65536u), secondMetadata(65536u);
+    std::memset(memory.Data(), 0x3c, bytes);
+    std::memset(firstMetadata.Data(), 0, 65536u);
+    std::memset(secondMetadata.Data(), 0, 65536u);
+    struct Cleanup {
+        AgcDriver::VulkanDevice& device;
+        ~Cleanup() { device.WaitIdle(); ClearCachedTextures(device.Device()); ClearDepthSurfaces(device.Device()); }
+    } cleanup{*device};
+    DepthTarget first{memory.Address(), 0u, extent, VK_FORMAT_D32_SFLOAT, 0.0f, 0u, firstMetadata.Address(), false, samples};
+    first.depthSwizzle = 24u;
+    first.iterate256 = first.htilePipeAligned = first.htileDepthOnly = true;
+    const auto firstView = DepthSurfaceView(context, first);
+    const auto words = DepthSampleWords(first);
+    auto kept = DepthSurfaceTexture(context, words, DecodeTextureResource(words), VkComponentMapping{});
+    Require(kept != nullptr, "attachment alias fixture has no held native depth view");
+    auto second = first;
+    second.extent.width = 6u;
+    second.htileAddress = secondMetadata.Address();
+    Require(MultisampledDepthBytes_nid_no_patch(second.extent, 4u, samples) <= bytes, "attachment alias fixture exceeds its backing");
+    const auto secondView = DepthSurfaceView(context, second);
+    Require(secondView != VK_NULL_HANDLE && secondView != firstView, "an aliased depth attachment reused the previous layout's view");
+    Require(DepthSurfaceView(context, second) == secondView, "an unchanged native depth attachment was not reused");
+    for (std::size_t offset = 0u; offset < bytes; ++offset) Require(memory.Data()[offset] == std::byte{0x3c}, "retiring an unwritten attachment alias changed guest bytes");
+    std::puts("native depth attachment alias changes layout while retaining held GPU views");
+}
+
 enum class ClearFault { None, PartialStore, CpuWrite, DepthWrite, BadMetadata, UnprovenDispatch, PendingWrite, Overlap, PossibleStore, CompletedPossibleStore, DriverDispatch, ImportWindowWrite };
 
 void CheckClearDispatch() {
@@ -926,10 +959,10 @@ void CheckDepthTarget(Context context, VkFormat format, std::uint32_t fragments,
     Require(probe[0] == std::byte{0xa5}, "resident depth guard blocked an unrelated read");
     auto alias = target;
     alias.htileAddress += 65536u;
-    TargetRefuses([&] { DepthSurfaceView(context, alias); }, "layout or metadata changed");
+    TargetRefuses([&] { DepthSurfaceView(context, alias); }, "sample-preserving guest publication");
     alias = target;
     alias.htileDepthOnly = !target.htileDepthOnly;
-    TargetRefuses([&] { DepthSurfaceView(context, alias); }, "layout or metadata changed");
+    TargetRefuses([&] { DepthSurfaceView(context, alias); }, "sample-preserving guest publication");
     auto resource = DecodeTextureResource(words);
     for (std::uint32_t field = 0u; field < 18u; ++field) {
         auto changed = resource;
@@ -1041,7 +1074,7 @@ void CheckHtileRouting(Context context, VkFormat format, std::uint32_t fragments
     auto alias = target;
     alias.samples = 1u;
     alias.address = single.Address();
-    TargetRefuses([&] { DepthSurfaceView(context, alias); }, "layout or metadata changed");
+    Require(DepthSurfaceView(context, alias) != VK_NULL_HANDLE, "a single-sample bind could not retire an unwritten native HTILE owner");
     Require(untouched(), "a single-sample bind over a native owner's HTILE stored expanded words into it");
     std::printf("Native depth format=%u samples=%u: binds over the owner's HTILE left its words alone\n", static_cast<std::uint32_t>(format), fragments);
 }
@@ -1303,6 +1336,7 @@ int main(int argc, char** argv) {
         }
         CheckDrawFeedback();
         const auto context = device->GetContext();
+        if (argc == 1) CheckDepthAttachmentAlias();
         std::unique_lock lock(AgcDriver::GuestMemory::GpuMutex());
         Require(DepthSupported(context, VK_FORMAT_D32_SFLOAT, 4u), "the native depth regression requires sampled D32 with four samples");
         if (argc == 2 && std::string_view(argv[1]) == "--clear-refusals") {

core/libs/prx/libSceAgcDriver/tests/execution/PipelinedDispatch.cpp

@@ -132,10 +132,8 @@ void Submit(std::vector<std::uint32_t>& words) {
 int main() {
 #ifdef _WIN32
     _putenv_s("APS5_PIPELINED_DRAWS", "1");
-    _putenv_s("APS5_PIPELINE_DMA", "1");
 #else
     setenv("APS5_PIPELINED_DRAWS", "1", 1);
-    setenv("APS5_PIPELINE_DMA", "1", 1);
 #endif
     try {
         {

core/libs/prx/libSceAgcDriver/tests/execution/PipelinedDma.cpp

@@ -74,10 +74,8 @@ void ExpectNoDrain(std::uint32_t opcode, std::uint64_t before, const char* what)
 int main() {
 #ifdef _WIN32
     _putenv_s("APS5_PIPELINED_DRAWS", "1");
-    _putenv_s("APS5_PIPELINE_DMA", "1");
 #else
     setenv("APS5_PIPELINED_DRAWS", "1", 1);
-    setenv("APS5_PIPELINE_DMA", "1", 1);
 #endif
     try {
         alignas(256) static std::array<std::uint32_t, 1024> first{};

core/libs/prx/libSceAgcDriver/tests/execution/PipelinedMetadataPass.cpp

@@ -120,11 +120,9 @@ void expectEliminated(const std::string& what) {
 int main() {
 #ifdef _WIN32
     _putenv_s("APS5_PIPELINED_DRAWS", "1");
-    _putenv_s("APS5_PIPELINE_DMA", "1");
     _putenv_s("APS5_DRAW_PIPELINE", "1");
 #else
     setenv("APS5_PIPELINED_DRAWS", "1", 1);
-    setenv("APS5_PIPELINE_DMA", "1", 1);
     setenv("APS5_DRAW_PIPELINE", "1", 1);
 #endif
     try {

core/libs/prx/libSceAgcDriver/tests/execution/PipelinedPrefetch.cpp

@@ -106,12 +106,12 @@ int main() {
         const auto prefetched = SubmitBehindBusyPipeline(prefetch, 0x11111111u, true);
         check(prefetched.outstanding >= 2, "a DMA_DATA prefetch drained the draw pipeline: the label behind it never queued behind the held commit (" + std::to_string(prefetched.outstanding) + " outstanding)");
         check(prefetched.drains == 0, "a DMA_DATA prefetch drained the draw pipeline (" + std::to_string(prefetched.drains) + " packet drains)");
-        const auto copied = SubmitBehindBusyPipeline(copy, 0x22222222u, false);
-        check(copied.outstanding == 1, "a DMA_DATA copy did not hold the queue worker (" + std::to_string(copied.outstanding) + " outstanding)");
-        check(copied.drains == 1, "a DMA_DATA copy did not drain the draw pipeline (" + std::to_string(copied.drains) + " packet drains)");
+        const auto copied = SubmitBehindBusyPipeline(copy, 0x22222222u, true);
+        check(copied.outstanding >= 2, "a DMA_DATA copy that misses every queued write drained the draw pipeline: the label behind it never queued behind the held commit (" + std::to_string(copied.outstanding) + " outstanding)");
+        check(copied.drains == 0, "a DMA_DATA copy that misses every queued write drained the draw pipeline (" + std::to_string(copied.drains) + " packet drains)");
         for (std::size_t i = 0; i < Destination.size(); ++i) check(Destination[i] == (i < 128 ? Source[i] : 0u), "the copy did not land, or wrote past its bytes");
         LibcRunShutdown_nid_postfix();
-        std::puts("a DMA_DATA prefetch keeps the draw pipeline, a copy drains it");
+        std::puts("a DMA_DATA prefetch and a copy that misses every queued write keep the draw pipeline");
         return 0;
     } catch (const std::exception& error) {
         std::cerr << error.what() << '\n';