Merge main into fix/agc-sample-location-grid-exact

.github/workflows/build.yml

@@ -70,23 +70,77 @@ jobs:
             const { owner, repo } = context.repo;
             const workflow = '.github/workflows/build.yml';
             const libc = 'core/libs/prx/libc/';
+            const jobOfLine = (text) => {
+              let inJobs = false;
+              let job = '';
+              return text.split('\n').map((line) => {
+                if (/^\S/.test(line)) {
+                  inJobs = /^jobs:\s*$/.test(line);
+                  job = '';
+                } else if (inJobs) {
+                  job = /^  ([\w-]+):\s*$/.exec(line)?.[1] ?? job;
+                }
+                return job;
+              });
+            };
+            const patchedJobs = (patch, text) => {
+              const jobs = jobOfLine(text);
+              const touched = new Set();
+              let line = 1;
+              for (const row of patch.split('\n')) {
+                const hunk = /^@@ -\d+(?:,\d+)? \+(\d+)/.exec(row);
+                if (hunk) {
+                  line = Number(hunk[1]);
+                } else if (row.startsWith('+')) {
+                  touched.add(jobs[line - 1] ?? '');
+                  line += 1;
+                } else if (row.startsWith('-')) {
+                  touched.add(jobs[line - 2] ?? '').add(jobs[line - 1] ?? '');
+                } else if (row.startsWith(' ')) {
+                  line += 1;
+                }
+              }
+              return touched;
+            };
+            const workflowJobs = new Set();
+            const readWorkflow = async (files, ref) => {
+              const file = files.find((file) => file.filename === workflow || file.previous_filename === workflow);
+              if (!file) {
+                return;
+              }
+              if (file.filename !== workflow || !file.patch) {
+                workflowJobs.add('*');
+                return;
+              }
+              try {
+                const { data } = await github.rest.repos.getContent({ owner, repo, path: workflow, ref });
+                const text = Buffer.from(data.content, 'base64').toString('utf8');
+                patchedJobs(file.patch, text).forEach((job) => workflowJobs.add(job));
+              } catch (error) {
+                core.warning(`Could not read ${workflow} at ${ref}, so every test job runs: ${error.message}`);
+                workflowJobs.add('*');
+              }
+            };
             const select = (paths, all) => {
               const touches = (...prefixes) => all ||
                 paths.some((path) => prefixes.some((prefix) => prefix.endsWith('/') ? path.startsWith(prefix) : path === prefix));
+              const edits = (job) => ['*', '', 'eligibility', job].some((name) => workflowJobs.has(name));
               return {
-                relinker: touches(
+                relinker: edits('relinker') || touches(
                   'core/relinker/', libc, 'core/libs/prx/libkernel/AppMetadata/', 'core/libs/tests/ParamJsonParser.cpp',
                   'core/shader/recompiler/tests/CacheFileReadTests.cpp', 'core/shader/recompiler/ShaderCacheDirectory.hpp',
-                  'tools/', 'cmake/', 'CMakeLists.txt', workflow
+                  'tools/', 'cmake/', 'CMakeLists.txt'
                 ),
-                registerContext: touches('core/libs/tests/register_context/', 'core/libs/tests/RegisterContext.cpp', libc, workflow),
-                writeWatch: touches('core/libs/tests/write_watch/', 'core/libs/tests/WriteWatch.cpp', libc, workflow),
+                registerContext: edits('register-context') ||
+                  touches('core/libs/tests/register_context/', 'core/libs/tests/RegisterContext.cpp', libc),
+                writeWatch: edits('write-watch') || touches('core/libs/tests/write_watch/', 'core/libs/tests/WriteWatch.cpp', libc),
               };
             };
             const relinkerHosts = (full, linux) => JSON.stringify([
               ...(full && linux ? [] : [{ os: 'ubuntu-latest', compiler: 'g++' }]),
               ...(full ? [] : [{ os: 'windows-latest', compiler: 'g++' }]),
               { os: 'macos-latest', compiler: 'clang++' },
+              { os: 'macos-latest', compiler: 'clang++', architectures: 'x86_64' },
             ]);
             if (context.eventName === 'push') {
               if (context.ref !== 'refs/heads/main') {
@@ -102,6 +156,7 @@ jobs:
                   const files = data.files ?? [];
                   all = files.length >= 300;
                   paths = files.flatMap((file) => [file.filename, file.previous_filename].filter(Boolean));
+                  await readWorkflow(files, context.sha);
                 } catch (error) {
                   core.warning(`Could not list the files of this push, so every test job runs: ${error.message}`);
                 }
@@ -167,6 +222,7 @@ jobs:
               const files = await github.paginate(github.rest.pulls.listFiles, { owner, repo, pull_number, per_page: 100 });
               truncated ||= files.length >= 3000;
               paths.push(...files.flatMap((file) => [file.filename, file.previous_filename].filter(Boolean)));
+              await readWorkflow(files, pr.head.sha);
               pulls.push({ number: pull_number, head: pr.head.sha });
             }
             const run = select(paths, truncated);
@@ -259,9 +315,40 @@ jobs:
           7z x "$RUNNER_TEMP/winlibs.zip" -oC:/winlibs > /dev/null
       - if: runner.os == 'Windows'
         run: echo 'C:\winlibs\mingw64\bin' >> "$GITHUB_PATH"
-      - run: cmake -S . -B build-relinker -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DANYPS5_RELINKER_ONLY=ON -DBUILD_TESTING=ON
+      - if: ${{ matrix.architectures == 'x86_64' }}
+        run: sudo softwareupdate --install-rosetta --agree-to-license
+      - run: cmake -S . -B build-relinker -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DANYPS5_RELINKER_ONLY=ON -DBUILD_TESTING=ON -DCMAKE_OSX_ARCHITECTURES=${{ matrix.architectures }}
       - run: cmake --build build-relinker --parallel 3
-      - run: ctest --test-dir build-relinker --output-on-failure --timeout 120
+      - run: ctest --test-dir build-relinker --output-on-failure --timeout 120 --parallel 4
+
+  relinker-asan:
+    needs: eligibility
+    if: ${{ needs.eligibility.outputs.relinker == 'true' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ needs.eligibility.outputs.sha }}
+          persist-credentials: false
+      - name: Merge the pull request into main
+        if: ${{ needs.eligibility.outputs.head != '' }}
+        env:
+          PR_NUMBER: ${{ needs.eligibility.outputs.pr }}
+          PR_HEAD: ${{ needs.eligibility.outputs.head }}
+        run: |
+          read -ra numbers <<< "$PR_NUMBER"
+          read -ra heads <<< "$PR_HEAD"
+          refspecs=()
+          for number in "${numbers[@]}"; do refspecs+=("+refs/pull/$number/head:refs/remotes/pull/$number"); done
+          git fetch --no-tags --unshallow origin "${refspecs[@]}"
+          for i in "${!numbers[@]}"; do
+            if [ "$(git rev-parse "refs/remotes/pull/${numbers[i]}")" != "${heads[i]}" ]; then echo "::error::Pull request #${numbers[i]} has newer commits; start a new build"; exit 1; fi
+            if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit "refs/remotes/pull/${numbers[i]}"; then echo "::error::Pull request #${numbers[i]} does not merge cleanly into main and the pull requests before it; rebase it first"; exit 1; fi
+          done
+      - run: cmake -S . -B build-relinker-asan -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=g++ -DANYPS5_RELINKER_ONLY=ON -DANYPS5_ENABLE_ASAN=ON -DBUILD_TESTING=ON
+      - run: cmake --build build-relinker-asan --parallel 4
+      - run: ctest --test-dir build-relinker-asan --output-on-failure --timeout 120 --label-regex '^tls_function_coverage$' --no-tests=error
 
   register-context:
     needs: eligibility
@@ -275,6 +362,25 @@ jobs:
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@v7
+        with:
+          ref: ${{ needs.eligibility.outputs.sha }}
+          persist-credentials: false
+      - name: Merge the pull request into main
+        if: ${{ needs.eligibility.outputs.head != '' }}
+        shell: bash
+        env:
+          PR_NUMBER: ${{ needs.eligibility.outputs.pr }}
+          PR_HEAD: ${{ needs.eligibility.outputs.head }}
+        run: |
+          read -ra numbers <<< "$PR_NUMBER"
+          read -ra heads <<< "$PR_HEAD"
+          refspecs=()
+          for number in "${numbers[@]}"; do refspecs+=("+refs/pull/$number/head:refs/remotes/pull/$number"); done
+          git fetch --no-tags --unshallow origin "${refspecs[@]}"
+          for i in "${!numbers[@]}"; do
+            if [ "$(git rev-parse "refs/remotes/pull/${numbers[i]}")" != "${heads[i]}" ]; then echo "::error::Pull request #${numbers[i]} has newer commits; start a new build"; exit 1; fi
+            if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit "refs/remotes/pull/${numbers[i]}"; then echo "::error::Pull request #${numbers[i]} does not merge cleanly into main and the pull requests before it; rebase it first"; exit 1; fi
+          done
       - run: cmake -S core/libs/tests/register_context -B build/register-context -DCMAKE_BUILD_TYPE=${{ matrix.configuration }}
       - run: cmake --build build/register-context --parallel
       - run: ctest --test-dir build/register-context --output-on-failure --repeat until-fail:3 --no-tests=error
@@ -290,6 +396,25 @@ jobs:
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@v7
+        with:
+          ref: ${{ needs.eligibility.outputs.sha }}
+          persist-credentials: false
+      - name: Merge the pull request into main
+        if: ${{ needs.eligibility.outputs.head != '' }}
+        shell: bash
+        env:
+          PR_NUMBER: ${{ needs.eligibility.outputs.pr }}
+          PR_HEAD: ${{ needs.eligibility.outputs.head }}
+        run: |
+          read -ra numbers <<< "$PR_NUMBER"
+          read -ra heads <<< "$PR_HEAD"
+          refspecs=()
+          for number in "${numbers[@]}"; do refspecs+=("+refs/pull/$number/head:refs/remotes/pull/$number"); done
+          git fetch --no-tags --unshallow origin "${refspecs[@]}"
+          for i in "${!numbers[@]}"; do
+            if [ "$(git rev-parse "refs/remotes/pull/${numbers[i]}")" != "${heads[i]}" ]; then echo "::error::Pull request #${numbers[i]} has newer commits; start a new build"; exit 1; fi
+            if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit "refs/remotes/pull/${numbers[i]}"; then echo "::error::Pull request #${numbers[i]} does not merge cleanly into main and the pull requests before it; rebase it first"; exit 1; fi
+          done
       - run: cmake -S core/libs/tests/write_watch -B build/write-watch -DCMAKE_BUILD_TYPE=Release
       - run: cmake --build build/write-watch --parallel
       - run: ctest --test-dir build/write-watch --output-on-failure --repeat until-fail:3 --no-tests=error
@@ -503,8 +628,9 @@ jobs:
       - run: echo 'C:\winlibs\mingw64\bin' >> "$GITHUB_PATH"
       - uses: hendrikmuhs/[email protected]
         with:
-          key: windows
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+          key: ${{ github.event_name == 'workflow_dispatch' && 'dispatch-windows' || 'windows' }}
+          restore-keys: ${{ github.event_name == 'workflow_dispatch' && 'windows' || '' }}
+          save: ${{ github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }}
           evict-old-files: job
       - id: ffmpeg-cache
         uses: actions/cache/restore@v6
@@ -577,3 +703,4 @@ jobs:
           [ -f ctest.log ] || exit 0
           awk '/Test +#[0-9]+: / && $NF == "sec" { for (i = 1; i <= NF; i++) if ($i ~ /^#[0-9]+:$/) name = $(i + 1); t = $(NF - 1) + 0; if (t > max[name]) max[name] = t }
             END { for (name in max) if (max[name] > 30) printf "::warning title=Slow test::%s took %.1f s on %s; tests must finish in 30 s (CONTRIBUTING.md)\n", name, max[name], ENVIRON["RUNNER_OS"] }' ctest.log
+

.github/workflows/pr-intake.yml

@@ -27,6 +27,27 @@ jobs:
             if (pr.state !== 'open' || pr.base.ref !== 'main') {
               return;
             }
+            if (context.payload.action === 'reopened') {
+              let maintainer = false;
+              try {
+                const { data: access } = await github.rest.repos.getCollaboratorPermissionLevel({
+                  owner, repo, username: context.payload.sender.login
+                });
+                maintainer = ['admin', 'maintain', 'write'].includes(access.role_name ?? access.permission);
+              } catch (error) {
+                core.warning(`Could not check the permission of ${context.payload.sender.login}, so the open PR limit applies: ${error.message}`);
+              }
+              if (maintainer) {
+                try {
+                  await github.rest.issues.removeLabel({ owner, repo, issue_number: pull_number, name: 'open PR limit' });
+                } catch (error) {
+                  if (error.status !== 404) {
+                    core.warning(`Could not remove the open PR limit label: ${error.message}`);
+                  }
+                }
+                return;
+              }
+            }
             const { data: mergedPRs } = await github.rest.search.issuesAndPullRequests({
               q: `repo:${owner}/${repo} is:pr is:merged base:main author:${pr.user.login}`,
               per_page: 1

CMakeLists.txt

@@ -17,6 +17,12 @@ function(add_test_executable target)
     endif()
 endfunction()
 
+function(add_asan_test_options test options)
+    if(ANYPS5_ENABLE_ASAN)
+        set_property(TEST ${test} APPEND PROPERTY ENVIRONMENT_MODIFICATION "ASAN_OPTIONS=string_append::${options}")
+    endif()
+endfunction()
+
 option(BUILD_TESTING "Build and register relinker tests" OFF)
 option(ANYPS5_RELINKER_ONLY "Build the relinker without system libraries or third-party dependencies" OFF)
 option(ANYPS5_BUILD_WASM "Build the browser relinker" OFF)
@@ -91,6 +97,17 @@ if(BUILD_TESTING)
     add_test(NAME request_serializer COMMAND request_serializer_tests)
     set_tests_properties(request_serializer PROPERTIES TIMEOUT 20)
     find_package(Python3 QUIET COMPONENTS Interpreter)
+    add_test_executable(shader_ir_use_rewrite_tests
+        core/shader/recompiler/IntermediateRepresentation/tests/UseRewrite.cpp
+        core/shader/recompiler/IntermediateRepresentation/tests/UseRewriteAllocation.cpp
+        core/shader/recompiler/IntermediateRepresentation/src/IrValue.cpp
+        core/shader/recompiler/IntermediateRepresentation/src/IrBlock.cpp
+        core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp
+        core/shader/recompiler/IntermediateRepresentation/src/GuestRegister.cpp)
+    target_include_directories(shader_ir_use_rewrite_tests PRIVATE
+        core/shader/recompiler/IntermediateRepresentation/include)
+    add_test(NAME shader_ir_use_rewrite COMMAND shader_ir_use_rewrite_tests)
+    set_tests_properties(shader_ir_use_rewrite PROPERTIES TIMEOUT 20)
     if(Python3_Interpreter_FOUND)
         add_test(NAME shader_cache_native_paths COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/tests/test_shader_cache_native_paths.py
@@ -118,14 +135,14 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_import_audit.py)
         add_test(NAME unwrap_self COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_unwrap_self.py)
-        foreach(shard RANGE 0 5)
+        foreach(shard RANGE 0 11)
             if(shard EQUAL 0)
                 set(conventions_test_name conventions_function_bodies)
             else()
                 set(conventions_test_name conventions_function_bodies_shard_${shard})
             endif()
             add_test(NAME ${conventions_test_name} COMMAND ${Python3_EXECUTABLE}
-                ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_comments.py --shard ${shard}/6)
+                ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_comments.py --shard ${shard}/12)
             set_tests_properties(${conventions_test_name} PROPERTIES TIMEOUT 30)
         endforeach()
         add_test(NAME conventions_wrapped_declarations COMMAND ${Python3_EXECUTABLE}

core/libs/CMakeLists.txt

@@ -242,6 +242,17 @@ if(BUILD_TESTING)
         target_link_options(patched_libraries_tests PRIVATE -static -static-libgcc -static-libstdc++)
     endif()
     add_test(NAME patched_libraries COMMAND patched_libraries_tests ${PATCHED_DIR})
+    if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        add_library(host_libstdcxx_exception_module SHARED tests/HostLibstdcxxExceptionModule.cpp)
+        target_link_libraries(host_libstdcxx_exception_module PRIVATE ${PATCHED_DIR}/libc.prx)
+        add_dependencies(host_libstdcxx_exception_module libs)
+        add_test_executable(host_libstdcxx_exception_tests tests/HostLibstdcxxException.cpp)
+        target_link_libraries(host_libstdcxx_exception_tests PRIVATE host_libstdcxx_exception_module ${PATCHED_DIR}/libc.prx)
+        add_dependencies(host_libstdcxx_exception_tests libs)
+        set_target_properties(host_libstdcxx_exception_module host_libstdcxx_exception_tests PROPERTIES BUILD_RPATH ${PATCHED_DIR})
+        add_test(NAME host_libstdcxx_exception COMMAND host_libstdcxx_exception_tests)
+        set_tests_properties(host_libstdcxx_exception PROPERTIES TIMEOUT 20)
+    endif()
     if(Python3_Interpreter_FOUND)
         add_test(NAME nid_elf_layout COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_elf_layout.py $<TARGET_FILE:nid_patcher>)
@@ -384,6 +395,16 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceRudp>")
     endif()
 
+    add_test_executable(guest_rudp_context_lifecycle_tests tests/GuestRudpContextLifecycle.cpp)
+    target_include_directories(guest_rudp_context_lifecycle_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_rudp_context_lifecycle_tests PRIVATE libSceRudp libc)
+    configure_windows_unwind(guest_rudp_context_lifecycle_tests)
+    add_test(NAME guest_rudp_context_lifecycle COMMAND guest_rudp_context_lifecycle_tests)
+    if(WIN32)
+        set_tests_properties(guest_rudp_context_lifecycle PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceRudp>")
+    endif()
+
     add_test_executable(guest_memory_pool_stats_tests tests/GuestMemoryPoolStats.cpp)
     target_include_directories(guest_memory_pool_stats_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_memory_pool_stats_tests PRIVATE libSceSsl libSceHttp2 libSceNet libkernel libc)

core/libs/SceTypes.hpp

@@ -921,6 +921,14 @@ struct VoicePortInfo {
     std::uint16_t reserved;
 };
 
+struct VoiceResourceInfo {
+    std::uint16_t max_voice_input_ports;
+    std::uint16_t max_voice_output_ports;
+    std::uint16_t max_device_input_ports;
+    std::uint16_t max_device_output_ports;
+    std::uint16_t max_ports;
+};
+
 struct VoiceStartParam {
     void* container;
     std::uint32_t mem_size;
@@ -2032,6 +2040,11 @@ struct VideoOutColorSettings {
     std::uint32_t reserved[3] = {};
 };
 
+struct VideoOutColorSpaceConversionSettings {
+    std::uint32_t internalData[8] = {};
+};
+static_assert(sizeof(VideoOutColorSpaceConversionSettings) == 32);
+
 
 using atexit_func_t = void (*)();
 

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -204,6 +204,7 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Synchronization/WaitStatistics.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/WorkerSampler.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/PerformanceTrace.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/PerformanceTraceThread.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/TraceFile.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/ThreadPriority.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/GuestMemory.cpp
@@ -405,6 +406,17 @@ if(BUILD_TESTING)
         set_tests_properties(agc_performance_trace PROPERTIES ENVIRONMENT_MODIFICATION
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+
+    add_test_executable(agc_handoff_spin_tests tests/HandoffSpin.cpp)
+    target_include_directories(agc_handoff_spin_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(agc_handoff_spin_tests PRIVATE libc)
+    configure_windows_unwind(agc_handoff_spin_tests)
+    add_test(NAME agc_handoff_spin COMMAND agc_handoff_spin_tests)
+    set_tests_properties(agc_handoff_spin PROPERTIES TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_handoff_spin PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
 endif()
 
 if(WIN32)
@@ -504,6 +516,7 @@ add_test_executable(agc_shader_memory_tests EXCLUDE_FROM_ALL
         tests/ShaderMemory.cpp
         Execution/src/ShaderMemory.cpp
         Execution/src/GuestMemory.cpp
+        Execution/src/PerformanceTraceThread.cpp
 )
 target_include_directories(agc_shader_memory_tests PRIVATE ${LIBS_INCLUDE_DIR})
 target_link_libraries(agc_shader_memory_tests PRIVATE shader_recompiler_agc_driver libc)
@@ -519,6 +532,34 @@ target_link_libraries(agc_vertex_input_tests PRIVATE shader_recompiler_agc_drive
 set_target_properties(agc_vertex_input_tests PROPERTIES CXX_EXTENSIONS OFF)
 configure_windows_unwind(agc_vertex_input_tests)
 
+add_test_executable(agc_guarded_memory_loads_tests EXCLUDE_FROM_ALL tests/GuardedMemoryLoads.cpp)
+target_link_libraries(agc_guarded_memory_loads_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_guarded_memory_loads_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_guarded_memory_loads_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_guarded_memory_loads_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_guarded_memory_loads COMMAND agc_guarded_memory_loads_tests)
+    set_tests_properties(agc_guarded_memory_loads PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_guarded_memory_loads PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
+add_test_executable(agc_lds_lock_budget_tests EXCLUDE_FROM_ALL tests/LdsLockBudget.cpp)
+target_link_libraries(agc_lds_lock_budget_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_lds_lock_budget_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_lds_lock_budget_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_lds_lock_budget_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_lds_lock_budget COMMAND agc_lds_lock_budget_tests)
+    set_tests_properties(agc_lds_lock_budget PROPERTIES TIMEOUT 30 ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_lds_lock_budget PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_shared_write_tests EXCLUDE_FROM_ALL tests/SharedWrites.cpp)
 target_include_directories(agc_shared_write_tests PRIVATE ${LIBS_INCLUDE_DIR})
 target_link_libraries(agc_shared_write_tests PRIVATE libSceAgcDriver libkernel libc)
@@ -528,6 +569,7 @@ configure_windows_unwind(agc_shared_write_tests)
 add_test_executable(agc_write_tracking_tests EXCLUDE_FROM_ALL
         tests/WriteTracking.cpp
         Execution/src/GuestMemory.cpp
+        Execution/src/PerformanceTraceThread.cpp
 )
 target_include_directories(agc_write_tracking_tests PRIVATE ${LIBS_INCLUDE_DIR})
 target_link_libraries(agc_write_tracking_tests PRIVATE shader_recompiler_agc_driver libc)
@@ -797,6 +839,24 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_pipeline_queue_failure_tests EXCLUDE_FROM_ALL tests/PipelineQueueFailure.cpp
+    Graphics/src/PipelineLibrary.cpp tests/AllocationFailure.cpp)
+target_include_directories(agc_pipeline_queue_failure_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
+target_link_libraries(agc_pipeline_queue_failure_tests PRIVATE libSceAgcDriver libc)
+set_target_properties(agc_pipeline_queue_failure_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_pipeline_queue_failure_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_pipeline_queue_failure_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    foreach(mode wait clear stop)
+        add_test(NAME agc_pipeline_queue_failure_${mode} COMMAND agc_pipeline_queue_failure_tests ${mode})
+        set_tests_properties(agc_pipeline_queue_failure_${mode} PROPERTIES TIMEOUT 15)
+        if(WIN32)
+            set_tests_properties(agc_pipeline_queue_failure_${mode} PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        endif()
+    endforeach()
+endif()
+
 add_test_executable(agc_pass_hazards_tests EXCLUDE_FROM_ALL tests/PassHazards.cpp Graphics/src/PassHazards.cpp)
 target_include_directories(agc_pass_hazards_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
 target_link_libraries(agc_pass_hazards_tests PRIVATE libc)
@@ -826,8 +886,14 @@ configure_windows_unwind(agc_buffer_pool_tests)
 if(BUILD_TESTING)
     set_target_properties(agc_buffer_pool_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
     add_test(NAME agc_buffer_pool COMMAND agc_buffer_pool_tests)
+    add_test(NAME agc_buffer_pool_small_device_shared COMMAND agc_buffer_pool_tests --small-device-retention)
+    set_tests_properties(agc_buffer_pool_small_device_shared PROPERTIES ENVIRONMENT
+        "APS5_STAGING_POOL_MIB=256;APS5_BUFFER_POOL_SHARED=1")
+    add_test(NAME agc_buffer_pool_small_device_no_staging COMMAND agc_buffer_pool_tests --small-device-retention)
+    set_tests_properties(agc_buffer_pool_small_device_no_staging PROPERTIES ENVIRONMENT
+        "APS5_STAGING_POOL_MIB=0")
     if(WIN32)
-        set_tests_properties(agc_buffer_pool PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_buffer_pool agc_buffer_pool_small_device_shared agc_buffer_pool_small_device_no_staging PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 endif()
@@ -1127,6 +1193,12 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_storage_upload_lifetime COMMAND agc_driver_recorder_tests --storage-upload-lifetime-only)
+    add_test(NAME agc_sampled_storage_base_level_usage COMMAND agc_driver_recorder_tests --sampled-storage-base-level-usage-only)
+    set_tests_properties(agc_sampled_storage_base_level_usage PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_sampled_storage_base_level_usage PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_storage_upload_failure COMMAND agc_driver_recorder_tests --storage-upload-failure-only)
     set_tests_properties(agc_storage_upload_lifetime agc_storage_upload_failure PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
     if(WIN32)
@@ -1219,8 +1291,10 @@ if(BUILD_TESTING)
     set_tests_properties(agc_shadowed_key_stores PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
     add_test(NAME agc_sample_dumps COMMAND agc_driver_recorder_tests --sample-dumps-only)
     set_tests_properties(agc_sample_dumps PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    add_test(NAME agc_draw_descriptors COMMAND agc_driver_recorder_tests --draw-descriptors-only)
+    set_tests_properties(agc_draw_descriptors PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 60)
     if(WIN32)
-        set_tests_properties(agc_pending_key_stores agc_shadowed_key_stores agc_sample_dumps PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_pending_key_stores agc_shadowed_key_stores agc_sample_dumps agc_draw_descriptors PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 endif()
@@ -1239,6 +1313,23 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_command_batch_failure_tests EXCLUDE_FROM_ALL
+        tests/CommandBatch.cpp tests/CommandBatchFailure.cpp Graphics/src/Resources.cpp tests/AllocationFailure.cpp)
+target_include_directories(agc_command_batch_failure_tests PRIVATE
+        ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
+target_link_libraries(agc_command_batch_failure_tests PRIVATE libSceAgcDriver libc)
+set_target_properties(agc_command_batch_failure_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_command_batch_failure_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_command_batch_failure_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_command_batch_failure COMMAND agc_command_batch_failure_tests)
+    set_tests_properties(agc_command_batch_failure PROPERTIES TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_command_batch_failure PROPERTIES
+                ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_host_import_retire_tests EXCLUDE_FROM_ALL tests/HostImportRetireHold.cpp)
 target_include_directories(agc_host_import_retire_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
 target_link_libraries(agc_host_import_retire_tests PRIVATE libSceAgcDriver libc)
@@ -1328,6 +1419,22 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL
         tests/DepthSurfaceReuse.cpp
 )
 
+if(BUILD_TESTING)
+    add_test_executable(agc_descriptor_cache_failure_tests
+        tests/DescriptorCacheFailure.cpp
+        Graphics/src/ShaderResources.cpp)
+    target_include_directories(agc_descriptor_cache_failure_tests PRIVATE
+        $<TARGET_PROPERTY:libSceAgcDriver,INCLUDE_DIRECTORIES>)
+    target_link_libraries(agc_descriptor_cache_failure_tests PRIVATE libSceAgcDriver shader_recompiler_agc_driver libc libkernel)
+    configure_windows_unwind(agc_descriptor_cache_failure_tests)
+    add_test(NAME agc_descriptor_cache_failure COMMAND agc_descriptor_cache_failure_tests)
+    set_tests_properties(agc_descriptor_cache_failure PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(agc_descriptor_cache_failure PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 target_include_directories(agc_driver_graphics_tests PRIVATE
         ${LIBS_INCLUDE_DIR}
         $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>
@@ -1428,6 +1535,14 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources)
     configure_windows_unwind(${agcExecutionTest})
     list(APPEND agcExecutionTests ${agcExecutionTest})
 endforeach()
+if(BUILD_TESTING)
+    add_test(NAME agc_driver_address_passes_targets_split COMMAND agc_driver_address_passes_tests --targets)
+    set_tests_properties(agc_driver_address_passes_targets_split PROPERTIES ENVIRONMENT "APS5_NO_ADDRESS_READ_PASSES=1" SKIP_RETURN_CODE 77 TIMEOUT 60)
+    if(WIN32)
+        set_tests_properties(agc_driver_address_passes_targets_split PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
 foreach(agcExecutionTestName IN LISTS ANYPS5_AGC_EXECUTION_TESTS)
     if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/execution/${agcExecutionTestName}.cpp)
         message(FATAL_ERROR "ANYPS5_AGC_EXECUTION_TESTS names ${agcExecutionTestName}, which has no source in tests/execution")
@@ -1435,11 +1550,14 @@ foreach(agcExecutionTestName IN LISTS ANYPS5_AGC_EXECUTION_TESTS)
 endforeach()
 
 if(NOT ANYPS5_AGC_EXECUTION_TESTS OR "PooledBufferRange" IN_LIST ANYPS5_AGC_EXECUTION_TESTS)
-    target_sources(agc_driver_pooled_buffer_range_tests PRIVATE Graphics/src/BufferPool.cpp tests/AllocationFailure.cpp)
+    target_sources(agc_driver_pooled_buffer_range_tests PRIVATE
+            Graphics/src/BufferPool.cpp Graphics/src/Resources.cpp tests/AllocationFailure.cpp tests/CommandBatchFailure.cpp)
 endif()
 
 target_link_libraries(agc_driver_gpu_map_on_fiber_tests PRIVATE libSceFiber)
 
+target_link_libraries(agc_driver_queue_zero_late_interrupt_tests PRIVATE libkernel)
+
 target_link_libraries(agc_driver_preparation_scope_settle_tests PRIVATE libSceAgc)
 
 target_include_directories(agc_driver_alpha_to_coverage_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
@@ -1457,6 +1575,13 @@ if(BUILD_TESTING)
 endif()
 
 if(BUILD_TESTING)
+    add_test(NAME agc_driver_native_msaa_vertex_input COMMAND agc_driver_sample_multisample_target_tests --native-vertex-input)
+    set_tests_properties(agc_driver_native_msaa_vertex_input PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" TIMEOUT 30 SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_native_msaa_vertex_input PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_undithered_alpha_to_coverage_state COMMAND agc_driver_graphics_tests --undithered-alpha-to-coverage-state)
     add_test(NAME agc_driver_msaa_dcc_state COMMAND agc_driver_graphics_tests --multisample-dcc-state)
     set_tests_properties(agc_driver_undithered_alpha_to_coverage_state PROPERTIES
@@ -1491,10 +1616,16 @@ target_include_directories(agc_driver_reversed_order_blend_tests PRIVATE $<TARGE
 
 target_include_directories(agc_driver_swap_alt_blend_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_packed_float_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
+target_link_libraries(agc_driver_job_manager_compute_tests PRIVATE libSceJobManager libSceAgc libkernel)
+
 target_include_directories(agc_driver_packed_unorm_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
 target_include_directories(agc_driver_color1555_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_color565_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
 foreach(stage IN ITEMS vert frag)
     set(agcProvokingShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/ProvokingVertex.${stage})
     set(agcProvokingShaderSpv ${agcTestShaderGeneratedRoot}/ProvokingVertex_${stage}.spv)
@@ -1540,6 +1671,8 @@ target_include_directories(agc_driver_pipeline_journal_tests PRIVATE $<TARGET_PR
 
 target_include_directories(agc_driver_buffer_descriptor_runtime_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_shutdown_threads_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
 target_include_directories(agc_driver_prepared_publication_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
 target_include_directories(agc_driver_prepared_graphics_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
@@ -1621,11 +1754,28 @@ if(BUILD_TESTING)
     if(WIN32)
         set_tests_properties(agc_driver_partially_resident_pending_stores PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_shader_disk_cache_large_file COMMAND agc_shader_disk_cache_tests --large-file)
+    set_tests_properties(agc_shader_disk_cache_large_file PROPERTIES SKIP_RETURN_CODE 77 RUN_SERIAL TRUE WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 240)
+    if(WIN32)
+        set_tests_properties(agc_shader_disk_cache_large_file PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     if(WIN32)
         add_test(NAME agc_driver_untracked_snapshot_hint COMMAND agc_driver_partially_resident_snapshot_tests --untracked-hint)
         set_tests_properties(agc_driver_untracked_snapshot_hint PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30
             ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    set_tests_properties(agc_driver_registration_trace PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TRACE_FRAMES=0:1;APS5_TRACE_EVENTS=64" TIMEOUT 30)
+    set_property(TEST agc_driver_registration_trace APPEND PROPERTY ENVIRONMENT_MODIFICATION
+            "APS5_TRACE_SECONDS=unset:" "APS5_TRACE_SUMMARY_EVERY=unset:")
+    add_test(NAME agc_driver_registration_trace_disabled COMMAND ${CMAKE_COMMAND} -E env
+            --unset=APS5_TRACE_FRAMES --unset=APS5_TRACE_SECONDS --unset=APS5_TRACE_SUMMARY_EVERY
+            $<TARGET_FILE:agc_driver_registration_trace_tests> disabled)
+    set_tests_properties(agc_driver_registration_trace_disabled PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_driver_registration_trace_disabled PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_sparse_buffer_holes COMMAND agc_driver_graphics_tests --sparse-buffer-holes)
     set_tests_properties(agc_driver_native_multisample_depth PROPERTIES TIMEOUT 30)
     set_tests_properties(agc_driver_sample_order PROPERTIES TIMEOUT 30)
@@ -1647,15 +1797,24 @@ if(BUILD_TESTING)
     set_tests_properties(agc_sparse_buffer_holes PROPERTIES TIMEOUT 30)
     add_test(NAME agc_pipeline_journal COMMAND agc_driver_graphics_tests --pipeline-journal)
     add_test(NAME agc_pipeline_library_clear_waits_for_build COMMAND agc_driver_graphics_tests --pipeline-library-clear-waits-for-build)
+    add_test(NAME agc_pipeline_store_lazy_buffer_pool COMMAND agc_driver_graphics_tests --pipeline-store-lazy-buffer-pool)
     set_tests_properties(agc_pipeline_journal PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     set_tests_properties(agc_pipeline_library_clear_waits_for_build PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    set_tests_properties(agc_pipeline_store_lazy_buffer_pool PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     if(WIN32)
-        set_tests_properties(agc_pipeline_journal agc_pipeline_library_clear_waits_for_build PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_pipeline_journal agc_pipeline_library_clear_waits_for_build agc_pipeline_store_lazy_buffer_pool PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     set_tests_properties(agc_driver_sparse_buffer_holes PROPERTIES TIMEOUT 30)
     if(WIN32)
         set_tests_properties(agc_sparse_buffer_holes PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_driver_lds_lock_budget COMMAND agc_driver_lds_mixed_width_atomics_tests --lock-budget)
+    set_tests_properties(agc_driver_lds_lock_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_driver_lds_lock_budget PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+
     add_test(NAME agc_driver_emulated_color_compare_srgb_decode COMMAND agc_driver_emulated_color_compare_tests)
     set_tests_properties(agc_driver_emulated_color_compare_srgb_decode PROPERTIES
             ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_SRGB_SHADER_DECODE=1"
@@ -1791,12 +1950,13 @@ if(BUILD_TESTING)
         endif()
     endforeach()
     add_test(NAME agc_descriptor_pool_reset_failure COMMAND agc_driver_graphics_tests --descriptor-pool-reset-failure)
+    add_test(NAME agc_descriptor_cache COMMAND agc_driver_graphics_tests --descriptor-cache)
     add_test(NAME agc_driver_address_space_teardown COMMAND agc_driver_graphics_tests --address-space-teardown)
     set_tests_properties(agc_driver_address_space_teardown PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_driver_pm4_failure_inline COMMAND agc_driver_pm4_tests failure)
     set_tests_properties(agc_driver_pm4_failure_inline PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=0" SKIP_RETURN_CODE 77)
     if(WIN32)
-        set_tests_properties(agc_driver_pm4_failure agc_driver_pm4_failure_inline agc_descriptor_pool_reset_failure agc_driver_address_space_teardown PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_driver_pm4_failure agc_driver_pm4_failure_inline agc_descriptor_pool_reset_failure agc_descriptor_cache agc_driver_address_space_teardown PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     if(NOT WIN32)
@@ -1904,6 +2064,17 @@ if(BUILD_TESTING)
             set_tests_properties(${preparedTest} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
         endif()
     endforeach()
+    add_test(NAME agc_driver_shutdown_threads_joined COMMAND agc_driver_shutdown_threads_tests --joined)
+    add_test(NAME agc_driver_shutdown_threads_compare_helpers COMMAND agc_driver_shutdown_threads_tests --compare-helpers)
+    add_test(NAME agc_driver_shutdown_threads_draw_committer COMMAND agc_driver_shutdown_threads_tests --draw-committer)
+    add_test(NAME agc_driver_shutdown_threads_shader_cache_writer COMMAND agc_driver_shutdown_threads_tests --shader-cache-writer)
+    add_test(NAME agc_driver_shutdown_threads_late_shader_cache COMMAND agc_driver_shutdown_threads_tests --late-shader-cache)
+    add_test(NAME agc_driver_shutdown_threads_async_log_held COMMAND agc_driver_shutdown_threads_tests --async-log-held)
+    add_test(NAME agc_driver_shutdown_threads_exit_after_start COMMAND agc_driver_shutdown_threads_tests --exit-after-start)
+    set_tests_properties(agc_driver_shutdown_threads_joined agc_driver_shutdown_threads_compare_helpers agc_driver_shutdown_threads_draw_committer agc_driver_shutdown_threads_shader_cache_writer agc_driver_shutdown_threads_late_shader_cache agc_driver_shutdown_threads_async_log_held agc_driver_shutdown_threads_exit_after_start PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_driver_shutdown_threads_joined agc_driver_shutdown_threads_compare_helpers agc_driver_shutdown_threads_draw_committer agc_driver_shutdown_threads_shader_cache_writer agc_driver_shutdown_threads_late_shader_cache agc_driver_shutdown_threads_async_log_held agc_driver_shutdown_threads_exit_after_start PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_compute_user_data_calls_registered COMMAND agc_driver_compute_user_data_calls_tests --registered)
     set_tests_properties(agc_driver_compute_user_data_calls_registered PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     if(WIN32)
@@ -2517,6 +2688,19 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_constant_select_tests EXCLUDE_FROM_ALL tests/ConstantSelect.cpp)
+target_link_libraries(agc_constant_select_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_constant_select_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_constant_select_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_constant_select_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_constant_select COMMAND agc_constant_select_tests)
+    if(WIN32)
+        set_tests_properties(agc_constant_select PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 if(BUILD_TESTING)
     get_directory_property(agcRegisteredTests TESTS)
     list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal|agc_driver_pipeline_journal)$")

core/libs/prx/libSceAgcDriver/Execution/include/AsyncLog.hpp

@@ -10,6 +10,7 @@
 #include <cstdio>
 #include <cstdlib>
 #include <deque>
+#include <initializer_list>
 #include <map>
 #include <memory>
 #include <mutex>
@@ -171,6 +172,16 @@ public:
         return true;
     }
 
+    static void FlushAtEnd() {
+        auto& log = Get();
+        if (log.abandoned.load(std::memory_order_acquire)) return;
+        if (!log.FlushFor(EndFlushLimit)) log.abandoned.store(true, std::memory_order_release);
+    }
+
+    std::unique_lock<std::mutex> Hold() {
+        return std::unique_lock(mutex);
+    }
+
 private:
     struct Window {
         Clock::time_point first;
@@ -193,14 +204,14 @@ private:
         if (!asynchronous) return;
         LogSink().store(&AsyncLog::Sink, std::memory_order_release);
         std::atexit(&AsyncLog::FlushAtEnd);
+        InitializeWaits();
         std::thread([this] { Run(); }).detach();
         LibcSetEndFlush_nid_no_patch(&AsyncLog::FlushAtEnd);
     }
 
-    static void FlushAtEnd() {
-        auto& log = Get();
-        if (log.abandoned.load(std::memory_order_acquire)) return;
-        if (!log.FlushFor(EndFlushLimit)) log.abandoned.store(true, std::memory_order_release);
+    void InitializeWaits() {
+        std::unique_lock lock(mutex);
+        for (auto* condition : {&wake, &space, &idle}) static_cast<void>(condition->wait_for(lock, std::chrono::nanoseconds(0)));
     }
 
     static int Sink(std::FILE* stream, const char* text, std::size_t length) {
@@ -383,6 +394,12 @@ inline void FlushAsyncLog_nid_no_patch() {
     AsyncLog::Get().Flush();
 }
 
+inline void FlushAsyncLogAtEnd_nid_no_patch() {
+    AsyncLog::FlushAtEnd();
+}
+
+std::unique_lock<std::mutex> HoldAsyncLog();
+
 template<typename... TArgs>
 void LogCoalescedStderr_nid_no_patch(const char* format, TArgs... args) {
     char small[1024];

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp

@@ -30,9 +30,11 @@ public:
     static bool& Active();
     static std::atomic<std::uint64_t>& EpochToken();
     static void FollowEpoch(std::uint64_t token);
+    static unsigned Committers();
 
     void Enqueue(Commit commit, std::vector<Range> writes, std::uint64_t labelAddress = 0, std::vector<std::byte> labelBytes = {});
     void Drain(DrainReason reason, std::uint32_t opcode = 0x100);
+    void Stop();
     static std::uint64_t DrainRequestsByOpcode(std::uint32_t opcode);
     bool Busy() const { return outstanding.load(std::memory_order_acquire) != 0; }
     std::size_t Outstanding() const { return outstanding.load(std::memory_order_acquire); }
@@ -60,6 +62,7 @@ private:
     std::uint32_t idleWaiters = 0;
     bool committerWaiting = false;
     bool stopped = false;
+    bool closing = false;
     std::thread thread;
     std::uint64_t commits = 0;
     std::uint64_t commitNs = 0;

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -103,6 +103,7 @@ private:
     static bool& completionDeferred();
     void deferCompletion(const Submission& submission);
     void submitOpenWork();
+    void wakeQueueZeroForCompletions();
     void completeSubmission(std::uint64_t serial, std::uint64_t received, std::span<const std::uint64_t> labelWrites);
     static bool waitFree(const Submission& submission);
     bool queue0Before(std::uint64_t received) const;

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp

@@ -119,6 +119,7 @@ void AwaitRegistrationPreparation();
 void SubmitLibraryPrebuild(std::function<void()> task);
 void AwaitLibraryPrebuilds();
 void CloseRegistrationPreparation();
+unsigned RegistrationPreparationThreads();
 
 void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target);
 

core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp

@@ -26,6 +26,7 @@ struct Commitment {
     bool whole = false;
 };
 Commitment DescribeCommitted(std::uint64_t address, std::size_t bytes, bool writable = false);
+void CopyGuestBytes(void* destination, const void* source, std::size_t bytes);
 void Read(std::uint64_t address, std::span<std::byte> destination, std::size_t alignment = 1);
 // Variants for resources in heaps the guest commits on demand (partially resident textures, GPU
 // scratch heaps): uncommitted pages read as zeros, are not compared, and are never stored.

core/libs/prx/libSceAgcDriver/Execution/include/HandoffSpin.hpp

@@ -0,0 +1,61 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_HANDOFFSPIN_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_HANDOFFSPIN_HPP
+
+#include "prx/libc/include/CpuTopology.hpp"
+#include <algorithm>
+#include <bitset>
+#include <chrono>
+#include <cstdint>
+#include <cstdlib>
+#include <thread>
+#if defined(__x86_64__) || defined(_M_X64)
+#include <immintrin.h>
+#endif
+
+namespace AgcDriver {
+
+inline void SpinPause() {
+#if defined(__x86_64__) || defined(_M_X64)
+    _mm_pause();
+#endif
+}
+
+inline unsigned UsableProcessors() {
+    const auto count = static_cast<unsigned>(std::bitset<64>(CpuTopology::Get().process).count());
+    return count != 0 ? count : std::thread::hardware_concurrency();
+}
+
+inline std::chrono::nanoseconds HandoffSpinWindowFor(const char* configured, unsigned usableProcessors) {
+    constexpr std::chrono::microseconds automaticWindow{150};
+    constexpr unsigned automaticProcessorsAbove = 8;
+    constexpr long long longestWindowUs = 10000;
+    if (configured != nullptr) return std::chrono::microseconds(std::clamp<long long>(std::strtoll(configured, nullptr, 10), 0, longestWindowUs));
+    return usableProcessors > automaticProcessorsAbove ? std::chrono::nanoseconds(automaticWindow) : std::chrono::nanoseconds::zero();
+}
+
+inline std::chrono::nanoseconds HandoffSpinWindow() {
+    static const std::chrono::nanoseconds window = HandoffSpinWindowFor(std::getenv("APS5_HANDOFF_SPIN_US"), UsableProcessors());
+    return window;
+}
+
+template<typename TReady>
+bool SpinUntil(std::chrono::nanoseconds window, TReady&& ready) {
+    if (ready()) return true;
+    if (window.count() <= 0) return false;
+    const auto until = std::chrono::steady_clock::now() + window;
+    do {
+        SpinPause();
+        if (ready()) return true;
+    } while (std::chrono::steady_clock::now() < until);
+    return false;
+}
+
+template<typename TMutex>
+void LockSpinning(TMutex& mutex, std::chrono::nanoseconds window) {
+    if (window.count() > 0 && SpinUntil(window, [&] { return mutex.try_lock(); })) return;
+    mutex.lock();
+}
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp

@@ -21,6 +21,13 @@
 #include <utility>
 #include <vector>
 
+namespace AgcDriver {
+class TraceThread;
+}
+
+extern "C" bool AgcDriverTraceRecording_nid_postfix();
+extern "C" AgcDriver::TraceThread* AgcDriverTraceThread_nid_postfix();
+
 namespace AgcDriver {
 
 class VulkanDevice;
@@ -532,23 +539,26 @@ public:
     }
 
     static bool Recording() {
-        return recording.load(std::memory_order_relaxed);
+        return AgcDriverTraceRecording_nid_postfix();
     }
 
     static TraceThread* Thread() {
-        return local;
+        return AgcDriverTraceThread_nid_postfix();
     }
 
     static void StopOutput();
 
 private:
+    friend bool ::AgcDriverTraceRecording_nid_postfix();
+    friend TraceThread* ::AgcDriverTraceThread_nid_postfix();
+
     static void OpenInterval(std::chrono::steady_clock::time_point now, std::uint64_t present);
     static void FlushInterval(std::chrono::steady_clock::time_point now, std::uint32_t width, std::uint32_t height, const VulkanDevice* device);
 
-    inline static std::atomic<bool> recording{false};
+    static std::atomic<bool> recording;
     inline static std::atomic<bool> buffering{false};
     inline static std::atomic<std::uint64_t> presentCount{0};
-    inline static thread_local TraceThread* local = nullptr;
+    static thread_local TraceThread* local;
     inline static std::mutex gpuMutex;
     inline static std::vector<TraceGpuRange> gpuRanges;
     inline static std::uint64_t gpuLost = 0;

core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp

@@ -47,7 +47,7 @@ struct Recipe {
     std::uint64_t dataWordsHash = 0;
     // The template's PresyncSurfaces() as of the attach and the host-import identity they were
     // computed under (a surface is CPU-read only while its memory has no import).
-    std::vector<std::pair<std::uint64_t, std::uint64_t>> presyncSurfaces;
+    std::vector<Graphics::PresyncSurface> presyncSurfaces;
     Graphics::HostImportsProof presyncProof;
     // Both false for every recipe (only reusable objects get one); kept for the asserts.
     bool needsCompletion = false;

core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp

@@ -56,6 +56,7 @@ public:
     ShaderRecompiler::SpirvTarget Target() const;
     Graphics::SampleOrder HostSampleOrder(const Graphics::State& graphics) const;
     std::optional<ShaderRecompiler::GeometryStageLimits> GeometryLimits() const;
+    std::uint32_t MaxFragmentInputComponents() const;
     const std::uint32_t* WorkGroupCountLimit() const;
     std::uint32_t VendorId() const;
     ShaderRecompiler::SpirvTarget ComputeTarget(std::uint32_t waveSize) const;
@@ -186,6 +187,10 @@ public:
         std::uint64_t created, underGpuMutex;
     };
     static ComputePipelineCreation ComputePipelineCounts();
+    struct DispatchMoveCount {
+        std::uint64_t checked, exact, moved, refused, filtered, rebuilt, kept, served;
+    };
+    static DispatchMoveCount DispatchMoveCounts();
     void ResolveMemory(std::uint64_t address, std::size_t bytes, bool writable);
     void* Window() const;
     void Resize(std::uint32_t width, std::uint32_t height);
@@ -245,6 +250,10 @@ public:
         double gpuBusyMs, gpuGapMs, lastSubmitAfterFlipMs, blitSubmitAfterFlipMs;
     };
     static PresentStatistics PresentCounts();
+    struct PresyncStatistics {
+        std::uint64_t checked, presyncs, keyChecks, keyWaits;
+    };
+    static PresyncStatistics PresyncCounts();
     // Stage A of a dispatch's resource build, run WITHOUT GuestMemory::GpuMutex before Dispatch or
     // DispatchIndirect: the binding plan, data buffers, imports already serving the guest buffers,
     // read-only copies and the descriptor set (see ShaderResources). The dispatch completes it
@@ -338,7 +347,7 @@ private:
     // Body of Dispatch and DispatchIndirect: `arguments` 0 dispatches x, y, z groups.
     IndirectOutcome dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t programAddress, std::shared_ptr<PreparedDispatch> prepared, std::shared_ptr<const Recipe>* recipe);
     // The stage-A pre-sync over `surfaces` (see PrepareDispatch): the serial waited for, 0 none.
-    std::uint64_t presync(std::span<const std::pair<std::uint64_t, std::uint64_t>> surfaces);
+    std::uint64_t presync(std::span<const Graphics::PresyncSurface> surfaces);
     std::shared_ptr<ComputePipelineObjects> computePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span<const std::uint32_t> layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress);
     // A DISPATCH_INDIRECT's argument path (see DispatchIndirect): GPU-side from the host import, or
     // the counts read on the CPU (the record then continues as a direct dispatch).

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp

@@ -272,8 +272,10 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     bool verifyHit = false;
     const bool stageEntries = useDrawEntries && !(graphics.stages.tessellation && !drawParameters.indirect && drawParameters.instanceCount > 1u);
     lookupDraw(submission, localDevice, graphics, pixel, programs, roles, vertexInfos, stageEntries, registerKey, profile, drawKey, entry, matched, matchedRegions, drawHit, verifyHit, phaseTiming, phaseMs, dataEntry, dataCandidates, stageData);
-    for (auto& variant : matched) {
-        if (variant != nullptr) applyShaderDataPatches(*variant);
+    for (std::size_t i = 0; i < matched.size(); ++i) {
+        if (matched[i] == nullptr || matched[i]->shaderDataPatches.empty()) continue;
+        applyShaderDataPatches(*matched[i]);
+        if (stageData[i].compiled != nullptr) stageData[i].compiled = patchDataWords(*matched[i], stageData[i].words);
     }
     const auto relocatedDecode = decode;
     if (!userPatches.empty()) {
@@ -434,15 +436,16 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         phaseTiming.Phase(DrawRowVectors);
         require(programs.size() == 2 && programResults[0] != nullptr && programResults[1] != nullptr, "rect-list requires vertex and fragment programs");
         auto rectangle = DrawRectangle(*programs[0].snapshot, programs[1].snapshot, programResults[0]->variantId, programResults[1]->variantId, localDevice->Target());
+        auto& control = graphics.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT ? rectangle.lastVertexControl : rectangle.control;
         if (rectListBuilt) {
-            results[rectIndex] = std::move(rectangle.control);
+            results[rectIndex] = std::move(control);
             results[rectIndex + 1] = std::move(rectangle.evaluation);
             phaseTiming.Phase(DrawRowRectList);
             return;
         }
         require(stages.size() == 2, "rect-list requires vertex and fragment programs");
         rectIndex = results.size();
-        results.push_back(std::move(rectangle.control));
+        results.push_back(std::move(control));
         results.push_back(std::move(rectangle.evaluation));
         stages.insert(stages.begin() + 1, {{Stage::TessellationControl, &results[rectIndex], 0}, {Stage::TessellationEvaluation, &results[rectIndex + 1], 0}});
         rectListBuilt = true;
@@ -451,8 +454,18 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     if (graphics.rectList) buildRectList();
     if (!programs.empty() && programResults.back() != nullptr && programResults.back()->barycentricEmulation.active) {
         require(!graphics.rectList && graphics.stages.path == Graphics::ShaderPath::Vertex && programs.size() == 2 && programResults[0] != nullptr && stages.size() == 2, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric needs a vertex shader before it");
-        results.push_back(ShaderRecompiler::BuildBarycentricGeometryShader(*programResults[0], *programResults[1], localDevice->Target(), localDevice->GeometryLimits()));
-        stages.insert(stages.begin() + 1, Graphics::CompiledShader{Stage::Geometry, &results.back(), 0});
+        const auto geometryLimits = localDevice->GeometryLimits();
+        if (geometryLimits || !localDevice->Target().tessellation) {
+            results.push_back(ShaderRecompiler::BuildBarycentricGeometryShader(*programResults[0], *programResults[1], localDevice->Target(), geometryLimits));
+            stages.insert(stages.begin() + 1, Graphics::CompiledShader{Stage::Geometry, &results.back(), 0});
+        } else {
+            require(graphics.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST && !graphics.primitiveRestart, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric or geometry shaders needs a triangle list without primitive restart");
+            auto patches = ShaderRecompiler::BuildBarycentricTessellationShaders(*programResults[0], *programResults[1], localDevice->Target(), localDevice->MaxFragmentInputComponents());
+            const auto first = results.size();
+            results.push_back(std::move(patches.control));
+            results.push_back(std::move(patches.evaluation));
+            stages.insert(stages.begin() + 1, {{Stage::TessellationControl, &results[first], 0}, {Stage::TessellationEvaluation, &results[first + 1], 0}});
+        }
     }
     std::vector<Graphics::GuestMemorySnapshot> snapshots;
     const auto snapshot = [&] {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp

@@ -1,6 +1,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/HandoffSpin.hpp"
 #include <algorithm>
 #include <cstdio>
 #include <cstdlib>
@@ -14,6 +15,8 @@ namespace {
 
 constexpr std::array<const char*, static_cast<std::size_t>(DrawPipeline::DrainReason::Count)> DrainNames{"packet", "flush", "labels", "capture", "indirect", "submission"};
 
+std::atomic<unsigned> runningCommitters{0};
+
 std::uint64_t elapsedNs(std::chrono::steady_clock::time_point since) {
     return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - since).count());
 }
@@ -59,6 +62,10 @@ void DrawPipeline::FollowEpoch(std::uint64_t token) {
     GuestMemory::BumpCollectEpoch();
 }
 
+unsigned DrawPipeline::Committers() {
+    return runningCommitters.load(std::memory_order_acquire);
+}
+
 void DrawPipeline::rethrowFailure() {
     if (failure == nullptr) return;
     auto error = std::exchange(failure, nullptr);
@@ -69,6 +76,11 @@ void DrawPipeline::Enqueue(Commit commit, std::vector<Range> writes, std::uint64
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
     std::unique_lock lock(mutex);
     rethrowFailure();
+    if (closing) {
+        lock.unlock();
+        commit();
+        return;
+    }
     if (items.size() >= Depth()) {
         if (GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw pipeline full while the enqueuing thread holds the GPU lock");
         const auto start = std::chrono::steady_clock::now();
@@ -82,7 +94,10 @@ void DrawPipeline::Enqueue(Commit commit, std::vector<Range> writes, std::uint64
     ++enqueued;
     items.push_back({std::move(commit), std::move(writes), labelAddress, std::move(labelBytes)});
     outstanding.fetch_add(1, std::memory_order_release);
-    if (!thread.joinable()) thread = std::thread([this] { run(); });
+    if (!thread.joinable()) {
+        thread = std::thread([this] { run(); });
+        runningCommitters.fetch_add(1, std::memory_order_acq_rel);
+    }
     if (committerWaiting) wake.notify_one();
     if (!profile) return;
     const auto now = std::chrono::steady_clock::now();
@@ -119,11 +134,13 @@ void DrawPipeline::report(std::chrono::steady_clock::time_point now) {
 void DrawPipeline::run() {
     for (;;) {
         Commit* commit = nullptr;
+        SpinUntil(HandoffSpinWindow(), [&] { return outstanding.load(std::memory_order_acquire) != 0; });
         {
             std::unique_lock lock(mutex);
             committerWaiting = true;
-            wake.wait(lock, [&] { return !items.empty(); });
+            wake.wait(lock, [&] { return !items.empty() || closing; });
             committerWaiting = false;
+            if (items.empty()) break;
             commit = stopped ? nullptr : &items.front().commit;
         }
         const auto start = std::chrono::steady_clock::now();
@@ -155,6 +172,20 @@ void DrawPipeline::run() {
         outstanding.fetch_sub(1, std::memory_order_release);
         if (idleWaiters != 0) idle.notify_all();
     }
+    runningCommitters.fetch_sub(1, std::memory_order_acq_rel);
+}
+
+void DrawPipeline::Stop() {
+    std::thread committer;
+    {
+        std::lock_guard lock(mutex);
+        closing = true;
+        committer = std::move(thread);
+    }
+    wake.notify_all();
+    if (!committer.joinable()) return;
+    if (committer.get_id() == std::this_thread::get_id()) committer.detach();
+    else committer.join();
 }
 
 void DrawPipeline::Drain(DrainReason reason, std::uint32_t opcode) {
@@ -162,6 +193,7 @@ void DrawPipeline::Drain(DrainReason reason, std::uint32_t opcode) {
     if (!Busy()) return;
     if (GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw pipeline drain under the GPU lock (reason " + std::to_string(static_cast<unsigned>(reason)) + ", opcode " + std::to_string(opcode) + ")");
     const auto start = std::chrono::steady_clock::now();
+    SpinUntil(HandoffSpinWindow() * 2, [&] { return !Busy(); });
     std::unique_lock lock(mutex);
     ++idleWaiters;
     idle.wait(lock, [&] { return items.empty(); });

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp

@@ -217,6 +217,7 @@ void Driver::relocateDrawEntry(const QueueState& queue, std::uint64_t alias, std
             copy->words = variant->words;
             copy->forgetSerial = variant->forgetSerial;
             copy->compiled = variant->compiled;
+            copy->meshIndexDescriptor = variant->meshIndexDescriptor;
             copy->shader = variant->shader;
             copy->dataPositions = variant->dataPositions;
             copy->dataSlots = variant->dataSlots;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp

@@ -2,13 +2,17 @@
 #include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
+#include "ShaderDiskCache.hpp"
 #include <atomic>
 #include <cstdint>
 #include <cstdio>
@@ -69,8 +73,7 @@ bool StackHoldsImport() {
     const auto low = bounds.low;
     const auto high = bounds.high;
 #endif
-    const volatile unsigned char marker = 0;
-    const auto frame = reinterpret_cast<std::uintptr_t>(&marker);
+    const auto frame = reinterpret_cast<std::uintptr_t>(__builtin_frame_address(0));
     return frame >= low && frame < high && frame - low >= InlineImportStackBytes;
 }
 
@@ -115,15 +118,19 @@ void Driver::stop() {
     for (auto& [queue, worker] : workers) {
         if (worker.thread.joinable()) worker.thread.join();
     }
+    DrawPipeline::Queue0().Stop();
     StopWorkerSampler();
     CloseRegistrationPreparation();
+    Graphics::StopPipelineLibraryWorker();
     std::lock_guard gpuLock(GuestMemory::GpuMutex());
     device.Reset();
     replacedDevices.clear();
     Graphics::FlushPipelineJournal();
     Graphics::ShutdownGuestBufferWorkers();
+    Graphics::StopCompareHelpers();
+    ShaderRecompiler::ShaderDiskCache::Shutdown();
     PerformanceTrace::StopOutput();
-    FlushAsyncLog_nid_no_patch();
+    FlushAsyncLogAtEnd_nid_no_patch();
     StopProfileOutput_nid_no_patch();
     stopped = true;
 }
@@ -222,3 +229,11 @@ bool& Driver::onWorkerThread() {
 }
 
 }
+
+namespace AgcDriver {
+
+std::unique_lock<std::mutex> HoldAsyncLog() {
+    return AsyncLog::Get().Hold();
+}
+
+}

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp

@@ -411,7 +411,10 @@ void Driver::execute(const Submission& submission) {
             return;
         }
         if ((std::atomic_ref<std::uint32_t>(*const_cast<std::uint32_t*>(submission.rewindTail - 1)).load(std::memory_order_acquire) & 0x80000000u) != 0) {
-            DrawPipeline::Queue0().Enqueue([this] { submitOpenWork(); }, {});
+            DrawPipeline::Queue0().Enqueue([this] {
+                submitOpenWork();
+                wakeQueueZeroForCompletions();
+            }, {});
             executeRewindTail(submission);
             return;
         }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp

@@ -85,6 +85,12 @@ void Driver::submitOpenWork() {
     if (localDevice != nullptr) localDevice->SubmitRecorded(true);
 }
 
+void Driver::wakeQueueZeroForCompletions() {
+    if (!completionsPending()) return;
+    std::lock_guard lock(mutex);
+    changed.notify_all();
+}
+
 void Driver::completeSubmission(std::uint64_t serial, std::uint64_t received, std::span<const std::uint64_t> labelWrites) {
     struct Complete {
         Driver& driver;
@@ -98,7 +104,7 @@ void Driver::completeSubmission(std::uint64_t serial, std::uint64_t received, st
                 driver.forgetUnfinishedWrites(driver.workers.at(0), labelWrites);
                 const auto found = std::find(driver.queue0Uncommitted.begin(), driver.queue0Uncommitted.end(), received);
                 if (found != driver.queue0Uncommitted.end()) driver.queue0Uncommitted.erase(found);
-                notify = driver.idleWaiters != 0 || driver.orderHolders.load(std::memory_order_acquire) != 0;
+                notify = driver.idleWaiters != 0 || driver.orderHolders.load(std::memory_order_acquire) != 0 || completionsPending();
             }
             if (notify) driver.changed.notify_all();
         }
@@ -214,9 +220,9 @@ void Driver::run(std::uint32_t id) noexcept {
                             && ((!storeIdleResults && Graphics::StorageTexture::AnyPending()) || idleEpoch.load(std::memory_order_acquire) != idleEpochAtCheck)) continue;
                         if (id == 0) queue0Dormant.store(true, std::memory_order_relaxed);
                         const auto idleEpochAtWait = idleEpoch.load(std::memory_order_acquire);
-                        changed.wait(lock, [&] { return ready() || (id == 0 && idleEpoch.load(std::memory_order_acquire) != idleEpochAtWait); });
+                        changed.wait(lock, [&] { return ready() || (id == 0 && (idleEpoch.load(std::memory_order_acquire) != idleEpochAtWait || completionsPending())); });
                         if (id == 0) queue0Dormant.store(false, std::memory_order_relaxed);
-                        if (id == 0 && !ready() && idleEpoch.load(std::memory_order_acquire) != idleEpochAtWait) continue;
+                        if (id == 0 && !ready() && (idleEpoch.load(std::memory_order_acquire) != idleEpochAtWait || completionsPending())) continue;
                         break;
                     }
                     if (poll(ready)) break;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp

@@ -11,6 +11,7 @@
 #include "CompiledVariant.hpp"
 #include "Optimization/ResourceProgram.hpp"
 #include <algorithm>
+#include <array>
 #include <atomic>
 #include <chrono>
 #include <set>
@@ -933,7 +934,8 @@ class RegistrationPreparePool {
 public:
     static RegistrationPreparePool& Instance() {
         static auto* pool = [] {
-            auto* created = new RegistrationPreparePool(std::max(2u, std::thread::hardware_concurrency() / 2u));
+            std::lock_guard lifecycle(Lifecycle());
+            auto* created = new RegistrationPreparePool(Closing() ? 0u : std::max(2u, std::thread::hardware_concurrency() / 2u));
             PreparationSlot().store(created);
             return created;
         }();
@@ -947,7 +949,8 @@ public:
     static RegistrationPreparePool& Prebuild() {
         static auto* pool = [] {
             const char* text = std::getenv("APS5_PREBUILD_THREADS");
-            auto* created = new RegistrationPreparePool(text != nullptr ? std::max(1u, static_cast<unsigned>(std::strtoul(text, nullptr, 10))) : 1u);
+            std::lock_guard lifecycle(Lifecycle());
+            auto* created = new RegistrationPreparePool(Closing() ? 0u : text != nullptr ? std::max(1u, static_cast<unsigned>(std::strtoul(text, nullptr, 10))) : 1u);
             PrebuildSlot().store(created);
             return created;
         }();
@@ -958,7 +961,33 @@ public:
         return PrebuildSlot().load();
     }
 
+    static unsigned LiveWorkers() {
+        return liveWorkers.load(std::memory_order_acquire);
+    }
+
+    static void CloseAll() {
+        std::array<RegistrationPreparePool*, 2> pools{};
+        {
+            std::lock_guard lifecycle(Lifecycle());
+            Closing() = true;
+            pools = {Existing(), ExistingPrebuild()};
+        }
+        for (auto* pool : pools) {
+            if (pool != nullptr) pool->Close();
+        }
+    }
+
 private:
+    static std::mutex& Lifecycle() {
+        static std::mutex lifecycle;
+        return lifecycle;
+    }
+
+    static bool& Closing() {
+        static bool closing = false;
+        return closing;
+    }
+
     static std::atomic<RegistrationPreparePool*>& PreparationSlot() {
         static std::atomic<RegistrationPreparePool*> slot{nullptr};
         return slot;
@@ -1002,9 +1031,17 @@ public:
     }
 
     void Close() {
-        std::unique_lock lock(mutex);
-        closed = true;
-        idle.wait(lock, [&] { return tasks.empty() && running == 0; });
+        std::vector<pthread_t> joined;
+        {
+            std::unique_lock lock(mutex);
+            closed = true;
+            idle.wait(lock, [&] { return tasks.empty() && running == 0; });
+            joined.swap(workers);
+        }
+        ready.notify_all();
+        for (const auto worker : joined) {
+            if (!pthread_equal(worker, pthread_self())) pthread_join(worker, nullptr);
+        }
     }
 
 private:
@@ -1015,8 +1052,9 @@ private:
         for (unsigned index = 0; index < count; ++index) {
             pthread_t thread;
             if (pthread_create(&thread, &attributes, &RegistrationPreparePool::Main, this) == 0) {
-                pthread_detach(thread);
+                workers.push_back(thread);
                 ++threads;
+                liveWorkers.fetch_add(1, std::memory_order_acq_rel);
             }
         }
         pthread_attr_destroy(&attributes);
@@ -1029,7 +1067,11 @@ private:
             Task task;
             {
                 std::unique_lock lock(pool.mutex);
-                pool.ready.wait(lock, [&] { return !pool.tasks.empty(); });
+                pool.ready.wait(lock, [&] { return !pool.tasks.empty() || pool.closed; });
+                if (pool.tasks.empty()) {
+                    liveWorkers.fetch_sub(1, std::memory_order_acq_rel);
+                    return nullptr;
+                }
                 task = std::move(pool.tasks.front());
                 pool.tasks.pop_front();
                 ++pool.running;
@@ -1062,7 +1104,9 @@ private:
     std::condition_variable ready;
     std::condition_variable idle;
     std::deque<Task> tasks;
+    std::vector<pthread_t> workers;
     unsigned threads = 0;
+    inline static std::atomic<unsigned> liveWorkers{0};
     unsigned running = 0;
     std::uint64_t submitted = 0;
     std::set<std::uint64_t> unfinished;
@@ -1124,8 +1168,11 @@ void AwaitLibraryPrebuilds() {
 }
 
 void CloseRegistrationPreparation() {
-    RegistrationPreparePool::Instance().Close();
-    if (auto* prebuild = RegistrationPreparePool::ExistingPrebuild()) prebuild->Close();
+    RegistrationPreparePool::CloseAll();
+}
+
+unsigned RegistrationPreparationThreads() {
+    return RegistrationPreparePool::LiveWorkers();
 }
 
 std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target) {
@@ -1198,7 +1245,16 @@ std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decod
     return prepared;
 }
 
+namespace {
+
+void BindTitleThread() {
+    if (PerformanceTrace::Configured()) PerformanceTrace::BindThread("title", TraceThread::DefaultCapacity / 8);
+}
+
+}
+
 void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive) {
+    BindTitleThread();
     PerformanceContext timingContext(FrameTiming::Preparation());
     PerformanceTimer timing("Shader.ResolveGraphicsStagesAbi");
     ShaderPreparationTransaction transaction;
@@ -1275,6 +1331,7 @@ void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std
 }
 
 void Driver::ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive) {
+    BindTitleThread();
     PerformanceContext timingContext(FrameTiming::Preparation());
     PerformanceTimer timing("Shader.ResolveAbi");
     ShaderPreparationTransaction transaction;
@@ -1346,6 +1403,7 @@ std::uint64_t NullPixelProgramAddress() {
 }
 
 void Driver::ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType) {
+    BindTitleThread();
     PerformanceContext timingContext(FrameTiming::Preparation());
     PerformanceTimer timing("Shader.ResolveGraphicsAbi");
     CheckFailure();
@@ -1472,6 +1530,7 @@ std::shared_ptr<const ShaderRegistry> Driver::RegisteredShaders() {
 }
 
 void Driver::RegisterShader(const Shader* shader) {
+    BindTitleThread();
     PerformanceContext timingContext(FrameTiming::Preparation());
     PerformanceTimer timing("Shader.Register");
     ShaderPreparationTransaction transaction;

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/HandoffSpin.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp"
 #include "prx/libSceAgcDriver/Execution/include/WriteWatchCoverage.hpp"
 #include "ThreadOwned.hpp"
@@ -657,8 +658,7 @@ bool onOwnLiveStack(std::uintptr_t address, std::size_t bytes) {
         }
     };
     thread_local const Bounds bounds;
-    const volatile unsigned char marker = 0;
-    const auto frame = reinterpret_cast<std::uintptr_t>(&marker);
+    const auto frame = reinterpret_cast<std::uintptr_t>(__builtin_frame_address(0));
     return bounds.high != 0 && frame >= bounds.low && frame < bounds.high && address >= frame && address < bounds.high && bytes <= bounds.high - address;
 }
 
@@ -1931,12 +1931,12 @@ void GpuMutexType::lock() {
     if (!GpuLockProfiled()) {
         auto* trace = PerformanceTrace::Recording() ? PerformanceTrace::Thread() : nullptr;
         if (trace == nullptr) {
-            mutex.lock();
+            LockSpinning(mutex, HandoffSpinWindow());
         } else {
             const auto site = static_cast<std::size_t>(std::exchange(LockStats().nextSite, GpuLockSite::Other));
             if (!mutex.try_lock()) {
                 const auto start = std::chrono::steady_clock::now();
-                mutex.lock();
+                LockSpinning(mutex, HandoffSpinWindow());
                 trace->Add("GpuMutex.wait", GpuLockSiteNames[site], start, std::chrono::steady_clock::now());
             }
         }
@@ -1954,7 +1954,7 @@ void GpuMutexType::lock() {
         HolderMatrixValues before{};
         if (holders) Holders().Snapshot(before);
         const auto start = std::chrono::steady_clock::now();
-        mutex.lock();
+        LockSpinning(mutex, HandoffSpinWindow());
         acquired();
         const auto now = std::chrono::steady_clock::now();
         if (auto* trace = PerformanceTrace::Recording() ? PerformanceTrace::Thread() : nullptr) trace->Add("GpuMutex.wait", GpuLockSiteNames[site], start, now);
@@ -2100,9 +2100,26 @@ void Read(std::uint64_t address, std::span<std::byte> destination, std::size_t a
         if (trace) std::fprintf(stderr, "[unreadable]   read from +0x%llx\n", ModuleOffset(__builtin_return_address(0)));
         throw;
     }
-    std::memcpy(destination.data(), source, destination.size());
+    CopyGuestBytes(destination.data(), source, destination.size());
 }
 
+#ifdef ANYPS5_ENABLE_ASAN
+[[gnu::no_sanitize("address")]] void CopyGuestBytes(void* destination, const void* source, std::size_t bytes) {
+    auto* to = static_cast<unsigned char*>(destination);
+    const auto* from = static_cast<const unsigned char*>(source);
+    std::size_t index = 0;
+    for (; index + sizeof(std::uint64_t) <= bytes; index += sizeof(std::uint64_t)) {
+        const std::uint64_t word = *reinterpret_cast<const volatile std::uint64_t*>(from + index);
+        std::memcpy(to + index, &word, sizeof(word));
+    }
+    for (; index < bytes; ++index) to[index] = static_cast<const volatile unsigned char*>(from)[index];
+}
+#else
+void CopyGuestBytes(void* destination, const void* source, std::size_t bytes) {
+    std::memcpy(destination, source, bytes);
+}
+#endif
+
 namespace {
 
 void copyCommitted(std::uint64_t address, std::span<std::byte> destination) {

core/libs/prx/libSceAgcDriver/Execution/src/PerformanceTraceThread.cpp

@@ -0,0 +1,16 @@
+#include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp"
+
+namespace AgcDriver {
+
+std::atomic<bool> PerformanceTrace::recording{false};
+thread_local TraceThread* PerformanceTrace::local = nullptr;
+
+}
+
+extern "C" bool AgcDriverTraceRecording_nid_postfix() {
+    return AgcDriver::PerformanceTrace::recording.load(std::memory_order_relaxed);
+}
+
+extern "C" AgcDriver::TraceThread* AgcDriverTraceThread_nid_postfix() {
+    return AgcDriver::PerformanceTrace::local;
+}

core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp

@@ -219,7 +219,7 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val
     const auto index = static_cast<std::size_t>((address % PageBytes) / sizeof(*value));
     if (!page.valid.Test(index) && page.lazy) {
         const auto first = index / BlockWords * BlockWords;
-        std::memcpy(&page.words[first], reinterpret_cast<const void*>((address & ~static_cast<std::uint64_t>(PageBytes - 1)) + first * sizeof(std::uint32_t)), BlockWords * sizeof(std::uint32_t));
+        GuestMemory::CopyGuestBytes(&page.words[first], reinterpret_cast<const void*>((address & ~static_cast<std::uint64_t>(PageBytes - 1)) + first * sizeof(std::uint32_t)), BlockWords * sizeof(std::uint32_t));
         page.valid.SetBlock(index);
     }
     if (!page.valid.Test(index)) {

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -281,6 +281,7 @@ struct VulkanDevice::State {
     VkDeviceSize uploadSize = 0;
     VkPhysicalDeviceProperties properties{};
     std::string driverInfo;
+    bool individuallyFreeDescriptorSets = false;
     PFN_vkGetCalibratedTimestampsKHR calibratedTimestamps = nullptr;
     PFN_vkCmdSetCheckpointNV checkpointMarker = nullptr;
     bool deviceLostHook = false;
@@ -304,6 +305,7 @@ struct VulkanDevice::State {
     bool float16DenormPreserve = false;
     bool demoteToHelperInvocation = false;
     bool earlyFragmentSampleMaskTestBeforeSampleCounting = false;
+    bool robustBufferAccess2 = false;
     // VK_EXT_descriptor_indexing with non-uniform image array indexing (bindless image tables in
     // graphics stages, and compute workgroups wider than a wave).
     bool descriptorIndexing = false;
@@ -1134,6 +1136,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         VkPhysicalDeviceProperties2 driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &driver};
         state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &driverProperties);
         state->driverInfo = std::string(driver.driverName) + " " + driver.driverInfo;
+        state->individuallyFreeDescriptorSets = driver.driverID == VK_DRIVER_ID_MESA_LLVMPIPE;
     }
     auto byteFeatures = QueryBdaByteFeatures(selected, state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2"), availableExtensions);
     auto bdaFeatures = QueryBdaFeatures(selected, state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2"), availableExtensions);
@@ -1844,9 +1847,12 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         VkPhysicalDeviceFeatures2 queried{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &robustness2};
         state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(selected, &queried);
     }
-    if (robustness2.nullDescriptor == VK_TRUE) {
+    state->robustBufferAccess2 = robustness2.robustBufferAccess2 == VK_TRUE && enabled.robustBufferAccess == VK_TRUE;
+    if (robustness2.nullDescriptor == VK_TRUE || state->robustBufferAccess2) {
+        const bool nullDescriptor = robustness2.nullDescriptor == VK_TRUE;
         robustness2 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT};
-        robustness2.nullDescriptor = VK_TRUE;
+        robustness2.nullDescriptor = nullDescriptor ? VK_TRUE : VK_FALSE;
+        robustness2.robustBufferAccess2 = state->robustBufferAccess2 ? VK_TRUE : VK_FALSE;
         robustness2.pNext = bdaFeatures.pNext;
         bdaFeatures.pNext = &robustness2;
         deviceExtensions.push_back(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME);
@@ -3507,6 +3513,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const {
     target.spannedWaves = state->spannedWaves;
     target.fixedPushSlots = state->graphicsPipelineLibrary;
     target.registrationModules = state->graphicsPipelineLibrary;
+    target.robustBufferAccess2 = state->robustBufferAccess2;
     target.srgbDecodeFormats = state->srgbDecodeFormats;
     target.vertexSubgroups = (state->subgroup.supportedStages & VK_SHADER_STAGE_VERTEX_BIT) != 0;
     if (state->meshShader) {
@@ -3523,6 +3530,10 @@ std::optional<ShaderRecompiler::GeometryStageLimits> VulkanDevice::GeometryLimit
     return ShaderRecompiler::GeometryStageLimits{limits.maxGeometryInputComponents, limits.maxGeometryOutputComponents, limits.maxGeometryOutputVertices, limits.maxGeometryTotalOutputComponents, limits.maxFragmentInputComponents};
 }
 
+std::uint32_t VulkanDevice::MaxFragmentInputComponents() const {
+    return state->properties.limits.maxFragmentInputComponents;
+}
+
 bool VulkanDevice::PrimitiveListRestart() const {
     return state->primitiveListRestart;
 }
@@ -3587,6 +3598,7 @@ Graphics::Context VulkanDevice::buildContext() const {
         state->hostImportAlignment
     };
     context.pipelineCacheSynchronization = state->pipelineCache ? &state->pipelineCache->Synchronization() : nullptr;
+    context.individuallyFreeDescriptorSets = state->individuallyFreeDescriptorSets;
     context.dmaBufImport = state->dmaBufImport;
     context.sparseBinding = state->sparseBinding;
     context.queueFamily = state->queueFamily;
@@ -3649,6 +3661,7 @@ Graphics::Context VulkanDevice::buildContext() const {
     context.shaderStencilExport = state->shaderStencilExport;
     context.memoryProperties2 =state->memoryBudget ? state->InstanceFunction<PFN_vkGetPhysicalDeviceMemoryProperties2>("vkGetPhysicalDeviceMemoryProperties2") : nullptr;
     context.singlePassStorage = state->storageImageWithoutFormat;
+    context.cacheReclaim = Graphics::ReleaseTextureCaches;
     return context;
 }
 
@@ -3736,11 +3749,24 @@ bool TemplateDataRefresh() {
     return enabled;
 }
 
+bool DispatchMove() {
+    static const bool enabled = std::getenv("APS5_NO_DISPATCH_MOVE") == nullptr && TemplateDataRefresh();
+    return enabled;
+}
+
+std::size_t DispatchMoveBytes() {
+    static const std::size_t bytes = [] {
+        const char* text = std::getenv("APS5_DISPMOVE_MAX_KIB");
+        return static_cast<std::size_t>(text != nullptr ? std::strtoull(text, nullptr, 10) : 1024ull) * 1024u;
+    }();
+    return bytes;
+}
+
 // The content key of a compute stage, naming the device: the cache is process-wide and the driver
 // replaces the headless device with the windowed one while workers may still use the old one, so
 // the key names the device that built the entry (its descriptor set and pooled buffers belong to it).
-ResourceCache::Key DispatchContentKey(const Graphics::CompiledShader& shader, VkDevice device) {
-    auto key = Graphics::ShaderResources::ContentKey(shader, !TemplateDataRefresh());
+ResourceCache::Key DispatchContentKey(const Graphics::CompiledShader& shader, VkDevice device, bool movable = true) {
+    auto key = Graphics::ShaderResources::ContentKey(shader, !TemplateDataRefresh(), movable && DispatchMove());
     const auto deviceHandle = reinterpret_cast<std::uint64_t>(device);
     key.push_back(static_cast<std::uint32_t>(deviceHandle));
     key.push_back(static_cast<std::uint32_t>(deviceHandle >> 32u));
@@ -3771,6 +3797,7 @@ struct PreparedDispatch {
     std::shared_ptr<Graphics::ShaderResources> resources;
     std::shared_ptr<Graphics::ShaderResources> cached;
     ResourceCache::Key key;
+    bool exact = false;
     // APS5_PROFILE_DRAW: stage A's time, added to the [dispatch] phase totals by the dispatch, and
     // the prepare's parts for the driver's 'prepare:' rows (PreparePhase order).
     double prepareMs = 0;
@@ -3803,6 +3830,8 @@ struct PresyncCounters {
     std::atomic<std::uint64_t> checked{0};
     std::atomic<std::uint64_t> presyncs{0};
     std::atomic<std::uint64_t> waitedUs{0};
+    std::atomic<std::uint64_t> keyChecks{0};
+    std::atomic<std::uint64_t> keyWaits{0};
     std::atomic<std::int64_t> lastReport{0};
 };
 
@@ -3968,6 +3997,63 @@ RecipeCounters& Recipes() {
     return counters;
 }
 
+constexpr std::size_t MoveRefusals = static_cast<std::size_t>(Graphics::ShaderResources::MoveRefusal::Count);
+
+struct DispatchMoveCounters {
+    std::atomic<std::uint64_t> checked{0}, exact{0}, moved{0}, buffers{0}, bytes{0};
+    std::atomic<std::uint64_t> rebuilt{0}, kept{0}, served{0};
+    std::array<std::atomic<std::uint64_t>, 5> sizes{};
+    std::array<std::atomic<std::uint64_t>, MoveRefusals> refused{};
+    std::array<std::atomic<std::uint64_t>, MoveRefusals> filtered{};
+    std::atomic<std::int64_t> lastReport{0};
+};
+
+DispatchMoveCounters& DispatchMoves() {
+    static DispatchMoveCounters counters;
+    return counters;
+}
+
+void reportDispatchMove() {
+    auto& moves = DispatchMoves();
+    const auto nowMs = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
+    auto last = moves.lastReport.load();
+    if (nowMs - last < 10000 || !moves.lastReport.compare_exchange_strong(last, nowMs)) return;
+    const auto take = [](std::atomic<std::uint64_t>& counter) { return static_cast<unsigned long long>(counter.exchange(0, std::memory_order_relaxed)); };
+    const auto list = [&](std::array<std::atomic<std::uint64_t>, MoveRefusals>& counters) {
+        std::string text;
+        unsigned long long total = 0;
+        char item[80];
+        for (std::size_t i = 1; i < MoveRefusals; ++i) {
+            const auto count = take(counters[i]);
+            total += count;
+            if (count == 0) continue;
+            std::snprintf(item, sizeof(item), "%s %s %llu", text.empty() ? "" : ",", Graphics::ShaderResources::MoveRefusalName(static_cast<Graphics::ShaderResources::MoveRefusal>(i)), count);
+            text += item;
+        }
+        std::snprintf(item, sizeof(item), "%llu (", total);
+        return std::string(item) + text + ")";
+    };
+    const auto checked = take(moves.checked);
+    const auto exact = take(moves.exact);
+    const auto moved = take(moves.moved);
+    const auto buffers = take(moves.buffers);
+    const auto bytes = take(moves.bytes);
+    const auto rebuilt = take(moves.rebuilt);
+    const auto kept = take(moves.kept);
+    const auto served = take(moves.served);
+    unsigned long long sizes[5];
+    for (std::size_t i = 0; i < 5; ++i) sizes[i] = take(moves.sizes[i]);
+    const auto refused = list(moves.refused);
+    const auto filtered = list(moves.filtered);
+    AgcDriver::ProfilePrint_nid_no_patch("[dispmove] (10 s) %llu template hits checked: %llu with every read-only buffer where it was built, %llu with %llu moved ones (%.2f MiB snapshotted; per dispatch up to 1 KiB %llu, 16 KiB %llu, 256 KiB %llu, 4 MiB %llu, more %llu); refused %s at stage A (the hit was dropped before the record), %s at record; after a refusal: %llu rebuilt, %llu inserted beside the movable template (kept), %llu served by an exact entry\n", checked, exact, moved, buffers, static_cast<double>(bytes) / 1048576.0, sizes[0], sizes[1], sizes[2], sizes[3], sizes[4], filtered.c_str(), refused.c_str(), rebuilt, kept, served);
+}
+
+void noteDispatchMoveSize(std::size_t bytes) {
+    auto& moves = DispatchMoves();
+    const std::size_t bucket = bytes <= 1024 ? 0 : bytes <= 16384 ? 1 : bytes <= 262144 ? 2 : bytes <= 4194304 ? 3 : 4;
+    moves.sizes[bucket].fetch_add(1, std::memory_order_relaxed);
+}
+
 void reportRecipes() {
     auto& recipes = Recipes();
     const auto nowMs = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
@@ -4118,12 +4204,23 @@ struct RecordedDispatch {
     // Whether the data refresh recorded anything.
     bool refreshed = false;
     bool resourceKeyMatched = false;
+    std::shared_ptr<Graphics::ShaderResources::DrawBindings> bindings{};
 };
 
 VkDevice VulkanDevice::Device() const {
     return state->device;
 }
 
+VulkanDevice::DispatchMoveCount VulkanDevice::DispatchMoveCounts() {
+    auto& moves = DispatchMoves();
+    DispatchMoveCount count{moves.checked.load(), moves.exact.load(), moves.moved.load(), 0, 0, moves.rebuilt.load(), moves.kept.load(), moves.served.load()};
+    for (std::size_t i = 1; i < MoveRefusals; ++i) {
+        count.refused += moves.refused[i].load();
+        count.filtered += moves.filtered[i].load();
+    }
+    return count;
+}
+
 bool VulkanDevice::DispatchRecipes() {
     static const bool noDispatchRecipe = std::getenv("APS5_NO_DISPATCH_RECIPE") != nullptr;
     return !noDispatchRecipe;
@@ -4185,23 +4282,31 @@ RecipeOutcome VulkanDevice::DrawFromRecipe(const Graphics::State& graphics, cons
     return RecipeOutcome::Recorded;
 }
 
-std::uint64_t VulkanDevice::presync(std::span<const std::pair<std::uint64_t, std::uint64_t>> surfaces) {
+std::uint64_t VulkanDevice::presync(std::span<const Graphics::PresyncSurface> surfaces) {
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
     auto& counters = Presyncs();
     bool overlaps = false;
-    for (const auto& [address, bytes] : surfaces) overlaps = overlaps || Graphics::Recorder::SnapshotWriteOverlaps(address, static_cast<std::size_t>(bytes));
+    for (const auto& surface : surfaces) {
+        const auto bytes = static_cast<std::size_t>(surface.bytes);
+        if (surface.dccKeys) counters.keyChecks.fetch_add(1, std::memory_order_relaxed);
+        overlaps = overlaps || (surface.dccKeys ? Graphics::DccKeyWritersPending(surface.address, bytes) : Graphics::Recorder::SnapshotWriteOverlaps(surface.address, bytes));
+    }
     std::uint64_t serial = 0;
+    bool texelsPending = false;
     if (overlaps) {
         // No reap here (SubmitAndEpoch's would run completions, which a compute worker must not do
         // on queue 0's behalf, see SubmitRecorded): the dispatch reaps after the wait.
         std::lock_guard lock(GuestMemory::GpuMutex());
         if (state->recorder) {
             bool open = false;
-            for (const auto& [address, bytes] : surfaces) {
-                const auto info = state->recorder->DescribePendingWrite(address, static_cast<std::size_t>(bytes));
+            for (const auto& surface : surfaces) {
+                const auto bytes = static_cast<std::size_t>(surface.bytes);
+                if (surface.dccKeys && !Graphics::DccKeyWriterNeedsSync(surface.address, bytes)) continue;
+                const auto info = state->recorder->DescribePendingWrite(surface.address, bytes);
                 if (!info.has_value()) continue;
                 open = open || info->open;
                 serial = std::max(serial, info->serial);
+                texelsPending = texelsPending || !surface.dccKeys;
             }
             if (open) serial = state->recorder->SubmitAndEpoch();
         }
@@ -4210,16 +4315,25 @@ std::uint64_t VulkanDevice::presync(std::span<const std::pair<std::uint64_t, std
         const auto waitStart = std::chrono::steady_clock::now();
         WaitRecorded(serial);
         counters.presyncs.fetch_add(1, std::memory_order_relaxed);
+        if (!texelsPending) counters.keyWaits.fetch_add(1, std::memory_order_relaxed);
         counters.waitedUs.fetch_add(static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - waitStart).count()), std::memory_order_relaxed);
     }
     if (profile) {
         const auto nowMs = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
         auto last = counters.lastReport.load();
-        if (nowMs - last >= 10000 && counters.lastReport.compare_exchange_strong(last, nowMs)) AgcDriver::ProfilePrint_nid_no_patch("[presync] %llu dispatches checked, %llu pre-syncs waited %.0f ms (cumulative)\n", static_cast<unsigned long long>(counters.checked.load()), static_cast<unsigned long long>(counters.presyncs.load()), counters.waitedUs.load() / 1000.0);
+        if (nowMs - last >= 10000 && counters.lastReport.compare_exchange_strong(last, nowMs)) {
+            const auto keyWaitCounts = Graphics::KeyWaitCounts();
+            AgcDriver::ProfilePrint_nid_no_patch("[presync] %llu dispatches checked, %llu pre-syncs waited %.0f ms (cumulative); key ranges %llu checked, %llu pre-syncs that only the keys required; WaitForKeyWriters %llu calls: %llu without a pending writer, %llu known keys, %llu fills leaving uncompressed, %llu synced\n", static_cast<unsigned long long>(counters.checked.load()), static_cast<unsigned long long>(counters.presyncs.load()), counters.waitedUs.load() / 1000.0, static_cast<unsigned long long>(counters.keyChecks.load()), static_cast<unsigned long long>(counters.keyWaits.load()), static_cast<unsigned long long>(keyWaitCounts.calls), static_cast<unsigned long long>(keyWaitCounts.noPending), static_cast<unsigned long long>(keyWaitCounts.known), static_cast<unsigned long long>(keyWaitCounts.fillsUncompressed), static_cast<unsigned long long>(keyWaitCounts.synced));
+        }
     }
     return serial;
 }
 
+VulkanDevice::PresyncStatistics VulkanDevice::PresyncCounts() {
+    const auto& counters = Presyncs();
+    return {counters.checked.load(std::memory_order_relaxed), counters.presyncs.load(std::memory_order_relaxed), counters.keyChecks.load(std::memory_order_relaxed), counters.keyWaits.load(std::memory_order_relaxed)};
+}
+
 std::shared_ptr<PreparedDispatch> VulkanDevice::PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t dispatchThreads, std::uint64_t programAddress) {
     ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion);
     // APS5_LOCKED_BUILD=1: the whole build under the mutex, as before the split.
@@ -4246,6 +4360,15 @@ std::shared_ptr<PreparedDispatch> VulkanDevice::PrepareDispatch(const ShaderReco
         prepared->key = DispatchContentKey(compute, context.device);
         phase(PreparedDispatch::PrepareKey);
         cached = state->resourceCache.Find(prepared->key);
+        if (cached != nullptr && DispatchMove()) {
+            Graphics::ShaderResources::MoveProof proof;
+            if (!cached->MovedDispatchBuffers(compute, nullptr, DispatchMoveBytes(), proof).has_value()) {
+                DispatchMoves().filtered[static_cast<std::size_t>(proof.refusal)].fetch_add(1, std::memory_order_relaxed);
+                prepared->key = DispatchContentKey(compute, context.device, false);
+                prepared->exact = true;
+                cached = state->resourceCache.Find(prepared->key);
+            }
+        }
         prepared->cached = cached;
         phase(PreparedDispatch::PrepareFind);
     }
@@ -4443,6 +4566,10 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
     recorder.Keep(record.objects);
     recorder.Keep(record.resources);
     if (record.resources != nullptr) recorder.KeepBytes(record.resources.get(), record.resources->CopiedBytes());
+    if (record.bindings != nullptr) {
+        recorder.Keep(record.bindings);
+        for (const auto& snapshot : record.bindings->snapshots) recorder.KeepBytes(snapshot.buffer.get(), snapshot.buffer->Bytes().size());
+    }
     recordStep(PhaseRecordKeeps);
     if (record.dataRefresh != RecordedDispatch::DataRefresh::None) {
         // The template's data buffers take this dispatch's words: a transfer write the pre-dispatch
@@ -4492,7 +4619,9 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
         ++d.preBarriersRecorded;
     }
     context.Resolved(&Graphics::DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->pipeline);
-    resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout);
+    if (record.bindings != nullptr) Graphics::Require(record.bindings->allocation.set != VK_NULL_HANDLE, "dispatch overlay has no descriptor set");
+    if (record.bindings != nullptr) context.Resolved(&Graphics::DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, 0, 1, &record.bindings->allocation.set, 0, nullptr);
+    else resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout);
     if (const auto dwords = record.shader->program->workgroupMemoryDwords; dwords != 0) {
         auto memory = state->WorkgroupMemoryFor(context, record.x, record.y, record.z, dwords);
         context.Resolved(&Graphics::DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, ShaderRecompiler::WorkgroupMemoryDescriptorSet, 1, &memory->set, 0, nullptr);
@@ -4705,7 +4834,9 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
     // The object came from the resource cache: a compute template whose data buffers take this
     // dispatch's words at the record (ShaderResources::RefreshData).
     bool fromCache = false;
+    bool refusedMove = false;
     double revalidateMs = 0;
+    std::shared_ptr<Graphics::ShaderResources::DrawBindings> overlay;
     // The batches the pre-sync (PrepareDispatch) waited for are reaped first: their completions
     // (CPU write-backs) land before stage B or a cached object's Revalidate reads, and the flush
     // hook then finds nothing pending over their surfaces.
@@ -4725,6 +4856,11 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
     // The image lookups of this call by outcome (stage B, a Revalidate or a locked build), for the
     // [indirect] line: reset here, read after the resources phase.
     if (profile && arguments != 0) Graphics::ThreadLookupOutcomes() = {};
+    const auto noteRefusalBuild = [](bool inserted) {
+        auto& moves = DispatchMoves();
+        moves.rebuilt.fetch_add(1, std::memory_order_relaxed);
+        if (inserted) moves.kept.fetch_add(1, std::memory_order_relaxed);
+    };
     // The resource cache insert evicts the oldest entry, destroying its ShaderResources here when the
     // cache was the last owner: timed apart from the build.
     const auto insert = [&] {
@@ -4773,6 +4909,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
         }
         if (cacheable) {
             ++d.cacheMisses;
+            if (prepared->exact) noteRefusalBuild(resources->Reusable());
             if (resources->Reusable()) insert();
         }
     } else if (cacheable) {
@@ -4780,13 +4917,54 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
         // object it found: revalidated as a map hit would be (an entry replaced meanwhile is left
         // alone by the pointer-conditional Remove). APS5_NO_PREPARED_FIND=1 looks the key up again.
         static const bool preparedFind = std::getenv("APS5_NO_PREPARED_FIND") == nullptr;
+        const bool exactEntry = prepared != nullptr && prepared->exact;
         contentKey = prepared != nullptr && !prepared->key.empty() ? std::move(prepared->key) : DispatchContentKey(shaders[0], context.device);
         auto cached = preparedFind && prepared != nullptr && prepared->cached != nullptr ? std::move(prepared->cached) : state->resourceCache.Find(contentKey);
         if (cached != nullptr) {
             if (cached->Revalidate(shaders[0])) {
-                resources = std::move(cached);
-                fromCache = true;
-                ++d.cacheHits;
+                bool usable = true;
+                if (DispatchMove() && !exactEntry) {
+                    auto& moves = DispatchMoves();
+                    Graphics::ShaderResources::MoveProof proof;
+                    auto moved = cached->MovedDispatchBuffers(shaders[0], state->recorder.get(), DispatchMoveBytes(), proof);
+                    moves.checked.fetch_add(1, std::memory_order_relaxed);
+                    if (moved.has_value() && !moved->empty()) {
+                        overlay = cached->PrepareDrawBindings(*state->recorder, *moved, true);
+                        if (overlay != nullptr && overlay->allocation.set == VK_NULL_HANDLE) overlay.reset();
+                        if (overlay == nullptr) {
+                            proof.refusal = Graphics::ShaderResources::MoveRefusal::Bindings;
+                            moved.reset();
+                        }
+                    }
+                    if (!moved.has_value()) {
+                        usable = false;
+                        moves.refused[static_cast<std::size_t>(proof.refusal)].fetch_add(1, std::memory_order_relaxed);
+                    } else if (moved->empty()) {
+                        moves.exact.fetch_add(1, std::memory_order_relaxed);
+                    } else {
+                        moves.moved.fetch_add(1, std::memory_order_relaxed);
+                        moves.buffers.fetch_add(proof.buffers, std::memory_order_relaxed);
+                        moves.bytes.fetch_add(proof.bytes, std::memory_order_relaxed);
+                        noteDispatchMoveSize(proof.bytes);
+                    }
+                }
+                if (!usable) {
+                    contentKey = DispatchContentKey(shaders[0], context.device, false);
+                    refusedMove = true;
+                    cached = state->resourceCache.Find(contentKey);
+                    if (cached != nullptr && cached->Revalidate(shaders[0])) usable = true;
+                    else if (cached != nullptr) {
+                        state->resourceCache.Remove(contentKey, cached.get());
+                        ++d.cacheInvalidated;
+                        state->recorder->Keep(std::move(cached));
+                    }
+                }
+                if (usable) {
+                    if (refusedMove || exactEntry) DispatchMoves().served.fetch_add(1, std::memory_order_relaxed);
+                    resources = std::move(cached);
+                    fromCache = true;
+                    ++d.cacheHits;
+                }
             } else {
                 state->resourceCache.Remove(contentKey, cached.get());
                 ++d.cacheInvalidated;
@@ -4805,6 +4983,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
         if (profile) timer.add(PhaseResourcesFullBuild, std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - buildStart).count());
         if (cacheable) {
             ++d.cacheMisses;
+            if (refusedMove) noteRefusalBuild(resources->Reusable());
             if (resources->Reusable()) insert();
         }
     }
@@ -4827,6 +5006,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
         AgcDriver::ProfilePrint_nid_no_patch("[rescache] %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw); template hits: %llu refreshed the data buffers (revalidate %.1f ms), %llu had the same words; Find calls %llu, Touch calls %llu\n", static_cast<unsigned long long>(d.cacheHits), static_cast<unsigned long long>(d.cacheMisses), static_cast<unsigned long long>(d.cacheInvalidated), state->resourceCache.Size(), static_cast<unsigned long long>(d.templateRefreshed), d.templateRevalidateMs, static_cast<unsigned long long>(d.templateSameWords), static_cast<unsigned long long>(ResourceCache::Finds()), static_cast<unsigned long long>(ResourceCache::Touches()));
         AgcDriver::ProfilePrint_nid_no_patch("[vk] deviceProc lookups inside the device call: %llu (%.2f per dispatch); pre-dispatch barriers recorded %llu, skipped %llu; compute pipelines created %llu (%llu under the GPU mutex)\n", static_cast<unsigned long long>(d.procLookups), d.profiledDispatches != 0 ? static_cast<double>(d.procLookups) / static_cast<double>(d.profiledDispatches) : 0.0, static_cast<unsigned long long>(d.preBarriersRecorded), static_cast<unsigned long long>(d.preBarriersSkipped), static_cast<unsigned long long>(computePipelinesCreated.load(std::memory_order_relaxed)), static_cast<unsigned long long>(computePipelinesCreatedLocked.load(std::memory_order_relaxed)));
         reportRecipes();
+        reportDispatchMove();
     }
     timing.Mark("shader_resources");
     timer.phase(PhaseResources);
@@ -4848,6 +5028,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
     IndirectOutcome outcome{0, 0};
     RecordedDispatch record{&context, &shaders[0], resources, objects, pushStages, &pushBytes, x, y, z, arguments, nullptr, programAddress, fromCache && TemplateDataRefresh() ? RecordedDispatch::DataRefresh::Words : RecordedDispatch::DataRefresh::None, 0, revalidateMs, &timer};
     record.resourceKeyMatched = fromCache;
+    record.bindings = overlay;
     if (arguments != 0) decideIndirect(record, outcome, groupsText);
     // The indirect hold total below still covers the whole call of a CPU-resolved one.
     const bool indirect = record.arguments != 0 || outcome.cpuReason != 0;
@@ -4901,7 +5082,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
     // object the resource cache serves under this content key (reusable: no lease, no copied
     // writes, every direct region import- or mirror-served), so a hit's proof is the template's
     // Revalidate and nothing needs completion work.
-    if (recipeOut != nullptr && DispatchRecipes() && cacheable && resources->Reusable() && !resources->NeedsCompletion() && objects != nullptr && !contentKey.empty()) {
+    if (recipeOut != nullptr && DispatchRecipes() && cacheable && overlay == nullptr && resources->Reusable() && !resources->NeedsCompletion() && objects != nullptr && !contentKey.empty()) {
         auto recipe = std::make_shared<Recipe>();
         recipe->device = state->device;
         recipe->templateRef = resources;

core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp

@@ -54,7 +54,8 @@ struct SlabSlot {
 class BufferPool {
 public:
     explicit BufferPool(const Context& context);
-    static VkDeviceSize DeviceBudget(const VkPhysicalDeviceMemoryProperties& memory);
+    static VkDeviceSize DeviceBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported = nullptr);
+    static VkDeviceSize ReclaimTarget(VkDeviceSize bytes);
     ~BufferPool();
     BufferPool(const BufferPool&) = delete;
     BufferPool& operator=(const BufferPool&) = delete;
@@ -116,6 +117,7 @@ private:
     // when shared; the device tier for device-local memory while it has a budget).
     Tier& tierFor(std::size_t capacity, VkMemoryPropertyFlags properties);
     static bool DeviceTierEnabled();
+    static VkDeviceSize deviceBudget(const Context& context);
     static bool DeviceTiered(VkMemoryPropertyFlags properties);
     static std::size_t NextCapacity(std::size_t capacity);
     VkDeviceSize destroy(const BufferAllocation& allocation) noexcept;
@@ -127,6 +129,7 @@ private:
     // The retained-slot bound of each tier (APS5_BUFFER_POOL_SLOTS, default `defaultSlots`), read once.
     static std::size_t MaxSlots();
     std::size_t maxSlots;
+    VkDeviceSize (*cacheReclaim)(VkDeviceSize bytes);
     VkDevice device;
     PFN_vkUnmapMemory unmap;
     PFN_vkDestroyBuffer destroyBuffer;

core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp

@@ -172,6 +172,7 @@ struct Context {
     std::uint32_t settleFamily = 0;
     std::mutex* settleQueueMutex = nullptr;
     PFN_vkGetPhysicalDeviceMemoryProperties2 memoryProperties2 = nullptr;
+    VkDeviceSize (*cacheReclaim)(VkDeviceSize bytes) = nullptr;
     RenderCache* renderCache = nullptr;
     DrawQueue* drawQueue = nullptr;
     GraphicsPipelineCache* graphicsPipelines = nullptr;
@@ -211,6 +212,7 @@ struct Context {
     bool variableDescriptorCount = false;
     std::uint32_t imageTableCapacity = 0;
     std::uint32_t samplerTableCapacity = 0;
+    bool individuallyFreeDescriptorSets = false;
     bool geometryShader = false;
     bool sampleRateShading = false;
     bool storageImageMultisample = false;
@@ -322,6 +324,7 @@ inline void EndRenderPass(const Context& context, VkCommandBuffer commands) {
 }
 
 std::uint32_t ImageTableCapacity(const Context& context);
+std::uint32_t StorageTableCapacity(const Context& context);
 std::uint32_t SamplerTableCapacity(const Context& context);
 
 }

core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp

@@ -99,6 +99,16 @@ DccKeys ProvedCurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceByt
 void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span<std::byte> bytes);
 void NoteKeysFillOnGpu(std::uint64_t begin, std::size_t count, DccKeys keys);
 std::optional<DccKeys> WaitForKeyWriters(const GuestTextureResource& resource, std::uint64_t guestBytes);
+bool DccKeyWritersPending(std::uint64_t metaAddress, std::size_t count);
+bool DccKeyWriterNeedsSync(std::uint64_t metaAddress, std::size_t count);
+struct DccKeyWaitCounts {
+    std::uint64_t calls;
+    std::uint64_t noPending;
+    std::uint64_t known;
+    std::uint64_t fillsUncompressed;
+    std::uint64_t synced;
+};
+DccKeyWaitCounts KeyWaitCounts();
 
 }
 

core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp

@@ -29,6 +29,7 @@ void RequireNoNativeDepthFeedback_nid_no_patch(const Context& context, const Dep
 void ClearDepthSurfaces(VkDevice device);
 void RetireDepthSurfaces(VkDevice device, std::uint64_t address, std::uint64_t bytes);
 bool DepthSurfaceAt(std::uint64_t address);
+bool DepthSurfaceOverlaps(VkDevice device, std::uint64_t address, std::uint64_t bytes);
 bool DepthSurfaceAt(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& resource);
 std::uint64_t HtileDepthClearAddress(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const std::array<std::uint32_t, 3>& numThreads);
 std::size_t HtileDepthClearBytes_nid_no_patch(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, std::span<const std::uint32_t> packet, std::uint64_t indirectArguments);

core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp

@@ -94,6 +94,15 @@ bool DrawRecipes();
 enum class IndirectDrawPath : std::uint8_t { Gpu, NotFolded, FetchUnknown, NonVertexPath, IndxOffset, DrawIndex, VertexRange, FeatureGap, PendingImage, PendingLabelOrCopy, NotImported, Disabled, Count };
 void CountIndirectDraw(IndirectDrawPath path, double readMs, bool rewritten = false);
 std::array<std::uint64_t, static_cast<std::size_t>(IndirectDrawPath::Count)> IndirectDrawCounts();
+
+struct DrawPassCounts {
+    std::uint64_t draws = 0;
+    std::uint64_t passesBegun = 0;
+    std::uint64_t passesContinued = 0;
+    std::uint64_t keyStoreFlushes = 0;
+    std::uint64_t storeFlushes = 0;
+};
+std::array<DrawPassCounts, 2> DrawPassTotals();
 const char* IndirectDrawPathName(IndirectDrawPath path);
 
 // Draw packets that drew nothing, for the [draws] line: Nothing (an empty count, or no color writes

core/libs/prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp

@@ -16,6 +16,8 @@ struct CompareSpan {
 void CompareSpans(std::span<const CompareSpan> spans, std::span<std::uint8_t> equal);
 void CompareSpansFrom(std::span<const CompareSpan> spans, std::span<std::uint8_t> equal, std::size_t parallelBytes);
 unsigned CompareHelpers();
+void StopCompareHelpers();
+unsigned CompareHelpersRunning();
 
 }
 

core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp

@@ -140,6 +140,7 @@ bool HasStencil(VkFormat format);
 void ValidateViewport(const Context& context, const VkViewport& viewport);
 void ValidateDepthBounds(const Context& context, const State& state);
 void ValidateProvokingVertex(const Context& context, const State& state, std::span<const CompiledShader> shaders = {});
+bool BarycentricPatches(const State& state, std::span<const CompiledShader> shaders);
 bool ValidateSampleLocations(const Context& context, const State& state);
 
 void ValidateShaderPair(const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment);

core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp

@@ -48,6 +48,9 @@ void WaitForOptimizedPipelines(VkDevice device);
 
 void ClearPipelineLibraries(VkDevice device);
 
+void StopPipelineLibraryWorker();
+unsigned PipelineLibraryWorkers();
+
 struct PipelineLibraryCounters {
     std::array<std::uint64_t, 4> built{};
     std::uint64_t linked = 0;

core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp

@@ -64,6 +64,7 @@ private:
     VkBufferUsageFlags usage;
     std::shared_ptr<BufferPool> cache;
     bool hostPlaced = false;
+    bool ready = false;
 };
 
 // Records a whole-range buffer copy.

core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp

@@ -28,10 +28,12 @@ class Recorder;
 
 void FlushCachedTextures(VkDevice device);
 void ClearCachedTextures(VkDevice device);
+std::uint64_t ReleaseTextureCaches(std::uint64_t bytes);
 std::uint64_t TextureCacheBudget(const VkPhysicalDeviceMemoryProperties& memory);
-std::uint64_t StorageTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported);
+std::uint64_t StorageTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t sampledBytes, std::uint64_t storageBytes);
 std::uint64_t SampledTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t textureBytes);
 bool SampledBudgetReportDue(std::uint64_t reported, std::uint64_t budget, std::chrono::steady_clock::duration sinceReport);
+std::uint64_t StorageTextureCacheBudget(const Context& context);
 struct TextureCacheUse {
     std::size_t sampledEntries = 0;
     std::uint64_t sampledBytes = 0;
@@ -42,6 +44,7 @@ std::uint64_t SampledTextureCacheBudget(const Context& context);
 std::shared_ptr<Texture> CachedSampledTexture(const Context& context, std::span<const std::uint32_t> words);
 bool SampledTexturesShareEntry(std::span<const std::uint32_t> first, std::span<const std::uint32_t> second);
 bool CompletePendingCpuWrites(const Context& context, std::uint64_t address, std::size_t bytes);
+bool SpirvStoresThroughAddresses(std::span<const std::uint32_t> words);
 
 // The cached storage image of a surface (render targets use it as their resident image); brought up
 // to date with guest memory before it is returned.
@@ -98,6 +101,7 @@ private:
     PFN_vkDestroyDescriptorSetLayout destroyLayout;
     PFN_vkDestroyDescriptorPool destroyPool;
     PFN_vkResetDescriptorPool resetPool;
+    PFN_vkFreeDescriptorSets freeDescriptorSets = nullptr;
     mutable std::mutex mutex;
     std::map<std::vector<std::uint32_t>, VkDescriptorSetLayout> layouts;
     struct Pool {
@@ -122,6 +126,12 @@ struct ImageTableReuseCounts {
 };
 ImageTableReuseCounts ImageTableReuses();
 
+struct PresyncSurface {
+    std::uint64_t address;
+    std::uint64_t bytes;
+    bool dccKeys;
+};
+
 class ShaderResources {
 public:
     ShaderResources(const Context& context, const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes);
@@ -168,8 +178,16 @@ public:
         std::size_t size;
         std::vector<std::uint32_t> words;
     };
-    std::shared_ptr<DrawBindings> PrepareDrawBindings(Recorder& recorder, std::span<const MovedBuffer> moved = {}) const;
+    std::shared_ptr<DrawBindings> PrepareDrawBindings(Recorder& recorder, std::span<const MovedBuffer> moved = {}, bool movedOnly = false) const;
     std::optional<std::vector<MovedBuffer>> MovedReadOnlyBuffers(std::span<const CompiledShader> shaders, Recorder& recorder) const;
+    enum class MoveRefusal : std::size_t { None, Shape, NotGuest, Written, Limit, Inaccessible, Overlap, Bindings, Count };
+    struct MoveProof {
+        MoveRefusal refusal = MoveRefusal::None;
+        std::size_t buffers = 0;
+        std::size_t bytes = 0;
+    };
+    static const char* MoveRefusalName(MoveRefusal refusal);
+    std::optional<std::vector<MovedBuffer>> MovedDispatchBuffers(const CompiledShader& shader, Recorder* recorder, std::size_t maxBytes, MoveProof& proof) const;
     static std::size_t MaxDrawSnapshotBytes();
     void WriteBack();
     // Deferred completion: MarkGpuWrites registers the results the recorded work leaves on the GPU
@@ -182,6 +200,7 @@ public:
     // Whether a written buffer was copied (its results reach guest memory by the CPU write-back).
     bool HasCopiedWrites() const { return guestMemory.HasCopiedWrites(); }
     bool HoldsLease() const { return guestMemory.HoldsLease(); }
+    bool BdaReadOnly() const { return usesBda && bdaReadOnly; }
     bool UsesAddressTable() const { return usesBda; }
     std::size_t CopiedBytes() const { return guestMemory.CopiedBytes(); }
     bool WritesOverlap(std::uint64_t address, std::size_t bytes) const { return guestMemory.WritesOverlap(address, bytes); }
@@ -226,6 +245,7 @@ public:
     bool Reusable() const { return reusable; }
     const std::vector<std::uint64_t>& MultisampleSources() const { return multisampleSources; }
     static bool NeverReusable(std::span<const CompiledShader> shaders);
+    static std::uint64_t DepthGuardRefusals();
     const std::vector<std::shared_ptr<Texture>>& SampledTextures() const { return textures; }
     bool ChecksFaults() const { return bda != nullptr; }
     void CheckFaults() const;
@@ -280,7 +300,7 @@ public:
     // Usable right after the deferred constructor, for address-based builds too (it decodes the
     // compiled shader's bindings itself when stage A has not run), and on a completed object from
     // the resource cache (its Revalidate repeats the same lookups), without the device lock.
-    std::vector<std::pair<std::uint64_t, std::uint64_t>> PresyncSurfaces() const;
+    std::vector<PresyncSurface> PresyncSurfaces() const;
 
 private:
     friend class AgcDriver::VulkanDevice;
@@ -338,12 +358,15 @@ private:
         std::uint32_t set = 0;
         std::size_t firstSampler = 0;
         std::size_t samplerCount = 0;
+        bool sampled = false;
+        bool storage = false;
         std::vector<std::size_t> elementTextures;
         std::vector<std::pair<std::uint32_t, std::uint32_t>> samplerRejections;
     };
     std::vector<TableStage> tableStages;
     std::array<std::size_t, 2> tableBindings{std::numeric_limits<std::size_t>::max(), std::numeric_limits<std::size_t>::max()};
     std::array<std::uint32_t, 2> tableElements{};
+    std::array<std::size_t, 2> storageTableBindings{std::numeric_limits<std::size_t>::max(), std::numeric_limits<std::size_t>::max()};
     TableStage& tableStage(const CompiledShader& shader);
     static const ShaderRecompiler::DescriptorBinding* tableImages(std::span<const CompiledShader> shaders, ShaderRecompiler::ShaderStage stage);
     std::uint32_t rejectTableElement(TableStage& table, std::span<const std::uint32_t> words, const std::string& message);
@@ -375,6 +398,7 @@ private:
     // same way but is left out of the write set (no write-back, no pending-write note). `atomic`
     // is its bufferAtomic (see GuestBufferMemory::AddWritable).
     std::size_t addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read);
+    std::uint64_t guestBufferBytes(std::span<const std::uint32_t> words) const;
     std::size_t addDataBuffer(std::span<const std::uint32_t> words);
     // Stage A: the layout entry of an image binding (samplers are taken at once, the sampler cache
     // locks itself); stage B looks the sampled textures and storage images up (resolveImageBinding).
@@ -486,6 +510,7 @@ private:
     std::size_t maxDrawSnapshotBytes = MaxDrawSnapshotBytes();
     std::unique_ptr<BdaResources> bda;
     bool usesBda = false;
+    bool bdaReadOnly = true;
     std::vector<std::uint64_t> multisampleSources;
     bool usesFaultBuffer = false;
     bool faultChecks = false;

core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp

@@ -186,6 +186,7 @@ public:
     void MarkDirty();
     void MarkSliceDirty(std::uint32_t slice);
     void Flush();
+    bool Dirty() const;
     // Stores every pending image overlapping the range, except `except`; returns whether any was.
     // Stores into host-imported memory are recorded (not waited for): a CPU reader syncs afterwards.
     // Then the unit shadows over the range are published into their imports as `scope` says (see

core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp

@@ -84,6 +84,11 @@ inline constexpr TextureSwizzleEquation kTextureSwizzleEquations[] = {
     {0x109u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00002000u, 0x00000002u, 0x02000000u, 0x00004000u, 0x00000004u, 0x04000000u, 0x00008000u, 0x00000008u, 0x08000000u, 0x00010000u, 0x00000010u}},
     {0x109u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x00000002u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x00004000u, 0x00000008u, 0x08000000u, 0x00008000u, 0x00000010u}},
     {0x109u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x00001000u, 0x00000001u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x00004000u, 0x00000004u, 0x08000000u, 0x00008000u, 0x00000008u}},
+    {0x115u, 1u, {0x00000001u, 0x00000002u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x40004040u, 0x20020008u, 0x08010020u, 0x10008010u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}},
+    {0x115u, 2u, {0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x40004020u, 0x20020004u, 0x08010010u, 0x10008008u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}},
+    {0x115u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00002000u, 0x00000002u, 0x02000000u, 0x20004020u, 0x10020004u, 0x04010010u, 0x08008008u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}},
+    {0x115u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x00000002u, 0x02000000u, 0x20002020u, 0x10010004u, 0x04008010u, 0x08004008u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}},
+    {0x115u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x00001000u, 0x00000001u, 0x02000000u, 0x20002010u, 0x10010002u, 0x04008008u, 0x08004004u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}},
     {0x119u, 1u, {0x00000001u, 0x00000002u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000004u, 0x04000000u, 0x40004040u, 0x20020008u, 0x08010020u, 0x10008010u, 0x00000010u, 0x10000000u, 0x00010000u, 0x00000020u}},
     {0x119u, 2u, {0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00002000u, 0x00000002u, 0x04000000u, 0x40004020u, 0x20020004u, 0x08010010u, 0x10008008u, 0x00000008u, 0x10000000u, 0x00010000u, 0x00000010u}},
     {0x119u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00002000u, 0x00000002u, 0x02000000u, 0x20004020u, 0x10020004u, 0x04010010u, 0x08008008u, 0x00000008u, 0x08000000u, 0x00010000u, 0x00000010u}},

core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp

@@ -36,10 +36,10 @@ constexpr VkBufferUsageFlags deviceUsage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
 
 }
 
-BufferPool::BufferPool(const Context& context) : device(context.device), unmap(context.Function<PFN_vkUnmapMemory>("vkUnmapMemory")), destroyBuffer(context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")), freeMemory(context.Function<PFN_vkFreeMemory>("vkFreeMemory")), allocateMemory(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")), mapMemory(context.Function<PFN_vkMapMemory>("vkMapMemory")), maxSlots(std::max<std::size_t>(1, std::min<std::size_t>(MaxSlots(), context.limits.maxMemoryAllocationCount / 6u))) {
+BufferPool::BufferPool(const Context& context) : device(context.device), unmap(context.Function<PFN_vkUnmapMemory>("vkUnmapMemory")), destroyBuffer(context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")), freeMemory(context.Function<PFN_vkFreeMemory>("vkFreeMemory")), allocateMemory(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")), mapMemory(context.Function<PFN_vkMapMemory>("vkMapMemory")), maxSlots(std::max<std::size_t>(1, std::min<std::size_t>(MaxSlots(), context.limits.maxMemoryAllocationCount / 6u))), cacheReclaim(context.cacheReclaim) {
     smallTier.budget = smallBudget;
     largeTier.budget = budget;
-    deviceTier.budget = DeviceTierEnabled() ? DeviceBudget(context.memory) : 0;
+    deviceTier.budget = deviceBudget(context);
 }
 
 BufferPool::~BufferPool() {
@@ -57,7 +57,9 @@ VkDeviceMemory BufferPool::AllocateMemory(const VkMemoryAllocateInfo& allocation
     const auto initialResult = result;
     unsigned attempts = 1;
     VkDeviceSize reclaimed = 0;
-    if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY || result == VK_ERROR_OUT_OF_HOST_MEMORY) {
+    VkDeviceSize released = 0;
+    const auto exhausted = [](VkResult value) { return value == VK_ERROR_OUT_OF_DEVICE_MEMORY || value == VK_ERROR_OUT_OF_HOST_MEMORY; };
+    if (exhausted(result)) {
         reclaimed = trim();
         if (reclaimed != 0) {
             memory = VK_NULL_HANDLE;
@@ -65,16 +67,28 @@ VkDeviceMemory BufferPool::AllocateMemory(const VkMemoryAllocateInfo& allocation
             ++attempts;
         }
     }
+    if (exhausted(result)) {
+        if (cacheReclaim != nullptr) released = cacheReclaim(ReclaimTarget(allocation.allocationSize));
+        if (released != 0) {
+            memory = VK_NULL_HANDLE;
+            result = allocateMemory(device, &allocation, nullptr, &memory);
+            ++attempts;
+        }
+    }
     if (result == VK_SUCCESS) {
-        if (attempts == 2) std::fprintf(stderr, "[bufferpool] %s recovered: allocation=%llu reclaimed=%llu initialResult=%d attempts=%u\n", operation, static_cast<unsigned long long>(allocation.allocationSize), static_cast<unsigned long long>(reclaimed), static_cast<int>(initialResult), attempts);
+        if (attempts > 1) std::fprintf(stderr, "[bufferpool] %s recovered: allocation=%llu reclaimed=%llu released=%llu initialResult=%d attempts=%u\n", operation, static_cast<unsigned long long>(allocation.allocationSize), static_cast<unsigned long long>(reclaimed), static_cast<unsigned long long>(released), static_cast<int>(initialResult), attempts);
         return memory;
     }
     char details[512];
-    std::snprintf(details, sizeof(details), "%s: allocation=%llu memoryType=%u reclaimed=%llu initialResult=%d attempts=%u", operation, static_cast<unsigned long long>(allocation.allocationSize), allocation.memoryTypeIndex, static_cast<unsigned long long>(reclaimed), static_cast<int>(initialResult), attempts);
+    std::snprintf(details, sizeof(details), "%s: allocation=%llu memoryType=%u reclaimed=%llu released=%llu initialResult=%d attempts=%u", operation, static_cast<unsigned long long>(allocation.allocationSize), allocation.memoryTypeIndex, static_cast<unsigned long long>(reclaimed), static_cast<unsigned long long>(released), static_cast<int>(initialResult), attempts);
     Check(result, details);
     return VK_NULL_HANDLE;
 }
 
+VkDeviceSize BufferPool::ReclaimTarget(VkDeviceSize bytes) {
+    return bytes + std::max<VkDeviceSize>(bytes, VkDeviceSize{256} << 20u);
+}
+
 bool BufferPool::SlabEligible(std::size_t capacity, VkDeviceSize alignment, VkDeviceSize size, VkDeviceSize atom) {
     if (capacity >= classLimit || size > capacity || !std::has_single_bit(capacity)) return false;
     const auto alignmentOk = alignment == 0 || (std::has_single_bit(alignment) && alignment <= capacity);
@@ -180,18 +194,27 @@ bool BufferPool::DeviceTierEnabled() {
     return enabled;
 }
 
-VkDeviceSize BufferPool::DeviceBudget(const VkPhysicalDeviceMemoryProperties& memory) {
+VkDeviceSize BufferPool::DeviceBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported) {
     static const char* value = std::getenv("APS5_STAGING_POOL_MIB");
     if (value != nullptr) return static_cast<VkDeviceSize>(std::strtoull(value, nullptr, 10)) << 20u;
-    VkDeviceSize largest = 0;
-    bool systemMemory = false;
+    std::optional<std::uint32_t> largest;
     for (std::uint32_t heap = 0; heap < memory.memoryHeapCount && heap < VK_MAX_MEMORY_HEAPS; ++heap) {
-        if ((memory.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0 || memory.memoryHeaps[heap].size <= largest) continue;
-        largest = memory.memoryHeaps[heap].size;
-        systemMemory = systemMemoryHeap(memory, heap);
+        if ((memory.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0 && (!largest || memory.memoryHeaps[heap].size > memory.memoryHeaps[*largest].size)) largest = heap;
     }
-    if (systemMemory) return std::max<VkDeviceSize>(VkDeviceSize{128} << 20u, largest / 64u);
-    return std::max<VkDeviceSize>(VkDeviceSize{512} << 20u, largest / 8u);
+    VkDeviceSize size = largest ? memory.memoryHeaps[*largest].size : 0;
+    if (largest && reported != nullptr && reported->heapBudget[*largest] != 0) size = std::min(size, reported->heapBudget[*largest]);
+    if (largest && systemMemoryHeap(memory, *largest)) return std::max<VkDeviceSize>(VkDeviceSize{128} << 20u, size / 64u);
+    return std::max<VkDeviceSize>(VkDeviceSize{512} << 20u, size / 8u);
+}
+
+VkDeviceSize BufferPool::deviceBudget(const Context& context) {
+    if (!DeviceTierEnabled()) return 0;
+    if (context.memoryProperties2 == nullptr || context.physical == VK_NULL_HANDLE) return DeviceBudget(context.memory);
+    VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
+    VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2};
+    properties.pNext = &reported;
+    context.memoryProperties2(context.physical, &properties);
+    return DeviceBudget(context.memory, &reported);
 }
 
 VkDeviceSize BufferPool::destroy(const BufferAllocation& allocation) noexcept {
@@ -228,7 +251,7 @@ VkBufferUsageFlags BufferPool::Usage(VkBufferUsageFlags usage, VkMemoryPropertyF
 }
 
 BufferPool::Tier& BufferPool::tierFor(std::size_t capacity, VkMemoryPropertyFlags properties) {
-    if (DeviceTiered(properties)) return deviceTier;
+    if (DeviceTiered(properties) && (sharedTiers() || capacity >= classLimit)) return deviceTier;
     return !sharedTiers() && capacity < classLimit ? smallTier : largeTier;
 }
 

core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp

@@ -761,13 +761,62 @@ bool PendingFillsLeaveUncompressed(const Recorder& recorder, const GuestTextureR
 
 }
 
+namespace {
+
+struct KeyWaitTotals {
+    std::atomic<std::uint64_t> calls{0};
+    std::atomic<std::uint64_t> noPending{0};
+    std::atomic<std::uint64_t> known{0};
+    std::atomic<std::uint64_t> fillsUncompressed{0};
+    std::atomic<std::uint64_t> synced{0};
+};
+
+KeyWaitTotals& KeyWaits() {
+    static KeyWaitTotals totals;
+    return totals;
+}
+
+}
+
+DccKeyWaitCounts KeyWaitCounts() {
+    const auto& totals = KeyWaits();
+    return {totals.calls.load(std::memory_order_relaxed), totals.noPending.load(std::memory_order_relaxed), totals.known.load(std::memory_order_relaxed), totals.fillsUncompressed.load(std::memory_order_relaxed), totals.synced.load(std::memory_order_relaxed)};
+}
+
+bool DccKeyWritersPending(std::uint64_t metaAddress, std::size_t count) {
+    if (metaAddress == 0 || count == 0 || !Recorder::SnapshotWriteOverlaps(metaAddress, count)) return false;
+    auto& memo = Memo();
+    std::lock_guard lock(memo.mutex);
+    MemoRecorder(memo);
+    return std::none_of(memo.entries.begin(), memo.entries.end(), [&](const GpuKeyStore& store) { return store.note != 0 && store.begin <= metaAddress && metaAddress + count <= store.end; });
+}
+
+bool DccKeyWriterNeedsSync(std::uint64_t metaAddress, std::size_t count) {
+    if (metaAddress == 0 || count == 0 || !GuestMemory::GpuMutex().HeldByThisThread()) return false;
+    auto* recorder = Recorder::Active();
+    return recorder != nullptr && recorder->PendingWriteOverlaps(metaAddress, count) && !PendingStoreKeys(*recorder, metaAddress, count).has_value();
+}
+
 std::optional<DccKeys> WaitForKeyWriters(const GuestTextureResource& resource, std::uint64_t guestBytes) {
     const auto count = static_cast<std::size_t>(guestBytes / KeyBytes);
     const auto metaAddress = resource.dccAddress;
     if (metaAddress == 0 || count == 0 || !GuestMemory::GpuMutex().HeldByThisThread()) return std::nullopt;
+    auto& waits = KeyWaits();
+    waits.calls.fetch_add(1, std::memory_order_relaxed);
     auto* recorder = Recorder::Active();
-    if (recorder == nullptr || !recorder->PendingWriteOverlaps(metaAddress, count) || PendingStoreKeys(*recorder, metaAddress, count).has_value()) return std::nullopt;
-    if (PendingFillsLeaveUncompressed(*recorder, resource, count)) return DccKeys::Uncompressed;
+    if (recorder == nullptr || !recorder->PendingWriteOverlaps(metaAddress, count)) {
+        waits.noPending.fetch_add(1, std::memory_order_relaxed);
+        return std::nullopt;
+    }
+    if (PendingStoreKeys(*recorder, metaAddress, count).has_value()) {
+        waits.known.fetch_add(1, std::memory_order_relaxed);
+        return std::nullopt;
+    }
+    if (PendingFillsLeaveUncompressed(*recorder, resource, count)) {
+        waits.fillsUncompressed.fetch_add(1, std::memory_order_relaxed);
+        return DccKeys::Uncompressed;
+    }
+    waits.synced.fetch_add(1, std::memory_order_relaxed);
     Recorder::CountSync(2);
     recorder->SyncThrough(metaAddress, count);
     return std::nullopt;

core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp

@@ -257,7 +257,9 @@ public:
         const VkImageSubresourceRange range{aspects, 0, 1, 0, 1};
         constexpr VkAccessFlags access = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
         RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, access, VK_ACCESS_TRANSFER_WRITE_BIT);
-        const VkClearDepthStencilValue clear{clearDepth, clearStencil};
+        const VkClearDepthStencilValue clear{depthFill.value_or(clearDepth), stencilFill.value_or(clearStencil)};
+        depthFill.reset();
+        stencilFill.reset();
         context.Function<PFN_vkCmdClearDepthStencilImage>("vkCmdClearDepthStencilImage")(commands, image, VK_IMAGE_LAYOUT_GENERAL, &clear, 1, &range);
         RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, access);
         if (batch) batch->SubmitAndWait();
@@ -322,8 +324,25 @@ public:
         return texture;
     }
 
+    void ForgetClear(VkImageAspectFlags aspects) {
+        pendingClear &= ~aspects;
+        if ((aspects & VK_IMAGE_ASPECT_DEPTH_BIT) != 0) depthFill.reset();
+        if ((aspects & VK_IMAGE_ASPECT_STENCIL_BIT) != 0) stencilFill.reset();
+    }
+
+    void NoteHtileClear(VkImageAspectFlags aspects) {
+        ForgetClear(aspects);
+        pendingClear |= aspects;
+    }
+
+    void NotePlaneFill(VkImageAspectFlagBits aspect, float depth, std::uint8_t stencil) {
+        pendingClear |= aspect;
+        if (aspect == VK_IMAGE_ASPECT_DEPTH_BIT) depthFill = depth;
+        else stencilFill = stencil;
+    }
+
     void CopyAspectFrom(const DepthSurface& source, VkImageAspectFlagBits aspect) {
-        pendingClear &= ~static_cast<VkImageAspectFlags>(aspect);
+        ForgetClear(aspect);
         auto* recorder = RecorderFor(context);
         std::unique_ptr<CommandBatch> batch;
         if (recorder == nullptr) batch = std::make_unique<CommandBatch>(context);
@@ -370,6 +389,8 @@ public:
     VkImageAspectFlags pendingClear = 0;
     float clearDepth = 0.0f;
     std::uint8_t clearStencil = 0;
+    std::optional<float> depthFill;
+    std::optional<std::uint8_t> stencilFill;
 
     bool StorageViewMatches(std::span<const std::uint32_t> words, const GuestTextureResource& resource) const {
         if (resource.width != target.extent.width || resource.height != target.extent.height) return false;
@@ -951,8 +972,13 @@ void takePlane(DepthSurface& surface, PlaneOwners& owners, std::uint64_t address
     auto& entry = owners[{surface.context.device, address}];
     auto* owner = entry.surface;
     if (owner != nullptr && owner->active->load(std::memory_order_acquire) && owner != &surface && owner->target.format == surface.target.format && owner->target.samples == surface.target.samples && owner->target.extent.width == surface.target.extent.width && owner->target.extent.height == surface.target.extent.height) {
-        if ((owner->pendingClear & aspect) != 0) surface.pendingClear |= aspect;
-        else surface.CopyAspectFrom(*owner, aspect);
+        if ((owner->pendingClear & aspect) != 0) {
+            surface.pendingClear |= aspect;
+            if (aspect == VK_IMAGE_ASPECT_DEPTH_BIT) surface.depthFill = owner->depthFill;
+            else surface.stencilFill = owner->stencilFill;
+        } else {
+            surface.CopyAspectFrom(*owner, aspect);
+        }
     }
     entry = {&surface, ++planeBinds};
 }
@@ -1222,9 +1248,9 @@ VkImageView DepthSurfaceView(const Context& context, const DepthTarget& target)
     if (created && !inherits && target.htileAddress == 0) surface->TakeStorage();
     if (cleared && (!created || inherits)) {
         if (target.samples != 1u) surface->sampling.refusal = "native multisample depth fast-clear transition is not implemented";
-        surface->pendingClear |= VK_IMAGE_ASPECT_DEPTH_BIT;
+        surface->NoteHtileClear(VK_IMAGE_ASPECT_DEPTH_BIT);
     }
-    surface->pendingClear |= htileCleared;
+    surface->NoteHtileClear(htileCleared);
     surface->ApplyFastClear();
     if (!created || inherits) surface->written = ++depthWrites;
     surface->NoteMemory();
@@ -1316,7 +1342,7 @@ void NoteHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes, std::siz
         }
         if (!HtileFillCovers(surface->target.htileAddress, surface->target.extent, htileAddress, bytes)) continue;
         if (surface->target.samples != 1u) surface->sampling.refusal = "native multisample depth fast-clear transition is not implemented";
-        surface->pendingClear |= VK_IMAGE_ASPECT_DEPTH_BIT;
+        surface->NoteHtileClear(VK_IMAGE_ASPECT_DEPTH_BIT);
         existing = true;
     }
     if (!existing) clearedHtiles()[htileAddress] = bytes;
@@ -1444,6 +1470,26 @@ bool HtileFillCovers(std::uint64_t htile, VkExtent2D extent, std::uint64_t addre
     return address + bytes - htile >= HtileMetadataBytes(extent);
 }
 
+bool PlaneFillCovers(std::uint64_t plane, std::uint64_t planeBytes, std::uint64_t address, std::size_t bytes) {
+    return plane != 0 && plane >= address && plane + planeBytes <= address + bytes;
+}
+
+std::optional<std::uint8_t> StencilFillValue(std::uint32_t pattern) {
+    const auto byte = static_cast<std::uint8_t>(pattern & 0xffu);
+    if (pattern != byte * 0x01010101u) return std::nullopt;
+    return byte;
+}
+
+std::optional<float> DepthFillValue(std::uint32_t pattern, bool d16) {
+    if (d16) {
+        if ((pattern >> 16u) != (pattern & 0xffffu)) return std::nullopt;
+        return static_cast<float>(pattern & 0xffffu) / 65535.0f;
+    }
+    const auto value = std::bit_cast<float>(pattern);
+    if (!(value >= 0.0f && value <= 1.0f)) return std::nullopt;
+    return value;
+}
+
 void NoteDepthMetadataFill(std::uint64_t address, std::size_t bytes, std::uint32_t pattern) {
     std::lock_guard lock(surfacesMutex());
     forgetHtileReads(address, bytes);
@@ -1451,9 +1497,20 @@ void NoteDepthMetadataFill(std::uint64_t address, std::size_t bytes, std::uint32
         if (!surface->active->load(std::memory_order_acquire)) continue;
         const auto& target = surface->target;
         if (target.samples != 1u && target.htileAddress != 0u && address < target.htileAddress + HtileMetadataBytes(target.extent) && target.htileAddress < address + bytes) surface->sampling.refusal = "native multisample depth metadata fill transition is not implemented";
-        if (!HtileFillCovers(target.htileAddress, target.extent, address, bytes)) continue;
-        const VkImageAspectFlags written = VK_IMAGE_ASPECT_DEPTH_BIT | (target.htileStencil ? VK_IMAGE_ASPECT_STENCIL_BIT : 0u);
-        surface->pendingClear = (surface->pendingClear & ~written) | HtileFillClears(pattern, target.htileStencil);
+        if (HtileFillCovers(target.htileAddress, target.extent, address, bytes)) {
+            const VkImageAspectFlags written = VK_IMAGE_ASPECT_DEPTH_BIT | (target.htileStencil ? VK_IMAGE_ASPECT_STENCIL_BIT : 0u);
+            surface->ForgetClear(written);
+            surface->NoteHtileClear(HtileFillClears(pattern, target.htileStencil));
+            continue;
+        }
+        if (target.samples != 1) continue;
+        const bool d16 = target.format == VK_FORMAT_D16_UNORM || target.format == VK_FORMAT_D16_UNORM_S8_UINT;
+        if (PlaneFillCovers(target.address, DepthSliceBytes(target.extent, d16 ? 2u : 4u), address, bytes)) {
+            if (const auto depth = DepthFillValue(pattern, d16)) surface->NotePlaneFill(VK_IMAGE_ASPECT_DEPTH_BIT, *depth, 0);
+        }
+        if (PlaneFillCovers(target.stencilAddress, DepthSliceBytes(target.extent, 1u), address, bytes)) {
+            if (const auto stencil = StencilFillValue(pattern)) surface->NotePlaneFill(VK_IMAGE_ASPECT_STENCIL_BIT, 0.0f, *stencil);
+        }
     }
 }
 
@@ -1545,6 +1602,11 @@ bool DepthSurfaceAt(std::uint64_t address) {
     return std::any_of(surfaces().begin(), surfaces().end(), [&](const auto& surface) { return surface->active->load(std::memory_order_acquire) && !surface->retired && (surface->target.address == address || surface->target.stencilAddress == address); });
 }
 
+bool DepthSurfaceOverlaps(VkDevice device, std::uint64_t address, std::uint64_t bytes) {
+    std::lock_guard lock(surfacesMutex());
+    return std::any_of(surfaces().begin(), surfaces().end(), [&](const auto& surface) { return surface->active->load(std::memory_order_acquire) && !surface->retired && surface->context.device == device && surface->Overlaps(address, bytes); });
+}
+
 void RetireDepthSurfaces(VkDevice device, std::uint64_t address, std::uint64_t bytes) {
     std::lock_guard lock(surfacesMutex());
     for (const auto& surface : surfaces()) {

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -354,6 +354,9 @@ struct DrawOutcome {
     // A recorded draw began a render pass of its own, or continued the previous draw's.
     bool passBegun = false;
     bool passContinued = false;
+    bool keyStoresFlushed = false;
+    bool storesFlushed = false;
+    bool addressDraw = false;
     SyncReason reason = SyncNone;
     // Shader validation memo (see CachedFragmentOutputs).
     bool validateMemoized = false;
@@ -426,6 +429,7 @@ struct DrawProfile {
     // Render passes recorded draws began, and draws that continued the previous draw's pass.
     std::uint64_t passesBegun = 0;
     std::uint64_t passesContinued = 0;
+    std::array<DrawPassCounts, 2> passTotals{};
     std::array<std::uint64_t, SyncCount> reasons{};
     // The CPU inside Graphics::Draw, split by outcome: a synchronous draw costs ten times a recorded
     // one, so one average would only show the synchronous population.
@@ -496,6 +500,14 @@ void reportDraw(const std::array<double, PhaseCount>& us, const ShaderResources:
     if (outcome.completion) ++profile.completion;
     if (outcome.passBegun) ++profile.passesBegun;
     if (outcome.passContinued) ++profile.passesContinued;
+    if (outcome.recorded) {
+        auto& totals = profile.passTotals[outcome.addressDraw ? 1 : 0];
+        ++totals.draws;
+        if (outcome.passBegun) ++totals.passesBegun;
+        if (outcome.passContinued) ++totals.passesContinued;
+        if (outcome.keyStoresFlushed) ++totals.keyStoreFlushes;
+        if (outcome.storesFlushed) ++totals.storeFlushes;
+    }
     ++profile.reasons[outcome.reason];
     (outcome.recorded ? (outcome.waited ? profile.waitedUs : profile.recordedUs) : profile.synchronousUs) += drawUs;
     profile.targetLookups += outcome.targetLookups;
@@ -841,6 +853,12 @@ std::array<std::uint64_t, IndirectPathCount> IndirectDrawCounts() {
     return stats.indirect;
 }
 
+std::array<DrawPassCounts, 2> DrawPassTotals() {
+    auto& stats = Profile();
+    std::lock_guard lock(stats.mutex);
+    return stats.passTotals;
+}
+
 void CountDrawSkip(DrawSkip kind, double us) {
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
     if (!profile) return;
@@ -1049,6 +1067,7 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P
     if (draw.indexed) GuestMemory::CheckRange(reinterpret_cast<const void*>(draw.indexAddress), static_cast<std::size_t>(indexBytes), draw.indexSize);
     APS5_LOG_CHARS_OUT_DEBUG("Index buffer range OK");
     Require(!draw.indexed || !state.hasColorTarget || draw.indexAddress + indexBytes <= state.color.address || state.color.address + state.color.bytes <= draw.indexAddress, "index buffer aliases the render target");
+    if (draw.indexed) RequireNoMultisampleTarget(draw.indexAddress, static_cast<std::size_t>(indexBytes), "an index buffer read");
     APS5_LOG_CHARS_OUT_DEBUG("ValidateShaders");
     if (recipe != nullptr) {
         inputs.fragmentOutputs = recipe->fragmentOutputs;
@@ -1198,6 +1217,7 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P
         const auto [begin, end] = plan.copies[range];
         const auto bytes = static_cast<std::size_t>(end - begin);
         GuestMemory::CheckRange(reinterpret_cast<const void*>(begin), bytes, 1);
+        RequireNoMultisampleTarget(begin, bytes, "a vertex buffer read");
         if (const auto* import = InPlaceDrawInput(context, begin, bytes, plan.alignments[range], &inputs.vertexInputBehindGpuWrite)) {
             rangeHandles.push_back(import->buffer);
             rangeOffsets.push_back(begin - import->base);
@@ -1755,9 +1775,15 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
         if (!gpuIndirect) return false;
         return (begin < args->arguments + args->RangeBytes() && args->arguments < end) || (args->countIndirect && begin < args->countAddress + 4 && args->countAddress < end);
     };
-    if (recorder->HasQueuedKeyStores() && (resources.HoldsLease() || recorder->AnyQueuedKeyStore(touches))) recorder->FlushKeyStores();
+    if (recorder->HasQueuedKeyStores() && (resources.HoldsLease() || recorder->AnyQueuedKeyStore(touches))) {
+        recorder->FlushKeyStores();
+        outcome.keyStoresFlushed = true;
+    }
     // A queued label store over such memory likewise (Recorder::RecordStore).
-    if (recorder->HasQueuedStores() && (resources.HoldsLease() || recorder->AnyQueuedStore(touches))) recorder->FlushStores();
+    if (recorder->HasQueuedStores() && (resources.HoldsLease() || recorder->AnyQueuedStore(touches))) {
+        recorder->FlushStores();
+        outcome.storesFlushed = true;
+    }
     const auto drawBindings = resources.PrepareDrawBindings(*recorder, record.moved);
     const bool capture = CaptureInputsEnabled();
     const bool meshIndirect = state.stages.mesh && args != nullptr;
@@ -1774,7 +1800,11 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
         if (target != nullptr) passAttachments.emplace_back(target->Image(), true);
     }
     const bool unknownMemory = addressBased || resources.UsesAddressTable() || resources.HasCopiedWrites();
-    const PassAccess passAccess{passReads, resources.GpuWrites(), passImages, passAttachments, unknownMemory, unknownMemory};
+    static const bool addressReadPasses = std::getenv("APS5_NO_ADDRESS_READ_PASSES") == nullptr;
+    const bool readOnlyBda = addressReadPasses && resources.BdaReadOnly();
+    outcome.addressDraw = addressBased;
+    const bool unknownWrites = ((addressBased || resources.UsesAddressTable()) && !readOnlyBda) || resources.HasCopiedWrites();
+    const PassAccess passAccess{passReads, resources.GpuWrites(), passImages, passAttachments, unknownMemory, unknownWrites};
     const bool fenced = state.depth.has_value() || !record.proxies.empty();
     const auto start = recorder->StartDrawPass(passKey, capture || readsTarget || gpuIndirect || meshIndirect, fenced, passAccess);
     const bool continued = start.continued;

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -160,6 +160,16 @@ struct ImportFailure {
     GuestAllocations::Mapped ranges;
 };
 
+struct ImportSettle {
+    VkBuffer settleTarget = VK_NULL_HANDLE;
+    VkDeviceMemory settleMemory = VK_NULL_HANDLE;
+    VkCommandBuffer settleCommands = VK_NULL_HANDLE;
+    VkFence settleFence = VK_NULL_HANDLE;
+    PFN_vkDestroyFence destroyFence = nullptr;
+    PFN_vkDestroyBuffer settleDestroyBuffer = nullptr;
+    PFN_vkFreeMemory settleFreeMemory = nullptr;
+};
+
 // Imports persist across draws, keyed by allocation base, and are dropped when their allocation leaves
 // the registered set; a dropped import is destroyed once the recorded work that may read it completed.
 struct HostImports {
@@ -184,14 +194,7 @@ struct HostImports {
     bool unwatchImports = false;
     bool unwatchDmaBufImports = false;
     bool watchPrivateImports = false;
-    VkDevice settleDevice = VK_NULL_HANDLE;
-    VkBuffer settleTarget = VK_NULL_HANDLE;
-    VkDeviceMemory settleMemory = VK_NULL_HANDLE;
-    VkCommandBuffer settleCommands = VK_NULL_HANDLE;
-    VkFence settleFence = VK_NULL_HANDLE;
-    PFN_vkDestroyFence destroyFence = nullptr;
-    PFN_vkDestroyBuffer settleDestroyBuffer = nullptr;
-    PFN_vkFreeMemory settleFreeMemory = nullptr;
+    std::map<VkDevice, ImportSettle> settles;
     std::uint64_t useCounter = 0;
     std::uint64_t buildFloor = 0;
 };
@@ -678,51 +681,45 @@ std::atomic<std::uint64_t> settledImports{0};
 
 const char* settleImport(const Context& context, HostImports& state, const HostImport& entry, VkResult& result) {
     std::unique_lock queueLock(*context.settleQueueMutex);
-    if (state.settleDevice != context.device) {
-        state.settleDevice = context.device;
-        state.settleTarget = VK_NULL_HANDLE;
-        state.settleMemory = VK_NULL_HANDLE;
-        state.settleCommands = VK_NULL_HANDLE;
-        state.settleFence = VK_NULL_HANDLE;
-        state.destroyFence = context.Function<PFN_vkDestroyFence>("vkDestroyFence");
-        state.settleDestroyBuffer = context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer");
-        state.settleFreeMemory = context.Function<PFN_vkFreeMemory>("vkFreeMemory");
-    }
-    if (state.settleTarget == VK_NULL_HANDLE) {
+    auto& settle = state.settles[context.device];
+    settle.destroyFence = context.Function<PFN_vkDestroyFence>("vkDestroyFence");
+    settle.settleDestroyBuffer = context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer");
+    settle.settleFreeMemory = context.Function<PFN_vkFreeMemory>("vkFreeMemory");
+    if (settle.settleTarget == VK_NULL_HANDLE) {
         VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
         info.size = 256;
         info.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
-        if ((result = context.Function<PFN_vkCreateBuffer>("vkCreateBuffer")(context.device, &info, nullptr, &state.settleTarget)) != VK_SUCCESS) {
-            state.settleTarget = VK_NULL_HANDLE;
+        if ((result = context.Function<PFN_vkCreateBuffer>("vkCreateBuffer")(context.device, &info, nullptr, &settle.settleTarget)) != VK_SUCCESS) {
+            settle.settleTarget = VK_NULL_HANDLE;
             return "vkCreateBuffer (settle)";
         }
         VkMemoryRequirements requirements{};
-        context.Function<PFN_vkGetBufferMemoryRequirements>("vkGetBufferMemoryRequirements")(context.device, state.settleTarget, &requirements);
+        context.Function<PFN_vkGetBufferMemoryRequirements>("vkGetBufferMemoryRequirements")(context.device, settle.settleTarget, &requirements);
         VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
         allocation.allocationSize = requirements.size;
         allocation.memoryTypeIndex = static_cast<std::uint32_t>(std::countr_zero(requirements.memoryTypeBits));
-        if ((result = context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &state.settleMemory)) != VK_SUCCESS || (result = context.Function<PFN_vkBindBufferMemory>("vkBindBufferMemory")(context.device, state.settleTarget, state.settleMemory, 0)) != VK_SUCCESS) {
-            context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(context.device, state.settleTarget, nullptr);
-            if (state.settleMemory != VK_NULL_HANDLE) context.Function<PFN_vkFreeMemory>("vkFreeMemory")(context.device, state.settleMemory, nullptr);
-            state.settleTarget = VK_NULL_HANDLE;
-            state.settleMemory = VK_NULL_HANDLE;
+        if ((result = context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &settle.settleMemory)) != VK_SUCCESS || (result = context.Function<PFN_vkBindBufferMemory>("vkBindBufferMemory")(context.device, settle.settleTarget, settle.settleMemory, 0)) != VK_SUCCESS) {
+            context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(context.device, settle.settleTarget, nullptr);
+            if (settle.settleMemory != VK_NULL_HANDLE) context.Function<PFN_vkFreeMemory>("vkFreeMemory")(context.device, settle.settleMemory, nullptr);
+            settle.settleTarget = VK_NULL_HANDLE;
+            settle.settleMemory = VK_NULL_HANDLE;
             return "the settle target";
         }
         VkCommandBufferAllocateInfo allocate{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
         allocate.commandPool = context.settlePool;
         allocate.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
         allocate.commandBufferCount = 1;
-        if ((result = context.Function<PFN_vkAllocateCommandBuffers>("vkAllocateCommandBuffers")(context.device, &allocate, &state.settleCommands)) != VK_SUCCESS) {
-            state.settleCommands = VK_NULL_HANDLE;
+        if ((result = context.Function<PFN_vkAllocateCommandBuffers>("vkAllocateCommandBuffers")(context.device, &allocate, &settle.settleCommands)) != VK_SUCCESS) {
+            settle.settleCommands = VK_NULL_HANDLE;
             return "vkAllocateCommandBuffers (settle)";
         }
         const VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
-        if ((result = context.Function<PFN_vkCreateFence>("vkCreateFence")(context.device, &fenceInfo, nullptr, &state.settleFence)) != VK_SUCCESS) {
-            state.settleFence = VK_NULL_HANDLE;
+        if ((result = context.Function<PFN_vkCreateFence>("vkCreateFence")(context.device, &fenceInfo, nullptr, &settle.settleFence)) != VK_SUCCESS) {
+            settle.settleFence = VK_NULL_HANDLE;
             return "vkCreateFence (settle)";
         }
     }
-    if (state.settleCommands == VK_NULL_HANDLE || state.settleFence == VK_NULL_HANDLE) return "the settle commands";
+    if (settle.settleCommands == VK_NULL_HANDLE || settle.settleFence == VK_NULL_HANDLE) return "the settle commands";
     VkBuffer source = entry.buffer;
     VkBuffer alias = VK_NULL_HANDLE;
     const auto run = [&]() -> const char* {
@@ -738,19 +735,19 @@ const char* settleImport(const Context& context, HostImports& state, const HostI
             source = alias;
         }
 #endif
-        if ((result = context.Function<PFN_vkResetFences>("vkResetFences")(context.device, 1, &state.settleFence)) != VK_SUCCESS) return "vkResetFences (settle)";
-        if ((result = context.Function<PFN_vkResetCommandBuffer>("vkResetCommandBuffer")(state.settleCommands, 0)) != VK_SUCCESS) return "vkResetCommandBuffer (settle)";
+        if ((result = context.Function<PFN_vkResetFences>("vkResetFences")(context.device, 1, &settle.settleFence)) != VK_SUCCESS) return "vkResetFences (settle)";
+        if ((result = context.Function<PFN_vkResetCommandBuffer>("vkResetCommandBuffer")(settle.settleCommands, 0)) != VK_SUCCESS) return "vkResetCommandBuffer (settle)";
         VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
         begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
-        if ((result = context.Function<PFN_vkBeginCommandBuffer>("vkBeginCommandBuffer")(state.settleCommands, &begin)) != VK_SUCCESS) return "vkBeginCommandBuffer (settle)";
+        if ((result = context.Function<PFN_vkBeginCommandBuffer>("vkBeginCommandBuffer")(settle.settleCommands, &begin)) != VK_SUCCESS) return "vkBeginCommandBuffer (settle)";
         const VkBufferCopy region{0, 0, 4};
-        context.Function<PFN_vkCmdCopyBuffer>("vkCmdCopyBuffer")(state.settleCommands, source, state.settleTarget, 1, &region);
-        if ((result = context.Function<PFN_vkEndCommandBuffer>("vkEndCommandBuffer")(state.settleCommands)) != VK_SUCCESS) return "vkEndCommandBuffer (settle)";
+        context.Function<PFN_vkCmdCopyBuffer>("vkCmdCopyBuffer")(settle.settleCommands, source, settle.settleTarget, 1, &region);
+        if ((result = context.Function<PFN_vkEndCommandBuffer>("vkEndCommandBuffer")(settle.settleCommands)) != VK_SUCCESS) return "vkEndCommandBuffer (settle)";
         VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO};
         submit.commandBufferCount = 1;
-        submit.pCommandBuffers = &state.settleCommands;
-        if ((result = context.Function<PFN_vkQueueSubmit>("vkQueueSubmit")(context.settleQueue, 1, &submit, state.settleFence)) != VK_SUCCESS) return "vkQueueSubmit (settle)";
-        if ((result = context.Function<PFN_vkWaitForFences>("vkWaitForFences")(context.device, 1, &state.settleFence, VK_TRUE, 10'000'000'000ull)) != VK_SUCCESS) return "vkWaitForFences (settle)";
+        submit.pCommandBuffers = &settle.settleCommands;
+        if ((result = context.Function<PFN_vkQueueSubmit>("vkQueueSubmit")(context.settleQueue, 1, &submit, settle.settleFence)) != VK_SUCCESS) return "vkQueueSubmit (settle)";
+        if ((result = context.Function<PFN_vkWaitForFences>("vkWaitForFences")(context.device, 1, &settle.settleFence, VK_TRUE, 10'000'000'000ull)) != VK_SUCCESS) return "vkWaitForFences (settle)";
         return nullptr;
     };
     const char* step = run();
@@ -1025,6 +1022,7 @@ ImportRun windowRun(const HostImports& state, ImportRun run, std::uint64_t begin
     return run;
 }
 
+const HostImport* peekImport(const HostImports& state, std::uint64_t begin, std::uint64_t end);
 const HostImport* findImport(HostImports& state, std::uint64_t begin, std::uint64_t end);
 bool importsStale(const Context& context, const HostImports& state);
 const HostImport* importRunAt(const Context& context, HostImports& state, ImportRun run, bool joined, std::uint64_t address, const GuestAllocations::Lease& lease, std::uint64_t end, bool evict);
@@ -1057,8 +1055,8 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
 }
 
 const HostImport* importJoined(const Context& context, HostImports& state, std::uint64_t begin, std::uint64_t end, const GuestAllocations::Lease& lease) {
-    if (const auto* low = findImport(state, begin, begin + 1); low != nullptr && low->joined) begin = low->base;
-    if (const auto* high = findImport(state, end - 1, end); high != nullptr && high->joined) end = high->base + high->bytes;
+    if (const auto* low = peekImport(state, begin, begin + 1); low != nullptr && low->joined) begin = low->base;
+    if (const auto* high = peekImport(state, end - 1, end); high != nullptr && high->joined) end = high->base + high->bytes;
     auto run = joinedRun(lease, begin, end);
     if (run.ranges.empty()) return nullptr;
     if (const auto failed = state.failedJoins.find(run.base); failed != state.failedJoins.end() && sameRun(failed->second, run)) return nullptr;
@@ -1158,6 +1156,7 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
             state.unaliasedSpans.insert(base);
             return importAllocation(context, state, address, lease, end, evict);
         }
+        if (entry.alias == nullptr) return nullptr;
     }
 #else
     if (!GuestMemory::Accessible(reinterpret_cast<const void*>(base), static_cast<std::size_t>(bytes), true)) {
@@ -1308,14 +1307,18 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
     for (auto it = state.failedJoins.begin(); it != state.failedJoins.end();) it = sameRun(it->second, joinedRun(lease, it->second.base, it->second.base + it->second.bytes)) ? std::next(it) : state.failedJoins.erase(it);
 }
 
-const HostImport* findImport(HostImports& state, std::uint64_t begin, std::uint64_t end) {
+const HostImport* peekImport(const HostImports& state, std::uint64_t begin, std::uint64_t end) {
     auto found = state.imports.upper_bound(begin);
     if (found == state.imports.begin()) return nullptr;
     --found;
     const auto& entry = found->second;
-    if (begin < entry.base || end > entry.base + entry.bytes) return nullptr;
-    entry.lastUse = ++state.useCounter;
-    return &entry;
+    return begin < entry.base || end > entry.base + entry.bytes ? nullptr : &entry;
+}
+
+const HostImport* findImport(HostImports& state, std::uint64_t begin, std::uint64_t end) {
+    const auto* entry = peekImport(state, begin, end);
+    if (entry != nullptr) entry->lastUse = ++state.useCounter;
+    return entry;
 }
 
 const HostImport* importAccess(const Context& context, HostImports& state, std::uint64_t begin, std::uint64_t end, const GuestAllocations::Lease& lease) {
@@ -2331,15 +2334,12 @@ void ClearHostImports(VkDevice device) {
     auto& state = Imports();
     std::lock_guard lock(state.mutex);
     state.parked.erase(device);
-    if (state.settleDevice == device) {
-        if (state.settleFence != VK_NULL_HANDLE) state.destroyFence(device, state.settleFence, nullptr);
-        if (state.settleTarget != VK_NULL_HANDLE) state.settleDestroyBuffer(device, state.settleTarget, nullptr);
-        if (state.settleMemory != VK_NULL_HANDLE) state.settleFreeMemory(device, state.settleMemory, nullptr);
-        state.settleDevice = VK_NULL_HANDLE;
-        state.settleTarget = VK_NULL_HANDLE;
-        state.settleMemory = VK_NULL_HANDLE;
-        state.settleCommands = VK_NULL_HANDLE;
-        state.settleFence = VK_NULL_HANDLE;
+    if (const auto found = state.settles.find(device); found != state.settles.end()) {
+        const auto& settle = found->second;
+        if (settle.settleFence != VK_NULL_HANDLE) settle.destroyFence(device, settle.settleFence, nullptr);
+        if (settle.settleTarget != VK_NULL_HANDLE) settle.settleDestroyBuffer(device, settle.settleTarget, nullptr);
+        if (settle.settleMemory != VK_NULL_HANDLE) settle.settleFreeMemory(device, settle.settleMemory, nullptr);
+        state.settles.erase(found);
     }
     if (state.device != device) return;
     for (const auto& [address, entry] : state.imports) {
@@ -2680,7 +2680,7 @@ bool HostImportCovers(const Context& context, std::uint64_t address, std::size_t
     std::lock_guard lock(state.mutex);
     // Imports of a replaced device are not this device's; a stale registry only means an import
     // may be dropped at the next reconcile, which the binding path handles.
-    return state.device == context.device && findImport(state, address, address + bytes) != nullptr;
+    return state.device == context.device && peekImport(state, address, address + bytes) != nullptr;
 }
 
 GuestBufferMemory::GuestBufferMemory(const Context& context) : context(context) {}
@@ -3743,7 +3743,7 @@ void GuestBufferMemory::UploadFinish(bool addressable) {
                 const auto& lease = importRanges();
                 const auto index = leasedIndexContaining(lease, region.begin);
                 if (entry == nullptr && index != lease.size() && lease[index]->address == region.begin) {
-                    const auto* low = findImport(state, region.begin, region.begin + 1);
+                    const auto* low = peekImport(state, region.begin, region.begin + 1);
                     const auto joinBase = low != nullptr && low->joined ? low->base : region.begin;
                     const auto joins = state.sparseRunJoins.find(joinBase);
                     const auto made = joins == state.sparseRunJoins.end() ? 0u : joins->second;
@@ -3851,7 +3851,7 @@ void GuestBufferMemory::copyRegion(Region& region, bool addressable) {
     if (profile && context.hostImportAlignment != 0) {
         auto& state = Imports();
         std::lock_guard lock(state.mutex);
-        inImport = findImport(state, region.begin, region.end) != nullptr;
+        inImport = peekImport(state, region.begin, region.end) != nullptr;
     }
     const auto allocateStart = std::chrono::steady_clock::now();
     if (region.sparse && region.buffer == nullptr && (region.hostBacked || region.writable)) {

core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp

@@ -368,7 +368,7 @@ private:
         for (const auto& range : dccRanges) {
             Require(GuestMemory::Watched(range.address, range.bytes), "multisampled DCC target reuse requires watched guest memory");
             GuestMemory::CollectWritesUncached(range.address, range.bytes);
-            Require(GuestMemory::UnchangedSince(range.address, range.bytes, range.generation), "writes to a resident multisampled DCC target or its metadata are not implemented");
+            Require(GuestMemory::UnchangedSince(range.address, range.bytes, range.generation) || !GuestMemory::StoredOver(range.address, range.bytes, range.generation), "writes to a resident multisampled DCC target or its metadata are not implemented");
             if (auto* recorder = Recorder::Active(); recorder != nullptr) Require(!recorder->PendingWriteOverlaps(range.address, range.bytes), "pending writes to a resident multisampled DCC target or its metadata are not implemented");
         }
     }
@@ -707,6 +707,25 @@ VkImageView MultisampleTargetView(const Context& context, const ColorTarget& tar
     return acquire(context, target).view;
 }
 
+namespace {
+
+bool redBlueSwapped(VkFormat rendered, VkFormat sampled) {
+    const auto pair = [&](VkFormat a, VkFormat b) { return (rendered == a && sampled == b) || (rendered == b && sampled == a); };
+    return pair(VK_FORMAT_B8G8R8A8_UNORM, VK_FORMAT_R8G8B8A8_UNORM) || pair(VK_FORMAT_B8G8R8A8_SRGB, VK_FORMAT_R8G8B8A8_SRGB) || pair(VK_FORMAT_B8G8R8A8_SNORM, VK_FORMAT_R8G8B8A8_SNORM) || pair(VK_FORMAT_B8G8R8A8_UINT, VK_FORMAT_R8G8B8A8_UINT) || pair(VK_FORMAT_B8G8R8A8_SINT, VK_FORMAT_R8G8B8A8_SINT);
+}
+
+VkComponentMapping swapRedBlue(VkComponentMapping components) {
+    const auto swapped = [](VkComponentSwizzle swizzle, VkComponentSwizzle identity) {
+        if (swizzle == VK_COMPONENT_SWIZZLE_IDENTITY) swizzle = identity;
+        if (swizzle == VK_COMPONENT_SWIZZLE_R) return VK_COMPONENT_SWIZZLE_B;
+        if (swizzle == VK_COMPONENT_SWIZZLE_B) return VK_COMPONENT_SWIZZLE_R;
+        return swizzle;
+    };
+    return {swapped(components.r, VK_COMPONENT_SWIZZLE_R), swapped(components.g, VK_COMPONENT_SWIZZLE_G), swapped(components.b, VK_COMPONENT_SWIZZLE_B), swapped(components.a, VK_COMPONENT_SWIZZLE_A)};
+}
+
+}
+
 std::shared_ptr<Texture> MultisampleTargetTexture(const Context& context, const GuestTextureResource& resource, VkComponentMapping components) {
     std::lock_guard lock(targetsMutex());
     const auto format = ResolveTextureFormat(resource.format);
@@ -715,7 +734,9 @@ std::shared_ptr<Texture> MultisampleTargetTexture(const Context& context, const
         if (!target->active->load(std::memory_order_acquire)) continue;
         if (target->context.device != context.device || target->target.surfaceAddress != resource.baseAddress) continue;
         const auto& rendered = target->target;
-        if (rendered.format == format && rendered.extent.width == resource.width && rendered.extent.height == resource.height && rendered.samples == resource.samples && rendered.mip == 0) return target->Sampled(components);
+        const bool sameShape = rendered.extent.width == resource.width && rendered.extent.height == resource.height && rendered.samples == resource.samples && rendered.mip == 0;
+        if (sameShape && rendered.format == format) return target->Sampled(components);
+        if (sameShape && redBlueSwapped(rendered.format, format)) return target->Sampled(swapRedBlue(components));
         other = target.get();
     }
     if (other == nullptr) {

core/libs/prx/libSceAgcDriver/Graphics/src/ParallelCompare.cpp

@@ -5,14 +5,21 @@
 #include <cstring>
 #include <mutex>
 #include <thread>
+#include <vector>
 
 namespace AgcDriver::Graphics {
 namespace {
 
+std::atomic<unsigned> runningHelpers{0};
+
 class ComparePool {
 public:
     explicit ComparePool(unsigned count) {
-        for (unsigned i = 0; i < count; ++i) std::thread([this] { work(); }).detach();
+        stopping = count == 0;
+        for (unsigned i = 0; i < count; ++i) {
+            workers.emplace_back([this] { work(); });
+            runningHelpers.fetch_add(1, std::memory_order_acq_rel);
+        }
     }
 
     bool Run(std::span<const CompareSpan> spans, std::span<std::uint8_t> equal) {
@@ -20,6 +27,7 @@ public:
         if (!caller.owns_lock()) return false;
         {
             std::unique_lock lock(mutex);
+            if (stopping) return false;
             finished.wait(lock, [&] { return working == 0; });
             jobSpans = spans;
             jobEqual = equal;
@@ -36,6 +44,21 @@ public:
         return true;
     }
 
+    void Stop() {
+        std::vector<std::thread> joined;
+        {
+            std::lock_guard caller(callers);
+            std::lock_guard lock(mutex);
+            stopping = true;
+            joined.swap(workers);
+        }
+        wake.notify_all();
+        for (auto& worker : joined) {
+            if (worker.get_id() == std::this_thread::get_id()) worker.detach();
+            else worker.join();
+        }
+    }
+
 private:
     void drain(std::span<const CompareSpan> spans, std::span<std::uint8_t> equal) {
         std::size_t compared = 0;
@@ -51,7 +74,8 @@ private:
         std::uint64_t seen = 0;
         std::unique_lock lock(mutex);
         for (;;) {
-            wake.wait(lock, [&] { return generation != seen; });
+            wake.wait(lock, [&] { return generation != seen || stopping; });
+            if (stopping) break;
             seen = generation;
             const auto spans = jobSpans;
             const auto equal = jobEqual;
@@ -62,6 +86,7 @@ private:
             --working;
             if (working == 0) finished.notify_all();
         }
+        runningHelpers.fetch_sub(1, std::memory_order_acq_rel);
     }
 
     std::mutex callers;
@@ -74,10 +99,32 @@ private:
     std::span<std::uint8_t> jobEqual;
     std::atomic<std::size_t> next{0};
     std::atomic<std::size_t> done{0};
+    std::vector<std::thread> workers;
+    bool stopping = false;
 };
 
+std::mutex& poolLifecycle() {
+    static std::mutex lifecycle;
+    return lifecycle;
+}
+
+bool& poolStopped() {
+    static bool stopped = false;
+    return stopped;
+}
+
+std::atomic<ComparePool*>& createdPool() {
+    static std::atomic<ComparePool*> created{nullptr};
+    return created;
+}
+
 ComparePool* sharedPool() {
-    static ComparePool* const pool = new ComparePool(CompareHelpers());
+    static ComparePool* const pool = [] {
+        std::lock_guard lifecycle(poolLifecycle());
+        auto* created = new ComparePool(poolStopped() ? 0u : CompareHelpers());
+        createdPool().store(created, std::memory_order_release);
+        return created;
+    }();
     return pool;
 }
 
@@ -97,6 +144,20 @@ void CompareSpansFrom(std::span<const CompareSpan> spans, std::span<std::uint8_t
     for (std::size_t index = 0; index < spans.size(); ++index) equal[index] = std::memcmp(spans[index].first, spans[index].second, spans[index].bytes) == 0 ? 1 : 0;
 }
 
+void StopCompareHelpers() {
+    ComparePool* pool = nullptr;
+    {
+        std::lock_guard lifecycle(poolLifecycle());
+        poolStopped() = true;
+        pool = createdPool().load(std::memory_order_acquire);
+    }
+    if (pool != nullptr) pool->Stop();
+}
+
+unsigned CompareHelpersRunning() {
+    return runningHelpers.load(std::memory_order_acquire);
+}
+
 void CompareSpans(std::span<const CompareSpan> spans, std::span<std::uint8_t> equal) {
     CompareSpansFrom(spans, equal, std::size_t{1} << 20u);
 }

core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp

@@ -98,6 +98,10 @@ void ValidateDepthBounds(const Context& context, const State& state) {
     Require(!state.depthBoundsTest || context.depthRangeUnrestricted || (state.minDepthBounds >= 0.0f && state.minDepthBounds <= 1.0f && state.maxDepthBounds >= 0.0f && state.maxDepthBounds <= 1.0f), "depth bounds outside [0, 1] require VK_EXT_depth_range_unrestricted");
 }
 
+bool BarycentricPatches(const State& state, std::span<const CompiledShader> shaders) {
+    return !state.rectList && state.stages.path == ShaderPath::Vertex && std::any_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.stage == ShaderRecompiler::ShaderStage::TessellationControl; });
+}
+
 void ValidateProvokingVertex(const Context& context, const State& state, std::span<const CompiledShader> shaders) {
     Require(state.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT || state.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT, "invalid provoking vertex mode");
     if (state.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT) {
@@ -108,8 +112,8 @@ void ValidateProvokingVertex(const Context& context, const State& state, std::sp
         return;
     }
     Require(context.provokingVertexLast, "last provoking vertex requires VK_EXT_provoking_vertex with provokingVertexLast enabled");
-    Require(!state.rectList && state.stages.path == ShaderPath::Vertex && !state.stages.mesh && !state.stages.tessellation, "last provoking vertex is unsupported for generated primitive pipelines");
-    Require(state.topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_LINE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_LINE_STRIP || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN, "last provoking vertex is unsupported for this primitive topology");
+    Require(!BarycentricPatches(state, shaders) && state.stages.path == ShaderPath::Vertex && !state.stages.mesh && !state.stages.tessellation, "last provoking vertex is unsupported for generated primitive pipelines");
+    Require(state.rectList || state.topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_LINE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_LINE_STRIP || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN, "last provoking vertex is unsupported for this primitive topology");
     for (const auto& shader : shaders) {
         if (shader.stage != ShaderRecompiler::ShaderStage::Fragment) continue;
         Require(shader.program != nullptr, "missing compiled fragment shader");
@@ -194,7 +198,7 @@ void appendKey(std::vector<std::byte>& key, const TValue& value) {
 
 bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& state, const VertexInputLayout& input, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders) {
     using Stage = ShaderRecompiler::ShaderStage;
-    if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || state.depthBiasPerFace || state.rectList || SplitsAlternateAlpha(state)) return false;
+    if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || state.depthBiasPerFace || state.rectList || BarycentricPatches(state, shaders) || SplitsAlternateAlpha(state)) return false;
     keys = {};
     for (const auto& shader : shaders) {
         const auto identity = shader.program->Identity();
@@ -321,6 +325,8 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
     Require(state.conservativeRasterization == VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT || context.conservativeRasterization, "conservative rasterization requires VK_EXT_conservative_rasterization with at most 1/256 pixel of overestimation and degenerate triangles rasterized");
     ValidateProvokingVertex(context, state, shaders);
     if (state.rectList) Require(context.tessellationShader && context.limits.maxTessellationPatchSize >= 4, "rect-list requires tessellation with four output control points");
+    const bool barycentricPatches = BarycentricPatches(state, shaders);
+    if (barycentricPatches) Require(context.tessellationShader && state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST && !state.primitiveRestart, "emulated barycentrics through tessellation require a triangle list without primitive restart");
     if (std::any_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.stage == ShaderRecompiler::ShaderStage::Geometry; })) Require(context.geometryShader, "device does not support geometry shaders");
     if (state.stages.tessellation) {
         Require(context.tessellationShader, "device does not support tessellation shaders");
@@ -436,7 +442,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
         input.vertexAttributeDescriptionCount = static_cast<std::uint32_t>(vertexInput.attributes.size());
         input.pVertexAttributeDescriptions = vertexInput.attributes.data();
         VkPipelineInputAssemblyStateCreateInfo assembly{VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO};
-        assembly.topology = state.topology;
+        assembly.topology = BarycentricPatches(state, shaders) ? VK_PRIMITIVE_TOPOLOGY_PATCH_LIST : state.topology;
         const bool listTopology = state.topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_LINE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
         assembly.primitiveRestartEnable = state.primitiveRestart && (!listTopology || context.primitiveListRestart) ? VK_TRUE : VK_FALSE;
         // Viewport and scissor are set per draw (Begin), so they do not multiply pipelines; the depth
@@ -523,8 +529,8 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
         pipelineInfo.stageCount = static_cast<std::uint32_t>(stages.size());
         pipelineInfo.pStages = stages.data();
         VkPipelineTessellationStateCreateInfo tessellation{VK_STRUCTURE_TYPE_PIPELINE_TESSELLATION_STATE_CREATE_INFO};
-        if (state.rectList || state.stages.tessellation) {
-            tessellation.patchControlPoints = state.rectList ? 3u : state.stages.tessellation->inputControlPoints;
+        if (state.rectList || state.stages.tessellation || barycentricPatches) {
+            tessellation.patchControlPoints = state.stages.tessellation ? state.stages.tessellation->inputControlPoints : 3u;
             pipelineInfo.pTessellationState = &tessellation;
         }
         pipelineInfo.pVertexInputState = state.stages.mesh ? nullptr : &input;
@@ -926,6 +932,7 @@ struct PipelineStore {
         // The device's buffer pool at insertion: it is made and reset with the device, so it tells
         // the device instance apart from a later one the loader gave the same handle value.
         std::weak_ptr<BufferPool> pool;
+        bool tracksPool;
         std::uint64_t hash;
         std::vector<std::byte> key;
         std::shared_ptr<Pipeline> pipeline;
@@ -951,8 +958,12 @@ PipelineStore& Pipelines() {
 
 // Whether the entry's objects belong to the device the context names. A context without a buffer
 // pool (tests) is identified by the handle alone.
-bool alive(const PipelineStore::Entry& entry, const Context& context) {
+bool alive(PipelineStore::Entry& entry, const Context& context) {
     if (entry.device != context.device) return false;
+    if (!entry.tracksPool && context.bufferPool != nullptr) {
+        entry.pool = context.bufferPool;
+        entry.tracksPool = true;
+    }
     return context.bufferPool == nullptr || entry.pool.lock() == context.bufferPool;
 }
 
@@ -1063,10 +1074,10 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
         // the title's own pipelines evict it first, and a full store takes none.
         if (replaying && store.entries.size() >= bound) return pipeline;
         if (replaying) {
-            store.entries.push_front({context.device, context.bufferPool, hash, std::vector<std::byte>(key), pipeline});
+            store.entries.push_front({context.device, context.bufferPool, context.bufferPool != nullptr, hash, std::vector<std::byte>(key), pipeline});
             store.index[hash] = store.entries.begin();
         } else {
-            store.entries.push_back({context.device, context.bufferPool, hash, std::vector<std::byte>(key), pipeline});
+            store.entries.push_back({context.device, context.bufferPool, context.bufferPool != nullptr, hash, std::vector<std::byte>(key), pipeline});
             store.index[hash] = std::prev(store.entries.end());
         }
         while (store.entries.size() > bound) {

core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp

@@ -2,6 +2,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include <array>
+#include <atomic>
 #include <condition_variable>
 #include <cstdlib>
 #include <deque>
@@ -54,9 +55,12 @@ struct LibraryStore {
     std::deque<std::pair<VkDevice, std::function<void()>>> jobs;
     std::map<VkDevice, std::size_t> pending;
     std::map<VkDevice, std::size_t> activeBuilds;
-    bool started = false;
+    std::thread worker;
+    bool stopping = false;
 };
 
+std::atomic<unsigned> runningWorkers{0};
+
 LibraryStore& Libraries() {
     static auto* store = new LibraryStore();
     return *store;
@@ -65,7 +69,8 @@ LibraryStore& Libraries() {
 void runJobs(LibraryStore& store) {
     std::unique_lock lock(store.mutex);
     for (;;) {
-        store.work.wait(lock, [&] { return !store.jobs.empty(); });
+        store.work.wait(lock, [&] { return !store.jobs.empty() || store.stopping; });
+        if (store.stopping) break;
         auto [device, job] = std::move(store.jobs.front());
         store.jobs.pop_front();
         lock.unlock();
@@ -73,15 +78,18 @@ void runJobs(LibraryStore& store) {
         lock.lock();
         if (--store.pending[device] == 0) store.idle.notify_all();
     }
+    runningWorkers.fetch_sub(1, std::memory_order_acq_rel);
 }
 
 void enqueue(LibraryStore& store, VkDevice device, std::function<void()> job) {
-    if (!store.started) {
-        std::thread(runJobs, std::ref(store)).detach();
-        store.started = true;
+    if (store.stopping) return;
+    if (!store.worker.joinable()) {
+        store.worker = std::thread(runJobs, std::ref(store));
+        runningWorkers.fetch_add(1, std::memory_order_acq_rel);
     }
-    ++store.pending[device];
+    auto& pending = store.pending[device];
     store.jobs.emplace_back(device, std::move(job));
+    ++pending;
     store.work.notify_one();
 }
 
@@ -335,6 +343,27 @@ void ClearPipelineLibraries(VkDevice device) {
     store.devices.erase(found);
 }
 
+void StopPipelineLibraryWorker() {
+    auto& store = Libraries();
+    std::thread worker;
+    {
+        std::lock_guard lock(store.mutex);
+        store.stopping = true;
+        for (const auto& [device, job] : store.jobs) --store.pending[device];
+        store.jobs.clear();
+        worker = std::move(store.worker);
+    }
+    store.idle.notify_all();
+    store.work.notify_all();
+    if (!worker.joinable()) return;
+    if (worker.get_id() == std::this_thread::get_id()) worker.detach();
+    else worker.join();
+}
+
+unsigned PipelineLibraryWorkers() {
+    return runningWorkers.load(std::memory_order_acquire);
+}
+
 PipelineLibraryCounters PipelineLibraryCountersOf(VkDevice device) {
     auto& store = Libraries();
     std::lock_guard lock(store.mutex);

core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp

@@ -151,7 +151,10 @@ DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsa
         memory = allocation->memory;
         allocationBytes = allocation->allocationBytes;
         capacity = allocation->bytes;
-        if (allocation->bufferBytes == size) return;
+        if (allocation->bufferBytes == size) {
+            ready = true;
+            return;
+        }
         context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(context.device, buffer, nullptr);
         buffer = VK_NULL_HANDLE;
         try {
@@ -160,6 +163,7 @@ DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsa
             release();
             throw;
         }
+        ready = true;
         return;
     }
     try {
@@ -177,6 +181,7 @@ DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsa
         hostPlaced = allocation.memoryTypeIndex != context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
         memory = cache->AllocateMemory(allocation, "vkAllocateMemory device buffer");
         Check(context.Function<PFN_vkBindBufferMemory>("vkBindBufferMemory")(context.device, buffer, memory, 0), "vkBindBufferMemory device");
+        ready = true;
     } catch (...) {
         release();
         throw;
@@ -188,7 +193,7 @@ DeviceBuffer::~DeviceBuffer() {
 }
 
 void DeviceBuffer::release() noexcept {
-    if (buffer && memory && cache && !hostPlaced) {
+    if (ready && buffer && memory && cache && !hostPlaced) {
         cache->Put({buffer, memory, nullptr, 0, allocationBytes, capacity, usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 0, false, size});
         return;
     }
@@ -373,9 +378,9 @@ void CommandBatch::Submit() {
     submission.pCommandBuffers = &commands;
     timing.Mark("command_end");
     Check(context.Function<PFN_vkQueueSubmit>("vkQueueSubmit")(context.queue, 1, &submission, fence), "vkQueueSubmit graphics");
-    timing.Mark("queue_submit");
     pending = true;
     submitted = true;
+    timing.Mark("queue_submit");
 }
 
 void CommandBatch::Wait() {

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp

@@ -1,10 +1,119 @@
 #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
+#include <spirv/unified1/spirv.hpp>
 #include <algorithm>
 #include <chrono>
 #include <cstdlib>
+#include <map>
+#include <mutex>
+#include <unordered_set>
+#include <utility>
 
 namespace AgcDriver::Graphics {
 
+namespace {
+
+std::uint32_t pointerOperandStored(std::uint32_t opcode, std::span<const std::uint32_t> instruction) {
+    switch (static_cast<spv::Op>(opcode)) {
+    case spv::OpStore:
+    case spv::OpCopyMemory:
+    case spv::OpCopyMemorySized:
+    case spv::OpAtomicStore:
+    case spv::OpAtomicFlagClear:
+    case spv::OpCooperativeMatrixStoreKHR:
+    case spv::OpCooperativeMatrixStoreNV:
+    case spv::OpCooperativeMatrixStoreTensorNV:
+        return instruction.size() > 1 ? instruction[1] : 0u;
+    case spv::OpAtomicExchange:
+    case spv::OpAtomicCompareExchange:
+    case spv::OpAtomicCompareExchangeWeak:
+    case spv::OpAtomicIIncrement:
+    case spv::OpAtomicIDecrement:
+    case spv::OpAtomicIAdd:
+    case spv::OpAtomicISub:
+    case spv::OpAtomicSMin:
+    case spv::OpAtomicUMin:
+    case spv::OpAtomicSMax:
+    case spv::OpAtomicUMax:
+    case spv::OpAtomicAnd:
+    case spv::OpAtomicOr:
+    case spv::OpAtomicXor:
+    case spv::OpAtomicFlagTestAndSet:
+    case spv::OpAtomicFMinEXT:
+    case spv::OpAtomicFMaxEXT:
+    case spv::OpAtomicFAddEXT:
+        return instruction.size() > 3 ? instruction[3] : 0u;
+    default:
+        return 0u;
+    }
+}
+
+bool definesNoPointerValue(std::uint32_t opcode) {
+    switch (static_cast<spv::Op>(opcode)) {
+    case spv::OpTypePointer:
+    case spv::OpTypeUntypedPointerKHR:
+    case spv::OpTypeForwardPointer:
+    case spv::OpName:
+    case spv::OpMemberName:
+    case spv::OpDecorate:
+    case spv::OpMemberDecorate:
+    case spv::OpDecorateId:
+    case spv::OpDecorateString:
+    case spv::OpMemberDecorateString:
+        return true;
+    default:
+        return false;
+    }
+}
+
+template<typename TVisit>
+bool forEachInstruction(std::span<const std::uint32_t> words, const TVisit& visit) {
+    for (std::size_t at = 5; at < words.size();) {
+        const auto count = words[at] >> 16u;
+        if (count == 0 || at + count > words.size()) return false;
+        visit(words[at] & 0xffffu, words.subspan(at, count));
+        at += count;
+    }
+    return true;
+}
+
+bool storesThroughAddressesCached(const ShaderRecompiler::SharedSpirv& spirv) {
+    static std::mutex mutex;
+    static std::map<std::pair<std::uint64_t, std::uint64_t>, bool> known;
+    const auto hash = spirv.ContentHash();
+    const std::pair key{hash.low, hash.high};
+    {
+        std::lock_guard lock(mutex);
+        if (const auto found = known.find(key); found != known.end()) return found->second;
+    }
+    const bool stores = SpirvStoresThroughAddresses(spirv.Words());
+    std::lock_guard lock(mutex);
+    known.emplace(key, stores);
+    return stores;
+}
+
+}
+
+bool SpirvStoresThroughAddresses(std::span<const std::uint32_t> words) {
+    if (words.size() < 5 || words[0] != spv::MagicNumber) return true;
+    std::unordered_set<std::uint32_t> pointerTypes;
+    const bool readable = forEachInstruction(words, [&](std::uint32_t opcode, std::span<const std::uint32_t> instruction) {
+        if ((opcode == spv::OpTypePointer || opcode == spv::OpTypeUntypedPointerKHR) && instruction.size() > 2 && instruction[2] == spv::StorageClassPhysicalStorageBuffer) pointerTypes.insert(instruction[1]);
+    });
+    if (!readable) return true;
+    if (pointerTypes.empty()) return false;
+    std::unordered_set<std::uint32_t> pointers;
+    forEachInstruction(words, [&](std::uint32_t opcode, std::span<const std::uint32_t> instruction) {
+        if (instruction.size() > 2 && !definesNoPointerValue(opcode) && pointerTypes.contains(instruction[1])) pointers.insert(instruction[2]);
+    });
+    bool stores = false;
+    forEachInstruction(words, [&](std::uint32_t opcode, std::span<const std::uint32_t> instruction) {
+        const auto pointer = pointerOperandStored(opcode, instruction);
+        const bool extendedWithPointer = opcode == spv::OpExtInst && std::any_of(instruction.begin() + static_cast<std::ptrdiff_t>(std::min<std::size_t>(5u, instruction.size())), instruction.end(), [&](std::uint32_t operand) { return pointers.contains(operand); });
+        stores = stores || extendedWithPointer || (pointer != 0 && pointers.contains(pointer));
+    });
+    return stores;
+}
+
 void ShaderResources::prepareAddressBindings(std::span<const CompiledShader> shaders, std::span<const GuestMemorySnapshot> snapshots) {
     static const bool loopGuarded = std::getenv("APS5_LOOP_GUARD") != nullptr;
     for (const auto& shader : shaders) {
@@ -22,6 +131,7 @@ void ShaderResources::prepareAddressBindings(std::span<const CompiledShader> sha
         Require(shader.program->bdaAbiVersion == (faults == 0 ? 0u : ShaderRecompiler::BdaAbi::Version), "incompatible BDA ABI version");
         // Rect-list validation needs a fault buffer, but never accesses guest addresses.
         usesBda = usesBda || tables != 0;
+        if (tables != 0 && storesThroughAddressesCached(shader.program->spirv)) bdaReadOnly = false;
         usesFaultBuffer = usesFaultBuffer || faults != 0;
         const bool tableFaultsOnly = std::any_of(shader.program->bindings.begin(), shader.program->bindings.end(), [](const ShaderRecompiler::DescriptorBinding& binding) { return binding.role == ShaderRecompiler::DescriptorRole::ImageTableMap; });
         faultChecks = faultChecks || tables != 0 || (faults != 0 && (!tableFaultsOnly || shader.program->imageTableFaults != 0 || shader.program->poisonedSrtReads != 0 || loopGuarded));

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp

@@ -14,6 +14,7 @@
 #include <atomic>
 #include <memory>
 #include <mutex>
+#include <thread>
 #include <array>
 #include <chrono>
 #include <cstdio>
@@ -145,6 +146,7 @@ struct TextureCache {
     std::chrono::steady_clock::time_point budgetRead{};
     std::uint64_t budget = 0;
     std::uint64_t reportedBudget = 0;
+    std::uint64_t reportedStorage = 0;
     std::chrono::steady_clock::time_point reportedAt{};
     std::uint64_t erasures = 0;
 };
@@ -154,6 +156,26 @@ TextureCache& Textures() {
     return cache;
 }
 
+thread_local unsigned sampledCacheHeld = 0;
+thread_local unsigned storageCacheHeld = 0;
+
+struct CacheHold {
+    unsigned& depth;
+    explicit CacheHold(unsigned& depth) : depth(depth) { ++depth; }
+    ~CacheHold() { --depth; }
+    CacheHold(const CacheHold&) = delete;
+    CacheHold& operator=(const CacheHold&) = delete;
+};
+
+bool AcquireCache(std::mutex& mutex) {
+    const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(20);
+    do {
+        if (mutex.try_lock()) return true;
+        std::this_thread::yield();
+    } while (std::chrono::steady_clock::now() < deadline);
+    return false;
+}
+
 struct UncachedImages {
     std::mutex mutex;
     std::vector<std::weak_ptr<StorageTexture>> images;
@@ -516,7 +538,7 @@ std::uint64_t storageBudget(const Context& context) {
     context.memoryProperties2(context.physical, &properties);
     cached.device = context.physical;
     cached.read = now;
-    cached.budget = StorageTextureBudget(context.memory, &reported);
+    cached.budget = StorageTextureBudget(context.memory, &reported, SampledTextureMemory(), storageCacheBytes());
     return cached.budget;
 }
 
@@ -533,11 +555,17 @@ std::uint64_t sampledBudget(const Context& context, TextureCache& cache) {
     const auto storage = storageCacheBytes();
     const auto budget = SampledTextureBudget(context.memory, &reported, sampled + storage);
     const bool fresh = cache.budgetDevice != context.physical;
-    if (const auto heap = largestDeviceLocalHeap(context.memory); heap && (fresh || SampledBudgetReportDue(cache.reportedBudget, budget, now - cache.reportedAt))) {
+    const auto storageLimit = storageBudget(context);
+    if (const auto heap = largestDeviceLocalHeap(context.memory); heap && (fresh || SampledBudgetReportDue(cache.reportedBudget, budget, now - cache.reportedAt) || SampledBudgetReportDue(cache.reportedStorage, storageLimit, now - cache.reportedAt))) {
         char was[48] = "";
-        if (!fresh) std::snprintf(was, sizeof(was), " (was %llu MiB)", static_cast<unsigned long long>(cache.reportedBudget >> 20u));
-        std::fprintf(stderr, "[gpu] sampled texture cache budget %llu MiB%s: device-local heap %u has a VK_EXT_memory_budget budget of %llu MiB and uses %llu MiB, %llu MiB of it sampled textures (%llu MiB in the cache) and %llu MiB cached storage images; storage image cache budget %llu MiB\n", static_cast<unsigned long long>(budget >> 20u), was, *heap, static_cast<unsigned long long>(reported.heapBudget[*heap] >> 20u), static_cast<unsigned long long>(reported.heapUsage[*heap] >> 20u), static_cast<unsigned long long>(sampled >> 20u), static_cast<unsigned long long>(cache.bytes >> 20u), static_cast<unsigned long long>(storage >> 20u), static_cast<unsigned long long>(storageBudget(context) >> 20u));
+        char wasStorage[48] = "";
+        if (!fresh) {
+            std::snprintf(was, sizeof(was), " (was %llu MiB)", static_cast<unsigned long long>(cache.reportedBudget >> 20u));
+            std::snprintf(wasStorage, sizeof(wasStorage), " (was %llu MiB)", static_cast<unsigned long long>(cache.reportedStorage >> 20u));
+        }
+        std::fprintf(stderr, "[gpu] sampled texture cache budget %llu MiB%s: device-local heap %u has a VK_EXT_memory_budget budget of %llu MiB and uses %llu MiB, %llu MiB of it sampled textures (%llu MiB in the cache) and %llu MiB cached storage images; storage image cache budget %llu MiB%s\n", static_cast<unsigned long long>(budget >> 20u), was, *heap, static_cast<unsigned long long>(reported.heapBudget[*heap] >> 20u), static_cast<unsigned long long>(reported.heapUsage[*heap] >> 20u), static_cast<unsigned long long>(sampled >> 20u), static_cast<unsigned long long>(cache.bytes >> 20u), static_cast<unsigned long long>(storage >> 20u), static_cast<unsigned long long>(storageLimit >> 20u), wasStorage);
         cache.reportedBudget = budget;
+        cache.reportedStorage = storageLimit;
         cache.reportedAt = now;
     }
     cache.budgetDevice = context.physical;
@@ -713,6 +741,7 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
     auto& cache = Textures();
     const auto key = MakeTextureKey(context.device, words, components, depthCompare);
     std::lock_guard lock(cache.mutex);
+    const CacheHold held(sampledCacheHeld);
     dropUncachedViews(cache);
     if (auto it = findTexture(cache, key); it != cache.entries.end()) {
         if (it->source != nullptr) {
@@ -953,6 +982,7 @@ std::shared_ptr<StorageTexture> cachedStorageTexture(const Context& context, std
     std::shared_ptr<StorageTexture> texture;
     {
         std::lock_guard lock(cache.mutex);
+        const CacheHold held(storageCacheHeld);
         if (resource.mipCount > AllocatedLevels(resource)) {
             auto allocated = resource;
             allocated.mipCount = AllocatedLevels(resource);
@@ -1010,10 +1040,14 @@ std::uint64_t TextureCacheBudget(const VkPhysicalDeviceMemoryProperties& memory)
     return std::max<std::uint64_t>(2048ull << 20u, heap ? memory.memoryHeaps[*heap].size / 4u : 0u);
 }
 
-std::uint64_t StorageTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported) {
+std::uint64_t StorageTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t sampledBytes, std::uint64_t storageBytes) {
     const auto heap = largestDeviceLocalHeap(memory);
     if (reported == nullptr || !heap || reported->heapBudget[*heap] == 0) return TextureCacheBudget(memory);
-    return std::max<std::uint64_t>(512ull << 20u, std::min<std::uint64_t>(TextureCacheBudget(memory), reported->heapBudget[*heap] / 4u));
+    const auto budget = reported->heapBudget[*heap];
+    const auto usage = reported->heapUsage[*heap];
+    const auto reserved = usage - std::min<std::uint64_t>(usage, sampledBytes + storageBytes) + std::max<std::uint64_t>(2048ull << 20u, sampledBytes) + budget / 8u;
+    const auto cap = std::min<std::uint64_t>(TextureCacheBudget(memory), budget / 4u);
+    return std::max<std::uint64_t>(512ull << 20u, std::min<std::uint64_t>(cap, budget > reserved ? budget - reserved : 0u));
 }
 
 std::uint64_t SampledTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t textureBytes) {
@@ -1030,6 +1064,10 @@ bool SampledBudgetReportDue(std::uint64_t reported, std::uint64_t budget, std::c
     return change > reported / 10u && sinceReport >= std::chrono::seconds(10);
 }
 
+std::uint64_t StorageTextureCacheBudget(const Context& context) {
+    return storageBudget(context);
+}
+
 TextureCacheUse TextureCacheUsage() {
     TextureCacheUse use;
     {
@@ -1109,6 +1147,90 @@ void ClearCachedTextures(VkDevice device) {
     }
 }
 
+std::uint64_t ReleaseTextureCaches(std::uint64_t bytes) {
+    thread_local bool releasing = false;
+    if (releasing) return 0;
+    releasing = true;
+    struct Done {
+        bool& flag;
+        ~Done() { flag = false; }
+    } done{releasing};
+    std::uint64_t freed = 0;
+    std::size_t sampledEntries = 0;
+    std::size_t storageEntries = 0;
+    std::size_t storageKept = 0;
+    bool sampledSkipped = false;
+    bool storageSkipped = false;
+    {
+        auto& cache = Textures();
+        if (sampledCacheHeld != 0 || !AcquireCache(cache.mutex)) {
+            sampledSkipped = true;
+        } else {
+            std::lock_guard lock(cache.mutex, std::adopt_lock);
+            dropUncachedViews(cache);
+            while (freed < bytes && !cache.entries.empty()) {
+                const auto back = std::prev(cache.entries.end());
+                const auto accounted = back->accounted;
+                const bool last = back->texture.use_count() == 1;
+                eraseTexture(cache, back);
+                ++cache.counts.evicted;
+                ++sampledEntries;
+                if (last) freed += accounted;
+            }
+        }
+    }
+    if (freed < bytes) {
+        auto& cache = StorageTextures();
+        std::unique_lock<GuestMemory::GpuMutexType> gpu(GuestMemory::GpuMutex(), std::defer_lock);
+        if (storageCacheHeld != 0 || !gpu.try_lock()) {
+            storageSkipped = true;
+        } else {
+            try {
+                while (freed < bytes) {
+                    std::vector<std::pair<std::shared_ptr<StorageTexture>, std::uint64_t>> evicted;
+                    {
+                        if (!AcquireCache(cache.mutex)) {
+                            storageSkipped = true;
+                            break;
+                        }
+                        std::lock_guard lock(cache.mutex, std::adopt_lock);
+                        std::uint64_t batch = 0;
+                        storageKept = 0;
+                        for (auto it = cache.entries.end(); it != cache.entries.begin() && batch + freed < bytes;) {
+                            --it;
+                            if (it->texture->Dirty()) {
+                                ++storageKept;
+                                continue;
+                            }
+                            const auto accounted = it->accounted;
+                            const auto next = std::next(it);
+                            evicted.emplace_back(evictStorage(cache, it), accounted);
+                            batch += accounted;
+                            it = next;
+                        }
+                        if (evicted.empty()) break;
+                    }
+                    for (auto& [image, accounted] : evicted) {
+                        std::vector<std::shared_ptr<ShaderResources>> resources;
+                        SharedResourceCache().RemoveImage(image.get(), &resources);
+                        if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->Recording()) {
+                            for (auto& resource : resources) recorder->Keep(std::move(resource));
+                        }
+                        resources.clear();
+                        ++storageEntries;
+                        if (image.use_count() == 1) freed += accounted;
+                        image.reset();
+                    }
+                }
+            } catch (const std::exception& error) {
+                std::fprintf(stderr, "[gpu] texture cache release stopped: %s\n", error.what());
+            }
+        }
+    }
+    std::fprintf(stderr, "[gpu] texture caches released %llu MiB of the %llu MiB asked for: %zu sampled entries%s, %zu storage images%s, %zu with pending results kept\n", static_cast<unsigned long long>(freed >> 20u), static_cast<unsigned long long>(bytes >> 20u), sampledEntries, sampledSkipped ? " (cache busy, skipped)" : "", storageEntries, storageSkipped ? " (cache or device busy, skipped)" : "", storageKept);
+    return freed;
+}
+
 bool StorageImageCached(const Context& context, const StorageTexture* image) {
     auto& cache = StorageTextures();
     std::lock_guard lock(cache.mutex);
@@ -1160,6 +1282,7 @@ const char* roleName(ShaderRecompiler::DescriptorRole role) {
         case ShaderRecompiler::DescriptorRole::ImageTable: return "ImageTable";
         case ShaderRecompiler::DescriptorRole::SamplerTable: return "SamplerTable";
         case ShaderRecompiler::DescriptorRole::ImageTableMap: return "ImageTableMap";
+        case ShaderRecompiler::DescriptorRole::ImageStorageTable: return "ImageStorageTable";
     }
     throw std::runtime_error("AGC graphics: unknown descriptor role");
 }
@@ -1585,10 +1708,13 @@ void ShaderResources::buildPrepare(std::span<const CompiledShader> shaders, cons
             };
             for (const auto& binding : shader.program->bindings) {
                 Require(binding.descriptorSet == ShaderRecompiler::RuntimeAbi::DescriptorSet || (shader.stage == ShaderRecompiler::ShaderStage::Fragment && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet), "unexpected descriptor set: a shader resource must use its stage's descriptor set");
-                if (binding.role == ShaderRecompiler::DescriptorRole::ImageTable) {
-                    Require(binding.binding == ShaderRecompiler::RuntimeAbi::ImageTableBinding && binding.kind == ShaderRecompiler::DescriptorKind::SampledImage && binding.guestDescriptor.size() == static_cast<std::size_t>(binding.count) * 8u && binding.imageShapes.size() == binding.count && binding.imageDepthCompare.size() == binding.count && binding.imageSamplers.size() == binding.count, "image table binding is malformed");
+                if (binding.role == ShaderRecompiler::DescriptorRole::ImageTable || binding.role == ShaderRecompiler::DescriptorRole::ImageStorageTable) {
+                    const bool storage = binding.role == ShaderRecompiler::DescriptorRole::ImageStorageTable;
+                    Require(binding.binding == (storage ? ShaderRecompiler::RuntimeAbi::ImageStorageTableBinding : ShaderRecompiler::RuntimeAbi::ImageTableBinding) && binding.kind == (storage ? ShaderRecompiler::DescriptorKind::StorageImage : ShaderRecompiler::DescriptorKind::SampledImage) && binding.guestDescriptor.size() == static_cast<std::size_t>(binding.count) * 8u && binding.imageShapes.size() == binding.count && binding.imageDepthCompare.size() == binding.count && binding.imageSamplers.size() == binding.count, "image table binding is malformed");
                     auto& table = tableStage(shader);
-                    Require(table.images == nullptr, "a stage binds two image tables");
+                    Require(storage ? !table.storage : !table.sampled, "a stage binds two image tables");
+                    Require(table.images == nullptr || table.images->count == binding.count, "a stage's image tables disagree on their elements");
+                    (storage ? table.storage : table.sampled) = true;
                     table.images = &binding;
                     table.firstSampler = firstSampler;
                     continue;
@@ -1683,6 +1809,7 @@ void ShaderResources::buildPrepare(std::span<const CompiledShader> shaders, cons
         }
         splitSets = SplitsSets(context, shaders);
         std::array<VkShaderStageFlags, 2> tableFlags{};
+        std::array<VkShaderStageFlags, 2> storageTableFlags{};
         for (auto& table : tableStages) {
             Require(table.mapAllocation != std::numeric_limits<std::size_t>::max(), "an image table stage has no map");
             const auto& map = table.map;
@@ -1697,7 +1824,13 @@ void ShaderResources::buildPrepare(std::span<const CompiledShader> shaders, cons
             table.elementBase = tableElements[table.set];
             table.map[ShaderRecompiler::ImageTableAbi::HeaderElementBase] = table.elementBase;
             tableElements[table.set] += elements;
-            tableFlags[table.set] |= VulkanStage(table.stage);
+            if (table.sampled) tableFlags[table.set] |= VulkanStage(table.stage);
+            if (table.storage) storageTableFlags[table.set] |= VulkanStage(table.stage);
+        }
+        for (std::uint32_t set = 0; set < storageTableFlags.size(); ++set) {
+            if (storageTableFlags[set] == 0) continue;
+            bindings.push_back({{ShaderRecompiler::RuntimeAbi::ImageStorageTableBinding, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, StorageTableCapacity(context), storageTableFlags[set], nullptr}, {}, {}, VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT, static_cast<std::int32_t>(set)});
+            storageTableBindings[set] = bindings.size() - 1u;
         }
         for (std::uint32_t set = 0; set < tableFlags.size(); ++set) {
             if (tableFlags[set] == 0) continue;
@@ -1798,7 +1931,8 @@ void ShaderResources::buildPrepare(std::span<const CompiledShader> shaders, cons
             if (storageBuffers != 0) sizes.push_back({VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, static_cast<std::uint32_t>(storageBuffers)});
             const auto tableDescriptors = tableBindings[0] != std::numeric_limits<std::size_t>::max() ? tableDescriptorCount(0) : 0u;
             if (plannedSampledImages + tableDescriptors != 0) sizes.push_back({VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, plannedSampledImages + tableDescriptors});
-            if (plannedStorageImages != 0) sizes.push_back({VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, plannedStorageImages});
+            const auto storageTableDescriptors = storageTableBindings[0] != std::numeric_limits<std::size_t>::max() ? bindings[storageTableBindings[0]].layout.descriptorCount : 0u;
+            if (plannedStorageImages + storageTableDescriptors != 0) sizes.push_back({VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, plannedStorageImages + storageTableDescriptors});
             std::uint32_t samplerDescriptors = 0;
             for (const auto& binding : bindings) {
                 if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) samplerDescriptors += binding.layout.descriptorCount;
@@ -1848,6 +1982,7 @@ DescribedSetLayouts DescribeSetLayouts(const Context& context, std::span<const C
     const bool split = SplitsSets(context, shaders);
     std::array<std::vector<VkDescriptorBindingFlags>, 2> bindingFlags;
     std::array<VkShaderStageFlags, 2> tableFlags{};
+    std::array<VkShaderStageFlags, 2> storageTableFlags{};
     bool updateAfterBind = false;
     std::uint64_t storageBuffers = 0;
     for (const auto& shader : shaders) {
@@ -1861,6 +1996,10 @@ DescribedSetLayouts DescribeSetLayouts(const Context& context, std::span<const C
                 tableFlags[split && shader.stage == ShaderRecompiler::ShaderStage::Fragment ? 1 : 0] |= flags;
                 continue;
             }
+            if (binding.role == ShaderRecompiler::DescriptorRole::ImageStorageTable) {
+                storageTableFlags[split && shader.stage == ShaderRecompiler::ShaderStage::Fragment ? 1 : 0] |= flags;
+                continue;
+            }
             if (binding.role == ShaderRecompiler::DescriptorRole::SamplerTable) {
                 descriptions[set].push_back({binding.binding, VK_DESCRIPTOR_TYPE_SAMPLER, context.samplerTableCapacity, flags, nullptr});
                 bindingFlags[set].push_back(VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT);
@@ -1878,6 +2017,11 @@ DescribedSetLayouts DescribeSetLayouts(const Context& context, std::span<const C
             bindingFlags[set].push_back(0);
         }
     }
+    for (std::size_t set = 0; set < storageTableFlags.size(); ++set) {
+        if (storageTableFlags[set] == 0) continue;
+        descriptions[set].push_back({ShaderRecompiler::RuntimeAbi::ImageStorageTableBinding, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, StorageTableCapacity(context), storageTableFlags[set], nullptr});
+        bindingFlags[set].push_back(VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT);
+    }
     for (std::size_t set = 0; set < tableFlags.size(); ++set) {
         if (tableFlags[set] == 0) continue;
         descriptions[set].push_back({ShaderRecompiler::RuntimeAbi::ImageTableBinding, VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, context.imageTableCapacity, tableFlags[set], nullptr});
@@ -1968,6 +2112,7 @@ void ShaderResources::buildComplete() {
                         for (auto element = first; element < last; ++element) {
                             const auto index = binding.imageAllocations[element];
                             if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) images.push_back({samplers[index]->Handle(), VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED});
+                            else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) images.push_back({VK_NULL_HANDLE, storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL});
                             else images.push_back({VK_NULL_HANDLE, textureFirstLayer[index] ? textures[index]->FirstLayerView() : textures[index]->View(), textures[index]->Layout()});
                         }
                         writes.push_back(write);
@@ -2072,6 +2217,7 @@ void mixDataWords(std::uint64_t& hash, std::span<const std::uint32_t> words) {
 // ResourceCache::Find and Touch calls (the [rescache] line).
 std::atomic<std::uint64_t> resourceCacheFinds{0};
 std::atomic<std::uint64_t> resourceCacheTouches{0};
+std::atomic<std::uint64_t> depthGuardRefusals{0};
 
 }
 
@@ -2114,6 +2260,10 @@ void ShaderResources::noteReusable() {
     reusable = true;
 }
 
+std::uint64_t ShaderResources::DepthGuardRefusals() {
+    return depthGuardRefusals.load(std::memory_order_relaxed);
+}
+
 bool ShaderResources::NeverReusable(std::span<const CompiledShader> shaders) {
     const bool loopGuard = LoopGuardEnabled();
     return std::any_of(shaders.begin(), shaders.end(), [loopGuard](const CompiledShader& shader) {
@@ -2584,6 +2734,10 @@ bool ShaderResources::fastRevalidate(std::uint64_t serialBefore, std::span<const
         const auto& own = image->Descriptor();
         const auto address = own.baseAddress;
         const auto bytes = static_cast<std::size_t>(image->GuestBytes());
+        if (DepthSurfaceOverlaps(context.device, address, bytes)) {
+            depthGuardRefusals.fetch_add(1, std::memory_order_relaxed);
+            return fail(FastFail::Depth);
+        }
         if (GuestMemory::CollectWrites(address, bytes) == 0) return fail(FastFail::Collect);
         if (own.dccAddress != 0) {
             // Refresh's unchanged branch (its key compare against the keys the content was
@@ -2687,6 +2841,10 @@ bool ShaderResources::fastRevalidateEach() {
         if (own.dccAddress != 0) return false;
         const auto address = own.baseAddress;
         const auto bytes = static_cast<std::size_t>(image->GuestBytes());
+        if (DepthSurfaceOverlaps(context.device, address, bytes)) {
+            depthGuardRefusals.fetch_add(1, std::memory_order_relaxed);
+            return false;
+        }
         if (GuestMemory::CollectWrites(address, bytes) == 0) return false;
         if (PendingStorageOverlaps(address, bytes, image)) return false;
         if (!StorageImageCached(context, image)) return false;
@@ -3383,7 +3541,7 @@ ResourceCache& SharedResourceCache() {
     return *cache;
 }
 
-DescriptorCache::DescriptorCache(const Context& context) : context(context), destroyLayout(context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")), destroyPool(context.Function<PFN_vkDestroyDescriptorPool>("vkDestroyDescriptorPool")), resetPool(context.Function<PFN_vkResetDescriptorPool>("vkResetDescriptorPool")) {}
+DescriptorCache::DescriptorCache(const Context& context) : context(context), destroyLayout(context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")), destroyPool(context.Function<PFN_vkDestroyDescriptorPool>("vkDestroyDescriptorPool")), resetPool(context.Function<PFN_vkResetDescriptorPool>("vkResetDescriptorPool")), freeDescriptorSets(context.individuallyFreeDescriptorSets ? context.Function<PFN_vkFreeDescriptorSets>("vkFreeDescriptorSets") : nullptr) {}
 
 DescriptorCache::~DescriptorCache() {
     for (const auto& pool : pools) destroyPool(context.device, pool.handle, nullptr);
@@ -3410,7 +3568,12 @@ VkDescriptorSetLayout DescriptorCache::Layout(std::span<const std::uint32_t> key
     ValidateDescriptorLayout(context, bindings, updateAfterBind);
     ++stats.layoutMisses;
     const auto layout = CreateSetLayout(context, bindings, updateAfterBind, flags);
-    layouts.emplace(std::move(keyCopy), layout);
+    try {
+        layouts.emplace(std::move(keyCopy), layout);
+    } catch (...) {
+        destroyLayout(context.device, layout, nullptr);
+        throw;
+    }
     return layout;
 }
 
@@ -3481,9 +3644,11 @@ DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout l
     }
     VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
     poolInfo.flags = updateAfterBind ? VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT : 0;
+    if (context.individuallyFreeDescriptorSets) poolInfo.flags |= VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT;
     poolInfo.maxSets = ChainPoolSets;
     poolInfo.poolSizeCount = static_cast<std::uint32_t>(ChainPoolCreateSizes.size());
     poolInfo.pPoolSizes = ChainPoolCreateSizes.data();
+    chain.reserve(chain.size() + 1);
     VkDescriptorPool pool = VK_NULL_HANDLE;
     Check(context.Function<PFN_vkCreateDescriptorPool>("vkCreateDescriptorPool")(context.device, &poolInfo, nullptr, &pool), "vkCreateDescriptorPool");
     chain.push_back({pool});
@@ -3536,6 +3701,7 @@ void DescriptorCache::Free(const SetAllocation& allocation) noexcept {
     if (allocation.poolIndex >= chain.size()) std::terminate();
     auto& pool = chain[allocation.poolIndex];
     if (pool.handle != allocation.pool || pool.liveSets == 0) std::terminate();
+    if (freeDescriptorSets != nullptr && freeDescriptorSets(context.device, allocation.pool, 1, &allocation.set) != VK_SUCCESS) std::terminate();
     if (--pool.liveSets != 0) return;
     if (resetPool(context.device, pool.handle, 0) != VK_SUCCESS) std::terminate();
     pool.exhausted = false;
@@ -3546,11 +3712,8 @@ DescriptorCache::Stats DescriptorCache::Counters() const {
     return stats;
 }
 
-std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read) {
-    Require(words.size() == 4, "buffer descriptor must contain four DWORDs");
-    Require((words[1] & 0x40000000u) == 0, "buffer descriptor has reserved bits set");
+std::uint64_t ShaderResources::guestBufferBytes(std::span<const std::uint32_t> words) const {
     const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}};
-    Require(descriptor.Type() == 0u, "buffer descriptor uses an unsupported type");
     const auto address = descriptor.Base48();
     auto byteSize = descriptor.GetSize();
     if (byteSize > context.limits.maxStorageBufferRange) {
@@ -3558,22 +3721,33 @@ std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words
         if (mappedEnd <= address) byteSize = 0;
         else if (mappedEnd - address < context.limits.maxStorageBufferRange) byteSize = mappedEnd - address;
     }
+    if (byteSize == 0 || address == 0) return 0;
+    if (const std::uint64_t stride = descriptor.Stride(); stride != 0 && byteSize <= context.limits.maxStorageBufferRange) {
+        constexpr std::uint64_t GuestPageBytes = 0x4000;
+        const auto end = address + byteSize;
+        const auto overhang = end % GuestPageBytes;
+        if (overhang != 0 && overhang < stride) {
+            const auto allocationEnd = RegisteredReadableEnd(address);
+            if (allocationEnd > address && allocationEnd < end && end - allocationEnd < stride) byteSize = allocationEnd - address;
+        }
+    }
+    return byteSize;
+}
+
+std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read) {
+    Require(words.size() == 4, "buffer descriptor must contain four DWORDs");
+    Require((words[1] & 0x40000000u) == 0, "buffer descriptor has reserved bits set");
+    const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}};
+    Require(descriptor.Type() == 0u, "buffer descriptor uses an unsupported type");
+    const auto address = descriptor.Base48();
+    const auto byteSize = guestBufferBytes(words);
     if (byteSize == 0 || address == 0) {
         allocations.push_back({0, EmptyBufferBytes, false, nullptr, ShaderRecompiler::DescriptorRole::GuestBuffers, false});
         return allocations.size() - 1;
     }
     Require(byteSize <= context.limits.maxStorageBufferRange, "shader buffer exceeds descriptor range limit");
     Require(byteSize <= std::numeric_limits<std::size_t>::max(), "shader buffer size exceeds host address space");
-    auto size = static_cast<std::size_t>(byteSize);
-    if (const std::uint64_t stride = descriptor.Stride(); stride != 0) {
-        constexpr std::uint64_t GuestPageBytes = 0x4000;
-        const auto end = address + size;
-        const auto overhang = end % GuestPageBytes;
-        if (overhang != 0 && overhang < stride) {
-            const auto allocationEnd = RegisteredReadableEnd(address);
-            if (allocationEnd > address && allocationEnd < end && end - allocationEnd < stride) size = static_cast<std::size_t>(allocationEnd - address);
-        }
-    }
+    const auto size = static_cast<std::size_t>(byteSize);
     Require(target == nullptr || !overlap(address, size, target->address, target->bytes), "shader buffer aliases the render target");
     // APS5_ALL_BUFFERS_WRITTEN=1: every element is noted as written, as before bufferWritten existed.
     static const bool allWritten = std::getenv("APS5_ALL_BUFFERS_WRITTEN") != nullptr;
@@ -4098,6 +4272,10 @@ void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBind
     }
 }
 
+std::uint32_t StorageTableCapacity(const Context& context) {
+    return std::min<std::uint32_t>({context.imageTableCapacity, 4096u, context.limits.maxPerStageDescriptorStorageImages / 4u});
+}
+
 std::uint32_t ImageTableCapacity(const Context& context) {
     if (!context.runtimeDescriptorArray || !context.partiallyBound || !context.variableDescriptorCount || !context.descriptorIndexing) return 0u;
     constexpr std::uint64_t sampledHeaps = static_cast<std::uint64_t>(ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding - ShaderRecompiler::RuntimeAbi::FirstImageBinding) * ShaderRecompiler::RuntimeAbi::SampledHeapCapacity;
@@ -4193,14 +4371,16 @@ void ShaderResources::addSamplerTable(const ShaderRecompiler::DescriptorBinding&
 }
 
 void ShaderResources::resolveImageTables() {
-    if (tableBindings[0] == std::numeric_limits<std::size_t>::max() && tableBindings[1] == std::numeric_limits<std::size_t>::max()) return;
+    const auto none = [](const std::array<std::size_t, 2>& set) { return set[0] == std::numeric_limits<std::size_t>::max() && set[1] == std::numeric_limits<std::size_t>::max(); };
+    if (none(tableBindings) && none(storageTableBindings)) return;
     auto& counters = TextureCounts();
     for (std::size_t set = 0; set < tableBindings.size(); ++set) {
+        if (storageTableBindings[set] != std::numeric_limits<std::size_t>::max()) bindings[storageTableBindings[set]].imageAllocations.assign(std::min(tableElements[set], StorageTableCapacity(context)), std::numeric_limits<std::size_t>::max());
         if (tableBindings[set] != std::numeric_limits<std::size_t>::max()) bindings[tableBindings[set]].imageAllocations.assign(std::min(tableElements[set], context.imageTableCapacity), std::numeric_limits<std::size_t>::max());
     }
     for (auto& table : tableStages) {
         if (table.images == nullptr) continue;
-        auto& item = bindings[tableBindings[table.set]];
+        auto* item = tableBindings[table.set] != std::numeric_limits<std::size_t>::max() ? &bindings[tableBindings[table.set]] : nullptr;
         const auto& binding = *table.images;
         const auto faults = table.map[ShaderRecompiler::ImageTableAbi::HeaderElementFaults];
         const auto stageSamplers = std::span<const std::shared_ptr<Sampler>>(samplers).subspan(table.firstSampler, table.samplerCount);
@@ -4229,11 +4409,12 @@ void ShaderResources::resolveImageTables() {
             textureFirstLayer.push_back(firstLayer);
             describedRanges.push_back({"texture", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast<int>(resource.tileMode), resource.dccAddress});
             table.elementTextures[element] = textures.size() - 1u;
-            item.imageAllocations[static_cast<std::size_t>(table.elementBase) + element] = textures.size() - 1u;
+            item->imageAllocations[static_cast<std::size_t>(table.elementBase) + element] = textures.size() - 1u;
         };
         const auto state = CurrentTableState();
         std::vector<std::shared_ptr<Texture>> cached;
-        if (auto* reused = FindReusedTable(context.device, binding, state); reused != nullptr && TableCurrent(*reused) && CachedTableTextures(*reused, cached)) {
+        const bool storageTable = std::find(binding.imageWritten.begin(), binding.imageWritten.end(), true) != binding.imageWritten.end();
+        if (auto* reused = storageTable ? nullptr : FindReusedTable(context.device, binding, state); reused != nullptr && TableCurrent(*reused) && CachedTableTextures(*reused, cached)) {
             auto next = cached.begin();
             for (std::uint32_t element = 0; element < binding.count; ++element) {
                 const auto words = std::span<const std::uint32_t>(binding.guestDescriptor).subspan(static_cast<std::size_t>(element) * 8u, 8u);
@@ -4277,6 +4458,45 @@ void ShaderResources::resolveImageTables() {
                 if (reusable) fresh.elements.push_back({{}, {}, {}, 0, false, true, error.what()});
                 continue;
             }
+            if (element < binding.imageWritten.size() && binding.imageWritten[element]) {
+                reusable = false;
+                const auto storageBinding = storageTableBindings[table.set];
+                if (storageBinding == std::numeric_limits<std::size_t>::max()) {
+                    table.map[faults + element] = rejectTableElement(table, words, "set " + std::to_string(table.set) + " has no image storage table, but the shader stores to its table element " + std::to_string(absolute));
+                    continue;
+                }
+                if (absolute >= StorageTableCapacity(context)) {
+                    table.map[faults + element] = rejectTableElement(table, words, "the stored table element " + std::to_string(absolute) + " of set " + std::to_string(table.set) + " is past the image storage table's capacity of " + std::to_string(StorageTableCapacity(context)) + " images");
+                    continue;
+                }
+                CompletePendingCpuWrites(context, resource.baseAddress, static_cast<std::size_t>(guestBytes));
+                if (resource.dccAddress != 0) CompletePendingCpuWrites(context, resource.dccAddress, DccKeyBytes(guestBytes));
+                const auto mip = std::min(resource.baseLevel, resource.mipCount - 1u);
+                if (resource.minLod > mip * 256u) {
+                    table.map[faults + element] = rejectTableElement(table, words, "the table entry's storage T# clamps its minimum LOD above the level it addresses, which is not implemented");
+                    continue;
+                }
+                try {
+                    storageTextures.push_back(cachedStorageTexture(context, words, resource, mip, guestBytes, true));
+                } catch (const std::runtime_error& error) {
+                    table.map[faults + element] = rejectTableElement(table, words, error.what());
+                    continue;
+                }
+                storageMips.push_back(mip);
+                storageKeys.push_back(resource.dccAddress);
+                storageFirstLayer.push_back(firstLayer);
+                storageWritten.push_back(true);
+                storageAtomic.push_back(false);
+                storageAtomic64.push_back(false);
+                describedRanges.push_back({"storage", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast<int>(resource.tileMode), resource.dccAddress});
+                bindings[storageBinding].imageAllocations[absolute] = storageTextures.size() - 1u;
+                continue;
+            }
+            if (item == nullptr) {
+                table.map[faults + element] = rejectTableElement(table, words, "a sampled image table entry has no sampled image table binding");
+                reusable = false;
+                continue;
+            }
             CompletePendingCpuWrites(context, resource.baseAddress, static_cast<std::size_t>(guestBytes));
             if (resource.dccAddress != 0) CompletePendingCpuWrites(context, resource.dccAddress, DccKeyBytes(guestBytes));
             std::shared_ptr<Texture> texture;
@@ -4363,15 +4583,20 @@ void ShaderResources::finishImageTables() {
     bda->SetImageTables(std::move(faults), std::move(ranges));
 }
 
-std::vector<std::pair<std::uint64_t, std::uint64_t>> ShaderResources::PresyncSurfaces() const {
-    std::vector<std::pair<std::uint64_t, std::uint64_t>> surfaces;
+std::vector<PresyncSurface> ShaderResources::PresyncSurfaces() const {
+    std::vector<PresyncSurface> surfaces;
     // A surface's lookup reads guest memory on the CPU unless it is served GPU-direct from a host
     // import: a storage image of imported memory uploads and refreshes from the import, and a
     // sampled texture of one views that image (SampledFromStorageEligible), so neither waits for
     // the producer.
     const auto consider = [&](std::uint32_t format, std::uint64_t address, std::uint64_t guestBytes, bool sampled) {
         const bool gpuDirect = sampled ? SampledFromStorageEligible(context, format, address, guestBytes) : HostImportCovers(context, address, static_cast<std::size_t>(guestBytes));
-        if (!gpuDirect) surfaces.emplace_back(address, guestBytes);
+        if (!gpuDirect) surfaces.push_back({address, guestBytes, false});
+    };
+    static const bool keyRanges = std::getenv("APS5_NO_DCC_PRESYNC") == nullptr;
+    const auto considerKeys = [&](std::uint64_t dccAddress, std::uint64_t guestBytes) {
+        const auto keyBytes = DccKeyBytes(guestBytes);
+        if (keyRanges && dccAddress != 0 && keyBytes != 0) surfaces.push_back({dccAddress, keyBytes, true});
     };
     if (completed) {
         // A cached object (the resource cache): Revalidate repeats the lookups of the objects the
@@ -4387,18 +4612,22 @@ std::vector<std::pair<std::uint64_t, std::uint64_t>> ShaderResources::PresyncSur
                 const auto& texture = textures[textureIndex++];
                 if (texture->ViewsStorageImage()) consider(range.format, range.address, range.bytes, false);
                 else consider(range.format, range.address, range.bytes, true);
+                considerKeys(range.dccAddress, range.bytes);
             }
         }
         for (std::size_t index = 0; index < storageTextures.size(); ++index) {
             if (storageTextures[index] == nullptr || (index != 0 && storageTextures[index] == storageTextures[index - 1])) continue;
             const auto& image = *storageTextures[index];
             consider(image.Descriptor().format, image.Descriptor().baseAddress, image.GuestBytes(), false);
+            considerKeys(image.Descriptor().dccAddress, image.GuestBytes());
         }
         return surfaces;
     }
     if (!imageRecords.empty()) {
         for (const auto& record : imageRecords) {
-            if (record.decoded) consider(record.resource.format, record.resource.baseAddress, record.guestBytes, record.sampled);
+            if (!record.decoded) continue;
+            consider(record.resource.format, record.resource.baseAddress, record.guestBytes, record.sampled);
+            considerKeys(record.resource.dccAddress, record.guestBytes);
         }
         return surfaces;
     }
@@ -4407,14 +4636,18 @@ std::vector<std::pair<std::uint64_t, std::uint64_t>> ShaderResources::PresyncSur
     forEachImageElement(*deferredCompute.program, [&](const ShaderRecompiler::DescriptorBinding& binding, std::uint32_t, std::span<const std::uint32_t> words) {
         if (IsNullTextureResource(words)) return;
         const auto resource = DecodeTextureResource(words);
-        consider(resource.format, resource.baseAddress, DescribeSurface(resource).guestBytes, binding.kind == ShaderRecompiler::DescriptorKind::SampledImage);
+        const auto guestBytes = DescribeSurface(resource).guestBytes;
+        consider(resource.format, resource.baseAddress, guestBytes, binding.kind == ShaderRecompiler::DescriptorKind::SampledImage);
+        considerKeys(resource.dccAddress, guestBytes);
     });
     for (const auto& binding : deferredCompute.program->bindings) {
         if (binding.role != ShaderRecompiler::DescriptorRole::ImageTable) continue;
         for (std::size_t element = 0; element < binding.guestDescriptor.size() / 8u; ++element) {
             try {
                 const auto resource = DecodeTextureResource(std::span<const std::uint32_t>(binding.guestDescriptor).subspan(element * 8u, 8u));
-                consider(resource.format, resource.baseAddress, DescribeSurface(resource).guestBytes, true);
+                const auto guestBytes = DescribeSurface(resource).guestBytes;
+                consider(resource.format, resource.baseAddress, guestBytes, true);
+                considerKeys(resource.dccAddress, guestBytes);
             } catch (const std::runtime_error&) {
             }
         }
@@ -4520,13 +4753,71 @@ std::optional<std::vector<ShaderResources::MovedBuffer>> ShaderResources::MovedR
     return moved;
 }
 
+const char* ShaderResources::MoveRefusalName(MoveRefusal refusal) {
+    switch (refusal) {
+        case MoveRefusal::None: return "none";
+        case MoveRefusal::Shape: return "bindings differ";
+        case MoveRefusal::NotGuest: return "not read in place";
+        case MoveRefusal::Written: return "written or emptied";
+        case MoveRefusal::Limit: return "over the limit";
+        case MoveRefusal::Inaccessible: return "unmapped";
+        case MoveRefusal::Overlap: return "pending write";
+        case MoveRefusal::Bindings: return "no bindings made";
+        case MoveRefusal::Count: break;
+    }
+    return "?";
+}
+
+std::optional<std::vector<ShaderResources::MovedBuffer>> ShaderResources::MovedDispatchBuffers(const CompiledShader& shader, Recorder* recorder, std::size_t maxBytes, MoveProof& proof) const {
+    proof = {};
+    const auto refuse = [&](MoveRefusal why) {
+        proof.refusal = why;
+        return std::optional<std::vector<MovedBuffer>>{};
+    };
+    std::vector<MovedBuffer> moved;
+    if (shader.program == nullptr || usesBda) return refuse(MoveRefusal::Shape);
+    if (_set == VK_NULL_HANDLE) return moved;
+    for (const auto& binding : shader.program->bindings) {
+        if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue;
+        const auto kept = std::find_if(bindings.begin(), bindings.end(), [&](const Binding& item) { return item.layout.binding == binding.binding; });
+        if (kept == bindings.end() || kept->allocations.size() != binding.count || binding.guestDescriptor.size() != static_cast<std::size_t>(binding.count) * 4u) return refuse(MoveRefusal::Shape);
+        for (std::uint32_t element = 0; element < binding.count; ++element) {
+            const auto* words = binding.guestDescriptor.data() + static_cast<std::size_t>(element) * 4u;
+            const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}};
+            const auto address = descriptor.Base48();
+            const auto size = guestBufferBytes(std::span<const std::uint32_t>(words, 4));
+            const auto& item = allocations[kept->allocations[element]];
+            const bool empty = size == 0 || address == 0;
+            if (!item.guest) {
+                if (empty) continue;
+                return refuse(MoveRefusal::NotGuest);
+            }
+            if (item.address == address && item.size == size) continue;
+            const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element];
+            if (written || empty || item.written) return refuse(MoveRefusal::Written);
+            if (size > context.limits.maxStorageBufferRange) return refuse(MoveRefusal::Limit);
+            const auto begin = address - item.adjustment;
+            const auto bytes = static_cast<std::size_t>(GuestBufferMemory::ViewBytes(size, item.adjustment));
+            if (maxBytes != 0 && proof.bytes + bytes > maxBytes) return refuse(MoveRefusal::Limit);
+            if (!GuestMemory::Accessible(reinterpret_cast<const void*>(begin), bytes)) return refuse(MoveRefusal::Inaccessible);
+            const bool pending = recorder != nullptr ? recorder->PendingWriteOverlaps(begin, bytes) : Recorder::SnapshotWriteOverlaps(begin, bytes);
+            const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> range{{{begin, begin + bytes}}};
+            if (guestMemory.WritesOverlap(begin, bytes) || pending || StorageTexture::AnyPendingOverlaps(range) || AnyShadowedOverlaps(begin, bytes)) return refuse(MoveRefusal::Overlap);
+            moved.push_back({kept->allocations[element], address, static_cast<std::size_t>(size)});
+            ++proof.buffers;
+            proof.bytes += bytes;
+        }
+    }
+    return moved;
+}
+
 std::size_t ShaderResources::MaxDrawSnapshotBytes() {
     const char* text = std::getenv("APS5_DRAW_SNAPSHOT_MAX_MIB");
     const auto mib = text != nullptr ? std::strtoull(text, nullptr, 10) : 64ull;
     return static_cast<std::size_t>(mib * 1024ull * 1024ull);
 }
 
-std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindings(Recorder& recorder, std::span<const MovedBuffer> moved) const {
+std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindings(Recorder& recorder, std::span<const MovedBuffer> moved, bool movedOnly) const {
     if ((_set == VK_NULL_HANDLE && fragment.set == VK_NULL_HANDLE) || usesBda) return {};
     const auto reads = guestMemory.InPlaceReads();
     const auto maxSnapshotBytes = maxDrawSnapshotBytes;
@@ -4566,7 +4857,7 @@ std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindi
             address = override->address;
             size = override->size;
         } else {
-            if (!item.guest || item.written || guestMemory.WritesOverlap(item.address, item.size)) continue;
+            if (movedOnly || !item.guest || item.written || guestMemory.WritesOverlap(item.address, item.size)) continue;
             const bool direct = std::any_of(reads.begin(), reads.end(), [&](const auto& range) { return item.address >= range.first && item.address < range.second && item.size <= range.second - item.address; });
             if (!direct || recorder.PendingWriteOverlaps(item.address, item.size)) continue;
         }
@@ -4637,7 +4928,8 @@ std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindi
     for (std::size_t index = 0; index < selected.size(); ++index) snapshotIndices[selected[index]] = index;
     auto nextWrite = writes.begin();
     for (const auto& binding : bindings) {
-        while (nextWrite != writes.end() && nextWrite->dstBinding == binding.layout.binding) {
+        const auto original = setOf(binding) == 0 ? _set : fragment.set;
+        while (nextWrite != writes.end() && nextWrite->dstBinding == binding.layout.binding && nextWrite->dstSet == original) {
             auto& write = *nextWrite++;
             write.dstSet = setOf(binding) == 0 ? result->allocation.set : result->fragmentAllocation.set;
             if (binding.allocations.empty()) continue;

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp

@@ -115,7 +115,7 @@ struct Module {
             return;
         }
         // One gl_ClipDistance array holds the guest clip distances and the recompiler's all-zero position distance.
-        if (!input && ((value == spv::BuiltInClipDistance && (vertex || stage == Stage::TessellationEvaluation || stage == Stage::Mesh)) || (value == spv::BuiltInCullDistance && (vertex || stage == Stage::Mesh)))) {
+        if (!input && ((value == spv::BuiltInClipDistance && (vertex || stage == Stage::TessellationControl || stage == Stage::TessellationEvaluation || stage == Stage::Mesh)) || (value == spv::BuiltInCullDistance && (vertex || stage == Stage::Mesh)))) {
             const auto signature = Signature(type);
             auto& count = value == spv::BuiltInClipDistance ? clipDistances : cullDistances;
             Require(count == 0 && signature.size() == 6 && signature.starts_with("f32[") && signature[4] >= '1' && signature[4] <= '8' && signature[5] == ']', "invalid or duplicate clip or cull distance output");
@@ -131,7 +131,8 @@ struct Module {
                 return;
             }
             if (stage == Stage::TessellationControl || stage == Stage::TessellationEvaluation) {
-                Require(input && ((value == spv::BuiltInPosition && signature == "f32x4") || (value == spv::BuiltInTessCoord && stage == Stage::TessellationEvaluation && signature == "f32x3") || ((value == spv::BuiltInInvocationId || value == spv::BuiltInPrimitiveId || value == spv::BuiltInPatchVertices) && (signature == "i32" || signature == "u32"))), "unsupported tessellation built-in");
+                const bool clipInput = value == spv::BuiltInClipDistance && signature.size() == 6 && signature.starts_with("f32[") && signature[4] >= '1' && signature[4] <= '8' && signature[5] == ']';
+                Require(input && (clipInput || (value == spv::BuiltInPosition && signature == "f32x4") || (value == spv::BuiltInTessCoord && stage == Stage::TessellationEvaluation && signature == "f32x3") || ((value == spv::BuiltInInvocationId || value == spv::BuiltInPrimitiveId || value == spv::BuiltInPatchVertices) && (signature == "i32" || signature == "u32"))), "unsupported tessellation built-in");
                 return;
             }
             if (!input && value == spv::BuiltInPrimitiveLineIndicesEXT) primitiveIndices = 2u;
@@ -252,7 +253,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
                     fragment &&
                     capability == spv::CapabilityStencilExportEXT;
 
-                if ((vertex || evaluation || mesh) && (capability == spv::CapabilityClipDistance || capability == spv::CapabilityCullDistance)) {
+                if ((vertex || control || evaluation || mesh) && (capability == spv::CapabilityClipDistance || capability == spv::CapabilityCullDistance)) {
                     char word[16];
                     std::snprintf(word, sizeof(word), "0x%08x", state.paClVsOutCntl);
                     Require(capability == spv::CapabilityClipDistance ? shaderClipDistance : (vertex || mesh) && shaderCullDistance, std::string("vertex clip or cull distance needs shaderClipDistance or shaderCullDistance, which the device lacks (PA_CL_VS_OUT_CNTL=") + word + ")");
@@ -264,7 +265,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
                     capability == spv::CapabilityImage1D ||
                     capability == spv::CapabilityImageGatherExtended ||
                     capability == spv::CapabilityImageQuery ||
-                    ((vertex || evaluation || mesh) && shaderClipDistance && capability == spv::CapabilityClipDistance) ||
+                    ((vertex || control || evaluation || mesh) && shaderClipDistance && capability == spv::CapabilityClipDistance) ||
                     (vertex && capability == spv::CapabilityDrawParameters) ||
                     (fragment && geometryShader && capability == spv::CapabilityGeometry) ||
                     (fragment && sampleRateShading && capability == spv::CapabilitySampleRateShading) ||
@@ -441,6 +442,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
         const auto it = module.modes.find(name);
         Require(it != module.modes.end() && it->second == operands, "missing or incompatible shader execution mode");
     };
+    const bool barycentricPatches = (control || evaluation) && !state.rectList && !state.stages.tessellation && state.stages.path == ShaderPath::Vertex;
     if (mesh) {
         Require(state.stages.mesh.has_value(), "mesh configuration is missing");
         const auto& config = *state.stages.mesh;
@@ -451,6 +453,12 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
         mode(spv::ExecutionModeOutputPrimitivesEXT, {config.maxPrimitives});
         mode(config.outputPrimitiveSize == 2u ? spv::ExecutionModeOutputLinesEXT : spv::ExecutionModeOutputTrianglesEXT, {});
         Require(module.modes.size() == 4, "unsupported mesh execution mode");
+    } else if (barycentricPatches) {
+        if (control) mode(spv::ExecutionModeOutputVertices, {3u});
+        mode(spv::ExecutionModeTriangles, {});
+        mode(spv::ExecutionModeSpacingEqual, {});
+        mode(spv::ExecutionModeVertexOrderCw, {});
+        Require(module.modes.size() == (control ? 4u : 3u), "unsupported generated tessellation execution mode");
     } else if (control) {
         Require(state.rectList || state.stages.tessellation.has_value(), "tessellation configuration is missing");
         mode(spv::ExecutionModeOutputVertices, {state.rectList ? 4u : state.stages.tessellation->outputControlPoints});
@@ -503,7 +511,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
             } else if (geometry) {
                 Require(length->second == 3u, "geometry interface array must hold the three vertices of a triangle");
             } else {
-                const auto expected = state.rectList ? (control && input ? 3u : 4u) : control && input ? state.stages.tessellation->inputControlPoints : state.stages.tessellation->outputControlPoints;
+                const auto expected = barycentricPatches ? 3u : state.rectList ? (control && input ? 3u : 4u) : control && input ? state.stages.tessellation->inputControlPoints : state.stages.tessellation->outputControlPoints;
                 Require(length->second == expected, "tessellation interface array disagrees with control-point count");
             }
             typeId = outer[2];
@@ -549,10 +557,10 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
             Require(decoration.set && decoration.binding, "unbound shader resource");
             const auto key = std::make_pair(*decoration.set, *decoration.binding);
             const auto binding = std::find_if(shader.bindings.begin(), shader.bindings.end(), [&](const auto& item) { return item.descriptorSet == key.first && item.binding == key.second; });
-            Require(descriptors.insert(key).second || (binding != shader.bindings.end() && binding->role == ShaderRecompiler::DescriptorRole::ImageTable), "duplicate SPIR-V resource binding");
+            Require(descriptors.insert(key).second || (binding != shader.bindings.end() && (binding->role == ShaderRecompiler::DescriptorRole::ImageTable || binding->role == ShaderRecompiler::DescriptorRole::ImageStorageTable)), "duplicate SPIR-V resource binding");
             Require(binding != shader.bindings.end(), "SPIR-V resource is absent from recompiler binding metadata");
             Require(binding->kind == ShaderRecompiler::DescriptorKind::Sampler || binding->kind == ShaderRecompiler::DescriptorKind::SampledImage || binding->kind == ShaderRecompiler::DescriptorKind::StorageImage, "SPIR-V descriptor type disagrees with recompiler binding metadata");
-            if (binding->role == ShaderRecompiler::DescriptorRole::ImageTable || binding->role == ShaderRecompiler::DescriptorRole::SamplerTable) {
+            if (binding->role == ShaderRecompiler::DescriptorRole::ImageTable || binding->role == ShaderRecompiler::DescriptorRole::ImageStorageTable || binding->role == ShaderRecompiler::DescriptorRole::SamplerTable) {
                 Require(type.size() == 3 && (type[0] & 0xffffu) == spv::OpTypeRuntimeArray, "image table must be declared as a runtime descriptor array");
                 continue;
             }
@@ -604,7 +612,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
         Require(shader.vertexAttributes.empty(), "vertex attribute metadata is invalid for this stage");
     }
     Require(descriptors.size() == shader.bindings.size(), "recompiler binding metadata contains undeclared resources");
-    Require(push == (!shader.pushConstants.empty() || mesh || (control && !rectListControl)), "recompiler push constant metadata disagrees with SPIR-V");
+    Require(push == (!shader.pushConstants.empty() || mesh || (control && !rectListControl && !barycentricPatches)), "recompiler push constant metadata disagrees with SPIR-V");
     for (const auto id : module.interface) Require(module.variables.contains(id), "entry point interface contains an unknown variable");
     if (mesh) Require(module.primitiveIndices == state.stages.mesh->outputPrimitiveSize, "mesh shader does not export primitive indices");
     if (stage == Stage::Vertex || mesh || evaluation || geometry) Require(module.position, "vertex shader does not export position");
@@ -635,7 +643,9 @@ std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders,
     Require(state.stages.mesh.has_value() == mesh && state.stages.tessellation.has_value() == tessellation, "graphics stage configuration disagrees with its path");
     Require(!state.rectList || (state.stages.path == ShaderPath::Vertex && state.topology == VK_PRIMITIVE_TOPOLOGY_PATCH_LIST && state.cullMode == VK_CULL_MODE_NONE), "invalid rect-list pipeline state");
     const bool geometry = state.stages.path == ShaderPath::Vertex && !state.rectList && shaders.size() == 3u;
-    Require(geometry || shaders.size() == (tessellation || state.rectList ? 4u : 2u), "incorrect graphics stage count");
+    const bool patches = BarycentricPatches(state, shaders) && shaders.back().program != nullptr && shaders.back().program->barycentricEmulation.active;
+    Require(geometry || shaders.size() == (tessellation || state.rectList || patches ? 4u : 2u), "incorrect graphics stage count");
+    if (patches) Require(state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, "emulated barycentrics through tessellation require a triangle list");
     if (geometry) {
         Require(shaders.back().program != nullptr && shaders.back().program->barycentricEmulation.active, "a geometry stage is generated only for emulated barycentrics");
         Require(state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN, "emulated barycentrics require triangle primitives");
@@ -644,7 +654,7 @@ std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders,
     static_cast<void>(AssemblePushConstants(shaders));
     Module previous;
     for (std::size_t i = 0; i < shaders.size(); ++i) {
-        const auto expected = state.rectList ? (i == 0 ? Stage::Vertex : tessStages[i]) : tessellation ? tessStages[i] : geometry && i == 1 ? Stage::Geometry : i == shaders.size() - 1 ? Stage::Fragment : state.stages.path == ShaderPath::Geometry ? Stage::Mesh : Stage::Vertex;
+        const auto expected = state.rectList || patches ? (i == 0 ? Stage::Vertex : tessStages[i]) : tessellation ? tessStages[i] : geometry && i == 1 ? Stage::Geometry : i == shaders.size() - 1 ? Stage::Fragment : state.stages.path == ShaderPath::Geometry ? Stage::Mesh : Stage::Vertex;
         Require(shaders[i].program != nullptr, "missing compiled shader");
         Require(shaders[i].stage == expected, "graphics stage order disagrees");
         for (const auto& binding : shaders[i].program->bindings) Require(binding.descriptorSet == ShaderRecompiler::RuntimeAbi::DescriptorSet || (shaders[i].stage == ShaderRecompiler::ShaderStage::Fragment && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet), "graphics resource uses a descriptor set other than its stage's");
@@ -675,7 +685,7 @@ std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders,
         const auto color = std::find_if(state.colors.begin(), state.colors.end(), [&](const ColorTarget& target) { return target.exportIndex == location; });
         const bool uintExport = color != state.colors.end() && color->uintExport;
         const bool packed = color != state.colors.end() && color->packing != ShaderRecompiler::ColorExportPacking::None;
-        Require(signature == (uintExport || packed ? "vertex:u32x4" : "vertex:f32x4"), packed ? "fragment shader must export uint4 words to its packed 10_11_11 unorm attachments" : uintExport ? "fragment shader must export uint4 colors to its unsigned integer attachments" : "fragment shader must export float4 colors to its attachments");
+        Require(signature == (uintExport || packed ? "vertex:u32x4" : "vertex:f32x4"), packed ? "fragment shader must export uint4 words to its packed color attachments" : uintExport ? "fragment shader must export uint4 colors to its unsigned integer attachments" : "fragment shader must export float4 colors to its attachments");
         locations.insert(location);
     }
     Require((!state.dualSourceBlend && state.reversedBlend == 0) || previous.secondSource == "f32x4", "dual-source blending needs a float4 second color at location 0 index 1");

core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp

@@ -743,6 +743,10 @@ struct DecodedColorFormat {
     ShaderRecompiler::ColorExportPacking packing = ShaderRecompiler::ColorExportPacking::None;
 };
 
+const char* PackedColorTargetName(ShaderRecompiler::ColorExportPacking packing) {
+    return packing == ShaderRecompiler::ColorExportPacking::Float11_11_10 ? "11_11_10 float" : "10_11_11 unorm";
+}
+
 // CB_COLOR_INFO FORMAT / NUMBER_TYPE / COMP_SWAP to a Vulkan attachment format. AMD formats list
 // components from the least significant bits, as Vulkan's non-packed formats do.
 DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number, std::uint32_t swap) {
@@ -753,7 +757,7 @@ DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number,
     };
     const bool alternate = swap == 1;
     const auto single = [&](VkFormat vkFormat, std::uint32_t bytes) { return DecodedColorFormat{vkFormat, bytes, static_cast<std::uint8_t>((0xe4u & ~3u) | swap)}; };
-    if (swap > 1 && format != 1 && format != 2 && format != 4 && format != 10 && format != 12) return fail();
+    if (swap > 1 && format != 1 && format != 2 && format != 4 && format != 7 && format != 10 && format != 12 && format != 16) return fail();
     switch (format) {
         case 1:
             if (number == unorm) return single(VK_FORMAT_R8_UNORM, 1);
@@ -791,6 +795,9 @@ DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number,
             if (swap != 0 || (number != floating && number != unorm)) return fail();
             if (number == unorm) return {VK_FORMAT_R32_UINT, 4, 0xe4u, ShaderRecompiler::ColorExportPacking::Unorm10_11_11};
             return {VK_FORMAT_B10G11R11_UFLOAT_PACK32, 4};
+        case 7:
+            if (swap != 2 || number != floating) return fail();
+            return {VK_FORMAT_R32_UINT, 4, 0xc6u, ShaderRecompiler::ColorExportPacking::Float11_11_10};
         case 9:
             // COLOR_2_10_10_10 keeps red in the low bits, the Vulkan A2B10G10R10 packing.
             if (number != unorm) return fail();
@@ -821,6 +828,9 @@ DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number,
             if (number == floating) return {VK_FORMAT_R32G32B32A32_SFLOAT, 16};
             if (number == uint) return {VK_FORMAT_R32G32B32A32_UINT, 16};
             return fail();
+        case 16:
+            if ((swap != 0 && swap != 2) || number != unorm) return fail();
+            return {VK_FORMAT_R5G6B5_UNORM_PACK16, 2, static_cast<std::uint8_t>(swap == 2 ? 0xe4u : 0xc6u)};
         case 17:
             if (swap != 0 || number != unorm) return fail();
             return {VK_FORMAT_A1R5G5B5_UNORM_PACK16, 2, 0xc6u};
@@ -1350,16 +1360,22 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
             }
         }
         if (color.packing != ShaderRecompiler::ColorExportPacking::None) {
+            const std::string packed = std::string(color.packing == ShaderRecompiler::ColorExportPacking::Float11_11_10 ? "an " : "a ") + PackedColorTargetName(color.packing);
             const auto slotExport = (exportFormat >> (4u * color.exportIndex)) & 0xfu;
-            if (slotExport == 5 || slotExport == 6) throw std::runtime_error("AGC graphics: color export format " + std::to_string(slotExport) + " into a 10_11_11 unorm target is unsupported");
-            Require(!state.blendEnable, "blending into a 10_11_11 unorm color target is unsupported");
-            Require((exportedMask & 7u) == 7u, "partial writes of a 10_11_11 unorm color target are unsupported");
+            if (slotExport == 5 || slotExport == 6) throw std::runtime_error("AGC graphics: color export format " + std::to_string(slotExport) + " into " + packed + " target is unsupported");
+            Require(!state.blendEnable, "blending into " + packed + " color target is unsupported");
+            Require((exportedMask & 7u) == 7u, "partial writes of " + packed + " color target are unsupported");
         }
         if (state.blendEnable && !reversed && (secondSource(state.srcColorBlendFactor) || secondSource(state.dstColorBlendFactor) || secondSource(state.srcAlphaBlendFactor) || secondSource(state.dstAlphaBlendFactor))) {
             const auto addition = [](VkBlendOp op) { return op == VK_BLEND_OP_ADD || op == VK_BLEND_OP_SUBTRACT || op == VK_BLEND_OP_REVERSE_SUBTRACT; };
             Require(addition(state.colorBlendOp) && addition(state.alphaBlendOp), "dual-source blending with a MIN or MAX operation is unsupported");
             result.dualSourceBlend = true;
         }
+        if (const auto info = read(cx, 0x31c + slot * 0xfu); state.blendEnable && (info & 0x8000u) == 0 && ((info >> 8u) & 7u) <= 1u) {
+            char detail[80];
+            std::snprintf(detail, sizeof(detail), " (CB_COLOR_INFO 0x%08x, CB_BLEND%u_CONTROL 0x%08x)", info, slot, blend);
+            throw std::runtime_error(std::string("AGC graphics: blending unclamped normalized color is not implemented") + detail);
+        }
         result.blends[color.exportIndex] = state;
     }
     for (std::uint32_t i = 0; i < 3; ++i) {
@@ -1517,9 +1533,9 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot, bool allo
     Require(color.samples <= 8u, "color targets with more than eight samples are unsupported");
     Require((1u << ((attrib >> 15u) & 3u)) == color.samples, "color targets with fewer fragments than samples (EQAA) are unsupported");
     const auto multisampleInfo = color.samples > 1 ? 0x0c006000u : 0u;
-    // ROUND_MODE (bit 18) only affects unorm rounding. With DCC_ENABLE (bit 28) the target is written
+    // ROUND_MODE (bit 18) only affects unorm rounding and packed float truncation. With DCC_ENABLE (bit 28) the target is written
     if ((info & ~(0x00039f7cu | 0x00040000u | 0x10000000u | 0x00002000u | multisampleInfo)) != 0) throw std::runtime_error("AGC graphics: color compression, DCC, endian conversion, nonstandard rounding or color optimization is unsupported (CB_COLOR_INFO 0x" + [&] { char text[16]; std::snprintf(text, sizeof(text), "%08x", info); return std::string(text); }() + ")");
-    Require((info & 0x8000u) != 0 || number == 7 || number == 4 || number == 5, "unclamped normalized color is unsupported");
+    Require((info & 0x8000u) != 0 || number <= 1u || number == 7 || number == 4 || number == 5, "unclamped normalized color is unsupported");
     const auto view = read(cx, 0x31b + stride);
     Require((view & ~0x3fffffffu) == 0, "reserved CB_COLOR_VIEW bits are set");
     const auto slice = view & 0x1fffu;
@@ -1536,7 +1552,11 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot, bool allo
     Require(((attrib3 >> 24u) & 3u) != 0u || (attrib2 & 0x3fffu) == 0u, "1D color targets taller than one row are unsupported");
     if (volume) {
         color.depth = (attrib3 & 0x1fffu) + 1u;
-        Require(maxMip == 0, "mipmapped 3D color targets are unsupported");
+        if (maxMip != 0) {
+            char message[384];
+            std::snprintf(message, sizeof(message), "mipmapped 3D color targets are unsupported (slot %u, %ux%ux%u, viewMip %u, maxMip %u, slices %u..%u, CB_COLOR_INFO 0x%08x, ATTRIB2 0x%08x, ATTRIB3 0x%08x, VIEW 0x%08x)", slot, ((attrib2 >> 14u) & 0x3fffu) + 1u, (attrib2 & 0x3fffu) + 1u, color.depth, viewMip, maxMip, slice, lastSlice, info, attrib2, attrib3, view);
+            Require(false, message);
+        }
         Require(slice < color.depth && lastSlice >= slice, "the color view slices are outside the 3D surface");
         color.depthSlice = slice;
         color.layers = std::min(lastSlice + 1u, color.depth) - slice;
@@ -1590,8 +1610,9 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot, bool allo
     color.format = decoded.format;
     color.componentMapping = decoded.componentMapping;
     color.packing = decoded.packing;
-    Require(color.packing == ShaderRecompiler::ColorExportPacking::None || (info & 0x10000000u) == 0, "DCC-compressed 10_11_11 unorm color targets are unsupported");
-    Require(color.packing == ShaderRecompiler::ColorExportPacking::None || (info & 0x00040000u) == 0, "truncating (ROUND_MODE) 10_11_11 unorm color targets are unsupported");
+    if (color.packing != ShaderRecompiler::ColorExportPacking::None && (info & 0x10000000u) != 0) throw std::runtime_error("AGC graphics: DCC-compressed " + std::string(PackedColorTargetName(color.packing)) + " color targets are unsupported");
+    Require(color.packing != ShaderRecompiler::ColorExportPacking::Unorm10_11_11 || (info & 0x00040000u) == 0, "truncating (ROUND_MODE) 10_11_11 unorm color targets are unsupported");
+    Require(color.packing != ShaderRecompiler::ColorExportPacking::Float11_11_10 || (info & 0x00040000u) != 0, "rounding (ROUND_MODE 0) 11_11_10 float color targets are unsupported");
     for (std::uint32_t word = 0; word < 2; ++word) {
         const auto clear = find(cx, 0x323 + word + stride);
         color.clearWords[word] = clear == cx.end() ? 0u : clear->second;
@@ -1690,6 +1711,7 @@ std::optional<ColorMetadataPass> DecodeColorMetadataPass(const QueueState& queue
         Require(destination.samples == 1, "CB resolve into a multisampled target is unsupported");
         Require(destination.cmaskAddress == 0, "CB resolve into a fast-clear color target is unsupported (the resolve does not update its CMASK)");
         Require(source.format == destination.format && source.extent.width == destination.extent.width && source.extent.height == destination.extent.height, "CB resolve between different formats or extents is unsupported");
+        if (source.packing != ShaderRecompiler::ColorExportPacking::None) throw std::runtime_error("AGC graphics: CB resolves of " + std::string(PackedColorTargetName(source.packing)) + " color targets are unsupported");
         requireCovered(destination);
         pass.source = source;
         pass.targets.push_back(destination);

core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp

@@ -4,6 +4,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/SlimMutex.hpp"
 #include "prx/libc/include/General.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
@@ -292,7 +293,7 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex
         allocation.allocationSize = requirements.size;
         allocationBytes = requirements.size;
         allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "texture");
-        Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &owned->memory), "vkAllocateMemory texture");
+        owned->memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory texture");
         Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory");
 
         {
@@ -583,7 +584,7 @@ Texture::Texture(const Context& context, const std::shared_ptr<StorageTexture>&
         allocation.allocationSize = requirements.size;
         allocationBytes = requirements.size;
         allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
-        Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &owned->memory), "vkAllocateMemory depth copy");
+        owned->memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory depth copy");
         Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory depth copy");
 
         const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u;
@@ -867,7 +868,13 @@ VkImageView Texture::SrgbView(bool firstLayer, bool baseLevel) const {
 VkImageView Texture::BaseLevelView() const {
     Require(baseLevelInfo.has_value(), "a base-level view needs a mipmapped 1D, 2D or 2D array texture");
     std::lock_guard lock(baseLevelMutex);
-    if (!baseLevelView) Check(context.Function<PFN_vkCreateImageView>("vkCreateImageView")(context.device, &*baseLevelInfo, nullptr, &baseLevelView), "vkCreateImageView base level");
+    if (!baseLevelView) {
+        auto info = *baseLevelInfo;
+        VkImageViewUsageCreateInfo usage{VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO};
+        usage.usage = VK_IMAGE_USAGE_SAMPLED_BIT;
+        info.pNext = &usage;
+        Check(context.Function<PFN_vkCreateImageView>("vkCreateImageView")(context.device, &info, nullptr, &baseLevelView), "vkCreateImageView base level");
+    }
     return baseLevelView;
 }
 
@@ -1144,7 +1151,7 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler,
         allocation.allocationSize = requirements.size;
         memoryBytes = requirements.size;
         allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "storage texture");
-        Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory storage texture");
+        memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory storage texture");
         Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory storage");
     } catch (...) {
         release();
@@ -3179,6 +3186,12 @@ void StorageTexture::reconcilePending() {
     BumpPendingSerial();
 }
 
+bool StorageTexture::Dirty() const {
+    auto& pending = Pending();
+    std::lock_guard lock(pending.mutex);
+    return dirty;
+}
+
 void StorageTexture::Flush() {
     {
         auto& live = Live();

core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp

@@ -144,8 +144,8 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem
     // per-bit XOR equation for the equation family (2); 20-21 override the block extent in elements.
     std::array<std::uint32_t, 22> values{elementBytes, BlockBytesFor(tileMode), tileMode == TextureTileMode::kLinear ? 0u : 1u, retile ? 1u : 0u};
     if (thick) {
-        const auto thickMode = tileMode == TextureTileMode::kStandard4KB ? 0x105u : tileMode == TextureTileMode::kStandard64KB ? 0x109u : tileMode == TextureTileMode::kS64KBX ? 0x119u : tileMode == TextureTileMode::kD64KBX ? 0x11au : 0u;
-        Require(thickMode != 0, "3D textures are only detiled thick from SW_4KB_S, SW_64KB_S, SW_64KB_S_X or SW_64KB_D_X");
+        const auto thickMode = tileMode == TextureTileMode::kStandard4KB ? 0x105u : tileMode == TextureTileMode::kS4KBX ? 0x115u : tileMode == TextureTileMode::kStandard64KB ? 0x109u : tileMode == TextureTileMode::kS64KBX ? 0x119u : tileMode == TextureTileMode::kD64KBX ? 0x11au : 0u;
+        Require(thickMode != 0, "3D textures are only detiled thick from SW_4KB_S, SW_4KB_S_X, SW_64KB_S, SW_64KB_S_X or SW_64KB_D_X");
         const auto* equation = FindTextureSwizzleEquation(thickMode, elementBytes);
         Require(equation != nullptr, "no thick swizzle equation for the element size");
         values[2] = 2u;

core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp

@@ -262,11 +262,12 @@ std::array<std::uint32_t, 3> ThickBlockExtent(TextureTileMode tileMode, std::uin
     Require(std::has_single_bit(bytesPerElement) && bytesPerElement <= 16u, "unsupported bytes per element for thick texture geometry");
     const auto index = static_cast<std::size_t>(std::countr_zero(bytesPerElement));
     switch (tileMode) {
-        case TextureTileMode::kStandard4KB: return {1u << thick4KB[index][0], 1u << thick4KB[index][1], 1u << thick4KB[index][2]};
+        case TextureTileMode::kStandard4KB:
+        case TextureTileMode::kS4KBX: return {1u << thick4KB[index][0], 1u << thick4KB[index][1], 1u << thick4KB[index][2]};
         case TextureTileMode::kStandard64KB:
         case TextureTileMode::kS64KBX:
         case TextureTileMode::kD64KBX: return {1u << thick64KB[index][0], 1u << thick64KB[index][1], 1u << thick64KB[index][2]};
-        default: throw std::runtime_error("AGC graphics: thick 3D textures are only supported in SW_4KB_S, SW_64KB_S, SW_64KB_S_X and SW_64KB_D_X, not in swizzle mode " + std::to_string(EquationSwizzleMode(tileMode)) + " at " + std::to_string(bytesPerElement) + " bytes per element");
+        default: throw std::runtime_error("AGC graphics: thick 3D textures are only supported in SW_4KB_S, SW_4KB_S_X, SW_64KB_S, SW_64KB_S_X and SW_64KB_D_X, not in swizzle mode " + std::to_string(EquationSwizzleMode(tileMode)) + " at " + std::to_string(bytesPerElement) + " bytes per element");
     }
 }
 
@@ -316,7 +317,7 @@ ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, s
     auto firstTailLevel = mipCount;
     if (tileMode != TextureTileMode::kLinear) {
         block = ThickBlockExtent(tileMode, bytesPerElement);
-        blockBytes = tileMode == TextureTileMode::kStandard4KB ? 4096u : 65536u;
+        blockBytes = tileMode == TextureTileMode::kStandard4KB || tileMode == TextureTileMode::kS4KBX ? 4096u : 65536u;
         const auto log2BlockBytes = static_cast<std::uint32_t>(std::countr_zero(blockBytes));
         const auto tailWidth = log2BlockBytes % 3u == 1u ? block[0] >> 1u : block[0];
         const auto tailHeight = log2BlockBytes % 3u == 0u ? block[1] >> 1u : block[1];

core/libs/prx/libSceAgcDriver/tests/BdaContracts.cpp

@@ -1,10 +1,120 @@
 #include "BdaShader.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
 #include "ControlFlow/RequestSerializer.hpp"
 #include "SpirvBackend/SpirvBda.hpp"
+#include <spirv/unified1/GLSL.std.450.h>
+#include <cstring>
+#include <functional>
+#include <initializer_list>
+#include <string>
+#include <vector>
 #include <array>
 
 namespace {
 
+struct ScanWords {
+    std::vector<std::uint32_t> words;
+    void Op(spv::Op opcode, std::initializer_list<std::uint32_t> operands) {
+        words.push_back((static_cast<std::uint32_t>(operands.size() + 1u) << 16u) | static_cast<std::uint32_t>(opcode));
+        words.insert(words.end(), operands.begin(), operands.end());
+    }
+    void OpString(spv::Op opcode, std::initializer_list<std::uint32_t> operands, const char* text) {
+        std::vector<std::uint32_t> packed((std::strlen(text) + 4u) / 4u, 0u);
+        std::memcpy(packed.data(), text, std::strlen(text));
+        words.push_back((static_cast<std::uint32_t>(operands.size() + packed.size() + 1u) << 16u) | static_cast<std::uint32_t>(opcode));
+        words.insert(words.end(), operands.begin(), operands.end());
+        words.insert(words.end(), packed.begin(), packed.end());
+    }
+};
+
+enum ScanId : std::uint32_t {
+    IdVoid = 1, IdVoidFunction, IdUint, IdUlong, IdUintPointer, IdAddress, IdOne, IdDeviceScope, IdNoSemantics, IdPointerPointer, IdPointerFunction, IdBool, IdTrue, IdFloat, IdFloatPointer, IdGlsl, IdFunctionUintPointer, IdUntypedPointer, IdFloatOne, IdMain, IdMainLabel, IdPointer, IdHelper, IdHelperParameter, IdHelperLabel, IdBody,
+};
+
+bool ScanStores(const std::function<void(ScanWords&)>& main, const std::function<void(ScanWords&)>& after = {}) {
+    ScanWords module{{spv::MagicNumber, 0x00010500u, 0u, 200u, 0u}};
+    module.Op(spv::OpCapability, {spv::CapabilityShader});
+    module.Op(spv::OpCapability, {spv::CapabilityInt64});
+    module.Op(spv::OpCapability, {spv::CapabilityPhysicalStorageBufferAddresses});
+    module.OpString(spv::OpExtInstImport, {IdGlsl}, "GLSL.std.450");
+    module.Op(spv::OpMemoryModel, {spv::AddressingModelPhysicalStorageBuffer64, spv::MemoryModelGLSL450});
+    module.OpString(spv::OpEntryPoint, {spv::ExecutionModelGLCompute, IdMain}, "main");
+    module.Op(spv::OpTypeVoid, {IdVoid});
+    module.Op(spv::OpTypeFunction, {IdVoidFunction, IdVoid});
+    module.Op(spv::OpTypeInt, {IdUint, 32u, 0u});
+    module.Op(spv::OpTypeInt, {IdUlong, 64u, 0u});
+    module.Op(spv::OpTypeFloat, {IdFloat, 32u});
+    module.Op(spv::OpTypeBool, {IdBool});
+    module.Op(spv::OpTypePointer, {IdUintPointer, spv::StorageClassPhysicalStorageBuffer, IdUint});
+    module.Op(spv::OpTypePointer, {IdPointerPointer, spv::StorageClassPhysicalStorageBuffer, IdUintPointer});
+    module.Op(spv::OpTypePointer, {IdFloatPointer, spv::StorageClassPhysicalStorageBuffer, IdFloat});
+    module.Op(spv::OpTypePointer, {IdPointerFunction, spv::StorageClassFunction, IdUint});
+    module.Op(spv::OpTypeUntypedPointerKHR, {IdUntypedPointer, spv::StorageClassPhysicalStorageBuffer});
+    module.Op(spv::OpTypeFunction, {IdFunctionUintPointer, IdVoid, IdUintPointer});
+    module.Op(spv::OpConstant, {IdUlong, IdAddress, 0x1000u, 0u});
+    module.Op(spv::OpConstant, {IdUint, IdOne, 1u});
+    module.Op(spv::OpConstant, {IdUint, IdDeviceScope, spv::ScopeDevice});
+    module.Op(spv::OpConstant, {IdUint, IdNoSemantics, 0u});
+    module.Op(spv::OpConstant, {IdFloat, IdFloatOne, 0x3f800000u});
+    module.Op(spv::OpConstantTrue, {IdBool, IdTrue});
+    module.Op(spv::OpFunction, {IdVoid, IdMain, spv::FunctionControlMaskNone, IdVoidFunction});
+    module.Op(spv::OpLabel, {IdMainLabel});
+    module.Op(spv::OpConvertUToPtr, {IdUintPointer, IdPointer, IdAddress});
+    main(module);
+    module.Op(spv::OpReturn, {});
+    module.Op(spv::OpFunctionEnd, {});
+    if (after) after(module);
+    return AgcDriver::Graphics::SpirvStoresThroughAddresses(module.words);
+}
+
+void RunAddressStoreScanTests() {
+    using AgcDriver::Graphics::Require;
+    Require(!ScanStores([](ScanWords& m) { m.Op(spv::OpLoad, {IdUint, IdBody, IdPointer}); }), "store scan: a load through a physical pointer was taken for a store");
+    Require(!ScanStores([](ScanWords& m) { m.Op(spv::OpAtomicLoad, {IdUint, IdBody, IdPointer, IdDeviceScope, IdNoSemantics}); }), "store scan: an atomic load was taken for a store");
+    Require(ScanStores([](ScanWords& m) { m.Op(spv::OpStore, {IdPointer, IdOne}); }), "store scan: a store was missed");
+    Require(ScanStores([](ScanWords& m) { m.Op(spv::OpAtomicStore, {IdPointer, IdDeviceScope, IdNoSemantics, IdOne}); }), "store scan: an atomic store was missed");
+    for (const auto opcode : {spv::OpAtomicExchange, spv::OpAtomicIAdd, spv::OpAtomicISub, spv::OpAtomicSMin, spv::OpAtomicUMin, spv::OpAtomicSMax, spv::OpAtomicUMax, spv::OpAtomicAnd, spv::OpAtomicOr, spv::OpAtomicXor}) {
+        Require(ScanStores([opcode](ScanWords& m) { m.Op(opcode, {IdUint, IdBody, IdPointer, IdDeviceScope, IdNoSemantics, IdOne}); }), "store scan: atomic opcode " + std::to_string(static_cast<std::uint32_t>(opcode)) + " was missed");
+    }
+    Require(ScanStores([](ScanWords& m) { m.Op(spv::OpAtomicIIncrement, {IdUint, IdBody, IdPointer, IdDeviceScope, IdNoSemantics}); }), "store scan: an atomic increment was missed");
+    Require(ScanStores([](ScanWords& m) { m.Op(spv::OpAtomicCompareExchange, {IdUint, IdBody, IdPointer, IdDeviceScope, IdNoSemantics, IdNoSemantics, IdOne, IdOne}); }), "store scan: an atomic compare-exchange was missed");
+    Require(ScanStores([](ScanWords& m) {
+        m.Op(spv::OpConvertUToPtr, {IdPointerPointer, IdBody, IdAddress});
+        m.Op(spv::OpLoad, {IdUintPointer, IdBody + 1u, IdBody});
+        m.Op(spv::OpStore, {IdBody + 1u, IdOne});
+    }), "store scan: a store through a pointer loaded from guest memory was missed");
+    Require(ScanStores([](ScanWords& m) {
+        m.Op(spv::OpSelect, {IdUintPointer, IdBody, IdTrue, IdPointer, IdPointer});
+        m.Op(spv::OpStore, {IdBody, IdOne});
+    }), "store scan: a store through a selected pointer was missed");
+    Require(ScanStores([](ScanWords& m) { m.Op(spv::OpFunctionCall, {IdVoid, IdBody, IdHelper, IdPointer}); }, [](ScanWords& m) {
+        m.Op(spv::OpFunction, {IdVoid, IdHelper, spv::FunctionControlMaskNone, IdFunctionUintPointer});
+        m.Op(spv::OpFunctionParameter, {IdUintPointer, IdHelperParameter});
+        m.Op(spv::OpLabel, {IdHelperLabel});
+        m.Op(spv::OpStore, {IdHelperParameter, IdOne});
+        m.Op(spv::OpReturn, {});
+        m.Op(spv::OpFunctionEnd, {});
+    }), "store scan: a store through a pointer parameter of a called function was missed");
+    Require(!ScanStores([](ScanWords& m) {
+        m.Op(spv::OpVariable, {IdPointerFunction, IdBody, spv::StorageClassFunction});
+        m.Op(spv::OpCopyMemory, {IdBody, IdPointer});
+    }), "store scan: a copy out of guest memory was taken for a store");
+    Require(ScanStores([](ScanWords& m) {
+        m.Op(spv::OpVariable, {IdPointerFunction, IdBody, spv::StorageClassFunction});
+        m.Op(spv::OpCopyMemory, {IdPointer, IdBody});
+    }), "store scan: a copy into guest memory was missed");
+    Require(ScanStores([](ScanWords& m) {
+        m.Op(spv::OpConvertUToPtr, {IdUntypedPointer, IdBody, IdAddress});
+        m.Op(spv::OpStore, {IdBody, IdOne});
+    }), "store scan: a store through an untyped physical pointer was missed");
+    Require(ScanStores([](ScanWords& m) {
+        m.Op(spv::OpConvertUToPtr, {IdFloatPointer, IdBody, IdAddress});
+        m.Op(spv::OpExtInst, {IdFloat, IdBody + 1u, IdGlsl, GLSLstd450Modf, IdFloatOne, IdBody});
+    }), "store scan: an extended instruction writing through a physical pointer was missed");
+    Require(ScanStores([](ScanWords& m) { m.Op(spv::OpCooperativeMatrixStoreKHR, {IdPointer, IdOne, IdNoSemantics}); }), "store scan: a cooperative matrix store was missed");
+}
+
+
 template<typename TAction>
 void reject(TAction action, const char* expected) {
     try { action(); }
@@ -19,6 +129,21 @@ void reject(TAction action, const char* expected) {
 
 void RunBdaContractTests() {
     using namespace ShaderRecompiler;
+    using AgcDriver::Graphics::SpirvStoresThroughAddresses;
+    RunAddressStoreScanTests();
+    AgcDriver::Graphics::Require(!SpirvStoresThroughAddresses(MakeBdaTestShader(0x1000, 32)), "a BDA read was taken for a store through an address");
+    AgcDriver::Graphics::Require(!SpirvStoresThroughAddresses(MakeBdaDwordReadTestShader(0x1000, 4, false, false)), "BDA dword reads were taken for a store through an address");
+    AgcDriver::Graphics::Require(SpirvStoresThroughAddresses(MakeBdaStoreTestShader(0x1000, 7, 32, false)), "a BDA store was not seen");
+    AgcDriver::Graphics::Require(SpirvStoresThroughAddresses(MakeBdaWriteTestShader(0x1000, 7, 32, false)), "a BDA write was not seen");
+    const std::array<std::uint32_t, 2> stored{7u, 9u};
+    AgcDriver::Graphics::Require(SpirvStoresThroughAddresses(MakeBdaDwordWriteTestShader(0x1000, static_cast<std::uint32_t>(stored.size()), stored.data())), "BDA dword writes were not seen");
+    const std::array<std::uint32_t, 5> headerOnly{spv::MagicNumber, 0x00010300u, 0u, 1u, 0u};
+    AgcDriver::Graphics::Require(!SpirvStoresThroughAddresses(headerOnly), "a module without physical pointers was taken for one storing through addresses");
+    const std::array<std::uint32_t, 2> truncated{spv::MagicNumber, 0u};
+    AgcDriver::Graphics::Require(SpirvStoresThroughAddresses(truncated), "an unreadable module was taken for one without stores");
+    auto unparsed = MakeBdaStoreTestShader(0x1000, 7, 32, false);
+    unparsed[5] |= 0xffff0000u;
+    AgcDriver::Graphics::Require(SpirvStoresThroughAddresses(unparsed), "a module with an instruction past its end was taken for one without stores");
     const std::array<std::uint32_t, 3> capabilities{spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess};
     const std::array<std::string_view, 2> extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"};
     SpirvTargetOptions target{0x00401000u, 0x00010300u, 32, BdaAbi::Version, capabilities, extensions};

core/libs/prx/libSceAgcDriver/tests/BdaDevice.cpp

@@ -102,6 +102,7 @@ public:
             context.bufferDeviceAddress = true;
             context.Function<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(context.device, family, 0, &context.queue);
             VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
+            pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
             pool.queueFamilyIndex = family;
             Check(context.Function<PFN_vkCreateCommandPool>("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool");
         } catch (...) {

core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp

@@ -550,6 +550,137 @@ void importJoinTests(const Context& context) {
     ::operator delete(block, std::align_val_t{window});
 }
 
+class ImportBudgetRanges {
+public:
+    static constexpr std::size_t piece = 65536;
+    static constexpr std::size_t bytes = 3 * piece;
+
+    explicit ImportBudgetRanges(const Context& context) : importing(context) {
+        block = static_cast<std::byte*>(::operator new(4 * bytes, std::align_val_t{piece}));
+        importing.hostImportAlignment = piece;
+        ClearHostImports(importing.device);
+        SetHostImportWindow(2 * piece);
+        SetHostImportRangeWindow(0);
+        SetHostImportBudget(2 * bytes);
+        GuestAllocations::Mutation mutation;
+        for (std::size_t index = 0; index < 4; ++index) {
+            std::memset(block + index * bytes, static_cast<int>(0x31 + index), bytes);
+            mutation.Add(block + index * bytes, bytes, true, false, true);
+        }
+    }
+
+    ~ImportBudgetRanges() {
+        ClearHostImports(importing.device);
+        {
+            GuestAllocations::Mutation mutation;
+            for (std::size_t index = 0; index < 4; ++index) mutation.Remove(block + index * bytes);
+        }
+        SetHostImportBudget(0);
+        SetHostImportRangeWindow(std::uint64_t{64} << 20u);
+        SetHostImportWindow(0);
+        ::operator delete(block, std::align_val_t{piece});
+    }
+
+    ImportBudgetRanges(const ImportBudgetRanges&) = delete;
+    ImportBudgetRanges& operator=(const ImportBudgetRanges&) = delete;
+
+    std::uintptr_t Address(std::size_t index) const { return reinterpret_cast<std::uintptr_t>(block + index * bytes); }
+
+    void CheckUnchanged() const {
+        for (std::size_t index = 0; index < 4; ++index) {
+            const auto expected = static_cast<std::byte>(0x31 + index);
+            Require(std::all_of(block + index * bytes, block + (index + 1) * bytes, [expected](std::byte value) { return value == expected; }), "import budget probes changed guest bytes");
+        }
+    }
+
+    Context importing;
+
+private:
+    std::byte* block = nullptr;
+};
+
+void importCoverageHintBudgetTests(const Context& context, const BdaTestAccess& access) {
+    for (const bool acquired : {true, false}) {
+        ImportBudgetRanges ranges(context);
+        const auto a = ranges.Address(0);
+        const auto c = ranges.Address(2);
+        const auto d = ranges.Address(3);
+        const auto* first = HostImportFor(ranges.importing, a, 16);
+        Require(first != nullptr && first->base == a && first->bytes == ranges.bytes, "coverage budget fixture did not import its first range");
+        const auto firstBuffer = first->buffer;
+        const auto firstSerial = HostImportSerial(ranges.importing, a, 16, false);
+        for (int use = 0; use < 300; ++use) Require(HostImportFor(ranges.importing, d, 16) != nullptr, "coverage budget fixture did not import its hot range");
+        const auto hotSerial = HostImportSerial(ranges.importing, d, 16, false);
+        Require(firstSerial != 0 && hotSerial != 0 && HostImportLiveBytes() == 2 * ranges.bytes, "coverage budget fixture has unexpected import ownership or usage");
+        {
+            GuestBufferMemory memory(ranges.importing);
+            memory.AddReadable(c, 32);
+            memory.UploadPrepare(false);
+            if (acquired) {
+                const auto* held = HostImportFor(ranges.importing, a, 16);
+                Require(held != nullptr && held->buffer == firstBuffer, "a genuine import acquisition replaced the retained range");
+            } else {
+                Require(HostImportCovers(ranges.importing, a, 16), "a coverage hint missed the existing range");
+            }
+            memory.UploadFinish(false);
+            std::uint32_t adjustment = 0;
+            const auto view = memory.Descriptor(c, 32, adjustment);
+            Require(view.range == 32 + adjustment, "a disjoint descriptor has an incorrect extent");
+            Require(HostImportSerial(ranges.importing, d, 16, false) == hotSerial && HostImportLiveBytes() == 2 * ranges.bytes, "a coverage probe displaced the hot import or changed the budget");
+            if (acquired) {
+                Require(view.buffer != firstBuffer, "a disjoint descriptor uses the retained range's import");
+                Require(HostImportSerial(ranges.importing, a, 16, false) == firstSerial && HostImportSerial(ranges.importing, c, 16, false) == 0, "a genuine import acquisition did not protect its buffer from eviction");
+                Require(memory.CopiedBytes() == 32, "a disjoint descriptor at the budget was not copied while an import was held");
+                const auto copied = access.bytes(view.buffer);
+                const auto offset = static_cast<std::size_t>(view.offset + adjustment);
+                Require(offset + 32 <= copied.size() && std::memcmp(copied.data() + offset, reinterpret_cast<const void*>(c), 32) == 0, "the descriptor copied behind a held import lost its guest bytes");
+            } else {
+                const auto importedSerial = HostImportSerial(ranges.importing, c, 16, false);
+                Require(HostImportSerial(ranges.importing, a, 16, false) == 0 && importedSerial != 0 && importedSerial != firstSerial, "a coverage hint prevented eviction of an unused import");
+                const auto* imported = ExistingHostImportFor(ranges.importing, c, 32);
+                Require(imported != nullptr && imported->base == c && imported->bytes == ranges.bytes && view.buffer == imported->buffer && view.offset + adjustment == c - imported->base, "a descriptor does not bind the import obtained after a coverage hint");
+                Require(memory.CopiedBytes() == 0, "a coverage hint forced the disjoint imported descriptor to copy");
+            }
+            memory.WriteBack();
+        }
+        ranges.CheckUnchanged();
+    }
+}
+
+void failedImportJoinProbeBudgetTests(const Context& context, const BdaTestAccess& access) {
+    ImportBudgetRanges ranges(context);
+    const auto a = ranges.Address(0);
+    const auto crossing = ranges.Address(1) - 16;
+    const auto c = ranges.Address(2);
+    const auto d = ranges.Address(3);
+    const auto* first = HostImportFor(ranges.importing, a, 16);
+    Require(first != nullptr && first->base == a && first->bytes == ranges.bytes, "failed join budget fixture did not import its first range");
+    const auto firstSerial = HostImportSerial(ranges.importing, a, 16, false);
+    for (int use = 0; use < 300; ++use) Require(HostImportFor(ranges.importing, d, 16) != nullptr, "failed join budget fixture did not import its hot range");
+    const auto hotSerial = HostImportSerial(ranges.importing, d, 16, false);
+    Require(firstSerial != 0 && hotSerial != 0 && HostImportLiveBytes() == 2 * ranges.bytes, "failed join budget fixture has unexpected import ownership or usage");
+    {
+        GuestBufferMemory memory(ranges.importing);
+        memory.AddReadable(crossing, 32);
+        memory.AddReadable(c, 32);
+        memory.Upload(false);
+        std::uint32_t adjustment = 0;
+        const auto copiedView = memory.Descriptor(crossing, 32, adjustment);
+        const auto copied = access.bytes(copiedView.buffer);
+        const auto offset = static_cast<std::size_t>(copiedView.offset + adjustment);
+        Require(copiedView.range == 32 + adjustment && offset + 32 <= copied.size() && std::memcmp(copied.data() + offset, reinterpret_cast<const void*>(crossing), 32) == 0, "the refused joined descriptor lost either side's guest bytes");
+        Require(HostImportSerial(ranges.importing, crossing, 32, false) == 0, "a join over the available budget unexpectedly produced an import");
+        const auto importedSerial = HostImportSerial(ranges.importing, c, 16, false);
+        Require(HostImportSerial(ranges.importing, a, 16, false) == 0 && importedSerial != 0 && importedSerial != firstSerial, "a failed join endpoint probe prevented eviction of an unused import");
+        const auto* imported = ExistingHostImportFor(ranges.importing, c, 32);
+        const auto view = memory.Descriptor(c, 32, adjustment);
+        Require(imported != nullptr && imported->base == c && imported->bytes == ranges.bytes && view.buffer == imported->buffer && view.offset + adjustment == c - imported->base && view.range == 32 + adjustment, "the descriptor after a failed join does not bind its complete import");
+        Require(memory.CopiedBytes() == 32 && HostImportLiveBytes() == 2 * ranges.bytes && HostImportSerial(ranges.importing, d, 16, false) == hotSerial, "a failed join forced a second copy or displaced the hot import");
+        memory.WriteBack();
+    }
+    ranges.CheckUnchanged();
+}
+
 // One registered range of `cells` import windows of `window` bytes, with the window and the import budget set for the
 // test and reset after it.
 class WindowedRange {
@@ -1934,6 +2065,8 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
     spaceWriteMergedPastImportTests(context, access);
     spaceReadOverWritableImportTableTests(context);
     importJoinTests(context);
+    importCoverageHintBudgetTests(context, access);
+    failedImportJoinProbeBudgetTests(context, access);
     importWholeRangeOverWindowTests(context);
     importRefusedWholeRangeTests(context);
     importNeighbouringEdgesTests(context);

core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp

@@ -3,6 +3,7 @@
 #include "AllocationFailure.hpp"
 #include <algorithm>
 #include <cstdint>
+#include <cstdlib>
 #include <cstring>
 #include <cstdio>
 #include <deque>
@@ -50,6 +51,8 @@ struct MockDevice {
     VkResult pressureResult = VK_ERROR_OUT_OF_DEVICE_MEMORY;
     std::uint64_t attempts = 0;
     VkResult mapResult = VK_SUCCESS;
+    VkResult bindResult = VK_SUCCESS;
+    unsigned binds = 0;
     unsigned maps = 0;
     unsigned unmaps = 0;
 };
@@ -90,6 +93,8 @@ VKAPI_ATTR VkResult VKAPI_CALL mockAllocateMemory(VkDevice, const VkMemoryAlloca
 }
 
 VKAPI_ATTR VkResult VKAPI_CALL mockBindBufferMemory(VkDevice, VkBuffer buffer, VkDeviceMemory memory, VkDeviceSize offset) {
+    ++mock.binds;
+    if (mock.bindResult != VK_SUCCESS) return mock.bindResult;
     if (offset + mock.sizes.at(buffer) > mock.memoryBytes.at(memory)) return VK_ERROR_OUT_OF_DEVICE_MEMORY;
     mock.bound[buffer] = {memory, offset};
     return VK_SUCCESS;
@@ -222,6 +227,31 @@ void SizeClassesAndDirections() {
     Expect(mock.sizes.at(writeBack.Handle()) == 10 * MiB + 512 * 1024, "a reused device buffer kept the VkBuffer of the earlier size");
 }
 
+void SmallDeviceRetention() {
+    mock = MockDevice{};
+    {
+        auto context = mockContext();
+        constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
+        const auto deviceBudget = BufferPool::DeviceBudget(context.memory);
+        const auto bytes = static_cast<std::size_t>((deviceBudget != 0 ? deviceBudget : 512 * MiB) / 2) + MiB;
+        VkDeviceMemory small = VK_NULL_HANDLE;
+        {
+            DeviceBuffer buffer(context, 256, usage);
+            small = MemoryOf(buffer.Handle());
+        }
+        std::vector<std::unique_ptr<DeviceBuffer>> large;
+        for (int i = 0; i < 2; ++i) large.push_back(std::make_unique<DeviceBuffer>(context, bytes, usage));
+        large.clear();
+        Expect(mock.frees != 0, "large device buffers did not exercise pool budget eviction");
+        const auto made = mock.allocations;
+        const bool isolated = std::getenv("APS5_BUFFER_POOL_SHARED") == nullptr;
+        DeviceBuffer again(context, 256, usage);
+        Expect((MemoryOf(again.Handle()) == small) == isolated, "small device buffer retention did not follow the separate or shared pool budget");
+        Expect(mock.allocations == made + (isolated ? 0 : 1), "large device buffer churn changed small buffer allocation reuse");
+    }
+    Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "device pool tier retention leaked Vulkan objects");
+}
+
 void LargerClass() {
     mock = MockDevice{};
     auto context = mockContext();
@@ -568,6 +598,101 @@ void AllocationFailure(VkResult initial, VkResult final, bool cached) {
     Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "failure cleanup leaked a resource");
 }
 
+struct ReclaimMock {
+    VkDeviceMemory external = VK_NULL_HANDLE;
+    VkDeviceSize externalBytes = 0;
+    std::uint64_t requests = 0;
+    VkDeviceSize asked = 0;
+};
+
+ReclaimMock reclaimMock;
+
+VkDeviceMemory ExternalAllocation(VkDeviceSize bytes) {
+    VkMemoryAllocateInfo info{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
+    info.allocationSize = bytes;
+    VkDeviceMemory memory = VK_NULL_HANDLE;
+    Expect(mockAllocateMemory(VK_NULL_HANDLE, &info, nullptr, &memory) == VK_SUCCESS, "the external allocation of the reclaim test failed");
+    reclaimMock.external = memory;
+    reclaimMock.externalBytes = bytes;
+    return memory;
+}
+
+VkDeviceSize mockCacheReclaim(VkDeviceSize bytes) {
+    ++reclaimMock.requests;
+    reclaimMock.asked = bytes;
+    if (reclaimMock.external == VK_NULL_HANDLE) return 0;
+    mockFreeMemory(VK_NULL_HANDLE, reclaimMock.external, nullptr);
+    reclaimMock.external = VK_NULL_HANDLE;
+    return reclaimMock.externalBytes;
+}
+
+void CacheReclaimRecovery() {
+    Expect(BufferPool::ReclaimTarget(4 * MiB) == 260 * MiB, "a 4 MiB request does not ask the caches for 256 MiB on top of itself");
+    Expect(BufferPool::ReclaimTarget(512 * MiB) == 1024 * MiB, "a 512 MiB request does not ask the caches for twice its size");
+    constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
+    mock = MockDevice{};
+    reclaimMock = ReclaimMock{};
+    {
+        auto context = mockContext();
+        context.cacheReclaim = mockCacheReclaim;
+        { DeviceBuffer idle(context, 2 * MiB, usage); }
+        ExternalAllocation(10 * MiB);
+        mock.limit = 12 * MiB;
+        const auto before = mock.attempts;
+        try {
+            DeviceBuffer recovered(context, 4 * MiB, usage);
+            Expect(mock.attempts == before + 3, "an allocation the pool's own slots could not satisfy did not retry once after the trim and once after the cache release");
+            Expect(reclaimMock.requests == 1 && reclaimMock.asked == 260 * MiB, "the caches were not asked once for the request plus its reserve");
+            Expect(mock.frees == 2 && mock.liveBytes == 4 * MiB, "the trim or the cache release did not free its memory");
+        } catch (const std::exception& error) {
+            Expect(false, std::string("the cache release did not permit recovery: ") + error.what());
+        }
+    }
+    Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "cache release recovery leaked a resource");
+    mock = MockDevice{};
+    reclaimMock = ReclaimMock{};
+    {
+        auto context = mockContext();
+        context.cacheReclaim = mockCacheReclaim;
+        ExternalAllocation(10 * MiB);
+        mock.limit = 12 * MiB;
+        const auto before = mock.attempts;
+        try {
+            DeviceBuffer recovered(context, 4 * MiB, usage);
+            Expect(mock.attempts == before + 2 && reclaimMock.requests == 1, "an empty pool did not go straight to the cache release and retry once");
+        } catch (const std::exception& error) {
+            Expect(false, std::string("the cache release alone did not permit recovery: ") + error.what());
+        }
+    }
+    Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "cache release recovery with an empty pool leaked a resource");
+    mock = MockDevice{};
+    reclaimMock = ReclaimMock{};
+    {
+        auto context = mockContext();
+        context.cacheReclaim = mockCacheReclaim;
+        mock.limit = 2 * MiB;
+        const auto before = mock.attempts;
+        std::string failure;
+        try {
+            DeviceBuffer buffer(context, 4 * MiB, usage);
+        } catch (const std::exception& error) {
+            failure = error.what();
+        }
+        Expect(reclaimMock.requests == 1 && mock.attempts == before + 1, "a cache release that freed nothing was retried after");
+        Expect(failure.find("Vulkan result -2") != std::string::npos && failure.find("released=0") != std::string::npos, "the failure after an empty cache release lost its result or the released bytes");
+        mock.results = {VK_ERROR_DEVICE_LOST};
+        mock.limit = 0;
+        failure.clear();
+        try {
+            DeviceBuffer buffer(context, 4 * MiB, usage);
+        } catch (const std::exception& error) {
+            failure = error.what();
+        }
+        Expect(reclaimMock.requests == 1 && failure.find("Vulkan result -4") != std::string::npos, "a device loss asked the caches for memory");
+    }
+    Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "the failed cache release leaked a resource");
+}
+
 void SuccessfulAllocationKeepsPool() {
     mock = MockDevice{};
     {
@@ -600,18 +725,55 @@ void PartialSlabDoesNotRetry() {
     Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "partial slab failure leaked a resource");
 }
 
+void DeviceBindingFailures() {
+    constexpr auto bytes = 3 * MiB;
+    constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
+    for (const auto result : {VK_ERROR_OUT_OF_HOST_MEMORY, VK_ERROR_OUT_OF_DEVICE_MEMORY}) {
+        mock = MockDevice{};
+        {
+            auto context = mockContext();
+            mock.bindResult = result;
+            bool threw = false;
+            try {
+                DeviceBuffer failed(context, bytes, usage);
+            } catch (const std::runtime_error& error) {
+                threw = std::string(error.what()).find("Vulkan result " + std::to_string(result)) != std::string::npos;
+            }
+            const auto retained = mock.liveBytes;
+            Expect(threw && mock.binds == 1, "a failed device buffer bind lost its error");
+            Expect(retained == 0 && mock.liveBuffers.empty(), "a failed device buffer was retained for reuse");
+            mock.bindResult = VK_SUCCESS;
+            unsigned invalid = 0;
+            for (unsigned retry = 0; retry < 3; ++retry) {
+                DeviceBuffer recovered(context, bytes, usage);
+                if (!mock.bound.contains(recovered.Handle())) ++invalid;
+            }
+            Expect(invalid == 0, "a retry reused an unbound device buffer");
+            Expect(mock.binds == 2 && mock.allocations == 2, "successfully bound device buffers stopped being reused");
+            std::printf("device bind result=%d retained=%llu invalid retries=%u binds=%u\n", static_cast<int>(result), static_cast<unsigned long long>(retained), invalid, mock.binds);
+        }
+        Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "device bind recovery leaked a resource");
+    }
 }
 
-int main() {
+}
+
+int main(int argc, char** argv) {
     try {
+        if (argc == 2 && std::string_view(argv[1]) == "--small-device-retention") {
+            SmallDeviceRetention();
+            return failures == 0 ? 0 : 1;
+        }
         SizeClassesAndDirections();
         LargerClass();
         KeptUntilFence();
+        SmallDeviceRetention();
         Budget();
         AddressAndHostUnchanged();
         SmallDeviceClasses();
         SlabMetadataFailure();
         SlabMapFailure();
+        DeviceBindingFailures();
         Slabs();
         for (const auto result : {VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_OUT_OF_HOST_MEMORY}) {
             AllocationRecovery(result, false);
@@ -625,6 +787,7 @@ int main() {
         AllocationFailure(VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_DEVICE_LOST, true);
         SuccessfulAllocationKeepsPool();
         PartialSlabDoesNotRetry();
+        CacheReclaimRecovery();
         HostPlacement();
     } catch (const std::exception& error) {
         std::fprintf(stderr, "FAIL: %s\n", error.what());

core/libs/prx/libSceAgcDriver/tests/CommandBatch.cpp

@@ -0,0 +1,20 @@
+#include "CommandBatchFailure.hpp"
+#include "AllocationFailure.hpp"
+#include <cstdint>
+#include <cstdio>
+#include <exception>
+
+int main() {
+    try {
+        AgcDriver::Graphics::Context context{};
+        context.device = reinterpret_cast<VkDevice>(std::uintptr_t{1});
+        context.queue = reinterpret_cast<VkQueue>(std::uintptr_t{2});
+        context.pool = reinterpret_cast<VkCommandPool>(std::uintptr_t{3});
+        AgcDriver::Tests::RunCommandBatchFailureTests(context, false);
+        return 0;
+    } catch (const std::exception& error) {
+        AgcDriver::Tests::SetAllocationFailureCountdown(-1);
+        std::fprintf(stderr, "%s\n", error.what());
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/CommandBatchFailure.cpp

@@ -0,0 +1,206 @@
+#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
+#include "AllocationFailure.hpp"
+#include "CommandBatchFailure.hpp"
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <exception>
+#include <memory>
+#include <mutex>
+#include <new>
+#include <string>
+
+namespace {
+
+using namespace AgcDriver::Graphics;
+using AgcDriver::Tests::SetAllocationFailureCountdown;
+
+enum class Fault { None, Timing, End, Submit };
+
+struct Probe {
+    PFN_vkGetDeviceProcAddr real = nullptr;
+    Fault fault = Fault::None;
+    VkFence fence = VK_NULL_HANDLE;
+    bool pending = false;
+    unsigned submitted = 0;
+    unsigned queries = 0;
+    unsigned idleWaits = 0;
+    unsigned fenceWaits = 0;
+    unsigned premature = 0;
+    unsigned safetyWaits = 0;
+    unsigned nativePending = 0;
+    unsigned freed = 0;
+    unsigned destroyed = 0;
+};
+
+Probe probe;
+VkDevice activeDevice = VK_NULL_HANDLE;
+VkQueue activeQueue = VK_NULL_HANDLE;
+
+void GuardRelease(VkDevice device, VkQueue queue) {
+    if (!probe.pending) return;
+    ++probe.premature;
+    ++probe.safetyWaits;
+    if (probe.real != nullptr) {
+        const auto status = reinterpret_cast<PFN_vkGetFenceStatus>(probe.real(device, "vkGetFenceStatus"))(device, probe.fence);
+        if (status == VK_NOT_READY) ++probe.nativePending;
+        else if (status != VK_SUCCESS) std::terminate();
+        const auto wait = reinterpret_cast<PFN_vkQueueWaitIdle>(probe.real(device, "vkQueueWaitIdle"));
+        if (wait(queue) != VK_SUCCESS) std::terminate();
+    }
+    probe.pending = false;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL AllocateCommands(VkDevice, const VkCommandBufferAllocateInfo*, VkCommandBuffer* commands) {
+    *commands = reinterpret_cast<VkCommandBuffer>(std::uintptr_t{4});
+    return VK_SUCCESS;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL CreateFence(VkDevice, const VkFenceCreateInfo*, const VkAllocationCallbacks*, VkFence* fence) {
+    *fence = reinterpret_cast<VkFence>(std::uintptr_t{5});
+    return VK_SUCCESS;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL BeginCommands(VkCommandBuffer, const VkCommandBufferBeginInfo*) { return VK_SUCCESS; }
+
+VKAPI_ATTR VkResult VKAPI_CALL EndCommands(VkCommandBuffer commands) {
+    if (probe.fault == Fault::End) return VK_ERROR_OUT_OF_HOST_MEMORY;
+    if (probe.real == nullptr) return VK_SUCCESS;
+    return reinterpret_cast<PFN_vkEndCommandBuffer>(probe.real(activeDevice, "vkEndCommandBuffer"))(commands);
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL Submit(VkQueue queue, std::uint32_t count, const VkSubmitInfo* submissions, VkFence fence) {
+    if (probe.fault == Fault::Submit) return VK_ERROR_OUT_OF_HOST_MEMORY;
+    const auto result = probe.real != nullptr ? reinterpret_cast<PFN_vkQueueSubmit>(probe.real(activeDevice, "vkQueueSubmit"))(queue, count, submissions, fence) : VK_SUCCESS;
+    if (result == VK_SUCCESS) {
+        ++probe.submitted;
+        probe.pending = true;
+        probe.fence = fence;
+        if (probe.fault == Fault::Timing) SetAllocationFailureCountdown(0);
+    }
+    return result;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL FenceStatus(VkDevice device, VkFence fence) {
+    ++probe.queries;
+    const auto result = probe.real != nullptr ? reinterpret_cast<PFN_vkGetFenceStatus>(probe.real(device, "vkGetFenceStatus"))(device, fence) : VK_NOT_READY;
+    if (result == VK_SUCCESS) probe.pending = false;
+    return result;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL QueueIdle(VkQueue queue) {
+    ++probe.idleWaits;
+    const auto result = probe.real != nullptr ? reinterpret_cast<PFN_vkQueueWaitIdle>(probe.real(activeDevice, "vkQueueWaitIdle"))(queue) : VK_SUCCESS;
+    if (result == VK_SUCCESS) probe.pending = false;
+    return result;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL WaitFences(VkDevice device, std::uint32_t count, const VkFence* fences, VkBool32 all, std::uint64_t timeout) {
+    ++probe.fenceWaits;
+    const auto result = probe.real != nullptr ? reinterpret_cast<PFN_vkWaitForFences>(probe.real(device, "vkWaitForFences"))(device, count, fences, all, timeout) : VK_SUCCESS;
+    if (result == VK_SUCCESS) probe.pending = false;
+    return result;
+}
+
+VKAPI_ATTR void VKAPI_CALL FreeCommands(VkDevice device, VkCommandPool pool, std::uint32_t count, const VkCommandBuffer* commands) {
+    GuardRelease(device, activeQueue);
+    probe.freed += count;
+    if (probe.real != nullptr) reinterpret_cast<PFN_vkFreeCommandBuffers>(probe.real(device, "vkFreeCommandBuffers"))(device, pool, count, commands);
+}
+
+VKAPI_ATTR void VKAPI_CALL DestroyFence(VkDevice device, VkFence fence, const VkAllocationCallbacks* allocator) {
+    GuardRelease(device, activeQueue);
+    ++probe.destroyed;
+    if (probe.real != nullptr) reinterpret_cast<PFN_vkDestroyFence>(probe.real(device, "vkDestroyFence"))(device, fence, allocator);
+}
+
+VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL DeviceProc(VkDevice device, const char* name) {
+    if (std::strcmp(name, "vkEndCommandBuffer") == 0) return reinterpret_cast<PFN_vkVoidFunction>(EndCommands);
+    if (std::strcmp(name, "vkQueueSubmit") == 0) return reinterpret_cast<PFN_vkVoidFunction>(Submit);
+    if (std::strcmp(name, "vkGetFenceStatus") == 0) return reinterpret_cast<PFN_vkVoidFunction>(FenceStatus);
+    if (std::strcmp(name, "vkQueueWaitIdle") == 0) return reinterpret_cast<PFN_vkVoidFunction>(QueueIdle);
+    if (std::strcmp(name, "vkWaitForFences") == 0) return reinterpret_cast<PFN_vkVoidFunction>(WaitFences);
+    if (std::strcmp(name, "vkFreeCommandBuffers") == 0) return reinterpret_cast<PFN_vkVoidFunction>(FreeCommands);
+    if (std::strcmp(name, "vkDestroyFence") == 0) return reinterpret_cast<PFN_vkVoidFunction>(DestroyFence);
+    if (probe.real != nullptr) return probe.real(device, name);
+    if (std::strcmp(name, "vkAllocateCommandBuffers") == 0) return reinterpret_cast<PFN_vkVoidFunction>(AllocateCommands);
+    if (std::strcmp(name, "vkCreateFence") == 0) return reinterpret_cast<PFN_vkVoidFunction>(CreateFence);
+    if (std::strcmp(name, "vkBeginCommandBuffer") == 0) return reinterpret_cast<PFN_vkVoidFunction>(BeginCommands);
+    return nullptr;
+}
+
+bool RunCase(const Context& original, Fault fault, bool wait, bool native) {
+    probe = {};
+    probe.real = native ? original.deviceProc : nullptr;
+    probe.fault = fault;
+    activeDevice = original.device;
+    activeQueue = original.queue;
+    auto context = original;
+    context.deviceProc = DeviceProc;
+    std::unique_ptr<Buffer> output;
+    constexpr std::size_t bytes = 256;
+    constexpr std::uint32_t pattern = 0x5a3c71e9u;
+    if (native) {
+        output = std::make_unique<Buffer>(context, bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT);
+        std::memset(output->Bytes().data(), 0, bytes);
+    }
+    bool allocationFailed = false;
+    bool apiFailed = false;
+    bool waitFailed = false;
+    {
+        AgcDriver::FrameTiming frame(1, true);
+        AgcDriver::PerformanceContext timing(&frame);
+        try {
+            CommandBatch batch(context);
+            if (output) {
+                context.Function<PFN_vkCmdFillBuffer>("vkCmdFillBuffer")(batch.Handle(), output->Handle(), 0, bytes, pattern);
+                RecordMemoryBarrier(context, batch.Handle(), VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT);
+            }
+            try {
+                batch.Submit();
+            } catch (const std::bad_alloc&) {
+                if (!wait) throw;
+                allocationFailed = true;
+            }
+            SetAllocationFailureCountdown(-1);
+            if (wait && probe.submitted != 0) {
+                try {
+                    batch.Wait();
+                } catch (const std::exception&) {
+                    waitFailed = true;
+                }
+            }
+        } catch (const std::bad_alloc&) {
+            allocationFailed = true;
+        } catch (const std::runtime_error& error) {
+            apiFailed = std::string(error.what()).find(fault == Fault::End ? "vkEndCommandBuffer" : "vkQueueSubmit graphics") != std::string::npos;
+        }
+        SetAllocationFailureCountdown(-1);
+    }
+    bool pixels = true;
+    if (output && probe.submitted != 0) {
+        output->Invalidate();
+        const auto* words = reinterpret_cast<const std::uint32_t*>(output->Bytes().data());
+        pixels = std::all_of(words, words + bytes / sizeof(*words), [](auto word) { return word == pattern; });
+    }
+    const bool rejected = fault == Fault::End || fault == Fault::Submit;
+    const bool complete = rejected ? probe.queries == 0 && probe.idleWaits == 0 && probe.fenceWaits == 0 : wait ? probe.fenceWaits == 1 : probe.queries == 1;
+    const bool passed = allocationFailed == (fault == Fault::Timing) && apiFailed == rejected && !waitFailed && !probe.pending && probe.submitted == (rejected ? 0u : 1u) && probe.premature == 0 && probe.freed == 1 && probe.destroyed == 1 && complete && pixels;
+    std::printf("command batch fault=%u wait=%u allocation failed=%u submitted=%u premature=%u safety waits=%u queries=%u idle waits=%u fence waits=%u readbacks=%u correct=%u wait failed=%u native pending=%u\n", static_cast<unsigned>(fault), wait, allocationFailed, probe.submitted, probe.premature, probe.safetyWaits, probe.queries, probe.idleWaits, probe.fenceWaits, output && probe.submitted != 0 ? 1u : 0u, pixels, waitFailed, probe.nativePending);
+    return passed;
+}
+
+}
+
+void AgcDriver::Tests::RunCommandBatchFailureTests(const AgcDriver::Graphics::Context& context, bool native) {
+    unsigned failures = 0;
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    for (const auto fault : {Fault::Timing, Fault::None}) {
+        for (const auto wait : {false, true}) if (!RunCase(context, fault, wait, native)) ++failures;
+    }
+    for (const auto fault : {Fault::End, Fault::Submit}) if (!RunCase(context, fault, false, native)) ++failures;
+    Require(failures == 0, "command batch failure handling released pending work or lost its submission state");
+}

core/libs/prx/libSceAgcDriver/tests/CommandBatchFailure.hpp

@@ -0,0 +1,12 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_COMMANDBATCHFAILURE_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_COMMANDBATCHFAILURE_HPP
+
+#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
+
+namespace AgcDriver::Tests {
+
+void RunCommandBatchFailureTests(const Graphics::Context& context, bool native);
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/tests/ConstantPhi.cpp

@@ -2,6 +2,8 @@
 #include "Optimization/ConstantFolder.hpp"
 #include "Optimization/DeadCodeEliminator.hpp"
 #include <cstdio>
+#include <pthread.h>
+#include <string>
 #include <stdexcept>
 
 using namespace ShaderRecompiler;
@@ -77,11 +79,54 @@ void CheckDiamond(unsigned mode) {
     }
 }
 
+void CheckLongAncillaryForwarding() {
+    IrProgram program;
+    auto& block = program.CreateBlock();
+    program.BlockOrder().push_back(&block);
+    BlockInfo info;
+    info.id = 0u;
+    info.terminator.kind = TerminatorKind::Return;
+    program.Metadata().blockInfo.push_back(info);
+    program.SetEntryBlock(block);
+    IrBuilder builder(program);
+    builder.SetInsertionPoint(block);
+    auto& ancillary = builder.Emit(IrOpcode::GetBuiltin, IrType::U32, {&builder.Constant(static_cast<std::uint32_t>(StageInputKind::PackedAncillary)), &builder.Constant(0u)});
+    auto& lane = builder.Emit(IrOpcode::LaneId, IrType::U32, {});
+    auto& condition = builder.Emit(IrOpcode::ULessThan32, IrType::Bool, {&lane, &builder.Constant(16u)});
+    auto& zero = builder.Constant(0u);
+    IrValue* forwarded = &ancillary;
+    for (std::uint32_t link = 0; link < 20000u; ++link) forwarded = &builder.Emit(IrOpcode::SelectU32, IrType::U32, {&condition, &zero, forwarded});
+    auto& field = builder.Emit(IrOpcode::BitFieldUExtract, IrType::U32, {forwarded, &builder.Constant(8u), &builder.Constant(4u)});
+    builder.Emit(IrOpcode::ReferenceU32, IrType::Void, {&field});
+    struct Run {
+        IrProgram& program;
+        std::string error;
+    } run{program, {}};
+    pthread_attr_t attributes;
+    pthread_attr_init(&attributes);
+    pthread_attr_setstacksize(&attributes, 256u * 1024u);
+    pthread_t thread;
+    const auto body = +[](void* argument) -> void* {
+        auto& state = *static_cast<Run*>(argument);
+        try {
+            ConstantFolder{}.Fold(state.program);
+        } catch (const std::exception& error) {
+            state.error = error.what();
+        }
+        return nullptr;
+    };
+    Require(pthread_create(&thread, &attributes, body, &run) == 0, "long ancillary forwarding: cannot start the folding thread");
+    pthread_join(thread, nullptr);
+    pthread_attr_destroy(&attributes);
+    Require(run.error.empty(), "long ancillary forwarding: folding failed");
+}
+
 }
 
 int main() {
     try {
         for (unsigned mode = 0; mode < 5u; ++mode) CheckDiamond(mode);
+        CheckLongAncillaryForwarding();
         std::puts("constant phi dominance tests passed");
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/ConstantSelect.cpp

@@ -0,0 +1,121 @@
+#include "IntermediateRepresentation/IrBuilder.hpp"
+#include "Optimization/ConstantFolder.hpp"
+#include "Optimization/DeadCodeEliminator.hpp"
+#include <cstdint>
+#include <cstdio>
+#include <stdexcept>
+
+using namespace ShaderRecompiler;
+
+namespace {
+
+void Require(bool condition, const char* message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+struct Program {
+    IrProgram program;
+    IrBuilder builder{program};
+    IrValue* lane = nullptr;
+    IrValue* condition = nullptr;
+    IrValue* other = nullptr;
+    std::uint32_t values = 0;
+
+    Program() {
+        auto& block = program.CreateBlock();
+        program.BlockOrder().push_back(&block);
+        BlockInfo info;
+        info.id = 0;
+        info.terminator.kind = TerminatorKind::Return;
+        program.Metadata().blockInfo.push_back(info);
+        program.SetEntryBlock(block);
+        builder.SetInsertionPoint(block);
+        lane = &builder.Emit(IrOpcode::LaneId, IrType::U32, {});
+        condition = &builder.Emit(IrOpcode::ULessThan32, IrType::Bool, {lane, &builder.Constant(16u)});
+        other = &builder.Emit(IrOpcode::ULessThan32, IrType::Bool, {lane, &builder.Constant(8u)});
+    }
+
+    IrValue& value() {
+        return builder.Emit(IrOpcode::IAdd32, IrType::U32, {lane, &builder.Constant(++values)});
+    }
+
+    IrValue& select(IrValue& selector, IrValue& onTrue, IrValue& onFalse) {
+        return builder.Emit(IrOpcode::SelectU32, IrType::U32, {&selector, &onTrue, &onFalse});
+    }
+
+    const IrValue* folded(IrValue& result) {
+        auto& reference = builder.Emit(IrOpcode::ReferenceU32, IrType::Void, {&result});
+        ValidateProgram(program, true);
+        ConstantFolder{}.Fold(program);
+        DeadCodeEliminator{}.RemoveIdentities(program);
+        DeadCodeEliminator{}.Eliminate(program);
+        ValidateProgram(program, true);
+        return reference.Argument(0)->Resolve();
+    }
+};
+
+bool Selects(const IrValue* value, const IrValue* selector, const IrValue* onTrue, const IrValue* onFalse) {
+    return value->Opcode() == IrOpcode::SelectU32 && value->Argument(0)->Resolve() == selector && value->Argument(1)->Resolve() == onTrue && value->Argument(2)->Resolve() == onFalse;
+}
+
+void CheckFalseArm() {
+    Program p;
+    auto& kept = p.value();
+    auto& dropped = p.value();
+    auto& fallback = p.value();
+    auto& inner = p.select(*p.condition, dropped, fallback);
+    const auto* result = p.folded(p.select(*p.condition, kept, inner));
+    Require(Selects(result, p.condition, &kept, &fallback), "a select in the false arm of a select on the same condition was not folded to its false arm");
+    Require(!inner.HasUses(), "the folded inner select is still used");
+}
+
+void CheckTrueArm() {
+    Program p;
+    auto& kept = p.value();
+    auto& dropped = p.value();
+    auto& fallback = p.value();
+    const auto* result = p.folded(p.select(*p.condition, p.select(*p.condition, kept, dropped), fallback));
+    Require(Selects(result, p.condition, &kept, &fallback), "a select in the true arm of a select on the same condition was not folded to its true arm");
+}
+
+void CheckChain() {
+    Program p;
+    auto& last = p.value();
+    auto& fallback = p.value();
+    auto* chain = &p.select(*p.condition, p.value(), fallback);
+    chain = &p.select(*p.condition, p.value(), *chain);
+    const auto* result = p.folded(p.select(*p.condition, last, *chain));
+    Require(Selects(result, p.condition, &last, &fallback), "a chain of selects on one condition did not fold to the last write and the fallback");
+}
+
+void CheckEqualArms() {
+    Program p;
+    auto& same = p.value();
+    const auto* result = p.folded(p.select(*p.condition, same, p.select(*p.condition, p.value(), same)));
+    Require(result == &same, "a select whose arms become equal after folding was not replaced by that value");
+}
+
+void CheckOtherCondition() {
+    Program p;
+    auto& kept = p.value();
+    auto& inner = p.select(*p.other, p.value(), p.value());
+    const auto* result = p.folded(p.select(*p.condition, kept, inner));
+    Require(Selects(result, p.condition, &kept, &inner), "a select nested under a different condition was folded");
+}
+
+}
+
+int main() {
+    try {
+        CheckFalseArm();
+        CheckTrueArm();
+        CheckChain();
+        CheckEqualArms();
+        CheckOtherCondition();
+        std::puts("constant select tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "%s\n", error.what());
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/DenormalFlushElimination.cpp

@@ -1,11 +1,20 @@
 #include "IntermediateRepresentation/IrProgram.hpp"
 #include "Optimization/DenormalFlushEliminator.hpp"
+#include "Recompiler.hpp"
+#include <array>
+#include <cstddef>
 #include <cstdint>
 #include <cstdio>
 #include <exception>
 #include <functional>
 #include <initializer_list>
+#include <set>
+#include <span>
+#include <spirv/unified1/spirv.hpp>
+#include <stdexcept>
+#include <string_view>
 #include <utility>
+#include <vector>
 
 using namespace ShaderRecompiler;
 
@@ -80,6 +89,46 @@ bool run(const char* name, const std::function<bool()>& test) {
     }
 }
 
+std::vector<std::uint32_t> Recompiled(std::span<const std::uint32_t> code) {
+    const std::array<std::uint32_t, 4> userData{0x10000000u, 0x00100000u, 0x40u, 0x00027facu};
+    const std::array<std::uint32_t, 2> capabilities{1u, 61u};
+    const std::array<std::string_view, 1> extensions{"SPV_KHR_storage_buffer_storage_class"};
+    RecompileRequest request{};
+    request.shader = {ShaderStage::Compute, 0x20000u, code, 0, {}};
+    request.context.waveSize = 32;
+    request.context.userDataBaseRegister = 0;
+    request.context.userData = userData;
+    request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
+    request.target.vulkanVersion = 0x00403000u;
+    request.target.spirvVersion = 0x00010600u;
+    request.target.subgroupSize = 32u;
+    request.target.bdaAbiVersion = 1;
+    request.target.supportedCapabilities = capabilities;
+    request.target.supportedExtensions = extensions;
+    request.target.maxWorkgroupSize = {1024u, 1024u, 64u};
+    request.target.maxWorkgroupInvocations = 1024;
+    request.target.maxWorkgroupSharedMemoryBytes = 49152;
+    request.layout = {0, 0, 0, 128};
+    request.useCache = false;
+    const auto& words = Recompile(request).spirv.Words();
+    return {words.begin(), words.end()};
+}
+
+std::size_t ExponentMasks(std::span<const std::uint32_t> words) {
+    if (words.size() < 5 || words[0] != spv::MagicNumber) throw std::runtime_error("the recompiler returned no SPIR-V module");
+    std::set<std::uint32_t> exponents;
+    std::size_t masks = 0;
+    for (std::size_t offset = 5; offset < words.size();) {
+        const auto count = words[offset] >> spv::WordCountShift;
+        if (count == 0 || count > words.size() - offset) throw std::runtime_error("the recompiler returned a malformed SPIR-V instruction");
+        const auto opcode = words[offset] & spv::OpCodeMask;
+        if (opcode == spv::OpConstant && count == 4 && words[offset + 3] == 0x7f800000u) exponents.insert(words[offset + 2]);
+        if (opcode == spv::OpBitwiseAnd && count == 5 && (exponents.contains(words[offset + 3]) || exponents.contains(words[offset + 4]))) ++masks;
+        offset += count;
+    }
+    return masks;
+}
+
 }
 
 int main() {
@@ -252,5 +301,79 @@ int main() {
         return loop(true) && !loop(false);
     });
 
+    passed &= run("a product behind its round-to-nearest tiny-product guard needs no flush, one behind another guard does", [] {
+        Builder b;
+        const auto value = [&](IrValue& bits) -> IrValue& { return b.emit(IrOpcode::BitCastF32U32, IrType::F32, {&bits}); };
+        const auto bitsOf = [&](IrValue& number) -> IrValue& { return b.emit(IrOpcode::BitCastU32F32, IrType::U32, {&number}); };
+        const auto guarded = [&](IrValue& lhs, IrValue& rhs, IrValue& guardLhs, IrValue& guardRhs, std::uint32_t threshold) -> IrValue& {
+            auto& product = b.emit(IrOpcode::FPMul32, IrType::F32, {&lhs, &rhs});
+            auto& scaled = b.emit(IrOpcode::FPMul32, IrType::F32, {&b.emit(IrOpcode::FPMul32, IrType::F32, {&guardLhs, &b.constant(0x5f800000u)}), &guardRhs});
+            auto& magnitude = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&bitsOf(scaled), &b.constant(0x7fffffffu)});
+            auto& tiny = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.emit(IrOpcode::ISub32, IrType::U32, {&magnitude, &b.constant(1u)}), &b.constant(threshold)});
+            auto& sign = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::BitwiseXor32, IrType::U32, {&bitsOf(lhs), &bitsOf(rhs)}), &b.constant(0x80000000u)});
+            return b.flush(bitsOf(b.emit(IrOpcode::SelectF32, IrType::F32, {&tiny, &value(sign), &product})));
+        };
+        auto& x = value(b.flush(b.unknown()));
+        auto& y = value(b.flush(b.unknown()));
+        auto& raw = value(b.unknown());
+        IrValue* removed[] = {&guarded(x, y, x, y, 0x207fffffu), &guarded(x, y, y, x, 0x207fffffu)};
+        IrValue* kept[] = {&guarded(x, y, x, raw, 0x207fffffu), &guarded(x, y, x, y, 0x20800000u), &guarded(x, raw, x, raw, 0x207fffffu)};
+        for (auto* flushed : removed) b.keep(*flushed);
+        for (auto* flushed : kept) b.keep(*flushed);
+        const bool counted = b.eliminate() == 2u;
+        bool flushes = true;
+        for (auto* flushed : removed) flushes &= !flushed->HasUses();
+        for (auto* flushed : kept) flushes &= flushed->HasUses();
+        return counted && flushes;
+    });
+
+    passed &= run("an arm a select takes only when its bits are NaN needs no flush, one under another value's NaN test does", [] {
+        Builder b;
+        auto& x = b.unknown();
+        auto& other = b.unknown();
+        const auto magnitude = [&](IrValue& bits) -> IrValue& { return b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&bits, &b.constant(0x7fffffffu)}); };
+        const auto isNan = [&](IrValue& bits) -> IrValue& { return b.emit(IrOpcode::UGreaterThan32, IrType::Bool, {&magnitude(bits), &b.constant(0x7f800000u)}); };
+        const auto chosen = [&](IrValue& condition, IrValue& arm) -> IrValue& { return b.flush(b.emit(IrOpcode::SelectU32, IrType::U32, {&condition, &arm, &b.flush(b.unknown())})); };
+        IrValue* removed[] = {
+            &chosen(isNan(x), x),
+            &chosen(isNan(x), b.emit(IrOpcode::BitwiseOr32, IrType::U32, {&x, &b.constant(0x00400000u)})),
+            &chosen(b.emit(IrOpcode::FPIsNan32, IrType::Bool, {&b.emit(IrOpcode::BitCastF32U32, IrType::F32, {&x})}), x),
+        };
+        IrValue* kept[] = {
+            &chosen(isNan(other), x),
+            &chosen(b.emit(IrOpcode::UGreaterThan32, IrType::Bool, {&b.constant(0x7f800000u), &magnitude(x)}), x),
+        };
+        for (auto* flushed : removed) b.keep(*flushed);
+        for (auto* flushed : kept) b.keep(*flushed);
+        const bool counted = b.eliminate() == 3u;
+        bool flushes = true;
+        for (auto* flushed : removed) flushes &= !flushed->HasUses();
+        for (auto* flushed : kept) flushes &= flushed->HasUses();
+        return counted && flushes;
+    });
+
+    passed &= run("the sign or exponent bits of any value need no flush, a mask that keeps significand bits does", [] {
+        Builder b;
+        auto& sign = b.flush(b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.unknown(), &b.constant(0x80000000u)}));
+        auto& exponent = b.flush(b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.constant(0xff800000u), &b.unknown()}));
+        auto& significand = b.flush(b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.unknown(), &b.constant(0x807fffffu)}));
+        b.keep(sign);
+        b.keep(exponent);
+        b.keep(significand);
+        return b.eliminate() == 2u && !sign.HasUses() && !exponent.HasUses() && significand.HasUses();
+    });
+
+    passed &= run("two chained v_mul_f32 keep only the input flush of the first source", [] {
+        const std::array<std::uint32_t, 5> code{
+            0x10020100u,
+            0x10040301u,
+            0xe0700000u, 0x80000200u,
+            0xbf810000u,
+        };
+        const auto masks = ExponentMasks(Recompiled(code));
+        if (masks != 1u) std::fprintf(stderr, "  %zu f32 exponent masks, expected 1\n", masks);
+        return masks == 1u;
+    });
+
     return passed ? 0 : 1;
 }

core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp

@@ -458,8 +458,8 @@ void RunDepthSurfaceReuseTests() {
     const auto otherWithA = pairing(otherDepth, stencilA);
     Require(imageCopies.size() == 1 && imageCopies[0] == std::make_tuple(withA, otherWithA, static_cast<VkImageAspectFlags>(VK_IMAGE_ASPECT_STENCIL_BIT)), "a stencil plane paired with another depth plane must carry its stencil over");
     {
-        constexpr std::uint64_t arrayBase = 0x80000000;
-        constexpr std::uint64_t htile = 0x81000000;
+        constexpr std::uint64_t arrayBase = 0x30000000;
+        constexpr std::uint64_t htile = 0x31000000;
         constexpr VkExtent2D extent{1024, 512};
         const auto sliceHtile = HtileMetadataBytes(extent);
         DepthTarget first{arrayBase, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0};

core/libs/prx/libSceAgcDriver/tests/DescriptorCacheFailure.cpp

@@ -0,0 +1,169 @@
+#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <new>
+
+namespace {
+thread_local bool failNextAllocation = false;
+}
+
+void* operator new(std::size_t size) {
+    if (failNextAllocation) {
+        failNextAllocation = false;
+        throw std::bad_alloc();
+    }
+    if (void* memory = std::malloc(size == 0 ? 1 : size)) return memory;
+    throw std::bad_alloc();
+}
+
+void* operator new[](std::size_t size) { return ::operator new(size); }
+void operator delete(void* memory) noexcept { std::free(memory); }
+void operator delete[](void* memory) noexcept { std::free(memory); }
+void operator delete(void* memory, std::size_t) noexcept { std::free(memory); }
+void operator delete[](void* memory, std::size_t) noexcept { std::free(memory); }
+
+namespace {
+using namespace AgcDriver::Graphics;
+
+struct VulkanObjects {
+    std::uintptr_t next = 1;
+    std::uint32_t layouts = 0;
+    std::uint32_t pools = 0;
+    std::uint32_t createdLayouts = 0;
+    std::uint32_t createdPools = 0;
+    bool failAfterLayout = false;
+    bool failAfterPool = false;
+    std::array<bool, 32> poolUsed{};
+} objects;
+
+void RequireTest(bool condition, const char* message) {
+    if (condition) return;
+    std::fprintf(stderr, "%s\n", message);
+    std::abort();
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL CreateLayout(VkDevice, const VkDescriptorSetLayoutCreateInfo*, const VkAllocationCallbacks*, VkDescriptorSetLayout* layout) {
+    *layout = reinterpret_cast<VkDescriptorSetLayout>(objects.next++);
+    ++objects.layouts;
+    ++objects.createdLayouts;
+    failNextAllocation = objects.failAfterLayout;
+    return VK_SUCCESS;
+}
+
+VKAPI_ATTR void VKAPI_CALL DestroyLayout(VkDevice, VkDescriptorSetLayout, const VkAllocationCallbacks*) {
+    RequireTest(objects.layouts != 0, "layout destroyed twice");
+    --objects.layouts;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL CreatePool(VkDevice, const VkDescriptorPoolCreateInfo*, const VkAllocationCallbacks*, VkDescriptorPool* pool) {
+    RequireTest(objects.next < objects.poolUsed.size(), "mock pool capacity exceeded");
+    *pool = reinterpret_cast<VkDescriptorPool>(objects.next++);
+    ++objects.pools;
+    ++objects.createdPools;
+    failNextAllocation = objects.failAfterPool;
+    return VK_SUCCESS;
+}
+
+VKAPI_ATTR void VKAPI_CALL DestroyPool(VkDevice, VkDescriptorPool, const VkAllocationCallbacks*) {
+    RequireTest(objects.pools != 0, "pool destroyed twice");
+    --objects.pools;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL AllocateSets(VkDevice, const VkDescriptorSetAllocateInfo* info, VkDescriptorSet* set) {
+    auto& used = objects.poolUsed.at(reinterpret_cast<std::uintptr_t>(info->descriptorPool));
+    if (used) return VK_ERROR_OUT_OF_POOL_MEMORY;
+    used = true;
+    *set = reinterpret_cast<VkDescriptorSet>(objects.next++);
+    return VK_SUCCESS;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL ResetPool(VkDevice, VkDescriptorPool pool, VkDescriptorPoolResetFlags) {
+    objects.poolUsed.at(reinterpret_cast<std::uintptr_t>(pool)) = false;
+    return VK_SUCCESS;
+}
+
+VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL Resolve(VkDevice, const char* name) {
+    if (std::strcmp(name, "vkCreateDescriptorSetLayout") == 0) return reinterpret_cast<PFN_vkVoidFunction>(CreateLayout);
+    if (std::strcmp(name, "vkDestroyDescriptorSetLayout") == 0) return reinterpret_cast<PFN_vkVoidFunction>(DestroyLayout);
+    if (std::strcmp(name, "vkCreateDescriptorPool") == 0) return reinterpret_cast<PFN_vkVoidFunction>(CreatePool);
+    if (std::strcmp(name, "vkDestroyDescriptorPool") == 0) return reinterpret_cast<PFN_vkVoidFunction>(DestroyPool);
+    if (std::strcmp(name, "vkAllocateDescriptorSets") == 0) return reinterpret_cast<PFN_vkVoidFunction>(AllocateSets);
+    if (std::strcmp(name, "vkResetDescriptorPool") == 0) return reinterpret_cast<PFN_vkVoidFunction>(ResetPool);
+    return nullptr;
+}
+
+Context TestContext() {
+    Context context{};
+    context.deviceProc = Resolve;
+    context.limits.maxPerStageResources = 128;
+    context.limits.maxDescriptorSetStorageBuffers = 4096;
+    context.descriptorIndexingLimits.maxPerStageUpdateAfterBindResources = 128;
+    context.descriptorIndexingLimits.maxDescriptorSetUpdateAfterBindStorageBuffers = 4096;
+    return context;
+}
+
+const VkDescriptorSetLayoutBinding binding{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr};
+const std::array<std::uint32_t, 4> key{binding.binding, binding.descriptorType, binding.descriptorCount, binding.stageFlags};
+const VkDescriptorPoolSize size{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1};
+
+bool LayoutFailure(bool updateAfterBind) {
+    objects = {};
+    bool failed = false;
+    {
+        DescriptorCache cache(TestContext());
+        objects.failAfterLayout = true;
+        try {
+            cache.Layout(key, std::span(&binding, 1), updateAfterBind);
+        } catch (const std::bad_alloc&) {
+            failed = true;
+        }
+        failNextAllocation = false;
+        objects.failAfterLayout = false;
+        RequireTest(failed && objects.createdLayouts == 1, "layout publication failure was not exercised");
+        const auto layout = cache.Layout(key, std::span(&binding, 1), updateAfterBind);
+        RequireTest(layout != VK_NULL_HANDLE && objects.createdLayouts == 2, "layout creation did not recover");
+        RequireTest(cache.Layout(key, std::span(&binding, 1), updateAfterBind) == layout && objects.createdLayouts == 2, "layout cache hit created another object");
+    }
+    std::printf("layout updateAfterBind=%d: unowned=%u\n", updateAfterBind, objects.layouts);
+    return objects.layouts == 0;
+}
+
+bool PoolFailure(bool updateAfterBind, bool grow) {
+    objects = {};
+    {
+        DescriptorCache cache(TestContext());
+        const auto layout = cache.Layout(key, std::span(&binding, 1), updateAfterBind);
+        DescriptorCache::SetAllocation first{};
+        if (grow) first = cache.Allocate(layout, std::span(&size, 1), updateAfterBind);
+        objects.failAfterPool = true;
+        DescriptorCache::SetAllocation allocation{};
+        try {
+            allocation = cache.Allocate(layout, std::span(&size, 1), updateAfterBind);
+        } catch (const std::bad_alloc&) {
+        }
+        failNextAllocation = false;
+        objects.failAfterPool = false;
+        RequireTest(objects.createdPools == (grow ? 2u : 1u), "pool publication path was not exercised");
+        if (allocation.set != VK_NULL_HANDLE) cache.Free(allocation);
+        if (grow) cache.Free(first);
+        const auto recovered = cache.Allocate(layout, std::span(&size, 1), updateAfterBind);
+        RequireTest(recovered.set != VK_NULL_HANDLE, "pool allocation did not recover");
+        cache.Free(recovered);
+    }
+    std::printf("pool updateAfterBind=%d grow=%d: unowned=%u\n", updateAfterBind, grow, objects.pools);
+    return objects.layouts == 0 && objects.pools == 0;
+}
+}
+
+int main() {
+    int failures = 0;
+    for (const bool updateAfterBind : {false, true}) {
+        if (!LayoutFailure(updateAfterBind)) ++failures;
+        for (const bool grow : {false, true}) if (!PoolFailure(updateAfterBind, grow)) ++failures;
+    }
+    return failures == 0 ? 0 : 1;
+}

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -30,6 +30,7 @@
 #include <spirv/unified1/spirv.hpp>
 #include <algorithm>
 #include <array>
+#include <atomic>
 #include <bit>
 #include <chrono>
 #include <condition_variable>
@@ -1338,6 +1339,48 @@ void BlendBypassTests() {
     Require(!bypassed.blends.at(0).blendEnable && bypassed.blends.at(0).colorWriteMask != 0, "a target with BLEND_BYPASS still blends");
 }
 
+void UnclampedNormalizedColorTests() {
+    using AgcDriver::Graphics::DecodeState;
+    constexpr std::uint32_t blending = (1u << 30u) | 4u | (5u << 8u);
+    const auto target = [](std::uint32_t info) {
+        auto queue = makeState();
+        for (std::uint32_t constant = 0x105; constant < 0x109; ++constant) queue.context[constant] = 0;
+        queue.context[0x31c] = info;
+        queue.context[0x1b3] = 2;
+        queue.context[0x1b4] = 2;
+        return queue;
+    };
+    auto queue = target(0x0000000cu);
+    auto state = DecodeState(queue);
+    Require(state.colors.size() == 1 && state.colors[0].format == VK_FORMAT_R8G8_UNORM && !state.blends.at(0).blendEnable && state.blends.at(0).colorWriteMask != 0, "an 8_8 UNORM target with BLEND_CLAMP clear (CB_COLOR_INFO 0x0000000c, PPSA26344) was refused without blending");
+    const auto rejection = AgcDriver::Graphics::DrawRejection(queue, false);
+    Require(rejection.empty(), "the precheck refused an unclamped UNORM target that does not blend: " + rejection);
+    Require(AgcDriver::Graphics::DecodeColorBuffer(queue.context, 0).format == VK_FORMAT_R8G8_UNORM, "an unclamped UNORM target was refused outside a draw");
+    auto eliminate = target(0x0000000cu);
+    eliminate.context[0x0] = 0;
+    eliminate.context[0x202] = 0xcc0020;
+    const auto pass = AgcDriver::Graphics::DecodeColorMetadataPass(eliminate);
+    Require(pass.has_value() && pass->mode == AgcDriver::Graphics::ColorMetadataPass::Mode::EliminateFastClear && pass->targets.size() == 1 && pass->targets[0].format == VK_FORMAT_R8G8_UNORM, "a fast-clear eliminate over an unclamped UNORM target was refused");
+    queue.context[0x1e0] = blending;
+    expectFailure([&] { DecodeState(queue); }, "blending unclamped normalized color is not implemented (CB_COLOR_INFO 0x0000000c, CB_BLEND0_CONTROL 0x40000504)");
+    queue.context[0x31c] |= 0x10000u;
+    state = DecodeState(queue);
+    Require(!state.blends.at(0).blendEnable && state.blends.at(0).colorWriteMask != 0, "an unclamped UNORM target with BLEND_BYPASS was refused or blended");
+    queue = target(0x0000010cu);
+    Require(DecodeState(queue).colors.at(0).format == VK_FORMAT_R8G8_SNORM, "an unclamped SNORM target that does not blend was refused");
+    queue.context[0x1e0] = blending;
+    expectFailure([&] { DecodeState(queue); }, "blending unclamped normalized color is not implemented");
+    queue = target(0x00000604u);
+    expectFailure([&] { DecodeState(queue); }, "unclamped normalized color is unsupported");
+    queue = target(0x0000800cu);
+    queue.context[0x1e0] = blending;
+    Require(DecodeState(queue).blends.at(0).blendEnable, "a clamped UNORM target stopped blending");
+    queue = target(0x00000708u);
+    queue.context[0x1e0] = blending;
+    state = DecodeState(queue);
+    Require(state.colors.at(0).format == VK_FORMAT_R16_SFLOAT && state.blends.at(0).blendEnable, "a 16 float target, which keeps BLEND_CLAMP clear, stopped blending");
+}
+
 void OpaqueDestinationAlphaTests() {
     auto queue = makeState();
     for (std::uint32_t constant = 0x105; constant < 0x109; ++constant) queue.context[constant] = 0;
@@ -1693,6 +1736,35 @@ void ReversedBlendRoutingTests() {
     Require(native == 121u && routed == 932u, "the reversed-order blend routing accepts " + std::to_string(native) + " native and " + std::to_string(routed) + " routed factor combinations, not 121 and 932");
 }
 
+void Color565TargetTests() {
+    auto queue = makeState();
+    const auto info = [&](std::uint32_t number, std::uint32_t swap) {
+        queue.context[0x31c] = (queue.context[0x31c] & ~((0x1fu << 2u) | (7u << 8u) | (3u << 11u))) | (16u << 2u) | (number << 8u) | (swap << 11u);
+    };
+    info(0, 2);
+    const auto reversed = AgcDriver::Graphics::DecodeState(queue);
+    Require(reversed.colors.size() == 1 && reversed.colors[0].format == VK_FORMAT_R5G6B5_UNORM_PACK16 && reversed.colors[0].elementBytes == 2 && reversed.colors[0].componentMapping == 0xe4u, "a 5_6_5 target with the reversed component order did not render through R5G6B5 with X in its high bits");
+    Require(reversed.blends[0].colorWriteMask == 0xfu, "a 5_6_5 target with the reversed component order did not write all its channels");
+    queue.context[0x8e] = 0x1u;
+    Require(AgcDriver::Graphics::DecodeState(queue).blends[0].colorWriteMask == VK_COLOR_COMPONENT_R_BIT, "a 5_6_5 target with the reversed component order writing only X did not write the R5G6B5 channel in its high bits");
+    info(0, 0);
+    const auto standard = AgcDriver::Graphics::DecodeState(queue);
+    Require(standard.colors[0].format == VK_FORMAT_R5G6B5_UNORM_PACK16 && standard.colors[0].componentMapping == 0xc6u, "a 5_6_5 target with the standard component order did not render through R5G6B5 with X and Z exchanged");
+    Require(AgcDriver::Graphics::ExportMappings(standard)[0] == 0xc6u, "a 5_6_5 target with the standard component order did not export through the X and Z exchange");
+    Require(standard.blends[0].colorWriteMask == VK_COLOR_COMPONENT_B_BIT, "a 5_6_5 target with the standard component order writing only X did not write the R5G6B5 channel in its low bits");
+    queue.context[0x8e] = 0xfu;
+    info(0, 2);
+    for (std::uint32_t i = 0; i < 4; ++i) queue.context[0x105 + i] = 0;
+    queue.context[0x1e0] = 0x40000504u;
+    const auto blended = AgcDriver::Graphics::DecodeState(queue);
+    Require(blended.blends[0].blendEnable && blended.blends[0].srcColorBlendFactor == VK_BLEND_FACTOR_SRC_ALPHA && blended.blends[0].dstColorBlendFactor == VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA, "alpha blending into a 5_6_5 target was not decoded");
+    queue.context[0x1e0] = 0;
+    for (const auto& [number, swap] : {std::pair{1u, 2u}, std::pair{4u, 0u}, std::pair{0u, 1u}, std::pair{0u, 3u}}) {
+        info(number, swap);
+        expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "unsupported color format 16 number type " + std::to_string(number) + " component swap " + std::to_string(swap));
+    }
+}
+
 void Color1555TargetTests() {
     auto queue = makeState();
     queue.context[0x31c] = (queue.context[0x31c] & ~((0x1fu << 2u) | (7u << 8u) | (3u << 11u))) | (17u << 2u);
@@ -2013,6 +2085,12 @@ void DepthStencilTests() {
     const auto volume = AgcDriver::Graphics::DecodeState(queue);
     Require(volume.color.address == sliced && volume.color.depth == 4u && volume.color.depthSlice == 2u, "a color view of one 3D depth slice did not keep the surface address and select the slice");
     Require(volume.color.layers == 1 && volume.renderLayers == 1 && volume.layerExports == 0, "a color view of one 3D depth slice became layered");
+    {
+        auto mipmapped = queue;
+        mipmapped.context[0x3b0] |= 2u << 28u;
+        mipmapped.context[0x31b] = (1u << 26u) | (1u << 13u) | 1u;
+        expectFailure([&] { AgcDriver::Graphics::DecodeColorBuffer(mipmapped.context, 0); }, "slot 0, 64x4x4, viewMip 1, maxMip 2, slices 1..1, CB_COLOR_INFO 0x00028028, ATTRIB2 0x200fc003, ATTRIB3 0x0a000003, VIEW 0x04002001");
+    }
     queue.context[0x31b] = 4u | (4u << 13u);
     expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "outside the 3D surface");
     queue.context[0x31b] = 2u | (1u << 13u);
@@ -2993,6 +3071,11 @@ void multisampleTests() {
     }
     queue.context[0x32b] = queue.context[0x31c] ^ (1u << 8u);
     expectFailure([&] { DecodeColorMetadataPass(queue); }, "different formats or extents");
+    for (const auto [packedInfo, name] : {std::pair{6u << 2u, "10_11_11 unorm"}, std::pair{(7u << 2u) | (7u << 8u) | (2u << 11u) | 0x40000u, "11_11_10 float"}}) {
+        auto packed = rasterizedResolve;
+        for (const auto offset : {0x31cu, 0x32bu}) packed.context[offset] = (packed.context[offset] & ~0x41f7cu) | packedInfo;
+        expectFailure([&] { DecodeColorMetadataPass(packed); }, std::string("CB resolves of ") + name + " color targets");
+    }
     queue = rasterizedResolve;
     queue.context[0x31c] |= 0x10000000u;
     queue.context[0x325] = 0;
@@ -3655,9 +3738,36 @@ struct MockDescriptorWrite {
     std::vector<VkDescriptorImageInfo> images;
 };
 
+struct LiveCounter {
+    std::atomic<std::int64_t> value{0};
+    LiveCounter() = default;
+    LiveCounter(const LiveCounter& other) : value(other.value.load()) {}
+    LiveCounter& operator=(const LiveCounter& other) {
+        value.store(other.value.load());
+        return *this;
+    }
+    operator std::int64_t() const { return value.load(); }
+    LiveCounter& operator++() {
+        ++value;
+        return *this;
+    }
+    LiveCounter& operator--() {
+        --value;
+        return *this;
+    }
+    LiveCounter& operator+=(std::int64_t count) {
+        value += count;
+        return *this;
+    }
+    LiveCounter& operator-=(std::int64_t count) {
+        value -= count;
+        return *this;
+    }
+};
+
 struct MockVulkan {
     std::uint64_t next = 1;
-    std::int64_t live = 0;
+    LiveCounter live;
     std::map<VkBuffer, VkDeviceSize> bufferSizes;
     std::map<VkBuffer, VkBufferUsageFlags> bufferUsage;
     std::map<VkDeviceMemory, VkMemoryAllocateFlags> allocationFlags;
@@ -3677,6 +3787,7 @@ struct MockVulkan {
     std::vector<VkDescriptorPoolSize> poolSizes;
     std::uint32_t poolMaxSets = 0;
     VkDescriptorPoolCreateFlags poolFlags = 0;
+    std::uint32_t freedDescriptorSets = 0;
     struct DescriptorPool {
         VkDescriptorPoolCreateFlags flags;
         std::uint32_t maxSets;
@@ -3844,6 +3955,15 @@ VKAPI_ATTR VkResult VKAPI_CALL mockResetDescriptorPool(VkDevice, VkDescriptorPoo
     return VK_SUCCESS;
 }
 
+VKAPI_ATTR VkResult VKAPI_CALL mockFreeDescriptorSets(VkDevice, VkDescriptorPool handle, std::uint32_t count, const VkDescriptorSet*) {
+    auto& pool = mock.descriptorPools.at(handle);
+    Require((pool.flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT) != 0, "a descriptor set was freed from a pool without free support");
+    Require(count <= pool.allocated, "more descriptor sets were freed than allocated");
+    pool.allocated -= count;
+    mock.freedDescriptorSets += count;
+    return VK_SUCCESS;
+}
+
 VKAPI_ATTR VkResult VKAPI_CALL mockCreateSampler(VkDevice, const VkSamplerCreateInfo* info, const VkAllocationCallbacks*, VkSampler* sampler) {
     *sampler = makeHandle<VkSampler>();
     mock.samplerFlags[*sampler] = info->flags;
@@ -4029,6 +4149,7 @@ PFN_vkVoidFunction VKAPI_CALL mockProc(VkDevice, const char* name) {
         {"vkCreateDescriptorPool", reinterpret_cast<PFN_vkVoidFunction>(mockCreateDescriptorPool)},
         {"vkDestroyDescriptorPool", reinterpret_cast<PFN_vkVoidFunction>(mockDestroyDescriptorPool)},
         {"vkAllocateDescriptorSets", reinterpret_cast<PFN_vkVoidFunction>(mockAllocateDescriptorSets)},
+        {"vkFreeDescriptorSets", reinterpret_cast<PFN_vkVoidFunction>(mockFreeDescriptorSets)},
         {"vkResetDescriptorPool", reinterpret_cast<PFN_vkVoidFunction>(mockResetDescriptorPool)},
         {"vkCreateSampler", reinterpret_cast<PFN_vkVoidFunction>(mockCreateSampler)},
         {"vkDestroySampler", reinterpret_cast<PFN_vkVoidFunction>(mockDestroySampler)},
@@ -4439,6 +4560,152 @@ void descriptorSnapshotTests() {
     }
 }
 
+alignas(16) std::array<std::uint32_t, 8> guestMovedFirst{0x51515151, 0x52525252, 0x53535353, 0x54545454, 0x55555555, 0x56565656, 0x57575757, 0x58585858};
+alignas(16) std::array<std::uint32_t, 8> guestMovedSecond{0x61616161, 0x62626262, 0x63636363, 0x64646464, 0x65656565, 0x66666666, 0x67676767, 0x68686868};
+
+void dispatchMoveTests() {
+    using AgcDriver::Graphics::CompiledShader;
+    using AgcDriver::Graphics::DescriptorCache;
+    using AgcDriver::Graphics::Recorder;
+    using AgcDriver::Graphics::ShaderResources;
+    using Refusal = ShaderResources::MoveRefusal;
+    mock = MockVulkan{};
+    mock.allowCopies = true;
+    auto context = mockContext();
+    {
+        DescriptorCache cache(context);
+        context.descriptorCache = &cache;
+        std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex());
+        Recorder recorder(context);
+        ShaderRecompiler::RecompileResult built;
+        auto inputs = makeBinding(Role::GuestBuffers, 0, 2, join(vsharp(guestFirst.data(), 16), vsharp(guestSecond.data(), 32)));
+        inputs.bufferWritten = {false, false};
+        built.bindings.push_back(inputs);
+        auto output = makeBinding(Role::GuestBuffers, 4, 1, vsharp(guestThird.data(), 8));
+        output.bufferWritten = {true};
+        built.bindings.push_back(output);
+        const CompiledShader shader{ShaderRecompiler::ShaderStage::Compute, &built, 0};
+        ShaderResources resources(context, shader);
+        const auto moveTo = [&](std::size_t element, const void* where, std::uint32_t bytes) {
+            auto program = built;
+            const auto words = vsharp(where, bytes);
+            std::copy(words.begin(), words.end(), program.bindings[0].guestDescriptor.begin() + static_cast<std::ptrdiff_t>(element) * 4);
+            return program;
+        };
+        const auto prove = [&](const ShaderRecompiler::RecompileResult& program, std::size_t maxBytes, ShaderResources::MoveProof& proof) {
+            return resources.MovedDispatchBuffers(CompiledShader{ShaderRecompiler::ShaderStage::Compute, &program, 0}, &recorder, maxBytes, proof);
+        };
+        const auto refuses = [&](const ShaderRecompiler::RecompileResult& program, std::size_t maxBytes, Refusal why, const char* what) {
+            ShaderResources::MoveProof refusal;
+            const auto listed = prove(program, maxBytes, refusal);
+            Require(!listed.has_value() && refusal.refusal == why, what);
+        };
+
+        ShaderResources::MoveProof proof;
+        auto moved = prove(built, 0, proof);
+        Require(moved.has_value() && moved->empty() && proof.refusal == Refusal::None && proof.buffers == 0, "a dispatch with every buffer where it was built moved something");
+
+        const auto relocated = moveTo(1, guestMovedSecond.data(), 24);
+        const CompiledShader relocatedShader{ShaderRecompiler::ShaderStage::Compute, &relocated, 0};
+        Require(ShaderResources::ContentKey(shader, false, true) == ShaderResources::ContentKey(relocatedShader, false, true), "moving a read-only buffer changed the movable content key");
+        Require(ShaderResources::ContentKey(shader, false, false) != ShaderResources::ContentKey(relocatedShader, false, false), "the fixed content key ignored a moved buffer");
+
+        moved = prove(relocated, 0, proof);
+        Require(moved.has_value() && moved->size() == 1 && proof.refusal == Refusal::None && proof.buffers == 1 && proof.bytes >= 24, "a moved read-only buffer was not listed");
+        Require(moved->front().address == reinterpret_cast<std::uintptr_t>(guestMovedSecond.data()) && moved->front().size == 24 && moved->front().words.empty(), "the listed buffer is not the moved range");
+
+        moved = prove(moveTo(0, guestFirst.data(), 8), 0, proof);
+        Require(moved.has_value() && moved->size() == 1 && moved->front().address == reinterpret_cast<std::uintptr_t>(guestFirst.data()) && moved->front().size == 8, "a buffer that only changed its size was not listed");
+
+        auto both = relocated;
+        const auto firstWords = vsharp(guestMovedFirst.data(), 12);
+        std::copy(firstWords.begin(), firstWords.end(), both.bindings[0].guestDescriptor.begin());
+        moved = prove(both, 1024, proof);
+        Require(moved.has_value() && moved->size() == 2 && proof.buffers == 2 && proof.bytes >= 36, "two moved buffers were not both listed");
+        refuses(both, 30, Refusal::Limit, "moved buffers past the byte budget together were served");
+
+        auto written = relocated;
+        written.bindings[0].bufferWritten[1] = true;
+        refuses(written, 0, Refusal::Written, "a moved buffer the dispatch writes was served");
+        refuses(moveTo(1, guestMovedSecond.data(), 20000), 0, Refusal::Written, "a moved buffer over the storage buffer range that no registered allocation holds was served");
+        refuses(relocated, 16, Refusal::Limit, "a moved buffer over the byte budget was served");
+        refuses(moveTo(1, reinterpret_cast<const void*>(std::uintptr_t{0x10}), 24), 0, Refusal::Inaccessible, "a moved buffer over unmapped memory was served");
+        refuses(moveTo(1, guestThird.data(), 8), 0, Refusal::Overlap, "a moved buffer over the dispatch's own written range was served");
+        auto shaped = relocated;
+        shaped.bindings[0].guestDescriptor.resize(4);
+        refuses(shaped, 0, Refusal::Shape, "a dispatch with another binding shape was served");
+        recorder.Sync();
+    }
+    {
+        constexpr std::size_t blockBytes = 65536;
+        void* block = ::operator new(blockBytes, std::align_val_t{blockBytes});
+        std::memset(block, 0x5a, blockBytes);
+        {
+            auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
+            GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, block, blockBytes, true, true, true);
+            GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
+        }
+        const auto* end = static_cast<const std::byte*>(block) + blockBytes;
+        const auto strided = [](const void* pointer, std::uint32_t stride, std::uint32_t records) {
+            auto words = vsharp(pointer, records);
+            words[1] |= stride << 16u;
+            return words;
+        };
+        DescriptorCache cache(context);
+        context.descriptorCache = &cache;
+        std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex());
+        Recorder recorder(context);
+        const auto proveAgainst = [&](const ShaderRecompiler::RecompileResult& built, const std::vector<std::uint32_t>& incoming, ShaderResources::MoveProof& proof, std::optional<std::vector<ShaderResources::MovedBuffer>>& moved) {
+            const CompiledShader shader{ShaderRecompiler::ShaderStage::Compute, &built, 0};
+            ShaderResources resources(context, shader);
+            auto program = built;
+            program.bindings[0].guestDescriptor = incoming;
+            moved = resources.MovedDispatchBuffers(CompiledShader{ShaderRecompiler::ShaderStage::Compute, &program, 0}, &recorder, 0, proof);
+        };
+        const auto readOnly = [&](const std::vector<std::uint32_t>& words) {
+            ShaderRecompiler::RecompileResult built;
+            auto binding = makeBinding(Role::GuestBuffers, 0, 1, words);
+            binding.bufferWritten = {false};
+            built.bindings.push_back(binding);
+            return built;
+        };
+        const auto address = [](const void* pointer) { return reinterpret_cast<std::uintptr_t>(pointer); };
+        ShaderResources::MoveProof proof;
+        std::optional<std::vector<ShaderResources::MovedBuffer>> moved;
+
+        const auto* tail = end - 8192;
+        const auto* nearEnd = end - 4096;
+        const auto oversized = readOnly(vsharp(tail, 0xffffffffu));
+        proveAgainst(oversized, vsharp(tail, 0xffffffffu), proof, moved);
+        Require(moved.has_value() && moved->empty() && proof.refusal == Refusal::None && proof.buffers == 0, "an unchanged V# the build cut to its allocation end was not an exact hit");
+        proveAgainst(oversized, vsharp(nearEnd, 0xffffffffu), proof, moved);
+        Require(moved.has_value() && moved->size() == 1 && proof.buffers == 1 && proof.bytes == 4096 && moved->front().address == address(nearEnd) && moved->front().size == 4096, "a moved V# past the range limit was not snapshotted at its clamped range");
+        proveAgainst(oversized, vsharp(nearEnd, 0xfffffff0u), proof, moved);
+        Require(moved.has_value() && moved->size() == 1 && moved->front().size == 4096, "another oversized size over the same allocation end listed another range");
+        proveAgainst(oversized, vsharp(tail, 4096), proof, moved);
+        Require(moved.has_value() && moved->size() == 1 && moved->front().address == address(tail) && moved->front().size == 4096, "a V# that now fits the range limit was not listed at its own size");
+
+        proveAgainst(oversized, vsharp(block, 0xffffffffu), proof, moved);
+        Require(!moved.has_value() && proof.refusal == Refusal::Limit, "a moved V# the allocation holds past the range limit was served");
+
+        const auto overhang = readOnly(strided(tail, 24, 342));
+        proveAgainst(overhang, strided(tail, 24, 342), proof, moved);
+        Require(moved.has_value() && moved->empty() && proof.refusal == Refusal::None, "an unchanged V# with a stride overhang past the allocation was not an exact hit");
+        proveAgainst(overhang, strided(nearEnd, 24, 171), proof, moved);
+        Require(moved.has_value() && moved->size() == 1 && moved->front().address == address(nearEnd) && moved->front().size == 4096 && proof.bytes == 4096, "a moved V# with a stride overhang was snapshotted past the allocation end");
+        proveAgainst(overhang, strided(nearEnd, 24, 100), proof, moved);
+        Require(moved.has_value() && moved->size() == 1 && moved->front().size == 2400, "a moved V# inside the allocation was not listed at its own size");
+        recorder.Sync();
+        {
+            auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
+            GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, block);
+            GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
+        }
+        ::operator delete(block, std::align_val_t{blockBytes});
+    }
+    Require(mock.live == 0, "dispatch move resources leaked Vulkan objects");
+}
+
 void resourceTests() {
     mock = MockVulkan{};
     const auto context = mockContext();
@@ -4741,6 +5008,28 @@ void descriptorCacheTests() {
         expectFailure([&] { cache.Allocate(fittingLayout, fitting, false, 3u); }, "descriptor set exceeds the device's per-set descriptor limit");
     }
     Require(mock.live == 0, "the combined descriptor cache leaked a pool or layout");
+    mock = MockVulkan{};
+    auto lavapipeContext = mockContext();
+    lavapipeContext.individuallyFreeDescriptorSets = true;
+    lavapipeContext.limits.maxDescriptorSetStorageBuffers = 16;
+    {
+        AgcDriver::Graphics::DescriptorCache cache(lavapipeContext);
+        const VkDescriptorSetLayoutBinding binding{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_FRAGMENT_BIT, nullptr};
+        const std::array<std::uint32_t, 4> key{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_FRAGMENT_BIT};
+        const auto layout = cache.Layout(key, std::span(&binding, 1));
+        const std::array<VkDescriptorPoolSize, 1> sizes{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1}}};
+        const auto first = cache.Allocate(layout, sizes);
+        const auto second = cache.Allocate(layout, sizes);
+        Require(first.pool == second.pool && mock.poolFlags == VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT, "lavapipe chain pools do not enable individual descriptor-set frees");
+        cache.Free(first);
+        Require(mock.freedDescriptorSets == 1 && mock.descriptorPools.at(first.pool).resets == 0, "freeing one of two live sets reset its chain pool");
+        cache.Free(second);
+        Require(mock.freedDescriptorSets == 2 && mock.descriptorPools.at(second.pool).resets == 1, "freeing the last live set did not free it and reset its chain pool");
+        const auto reused = cache.Allocate(layout, sizes);
+        Require(reused.pool == first.pool && mock.descriptorPools.at(reused.pool).attempts == 3, "an emptied lavapipe chain pool was not reused");
+        cache.Free(reused);
+    }
+    Require(mock.live == 0, "the lavapipe descriptor cache leaked a pool or layout");
 }
 
 void imageTableAllocationTests() {
@@ -5008,21 +5297,33 @@ void storageTextureBudgetTests() {
     memory.memoryHeaps[1] = {32 * GiB, 0};
     memory.memoryHeaps[2] = {16 * GiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT};
     VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
-    Require(StorageTextureBudget(memory, nullptr) == 4 * GiB, "without VK_EXT_memory_budget the storage image cache does not keep a quarter of the 16 GiB heap");
+    Require(StorageTextureBudget(memory, nullptr, 0, 0) == 4 * GiB, "without VK_EXT_memory_budget the storage image cache does not keep a quarter of the 16 GiB heap");
     reported.heapBudget[0] = 100 * GiB;
     reported.heapBudget[1] = 100 * GiB;
-    Require(StorageTextureBudget(memory, &reported) == 4 * GiB, "a zero budget for the largest device-local heap did not fall back to a quarter of the heap");
+    Require(StorageTextureBudget(memory, &reported, 0, 0) == 4 * GiB, "a zero budget for the largest device-local heap did not fall back to a quarter of the heap");
     reported.heapBudget[2] = 10 * GiB;
-    Require(StorageTextureBudget(memory, &reported) == 2560 * MiB, "a 10 GiB budget on the 16 GiB heap does not limit the storage image cache to a quarter of the budget");
+    Require(StorageTextureBudget(memory, &reported, 0, 0) == 2560 * MiB, "a 10 GiB budget on the 16 GiB heap does not limit the storage image cache to a quarter of the budget");
+    reported.heapUsage[2] = 5 * GiB;
+    Require(StorageTextureBudget(memory, &reported, 1 * GiB, 2 * GiB) == 2560 * MiB, "2 GiB used outside the texture caches on a 10 GiB budget lowered the storage image cache below its quarter");
+    reported.heapUsage[2] = 8 * GiB;
+    Require(StorageTextureBudget(memory, &reported, 1 * GiB, 2 * GiB) == 1792 * MiB, "a 10 GiB budget with 5 GiB used outside the texture caches does not leave 1792 MiB after the sampled textures' 2 GiB floor and 1280 MiB of headroom");
+    reported.heapUsage[2] = 9 * GiB;
+    Require(StorageTextureBudget(memory, &reported, 3 * GiB, 2 * GiB) == 1792 * MiB, "3 GiB of sampled textures were not reserved past their 2 GiB floor");
+    reported.heapUsage[2] = 1 * GiB;
+    Require(StorageTextureBudget(memory, &reported, 1 * GiB, 2 * GiB) == 2560 * MiB, "usage below the cached bytes is not read as no other device memory in use");
+    Require(StorageTextureBudget(memory, &reported, 7 * GiB, 1 * GiB) == 1792 * MiB, "7 GiB of sampled textures with the usage below the cached bytes do not leave 1792 MiB after their reservation and 1280 MiB of headroom");
+    reported.heapUsage[2] = 9 * GiB + 512 * MiB;
+    Require(StorageTextureBudget(memory, &reported, 2 * GiB, 512 * MiB) == 512 * MiB, "a heap whose other users leave no room does not keep the 512 MiB floor");
+    reported.heapUsage[2] = 0;
     reported.heapBudget[2] = 6 * GiB;
-    Require(StorageTextureBudget(memory, &reported) == 1536 * MiB, "a 6 GiB budget on the 16 GiB heap does not limit the storage image cache to a quarter of the budget");
+    Require(StorageTextureBudget(memory, &reported, 0, 0) == 1536 * MiB, "a 6 GiB budget on the 16 GiB heap does not limit the storage image cache to a quarter of the budget");
     reported.heapBudget[2] = 1 * GiB;
-    Require(StorageTextureBudget(memory, &reported) == 512 * MiB, "a 1 GiB budget does not keep the 512 MiB floor");
+    Require(StorageTextureBudget(memory, &reported, 0, 0) == 512 * MiB, "a 1 GiB budget does not keep the 512 MiB floor");
     reported.heapBudget[2] = 40 * GiB;
-    Require(StorageTextureBudget(memory, &reported) == 4 * GiB, "a budget above the heap size raised the storage image cache past a quarter of the heap");
+    Require(StorageTextureBudget(memory, &reported, 0, 0) == 4 * GiB, "a budget above the heap size raised the storage image cache past a quarter of the heap");
     memory.memoryHeaps[0].flags = 0;
     memory.memoryHeaps[2].flags = 0;
-    Require(StorageTextureBudget(memory, &reported) == 2 * GiB, "a device without a device-local heap does not keep 2 GiB of storage images");
+    Require(StorageTextureBudget(memory, &reported, 0, 0) == 2 * GiB, "a device without a device-local heap does not keep 2 GiB of storage images");
 }
 
 void sampledBudgetReportTests() {
@@ -6109,7 +6410,67 @@ void barycentricEmulationTests() {
     expectFailure([&] { static_cast<void>(BuildBarycentricGeometryShader(vertex, plain, target, limits)); }, "does not emulate barycentrics");
     expectFailure([&] { static_cast<void>(BuildBarycentricGeometryShader(vertex, fragment, target, std::nullopt)); }, "geometry shaders are unavailable");
     expectFailure([&] { static_cast<void>(BuildBarycentricGeometryShader(vertex, fragment, target, GeometryStageLimits{64u, 12u, 256u, 1024u, 128u})); }, "exceeds device limits");
+
+    const std::array<std::uint32_t, 2> tessellationCapabilities{spv::CapabilityShader, spv::CapabilityTessellation};
+    SpirvTarget tessellationTarget = target;
+    tessellationTarget.supportedCapabilities = tessellationCapabilities;
+    tessellationTarget.tessellation = TessellationTargetLimits{32u, 128u, 128u, 120u, 4096u, 128u, 128u};
+    const auto modes = [](const RecompileResult& result) {
+        std::vector<std::uint32_t> values;
+        const auto& words = result.spirv.Words();
+        for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
+            if (static_cast<spv::Op>(words[at] & 0xffffu) == spv::OpExecutionMode) values.push_back(words[at + 2]);
+        }
+        return values;
+    };
+    auto patches = BuildBarycentricTessellationShaders(vertex, fragment, tessellationTarget, 128u);
+    Require(modes(patches.control) == std::vector<std::uint32_t>{spv::ExecutionModeOutputVertices, spv::ExecutionModeTriangles, spv::ExecutionModeSpacingEqual, spv::ExecutionModeVertexOrderCw}, "the generated control stage does not declare three output vertices and the triangle domain");
+    Require(modes(patches.evaluation) == std::vector<std::uint32_t>{spv::ExecutionModeTriangles, spv::ExecutionModeSpacingEqual, spv::ExecutionModeVertexOrderCw}, "the generated evaluation stage does not declare the triangle domain");
+    const std::array<CompiledShader, 4> patchShaders{{{ShaderStage::Vertex, &vertex, 0}, {ShaderStage::TessellationControl, &patches.control, 0}, {ShaderStage::TessellationEvaluation, &patches.evaluation, 0}, {ShaderStage::Fragment, &fragment, 0}}};
+    Require(BarycentricPatches(state, patchShaders) && !BarycentricPatches(state, shaders), "generated tessellation stages were not recognised as barycentric patches");
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    ValidateShaders(patchShaders, state, subgroup, false);
+    for (const auto topology : {VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN, VK_PRIMITIVE_TOPOLOGY_POINT_LIST}) {
+        state.topology = topology;
+        expectFailure([&] { ValidateShaders(patchShaders, state, subgroup, false); }, "require a triangle list");
+    }
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    const std::array<CompiledShader, 4> plainPatchShaders{{{ShaderStage::Vertex, &vertex, 0}, {ShaderStage::TessellationControl, &patches.control, 0}, {ShaderStage::TessellationEvaluation, &patches.evaluation, 0}, {ShaderStage::Fragment, &plain, 0}}};
+    expectFailure([&] { ValidateShaders(plainPatchShaders, state, subgroup, false); }, "stage count");
+    expectFailure([&] { static_cast<void>(BuildBarycentricTessellationShaders(vertex, plain, tessellationTarget, 128u)); }, "does not emulate barycentrics");
+    auto noTessellation = tessellationTarget;
+    noTessellation.tessellation.reset();
+    expectFailure([&] { static_cast<void>(BuildBarycentricTessellationShaders(vertex, fragment, noTessellation, 128u)); }, "tessellation shaders are unavailable");
+    expectFailure([&] { static_cast<void>(BuildBarycentricTessellationShaders(vertex, fragment, target, 128u)); }, "tessellation shaders are unavailable");
+    auto noCapability = tessellationTarget;
+    noCapability.supportedCapabilities = std::span<const std::uint32_t>(capabilities).first(1);
+    expectFailure([&] { static_cast<void>(BuildBarycentricTessellationShaders(vertex, fragment, noCapability, 128u)); }, "tessellation capability is unavailable");
+    auto smallPatches = tessellationTarget;
+    smallPatches.tessellation->maxEvaluationOutputComponents = 4u;
+    expectFailure([&] { static_cast<void>(BuildBarycentricTessellationShaders(vertex, fragment, smallPatches, 128u)); }, "exceeds device limits");
+    expectFailure([&] { static_cast<void>(BuildBarycentricTessellationShaders(vertex, fragment, tessellationTarget, 0u)); }, "exceeds device limits");
+
+    RecompileResult clipped;
+    clipped.spirv = makeModule({.parameterOutput = true, .clipDistanceLength = 2});
+    clipped.parameterExports = {0u};
+    expectFailure([&] { static_cast<void>(BuildBarycentricTessellationShaders(clipped, fragment, tessellationTarget, 128u)); }, "need shaderClipDistance");
+    const std::array<std::uint32_t, 3> clipCapabilities{spv::CapabilityShader, spv::CapabilityTessellation, spv::CapabilityClipDistance};
+    auto clipTarget = tessellationTarget;
+    clipTarget.supportedCapabilities = clipCapabilities;
+    auto clippedPatches = BuildBarycentricTessellationShaders(clipped, fragment, clipTarget, 128u);
+    const auto memberBuiltIn = [](const RecompileResult& result, spv::BuiltIn builtin) {
+        const auto& words = result.spirv.Words();
+        for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
+            if (static_cast<spv::Op>(words[at] & 0xffffu) == spv::OpMemberDecorate && words[at + 3] == spv::DecorationBuiltIn && words[at + 4] == builtin) return true;
+        }
+        return false;
+    };
+    Require(!memberBuiltIn(patches.control, spv::BuiltInClipDistance) && !memberBuiltIn(patches.evaluation, spv::BuiltInClipDistance), "a generated tessellation stage declares clip distances the vertex shader does not write");
+    Require(memberBuiltIn(clippedPatches.control, spv::BuiltInClipDistance) && memberBuiltIn(clippedPatches.evaluation, spv::BuiltInClipDistance), "a generated tessellation stage does not forward the vertex clip distances");
+    const std::array<CompiledShader, 4> clippedShaders{{{ShaderStage::Vertex, &clipped, 0}, {ShaderStage::TessellationControl, &clippedPatches.control, 0}, {ShaderStage::TessellationEvaluation, &clippedPatches.evaluation, 0}, {ShaderStage::Fragment, &fragment, 0}}};
+    ValidateShaders(clippedShaders, state, subgroup, false, false, false, false, false, false, true);
     vertex.parameterExports.clear();
+    expectFailure([&] { static_cast<void>(BuildBarycentricTessellationShaders(vertex, fragment, tessellationTarget, 128u)); }, "not exported by the vertex shader");
     expectFailure([&] { static_cast<void>(BuildBarycentricGeometryShader(vertex, fragment, target, limits)); }, "not exported by the vertex shader");
 }
 
@@ -6240,6 +6601,23 @@ void registeredReadableCoverTests() {
     ::operator delete(memory, std::align_val_t{block});
 }
 
+std::size_t VariableBuiltIns(const ShaderRecompiler::RecompileResult& result, spv::BuiltIn builtin) {
+    std::size_t count = 0;
+    const auto& words = result.spirv.Words();
+    for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
+        if (static_cast<spv::Op>(words[at] & 0xffffu) == spv::OpDecorate && (words[at] >> 16u) == 4u && words[at + 2] == spv::DecorationBuiltIn && words[at + 3] == builtin) ++count;
+    }
+    return count;
+}
+
+bool MemberBuiltIn(const ShaderRecompiler::RecompileResult& result, spv::BuiltIn builtin) {
+    const auto& words = result.spirv.Words();
+    for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
+        if (static_cast<spv::Op>(words[at] & 0xffffu) == spv::OpMemberDecorate && words[at + 3] == spv::DecorationBuiltIn && words[at + 4] == builtin) return true;
+    }
+    return false;
+}
+
 void rectListTests() {
     using namespace ShaderRecompiler;
     using namespace AgcDriver::Graphics;
@@ -6273,17 +6651,35 @@ void rectListTests() {
             expectFailure([&] { ValidateShaders(shaders, state, VkPhysicalDeviceSubgroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}, false); }, "stage count");
         }
     }
+    {
+        RecompileResult clipped;
+        clipped.spirv = makeModule({.clipDistanceLength = 2});
+        const std::array<std::uint32_t, 3> clipCapabilities{spv::CapabilityShader, spv::CapabilityTessellation, spv::CapabilityClipDistance};
+        auto clipTarget = target;
+        clipTarget.supportedCapabilities = clipCapabilities;
+        auto clippedRect = BuildRectListShaders(clipped, fragment, clipTarget);
+        for (const auto* stage : {&clippedRect.control, &clippedRect.lastVertexControl, &clippedRect.evaluation}) {
+            Require(!MemberBuiltIn(*stage, spv::BuiltInClipDistance) && VariableBuiltIns(*stage, spv::BuiltInClipDistance) == 2u, "a rect-list stage does not take and forward the vertex clip distances through variables of their own, as the vertex shader writes them");
+        }
+        const std::array<CompiledShader, 4> clippedShaders{{{ShaderStage::Vertex, &clipped, 0}, {ShaderStage::TessellationControl, &clippedRect.control, 0}, {ShaderStage::TessellationEvaluation, &clippedRect.evaluation, 0}, {ShaderStage::Fragment, &fragment, 0}}};
+        auto queue = makeState();
+        queue.userConfig[0x242] = 17;
+        ValidateShaders(clippedShaders, DecodeState(queue), VkPhysicalDeviceSubgroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}, false, false, false, false, false, false, true);
+    }
     vertex.spirv = makeModule({.parameterOutput = true});
     fragment.spirv = makeModule({.fragment = true, .rectParameters = true});
     vertex.parameterExports = {0};
     fragment.fragmentParameters = {{0, 0, false, false}, {1, 0, true, false}};
     auto auxiliary = BuildRectListShaders(vertex, fragment, target);
     Require(!auxiliary.control.spirv.empty() && !auxiliary.evaluation.spirv.empty(), "rect-list parameter shaders are empty");
+    Require(faultAuxiliary.lastVertexControl.Identity() == faultAuxiliary.control.Identity() && auxiliary.lastVertexControl.Identity() != auxiliary.control.Identity() && auxiliary.lastVertexControl.bindings.size() == 1 && auxiliary.lastVertexControl.bdaAbiVersion == auxiliary.control.bdaAbiVersion, "the last-vertex rect-list control shader does not differ from the first-vertex one exactly when a flat parameter is present");
     auto parameterQueue = makeState();
     parameterQueue.userConfig[0x242] = 17;
     const auto parameterState = DecodeState(parameterQueue);
-    const std::array<CompiledShader, 4> parameterShaders{{{ShaderStage::Vertex, &vertex, 0}, {ShaderStage::TessellationControl, &auxiliary.control, 0}, {ShaderStage::TessellationEvaluation, &auxiliary.evaluation, 0}, {ShaderStage::Fragment, &fragment, 0}}};
-    ValidateShaders(parameterShaders, parameterState, VkPhysicalDeviceSubgroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}, false);
+    for (const auto* control : {&auxiliary.control, &auxiliary.lastVertexControl}) {
+        const std::array<CompiledShader, 4> parameterShaders{{{ShaderStage::Vertex, &vertex, 0}, {ShaderStage::TessellationControl, control, 0}, {ShaderStage::TessellationEvaluation, &auxiliary.evaluation, 0}, {ShaderStage::Fragment, &fragment, 0}}};
+        ValidateShaders(parameterShaders, parameterState, VkPhysicalDeviceSubgroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}, false);
+    }
     fragment.fragmentParameters[0].perVertex = true;
     auto explicitInterpolation = BuildRectListShaders(vertex, fragment, target);
     Require(!explicitInterpolation.control.spirv.empty() && !explicitInterpolation.evaluation.spirv.empty(), "rect-list shaders for an explicitly interpolated parameter are empty");
@@ -7148,13 +7544,16 @@ void ProvokingVertexTests() {
         generated.provokingVertexMode = VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT;
         ValidateProvokingVertex(unsupported, generated);
     }
-    for (std::uint32_t generated = 0; generated < 3; ++generated) {
+    for (std::uint32_t generated = 0; generated < 2; ++generated) {
         auto state = last;
-        if (generated == 0) state.rectList = true;
-        if (generated == 1) state.stages.mesh.emplace();
-        if (generated == 2) state.stages.tessellation.emplace();
+        if (generated == 0) state.stages.mesh.emplace();
+        if (generated == 1) state.stages.tessellation.emplace();
         expectFailure([&] { ValidateProvokingVertex(supported, state); }, "last provoking vertex is unsupported for generated primitive pipelines");
     }
+    auto rectangle = last;
+    rectangle.rectList = true;
+    rectangle.topology = VK_PRIMITIVE_TOPOLOGY_PATCH_LIST;
+    ValidateProvokingVertex(supported, rectangle, flatShaders);
     last.provokingVertexMode = VK_PROVOKING_VERTEX_MODE_MAX_ENUM_EXT;
     expectFailure([&] { ValidateProvokingVertex(supported, last); }, "invalid provoking vertex mode");
 }
@@ -7398,13 +7797,78 @@ void unorm10_11_11TargetTests() {
         return AgcDriver::Graphics::ValidateShaders(shaders, target, subgroup, false);
     };
     Require(validate(packed, state) == std::set<std::uint32_t>{0u}, "the packed export did not reach attachment 0");
-    expectFailure([&] { validate(unpacked, state); }, "uint4 words to its packed 10_11_11 unorm attachments");
+    expectFailure([&] { validate(unpacked, state); }, "uint4 words to its packed color attachments");
     auto plainState = state;
     plainState.colors[0].packing = ColorExportPacking::None;
     expectFailure([&] { validate(packed, plainState); }, "float4 colors");
     auto uintExport = request;
     uintExport.context.pixel->targetOutputMode[0] = 7;
-    expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(uintExport)); }, "fragment output 0 has no 10_11_11 unorm packing path");
+    expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(uintExport)); }, "fragment output 0 has no color export packing path");
+}
+
+void float11_11_10TargetTests() {
+    using ShaderRecompiler::ColorExportPacking;
+    auto queue = makeState();
+    queue.context[0x31c] = (queue.context.at(0x31c) & ~0x1f7cu) | (7u << 2u) | (7u << 8u) | (2u << 11u) | 0x40000u;
+    queue.context[0x1b3] = 2;
+    queue.context[0x1b4] = 2;
+    const auto state = AgcDriver::Graphics::DecodeState(queue);
+    Require(state.color.format == VK_FORMAT_R32_UINT && state.color.elementBytes == 4 && state.color.bytes == colorMemory.size() && state.color.componentMapping == 0xc6u && state.color.packing == ColorExportPacking::Float11_11_10, "an 11_11_10 float target did not decode as a packed R32_UINT attachment with blue in the low bits");
+    const auto packings = AgcDriver::Graphics::ExportPackings(state);
+    Require(packings[0] == ColorExportPacking::Float11_11_10 && std::all_of(packings.begin() + 1, packings.end(), [](ColorExportPacking packing) { return packing == ColorExportPacking::None; }), "export 0 did not carry the 11_11_10 float packing alone");
+    Require(AgcDriver::Graphics::ExportMappings(state)[0] == 0xc6u, "export 0 did not reverse its components into the 11_11_10 float target");
+    auto changed = queue;
+    changed.context[0x31c] &= ~0x40000u;
+    expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "rounding (ROUND_MODE 0) 11_11_10 float color targets");
+    for (const auto swap : {0u, 1u, 3u}) {
+        changed = queue;
+        changed.context[0x31c] = (changed.context[0x31c] & ~0x1800u) | (swap << 11u);
+        expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "unsupported color format 7 number type 7 component swap " + std::to_string(swap));
+    }
+    for (const auto number : {0u, 4u}) {
+        changed = queue;
+        changed.context[0x31c] = (changed.context[0x31c] & ~0x700u) | (number << 8u) | 0x8000u;
+        expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "unsupported color format 7 number type " + std::to_string(number) + " component swap 2");
+    }
+    changed = queue;
+    changed.context[0x1e0] = 0x60010101u;
+    for (std::uint32_t i = 0; i < 4; ++i) changed.context[0x105 + i] = 0;
+    expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "blending into an 11_11_10 float color target");
+    changed = queue;
+    changed.context[0x8e] = 6;
+    expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "partial writes of an 11_11_10 float color target");
+    changed.context[0x8e] = 7;
+    Require(AgcDriver::Graphics::DecodeState(changed).color.packing == ColorExportPacking::Float11_11_10, "an RGB write of an 11_11_10 float target was refused");
+    for (const auto format : {5u, 6u}) {
+        changed = queue;
+        changed.context[0x1c5] = format;
+        expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "color export format " + std::to_string(format) + " into an 11_11_10 float target");
+    }
+    changed = queue;
+    changed.context[0x1c5] = 4;
+    Require(AgcDriver::Graphics::DecodeState(changed).color.packing == ColorExportPacking::Float11_11_10, "an FP16_ABGR export into an 11_11_10 float target was refused");
+    changed = queue;
+    changed.context[0x31c] |= 0x10000000u;
+    changed.context[0x325] = 0x1234;
+    expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "DCC-compressed 11_11_10 float color targets");
+
+    constexpr std::array<std::uint32_t, 3> code{0xf800180fu, 0x07060504u, 0xbf810000u};
+    const auto request = packedExportRequest(queue, state, code);
+    auto unorm = request;
+    unorm.context.pixel->targetExportPacking[0] = ColorExportPacking::Unorm10_11_11;
+    const ShaderRecompiler::RequestSerializer serializer;
+    Require(serializer.Deserialize(serializer.Serialize(request)).request.context.pixel->targetExportPacking == request.context.pixel->targetExportPacking, "the 11_11_10 float packing was lost in serialization");
+    const auto packed = ShaderRecompiler::Recompile(request);
+    const auto unormPacked = ShaderRecompiler::Recompile(unorm);
+    Require(packed.variantId == unormPacked.variantId && packed.PipelineVariantId() != unormPacked.PipelineVariantId(), "the 11_11_10 float packing did not key its own specialization of the shared artifact");
+    ShaderRecompiler::RecompileResult vertex;
+    vertex.spirv = makeModule({});
+    const VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES};
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &packed, 0}}};
+    Require(AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false) == std::set<std::uint32_t>{0u}, "the packed float export did not reach attachment 0");
+    auto invalid = request;
+    invalid.context.pixel->targetExportPacking[0] = static_cast<ColorExportPacking>(3);
+    expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(invalid)); }, "invalid prepared fragment export packing");
 }
 
 AgcDriver::QueueState dualSourceQueue() {
@@ -7992,6 +8456,16 @@ void stagingPoolBudgetTests() {
     Require(AgcDriver::Graphics::BufferPool::DeviceBudget(memory) == VkDeviceSize{128} << 20u, "the staging pool budget on system memory fell below 128 MiB");
     memory.memoryTypes[0] = {VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 1};
     Require(AgcDriver::Graphics::BufferPool::DeviceBudget(memory) == VkDeviceSize{512} << 20u, "a heap with a device-only memory type was taken for system memory");
+    memory.memoryHeaps[1].size = VkDeviceSize{16} << 30u;
+    VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
+    Require(AgcDriver::Graphics::BufferPool::DeviceBudget(memory, &reported) == VkDeviceSize{2} << 30u, "a zero reported budget did not fall back to an eighth of the heap");
+    reported.heapBudget[0] = VkDeviceSize{1} << 30u;
+    reported.heapBudget[1] = VkDeviceSize{10} << 30u;
+    Require(AgcDriver::Graphics::BufferPool::DeviceBudget(memory, &reported) == VkDeviceSize{1280} << 20u, "a 10 GiB budget on the 16 GiB heap does not limit the staging pool to an eighth of the budget");
+    reported.heapBudget[1] = VkDeviceSize{2} << 30u;
+    Require(AgcDriver::Graphics::BufferPool::DeviceBudget(memory, &reported) == VkDeviceSize{512} << 20u, "a 2 GiB budget does not keep the 512 MiB floor");
+    reported.heapBudget[1] = VkDeviceSize{40} << 30u;
+    Require(AgcDriver::Graphics::BufferPool::DeviceBudget(memory, &reported) == VkDeviceSize{2} << 30u, "a budget above the heap size raised the staging pool past an eighth of the heap");
 }
 
 void meshArgumentTests() {
@@ -8265,6 +8739,35 @@ void ResourceContentKeyTests(bool benchmark) {
 // into a second device) rebuilds the recorded graphics pipeline so the same configuration hits the
 // store, hands the compute result to its hook, recovers a tail torn inside a record's padding and
 // stops when cancelled.
+void pipelineStoreLazyBufferPoolTests() {
+    using namespace AgcDriver::Graphics;
+    mock = MockVulkan{};
+    auto context = mockContext();
+    context.device = makeHandle<VkDevice>();
+    context.graphicsPipelineLibrary = true;
+    context.pipelineLinkTimeOptimization = false;
+    context.limits.framebufferNoAttachmentsSampleCounts = VK_SAMPLE_COUNT_1_BIT;
+    ShaderRecompiler::RecompileResult vertex, fragment;
+    vertex.spirv = makeModule({});
+    fragment.spirv = makeModule({.fragment = true});
+    vertex.variantId = 201;
+    fragment.variantId = 202;
+    const std::array<CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, PipelinePushSlotBytes}}};
+    std::array<std::uint32_t, 1> mainKey{31}, fragmentKey{32};
+    const PipelineSetLayouts resources{makeHandle<VkDescriptorSetLayout>(), makeHandle<VkDescriptorSetLayout>(), true, mainKey, fragmentKey};
+    Require(context.bufferPool == nullptr, "the lazy buffer pool fixture already owns a buffer pool");
+    auto first = CachedPipeline(context, State{}, VertexInputLayout{}, resources, shaders);
+    const auto created = mock.pipelineCreateCount;
+    static_cast<void>(GetBufferPool(context));
+    auto again = CachedPipeline(context, State{}, VertexInputLayout{}, resources, shaders);
+    Require(again == first && mock.pipelineCreateCount == created, "creating the buffer pool dropped a cached pipeline of the live device");
+    ClearCachedPipelines(context.device);
+    again.reset();
+    first.reset();
+    context.bufferPool.reset();
+    Require(mock.live == 0, "lazy buffer pool pipeline tests leaked Vulkan objects");
+}
+
 void pipelineJournalTests() {
     using namespace AgcDriver::Graphics;
     using Stage = ShaderRecompiler::ShaderStage;
@@ -8553,6 +9056,10 @@ int main(int argc, char** argv) {
     setenv("APS5_HEAP_MIRROR_MIB", "4", 1);
 #endif
     try {
+        if (argc == 2 && std::string_view(argv[1]) == "--descriptor-cache") {
+            descriptorCacheTests();
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--unused-stencil-reference") {
             stencilReferenceUnusedTests();
             return 0;
@@ -8567,6 +9074,11 @@ int main(int argc, char** argv) {
             std::cout << "pipeline library clear waits for active part builds\n";
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--pipeline-store-lazy-buffer-pool") {
+            pipelineStoreLazyBufferPoolTests();
+            std::cout << "cached pipelines survive lazy buffer pool creation\n";
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--sparse-buffer-holes") {
             bool passed = false;
             {
@@ -8692,6 +9204,7 @@ int main(int argc, char** argv) {
         RunGuestLeaseWaitTests();
         ResourceContentKeyTests(false);
         stateTests();
+        pipelineStoreLazyBufferPoolTests();
         retainedPixelStageTests();
         pipelineLibrarySetTests();
         pipelineLibraryClearWaitsForBuildTests();
@@ -8706,6 +9219,7 @@ int main(int argc, char** argv) {
         memoryValueFillTests();
         ZExportTests();
         unorm10_11_11TargetTests();
+        float11_11_10TargetTests();
         DepthBoundsBiasTests();
         conservativeZExportTests();
         earlyFragmentTestsTests();
@@ -8719,8 +9233,10 @@ int main(int argc, char** argv) {
         ConservativeRasterizationTests();
         DisabledColorTests();
         OpaqueDestinationAlphaTests();
+        Color565TargetTests();
         Color1555TargetTests();
         BlendBypassTests();
+        UnclampedNormalizedColorTests();
         CompactedExportTests();
         ReversedComponentOrderTests();
         ReversedComponentOrderBlendTests();
@@ -8761,6 +9277,7 @@ int main(int argc, char** argv) {
         oversizedDescriptorTests();
         guestBufferViewBytesTests();
         descriptorSnapshotTests();
+        dispatchMoveTests();
         descriptorCacheTests();
         bufferBudgetPlacementTests();
         imageTableAllocationTests();

core/libs/prx/libSceAgcDriver/tests/GuardedMemoryLoads.cpp

@@ -0,0 +1,370 @@
+#define SPV_ENABLE_UTILITY_CODE
+#include <spirv/unified1/spirv.hpp>
+#undef SPV_ENABLE_UTILITY_CODE
+#include "IntermediateRepresentation/IrProgram.hpp"
+#include "Optimization/ShaderStageInputInfo.hpp"
+#include "SpirvBackend/SpirvEmitter.hpp"
+#include "PipelineSpecialization.hpp"
+#include <array>
+#include <cstdint>
+#include <cstring>
+#include <iostream>
+#include <map>
+#include <span>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace {
+
+using namespace ShaderRecompiler;
+
+void require(bool condition, const std::string& message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+struct LoadCase {
+    ResourceKind kind;
+    IrOpcode opcode;
+    IrType type;
+    std::uint32_t bits = 32;
+    std::uint32_t components = 1;
+};
+
+std::vector<std::uint32_t> emitLoad(const LoadCase& test, bool active, std::uint32_t offset, std::uint32_t records, bool robust) {
+    IrProgram program;
+    program.SetWaveSize(32);
+    program.Resources().stage = IrShaderStage::Compute;
+    program.Resources().srtPlanComplete = true;
+    program.Resources().resourceTrackingComplete = true;
+    program.Metadata().shaderInfoComplete = true;
+    program.Metadata().bindingLayoutComplete = true;
+    auto& block = program.CreateBlock();
+    program.SetEntryBlock(block);
+    program.BlockOrder().push_back(&block);
+    program.Metadata().blockInfo.emplace_back();
+    const auto constant = [&](std::uint32_t number) -> IrValue& {
+        auto& value = program.CreateValue(IrOpcode::Void, IrType::U32);
+        value.SetImmediateU32(number);
+        return value;
+    };
+    const auto emit = [&](IrOpcode opcode, IrType type, std::initializer_list<IrValue*> arguments) -> IrValue& {
+        auto& value = program.CreateValue(opcode, type);
+        for (auto* argument : arguments) value.AddArgument(argument);
+        block.AppendInstruction(&value);
+        return value;
+    };
+    auto& enabled = program.CreateValue(IrOpcode::Void, IrType::Bool);
+    enabled.SetImmediateBool(active);
+    auto& zero = constant(0);
+    auto& address = constant(offset);
+    MemoryInfo memory;
+    memory.kind = test.kind;
+    memory.dataBits = test.bits;
+    memory.dataDwords = test.components;
+    memory.componentCount = test.components;
+    program.Resources().memoryInfo.push_back(memory);
+    if (test.kind == ResourceKind::Scratch) {
+        program.Info().scratchDwords = 16;
+        auto& handle = emit(IrOpcode::GetScratchResource, IrType::AddressResource, {});
+        emit(test.opcode, test.type, {&handle, &address, &zero, &enabled});
+    } else if (test.kind == ResourceKind::Buffer) {
+        program.Resources().memoryInfo.front().coherent = true;
+        program.Info().buffers.emplace_back().read = true;
+        program.Metadata().bindings.descriptors.push_back({DescriptorBindingKind::Buffers, {0}});
+        program.Metadata().bindings.pushDataStartDword = 0;
+        program.Metadata().bindings.memoryOffsetCount = 1;
+        auto& size = constant(records);
+        auto& handle = emit(IrOpcode::GetBufferResource, IrType::BufferResource, {&zero, &zero, &size, &zero});
+        emit(test.opcode, test.type, {&handle, &zero, &address, &zero, &enabled});
+    } else {
+        if (test.kind == ResourceKind::Gds) program.Metadata().bindings.descriptors.push_back({DescriptorBindingKind::Gds, {0}});
+        emit(test.opcode, test.type, {&address, &enabled});
+    }
+    ShaderComputeInputInfo compute;
+    compute.threadsNum[0] = 32;
+    compute.threadsNum[1] = 1;
+    compute.threadsNum[2] = 1;
+    compute.ldsSizeDwords = 16;
+    compute.scratchSizeDwords = 16;
+    compute.hostSubgroupSize = 32;
+    compute.waveSize = 32;
+    ShaderStageInputInfo inputs;
+    inputs.compute = &compute;
+    CompiledBindingLayout bindings;
+    bindings.layout = program.Metadata().bindings;
+    SpirvTargetOptions target{};
+    target.vulkanVersion = 0x00401000;
+    target.spirvVersion = 0x00010300;
+    target.subgroupSize = 32;
+    target.robustBufferAccess2 = robust;
+    return SpirvEmitter().Emit(program, inputs, bindings, target);
+}
+
+class AccessTrace {
+public:
+    AccessTrace(const std::vector<std::uint32_t>& module, std::uint32_t misalignment = 0, bool valid = true)
+        : words(module), misalignment(misalignment), valid(valid) {
+        require(words.size() >= 5, "SPIR-V module header is missing");
+        for (std::size_t offset = 5; offset < words.size();) {
+            const auto count = words[offset] >> 16;
+            require(count != 0 && count <= words.size() - offset, "SPIR-V instruction is truncated");
+            const auto instruction = std::span(words).subspan(offset, count);
+            const auto opcode = static_cast<spv::Op>(instruction[0] & 0xffff);
+            bool result = false;
+            bool type = false;
+            spv::HasResultAndType(opcode, &result, &type);
+            if (result) definitions.emplace(instruction[type ? 2 : 1], instruction);
+            if (opcode == spv::OpName) names.emplace(instruction[1], std::string(reinterpret_cast<const char*>(instruction.data() + 2)));
+            if (opcode == spv::OpDecorate && instruction[2] == spv::DecorationSpecId) specializations.emplace(instruction[1], instruction[3]);
+            if (opcode == spv::OpLabel) labels.emplace(instruction[1], offset);
+            if (opcode == spv::OpEntryPoint) entryFunction = instruction[2];
+            offset += count;
+        }
+    }
+
+    std::vector<std::uint32_t> Run() {
+        std::size_t cursor = 0;
+        bool entry = false;
+        for (std::size_t offset = 5; offset < words.size(); offset += words[offset] >> 16) {
+            const auto opcode = static_cast<spv::Op>(words[offset] & 0xffff);
+            if (opcode == spv::OpFunction) entry = words[offset + 2] == entryFunction;
+            if (entry && opcode == spv::OpLabel) { cursor = offset; break; }
+        }
+        require(cursor != 0, "SPIR-V entry block is missing");
+        std::vector<std::uint32_t> reads;
+        std::uint32_t previous = 0;
+        std::uint32_t current = 0;
+        for (std::size_t steps = 0; steps < 4096; ++steps) {
+            const auto instruction = std::span(words).subspan(cursor, words[cursor] >> 16);
+            const auto opcode = static_cast<spv::Op>(instruction[0] & 0xffff);
+            if (opcode == spv::OpLabel) current = instruction[1];
+            if (opcode == spv::OpPhi) {
+                bool found = false;
+                for (std::size_t index = 3; index + 1 < instruction.size(); index += 2) {
+                    if (instruction[index + 1] == previous) { phiValues[instruction[2]] = instruction[index]; found = true; }
+                }
+                require(found, "SPIR-V phi has no incoming edge for the executed path");
+            }
+            if (opcode == spv::OpLoad && isGuestPointer(instruction[3])) {
+                const auto pointer = definition(instruction[3]);
+                require((pointer[0] & 0xffff) == spv::OpAccessChain, "guest read is not an indexed access");
+                reads.push_back(value(pointer.back()));
+                if (rootName(instruction[3]) == "scratch_dwords") require(pointerStorage(instruction[3]) == spv::StorageClassFunction, "scratch selection accessed non-private storage");
+                if (rootName(instruction[3]).starts_with("buffers")) require(instruction.size() == 5 && (instruction[4] & spv::MemoryAccessVolatileMask), "coherent buffer read lost its volatile access");
+            }
+            if (opcode == spv::OpBranch || opcode == spv::OpBranchConditional) {
+                const auto destination = opcode == spv::OpBranch ? instruction[1] : instruction[value(instruction[1]) ? 2 : 3];
+                previous = current;
+                cursor = labels.at(destination);
+                continue;
+            }
+            if (opcode == spv::OpReturn) return reads;
+            require(opcode != spv::OpFunctionCall && opcode != spv::OpSwitch && opcode != spv::OpLoopMerge, "access fixture unexpectedly requires a call, switch or loop");
+            cursor += instruction.size();
+            require(cursor < words.size(), "SPIR-V execution fell outside the function");
+        }
+        throw std::runtime_error("SPIR-V access trace exceeded its bounded instruction budget");
+    }
+
+private:
+    std::span<const std::uint32_t> definition(std::uint32_t id) const {
+        const auto found = definitions.find(id);
+        require(found != definitions.end(), "SPIR-V access condition refers to an undefined value");
+        return found->second;
+    }
+
+    std::string rootName(std::uint32_t id) const {
+        const auto instruction = definition(id);
+        if ((instruction[0] & 0xffff) == spv::OpAccessChain) return rootName(instruction[3]);
+        const auto found = names.find(id);
+        return found == names.end() ? std::string() : found->second;
+    }
+
+    std::uint32_t pointerStorage(std::uint32_t id) const {
+        const auto instruction = definition(id);
+        if ((instruction[0] & 0xffff) == spv::OpAccessChain) return pointerStorage(instruction[3]);
+        require((instruction[0] & 0xffff) == spv::OpVariable, "access pointer has no variable root");
+        return instruction[3];
+    }
+
+    bool isGuestPointer(std::uint32_t id) const {
+        const auto name = rootName(id);
+        return name == "lds_dwords" || name == "gds" || name == "scratch_dwords" || name.starts_with("buffers");
+    }
+
+    std::uint32_t value(std::uint32_t id) const {
+        const auto instruction = definition(id);
+        const auto opcode = static_cast<spv::Op>(instruction[0] & 0xffff);
+        const auto argument = [&](std::size_t index) { return value(instruction[index]); };
+        switch (opcode) {
+        case spv::OpConstantTrue: return 1;
+        case spv::OpConstantFalse: case spv::OpConstantNull: return 0;
+        case spv::OpConstant: return instruction[3];
+        case spv::OpSpecConstant: {
+            const auto spec = specializations.at(id);
+            if (spec == PipelineSpecialization::BufferBase) return 0;
+            if (spec == PipelineSpecialization::BufferBase + 1) return 0x30001000;
+            if (spec == PipelineSpecialization::BufferBase + 2) return valid;
+            if (spec == PipelineSpecialization::BufferBase + 3) return misalignment;
+            if (spec == PipelineSpecialization::PushDataOffset) return 0;
+            throw std::runtime_error("unexpected specialization in access fixture");
+        }
+        case spv::OpSelect: return argument(3) ? argument(4) : argument(5);
+        case spv::OpPhi: return value(phiValues.at(id));
+        case spv::OpIAdd: return argument(3) + argument(4);
+        case spv::OpISub: return argument(3) - argument(4);
+        case spv::OpIMul: return argument(3) * argument(4);
+        case spv::OpShiftLeftLogical:
+            require(argument(4) < 32, "undefined shift in access condition");
+            return argument(3) << argument(4);
+        case spv::OpShiftRightLogical:
+            require(argument(4) < 32, "undefined shift in access condition");
+            return argument(3) >> argument(4);
+        case spv::OpBitwiseAnd: return argument(3) & argument(4);
+        case spv::OpBitwiseOr: return argument(3) | argument(4);
+        case spv::OpBitFieldUExtract: {
+            const auto count = argument(5);
+            require(count <= 32 && argument(4) <= 32 - count, "invalid bitfield in access condition");
+            return (argument(3) >> argument(4)) & (count == 32 ? 0xffffffffu : (1u << count) - 1);
+        }
+        case spv::OpIEqual: return argument(3) == argument(4);
+        case spv::OpINotEqual: return argument(3) != argument(4);
+        case spv::OpULessThan: return argument(3) < argument(4);
+        case spv::OpULessThanEqual: return argument(3) <= argument(4);
+        case spv::OpUGreaterThanEqual: return argument(3) >= argument(4);
+        case spv::OpLogicalAnd: return argument(3) && argument(4);
+        case spv::OpLogicalOr: return argument(3) || argument(4);
+        case spv::OpLogicalNot: return !argument(3);
+        case spv::OpArrayLength:
+            require(isGuestPointer(instruction[3]), "unknown array length in access fixture");
+            return 16;
+        case spv::OpLoad:
+            require(!isGuestPointer(instruction[3]), "guest data unexpectedly controls a fixture access");
+            require(rootName(instruction[3]) == "vsharp" || rootName(instruction[3]).find("subgroup") != std::string::npos, "unknown input to access condition");
+            return 0;
+        default: throw std::runtime_error("unsupported opcode in access condition: " + std::to_string(opcode));
+        }
+    }
+
+    const std::vector<std::uint32_t>& words;
+    std::uint32_t misalignment;
+    bool valid;
+    std::uint32_t entryFunction = 0;
+    std::map<std::uint32_t, std::span<const std::uint32_t>> definitions;
+    std::map<std::uint32_t, std::string> names;
+    std::map<std::uint32_t, std::uint32_t> specializations;
+    std::map<std::uint32_t, std::size_t> labels;
+    std::map<std::uint32_t, std::uint32_t> phiValues;
+};
+
+constexpr std::uint32_t Unreachable = 0xfffffffeu;
+
+std::uint32_t ldsLength(const std::vector<std::uint32_t>& words) {
+    std::map<std::uint32_t, std::span<const std::uint32_t>> definitions;
+    std::uint32_t variable = 0;
+    for (std::size_t offset = 5; offset < words.size(); offset += words[offset] >> 16) {
+        const auto instruction = std::span(words).subspan(offset, words[offset] >> 16);
+        const auto opcode = static_cast<spv::Op>(instruction[0] & 0xffff);
+        if (opcode == spv::OpName && std::string(reinterpret_cast<const char*>(instruction.data() + 2)) == "lds_dwords") variable = instruction[1];
+        bool result = false;
+        bool type = false;
+        spv::HasResultAndType(opcode, &result, &type);
+        if (result) definitions.emplace(instruction[type ? 2 : 1], instruction);
+    }
+    require(variable != 0, "LDS array is missing");
+    const auto pointer = definitions.at(definitions.at(variable)[1]);
+    const auto array = definitions.at(pointer[3]);
+    require((array[0] & 0xffff) == spv::OpTypeArray, "LDS variable is not an array");
+    return definitions.at(array[3])[3];
+}
+
+void expectReads(const LoadCase& test, bool robust, bool active, std::uint32_t offset, std::vector<std::uint32_t> expected, std::uint32_t records = 64, std::uint32_t misalignment = 0, bool valid = true) {
+    const auto module = emitLoad(test, active, offset, records, robust);
+    const auto actual = AccessTrace(module, misalignment, valid).Run();
+    if (actual != expected) {
+        std::string detail = std::string(robust ? "robust" : "plain") + " guest reads:";
+        for (const auto index : actual) detail += " " + std::to_string(index);
+        detail += "; expected:";
+        for (const auto index : expected) detail += " " + std::to_string(index);
+        throw std::runtime_error(detail);
+    }
+}
+
+void ldsLoads(IrOpcode opcode, IrType type, std::uint32_t bits, std::uint32_t components) {
+    const LoadCase test{ResourceKind::Lds, opcode, type, bits, components};
+    require(ldsLength(emitLoad(test, true, 8, 64, false)) == 16u, "LDS array declares more than the program's dwords");
+    for (const bool robust : {false, true}) {
+        expectReads(test, robust, false, 8, {});
+        expectReads(test, robust, true, 84, {});
+        std::vector<std::uint32_t> indices;
+        for (std::uint32_t index = 0; index < components; ++index) indices.push_back(2 + index);
+        expectReads(test, robust, true, 8, indices);
+        expectReads(test, robust, true, 60, {15});
+    }
+}
+
+}
+
+int main() {
+    unsigned failures = 0;
+    const auto run = [&](std::string_view name, const auto& test) {
+        try {
+            test();
+            std::cout << "guarded memory loads: " << name << " passed\n";
+        } catch (const std::exception& error) {
+            ++failures;
+            std::cout << "guarded memory loads: " << name << ": " << error.what() << '\n';
+        }
+    };
+    run("LDS word keeps its branch", [] { ldsLoads(IrOpcode::LoadSharedU32, IrType::U32, 32, 1); });
+    run("LDS subword keeps its branch", [] {
+        ldsLoads(IrOpcode::LoadSharedU8, IrType::U8, 8, 1);
+        ldsLoads(IrOpcode::LoadSharedU16, IrType::U16, 16, 1);
+    });
+    run("LDS wide keeps its branch", [] { ldsLoads(IrOpcode::LoadSharedU32x4, IrType::U32x4, 32, 4); });
+    run("GDS word", [] {
+        const LoadCase test{ResourceKind::Gds, IrOpcode::LoadSharedU32, IrType::U32, 32, 1};
+        expectReads(test, false, false, 8, {});
+        expectReads(test, false, true, 64, {});
+        expectReads(test, false, true, 8, {2});
+        expectReads(test, true, false, 8, {Unreachable});
+        expectReads(test, true, true, 64, {Unreachable});
+        expectReads(test, true, true, 8, {2});
+        expectReads(test, true, true, 60, {15});
+    });
+    run("coherent buffer bounds", [] {
+        const LoadCase test{ResourceKind::Buffer, IrOpcode::LoadBufferU32, IrType::U32};
+        for (const bool robust : {false, true}) expectReads(test, robust, false, 8, {});
+        expectReads(test, false, true, 64, {}, 128);
+        expectReads(test, false, true, 60, {}, 32);
+        expectReads(test, false, true, 8, {}, 64, 0, false);
+        expectReads(test, true, true, 64, {Unreachable, Unreachable}, 128);
+        expectReads(test, true, true, 60, {Unreachable, Unreachable}, 32);
+        expectReads(test, true, true, 8, {Unreachable, Unreachable}, 64, 0, false);
+    });
+    run("coherent buffer high word", [] {
+        const LoadCase test{ResourceKind::Buffer, IrOpcode::LoadBufferU32, IrType::U32};
+        expectReads(test, false, true, 8, {2});
+        expectReads(test, false, true, 8, {2, 3}, 64, 1);
+        expectReads(test, false, true, 60, {}, 64, 1);
+        expectReads(test, true, true, 8, {2, 2});
+        expectReads(test, true, true, 8, {2, 3}, 64, 1);
+        expectReads(test, true, true, 60, {Unreachable, Unreachable}, 64, 1);
+    });
+    run("private scratch selection", [] {
+        const LoadCase test{ResourceKind::Scratch, IrOpcode::LoadAddressU32, IrType::U32};
+        for (const bool robust : {false, true}) {
+            expectReads(test, robust, false, 8, {0});
+            expectReads(test, robust, true, 64, {0});
+            expectReads(test, robust, true, 8, {2});
+        }
+    });
+    if (failures != 0) {
+        std::cerr << "guarded memory loads: a guest load reached a location the guarded program could not\n";
+        return 1;
+    }
+    std::cout << "Guarded memory load tests passed\n";
+    return 0;
+}

core/libs/prx/libSceAgcDriver/tests/HandoffSpin.cpp

@@ -0,0 +1,134 @@
+#include "prx/libSceAgcDriver/Execution/include/HandoffSpin.hpp"
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <cstdint>
+#include <iostream>
+#include <mutex>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace {
+
+using namespace std::chrono_literals;
+using AgcDriver::HandoffSpinWindowFor;
+using AgcDriver::LockSpinning;
+using AgcDriver::SpinUntil;
+
+void Require(bool condition, const std::string& message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+void CheckWindowPolicy() {
+    Require(HandoffSpinWindowFor(nullptr, 20) == 150us, "20 usable processors do not spin for 150 us");
+    Require(HandoffSpinWindowFor(nullptr, 9) == 150us, "9 usable processors do not spin for 150 us");
+    Require(HandoffSpinWindowFor(nullptr, 8) == 0ns, "8 usable processors spin");
+    Require(HandoffSpinWindowFor(nullptr, 4) == 0ns, "4 usable processors spin");
+    Require(HandoffSpinWindowFor(nullptr, 0) == 0ns, "an unknown processor count spins");
+    Require(HandoffSpinWindowFor("0", 20) == 0ns, "APS5_HANDOFF_SPIN_US=0 still spins");
+    Require(HandoffSpinWindowFor("400", 2) == 400us, "APS5_HANDOFF_SPIN_US=400 does not set 400 us on 2 logical processors");
+    Require(HandoffSpinWindowFor("-5", 20) == 0ns, "a negative window spins");
+    Require(HandoffSpinWindowFor("text", 20) == 0ns, "a window that is no number spins");
+    Require(HandoffSpinWindowFor("99999999999", 20) == 10000us, "a huge window is not limited to 10000 us");
+}
+
+void CheckUsableProcessors() {
+    const unsigned usable = AgcDriver::UsableProcessors();
+    Require(usable >= 1, "no usable processor was counted");
+    Require(usable <= (std::max)(std::thread::hardware_concurrency(), 64u), "more usable processors than the machine has");
+}
+
+void CheckSpinUntil() {
+    int calls = 0;
+    Require(SpinUntil(0ns, [&] { ++calls; return true; }) && calls == 1, "a ready condition was evaluated more than once");
+    calls = 0;
+    Require(!SpinUntil(0ns, [&] { ++calls; return false; }) && calls == 1, "a zero window evaluated the condition more than once");
+    calls = 0;
+    Require(SpinUntil(1s, [&] { return ++calls == 5; }) && calls == 5, "a condition that holds on its fifth evaluation was not seen");
+    const auto start = std::chrono::steady_clock::now();
+    Require(!SpinUntil(3ms, [] { return false; }), "a condition that never holds was reported ready");
+    Require(std::chrono::steady_clock::now() - start >= 3ms, "the spin returned before its window ended");
+    std::atomic<bool> flag{false};
+    std::thread setter([&] {
+        std::this_thread::sleep_for(500us);
+        flag.store(true, std::memory_order_release);
+    });
+    const bool seen = SpinUntil(5s, [&] { return flag.load(std::memory_order_acquire); });
+    setter.join();
+    Require(seen, "a flag set by another thread was not seen within the window");
+}
+
+template<typename TMutex>
+void Contend(std::chrono::nanoseconds window, const char* what) {
+    constexpr int Workers = 4;
+    constexpr int Rounds = 20000;
+    TMutex mutex;
+    std::atomic<int> inside{0};
+    std::atomic<bool> overlapped{false};
+    std::uint64_t counter = 0;
+    std::vector<std::thread> workers;
+    for (int worker = 0; worker < Workers; ++worker) {
+        workers.emplace_back([&] {
+            for (int round = 0; round < Rounds; ++round) {
+                LockSpinning(mutex, window);
+                if (inside.fetch_add(1) != 0) overlapped.store(true);
+                ++counter;
+                inside.fetch_sub(1);
+                mutex.unlock();
+            }
+        });
+    }
+    for (auto& worker : workers) worker.join();
+    Require(!overlapped.load(), std::string(what) + ": two threads held the mutex at once");
+    Require(counter == static_cast<std::uint64_t>(Workers) * Rounds, std::string(what) + ": increments were lost");
+}
+
+void CheckBlockingFallback() {
+    std::mutex mutex;
+    std::atomic<bool> held{false};
+    std::atomic<bool> released{false};
+    std::thread holder([&] {
+        mutex.lock();
+        held.store(true);
+        std::this_thread::sleep_for(20ms);
+        released.store(true);
+        mutex.unlock();
+    });
+    while (!held.load()) std::this_thread::yield();
+    LockSpinning(mutex, 100us);
+    const bool afterRelease = released.load();
+    mutex.unlock();
+    holder.join();
+    Require(afterRelease, "a spin that outlasted its window did not wait for the holder");
+}
+
+void CheckRecursion() {
+    std::recursive_mutex mutex;
+    LockSpinning(mutex, 100us);
+    LockSpinning(mutex, 100us);
+    mutex.unlock();
+    mutex.unlock();
+}
+
+}
+
+int main() {
+    try {
+        CheckWindowPolicy();
+        CheckUsableProcessors();
+        CheckSpinUntil();
+        Contend<std::mutex>(0ns, "plain lock");
+        Contend<std::mutex>(50us, "spin 50 us");
+        Contend<std::mutex>(1ms, "spin 1 ms");
+        Contend<std::recursive_mutex>(150us, "recursive mutex");
+        CheckBlockingFallback();
+        CheckRecursion();
+        std::cout << "handoff spin tests passed\n";
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/HostImportSparseDmaBuf.cpp

@@ -52,6 +52,7 @@ struct Bind {
 struct MockDevice {
     std::uintptr_t next = 0x1000;
     std::set<VkBuffer> buffers;
+    std::set<VkFence> fences;
     std::set<VkBuffer> sparse;
     std::map<VkBuffer, VkDeviceSize> sizes;
     std::map<VkDeviceMemory, VkDeviceSize> memories;
@@ -64,10 +65,25 @@ MockDevice mock;
 
 VKAPI_ATTR VkResult VKAPI_CALL mockCreateFence(VkDevice, const VkFenceCreateInfo*, const VkAllocationCallbacks*, VkFence* fence) {
     *fence = reinterpret_cast<VkFence>(mock.next++);
+    mock.fences.insert(*fence);
     return VK_SUCCESS;
 }
 
-VKAPI_ATTR void VKAPI_CALL mockDestroyFence(VkDevice, VkFence, const VkAllocationCallbacks*) {}
+VKAPI_ATTR void VKAPI_CALL mockDestroyFence(VkDevice, VkFence fence, const VkAllocationCallbacks*) {
+    mock.fences.erase(fence);
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL mockAllocateCommandBuffers(VkDevice, const VkCommandBufferAllocateInfo*, VkCommandBuffer* commands) {
+    *commands = reinterpret_cast<VkCommandBuffer>(mock.next++);
+    return VK_SUCCESS;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL mockResetFences(VkDevice, std::uint32_t, const VkFence*) { return VK_SUCCESS; }
+VKAPI_ATTR VkResult VKAPI_CALL mockResetCommandBuffer(VkCommandBuffer, VkCommandBufferResetFlags) { return VK_SUCCESS; }
+VKAPI_ATTR VkResult VKAPI_CALL mockBeginCommandBuffer(VkCommandBuffer, const VkCommandBufferBeginInfo*) { return VK_SUCCESS; }
+VKAPI_ATTR VkResult VKAPI_CALL mockEndCommandBuffer(VkCommandBuffer) { return VK_SUCCESS; }
+VKAPI_ATTR VkResult VKAPI_CALL mockQueueSubmit(VkQueue, std::uint32_t, const VkSubmitInfo*, VkFence) { return VK_SUCCESS; }
+VKAPI_ATTR void VKAPI_CALL mockCmdCopyBuffer(VkCommandBuffer, VkBuffer, VkBuffer, std::uint32_t, const VkBufferCopy*) {}
 
 VKAPI_ATTR VkResult VKAPI_CALL mockWaitForFences(VkDevice, std::uint32_t, const VkFence*, VkBool32, std::uint64_t) {
     return VK_SUCCESS;
@@ -133,6 +149,13 @@ VKAPI_ATTR VkDeviceAddress VKAPI_CALL mockGetBufferDeviceAddress(VkDevice, const
 
 PFN_vkVoidFunction VKAPI_CALL mockProc(VkDevice, const char* name) {
     static const std::map<std::string_view, PFN_vkVoidFunction> table{
+        {"vkAllocateCommandBuffers", reinterpret_cast<PFN_vkVoidFunction>(mockAllocateCommandBuffers)},
+        {"vkResetFences", reinterpret_cast<PFN_vkVoidFunction>(mockResetFences)},
+        {"vkResetCommandBuffer", reinterpret_cast<PFN_vkVoidFunction>(mockResetCommandBuffer)},
+        {"vkBeginCommandBuffer", reinterpret_cast<PFN_vkVoidFunction>(mockBeginCommandBuffer)},
+        {"vkEndCommandBuffer", reinterpret_cast<PFN_vkVoidFunction>(mockEndCommandBuffer)},
+        {"vkQueueSubmit", reinterpret_cast<PFN_vkVoidFunction>(mockQueueSubmit)},
+        {"vkCmdCopyBuffer", reinterpret_cast<PFN_vkVoidFunction>(mockCmdCopyBuffer)},
         {"vkCreateFence", reinterpret_cast<PFN_vkVoidFunction>(mockCreateFence)},
         {"vkDestroyFence", reinterpret_cast<PFN_vkVoidFunction>(mockDestroyFence)},
         {"vkWaitForFences", reinterpret_cast<PFN_vkVoidFunction>(mockWaitForFences)},
@@ -228,6 +251,36 @@ std::uint64_t RangeBytes() {
     return (UdmabufLimitBytes() / ChunkBytes + 1) * ChunkBytes + (std::uint64_t{1} << 20u);
 }
 
+void SettlingSurvivesDeviceReplacement(bool reverse) {
+    const auto baselineBuffers = mock.buffers.size();
+    const auto baselineMemories = mock.memories.size();
+    const auto baselineFences = mock.fences.size();
+    SharedRange range(8u << 20u, 0, true);
+    auto first = mockContext();
+    first.device = reinterpret_cast<VkDevice>(mock.next++);
+    first.settleQueue = reinterpret_cast<VkQueue>(mock.next++);
+    first.settlePool = reinterpret_cast<VkCommandPool>(mock.next++);
+    std::mutex queueMutex;
+    first.settleQueueMutex = &queueMutex;
+    auto second = first;
+    second.device = reinterpret_cast<VkDevice>(mock.next++);
+    second.settlePool = reinterpret_cast<VkCommandPool>(mock.next++);
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    for (const auto& context : {first, second, first}) {
+        const auto* imported = HostImportFor(context, range.Address(), 4096);
+        Expect(imported != nullptr && imported->dmaBuf, "replacement device did not settle its dma-buf import");
+    }
+    ClearHostImports(reverse ? second.device : first.device);
+    Expect(mock.fences.size() == baselineFences + 1, "clearing one device did not retain exactly the other device's settle fence");
+    ClearHostImports(reverse ? first.device : second.device);
+    Expect(mock.buffers.size() == baselineBuffers, "device replacement leaked a settle buffer");
+    Expect(mock.memories.size() == baselineMemories, "device replacement leaked settle memory");
+    Expect(mock.fences.size() == baselineFences, "device replacement leaked a settle fence");
+    ClearHostImports(first.device);
+    ClearHostImports(second.device);
+    Expect(mock.fences.size() == baselineFences, "repeated device cleanup changed the live fences");
+}
+
 void ImportsALargeRangeAsOneSparseBuffer() {
     const auto bytes = RangeBytes();
     SharedRange range(bytes, 0, true);
@@ -299,6 +352,8 @@ int main() {
     SetHostImportBudget(~std::uint64_t{0});
     GuestArena::GuestArenaSetSharedBacking_nid_postfix(Resolve);
     try {
+        SettlingSurvivesDeviceReplacement(false);
+        SettlingSurvivesDeviceReplacement(true);
         ImportsALargeRangeAsOneSparseBuffer();
         UnalignedOffsetIsDecidedBeforeImporting();
         FailureAfterTheDecisionThrows(false, VK_SUCCESS, "UDMABUF_CREATE");

core/libs/prx/libSceAgcDriver/tests/LdsLockBudget.cpp

@@ -0,0 +1,115 @@
+#include "Recompiler.hpp"
+#include <spirv/unified1/spirv.hpp>
+#include <array>
+#include <cstdint>
+#include <functional>
+#include <iostream>
+#include <map>
+#include <span>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace {
+
+using namespace ShaderRecompiler;
+
+void require(bool condition, const std::string& message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+std::uint64_t workgroupBytes(std::span<const std::uint32_t> words) {
+    std::map<std::uint32_t, std::span<const std::uint32_t>> types;
+    std::map<std::uint32_t, std::uint32_t> constants;
+    std::vector<std::uint32_t> pointers;
+    for (std::size_t offset = 5; offset < words.size();) {
+        const auto count = words[offset] >> 16u;
+        require(count != 0u && count <= words.size() - offset, "malformed SPIR-V instruction");
+        const auto instruction = words.subspan(offset, count);
+        const auto opcode = static_cast<spv::Op>(instruction[0] & 0xffffu);
+        if (opcode == spv::OpTypeInt || opcode == spv::OpTypeFloat || opcode == spv::OpTypeVector || opcode == spv::OpTypeArray || opcode == spv::OpTypeStruct || opcode == spv::OpTypePointer) types.emplace(instruction[1], instruction);
+        if (opcode == spv::OpConstant) constants.emplace(instruction[2], instruction[3]);
+        if (opcode == spv::OpVariable && instruction[3] == spv::StorageClassWorkgroup) pointers.push_back(instruction[1]);
+        offset += count;
+    }
+    const std::function<std::uint64_t(std::uint32_t)> size = [&](std::uint32_t id) -> std::uint64_t {
+        const auto type = types.at(id);
+        switch (static_cast<spv::Op>(type[0] & 0xffffu)) {
+        case spv::OpTypeInt: case spv::OpTypeFloat: return type[2] / 8u;
+        case spv::OpTypeVector: return size(type[2]) * type[3];
+        case spv::OpTypeArray: return size(type[2]) * constants.at(type[3]);
+        case spv::OpTypeStruct: {
+            std::uint64_t bytes = 0;
+            for (const auto member : type.subspan(2)) bytes += size(member);
+            return bytes;
+        }
+        default: throw std::runtime_error("unexpected Workgroup variable type");
+        }
+    };
+    std::uint64_t bytes = 0;
+    for (const auto pointer : pointers) {
+        const auto type = types.at(pointer);
+        require((type[0] & 0xffffu) == spv::OpTypePointer, "Workgroup variable is not a pointer");
+        bytes += size(type[3]);
+    }
+    return bytes;
+}
+
+RecompileResult compile(std::uint32_t waveSize, std::uint32_t limit) {
+    static constexpr std::array<std::uint32_t, 8> code{
+        0x7e020280u, 0x7e0402c1u, 0x7e060280u, 0x7e080281u,
+        0xd9000000u, 0x00000201u, 0xbf8cc07fu, 0xbf810000u
+    };
+    static constexpr std::array<std::uint32_t, 7> capabilities{spv::CapabilityShader, spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess};
+    static constexpr std::array<std::string_view, 2> extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"};
+    static constexpr std::array<std::uint32_t, 4> userData{};
+    RecompileRequest request{};
+    request.shader = {ShaderStage::Compute, 0x20000u, code, 0, {}};
+    request.context.waveSize = waveSize;
+    request.context.userData = userData;
+    request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 16u, {false, false, false}, false, 1u};
+    request.target.vulkanVersion = 0x00403000u;
+    request.target.spirvVersion = 0x00010600u;
+    request.target.subgroupSize = 32;
+    request.target.bdaAbiVersion = 1;
+    request.target.supportedCapabilities = capabilities;
+    request.target.supportedExtensions = extensions;
+    request.target.maxWorkgroupSize = {1024u, 1024u, 64u};
+    request.target.maxWorkgroupInvocations = 1024;
+    request.target.maxWorkgroupSharedMemoryBytes = limit;
+    request.layout = {0, 0, 0, 128};
+    request.useCache = false;
+    return Recompile(request);
+}
+
+void verify(std::uint32_t waveSize, std::uint32_t limit) {
+    const auto result = compile(waveSize, limit);
+    require(!result.spirv.empty(), "LDS atomic program did not compile");
+    const auto declared = workgroupBytes(result.spirv.Words());
+    require(declared <= limit, "declared " + std::to_string(declared) + " Workgroup bytes exceeds limit " + std::to_string(limit));
+    require(result.workgroupMemoryDwords == (limit < 68u ? 17u : 0u), "incorrect device-memory LDS stride at limit " + std::to_string(limit));
+    if (limit >= 68u) require(declared != 0u, "in-budget LDS unexpectedly lost its Workgroup allocation");
+}
+
+}
+
+int main() {
+    unsigned failures = 0;
+    for (const auto waveSize : {32u, 64u}) {
+        for (const auto limit : {64u, 68u, 72u}) {
+            const auto name = "wave" + std::to_string(waveSize) + " limit" + std::to_string(limit);
+            try {
+                verify(waveSize, limit);
+                std::cout << "LDS lock budget: " << name << " passed\n";
+            } catch (const std::exception& error) {
+                ++failures;
+                const std::string message(error.what());
+                std::cerr << "LDS lock budget: " << name << ": " << message.substr(0, message.find("RecompileRequest:")) << '\n';
+            }
+        }
+    }
+    if (failures != 0) return 1;
+    std::cout << "LDS lock budget tests passed\n";
+    return 0;
+}

core/libs/prx/libSceAgcDriver/tests/PipelineQueueFailure.cpp

@@ -0,0 +1,133 @@
+#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp"
+#include "AllocationFailure.hpp"
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <functional>
+#include <mutex>
+#include <new>
+#include <string_view>
+#include <thread>
+#include <utility>
+
+using namespace AgcDriver::Graphics;
+using AgcDriver::Tests::SetAllocationFailureCountdown;
+using namespace std::chrono_literals;
+
+namespace {
+
+void Check(bool condition, const char* message) {
+    if (condition) return;
+    std::fprintf(stderr, "FAIL: %s\n", message);
+    std::fflush(stderr);
+    std::_Exit(1);
+}
+
+void Schedule(VkDevice device, std::function<void()> work) {
+    OptimizedPipeline pipeline;
+    pipeline.device = device;
+    pipeline.relink = std::move(work);
+    pipeline.deferred = true;
+    pipeline.Schedule();
+}
+
+}
+
+int main(int argc, char** argv) {
+    Check(argc == 2, "expected wait, clear or stop");
+    const std::string_view mode(argv[1]);
+    Check(mode == "wait" || mode == "clear" || mode == "stop", "unknown drain mode");
+    const auto device = reinterpret_cast<VkDevice>(std::uintptr_t{1});
+    const auto otherDevice = reinterpret_cast<VkDevice>(std::uintptr_t{2});
+    std::atomic<unsigned> completed{0};
+    Schedule(device, [&] { ++completed; });
+    WaitForOptimizedPipelines(device);
+    Check(completed == 1, "the initial job did not run");
+
+    bool allocationFailed = false;
+    OptimizedPipeline fresh;
+    fresh.device = otherDevice;
+    fresh.relink = [&] { ++completed; };
+    fresh.deferred = true;
+    SetAllocationFailureCountdown(0);
+    try { fresh.Schedule(); } catch (const std::bad_alloc&) { allocationFailed = true; }
+    SetAllocationFailureCountdown(-1);
+    Check(allocationFailed, "the new device allocation did not fail");
+    WaitForOptimizedPipelines(otherDevice);
+    Check(completed == 1, "a rejected job ran");
+
+    std::mutex mutex;
+    std::condition_variable changed;
+    bool entered = false;
+    bool released = false;
+    Schedule(device, [&] {
+        std::unique_lock lock(mutex);
+        entered = true;
+        changed.notify_all();
+        changed.wait(lock, [&] { return released; });
+    });
+    {
+        std::unique_lock lock(mutex);
+        Check(changed.wait_for(lock, 3s, [&] { return entered; }), "the worker did not start");
+    }
+
+    allocationFailed = false;
+    unsigned accepted = 0;
+    for (unsigned attempt = 0; attempt < 512 && !allocationFailed; ++attempt) {
+        OptimizedPipeline pipeline;
+        pipeline.device = device;
+        pipeline.relink = [&] { ++completed; };
+        pipeline.deferred = true;
+        SetAllocationFailureCountdown(0);
+        try {
+            pipeline.Schedule();
+            ++accepted;
+        } catch (const std::bad_alloc&) {
+            allocationFailed = true;
+        }
+        SetAllocationFailureCountdown(-1);
+    }
+    Check(allocationFailed, "the queued jobs never needed another allocation");
+    Check(completed == 1, "queued jobs ran while the worker was held");
+    {
+        std::lock_guard lock(mutex);
+        released = true;
+    }
+    changed.notify_all();
+
+    bool drained = false;
+    std::thread drain([&] {
+        if (mode == "stop") StopPipelineLibraryWorker();
+        if (mode == "clear") ClearPipelineLibraries(device);
+        else WaitForOptimizedPipelines(device);
+        {
+            std::lock_guard lock(mutex);
+            drained = true;
+        }
+        changed.notify_all();
+    });
+    {
+        std::unique_lock lock(mutex);
+        Check(changed.wait_for(lock, 3s, [&] { return drained; }), "a failed queue allocation left an unfinished job");
+    }
+    drain.join();
+    if (mode != "stop") {
+        Check(completed == accepted + 1, "an accepted job was lost or ran twice");
+        Schedule(device, [&] { ++completed; });
+        Schedule(otherDevice, [&] { ++completed; });
+        WaitForOptimizedPipelines(device);
+        WaitForOptimizedPipelines(otherDevice);
+        Check(completed == accepted + 3, "the queue did not recover after allocation failure");
+        StopPipelineLibraryWorker();
+    }
+    WaitForOptimizedPipelines(device);
+    ClearPipelineLibraries(device);
+    ClearPipelineLibraries(otherDevice);
+    Check(PipelineLibraryWorkers() == 0, "the pipeline worker did not stop");
+    std::printf("%.*s: rejected new-device and queue-growth allocations; %u jobs accepted before rejection, queue drained\n",
+        static_cast<int>(mode.size()), mode.data(), accepted);
+    return 0;
+}

core/libs/prx/libSceAgcDriver/tests/Recorder.cpp

@@ -28,6 +28,7 @@
 #include "prx/libc/include/GuestWriteWatch.hpp"
 #include "prx/libc/include/general/VabiMacros.hpp"
 #include "ResidentPresent.hpp"
+#include "ImageTableAbi.hpp"
 #include "SampleLod_spv.h"
 #include "BufferUpdate_spv.h"
 #include "SampleArray_spv.h"
@@ -311,6 +312,28 @@ public:
                 extensionsEnabled.insert(extensionsEnabled.end(), libraryExtensions.begin(), libraryExtensions.end());
                 address.pNext = &library;
             }
+            VkPhysicalDeviceDescriptorIndexingFeaturesEXT indexing{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES_EXT};
+            if (hasExtension(VK_EXT_DESCRIPTOR_INDEXING_EXTENSION_NAME)) {
+                VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &indexing};
+                function<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(context.physical, &features);
+                context.descriptorIndexing = indexing.shaderSampledImageArrayNonUniformIndexing == VK_TRUE && indexing.shaderStorageImageArrayNonUniformIndexing == VK_TRUE;
+                context.partiallyBound = indexing.descriptorBindingPartiallyBound == VK_TRUE;
+                context.variableDescriptorCount = indexing.descriptorBindingVariableDescriptorCount == VK_TRUE;
+                context.runtimeDescriptorArray = indexing.runtimeDescriptorArray == VK_TRUE;
+            }
+            indexing = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES_EXT};
+            if (context.descriptorIndexing && context.partiallyBound && context.variableDescriptorCount && context.runtimeDescriptorArray) {
+                extensionsEnabled.push_back(VK_EXT_DESCRIPTOR_INDEXING_EXTENSION_NAME);
+                indexing.shaderSampledImageArrayNonUniformIndexing = VK_TRUE;
+                indexing.shaderStorageImageArrayNonUniformIndexing = VK_TRUE;
+                indexing.descriptorBindingPartiallyBound = VK_TRUE;
+                indexing.descriptorBindingVariableDescriptorCount = VK_TRUE;
+                indexing.runtimeDescriptorArray = VK_TRUE;
+                indexing.pNext = address.pNext;
+                address.pNext = &indexing;
+            } else {
+                context.descriptorIndexing = context.partiallyBound = context.variableDescriptorCount = context.runtimeDescriptorArray = false;
+            }
             VkPhysicalDeviceRobustness2FeaturesEXT robustness{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT};
             if (hasExtension(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME)) {
                 VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &robustness};
@@ -339,6 +362,7 @@ public:
             std::cout << "Vulkan device: " << properties.deviceName << '\n';
             context.limits = properties.limits;
             context.bufferDeviceAddress = true;
+            context.imageTableCapacity = ImageTableCapacity(context);
             context.formatProperties = function<PFN_vkGetPhysicalDeviceFormatProperties>("vkGetPhysicalDeviceFormatProperties");
             context.imageFormatProperties = function<PFN_vkGetPhysicalDeviceImageFormatProperties>("vkGetPhysicalDeviceImageFormatProperties");
             context.Function<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(context.device, family, 0, &context.queue);
@@ -2484,6 +2508,7 @@ public:
     struct Contents {
         std::map<std::pair<std::uint32_t, std::uint32_t>, VkDescriptorBufferInfo> buffers;
         std::map<std::pair<std::uint32_t, std::uint32_t>, VkDescriptorImageInfo> images;
+        std::map<std::pair<VkDescriptorSet, std::uint32_t>, std::uint32_t> written;
     };
     explicit DescriptorUpdates(Context& context) : context(context), previous(context.functions),
         update(context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")) {
@@ -2502,6 +2527,7 @@ private:
         Contents contents;
         for (std::uint32_t i = 0; i < count; ++i) {
             const auto& write = writes[i];
+            contents.written[{write.dstSet, write.dstBinding}] += write.descriptorCount;
             for (std::uint32_t element = 0; element < write.descriptorCount; ++element) {
                 const auto key = std::pair{write.dstBinding, write.dstArrayElement + element};
                 if (write.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER) contents.buffers[key] = write.pBufferInfo[element];
@@ -2572,6 +2598,85 @@ void drawDescriptorWriteTests(const Device& device, Recorder& recorder) {
     recorder.Activate();
 }
 
+void drawSnapshotTableSetTests(const Device& device, Recorder& recorder) {
+    namespace Abi = ShaderRecompiler::ImageTableAbi;
+    using ShaderRecompiler::DescriptorKind;
+    using ShaderRecompiler::DescriptorRole;
+    const auto& base = device.GetContext();
+    TextureDetiler detiler(base);
+    auto context = base;
+    context.detiler = &detiler;
+    if (context.imageTableCapacity < 2u) {
+        std::cout << "image tables unavailable: draw snapshot table sets not tested\n";
+        return;
+    }
+    constexpr std::uint32_t width = 4;
+    constexpr std::uint32_t height = 4;
+    alignas(256) static std::array<float, width * height> texels{};
+    texels.fill(1.0f);
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(texels.data()));
+    const std::array<std::uint32_t, 8> texture{static_cast<std::uint32_t>(address >> 8u), static_cast<std::uint32_t>((address >> 40u) & 0xffu) | (22u << 20u) | (((width - 1u) & 3u) << 30u), ((width - 1u) >> 2u) | ((height - 1u) << 14u), 0xfacu | (9u << 28u), 0u, 0u, 0u, 0u};
+    constexpr auto bindingCount = static_cast<std::uint32_t>(ShaderRecompiler::RuntimeAbi::Binding::Count);
+    const auto addTable = [&](ShaderRecompiler::RecompileResult& program, ShaderRecompiler::RuntimeAbi::Stage stage, std::uint32_t set, std::uint32_t elements) {
+        ShaderRecompiler::DescriptorBinding map{};
+        map.kind = DescriptorKind::StorageBuffer;
+        map.role = DescriptorRole::ImageTableMap;
+        map.descriptorSet = set;
+        map.binding = static_cast<std::uint32_t>(stage) * bindingCount + static_cast<std::uint32_t>(ShaderRecompiler::RuntimeAbi::Binding::ImageTableMap);
+        map.count = 1;
+        map.guestDescriptor.assign(Abi::HeaderWords + elements, 0u);
+        map.guestDescriptor[Abi::HeaderVersion] = Abi::Version;
+        map.guestDescriptor[Abi::HeaderElementCount] = elements;
+        map.guestDescriptor[Abi::HeaderElementFaults] = Abi::HeaderWords;
+        map.guestDescriptor[Abi::HeaderSamplerFaults] = Abi::HeaderWords + elements;
+        program.bindings.push_back(map);
+        ShaderRecompiler::DescriptorBinding images{};
+        images.kind = DescriptorKind::SampledImage;
+        images.role = DescriptorRole::ImageTable;
+        images.descriptorSet = set;
+        images.binding = ShaderRecompiler::RuntimeAbi::ImageTableBinding;
+        images.count = elements;
+        for (std::uint32_t element = 0; element < elements; ++element) images.guestDescriptor.insert(images.guestDescriptor.end(), texture.begin(), texture.end());
+        images.imageShapes.assign(elements, ShaderRecompiler::DescriptorImageShape::Image2D);
+        images.imageDepthCompare.assign(elements, false);
+        images.imageSamplers.assign(elements, 0u);
+        program.bindings.push_back(std::move(images));
+    };
+    DescriptorUpdates updates(context);
+    {
+        DescriptorCache cache(context);
+        context.descriptorCache = &cache;
+        Recorder snapshots(context);
+        snapshots.Activate();
+        ShaderRecompiler::RecompileResult vertex, fragment;
+        ShaderRecompiler::DescriptorBinding data{};
+        data.kind = DescriptorKind::StorageBuffer;
+        data.role = DescriptorRole::FlattenedSrt;
+        data.binding = static_cast<std::uint32_t>(ShaderRecompiler::RuntimeAbi::Binding::FlattenedSrt);
+        data.count = 1;
+        data.guestDescriptor = {1, 2, 3, 4};
+        vertex.bindings.push_back(data);
+        addTable(vertex, ShaderRecompiler::RuntimeAbi::Stage::Main, 0u, 1u);
+        addTable(fragment, ShaderRecompiler::RuntimeAbi::Stage::Fragment, ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet, 2u);
+        const ColorTarget target{};
+        ShaderResources resources(context, vertex, fragment, target, 0, 0);
+        Require(resources.SplitSets(), "a fragment image table did not get its own descriptor set");
+        const ShaderResources::MovedBuffer moved{0, 0, 16, {5, 6, 7, 8}};
+        const auto draw = resources.PrepareDrawBindings(snapshots, std::span(&moved, 1));
+        Require(draw != nullptr && draw->snapshots.size() == 1, "moved shader data did not create a draw snapshot");
+        const auto& written = updates.calls.back().written;
+        const auto tableWrites = [&](VkDescriptorSet set) {
+            const auto it = written.find({set, ShaderRecompiler::RuntimeAbi::ImageTableBinding});
+            return it == written.end() ? 0u : it->second;
+        };
+        Require(tableWrites(draw->allocation.set) == 1u, "the draw snapshot wrote the wrong number of image table descriptors into the vertex set");
+        Require(tableWrites(draw->fragmentAllocation.set) == 2u, "the draw snapshot wrote the wrong number of image table descriptors into the fragment set");
+        snapshots.Sync();
+    }
+    ClearCachedTextures(context.device);
+    recorder.Activate();
+}
+
 void drawSnapshotReuseTests(const Device& device, Recorder& recorder) {
     const auto& context = device.GetContext();
     constexpr std::size_t bytes = 65536;
@@ -6532,7 +6637,7 @@ void fmaskDecompressMsaaTests(const Device& device, Recorder& recorder) {
 #ifdef _WIN32
     void* block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
 #else
-    void* block = std::aligned_alloc(65536, bytes);
+    void* block = std::aligned_alloc(65536, (bytes + 65535u) & ~std::size_t{65535u});
 #endif
     Require(block != nullptr, "cannot allocate the FMASK decompress block");
     std::memset(block, 0, bytes);
@@ -6998,6 +7103,70 @@ void singleCubeTests(const Device& device, Recorder& recorder) {
     }
 }
 
+struct SampledViewProbe {
+    Context& context;
+    PFN_vkGetDeviceProcAddr resolver;
+    std::uint32_t views = 0;
+    bool checking = false;
+    static inline SampledViewProbe* active = nullptr;
+    explicit SampledViewProbe(Context& context) : context(context), resolver(context.deviceProc) {
+        Require(active == nullptr, "sampled view probe is already active");
+        active = this;
+        context.deviceProc = resolve;
+    }
+    ~SampledViewProbe() {
+        context.deviceProc = resolver;
+        active = nullptr;
+    }
+    static VKAPI_ATTR VkResult VKAPI_CALL createView(VkDevice device, const VkImageViewCreateInfo* info, const VkAllocationCallbacks* callbacks, VkImageView* created) {
+        if (active->checking && info->format == VK_FORMAT_R8G8B8A8_SRGB) {
+            const auto* next = static_cast<const VkBaseInStructure*>(info->pNext);
+            while (next != nullptr && next->sType != VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO) next = next->pNext;
+            if (next == nullptr || reinterpret_cast<const VkImageViewUsageCreateInfo*>(next)->usage != VK_IMAGE_USAGE_SAMPLED_BIT) return VK_ERROR_VALIDATION_FAILED_EXT;
+            ++active->views;
+        }
+        return reinterpret_cast<PFN_vkCreateImageView>(active->resolver(device, "vkCreateImageView"))(device, info, callbacks, created);
+    }
+    static VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL resolve(VkDevice device, const char* name) {
+        if (std::strcmp(name, "vkCreateImageView") == 0) return reinterpret_cast<PFN_vkVoidFunction>(createView);
+        return active->resolver(device, name);
+    }
+};
+
+void sampledStorageBaseLevelUsageTests(const Device& device, Recorder& recorder) {
+    auto context = device.GetContext();
+    TextureDetiler detiler(context);
+    context.detiler = &detiler;
+    GuestTextureResource resource{};
+    resource.width = resource.height = 8;
+    resource.depthOrLastArray = 1;
+    resource.mipCount = 2;
+    resource.lastLevel = 1;
+    resource.minLod = context.imageViewMinLod ? 0x100 : 0;
+    resource.tileMode = TextureTileMode::kLinear;
+    resource.dimension = TextureDimension::k2DArray;
+    resource.format = 130;
+    resource.dstSelX = 4;
+    resource.dstSelY = 5;
+    resource.dstSelZ = 6;
+    resource.dstSelW = 7;
+    const auto geometry = DescribeSurface(resource);
+    std::vector<std::byte> memory(static_cast<std::size_t>(geometry.guestBytes) + 256, std::byte{0x80});
+    resource.baseAddress = (reinterpret_cast<std::uintptr_t>(memory.data()) + 255) & ~std::uintptr_t{255};
+    auto storage = StorageTexture::Create(context, detiler, resource, 0);
+    recorder.Keep(storage);
+    recorder.Sync();
+    SampleProgram program(context, recorder);
+    auto sampledContext = context;
+    SampledViewProbe probe(sampledContext);
+    Texture sampled(sampledContext, storage, resource, VkComponentMapping{});
+    probe.checking = true;
+    const auto base = sampled.BaseLevelView();
+    probe.checking = false;
+    expectRed(program.Red(base, sampled.Layout(), 0), 0.21586f, "sampled sRGB base level");
+    Require(sampled.BaseLevelView() == base && probe.views == 1, "the lazy base-level view of a sampled storage image must restrict usage to SAMPLED and be reused");
+}
+
 void firstLayerViewTests(const Device& device, Recorder& recorder) {
     const auto& context = device.GetContext();
     TextureDetiler detiler(context);
@@ -8230,6 +8399,10 @@ int main(int argc, char** argv) {
             std::cout << "Single cube snapshot and storage sampling tests passed\n";
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--sampled-storage-base-level-usage-only") {
+            sampledStorageBaseLevelUsageTests(device, recorder);
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--unchanged-cpu-stamps-only") {
             unchangedCpuStampTests(device, recorder);
             return 0;
@@ -8352,6 +8525,7 @@ int main(int argc, char** argv) {
         }
         if (argc == 2 && std::string_view(argv[1]) == "--draw-descriptors-only") {
             drawDescriptorWriteTests(device, recorder);
+            drawSnapshotTableSetTests(device, recorder);
             std::cout << "Draw descriptor writes and snapshot overrides passed\n";
             return 0;
         }
@@ -8409,6 +8583,7 @@ int main(int argc, char** argv) {
         misalignedRegionTests(device, recorder);
         drawSnapshotReuseTests(device, recorder);
         drawDescriptorWriteTests(device, recorder);
+        drawSnapshotTableSetTests(device, recorder);
         drawSnapshotCapTests(device, recorder);
         drawSnapshotEvictionTests(device);
         unwrittenSnapshotTests(device);
@@ -8447,6 +8622,7 @@ int main(int argc, char** argv) {
         imageMemoTests(device, recorder);
         minLodTests(device, recorder);
         firstLayerViewTests(device, recorder);
+        sampledStorageBaseLevelUsageTests(device, recorder);
         singleCubeTests(device, recorder);
         atomicViewTests(device, recorder);
         metadataPassTests(device, recorder);

core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp

@@ -17,7 +17,9 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <exception>
 #include <filesystem>
+#include <fstream>
 #include <functional>
 #include <future>
 #include <iostream>
@@ -29,6 +31,7 @@
 #include <stdexcept>
 #include <string>
 #include <string_view>
+#include <thread>
 #include <vector>
 #ifdef _WIN32
 #ifndef NOMINMAX
@@ -646,6 +649,7 @@ void verifyKeySensitivity() {
     changes("f64 denormal preservation", [](SampleRequest& sample) { sample.request.target.float64DenormPreserve = true; });
     changes("f16 denormal preservation", [](SampleRequest& sample) { sample.request.target.float16DenormPreserve = true; });
     changes("spanned waves", [](SampleRequest& sample) { sample.request.target.spannedWaves = true; });
+    changes("robust buffer access 2", [](SampleRequest& sample) { sample.request.target.robustBufferAccess2 = true; });
     changes("the workgroup size limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupSize[2] = 128; });
     changes("the invocation limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupInvocations = 512; });
     changes("the shared memory limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupSharedMemoryBytes = 32768; });
@@ -730,6 +734,64 @@ void verifyFloatModeSerialization() {
     require(back.request.context.floatMode == mode, "the float mode did not survive serialization");
 }
 
+int runLargeFileWrite() {
+    constexpr std::size_t size = (std::size_t{2} << 30u) + 4096u;
+    const auto directory = std::filesystem::temp_directory_path() / ("aps5-shader-disk-cache-large-file-test-" + std::to_string(std::random_device{}()));
+    std::error_code error;
+    std::filesystem::remove_all(directory, error);
+    std::filesystem::create_directories(directory, error);
+    const auto space = std::filesystem::space(directory, error);
+    if (error || space.available < size + (std::uintmax_t{1} << 30u)) {
+        std::cout << "skipping: not enough free disk space for a file of more than 2 GiB\n";
+        return 77;
+    }
+    std::vector<std::byte> bytes;
+    try {
+        bytes.resize(size);
+    } catch (const std::bad_alloc&) {
+        std::cout << "skipping: cannot allocate a buffer of more than 2 GiB\n";
+        return 77;
+    }
+    const std::array<std::size_t, 6> samples{0u, (std::size_t{1} << 30u) - 1u, std::size_t{1} << 30u, (std::size_t{2} << 30u) - 1u, std::size_t{2} << 30u, size - 1u};
+    for (std::size_t index = 0; index < samples.size(); ++index) bytes[samples[index]] = static_cast<std::byte>(0xa0u + index);
+    const auto path = directory / "pipelines.bin";
+    auto written = std::make_shared<std::promise<bool>>();
+    auto result = written->get_future();
+    std::thread([written, &bytes, path] {
+        try {
+            written->set_value(WriteFileAtomically(path, bytes));
+        } catch (...) {
+            written->set_exception(std::current_exception());
+        }
+    }).detach();
+    if (result.wait_for(std::chrono::seconds(200)) != std::future_status::ready) {
+        std::cerr << "WriteFileAtomically did not return within 200 s for a file of more than 2 GiB\n";
+        std::cerr.flush();
+        std::_Exit(1);
+    }
+    require(result.get(), "cannot write a cache file of more than 2 GiB");
+    require(std::filesystem::file_size(path) == size, "the cache file of more than 2 GiB has the wrong size");
+    {
+        std::ifstream file(path, std::ios::binary);
+        require(static_cast<bool>(file), "cannot open the cache file of more than 2 GiB");
+        for (std::size_t index = 0; index < samples.size(); ++index) {
+            file.seekg(static_cast<std::streamoff>(samples[index]));
+            char value = 0;
+            require(file.get(value) && static_cast<std::byte>(value) == bytes[samples[index]], "the cache file of more than 2 GiB differs at byte " + std::to_string(samples[index]));
+        }
+    }
+    std::size_t entries = 0;
+    for (const auto& entry : std::filesystem::directory_iterator(directory)) {
+        static_cast<void>(entry);
+        ++entries;
+    }
+    require(entries == 1u, "the cache file of more than 2 GiB left a temporary file behind");
+    std::filesystem::remove_all(directory, error);
+    require(!error, "cannot remove the large file test directory: " + error.message());
+    std::cout << "large cache file write test passed\n";
+    return 0;
+}
+
 void verifyAtomicReplacement(const std::filesystem::path& directory) {
     struct Reader {
 #ifdef _WIN32
@@ -1135,7 +1197,8 @@ void verifyBindingPlanSelection() {
         throw std::runtime_error("binding plan accepted an unsupported buffer alignment");
     };
     snapshot.buffers[0].dwords[0] |= 2u;
-    rejectsAlignment("buffer atomic on a V# whose base is not DWORD aligned");
+    const auto atomicPlan = builder.Prepare(compiled.layout, info, IrShaderStage::Compute, snapshot);
+    require(std::ranges::any_of(atomicPlan.specialization, [](const PipelineSpecializationConstant& constant) { return constant.id == PipelineSpecialization::BufferBase + 3u && constant.value == 2u; }), "a buffer atomic V# off a DWORD boundary did not record its misalignment");
     snapshot.buffers[0].dwords[0] &= ~3u;
     info.buffers[1].typedAlignment = 2u;
     snapshot.buffers[1].dwords[0] |= 2u;
@@ -1443,6 +1506,10 @@ std::vector<PlanFixture> planFixtures() {
     add("bindless material table", 64, {0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x93109010u, 0xf4200406u, 0x20000004u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, tables}, {PlanTableAddress, heap}, {PlanMaterialAddress, materials}});
     add("bindless whole table", 64, {0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, tables}, {PlanTableAddress, heap}, {PlanMaterialAddress, materials}});
     add("bindless sample in a loop latch", 64, {0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xbea80380u, 0xf09c0f08u, 0x00450000u, 0x80288128u, 0xbf0a8428u, 0xbf85fffbu, 0xe0700000u, 0x80070000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, tables}, {PlanTableAddress, heap}, {PlanMaterialAddress, materials}});
+    auto keyed = tables;
+    keyed.resize(20u, 0u);
+    keyed[16] = 3u;
+    add("bindless selected record", 64, {0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0xf4000400u, 0xfa000040u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, keyed}, {PlanTableAddress, heap}, {PlanMaterialAddress, materials}});
 
     std::vector<std::uint32_t> phis(32, 0u);
     place(phis, 0, imageWords(PlanTextureAddress));
@@ -1588,6 +1655,8 @@ IrResourcePlan handBuiltPlan() {
     table.dwordCount = 8u;
     table.dwords[0] = &readConst(3);
     table.tableColumn = TableColumn{2u, 32u, 16u, 4u, 8u, 0xffu, false, true, {KeyDomain{3u, 4u, 16u, false}, KeyDomain{2u, 8u, 4u, true}}, false};
+    table.tableColumn->selected = true;
+    table.tableColumn->selection = 2u;
     DescriptorSource heap;
     heap.dwordCount = 4u;
     heap.dwords[1] = &readConst(6);
@@ -1836,6 +1905,7 @@ void verifyMalformedPlans() {
     rejects("a uniform fill of five words", [](IrResourcePlan& plan) { plan.uniformFill.fill.words = 5u; });
     rejects("an image table heap past the sources", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->heapSource = 4u; });
     rejects("an image table key past the sources", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->keyDomains[1].source = 4u; });
+    rejects("an image table selection past the sources", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->selection = 4u; });
     auto foreign = handBuiltPlan();
     IrValue stray(IrOpcode::Void, IrType::U32, 99u);
     foreign.srtReads[0].value = &stray;
@@ -1895,6 +1965,7 @@ void verifyPlanRoundTrip() {
     require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.descriptorSources.begin(), plan.descriptorSources.end(), [](const DescriptorSource& source) { return source.tableColumn.has_value() && source.tableColumn->keyDomains.empty(); }); }), "no fixture plans a whole image table");
     require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.descriptorSources.begin(), plan.descriptorSources.end(), [](const DescriptorSource& source) { return source.tableColumn.has_value() && !source.tableColumn->keyDomains.empty(); }); }), "no fixture plans an image table with a key domain");
     require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.descriptorSources.begin(), plan.descriptorSources.end(), [](const DescriptorSource& source) { return source.tableColumn.has_value() && source.tableColumn->address; }); }), "no fixture plans an image table through a pointer");
+    require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.descriptorSources.begin(), plan.descriptorSources.end(), [](const DescriptorSource& source) { return source.tableColumn.has_value() && source.tableColumn->selected; }); }), "no fixture plans a selected image table record");
     require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.controlFlow.begin(), plan.controlFlow.end(), [](const ResourceBlock& block) { return block.condition != nullptr; }); }), "no fixture plans a control-flow condition");
     require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.valueStorage.begin(), plan.valueStorage.end(), [](const auto& value) { return value.IsPhi(); }); }), "no fixture plans a phi");
     require(covers([](const IrResourcePlan& plan) { return !plan.guardedSrtSlots.empty(); }), "no fixture plans a guarded SRT read");
@@ -2124,6 +2195,7 @@ int main(int argc, char** argv) {
             std::cout << "artifact codec tests passed\n";
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--large-file") return runLargeFileWrite();
         if (argc == 2 && std::string_view(argv[1]) == "--load") return runLoadingProcess();
         if (argc == 2 && std::string_view(argv[1]) == "--load-prebuilt") return runPrebuiltLoadingProcess();
         if (argc == 2 && std::string_view(argv[1]) == "--no-failure-memo") return runWithoutFailureMemo();

core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp

@@ -561,9 +561,25 @@ void verifyMayEvaluate() {
     counter.AddArgument(&next);
     auto& latch = make(IrOpcode::ULessThan32, IrType::Bool, {&next, &constant(4)});
     auto& sum = make(IrOpcode::CompositeExtractU32x2, IrType::U32, {&make(IrOpcode::IAddCarry32, IrType::U32x2, {&input, &constant(1)}), &constant(0)});
+    auto& firstInput = make(IrOpcode::ReadFirstLane, IrType::U32, {&input, &varying});
+    auto& firstLane = make(IrOpcode::ReadFirstLane, IrType::U32, {&lane, &varying});
+    auto& firstActive = make(IrOpcode::ReadFirstLane, IrType::U32, {&make(IrOpcode::SelectU32, IrType::U32, {&varying, &input, &lane}), &varying});
+    auto& firstInactive = make(IrOpcode::ReadFirstLane, IrType::U32, {&make(IrOpcode::SelectU32, IrType::U32, {&varying, &lane, &input}), &varying});
+    plan.srtReads = {{&input, 0u}, {&lane, 4u}, {nullptr, 8u}};
+    auto& constInput = make(IrOpcode::ReadConst, IrType::U32, {&constant(0), &constant(0)});
+    auto& constLane = make(IrOpcode::ReadConst, IrType::U32, {&constant(0), &constant(1)});
+    auto& constOutside = make(IrOpcode::ReadConst, IrType::U32, {&constant(0), &constant(4)});
+    auto& constItself = make(IrOpcode::ReadConst, IrType::U32, {&constant(0), &constant(2)});
+    plan.srtReads[2].value = &constItself;
+    auto& carried = make(IrOpcode::Phi, IrType::U32, {&input});
+    auto& firstCarried = make(IrOpcode::ReadFirstLane, IrType::U32, {&carried, &varying});
+    carried.AddArgument(&make(IrOpcode::IAdd32, IrType::U32, {&firstCarried, &constant(1)}));
+    plan.srtReads.push_back({nullptr, 12u});
+    auto& constFirstItself = make(IrOpcode::ReadConst, IrType::U32, {&constant(0), &constant(3)});
+    plan.srtReads[3].value = &make(IrOpcode::ReadFirstLane, IrType::U32, {&constFirstItself, &varying});
     std::array<std::uint32_t, 1> data{4};
     SrtRuntime runtime{data};
-    for (const auto& [value, expected, name] : std::array<std::tuple<IrValue*, bool, const char*>, 11>{{
+    for (const auto& [value, expected, name] : std::array<std::tuple<IrValue*, bool, const char*>, 21>{{
              {&constant(3), true, "a constant"},
              {&input, true, "user data"},
              {&uniform, true, "a comparison of user data"},
@@ -575,13 +591,23 @@ void verifyMayEvaluate() {
              {&divergent, false, "a phi whose operands differ"},
              {&latch, false, "a scalar loop counter's latch condition"},
              {&sum, true, "the low word of a carry add of user data"},
+             {&firstInput, true, "a first-lane read of user data"},
+             {&firstLane, false, "a first-lane read of a vector register"},
+             {&firstActive, true, "a first-lane read of a select whose active arm is user data"},
+             {&firstInactive, false, "a first-lane read of a select whose active arm is a vector register"},
+             {&constInput, true, "a constant read whose slot holds user data"},
+             {&constLane, false, "a constant read whose slot holds a vector register"},
+             {&constOutside, false, "a constant read past the last slot"},
+             {&constItself, false, "a constant read whose slot reads itself"},
+             {&firstCarried, false, "a first-lane read of a loop-carried value that reaches itself through the read"},
+             {&constFirstItself, false, "a constant read whose slot reads itself through a first-lane read"},
          }}) {
         require(Detail::Evaluator::MayEvaluate(plan, value) == expected, (std::string("MayEvaluate misjudged ") + name).c_str());
         Detail::Evaluator evaluator(plan, runtime);
         std::uint32_t result = 0;
         require(evaluator.Evaluate(Detail::IrNode(value), result) == expected, (std::string("MayEvaluate disagreed with the evaluator on ") + name).c_str());
     }
-    const std::array<IrOpcode, 9> takenAsEvaluable{IrOpcode::Identity, IrOpcode::Phi, IrOpcode::Void, IrOpcode::GetUserData, IrOpcode::GetShaderBase, IrOpcode::ReadFirstLane, IrOpcode::ReadConst, IrOpcode::LoadAddressU32, IrOpcode::ReadConstBuffer};
+    const std::array<IrOpcode, 7> takenAsEvaluable{IrOpcode::Identity, IrOpcode::Phi, IrOpcode::Void, IrOpcode::GetUserData, IrOpcode::GetShaderBase, IrOpcode::LoadAddressU32, IrOpcode::ReadConstBuffer};
     for (std::uint16_t raw = 0; raw < static_cast<std::uint16_t>(IrOpcode::Count); ++raw) {
         const auto opcode = static_cast<IrOpcode>(raw);
         if (std::ranges::find(takenAsEvaluable, opcode) != takenAsEvaluable.end()) continue;
@@ -1030,6 +1056,7 @@ struct TableGuest {
     std::array<std::uint32_t, 20> srt{};
 };
 
+constexpr std::uint32_t TableFormat16UNorm = 7;
 constexpr std::uint32_t TableFormat8888UNorm = 56;
 constexpr std::uint32_t TableFormat32UInt = 20;
 constexpr std::uint32_t TableFormat16SInt = 12;
@@ -1109,6 +1136,13 @@ TableCompiled CompileTable(ShaderRecompiler::RecompileRequest request) {
     return out;
 }
 
+std::array<std::uint32_t, 8> TableSelected(const ShaderRecompiler::ImageTableSnapshot& tables, std::uint32_t table, const char* message) {
+    namespace Abi = ShaderRecompiler::ImageTableAbi;
+    const auto& column = tables.tables.at(table);
+    require(column.records == 1u && column.keys == 1u && column.codes.size() == 1u && (column.codes[0] & Abi::PoisonFlag) == 0u, message);
+    return tables.words.at(column.codes[0]).dwords;
+}
+
 void verifyImageTableKeys() {
     using namespace ShaderRecompiler;
     namespace Abi = ImageTableAbi;
@@ -1240,6 +1274,43 @@ void verifyImageTableBulkReads() {
     require(cutByRun->snapshot.tables == cutByWord->snapshot.tables && cutWordReads.wordReads == 12u && cutRunReads.wordReads == 0u && cutRunReads.bulkWords == 12u, "image table bulk reads: a record cut by an unmapped page read other words in runs than word by word");
 }
 
+void verifyImageTableBulkKeyScan() {
+    using namespace ShaderRecompiler;
+    namespace Abi = ImageTableAbi;
+    static TableTexture textures[2];
+    alignas(4096) static TableGuest guest;
+    std::fill(guest.heap.begin(), guest.heap.end(), std::array<std::uint32_t, 8>{});
+    guest.heap[0] = TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D);
+    guest.heap[1] = TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D);
+    guest.heap[3] = guest.heap[0];
+    guest.materials = {{{0u, 1u, 0u, 0u}, {0u, 0x08000003u, 0u, 0u}, {0u, 1u, 0u, 0u}}};
+    FillTableSrt(guest, guest.heap.data(), 32u, 4u);
+    const auto userData = TableUserData(guest);
+    const auto request = TableRequest(TableMaterialCode, userData);
+    const auto capture = [&](CountingMemory& memory, bool bulk) {
+        SrtRuntime runtime;
+        runtime.userData = request.context.userData;
+        runtime.shaderBase = request.shader.codeAddress;
+        runtime.userContext = &memory;
+        runtime.readMemory = &CountingRead;
+        runtime.readSpecializationMemory = &CountingRead;
+        runtime.accessible = &CountingAccessible;
+        if (bulk) runtime.readWords = &CountingReadWords;
+        return CaptureResources(request, runtime);
+    };
+    const auto begin = reinterpret_cast<std::uintptr_t>(guest.materials.data());
+    CountingMemory byWordReads{begin, begin + sizeof(guest.materials)};
+    CountingMemory byRunReads = byWordReads;
+    const auto byWord = capture(byWordReads, false);
+    const auto byRun = capture(byRunReads, true);
+    require(byWordReads.wordReads == guest.materials.size(), "image table bulk key scan: without a bulk reader the keys were not read one word each");
+    require(byRunReads.wordReads == 0u, "image table bulk key scan: the keys were read word by word although a bulk reader exists");
+    require(byRun->snapshot.tables == byWord->snapshot.tables, "image table bulk key scan: scanning the keys in bulk changed the snapshot");
+    const auto& snapshot = byRun->snapshot.tables.tables.at(0);
+    const auto outside = Abi::PoisonCode(static_cast<std::uint32_t>(Abi::PoisonReason::OutsideDomain));
+    require(snapshot.keys == 4u && snapshot.codes[0] == snapshot.codes[3] && snapshot.codes[0] != snapshot.codes[1] && snapshot.codes[2] == outside, "image table bulk key scan: the narrowed codes are wrong");
+}
+
 void verifyImageTables() {
     using namespace ShaderRecompiler;
     namespace Abi = ImageTableAbi;
@@ -1421,7 +1492,6 @@ void verifyImageTables() {
 
 void verifyImageTableKeyUnion() {
     using namespace ShaderRecompiler;
-    namespace Abi = ImageTableAbi;
     static TableTexture textures[3];
     alignas(4096) static TableGuest guest;
     auto& heap = guest.heap;
@@ -1436,24 +1506,25 @@ void verifyImageTableKeyUnion() {
     const auto userData = TableUserData(guest);
     const auto request = TableRequest(TableTwoKeyCode, userData);
     const auto plan = GetResourcePlan(request);
-    require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u, "image table key union: two keyed reads of one table were not planned as one image of one column");
-    const auto& column = plan->descriptorSources.at(plan->info.images[0].source).tableColumn;
-    require(column.has_value() && column->keyDomains.size() == 2u && column->keyDomains[0].value && column->keyDomains[1].value && column->keyDomains[0].source != column->keyDomains[1].source, "image table key union: the column does not carry both value key domains");
+    require(plan->info.images.size() == 2u && plan->info.images[0].table == 0u && plan->info.images[1].table == 1u, "image table key union: two keyed reads of one heap were not planned as two tables");
+    const auto& first = plan->descriptorSources.at(plan->info.images[0].source).tableColumn;
+    const auto& second = plan->descriptorSources.at(plan->info.images[1].source).tableColumn;
+    require(first.has_value() && second.has_value() && first->selected && second->selected && first->heapSource == second->heapSource && first->selection != second->selection, "image table key union: the two keys do not each select a record of the one heap");
+    const auto held = [](const ImageTableSnapshot& tables, std::uint32_t table) { return TableSelected(tables, table, "image table key union: a table does not hold one readable selected record"); };
     const auto compiled = CompileTable(request);
-    const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0);
-    const auto outside = Abi::PoisonCode(static_cast<std::uint32_t>(Abi::PoisonReason::OutsideDomain));
-    require(compiled.capture->snapshot.tables.tables.size() == 1u && snapshot.keys == 4u && (snapshot.codes[0] & Abi::PoisonFlag) == 0u && (snapshot.codes[1] & Abi::PoisonFlag) == 0u && snapshot.codes[2] == outside && (snapshot.codes[3] & Abi::PoisonFlag) == 0u, "image table key union: the snapshot does not hold the union of the two keys' records");
+    const auto& tables = compiled.capture->snapshot.tables;
+    require(tables.tables.size() == 2u && held(tables, 0u) == heap[1] && held(tables, 1u) == heap[3], "image table key union: the tables do not hold the two keys' records");
+    require(TableCovered(compiled.regions, &heap[1], sizeof(heap[1])) && TableCovered(compiled.regions, &heap[3], sizeof(heap[3])) && !TableCovered(compiled.regions, &heap[0], sizeof(std::uint32_t)) && !TableCovered(compiled.regions, &heap[2], sizeof(std::uint32_t)), "image table key union: records no key selects were captured");
     const auto* images = TableBinding(*compiled.result, DescriptorRole::ImageTable);
-    require(images != nullptr && images->count == 3u, "image table key union: the binding does not hold the three distinct images of the selected records");
+    require(images != nullptr && images->count == 2u, "image table key union: the binding does not hold the two distinct images of the selected records");
     guest.srt[17] = 2u;
     const auto moved = CompileTable(request);
-    const auto& movedSnapshot = moved.capture->snapshot.tables.tables.at(0);
-    require((movedSnapshot.codes[2] & Abi::PoisonFlag) == 0u && movedSnapshot.codes[3] == outside && (movedSnapshot.codes[1] & Abi::PoisonFlag) == 0u && moved.result->spirv == compiled.result->spirv, "image table key union: a changed second key did not move the union or changed the module");
+    const auto& movedTables = moved.capture->snapshot.tables;
+    require(held(movedTables, 0u) == heap[1] && held(movedTables, 1u) == heap[2] && TableCovered(moved.regions, &heap[2], sizeof(heap[2])) && !TableCovered(moved.regions, &heap[3], sizeof(std::uint32_t)) && moved.result->spirv == compiled.result->spirv, "image table key union: a changed second key did not move its record alone or changed the module");
 }
 
 void verifyImageTableLateKey() {
     using namespace ShaderRecompiler;
-    namespace Abi = ImageTableAbi;
     static TableTexture textures[3];
     alignas(4096) static TableGuest guest;
     auto& heap = guest.heap;
@@ -1468,15 +1539,16 @@ void verifyImageTableLateKey() {
     const auto userData = TableUserData(guest);
     const auto request = TableRequest(TableLateKeyCode, userData);
     const auto plan = GetResourcePlan(request);
-    require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u, "image table late key: two keyed reads of one table were not planned as one image of one column");
-    const auto& column = plan->descriptorSources.at(plan->info.images[0].source).tableColumn;
-    require(column.has_value() && column->keyDomains.size() == 2u && column->keyDomains[0].value && column->keyDomains[1].value && column->keyDomains[0].source != column->keyDomains[1].source, "image table late key: the column does not carry both value key domains");
+    require(plan->info.images.size() == 2u && plan->info.images[0].table == 0u && plan->info.images[1].table == 1u, "image table late key: two keyed reads of one heap were not planned as two tables");
+    const auto& first = plan->descriptorSources.at(plan->info.images[0].source).tableColumn;
+    const auto& second = plan->descriptorSources.at(plan->info.images[1].source).tableColumn;
+    require(first.has_value() && second.has_value() && first->selected && second->selected && first->selection != second->selection, "image table late key: a key read after the first image operation does not select its own record");
+    const auto held = [](const ImageTableSnapshot& tables, std::uint32_t table) { return TableSelected(tables, table, "image table late key: a table does not hold one readable selected record"); };
     const auto compiled = CompileTable(request);
-    const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0);
-    const auto outside = Abi::PoisonCode(static_cast<std::uint32_t>(Abi::PoisonReason::OutsideDomain));
-    require(compiled.capture->snapshot.tables.tables.size() == 1u && snapshot.keys == 4u && (snapshot.codes[0] & Abi::PoisonFlag) == 0u && (snapshot.codes[1] & Abi::PoisonFlag) == 0u && snapshot.codes[2] == outside && (snapshot.codes[3] & Abi::PoisonFlag) == 0u, "image table late key: the snapshot does not hold the union of the two keys' records");
+    const auto& tables = compiled.capture->snapshot.tables;
+    require(tables.tables.size() == 2u && held(tables, 0u) == heap[1] && held(tables, 1u) == heap[3], "image table late key: the tables do not hold the two keys' records");
     const auto* images = TableBinding(*compiled.result, DescriptorRole::ImageTable);
-    require(images != nullptr && images->count == 3u, "image table late key: the binding does not hold the three distinct images of the selected records");
+    require(images != nullptr && images->count == 2u, "image table late key: the binding does not hold the two distinct images of the selected records");
 }
 
 void verifyImageTableUnkeyedAccess() {
@@ -1499,13 +1571,19 @@ void verifyImageTableUnkeyedAccess() {
         const auto userData = TableUserData(guest);
         const auto request = TableRequest(*code, userData);
         const auto plan = GetResourcePlan(request);
-        require(!plan->info.images.empty() && std::all_of(plan->info.images.begin(), plan->info.images.end(), [](const ImageResource& image) { return image.table == 0u; }), "image table unkeyed access: the two reads were not planned as one table");
-        const auto& column = plan->descriptorSources.at(plan->info.images[0].source).tableColumn;
-        require(column.has_value() && column->unkeyed && column->keyDomains.empty(), "image table unkeyed access: a column shared with a read that has no key domain kept a key domain");
+        const auto& planned = plan->info.images;
+        require(planned.size() == 2u && std::all_of(planned.begin(), planned.end(), [](const ImageResource& image) { return image.table != NoTable; }), "image table unkeyed access: the two reads were not planned as two tables");
+        const auto columnOf = [&](const ImageResource& image) { return *plan->descriptorSources.at(image.source).tableColumn; };
+        const auto selected = std::find_if(planned.begin(), planned.end(), [&](const ImageResource& image) { return columnOf(image).selected; });
+        const auto whole = std::find_if(planned.begin(), planned.end(), [&](const ImageResource& image) { return !columnOf(image).selected; });
+        require(selected != planned.end() && whole != planned.end(), "image table unkeyed access: the read keyed through the SRT does not select its record beside the read keyed per lane");
+        require(columnOf(*whole).unkeyed && columnOf(*whole).keyDomains.empty(), "image table unkeyed access: the read keyed per lane kept a key domain");
         const auto compiled = CompileTable(request);
-        const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0);
+        const auto& tables = compiled.capture->snapshot.tables;
+        require(TableSelected(tables, selected->table, "image table unkeyed access: a table does not hold one readable selected record") == heap[2], "image table unkeyed access: the read keyed through the SRT does not hold its record");
+        const auto& snapshot = tables.tables.at(whole->table);
         const auto outside = Abi::PoisonCode(static_cast<std::uint32_t>(Abi::PoisonReason::OutsideDomain));
-        require(snapshot.keys == 4u && std::none_of(snapshot.codes.begin(), snapshot.codes.end(), [&](std::uint32_t code) { return code == outside; }), "image table unkeyed access: the shared column was narrowed by the keyed read");
+        require(snapshot.keys == 4u && std::none_of(snapshot.codes.begin(), snapshot.codes.end(), [&](std::uint32_t entry) { return entry == outside; }), "image table unkeyed access: the table keyed per lane was narrowed");
         const auto* images = TableBinding(*compiled.result, DescriptorRole::ImageTable);
         require(images != nullptr && images->count == 4u, "image table unkeyed access: the binding does not hold the whole domain");
     }
@@ -1529,18 +1607,19 @@ void verifyImageTableSrtKey() {
     const auto plan = GetResourcePlan(request);
     require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u, "image table SRT key: the T# was not planned as a table");
     const auto& column = plan->descriptorSources.at(plan->info.images[0].source).tableColumn;
-    require(column.has_value() && !column->keyDomains.empty(), "image table SRT key: a key read through the SRT has no key domain");
+    require(column.has_value() && column->selected && column->selection < plan->descriptorSources.size(), "image table SRT key: a T# whose key is read through the SRT does not select its record");
     const auto compiled = CompileTable(request);
-    const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0);
-    const auto outside = Abi::PoisonCode(static_cast<std::uint32_t>(Abi::PoisonReason::OutsideDomain));
-    require(snapshot.records == 4u && snapshot.keys == 4u && snapshot.codes.size() == 4u && (snapshot.codes[0] & Abi::PoisonFlag) == 0u && snapshot.codes[1] == outside && (snapshot.codes[2] & Abi::PoisonFlag) == 0u && snapshot.codes[3] == outside, "image table SRT key: the snapshot was not narrowed to the key read through the SRT");
+    const auto& tables = compiled.capture->snapshot.tables;
+    const auto& snapshot = tables.tables.at(0);
+    require(snapshot.records == 1u && snapshot.keys == 1u && snapshot.codes.size() == 1u && (snapshot.codes[0] & Abi::PoisonFlag) == 0u && tables.words.at(snapshot.codes[0]).dwords == heap[2], "image table SRT key: the snapshot does not hold the selected record alone");
     const auto* images = TableBinding(*compiled.result, DescriptorRole::ImageTable);
-    require(images != nullptr && images->count == 2u, "image table SRT key: the binding holds other entries than the selected one and the first");
-    require(TableCovered(compiled.regions, &guest.srt[16], sizeof(std::uint32_t)), "image table SRT key: the key was not captured");
+    require(images != nullptr && images->count == 1u, "image table SRT key: the binding holds other entries than the selected one");
+    require(TableCovered(compiled.regions, &guest.srt[16], sizeof(std::uint32_t)) && TableCovered(compiled.regions, &heap[2], sizeof(heap[2])), "image table SRT key: the key or the selected record was not captured");
+    require(!TableCovered(compiled.regions, &heap[0], sizeof(std::uint32_t)) && !TableCovered(compiled.regions, &heap[1], sizeof(std::uint32_t)) && !TableCovered(compiled.regions, &heap[3], sizeof(std::uint32_t)), "image table SRT key: records the key does not select were captured");
     guest.srt[16] = 3u;
     const auto moved = CompileTable(request);
-    const auto& movedSnapshot = moved.capture->snapshot.tables.tables.at(0);
-    require((movedSnapshot.codes[3] & Abi::PoisonFlag) == 0u && movedSnapshot.codes[2] == outside && movedSnapshot.codes[1] == outside && moved.result->spirv == compiled.result->spirv && moved.key == compiled.key, "image table SRT key: a changed key did not move the snapshot alone");
+    const auto& movedTables = moved.capture->snapshot.tables;
+    require(movedTables.words.at(movedTables.tables.at(0).codes.at(0)).dwords == heap[3] && TableCovered(moved.regions, &heap[3], sizeof(heap[3])) && !TableCovered(moved.regions, &heap[2], sizeof(std::uint32_t)) && moved.result->spirv == compiled.result->spirv && moved.key == compiled.key, "image table SRT key: a changed key did not move the selected record alone");
 }
 
 void verifyImageTableMaskedOffset() {
@@ -1587,11 +1666,24 @@ void verifyImageTableRegistration() {
     const std::vector<std::uint32_t> storeCode{0xf4080100u, 0xfa000000u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf0200f08u, 0x00050200u, 0xbf810000u};
     const auto store = CompileTable(TableRequest(storeCode, userData));
     const auto& storePlan = *store.capture->plan;
-    require(storePlan.info.images.size() == 1u && storePlan.info.images[0].table == 0u && storePlan.info.images[0].tableOperation == TableOperation::Unsupported, "image table registration: a table T# feeding image_store was not an unsupported table operation");
+    require(storePlan.info.images.size() == 1u && storePlan.info.images[0].table == 0u && storePlan.info.images[0].tableOperation == TableOperation::Supported, "image table registration: a table T# feeding image_store was not a supported table operation");
     const auto& storeMap = TableMap(*store.result);
     const auto storeFault = storeMap[Abi::TableHeader(0) + Abi::TableFault];
-    require((storeFault & Abi::PoisonFlag) != 0u && store.result->imageTablePoison.at(storeFault & ~Abi::PoisonFlag).reason == static_cast<std::uint32_t>(Abi::PoisonReason::UnsupportedOperation) && store.result->imageTableFaults != 0u, "image table registration: the store's table does not fault on use");
-    require(TableBinding(*store.result, DescriptorRole::ImageTable) == nullptr || TableBinding(*store.result, DescriptorRole::ImageTable)->count == 0u, "image table registration: an unsupported table bound elements");
+    const auto* storeTable = TableBinding(*store.result, DescriptorRole::ImageStorageTable);
+    require((storeFault & Abi::PoisonFlag) == 0u && storeTable != nullptr && storeTable->count != 0u && std::all_of(storeTable->imageWritten.begin(), storeTable->imageWritten.end(), [](bool written) { return written; }), "image table registration: the store's table was not bound as written storage elements");
+    require(TableBinding(*store.result, DescriptorRole::ImageTable) == nullptr || TableBinding(*store.result, DescriptorRole::ImageTable)->count == 0u, "image table registration: a store-only table bound sampled elements");
+    const std::array<std::pair<std::uint32_t, const char*>, 3> unsupportedStores{{{0xf0240f08u, "image_store_mip"}, {0xf0280f08u, "image_store_pck"}, {0xf0440108u, "image_atomic_add"}}};
+    for (const auto& [instruction, name] : unsupportedStores) {
+        auto code = storeCode;
+        code[8] = instruction;
+        const auto unsupported = CompileTable(TableRequest(code, userData));
+        const auto& plan = *unsupported.capture->plan;
+        require(plan.info.images.size() == 1u && plan.info.images[0].table == 0u && (plan.info.images[0].tableOperation == TableOperation::Unsupported || plan.info.images[0].mipMode == ImageMipMode::DynamicStorage), (std::string("image table registration: a table T# feeding ") + name + " was not planned as an unsupported or dynamic-mip table store").c_str());
+        const auto fault = TableMap(*unsupported.result)[Abi::TableHeader(0) + Abi::TableFault];
+        require((fault & Abi::PoisonFlag) != 0u && unsupported.result->imageTablePoison.at(fault & ~Abi::PoisonFlag).reason == static_cast<std::uint32_t>(Abi::PoisonReason::UnsupportedOperation) && unsupported.result->imageTableFaults != 0u, (std::string("image table registration: the ") + name + " table does not fault on use").c_str());
+        const auto* storage = TableBinding(*unsupported.result, DescriptorRole::ImageStorageTable);
+        require(storage == nullptr || storage->count == 0u, (std::string("image table registration: the unsupported ") + name + " bound storage table elements").c_str());
+    }
 
     const std::vector<std::uint32_t> compareCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf0fc0108u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u};
     auto compareRequest = TableRequest(compareCode, userData);
@@ -1828,6 +1920,281 @@ void verifyPointerTableOffsetCarry() {
 #endif
 }
 
+void verifySelectedRecordFallbacks() {
+    using namespace ShaderRecompiler;
+    namespace Abi = ImageTableAbi;
+    static TableTexture textures[2];
+    alignas(4096) static TableGuest guest;
+    auto& heap = guest.heap;
+    std::fill(heap.begin(), heap.end(), std::array<std::uint32_t, 8>{});
+    heap[0] = TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D);
+    heap[1] = TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D);
+    heap[2] = heap[1];
+    heap[3] = heap[0];
+    FillTableSrt(guest, heap.data(), 32u, 4u);
+    guest.materials = {};
+    guest.srt[16] = 2u;
+    const auto userData = TableUserData(guest);
+    const auto request = TableRequest(TableSrtKeyCode, userData);
+    const auto valid = CompileTable(request);
+
+    heap[2][3] &= 0x0fffffffu;
+    const auto invalid = CompileTable(request);
+    const auto& invalidMap = TableMap(*invalid.result);
+    const auto invalidCodes = TableCodes(invalidMap, 0);
+    require(invalidCodes.size() == 1u && invalidCodes[0] != Abi::NullCode && (invalidCodes[0] & Abi::PoisonFlag) == 0u, "selected record fallbacks: an invalid selected T# has no entry");
+    const auto invalidFlags = invalidMap.at(invalidCodes[0] + Abi::ImageRecordFlags);
+    require((invalidFlags & Abi::PoisonFlag) != 0u, "selected record fallbacks: an invalid selected T# was bound");
+    const auto& poison = invalid.result->imageTablePoison.at(invalidFlags & ~Abi::PoisonFlag);
+    require(poison.reason == static_cast<std::uint32_t>(Abi::PoisonReason::NotImage) && poison.dwordCount == 8u && poison.words[3] == heap[2][3] && invalid.result->imageTableFaults != 0u, "selected record fallbacks: an invalid selected T# was not poisoned with its words");
+    require(invalid.result->variantId == valid.result->variantId, "selected record fallbacks: an invalid selected T# changed the artifact");
+    heap[2] = heap[1];
+
+    guest.srt[16] = 200u;
+    const auto beyond = CompileTable(request);
+    const auto beyondCodes = TableCodes(TableMap(*beyond.result), 0);
+    require(beyondCodes.size() == 1u && beyondCodes[0] == Abi::NullCode && beyond.result->imageTableFaults == 0u && !TableTouched(beyond.regions, heap.data(), sizeof(heap)), "selected record fallbacks: a key past the heap did not select a null image without reading the heap");
+    guest.srt[16] = 2u;
+
+    void* reserved = nullptr;
+#ifdef _WIN32
+    reserved = VirtualAlloc(nullptr, 4096, MEM_RESERVE, PAGE_NOACCESS);
+#else
+    reserved = mmap(nullptr, 4096, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (reserved == MAP_FAILED) reserved = nullptr;
+#endif
+    require(reserved != nullptr, "selected record fallbacks: cannot reserve an inaccessible page");
+    FillTableSrt(guest, reserved, 32u, 4u);
+    const auto unmapped = CompileTable(request);
+    const auto& unmappedTable = unmapped.capture->snapshot.tables.tables.at(0);
+    require(unmappedTable.keys == 1u && unmappedTable.codes.at(0) == Abi::PoisonCode(static_cast<std::uint32_t>(Abi::PoisonReason::Unmapped)) && !TableTouched(unmapped.regions, reserved, 4096u), "selected record fallbacks: an unmapped selected record is not Unmapped poison or was read");
+    FillTableSrt(guest, heap.data(), 32u, 4u);
+
+    std::vector<std::uint32_t> loadedKey(TableSrtKeyCode);
+    const auto shift = std::find(loadedKey.begin(), loadedKey.end(), 0x8f108510u);
+    require(shift != loadedKey.end(), "selected record fallbacks: the program has no s_lshl_b32");
+    loadedKey.insert(shift, {0xf4200406u, 0x20000000u});
+    guest.materials[0][1] = 3u;
+    guest.srt[16] = 4u;
+    const auto loadedRequest = TableRequest(loadedKey, userData);
+    const auto loadedPlan = GetResourcePlan(loadedRequest);
+    require(loadedPlan->info.images.size() == 1u && loadedPlan->descriptorSources.at(loadedPlan->info.images[0].source).tableColumn->selected, "selected record fallbacks: a key loaded from a constant buffer at an SRT offset does not select its record");
+    const auto loaded = CompileTable(loadedRequest);
+    require(TableSelected(loaded.capture->snapshot.tables, 0u, "selected record fallbacks: a loaded key does not hold one readable selected record") == heap[3] && TableCovered(loaded.regions, &guest.materials[0][1], sizeof(std::uint32_t)), "selected record fallbacks: the record of a loaded key was not selected or its key was not captured");
+    const auto materials = TableV(guest.materials.data(), 16u, 3u);
+    const auto unreadable = TableV(reserved, 16u, 3u);
+    std::copy(unreadable.begin(), unreadable.end(), guest.srt.begin() + 8);
+    const auto lost = CompileTable(loadedRequest);
+    const auto& lostTable = lost.capture->snapshot.tables.tables.at(0);
+    require(lostTable.keys == 0u && lostTable.fault == Abi::PoisonCode(static_cast<std::uint32_t>(Abi::PoisonReason::Unmapped)) && lost.result->imageTableFaults != 0u && lost.result->variantId == loaded.result->variantId, "selected record fallbacks: a key that cannot be read did not fault its table");
+    std::copy(materials.begin(), materials.end(), guest.srt.begin() + 8);
+#ifdef _WIN32
+    VirtualFree(reserved, 0, MEM_RELEASE);
+#else
+    munmap(reserved, 4096);
+#endif
+
+    auto writtenKey = loadedKey;
+    const auto store = std::find(writtenKey.begin(), writtenKey.end(), 0x80070000u);
+    require(store != writtenKey.end(), "selected record fallbacks: the program has no buffer_store_dword");
+    *store = 0x80030000u;
+    const auto writtenRequest = TableRequest(writtenKey, userData);
+    const auto writtenPlan = GetResourcePlan(writtenRequest);
+    const auto& writtenColumn = writtenPlan->descriptorSources.at(writtenPlan->info.images.at(0).source).tableColumn;
+    require(writtenColumn.has_value() && !writtenColumn->selected && writtenColumn->keyDomains.empty(), "selected record fallbacks: a key loaded from a buffer the shader writes selected its record");
+    const auto written = CompileTable(writtenRequest);
+    require(written.capture->snapshot.tables.tables.at(0).keys == 4u, "selected record fallbacks: a key loaded from a buffer the shader writes did not keep the whole table");
+    require(!loadedPlan->descriptorSources.at(loadedPlan->info.images[0].source).tableColumn->keyDomains.empty(), "selected record fallbacks: a key loaded from a constant buffer has no key domain");
+    auto pointerWritten = loadedKey;
+    const auto end = std::find(pointerWritten.begin(), pointerWritten.end(), 0xbf810000u);
+    require(end != pointerWritten.end(), "selected record fallbacks: the program has no s_endpgm");
+    pointerWritten.insert(end, {0xdc708000u, 0x001c0100u});
+    const auto pointerPlan = GetResourcePlan(TableRequest(pointerWritten, userData));
+    const auto& pointerColumn = pointerPlan->descriptorSources.at(pointerPlan->info.images.at(0).source).tableColumn;
+    require(pointerPlan->info.bdaWrites && pointerColumn.has_value() && !pointerColumn->selected && pointerColumn->keyDomains.empty(), "selected record fallbacks: a key loaded from a constant buffer in a shader that stores through a pointer selected its record or kept its key domain");
+
+    const auto lanePlan = GetResourcePlan(TableRequest(TableWholeCode, userData));
+    require(!lanePlan->info.images.empty() && !lanePlan->descriptorSources.at(lanePlan->info.images[0].source).tableColumn->selected, "selected record fallbacks: a T# selected per lane left the table");
+
+    alignas(4096) static std::array<std::array<std::uint32_t, 8>, 70000> large{};
+    large.back() = heap[1];
+    FillTableSrt(guest, large.data(), 32u, static_cast<std::uint32_t>(large.size()));
+    guest.srt[16] = static_cast<std::uint32_t>(large.size() - 1u);
+    const auto pastCap = CompileTable(request);
+    require(TableSelected(pastCap.capture->snapshot.tables, 0u, "selected record fallbacks: a record past the key cap does not hold one readable selected record") == heap[1] && !pastCap.capture->snapshot.tables.tables[0].outside && pastCap.result->imageTableFaults == 0u, "selected record fallbacks: a record past the key cap was not selected");
+    FillTableSrt(guest, heap.data(), 32u, 4u);
+}
+
+void verifySelectedRecordOffsetCarry() {
+    using namespace ShaderRecompiler;
+    namespace Abi = ImageTableAbi;
+    static TableTexture texture;
+    alignas(4096) static TableGuest guest;
+    std::fill(guest.heap.begin(), guest.heap.end(), std::array<std::uint32_t, 8>{});
+    const auto image = TableT(texture.bytes.data(), TableFormat8888UNorm, TableType2D);
+    auto* words = guest.heap.front().data();
+    std::copy(image.begin(), image.end(), words + 0x50u / 4u);
+    FillTableSrt(guest, guest.heap.data(), 0u, static_cast<std::uint32_t>(sizeof(guest.heap)));
+    const auto userData = TableUserData(guest);
+    const std::vector<std::uint32_t> code{0xf4080100u, 0xfa000000u, 0xf4080700u, 0xfa000030u, 0xf4080800u, 0xfa000010u, 0xf4000440u, 0xfa000040u, 0x8711ff11u,
+        0x000000ffu, 0x8f128511u, 0x8113ff12u, 0xffffff10u, 0x8114ff12u, 0xffffff20u, 0xf4280902u, 0x26000040u, 0xf4280a02u, 0x28000040u, 0xf09c0f08u,
+        0x01090000u, 0xe0700000u, 0x80070000u, 0xbf810000u};
+    const auto request = TableRequest(code, userData);
+    const auto plan = GetResourcePlan(request);
+    require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u, "selected record offset carry: a T# read by two loads of its record was not planned as a table");
+    const auto& column = *plan->descriptorSources.at(plan->info.images[0].source).tableColumn;
+    require(column.selected && !column.address && column.stride == 32u && column.addend == 0xffffff10u && column.offset == 0x40u && column.foldedAddend == 0xffffff10u && column.foldedCarry == 16u, "selected record offset carry: the column is wrong");
+    guest.srt[16] = 7u;
+    const auto carried = CompileTable(request);
+    const auto& carriedTable = carried.capture->snapshot.tables.tables.at(0);
+    require(carriedTable.keys == 1u && carriedTable.codes.at(0) == Abi::PoisonCode(static_cast<std::uint32_t>(Abi::PoisonReason::OffsetCarry)) && !TableTouched(carried.regions, guest.heap.data(), sizeof(guest.heap)), "selected record offset carry: the record whose second load wraps past 32 bits is not OffsetCarry poison or was read");
+    guest.srt[16] = 8u;
+    const auto wrapped = CompileTable(request);
+    require(TableSelected(wrapped.capture->snapshot.tables, 0u, "selected record offset carry: the record past the wrap does not hold one readable selected record") == image && TableCovered(wrapped.regions, words + 0x50u / 4u, sizeof(image)), "selected record offset carry: the record past the wrap was not selected");
+}
+
+std::vector<std::uint32_t> WithImageOperation(std::vector<std::uint32_t> code, std::uint32_t word0, bool keepSample) {
+    const auto sample = std::find(code.begin(), code.end(), 0xf09c0f08u);
+    require(sample != code.end() && sample + 1 != code.end(), "image operation variant: the program has no image_sample_lz");
+    if (keepSample) {
+        code.insert(sample + 2, {word0, 0x00050000u});
+    } else {
+        sample[0] = word0;
+        sample[1] = 0x00050000u;
+    }
+    return code;
+}
+
+void verifySelectedRecordDirectOperations() {
+    using namespace ShaderRecompiler;
+    static TableTexture textures[2];
+    alignas(4096) static TableGuest guest;
+    auto& heap = guest.heap;
+    std::fill(heap.begin(), heap.end(), std::array<std::uint32_t, 8>{});
+    heap[0] = TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D);
+    heap[1] = TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D);
+    heap[2] = heap[1];
+    heap[3] = heap[0];
+    FillTableSrt(guest, heap.data(), 32u, 4u);
+    guest.srt[16] = 2u;
+    const auto userData = TableUserData(guest);
+    const std::array<std::tuple<const char*, std::uint32_t, std::uint32_t>, 4> operations{{{"image_store", 0xf0200f08u, TableFormat32UInt}, {"image_atomic_add", 0xf0440108u, TableFormat32UInt}, {"image_load_pck", 0xf0080f08u, TableFormat32UInt}, {"image_load_by2", 0xf1080308u, TableFormat16UNorm}}};
+    for (const auto& [name, word0, format] : operations) {
+        const auto what = std::string("selected record direct operations: ") + name;
+        heap[2] = TableT(textures[1].bytes.data(), format, TableType2D);
+        const auto code = WithImageOperation(TableSrtKeyCode, word0, false);
+        const auto request = TableRequest(code, userData);
+        const auto plan = GetResourcePlan(request);
+        require(plan->info.images.size() == 1u && plan->info.images[0].table == NoTable && !plan->descriptorSources.at(plan->info.images[0].source).tableColumn.has_value(), (what + " through a T# whose record the CPU evaluates was planned through an image table").c_str());
+        const auto compiled = CompileTable(request);
+        require(TableBinding(*compiled.result, DescriptorRole::ImageTable) == nullptr && compiled.result->imageTableFaults == 0u && TableCovered(compiled.regions, &heap[2], sizeof(heap[2])), (what + " through a T# whose record the CPU evaluates was not bound from its record").c_str());
+        const auto wholeCode = WithImageOperation(TableWholeCode, word0, false);
+        const auto whole = GetResourcePlan(TableRequest(wholeCode, userData));
+        require(whole->info.images.size() == 1u && whole->info.images[0].table == 0u && whole->info.images[0].tableOperation == (word0 == 0xf0200f08u ? TableOperation::Supported : TableOperation::Unsupported), (what + " through a T# keyed per lane left the image table or has the wrong table operation").c_str());
+    }
+    const auto mixedCode = WithImageOperation(TableSrtKeyCode, 0xf0200f08u, true);
+    const auto mixed = GetResourcePlan(TableRequest(mixedCode, userData));
+    require(mixed->info.images.size() == 2u && std::ranges::all_of(mixed->info.images, [&](const ImageResource& image) { return image.table == NoTable && !mixed->descriptorSources.at(image.source).tableColumn.has_value(); }), "selected record direct operations: a T# sampled and stored was planned through an image table for one of its operations");
+    const auto sampled = GetResourcePlan(TableRequest(TableSrtKeyCode, userData));
+    require(sampled->info.images.size() == 1u && sampled->info.images[0].table == 0u && sampled->descriptorSources.at(sampled->info.images[0].source).tableColumn->selected, "selected record direct operations: a T# that is only sampled no longer selects its record");
+}
+
+void verifyEvaluableHeapSamplerDirect() {
+    using namespace ShaderRecompiler;
+    static TableTexture textures[2];
+    alignas(4096) static TableGuest guest;
+    auto& heap = guest.heap;
+    std::fill(heap.begin(), heap.end(), std::array<std::uint32_t, 8>{});
+    heap[0] = TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D);
+    heap[1] = TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D);
+    heap[2] = heap[1];
+    heap[3] = heap[0];
+    FillTableSrt(guest, heap.data(), 32u, 4u);
+    guest.srt[16] = 2u;
+    const auto userData = TableUserData(guest);
+    const auto heapSampler = [](std::vector<std::uint32_t> code) {
+        const auto direct = std::find(code.begin(), code.end(), 0xf4080200u);
+        require(direct != code.end() && direct[1] == 0xfa000010u, "evaluable heap sampler: the program has no S# load from the SRT");
+        code.erase(direct, direct + 2);
+        const auto shift = std::find(code.begin(), code.end(), 0x8f108510u);
+        require(shift != code.end(), "evaluable heap sampler: the program has no key shift");
+        code.insert(shift + 1, {0xf4280202u, 0x20000000u});
+        return code;
+    };
+    const auto evaluatedCode = heapSampler(TableSrtKeyCode);
+    const auto evaluated = GetResourcePlan(TableRequest(evaluatedCode, userData));
+    require(evaluated->info.samplers.size() == 1u && evaluated->info.samplers[0].table == NoTable && !evaluated->descriptorSources.at(evaluated->info.samplers[0].source).tableColumn.has_value(), "evaluable heap sampler: an S# read from a heap at an offset the SRT walker evaluates was planned through a sampler table");
+    const auto perLaneCode = heapSampler(TableWholeCode);
+    const auto perLane = GetResourcePlan(TableRequest(perLaneCode, userData));
+    require(perLane->info.samplers.size() == 1u && perLane->info.samplers[0].table != NoTable && perLane->descriptorSources.at(perLane->info.samplers[0].source).tableColumn.has_value(), "evaluable heap sampler: an S# read from a heap at an offset keyed per lane was not planned through a sampler table");
+}
+
+void verifySelectedRecordUniformIndex() {
+    using namespace ShaderRecompiler;
+    static TableTexture textures[2];
+    alignas(4096) static TableGuest guest;
+    auto& heap = guest.heap;
+    std::fill(heap.begin(), heap.end(), std::array<std::uint32_t, 8>{});
+    heap[0] = TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D);
+    heap[1] = TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D);
+    heap[2] = heap[1];
+    heap[3] = heap[0];
+    FillTableSrt(guest, heap.data(), 32u, 4u);
+    guest.srt[16] = 2u;
+    const auto userData = TableUserData(guest);
+    const auto twoWaves = [&](const std::vector<std::uint32_t>& code) {
+        auto request = TableRequest(code, userData);
+        request.context.compute->numThreads = {128u, 1u, 1u};
+        return CompileTable(request);
+    };
+    const auto nonUniform = [](const TableCompiled& compiled) {
+        const auto spirv = compiled.result->spirv.Words();
+        std::uint32_t count = 0;
+        for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) {
+            if ((spirv[cursor] & 0xffffu) == spv::OpDecorate && (spirv[cursor] >> 16u) == 3u && spirv[cursor + 2u] == spv::DecorationNonUniform) ++count;
+        }
+        return count;
+    };
+    const auto selected = twoWaves(TableSrtKeyCode);
+    require(TableBinding(*selected.result, DescriptorRole::ImageTable) != nullptr && nonUniform(selected) == 0u, "selected record uniform index: the image table index of a selected record in a two-wave workgroup is decorated NonUniform");
+    const auto whole = twoWaves(TableWholeCode);
+    require(nonUniform(whole) != 0u, "selected record uniform index: the image table index of a table keyed per lane in a two-wave workgroup is not decorated NonUniform");
+}
+
+void verifyChainedTableOffset() {
+    using namespace ShaderRecompiler;
+    namespace Abi = ImageTableAbi;
+    static TableTexture textures[2];
+    alignas(256) static std::array<std::uint32_t, 0x3000> root{};
+    static std::array<std::uint32_t, 8> keys{};
+    static std::array<std::uint32_t, 4> output{};
+    const auto keysV = TableV(keys.data(), 0u, static_cast<std::uint32_t>(sizeof(keys)));
+    const auto outputV = TableV(output.data(), 0u, static_cast<std::uint32_t>(sizeof(output)));
+    std::copy(keysV.begin(), keysV.end(), root.begin());
+    std::copy(outputV.begin(), outputV.end(), root.begin() + 4);
+    const auto at = [&](std::uint32_t offset, std::array<std::uint32_t, 8> words) {
+        std::copy(words.begin(), words.end(), root.begin() + offset / 4u);
+    };
+    at(0x60u + 3u * 32u, TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D));
+    at(0x60u + 5u * 32u, TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D));
+    const auto base = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(root.data()));
+    const std::array<std::uint32_t, 2> userData{static_cast<std::uint32_t>(base), static_cast<std::uint32_t>(base >> 32u)};
+    const std::vector<std::uint32_t> code{0xf4080200u, 0xfa000000u, 0xf4080700u, 0xfa000010u, 0xf4080800u, 0xfa000020u, 0x7e200500u, 0x8f108210u, 0xf4200444u,
+        0x20000000u, 0x8711ff11u, 0x000000ffu, 0x8f128511u, 0x8113a012u, 0x81149013u, 0xf4080900u, 0x26000040u, 0xf4080a00u, 0x28000040u, 0xf09c0f08u,
+        0x01090000u, 0xe0700000u, 0x80070000u, 0xbf810000u};
+    auto request = TableRequest(code, userData);
+    request.shader.codeAddress = 0x37000u;
+    const auto plan = GetResourcePlan(request);
+    require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u, "chained table offset: a T# whose second load offset adds to the first one's was not planned as a table");
+    const auto& column = *plan->descriptorSources.at(plan->info.images[0].source).tableColumn;
+    require(column.address && column.stride == 32u && column.addend == 0x20u && column.offset == 0x40u && column.maxKey == 0xffu && column.foldedAddend == 0x20u && column.foldedCarry == 16u, "chained table offset: the column is wrong");
+    const auto compiled = CompileTable(request);
+    const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0);
+    const auto& words = compiled.capture->snapshot.tables.words;
+    require(snapshot.keys == 256u && (snapshot.codes[3] & Abi::PoisonFlag) == 0u && (snapshot.codes[5] & Abi::PoisonFlag) == 0u, "chained table offset: the records were not resolved");
+    require(words.at(snapshot.codes[3]).dwords[0] == root[(0x60u + 3u * 32u) / 4u] && words.at(snapshot.codes[3]).dwords[7] == root[(0x60u + 3u * 32u) / 4u + 7u] && words.at(snapshot.codes[5]).dwords[0] == root[(0x60u + 5u * 32u) / 4u] && snapshot.codes[3] != snapshot.codes[5], "chained table offset: a record was read from the wrong offset");
+}
+
 void verifyDescriptorPhis() {
     using namespace ShaderRecompiler;
     constexpr std::uint32_t Format8888UNorm = 56;
@@ -2049,15 +2416,15 @@ void verifyUniformTableImage() {
     request.layout.pushConstantSizeBytes = 128;
     const auto plan = GetResourcePlan(request);
     const auto& column = plan->descriptorSources.at(plan->info.images.at(0).source).tableColumn;
-    require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u && column.has_value() && column->stride == 48u && column->dwordCount == 8u, "uniform table image: the T# read from a constant buffer at a user data offset was not planned as one table column");
+    require(plan->info.images.size() == 1u && plan->info.images[0].table == 0u && column.has_value() && column->stride == 48u && column->dwordCount == 8u && column->selected, "uniform table image: the T# read from a constant buffer at a user data offset does not select its record");
     for (const auto& [key, expected] : {std::pair{2u, second}, std::pair{1u, first}}) {
         userData[2] = key;
         AgcDriver::ShaderMemory memory({});
         const auto capture = memory.Capture(request);
         const auto& tables = capture->snapshot.tables;
-        require(tables.tables.size() == 1u && key < tables.tables[0].codes.size(), "uniform table image: the capture has no table for the user data key");
-        const auto code = tables.tables[0].codes[key];
-        require((code & ImageTableAbi::PoisonFlag) == 0u && std::equal(expected.begin(), expected.end(), tables.words.at(code).dwords.begin()), "uniform table image: the T# of the record the user data selects was not read");
+        require(tables.tables.size() == 1u && TableSelected(tables, 0u, "uniform table image: a table does not hold one readable selected record") == expected, "uniform table image: the T# of the record the user data selects was not read");
+        const auto regions = memory.Regions();
+        require(TableCovered(regions, guest.table[key].data() + 4, sizeof(expected)) && !TableCovered(regions, guest.table[3u - key].data() + 4, sizeof(std::uint32_t)), "uniform table image: the record the user data does not select was read");
     }
 }
 
@@ -4182,6 +4549,15 @@ void verifyUnnormalizedSamplers() {
     reject(0xf11c0108u, userData(unnormalized), "unnormalized guest sampler is used by a gather, which is not implemented", "unnormalized samplers: image_gather4_lz was accepted");
     reject(0xf1800308u, userData(unnormalized), "unnormalized guest sampler is used by image_get_lod, which is not implemented", "unnormalized samplers: image_get_lod was accepted");
     reject(0xf0800f09u, userData(unnormalized), "unnormalized guest sampler is used by an image_sample_*_a variant, which is not implemented", "unnormalized samplers: image_sample_a was accepted");
+    const auto reservedDword3 = [&](std::uint32_t mimg) {
+        auto code = program(mimg);
+        const std::array<std::uint32_t, 4> computed{0x7d8a0080u, 0x870bff6au, 0x000000ffu, 0x8f0b8c0bu};
+        code.insert(code.begin(), computed.begin(), computed.end());
+        return code;
+    };
+    const auto adjusted = recompile(reservedDword3(0xf0800f09u), userData(normalized));
+    require(proven(adjusted, false), "sampler dword 3: an image_sample_a whose S# dword 3 holds only a ballot in its reserved bits was not compiled");
+    expectFailure([&] { static_cast<void>(recompile(reservedDword3(0xf0800f08u), userData(normalized))); }, "GetSamplerResource dword 3 is not a valid runtime value", "sampler dword 3: an image_sample whose S# dword 3 holds a ballot was accepted");
     reject(0xf09c0f18u, imageData(unnormalized, TypeCube, Format8888UNorm, 5u), "unnormalized guest sampler samples a 1D-array, 2D-array, 3D, cube or multisampled image", "unnormalized samplers: a cube T# was accepted");
     reject(0xf09c0f10u, imageData(unnormalized, Type3D, Format8888UNorm, 3u), "unnormalized guest sampler samples a 1D-array, 2D-array, 3D, cube or multisampled image", "unnormalized samplers: a 3D T# was accepted");
     reject(0xf09c0f28u, imageData(unnormalized, Type2DArray, Format8888UNorm, 5u), "unnormalized guest sampler samples a 1D-array, 2D-array, 3D, cube or multisampled image", "unnormalized samplers: a 2D-array sample was accepted");
@@ -5233,6 +5609,7 @@ int main(int argc, char** argv) {
             verifyImageTableKeys();
             verifyImageTables();
             verifyImageTableBulkReads();
+            verifyImageTableBulkKeyScan();
             verifyImageTableSrtKey();
             verifyImageTableMaskedOffset();
             verifyImageTableKeyUnion();
@@ -5242,6 +5619,12 @@ int main(int argc, char** argv) {
             verifyScalarAddressRule();
             verifyPointerImageTable();
             verifyPointerTableOffsetCarry();
+            verifySelectedRecordFallbacks();
+            verifySelectedRecordOffsetCarry();
+            verifySelectedRecordDirectOperations();
+            verifyEvaluableHeapSamplerDirect();
+            verifySelectedRecordUniformIndex();
+            verifyChainedTableOffset();
             std::cout << "Image table keys, maps, records and registration passed\n";
             return 0;
         }
@@ -5273,6 +5656,7 @@ int main(int argc, char** argv) {
         verifyScalarAddressRule();
         verifyPointerImageTable();
         verifyPointerTableOffsetCarry();
+        verifyChainedTableOffset();
         verifyDescriptorPhis();
         verifyUniformTableImage();
         verifyLaneSelectedDescriptors();

core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp

@@ -32,6 +32,7 @@ struct ElementAddress {
     std::uint32_t y;
     std::uint32_t z;
     std::uint64_t address;
+    std::uint32_t pipeBankXor = 0;
 };
 
 std::uint64_t equationOffset(const TextureSwizzleEquation& equation, std::uint32_t x, std::uint32_t y, std::uint32_t z) {
@@ -60,7 +61,7 @@ void requireThickAddresses(const GuestTextureResource& resource, std::uint32_t b
     const auto geometry = DescribeSurface(resource);
     Require(geometry.thick && geometry.mips.size() == resource.mipCount && geometry.guestBytes == guestBytes, what + ": guest size differs from addrlib");
     for (std::uint32_t level = 0; level < resource.mipCount; ++level) Require(geometry.mips[level].tail == (level >= firstTailLevel), what + ": mip " + std::to_string(level) + " is on the wrong side of the mip tail");
-    const auto blockBytes = resource.tileMode == TextureTileMode::kStandard4KB ? 4096u : 65536u;
+    const auto blockBytes = resource.tileMode == TextureTileMode::kStandard4KB || resource.tileMode == TextureTileMode::kS4KBX ? 4096u : 65536u;
     const auto* equation = FindTextureSwizzleEquation(0x100u | (resource.tileMode == TextureTileMode::kStandard4KB ? 5u : resource.tileMode == TextureTileMode::kStandard64KB ? 9u : XorSwizzleMode(resource.tileMode)), bytesPerElement);
     Require(equation != nullptr, what + ": missing thick swizzle equation");
     const auto block = ThickBlockExtent(resource.tileMode, bytesPerElement);
@@ -73,6 +74,7 @@ void requireThickAddresses(const GuestTextureResource& resource, std::uint32_t b
         } else {
             address += (static_cast<std::uint64_t>(element.y / block[1]) * mip.blocksPerRow + element.x / block[0]) * blockBytes + equationOffset(*equation, element.x, element.y, element.z);
         }
+        address ^= element.pipeBankXor;
         Require(address == element.address, what + ": mip " + std::to_string(element.mip) + " element (" + std::to_string(element.x) + ", " + std::to_string(element.y) + ", " + std::to_string(element.z) + ") detiles from " + std::to_string(address) + " instead of addrlib's " + std::to_string(element.address));
     }
 }
@@ -258,6 +260,22 @@ void RunTextureTilingTests() {
         Require(odd.mips[0].blocksPerRow == 5 && odd.mips[0].tiledOffset == 12288 && odd.mips[0].tiledSize == 40960, "3D mip 0 of a non-power-of-two volume changed");
     }
 
+    {
+        constexpr ElementAddress s4xTail1[] = {{0, 0, 0, 0, 0x3000, 0x0}, {0, 32, 19, 11, 0x843c, 0x0}, {0, 8, 9, 8, 0x3e08, 0x0}, {0, 15, 2, 8, 0x3163, 0x700}, {1, 0, 0, 0, 0x1000, 0x0}, {1, 15, 9, 5, 0x1acf, 0x0}, {1, 12, 4, 0, 0x1340, 0x0}, {1, 7, 2, 5, 0x17e7, 0x700}, {2, 0, 0, 0, 0x800, 0x0}, {2, 7, 4, 2, 0x953, 0x0}, {2, 2, 2, 0, 0x822, 0x0}, {2, 0, 1, 1, 0xf0c, 0x700}, {3, 0, 0, 0, 0x300, 0x0}, {3, 3, 1, 0, 0x30b, 0x0}, {3, 3, 1, 0, 0x30b, 0x0}, {3, 0, 0, 0, 0x400, 0x700}, {4, 0, 0, 0, 0x200, 0x0}, {4, 1, 0, 0, 0x201, 0x0}, {4, 0, 0, 0, 0x200, 0x0}, {4, 0, 0, 0, 0x500, 0x700}, {5, 0, 0, 0, 0x100, 0x0}, {5, 0, 0, 0, 0x100, 0x0}, {5, 0, 0, 0, 0x100, 0x0}, {5, 0, 0, 0, 0x600, 0x700}};
+        requireThickAddresses(thickVolume(TextureTileMode::kS4KBX, 1, 33, 20, 12, 6), 1, 2, 36864, s4xTail1, "SW_4KB_S_X 8 bpp 33x20x12, 6 levels");
+        constexpr ElementAddress s4xTail2[] = {{0, 0, 0, 0, 0x6000, 0x0}, {0, 32, 19, 11, 0xf13c, 0x0}, {0, 8, 9, 8, 0x7408, 0x0}, {0, 15, 2, 8, 0x7962, 0x700}, {1, 0, 0, 0, 0x3000, 0x0}, {1, 15, 9, 5, 0x42ce, 0x0}, {1, 12, 4, 0, 0x4b00, 0x0}, {1, 7, 2, 5, 0x35e6, 0x700}, {2, 0, 0, 0, 0x1000, 0x0}, {2, 7, 4, 2, 0x1352, 0x0}, {2, 2, 2, 0, 0x1060, 0x0}, {2, 0, 1, 1, 0x170c, 0x700}, {3, 0, 0, 0, 0x800, 0x0}, {3, 3, 1, 0, 0x84a, 0x0}, {3, 3, 1, 0, 0x84a, 0x0}, {3, 0, 0, 0, 0xf00, 0x700}, {4, 0, 0, 0, 0x300, 0x0}, {4, 1, 0, 0, 0x302, 0x0}, {4, 0, 0, 0, 0x300, 0x0}, {4, 0, 0, 0, 0x400, 0x700}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x500, 0x700}};
+        requireThickAddresses(thickVolume(TextureTileMode::kS4KBX, 12, 33, 20, 12, 6), 2, 3, 65536, s4xTail2, "SW_4KB_S_X 16 bpp 33x20x12, 6 levels");
+        constexpr ElementAddress s4xWide4[] = {{0, 0, 0, 0, 0x20000, 0x0}, {0, 127, 127, 7, 0x9f4fc, 0x0}, {0, 24, 93, 0, 0x73108, 0x0}, {0, 42, 22, 4, 0x35f60, 0x700}, {1, 0, 0, 0, 0x0, 0x0}, {1, 63, 63, 3, 0x1f0fc, 0x0}, {1, 28, 38, 2, 0x13da0, 0x0}, {1, 55, 52, 1, 0x1e754, 0x700}};
+        requireThickAddresses(thickVolume(TextureTileMode::kS4KBX, 56, 128, 128, 8, 2), 4, 2, 655360, s4xWide4, "SW_4KB_S_X 32 bpp 128x128x8, 2 levels");
+        constexpr ElementAddress s4xTail4[] = {{0, 0, 0, 0, 0x6000, 0x0}, {0, 32, 19, 11, 0x1fdb8, 0x0}, {0, 8, 9, 8, 0x17808, 0x0}, {0, 15, 2, 8, 0x17564, 0x700}, {1, 0, 0, 0, 0x3000, 0x0}, {1, 15, 9, 5, 0x465c, 0x0}, {1, 12, 4, 0, 0x4b00, 0x0}, {1, 7, 2, 5, 0x3174, 0x700}, {2, 0, 0, 0, 0x1000, 0x0}, {2, 7, 4, 2, 0x13c4, 0x0}, {2, 2, 2, 0, 0x1060, 0x0}, {2, 0, 1, 1, 0x1718, 0x700}, {3, 0, 0, 0, 0x800, 0x0}, {3, 3, 1, 0, 0x84c, 0x0}, {3, 3, 1, 0, 0x84c, 0x0}, {3, 0, 0, 0, 0xf00, 0x700}, {4, 0, 0, 0, 0x300, 0x0}, {4, 1, 0, 0, 0x304, 0x0}, {4, 0, 0, 0, 0x300, 0x0}, {4, 0, 0, 0, 0x400, 0x700}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x500, 0x700}};
+        requireThickAddresses(thickVolume(TextureTileMode::kS4KBX, 56, 33, 20, 12, 6), 4, 3, 131072, s4xTail4, "SW_4KB_S_X 32 bpp 33x20x12, 6 levels");
+        constexpr ElementAddress s4xTail8[] = {{0, 0, 0, 0, 0x9000, 0x0}, {0, 32, 19, 11, 0x2fab0, 0x0}, {0, 8, 9, 8, 0x27420, 0x0}, {0, 15, 2, 8, 0x22448, 0x700}, {1, 0, 0, 0, 0x3000, 0x0}, {1, 15, 9, 5, 0x7a78, 0x0}, {1, 12, 4, 0, 0x4200, 0x0}, {1, 7, 2, 5, 0x3058, 0x700}, {2, 0, 0, 0, 0x1000, 0x0}, {2, 7, 4, 2, 0x1ac8, 0x0}, {2, 2, 2, 0, 0x1140, 0x0}, {2, 0, 1, 1, 0x1730, 0x700}, {3, 0, 0, 0, 0x800, 0x0}, {3, 3, 1, 0, 0x868, 0x0}, {3, 3, 1, 0, 0x868, 0x0}, {3, 0, 0, 0, 0xf00, 0x700}, {4, 0, 0, 0, 0x300, 0x0}, {4, 1, 0, 0, 0x308, 0x0}, {4, 0, 0, 0, 0x300, 0x0}, {4, 0, 0, 0, 0x400, 0x700}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x200, 0x0}, {5, 0, 0, 0, 0x500, 0x700}};
+        requireThickAddresses(thickVolume(TextureTileMode::kS4KBX, 71, 33, 20, 12, 6), 8, 3, 196608, s4xTail8, "SW_4KB_S_X 64 bpp 33x20x12, 6 levels");
+        constexpr ElementAddress s4xTail16[] = {{0, 0, 0, 0, 0x10000, 0x0}, {0, 32, 19, 11, 0x55bb0, 0x0}, {0, 8, 9, 8, 0x46820, 0x0}, {0, 15, 2, 8, 0x3e040, 0x700}, {1, 0, 0, 0, 0x6000, 0x0}, {1, 15, 9, 5, 0xee70, 0x0}, {1, 12, 4, 0, 0x9400, 0x0}, {1, 7, 2, 5, 0x7850, 0x700}, {2, 0, 0, 0, 0x3000, 0x0}, {2, 7, 4, 2, 0x42c0, 0x0}, {2, 2, 2, 0, 0x3300, 0x0}, {2, 0, 1, 1, 0x3730, 0x700}, {3, 0, 0, 0, 0x1000, 0x0}, {3, 3, 1, 0, 0x1260, 0x0}, {3, 3, 1, 0, 0x1260, 0x0}, {3, 0, 0, 0, 0x1700, 0x700}, {4, 0, 0, 0, 0x800, 0x0}, {4, 1, 0, 0, 0x840, 0x0}, {4, 0, 0, 0, 0x800, 0x0}, {4, 0, 0, 0, 0xf00, 0x700}, {5, 0, 0, 0, 0x300, 0x0}, {5, 0, 0, 0, 0x300, 0x0}, {5, 0, 0, 0, 0x300, 0x0}, {5, 0, 0, 0, 0x400, 0x700}};
+        requireThickAddresses(thickVolume(TextureTileMode::kS4KBX, 77, 33, 20, 12, 6), 16, 4, 352256, s4xTail16, "SW_4KB_S_X 128 bpp 33x20x12, 6 levels");
+        requireBlockVolumeLayout(TextureTileMode::kS4KBX, 169, 71, 60, 36, 20, 5, "SW_4KB_S_X BC1 60x36x20, 5 levels");
+    }
+
     {
         constexpr ElementAddress chainInTail[] = {{0, 0, 0, 0, 0x800}, {0, 7, 7, 7, 0xffc}, {0, 4, 2, 4, 0xe20}, {0, 2, 7, 0, 0x968}, {0, 7, 0, 7, 0xed4}, {1, 0, 0, 0, 0x300}, {1, 3, 3, 3, 0x3fc}, {1, 2, 1, 2, 0x3c8}, {1, 1, 3, 0, 0x32c}, {1, 3, 0, 3, 0x3d4}};
         requireThickAddresses(thickVolume(TextureTileMode::kStandard4KB, 56, 8, 8, 8, 2), 4, 0, 4096, chainInTail, "SW_4KB_S 32 bpp 8x8x8, 2 levels");

core/libs/prx/libSceAgcDriver/tests/execution/AddressPasses.cpp

@@ -0,0 +1,331 @@
+#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <initializer_list>
+#include <iostream>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+
+constexpr std::uint32_t VertexBuffers = 3;
+constexpr std::size_t VertexTableOffset = 0x300;
+constexpr std::size_t VertexDataOffset = 0x800;
+constexpr std::size_t ColorOffset = 0;
+constexpr std::size_t ColorBytes = 65536;
+constexpr std::uint32_t RenderTargetTileMode = 0x1b;
+constexpr std::size_t BlockBytes = 65536;
+
+alignas(256) constexpr std::array<std::uint32_t, 40> VertexCode{
+    0x7e000d05u, 0x080200f4u, 0x10020300u, 0x100202f4u, 0x060202f3u, 0x060400f3u, 0x10040500u, 0x060404f3u,
+    0x7e060280u, 0x7e0802f2u,
+    0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u,
+    0xbf8cc07fu,
+    0xe00c2000u, 0x80010a05u, 0xe00c2000u, 0x80020e05u, 0xe00c2000u, 0x80031205u,
+    0xbf8c3f70u,
+    0x0602030au, 0x0604050bu, 0x0606070cu, 0x0608090du,
+    0x0602030eu, 0x0604050fu, 0x06060710u, 0x06080911u,
+    0x06020312u, 0x06040513u, 0x06060714u, 0x06080915u,
+    0xf80008cfu, 0x04030201u,
+    0xbf810000u, 0xbf810000u,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 40> VertexCodeUser8 = [] {
+    auto code = VertexCode;
+    for (const std::size_t load : {10u, 12u, 14u}) code[load] |= 4u;
+    return code;
+}();
+
+alignas(256) constexpr std::array<std::uint32_t, 11> PixelRead{0x7e1a0280u, 0xdc308000u, 0x0c02000du, 0xbf8c3f70u, 0x7e10030cu, 0x7e120280u, 0x7e140280u, 0x7e160280u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 9> PixelColorWord{0x7e100280u, 0x7e120280u, 0x7e1402ffu, 0x3f008081u, 0x7e1602ffu, 0x3e7cfcfdu, 0xf800180fu, 0x0b0a0908u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 11> PixelDescriptorStore{0x7e0202f2u, 0x7e040280u, 0xe0701000u, 0x80010102u, 0x7e1002f2u, 0x7e1202f2u, 0x7e1402f2u, 0x7e1602f2u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 11> PixelStore{0x7e1a0280u, 0x7e1802f2u, 0xdc708000u, 0x00020c0du, 0x7e1002f2u, 0x7e1202f2u, 0x7e1402f2u, 0x7e1602f2u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u};
+
+struct Header {
+    Shader shader{};
+    ShaderUserData userData{};
+};
+
+alignas(8) Header VertexHeader{};
+alignas(8) Header ReadHeader{};
+alignas(8) Header StoreHeader{};
+alignas(8) Header DescriptorStoreHeader{};
+alignas(8) Header ColorWordHeader{};
+
+void Register(Header& header, const std::uint32_t* code, std::size_t bytes, std::uint8_t type) {
+    header.shader.file_header = 0x34333231u;
+    header.shader.version = 0x18u;
+    header.shader.header_size = sizeof(Header);
+    header.shader.shader_size = static_cast<std::uint32_t>(bytes);
+    header.shader.code = code;
+    header.shader.type = type;
+    header.shader.user_data = &header.userData;
+    AgcDriverRegisterShader_nid_postfix(&header.shader);
+}
+
+class GuestBlock {
+public:
+    GuestBlock() {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(BlockBytes, 65536));
+        if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes);
+#else
+        constexpr std::uintptr_t alignment = 65536;
+        void* mapped = mmap(nullptr, BlockBytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        if (mapped != MAP_FAILED) {
+            const auto begin = reinterpret_cast<std::uintptr_t>(mapped);
+            const auto aligned = (begin + alignment - 1) & ~(alignment - 1);
+            if (aligned != begin) munmap(mapped, aligned - begin);
+            if (aligned + BlockBytes != begin + BlockBytes + alignment) munmap(reinterpret_cast<void*>(aligned + BlockBytes), begin + alignment - aligned);
+            block = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, BlockBytes);
+        }
+#endif
+        Require(block != nullptr, "address passes: cannot allocate a guest block");
+        std::memset(block, 0, BlockBytes);
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    void Map(bool writable) const { GuestAllocations::Mutation().Add(block, BlockBytes, true, writable, true); }
+    std::uint64_t At(std::size_t offset) const { return static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(block + offset)); }
+    std::uint8_t* Bytes(std::size_t offset) const { return block + offset; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(std::uint64_t address, std::uint32_t stride, std::uint32_t records, std::uint32_t word3) {
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), records, word3};
+}
+
+void FillVertices(const GuestBlock& block) {
+    for (std::uint32_t buffer = 0; buffer < VertexBuffers; ++buffer) {
+        const auto vertices = BufferDescriptor(block.At(VertexDataOffset + buffer * 0x100u), 16, 3, 0x0104dfacu);
+        std::memcpy(block.Bytes(VertexTableOffset + buffer * 16u), vertices.data(), sizeof(vertices));
+    }
+}
+
+void Append(std::vector<std::uint32_t>& commands, std::uint32_t opcode, std::initializer_list<std::uint32_t> payload) {
+    commands.push_back(0xc0000000u | (static_cast<std::uint32_t>(payload.size() - 1) << 16u) | (opcode << 8u));
+    commands.insert(commands.end(), payload);
+}
+
+struct Step {
+    const std::uint32_t* pixel;
+    std::uint64_t pointer;
+    std::array<std::uint32_t, 4> descriptor{};
+};
+
+void Submit(const GuestBlock& vertices, const GuestBlock& target, std::initializer_list<Step> steps, bool linear = false) {
+    const auto color = target.At(ColorOffset);
+    const auto vertex = reinterpret_cast<std::uintptr_t>(VertexCodeUser8.data());
+    const auto table = vertices.At(VertexTableOffset);
+    const float scale = 32.0f;
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> context{
+        {0x2d5, 0x2000},
+        {0x1b6, 0}, {0x207, 0}, {0x200, 0}, {0x203, 0x800},
+        {0x2dc, 0xaa00}, {0x2f8, 0}, {0x292, 2}, {0x293, 0},
+        {0x80, 0}, {0x8d, 0}, {0x83, 0xffff}, {0x8c, 0xa},
+        {0x2f9, 0x2d}, {0x313, 0x6000}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff},
+        {0x206, 0x43f}, {0x204, 0x80000}, {0x205, 0x240},
+        {0x8e, 0xf}, {0x8f, 0xf}, {0x202, 0xcc0010},
+        {0x1c4, 0}, {0x1c5, 9}, {0x1c3, 4}, {0x31c, 0x28028},
+        {0x31b, 0}, {0x31d, 0}, {0x3b0, (63u << 14u) | 3u},
+        {0x3b8, linear ? 0x9000000u : 0x9000000u | (RenderTargetTileMode << 14u)}, {0x1e0, 0},
+        {0xc, 0}, {0xd, 0x40040},
+        {0x81, 0x80000000}, {0x82, 0x40040},
+        {0x90, 0x80000000}, {0x91, 0x40040},
+        {0x94, 0x80000000}, {0x95, 0x40040},
+        {0x318, static_cast<std::uint32_t>(color >> 8u)}, {0x390, static_cast<std::uint32_t>(color >> 40u)},
+        {0x10f, std::bit_cast<std::uint32_t>(scale)}, {0x110, std::bit_cast<std::uint32_t>(scale)},
+        {0x111, std::bit_cast<std::uint32_t>(-2.0f)}, {0x112, std::bit_cast<std::uint32_t>(2.0f)},
+        {0x113, std::bit_cast<std::uint32_t>(1.0f)}, {0x114, 0},
+        {0xb4, 0}, {0xb5, std::bit_cast<std::uint32_t>(1.0f)},
+        {0x2fe, 0}, {0x302, 0}, {0x306, 0}, {0x30a, 0},
+    };
+    std::vector<std::uint32_t> commands;
+    Append(commands, 0x79, {0x242, 4});
+    for (const auto& [offset, value] : context) Append(commands, 0x69, {offset, value});
+    for (const auto& [offset, value] : std::initializer_list<std::pair<std::uint32_t, std::uint32_t>>{{0xc8, static_cast<std::uint32_t>(vertex >> 8u)}, {0xc9, static_cast<std::uint32_t>(vertex >> 40u)}, {0x8b, 2u << 1u}, {0x8c, static_cast<std::uint32_t>(table)}, {0x8d, static_cast<std::uint32_t>(table >> 32u)}, {0x00b, 8u << 1u}, {0x00c, 0}, {0x00d, 0}}) Append(commands, 0x76, {offset, value});
+    for (const auto& step : steps) {
+        const auto pixel = reinterpret_cast<std::uintptr_t>(step.pixel);
+        Append(commands, 0x76, {0x008, static_cast<std::uint32_t>(pixel >> 8u), static_cast<std::uint32_t>(pixel >> 40u)});
+        Append(commands, 0x76, {0x00e, static_cast<std::uint32_t>(step.pointer), static_cast<std::uint32_t>(step.pointer >> 32u)});
+        Append(commands, 0x76, {0x010, step.descriptor[0], step.descriptor[1], step.descriptor[2], step.descriptor[3]});
+        Append(commands, 0x2d, {3, 2});
+    }
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    Require(sceAgcDriverSubmitDcb(&packet) == 0, "address passes: the draw submission failed");
+    AgcDriverWaitIdle_nid_postfix();
+}
+
+AgcDriver::Graphics::DrawPassCounts AddressDraws() {
+    return AgcDriver::Graphics::DrawPassTotals()[1];
+}
+
+AgcDriver::Graphics::DrawPassCounts OtherDraws() {
+    return AgcDriver::Graphics::DrawPassTotals()[0];
+}
+
+std::array<std::uint32_t, 4> StoredDword(const GuestBlock& data, std::size_t offset) {
+    return BufferDescriptor(data.At(offset), 0, 4, 0x31016facu);
+}
+
+void Expect(bool condition, const std::string& what) {
+    if (!condition) throw std::runtime_error("address passes: " + what);
+}
+
+std::string Counts(const AgcDriver::Graphics::DrawPassCounts& before, const AgcDriver::Graphics::DrawPassCounts& after, const char* draws = "address-based draws") {
+    return std::to_string(after.draws - before.draws) + " " + draws + ", " + std::to_string(after.passesBegun - before.passesBegun) + " began a pass, " + std::to_string(after.passesContinued - before.passesContinued) + " continued one";
+}
+
+void ReadersShareAPass(const GuestBlock& vertices, const GuestBlock& data, const GuestBlock& target) {
+    const auto before = AddressDraws();
+    Submit(vertices, target, {{PixelRead.data(), data.At(0)}, {PixelRead.data(), data.At(0)}});
+    const auto after = AddressDraws();
+    std::printf("two draws reading through the table: %s\n", Counts(before, after).c_str());
+    Expect(after.draws - before.draws == 2 && after.passesContinued - before.passesContinued == 1, "a draw reading through the table after another did not continue its pass: " + Counts(before, after));
+}
+
+void StoreAfterTableReadBreaks(const GuestBlock& vertices, const GuestBlock& data, const GuestBlock& target) {
+    const auto before = AddressDraws();
+    Submit(vertices, target, {{PixelRead.data(), data.At(32)}, {PixelStore.data(), data.At(32)}});
+    const auto after = AddressDraws();
+    std::printf("a draw storing through the table after a draw reading through it: %s\n", Counts(before, after).c_str());
+    Expect(after.draws - before.draws == 2 && after.passesBegun - before.passesBegun == 2, "a draw storing through the table continued the pass of a draw that read through it: " + Counts(before, after));
+}
+
+void ReadSeesStore(const GuestBlock& vertices, const GuestBlock& data, const GuestBlock& target) {
+    std::memset(target.Bytes(ColorOffset), 0, ColorBytes);
+    std::memset(data.Bytes(48), 0, 4);
+    const auto before = AddressDraws();
+    Submit(vertices, target, {{PixelStore.data(), data.At(48)}, {PixelRead.data(), data.At(48)}});
+    const auto after = AddressDraws();
+    AgcDriver::GuestMemory::FlushGpuWrites(static_cast<std::uintptr_t>(target.At(ColorOffset)), ColorBytes);
+    float stored = 0;
+    std::memcpy(&stored, data.Bytes(48), sizeof(stored));
+    unsigned brightest = 0;
+    for (std::size_t i = 0; i < ColorBytes; ++i) brightest = std::max(brightest, static_cast<unsigned>(target.Bytes(ColorOffset)[i]));
+    std::printf("a draw reading through the table what the previous draw stored: stored %g, brightest color byte %u, %s\n", static_cast<double>(stored), brightest, Counts(before, after).c_str());
+    Expect(stored == 1.0f, "the store through the table did not reach guest memory");
+    Expect(brightest == 255u, "the read through the table did not see the previous draw's store");
+    Expect(after.passesBegun - before.passesBegun == 2, "a draw reading through the table continued the pass of a draw that stored through it: " + Counts(before, after));
+}
+
+void ReadSeesDescriptorStore(const GuestBlock& vertices, const GuestBlock& data, const GuestBlock& target) {
+    std::memset(target.Bytes(ColorOffset), 0, ColorBytes);
+    std::memset(data.Bytes(64), 0, 4);
+    const auto before = AddressDraws();
+    const auto otherBefore = OtherDraws();
+    Submit(vertices, target, {{PixelDescriptorStore.data(), 0, StoredDword(data, 64)}, {PixelRead.data(), data.At(64)}});
+    const auto after = AddressDraws();
+    const auto otherAfter = OtherDraws();
+    AgcDriver::GuestMemory::FlushGpuWrites(static_cast<std::uintptr_t>(target.At(ColorOffset)), ColorBytes);
+    float stored = 0;
+    std::memcpy(&stored, data.Bytes(64), sizeof(stored));
+    unsigned brightest = 0;
+    for (std::size_t i = 0; i < ColorBytes; ++i) brightest = std::max(brightest, static_cast<unsigned>(target.Bytes(ColorOffset)[i]));
+    std::printf("a draw reading through the table what the previous draw stored through a buffer descriptor: stored %g, brightest color byte %u, %s; the storing draw: %s\n", static_cast<double>(stored), brightest, Counts(before, after).c_str(), Counts(otherBefore, otherAfter, "other draws").c_str());
+    Expect(otherAfter.draws - otherBefore.draws == 1, "the draw storing through a buffer descriptor was taken for an address-based one: " + Counts(otherBefore, otherAfter, "other draws"));
+    Expect(stored == 1.0f, "the store through the buffer descriptor did not reach guest memory");
+    Expect(after.draws - before.draws == 1 && after.passesBegun - before.passesBegun == 1, "a draw reading through the table continued the pass of a draw that stored through a buffer descriptor: " + Counts(before, after));
+    Expect(brightest == 255u, "the read through the table did not see the previous draw's store through a buffer descriptor");
+}
+
+void DescriptorStoreAfterTableReadBreaks(const GuestBlock& vertices, const GuestBlock& data, const GuestBlock& target) {
+    const auto before = AddressDraws();
+    const auto otherBefore = OtherDraws();
+    Submit(vertices, target, {{PixelRead.data(), data.At(80)}, {PixelDescriptorStore.data(), 0, StoredDword(data, 80)}});
+    const auto after = AddressDraws();
+    const auto otherAfter = OtherDraws();
+    std::printf("a draw storing through a buffer descriptor after a draw reading through the table: %s; the storing draw: %s\n", Counts(before, after).c_str(), Counts(otherBefore, otherAfter, "other draws").c_str());
+    Expect(after.draws - before.draws == 1 && otherAfter.draws - otherBefore.draws == 1, "the reading and the storing draw were not one address-based and one other draw");
+    Expect(otherAfter.passesBegun - otherBefore.passesBegun == 1, "a draw storing through a buffer descriptor continued the pass of a draw that read through the table: " + Counts(otherBefore, otherAfter, "other draws"));
+}
+
+void TargetReadAsWhenSplit(const GuestBlock& vertices, const GuestBlock& target, bool linear) {
+    const bool split = std::getenv("APS5_NO_ADDRESS_READ_PASSES") != nullptr;
+    Submit(vertices, target, {{PixelRead.data(), target.At(ColorOffset)}}, linear);
+    std::memset(target.Bytes(ColorOffset), 0, ColorBytes);
+    const auto before = AddressDraws();
+    Submit(vertices, target, {{PixelColorWord.data(), 0}, {PixelRead.data(), target.At(ColorOffset)}}, linear);
+    const auto after = AddressDraws();
+    AgcDriver::GuestMemory::FlushGpuWrites(static_cast<std::uintptr_t>(target.At(ColorOffset)), ColorBytes);
+    const unsigned red = target.Bytes(ColorOffset)[0];
+    const auto what = std::string(linear ? "linear" : "tiled") + " target" + (split ? " with APS5_NO_ADDRESS_READ_PASSES=1" : "");
+    std::printf("%s: a draw reading through the table the target the previous draw rendered read %s (red byte %u), %s\n", what.c_str(), red == 255u ? "that draw's color" : "the bytes before it", red, Counts(before, after).c_str());
+    Expect(red == (linear ? 255u : 0u), what + ": a draw reading through the table the target the previous draw rendered read " + (red == 255u ? "that draw's color" : "the bytes before it") + "; whether or not the pass breaks between them, it reads " + (linear ? "that draw's color (stored back after the draw)" : "the bytes before it (no write-back between them)"));
+    if (!linear && !split) Expect(after.passesContinued - before.passesContinued == 1, what + ": the reading draw did not continue the rendering draw's pass, so the case does not test it: " + Counts(before, after));
+}
+
+}
+
+int main(int argc, char** argv) {
+    try {
+#ifdef _WIN32
+        _putenv_s("APS5_PROFILE_DRAW", "1");
+#else
+        setenv("APS5_PROFILE_DRAW", "1", 1);
+#endif
+        {
+            const auto device = OpenVulkanTestDevice();
+            if (!device) return VulkanTestSkipped;
+        }
+        Register(VertexHeader, VertexCodeUser8.data(), sizeof(VertexCodeUser8), 2);
+        Register(ReadHeader, PixelRead.data(), sizeof(PixelRead), 1);
+        Register(StoreHeader, PixelStore.data(), sizeof(PixelStore), 1);
+        Register(DescriptorStoreHeader, PixelDescriptorStore.data(), sizeof(PixelDescriptorStore), 1);
+        Register(ColorWordHeader, PixelColorWord.data(), sizeof(PixelColorWord), 1);
+        GuestBlock readOnlyVertices;
+        GuestBlock data;
+        GuestBlock target;
+        FillVertices(readOnlyVertices);
+        readOnlyVertices.Map(false);
+        data.Map(true);
+        target.Map(true);
+        const std::string only = argc > 1 ? argv[1] : "";
+        if (only.empty() || only == "--readers") ReadersShareAPass(readOnlyVertices, data, target);
+        if (only.empty() || only == "--store-after-read") StoreAfterTableReadBreaks(readOnlyVertices, data, target);
+        if (only.empty() || only == "--read-sees-store") ReadSeesStore(readOnlyVertices, data, target);
+        if (only.empty() || only == "--read-sees-descriptor-store") ReadSeesDescriptorStore(readOnlyVertices, data, target);
+        if (only.empty() || only == "--descriptor-store-after-read") DescriptorStoreAfterTableReadBreaks(readOnlyVertices, data, target);
+        if (only.empty() || only == "--targets") {
+            TargetReadAsWhenSplit(readOnlyVertices, target, false);
+            TargetReadAsWhenSplit(readOnlyVertices, target, true);
+        }
+        AgcDriverShutdown_nid_postfix();
+        std::puts("address passes: draws reading through the BDA table share a render pass, and a store through it still begins one");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        try { AgcDriverShutdown_nid_postfix(); } catch (...) {}
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/AlphaToCoverage.cpp

@@ -514,6 +514,27 @@ void DccFastClearTests(AgcDriver::VulkanDevice& device, std::uint32_t selectedSa
                             }
                         }
                     }
+                    if (code == 0 && !alphaOnMsb && cmaskMode == 0) {
+                        const auto dccBytes = AgcDriver::Graphics::MultisampledDccBytes(target);
+                        Require(dccBytes + 8u <= metadata.bytes, "the DCC fixture has no adjacent-page control");
+                        const std::array<std::byte, 4> adjacent{std::byte{0x7b}, std::byte{0x7b}, std::byte{0x7b}, std::byte{0x7b}};
+                        AgcDriver::GuestMemory::Write(metadata.Address() + dccBytes, adjacent);
+                        device.ColorMetadataPass(pass);
+                        Require(ReadBack(device, resolved) == stored, "an adjacent driver store changed the resident DCC target");
+                        static_cast<volatile std::uint8_t*>(metadata.data)[dccBytes + 4u] = 0x7c;
+                        AgcDriver::GuestMemory::CollectWritesUncached(metadata.Address() + dccBytes, 8u);
+                        device.ColorMetadataPass(pass);
+                        Require(ReadBack(device, resolved) == stored, "an adjacent CPU store changed the resident DCC target");
+                        const std::array<std::byte, 1> same{std::byte{0}};
+                        AgcDriver::GuestMemory::Write(metadata.Address(), same);
+                        bool refused = false;
+                        try {
+                            device.ColorMetadataPass(pass);
+                        } catch (const std::runtime_error& error) {
+                            refused = std::string_view(error.what()).find("writes to a resident multisampled DCC") != std::string_view::npos;
+                        }
+                        Require(refused, "a same-value driver DCC store did not invalidate multisampled DCC residency");
+                    }
                     if (code == 0 && alphaOnMsb && cmaskMode == 0) {
                         static_cast<volatile std::uint8_t*>(source.data)[0] = 0;
                         bool refused = false;

core/libs/prx/libSceAgcDriver/tests/execution/BufferUnalignedBase.cpp

@@ -42,7 +42,10 @@ alignas(256) constexpr std::array<std::uint32_t, 11> PairCode{
     0xe0741000, 0x80020302, 0xbf810000,
 };
 
-alignas(256) constexpr std::array<std::uint32_t, 3> AtomicCode{0xe0c81000, 0x80000100, 0xbf810000};
+alignas(256) constexpr std::array<std::uint32_t, 10> AtomicCode{
+    0x34040082, 0x4a040408, 0x4a020081, 0xe0c85000, 0x80000102, 0xbf8c3f70, 0x34060082, 0xe0701000,
+    0x80010103, 0xbf810000,
+};
 
 constexpr std::array<std::uint32_t, 7> FormattedLoadCode(std::uint32_t format, std::uint32_t offset) {
     return {0x34020082, 0xe8001000u | (format << 19u) | offset, 0x80000201, 0xbf8c0000, 0xe0701000, 0x80010201, 0xbf810000};
@@ -218,20 +221,32 @@ void RunFormattedStore(AgcDriver::VulkanDevice& device, std::uint32_t viewOffset
     }
 }
 
-void RunAtomic(AgcDriver::VulkanDevice& device) {
+void DispatchAtomic(AgcDriver::VulkanDevice& device, std::uint32_t misalignment, std::uint32_t addend) {
+    Output.fill(Fill);
     std::vector<std::uint32_t> userData;
-    const auto descriptor = BufferDescriptor(reinterpret_cast<std::uint8_t*>(Output.data()) + 2u, 64u);
-    userData.insert(userData.end(), descriptor.begin(), descriptor.end());
-    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(AtomicCode.data()), std::as_bytes(std::span(AtomicCode))}}};
-    try {
-        const auto result = ShaderRecompiler::Recompile(Request(AtomicCode, userData, device.Target(), memory));
-        device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(AtomicCode.data()));
-        device.WaitIdle();
-    } catch (const std::exception& error) {
-        Require(std::string(error.what()).find("buffer atomic on a V# whose base is not DWORD aligned") != std::string::npos, std::string("buffer unaligned base atomic: unexpected error: ") + error.what());
-        return;
+    for (const auto& descriptor : {BufferDescriptor(reinterpret_cast<std::uint8_t*>(Output.data()) + misalignment, (Lanes + 2u) * 4u), BufferDescriptor(Output.data() + 128, Lanes * 4u)}) {
+        userData.insert(userData.end(), descriptor.begin(), descriptor.end());
     }
-    throw std::runtime_error("buffer unaligned base atomic: an atomic on a V# off a DWORD boundary ran");
+    userData.push_back(addend);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(AtomicCode.data()), std::as_bytes(std::span(AtomicCode))}}};
+    const auto result = ShaderRecompiler::Recompile(Request(AtomicCode, userData, device.Target(), memory));
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(AtomicCode.data()));
+    device.WaitIdle();
+}
+
+void RunAtomic(AgcDriver::VulkanDevice& device, std::uint32_t misalignment) {
+    DispatchAtomic(device, misalignment, 4u - misalignment);
+    const auto run = "buffer unaligned base atomic, view +" + std::to_string(misalignment) + ", offset lane * 4 + " + std::to_string(4u - misalignment);
+    for (std::uint32_t word = 0; word < 128u; ++word) {
+        const auto expected = word >= 1u && word <= Lanes ? Fill + word : Fill;
+        Require(Output[word] == expected, run + ": dword " + std::to_string(word) + " is " + Hex(Output[word]) + ", expected " + Hex(expected));
+    }
+    for (std::uint32_t lane = 0; lane < Lanes; ++lane) Require(Output[128u + lane] == Fill, run + ": lane " + std::to_string(lane) + " returned " + Hex(Output[128u + lane]) + ", expected " + Hex(Fill));
+}
+
+void RunAtomicFault(AgcDriver::VulkanDevice& device) {
+    DispatchAtomic(device, 2u, 0u);
+    for (std::uint32_t word = 0; word < Output.size(); ++word) Require(Output[word] == Fill, "buffer unaligned base atomic, view +2, offset lane * 4: dword " + std::to_string(word) + " is " + Hex(Output[word]) + ", but an atomic off a DWORD boundary faults the wave on RDNA2");
 }
 
 }
@@ -246,7 +261,7 @@ int main() {
         RunPair(*device, 1);
         RunPair(*device, 2);
         RunPair(*device, 3);
-        RunAtomic(*device);
+        for (const auto view : {1u, 2u, 3u}) RunAtomic(*device, view);
         RunFormattedLoad(*device, 2);
         RunFormattedStore(*device, 2);
         for (const auto view : {1u, 3u}) {
@@ -257,6 +272,7 @@ int main() {
             RunFormattedFault(*device, "tbuffer_load_format_x 32_UINT", Load32Code, view);
             RunFormattedFault(*device, "tbuffer_load_format_x 16_16_UINT", Load16_16Code, view);
         }
+        RunAtomicFault(*device);
         std::puts("buffer unaligned base tests passed");
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/execution/Color565Target.cpp

@@ -0,0 +1,307 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "Optimization/ResourceProgram.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <array>
+#include <bit>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::ColorTargetLayout;
+using AgcDriver::Graphics::ColorTileMode;
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 256;
+constexpr std::uint32_t Height = 128;
+constexpr std::uint32_t ScissorWidth = 128;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::uint32_t Color565StandardInfo = 0x8040u;
+constexpr std::uint32_t Color565ReversedInfo = 0x9040u;
+constexpr std::uint32_t TiledAttrib3 = 0x4dc6c000u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t Float32Export = 9;
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::array<std::uint32_t, 64>, 16> Programs{};
+std::size_t programCount = 0;
+
+using Color = std::array<float, 4>;
+
+std::uint16_t Texel(const Color& color, bool reversed) {
+    const auto unorm = [](float value, std::uint32_t maximum) { return static_cast<std::uint32_t>(std::lround(std::fmin(std::fmax(static_cast<double>(value), 0.0), 1.0) * maximum)); };
+    const auto low = unorm(color[reversed ? 2 : 0], 31);
+    const auto high = unorm(color[reversed ? 0 : 2], 31);
+    return static_cast<std::uint16_t>(low | (unorm(color[1], 63) << 5u) | (high << 11u));
+}
+
+std::span<const std::uint32_t> Program(const Color& color) {
+    Require(programCount < Programs.size(), "too many test programs");
+    auto& code = Programs[programCount++];
+    std::size_t at = 0;
+    for (std::uint32_t channel = 0; channel < 4; ++channel) {
+        code[at++] = 0x7e0002ffu | ((4u + channel) << 17u);
+        code[at++] = std::bit_cast<std::uint32_t>(color[channel]);
+    }
+    code[at++] = 0xf800180fu;
+    code[at++] = 0x07060504u;
+    code[at++] = 0xbf810000u;
+    return std::span<const std::uint32_t>(code.data(), at);
+}
+
+struct Block {
+    explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (watched) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        if (watched) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map the color target");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "cannot allocate the color target");
+        Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is not write-watched");
+        GuestAllocations::Mutation mutation;
+        mutation.Add(data, bytes, true, true, true);
+    }
+    ~Block() {
+        AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release");
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Remove(data);
+        }
+#ifdef _WIN32
+        if (watched) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+#else
+        if (watched) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Block(const Block&) = delete;
+    Block& operator=(const Block&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    std::size_t bytes;
+    bool watched;
+    std::uint8_t* data = nullptr;
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block, std::uint32_t attrib3, std::uint32_t info) {
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(block.Address() >> 8u);
+    cx[0x31b] = 0;
+    cx[0x31c] = info;
+    cx[0x31d] = 0;
+    cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u);
+    cx[0x3b8] = attrib3;
+    cx[0x390] = static_cast<std::uint32_t>(block.Address() >> 40u);
+    auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    Require(color.format == VK_FORMAT_R5G6B5_UNORM_PACK16 && color.elementBytes == 2 && color.bytes == block.bytes, "the 5_6_5 unorm target did not decode as an R5G6B5 surface");
+    return color;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::uint32_t exportFormat, std::span<const std::uint32_t> pixelCode) {
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto address = reinterpret_cast<std::uintptr_t>(Triangle.data());
+    const std::array<std::uint32_t, 4> vertexBuffer{static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (16u << 16u), static_cast<std::uint32_t>(Triangle.size()), 0x01016facu};
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = {{0, 0}, {ScissorWidth, Height}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.colorWriteMask = 0xf;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+
+    const auto codeAddress = reinterpret_cast<std::uintptr_t>(pixelCode.data());
+    const AgcDriver::Registers context{{0x1b6, 0x8000}, {0x1b3, 0}, {0x1b4, 0}, {0x203, 0}, {0x1c5, exportFormat}};
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{codeAddress, std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, codeAddress, pixelCode, 0, {}},
+        {waveSize, 0, {}, std::nullopt, AgcDriver::Graphics::DecodePixelStageInfo(context, {}), std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    const auto handle = ShaderRecompiler::PrepareShader(fragment);
+    AgcDriver::DriverDetail::ShaderSnapshot snapshot{codeAddress, 0, 1, {pixelCode.begin(), pixelCode.end()}, {}};
+    snapshot.prepared->entries.push_back({0, handle});
+    fragment.shader.code = snapshot.code;
+    fragment.context.pixel = AgcDriver::Graphics::DecodePixelStageInfo(context, AgcDriver::Graphics::ExportMappings(state));
+    const ShaderRecompiler::SrtRuntime runtime{};
+    const auto materialize = [&](const ShaderRecompiler::RecompileRequest& request) {
+        const auto invocation = AgcDriver::DriverDetail::InvocationFor(snapshot, 0, request);
+        return invocation.Materialize(*invocation.Capture(runtime));
+    };
+    const auto pixelResult = materialize(fragment);
+    Require(pixelResult->variantId == ShaderRecompiler::GetPreparedArtifact(*handle).variantId, "the draw did not use the registered pixel shader");
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, pixelResult.get(), PixelPushOffset(vertexPush, target)}
+    }};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+}
+
+std::vector<std::uint16_t> ReadBack(AgcDriver::VulkanDevice& device, const Block& block) {
+    AgcDriver::Graphics::StorageTexture::FlushPending(block.Address(), block.bytes, nullptr, "test read-back");
+    device.WaitIdle();
+    std::vector<std::uint16_t> stored(block.bytes / 2u);
+    AgcDriver::GuestMemory::Read(block.Address(), std::as_writable_bytes(std::span(stored)), 1);
+    return stored;
+}
+
+std::vector<std::uint16_t> Fill(Block& block) {
+    std::vector<std::uint16_t> halves(block.bytes / 2u);
+    for (std::size_t index = 0; index < halves.size(); ++index) halves[index] = static_cast<std::uint16_t>((index * 0x9e3779b9u) >> 16u);
+    std::memcpy(block.data, halves.data(), block.bytes);
+    return halves;
+}
+
+void ExpectSurface(AgcDriver::VulkanDevice& device, const Block& block, const ColorTargetLayout& layout, const std::vector<std::uint16_t>& before, std::uint16_t texel, const std::string& what) {
+    const auto stored = ReadBack(device, block);
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            const auto index = layout.Offset(x, y) / 2u;
+            const bool inside = x < ScissorWidth;
+            const auto expected = inside ? texel : before[index];
+            if (stored[index] == expected) continue;
+            char detail[160];
+            std::snprintf(detail, sizeof(detail), ": pixel (%u, %u) %s the scissor holds 0x%04x, expected 0x%04x", x, y, inside ? "inside" : "outside", stored[index], expected);
+            throw std::runtime_error(what + detail);
+        }
+    }
+}
+
+void TargetTests(AgcDriver::VulkanDevice& device, Block& block, std::uint32_t attrib3, ColorTileMode mode, bool reversed, const char* kind) {
+    const ColorTargetLayout layout(Width, Height, mode, 2);
+    Require(layout.Bytes() == block.bytes, "the test surface size disagrees with its layout");
+    const auto color = DecodeTarget(block, attrib3, reversed ? Color565ReversedInfo : Color565StandardInfo);
+    auto before = Fill(block);
+    struct Case {
+        Color color;
+        std::uint16_t reversedTexel;
+        std::uint16_t standardTexel;
+    };
+    const std::array<Case, 11> cases{{
+        {{1.0f, 0.0f, 0.0f, 1.0f}, 0xf800, 0x001f},
+        {{0.0f, 1.0f, 0.0f, 0.0f}, 0x07e0, 0x07e0},
+        {{0.0f, 0.0f, 1.0f, 0.0f}, 0x001f, 0xf800},
+        {{1.0f, 0.0f, 1.0f, 0.5f}, 0xf81f, 0xf81f},
+        {{1.0f, 1.0f, 1.0f, 1.0f}, 0xffff, 0xffff},
+        {{0.0f, 0.0f, 0.0f, 1.0f}, 0x0000, 0x0000},
+        {{3.0f / 31.0f, 40.0f / 63.0f, 21.0f / 31.0f, 1.0f}, (3u << 11u) | (40u << 5u) | 21u, 3u | (40u << 5u) | (21u << 11u)},
+        {{21.0f / 31.0f, 3.0f / 63.0f, 10.0f / 31.0f, 0.0f}, (21u << 11u) | (3u << 5u) | 10u, 21u | (3u << 5u) | (10u << 11u)},
+        {{1.0f / 31.0f, 62.0f / 63.0f, 30.0f / 31.0f, 0.75f}, (1u << 11u) | (62u << 5u) | 30u, 1u | (62u << 5u) | (30u << 11u)},
+        {{30.0f / 31.0f, 1.0f / 63.0f, 1.0f / 31.0f, 0.25f}, (30u << 11u) | (1u << 5u) | 1u, 30u | (1u << 5u) | (1u << 11u)},
+        {{2.0f, -1.0f, 10.0f / 31.0f, 0.25f}, (31u << 11u) | 10u, 31u | (10u << 11u)},
+    }};
+    for (const auto& test : cases) {
+        const auto& value = test.color;
+        const auto texel = reversed ? test.reversedTexel : test.standardTexel;
+        Require(Texel(value, reversed) == texel, "the test's expected texel disagrees with the 5_6_5 field layout");
+        Draw(device, color, Float32Export, Program(value));
+        char what[192];
+        std::snprintf(what, sizeof(what), "a 32_ABGR export of (%.9g, %.9g, %.9g, %.9g) into a %s 5_6_5 unorm target with the %s component order", value[0], value[1], value[2], value[3], kind, reversed ? "reversed" : "standard");
+        ExpectSurface(device, block, layout, before, texel, what);
+        before = ReadBack(device, block);
+        programCount = 0;
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Block tiled(ColorTargetLayout(Width, Height, ColorTileMode::RenderTarget, 2).Bytes());
+        Block linear(ColorTargetLayout(Width, Height, ColorTileMode::Linear, 2).Bytes());
+        for (const bool reversed : {true, false}) {
+            TargetTests(*device, tiled, TiledAttrib3, ColorTileMode::RenderTarget, reversed, "tiled");
+            TargetTests(*device, linear, LinearAttrib3, ColorTileMode::Linear, reversed, "linear");
+        }
+        std::puts("5_6_5 unorm color target tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/DccKeyPresync.cpp

@@ -0,0 +1,283 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <mutex>
+#include <optional>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Side = 128;
+constexpr std::uint32_t Format8888Unorm = 56;
+constexpr std::uint32_t TileRenderTarget64KB = 0x1b;
+constexpr std::uint32_t Type2D = 9;
+constexpr std::size_t SurfaceBytes = 65536;
+constexpr std::size_t KeyBytes = SurfaceBytes / 256;
+constexpr std::size_t KeysOffset = SurfaceBytes;
+constexpr std::size_t SourceKeysOffset = SurfaceBytes + 4096;
+constexpr std::size_t BlockBytes = 2 * 65536;
+
+alignas(256) constexpr std::array<std::uint32_t, 7> CopyKeysCode{
+    0x34020083, 0xe0341000, 0x80000201, 0xbf8c3f70, 0xe0741000, 0x80010201, 0xbf810000,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 11> SampleSecondTexelCode{
+    0x34060084, 0x7e2802ff, 0x3c400000, 0x7e2a02ff, 0x3b800000, 0xf09c0f08, 0x00610a14, 0xbf8c3f70,
+    0xe0781000, 0x80000a03, 0xbf810000,
+};
+
+constexpr std::array<std::uint32_t, 4> PointSampler{0x00000092u, 0x00fff000u, 0u, 0u};
+
+std::uint64_t AddressOf(const void* data) {
+    return static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(data));
+}
+
+class GuestBlock {
+public:
+    GuestBlock() {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint8_t*>(std::aligned_alloc(65536, BlockBytes));
+#endif
+        Require(block != nullptr, "DCC key pre-sync: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+#ifdef _WIN32
+        VirtualFree(block, 0, MEM_RELEASE);
+#else
+        std::free(block);
+#endif
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* Data() { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+std::array<std::uint32_t, 8> TextureDescriptor(const std::uint8_t* texels, const std::uint8_t* keys) {
+    const auto address = AddressOf(texels);
+    const auto meta = AddressOf(keys) >> 8u;
+    return {
+        static_cast<std::uint32_t>(address >> 8u),
+        static_cast<std::uint32_t>((address >> 40u) & 0xffu) | (Format8888Unorm << 20u) | (((Side - 1u) & 3u) << 30u),
+        ((Side - 1u) >> 2u) | ((Side - 1u) << 14u),
+        0xfacu | (TileRenderTarget64KB << 20u) | (Type2D << 28u),
+        0u,
+        0u,
+        (1u << 20u) | (1u << 21u) | (static_cast<std::uint32_t>(meta & 0xffu) << 24u),
+        static_cast<std::uint32_t>(meta >> 8u),
+    };
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = AddressOf(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData) {
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    return ShaderRecompiler::Recompile(request);
+}
+
+struct Scenario {
+    std::uint8_t* texels;
+    std::uint8_t* keys;
+    std::uint8_t* source;
+    std::uint32_t* output;
+    std::vector<std::uint32_t> copyData;
+    std::vector<std::uint32_t> sampleData;
+};
+
+Scenario Prepare(std::uint8_t* block, std::uint8_t* outputBlock) {
+    Scenario scenario{block, block + KeysOffset, block + SourceKeysOffset, reinterpret_cast<std::uint32_t*>(outputBlock), std::vector<std::uint32_t>(16, 0u), std::vector<std::uint32_t>(24, 0u)};
+    const auto source = BufferDescriptor(scenario.source, KeyBytes);
+    const auto destination = BufferDescriptor(scenario.keys, KeyBytes);
+    std::copy(source.begin(), source.end(), scenario.copyData.begin());
+    std::copy(destination.begin(), destination.end(), scenario.copyData.begin() + 4);
+    const auto results = BufferDescriptor(scenario.output, Threads * 16u);
+    const auto texture = TextureDescriptor(scenario.texels, scenario.keys);
+    std::copy(results.begin(), results.end(), scenario.sampleData.begin());
+    std::copy(texture.begin(), texture.end(), scenario.sampleData.begin() + 4);
+    std::copy(PointSampler.begin(), PointSampler.end(), scenario.sampleData.begin() + 12);
+    return scenario;
+}
+
+void Reset(const Scenario& scenario) {
+    std::memset(scenario.texels, 0x55, SurfaceBytes);
+    std::memset(scenario.keys, 0xff, KeyBytes);
+    std::memset(scenario.source, 0x00, KeyBytes);
+    std::fill(scenario.output, scenario.output + Threads * 4u, 0xdeadbeefu);
+    AgcDriver::GuestMemory::BumpCollectEpoch();
+}
+
+void RecordKeyCopy(AgcDriver::VulkanDevice& device, const Scenario& scenario) {
+    const auto copy = Compile(device, CopyKeysCode, scenario.copyData);
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    device.Dispatch(copy, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(CopyKeysCode.data()));
+    Require(AgcDriver::Graphics::DccKeyWritersPending(AddressOf(scenario.keys), KeyBytes), "a recorded kernel that copies data over the keys is not seen as a pending key writer");
+}
+
+void RequireSampledClear(const Scenario& scenario, const char* what) {
+    for (std::uint32_t index = 0; index < Threads * 4u; ++index) {
+        char message[256];
+        std::snprintf(message, sizeof(message), "%s: component %u of thread %u is 0x%08x, expected 0 (keys stored 0000 by the kernel)", what, index % 4u, index / 4u, scenario.output[index]);
+        Require(scenario.output[index] == 0u, message);
+    }
+}
+
+void RunLockedSync(AgcDriver::VulkanDevice& device, const Scenario& scenario) {
+    Reset(scenario);
+    RecordKeyCopy(device, scenario);
+    const auto sample = Compile(device, SampleSecondTexelCode, scenario.sampleData);
+    const auto waits = AgcDriver::Graphics::KeyWaitCounts();
+    {
+        std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+        device.Dispatch(sample, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(SampleSecondTexelCode.data()));
+        device.WaitIdle();
+    }
+    const auto after = AgcDriver::Graphics::KeyWaitCounts();
+    Require(after.synced > waits.synced, "a dispatch built under the GPU mutex did not wait for the kernel that copies data over the keys");
+    RequireSampledClear(scenario, "a dispatch built under the GPU mutex");
+}
+
+void RunPresync(AgcDriver::VulkanDevice& device, const Scenario& scenario) {
+    Reset(scenario);
+    RecordKeyCopy(device, scenario);
+    const auto sample = Compile(device, SampleSecondTexelCode, scenario.sampleData);
+    const auto waits = AgcDriver::Graphics::KeyWaitCounts();
+    const auto presyncs = AgcDriver::VulkanDevice::PresyncCounts();
+    auto prepared = device.PrepareDispatch(sample, {}, Threads, reinterpret_cast<std::uintptr_t>(SampleSecondTexelCode.data()));
+    const auto stageA = AgcDriver::VulkanDevice::PresyncCounts();
+    Require(stageA.keyChecks > presyncs.keyChecks, "stage A did not list the DCC keys of the sampled texture for the pre-sync");
+    Require(stageA.keyWaits == presyncs.keyWaits + 1 && stageA.presyncs == presyncs.presyncs + 1, "stage A did not wait for the kernel that copies data over the keys before the GPU mutex");
+    {
+        std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+        device.Dispatch(sample, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(SampleSecondTexelCode.data()), std::move(prepared));
+        device.WaitIdle();
+    }
+    const auto after = AgcDriver::Graphics::KeyWaitCounts();
+    Require(after.synced == waits.synced, "the dispatch waited for the key writer under the GPU mutex after the pre-sync waited for it");
+    RequireSampledClear(scenario, "a dispatch whose keys were pre-synced");
+}
+
+bool CanFillKeys(AgcDriver::VulkanDevice& device, const Scenario& scenario) {
+    const std::array<std::uint32_t, 4> pattern{0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu};
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    const bool filled = device.FillBuffer(AddressOf(scenario.keys), KeyBytes, pattern);
+    device.WaitIdle();
+    return filled;
+}
+
+void RunKnownKeys(AgcDriver::VulkanDevice& device, const Scenario& scenario) {
+    Reset(scenario);
+    const std::array<std::uint32_t, 4> pattern{0u, 0u, 0u, 0u};
+    {
+        std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+        Require(device.FillBuffer(AddressOf(scenario.keys), KeyBytes, pattern), "the fill of the keys was not recorded");
+        Require(AgcDriver::Graphics::DccKeyWritersPending(AddressOf(scenario.keys), KeyBytes), "a recorded fill of the keys is not seen as a pending key writer");
+        Require(!AgcDriver::Graphics::DccKeyWriterNeedsSync(AddressOf(scenario.keys), KeyBytes), "the keys of a recorded constant fill need a synchronization");
+    }
+    const auto sample = Compile(device, SampleSecondTexelCode, scenario.sampleData);
+    const auto waits = AgcDriver::Graphics::KeyWaitCounts();
+    const auto before = AgcDriver::VulkanDevice::PresyncCounts();
+    auto prepared = device.PrepareDispatch(sample, {}, Threads, reinterpret_cast<std::uintptr_t>(SampleSecondTexelCode.data()));
+    const auto after = AgcDriver::VulkanDevice::PresyncCounts();
+    Require(after.keyChecks > before.keyChecks, "stage A did not list the DCC keys of the sampled texture for the pre-sync");
+    Require(after.presyncs == before.presyncs, "stage A waited for a fill whose keys the locked lookup derives from the pending stores");
+    {
+        std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+        device.Dispatch(sample, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(SampleSecondTexelCode.data()), std::move(prepared));
+        device.WaitIdle();
+    }
+    Require(AgcDriver::Graphics::KeyWaitCounts().synced == waits.synced, "the dispatch synchronized with the GPU although the pending fill gives the keys");
+    RequireSampledClear(scenario, "a dispatch behind a pending fill of the keys");
+}
+
+void RunNoWriter(AgcDriver::VulkanDevice& device, const Scenario& scenario) {
+    const auto sample = Compile(device, SampleSecondTexelCode, scenario.sampleData);
+    const auto before = AgcDriver::VulkanDevice::PresyncCounts();
+    Require(!AgcDriver::Graphics::DccKeyWritersPending(AddressOf(scenario.keys), KeyBytes), "the idle device still reports a pending key writer");
+    auto prepared = device.PrepareDispatch(sample, {}, Threads, reinterpret_cast<std::uintptr_t>(SampleSecondTexelCode.data()));
+    const auto after = AgcDriver::VulkanDevice::PresyncCounts();
+    Require(after.keyChecks > before.keyChecks, "stage A did not list the DCC keys of the sampled texture for the pre-sync");
+    Require(after.keyWaits == before.keyWaits && after.presyncs == before.presyncs, "stage A waited although no recorded work writes the keys");
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    device.Dispatch(sample, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(SampleSecondTexelCode.data()), std::move(prepared));
+    device.WaitIdle();
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (!device->CanWaitUnlocked()) {
+            std::puts("skipped, the device has no timeline semaphores: the pre-sync is not tested");
+            return VulkanTestSkipped;
+        }
+        GuestBlock locked;
+        GuestBlock lockedOutput;
+        GuestBlock presynced;
+        GuestBlock presyncedOutput;
+        GuestBlock known;
+        GuestBlock knownOutput;
+        const auto control = Prepare(locked.Data(), lockedOutput.Data());
+        const auto changed = Prepare(presynced.Data(), presyncedOutput.Data());
+        const auto fills = Prepare(known.Data(), knownOutput.Data());
+        if (!CanFillKeys(*device, control)) {
+            std::puts("skipped, host imports are unavailable: the key writers of the test are copied writers");
+            return VulkanTestSkipped;
+        }
+        RunLockedSync(*device, control);
+        RunPresync(*device, changed);
+        RunNoWriter(*device, changed);
+        RunKnownKeys(*device, fills);
+        std::puts("DCC key pre-sync tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/DepthFastClearHarness.hpp

@@ -48,6 +48,8 @@ struct DrawOptions {
     VkCompareOp depthCompare = VK_COMPARE_OP_LESS;
     bool stencilTest = false;
     VkStencilOpState stencil{};
+    float minDepth = 0.0f;
+    float maxDepth = 1.0f;
 };
 
 inline std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
@@ -126,7 +128,7 @@ inline std::uint32_t Draw(AgcDriver::VulkanDevice& device, const Shaders& compil
     state.stencilBack = options.stencil;
     state.renderExtent = {Width, Height};
     state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
-    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), options.minDepth, options.maxDepth};
     state.negativeOneToOne = false;
     state.scissor = {{0, 0}, {Width, Height}};
     state.cullMode = VK_CULL_MODE_NONE;

core/libs/prx/libSceAgcDriver/tests/execution/DepthPlaneFill.cpp

@@ -0,0 +1,100 @@
+#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
+#include "DepthFastClearHarness.hpp"
+#include "VulkanTestDevice.hpp"
+#include <array>
+#include <cstdio>
+#include <iostream>
+
+namespace {
+
+using DepthFastClearHarness::Compile;
+using DepthFastClearHarness::Covered;
+using DepthFastClearHarness::Draw;
+using DepthFastClearHarness::DrawOptions;
+using DepthFastClearHarness::Height;
+using DepthFastClearHarness::HtileBytes;
+using DepthFastClearHarness::Require;
+using DepthFastClearHarness::Surface;
+using DepthFastClearHarness::Width;
+
+constexpr std::size_t DepthPlaneBytes = 128u * 128u * 4u;
+constexpr std::size_t StencilPlaneBytes = 256u * 256u;
+
+alignas(4096) std::array<std::byte, DepthPlaneBytes> Depth{};
+alignas(4096) std::array<std::byte, StencilPlaneBytes> Stencil{};
+alignas(4096) std::array<std::byte, DepthPlaneBytes> HtileDepth{};
+alignas(256) std::array<std::uint32_t, HtileBytes / 4u> Htile{};
+
+VkStencilOpState StencilState(VkCompareOp compare, VkStencilOp pass, std::uint32_t reference) {
+    VkStencilOpState state{};
+    state.failOp = VK_STENCIL_OP_KEEP;
+    state.passOp = pass;
+    state.depthFailOp = VK_STENCIL_OP_KEEP;
+    state.compareOp = compare;
+    state.compareMask = 0xffu;
+    state.writeMask = 0xffu;
+    state.reference = reference;
+    return state;
+}
+
+DrawOptions StencilEquals(std::uint32_t reference) {
+    return {.depthTest = false, .stencilTest = true, .stencil = StencilState(VK_COMPARE_OP_EQUAL, VK_STENCIL_OP_KEEP, reference)};
+}
+
+DrawOptions StencilWrite(std::uint32_t reference) {
+    return {.depthTest = false, .stencilTest = true, .stencil = StencilState(VK_COMPARE_OP_ALWAYS, VK_STENCIL_OP_REPLACE, reference)};
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Require(AgcDriver::Graphics::DepthSliceBytes({Width, Height}, 4u) == DepthPlaneBytes, "the depth plane size of the test surface changed");
+        Require(AgcDriver::Graphics::DepthSliceBytes({Width, Height}, 1u) == StencilPlaneBytes, "the stencil plane size of the test surface changed");
+        const auto shaders = Compile(*device);
+        const Surface surface{reinterpret_cast<std::uintptr_t>(Depth.data()), reinterpret_cast<std::uintptr_t>(Stencil.data()), 0, false, VK_FORMAT_D32_SFLOAT_S8_UINT};
+        const auto depth = surface.depth;
+        const auto stencil = surface.stencil;
+
+        Require(Draw(*device, shaders, surface, StencilWrite(2u)) == Covered, "the stencil mark draw did not cover the target");
+        Require(Draw(*device, shaders, surface, StencilEquals(2u)) == Covered, "the stencil mark was not kept");
+        AgcDriver::Graphics::NoteDepthMetadataFill(stencil, StencilPlaneBytes - 4u, 0x01010101u);
+        Require(Draw(*device, shaders, surface, StencilEquals(2u)) == Covered, "a fill shorter than the stencil plane changed the stencil");
+        AgcDriver::Graphics::NoteDepthMetadataFill(stencil, StencilPlaneBytes, 0x01020304u);
+        Require(Draw(*device, shaders, surface, StencilEquals(2u)) == Covered, "a fill whose bytes differ changed the stencil");
+        AgcDriver::Graphics::NoteDepthMetadataFill(stencil, StencilPlaneBytes, 0x01010101u);
+        Require(Draw(*device, shaders, surface, StencilEquals(2u)) == 0u, "a fill of the whole stencil plane with 1 left the stencil mark 2");
+        Require(Draw(*device, shaders, surface, StencilEquals(1u)) == Covered, "a fill of the whole stencil plane with 1 did not clear the stencil to 1");
+        Require(Draw(*device, shaders, surface, StencilWrite(2u)) == Covered, "the stencil could not be written after the fill");
+        AgcDriver::Graphics::NoteDepthMetadataFill(stencil - 4096u, StencilPlaneBytes + 8192u, 0u);
+        Require(Draw(*device, shaders, surface, StencilEquals(0u)) == Covered, "a fill around the whole stencil plane with 0 did not clear the stencil to 0");
+
+        Require(Draw(*device, shaders, surface, {.depthWrite = true}) == Covered, "the depth was not written at 0.5");
+        Require(Draw(*device, shaders, surface, {}) == 0u, "a draw at depth 0.5 passed LESS against the 0.5 written before it");
+        AgcDriver::Graphics::NoteDepthMetadataFill(depth, DepthPlaneBytes, 0x3f800000u);
+        Require(Draw(*device, shaders, surface, {}) == Covered, "a fill of the whole depth plane with 1.0 did not clear the depth to 1.0");
+        AgcDriver::Graphics::NoteDepthMetadataFill(depth, DepthPlaneBytes, 0u);
+        Require(Draw(*device, shaders, surface, {}) == 0u, "a fill of the whole depth plane with 0.0 did not clear the depth to 0.0");
+        AgcDriver::Graphics::NoteDepthMetadataFill(depth, DepthPlaneBytes, 0xff000000u);
+        Require(Draw(*device, shaders, surface, {}) == 0u, "a fill of the depth plane with a value outside 0 to 1 changed the depth");
+        AgcDriver::Graphics::NoteDepthMetadataFill(depth, DepthPlaneBytes - 4u, 0x3f800000u);
+        Require(Draw(*device, shaders, surface, {}) == 0u, "a fill shorter than the depth plane changed the depth");
+
+        Htile.fill(0xffffffffu);
+        const Surface htiled{reinterpret_cast<std::uintptr_t>(HtileDepth.data()), 0, reinterpret_cast<std::uintptr_t>(Htile.data()), false, VK_FORMAT_D32_SFLOAT};
+        Require(Draw(*device, shaders, htiled, {.depthWrite = true}) == Covered, "the depth of the surface with HTILE was not written at 0.5");
+        AgcDriver::Graphics::NoteDepthMetadataFill(htiled.depth, DepthPlaneBytes, 0u);
+        AgcDriver::Graphics::NoteDepthMetadataFill(htiled.htile, HtileBytes, 0u);
+        Require(Draw(*device, shaders, htiled, {.depthWrite = true}) == Covered, "an HTILE fill after a depth plane fill of 0.0 did not clear the depth to DB_DEPTH_CLEAR (1.0)");
+        AgcDriver::Graphics::NoteDepthMetadataFill(htiled.depth, DepthPlaneBytes, 0u);
+        AgcDriver::Graphics::NoteHtileDepthClear(htiled.htile, HtileBytes);
+        Require(Draw(*device, shaders, htiled, {}) == Covered, "an HTILE clear kernel after a depth plane fill of 0.0 did not clear the depth to DB_DEPTH_CLEAR (1.0)");
+        std::puts("depth plane fill tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/DepthStorage.cpp

@@ -109,6 +109,7 @@ public:
             if (hasExtension(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) context.memoryProperties2 = function<PFN_vkGetPhysicalDeviceMemoryProperties2>("vkGetPhysicalDeviceMemoryProperties2");
             context.Function<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(context.device, family, 0, &context.queue);
             VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
+            pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
             pool.queueFamilyIndex = family;
             Check(context.Function<PFN_vkCreateCommandPool>("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool");
         } catch (...) {

core/libs/prx/libSceAgcDriver/tests/execution/DispatchMove.cpp

@@ -0,0 +1,285 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Words = 64;
+constexpr std::uint32_t Bytes = Words * 4u;
+constexpr std::uint32_t Constant = 0x00010000u;
+constexpr std::uint32_t Sentinel = 0xdeadbeefu;
+constexpr std::size_t Piece = 65536;
+constexpr std::size_t BlockBytes = 5 * Piece;
+
+alignas(256) std::array<std::uint32_t, 8> Table{};
+
+constexpr std::uint32_t Vop2(std::uint32_t opcode, std::uint32_t destination, std::uint32_t source0, std::uint32_t source1) {
+    return (opcode << 25u) | (destination << 17u) | (source1 << 9u) | source0;
+}
+
+constexpr std::uint32_t BufferOperands = (0x80u << 24u) | (1u << 16u) | (2u << 8u) | 1u;
+
+alignas(256) constexpr std::array<std::uint32_t, 15> Code{
+    Vop2(0x1au, 1u, 0x82u, 0u),
+    0xf4080100u, 0xfa000000u,
+    0xbf8cc07fu,
+    0xe0301000u, BufferOperands,
+    0xbf8c3f70u,
+    0xf4080100u, 0xfa000010u,
+    0xbf8cc07fu,
+    Vop2(0x25u, 2u, 0xffu, 2u), Constant,
+    0xe0701000u, BufferOperands,
+    0xbf810000u,
+};
+
+class GuestBlock {
+public:
+    GuestBlock() {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(BlockBytes, Piece));
+        if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes);
+#else
+        constexpr std::uintptr_t alignment = Piece;
+        void* mapped = mmap(nullptr, BlockBytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        if (mapped != MAP_FAILED) {
+            const auto begin = reinterpret_cast<std::uintptr_t>(mapped);
+            const auto aligned = (begin + alignment - 1) & ~(alignment - 1);
+            if (aligned != begin) munmap(mapped, aligned - begin);
+            if (aligned + BlockBytes != begin + BlockBytes + alignment) munmap(reinterpret_cast<void*>(aligned + BlockBytes), begin + alignment - aligned);
+            block = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, BlockBytes);
+        }
+#endif
+        Require(block != nullptr, "dispatch move: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+#ifdef _WIN32
+        GuestArena::GuestArenaReset_nid_postfix(block, BlockBytes);
+        GuestArena::GuestArenaRelease_nid_postfix(block, BlockBytes);
+#else
+        munmap(block, BlockBytes);
+        GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, BlockBytes);
+#endif
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint32_t* InputA() const { return reinterpret_cast<std::uint32_t*>(block); }
+    std::uint32_t* InputB() const { return reinterpret_cast<std::uint32_t*>(block + Piece + 12); }
+    std::uint32_t* InputC() const { return reinterpret_cast<std::uint32_t*>(block + 2 * Piece + 32); }
+    std::uint32_t* OutputO() const { return reinterpret_cast<std::uint32_t*>(block + 3 * Piece); }
+    std::uint32_t* OutputP() const { return reinterpret_cast<std::uint32_t*>(block + 4 * Piece); }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Fill(std::uint32_t* where, std::uint32_t first) {
+    for (std::uint32_t index = 0; index < Words; ++index) where[index] = first + index;
+}
+
+void Reset(std::uint32_t* output) {
+    std::fill(output, output + Words, Sentinel);
+}
+
+ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, const std::uint32_t* input, std::uint32_t inputBytes, const std::uint32_t* output) {
+    const auto in = BufferDescriptor(input, inputBytes);
+    const auto out = BufferDescriptor(output, Bytes);
+    std::copy(in.begin(), in.end(), Table.begin());
+    std::copy(out.begin(), out.end(), Table.begin() + 4);
+    const auto table = reinterpret_cast<std::uintptr_t>(Table.data());
+    const std::vector<std::uint32_t> userData{static_cast<std::uint32_t>(table), static_cast<std::uint32_t>(table >> 32u)};
+    const std::span<const std::uint32_t> code(Code);
+    const std::span<const std::uint32_t> tableWords(Table);
+    const std::array<ShaderRecompiler::MemoryRegion, 2> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}, {table, std::as_bytes(tableWords)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    return ShaderRecompiler::Recompile(request);
+}
+
+void Run(AgcDriver::VulkanDevice& device, const ShaderRecompiler::RecompileResult& compiled, const std::uint32_t* input, std::uint32_t inputBytes, const std::uint32_t* output, bool wait) {
+    auto result = compiled;
+    std::size_t elements = 0;
+    for (auto& binding : result.bindings) {
+        if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue;
+        Require(binding.guestDescriptor.size() == static_cast<std::size_t>(binding.count) * 4u && binding.bufferWritten.size() == binding.count, "dispatch move: the program's buffer bindings have another shape");
+        for (std::uint32_t element = 0; element < binding.count; ++element, ++elements) {
+            const auto words = binding.bufferWritten[element] ? BufferDescriptor(output, Bytes) : BufferDescriptor(input, inputBytes);
+            std::copy(words.begin(), words.end(), binding.guestDescriptor.begin() + static_cast<std::ptrdiff_t>(element) * 4);
+        }
+    }
+    Require(elements == 2, "dispatch move: the program does not bind an input and an output");
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(Code.data()));
+    if (wait) device.WaitIdle();
+}
+
+void Expect(const std::uint32_t* input, const std::uint32_t* output, std::uint32_t added, const std::string& what) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const auto expected = input[tid] + added;
+        Require(output[tid] == expected, "dispatch move: " + what + ": lane " + std::to_string(tid) + " is " + Hex(output[tid]) + ", expected " + Hex(expected));
+    }
+}
+
+struct Step {
+    std::uint64_t checked, exact, moved, refused, rebuilt, kept, served;
+};
+
+Step Since(const AgcDriver::VulkanDevice::DispatchMoveCount& before) {
+    const auto now = AgcDriver::VulkanDevice::DispatchMoveCounts();
+    return {now.checked - before.checked, now.exact - before.exact, now.moved - before.moved, now.refused - before.refused, now.rebuilt - before.rebuilt, now.kept - before.kept, now.served - before.served};
+}
+
+void Print(const char* what, const Step& step) {
+    std::printf("dispatch move: %s: checked %llu, exact %llu, moved %llu, refused %llu\n", what, static_cast<unsigned long long>(step.checked), static_cast<unsigned long long>(step.exact), static_cast<unsigned long long>(step.moved), static_cast<unsigned long long>(step.refused));
+}
+
+bool RunTests(AgcDriver::VulkanDevice& device, const GuestBlock& guest) {
+    Fill(guest.InputA(), 0xa000u);
+    Fill(guest.InputB(), 0xb000u);
+    Fill(guest.InputC(), 0xc000u);
+
+    const auto compiled = Compile(device, guest.InputA(), Bytes, guest.OutputO());
+    Reset(guest.OutputO());
+    Run(device, compiled, guest.InputA(), Bytes, guest.OutputO(), true);
+    Expect(guest.InputA(), guest.OutputO(), Constant, "the first dispatch");
+    const auto start = AgcDriver::VulkanDevice::DispatchMoveCounts();
+
+    Reset(guest.OutputO());
+    Run(device, compiled, guest.InputB(), Bytes, guest.OutputO(), true);
+    Expect(guest.InputB(), guest.OutputO(), Constant, "an input moved to another block");
+    Reset(guest.OutputO());
+    Run(device, compiled, guest.InputA(), Bytes, guest.OutputO(), true);
+    Expect(guest.InputA(), guest.OutputO(), Constant, "the input back where the template was built");
+    Reset(guest.OutputO());
+    Run(device, compiled, guest.InputC(), Bytes, guest.OutputO(), true);
+    Expect(guest.InputC(), guest.OutputO(), Constant, "an input moved to a third block");
+    const auto repeats = Since(start);
+    Print("an exact repeat between two moved ones", repeats);
+    const bool templates = repeats.checked != 0;
+    if (templates) {
+        Require(repeats.exact == 1 && repeats.moved == 2 && repeats.refused == 0, "dispatch move: the repeats were not served as one exact hit and two moved ones");
+    } else {
+        std::puts("dispatch move: the device kept no template (no host imports); the moved-buffer path is not exercised");
+    }
+
+    Reset(guest.OutputO());
+    Run(device, compiled, guest.InputB(), 192, guest.OutputO(), true);
+    Expect(guest.InputB(), guest.OutputO(), Constant, "an input moved and shortened");
+
+    std::array<std::uint32_t, Words> changed{};
+    for (std::uint32_t index = 0; index < Words; ++index) changed[index] = 0xd000u + index * 3u;
+    AgcDriver::GuestMemory::Write(reinterpret_cast<std::uintptr_t>(guest.InputB()), std::as_bytes(std::span(changed)));
+    Reset(guest.OutputO());
+    Run(device, compiled, guest.InputB(), Bytes, guest.OutputO(), true);
+    Expect(changed.data(), guest.OutputO(), Constant, "an input moved to a range the driver stored over");
+
+    const auto refusals = AgcDriver::VulkanDevice::DispatchMoveCounts();
+    for (std::uint32_t index = 0; index < Words; ++index) guest.OutputO()[index] = 0x7000u + index;
+    Run(device, compiled, guest.OutputO() + Threads, Bytes / 2, guest.OutputO(), true);
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const auto expected = 0x7000u + Threads + tid + Constant;
+        Require(guest.OutputO()[tid] == expected, "dispatch move: an input moved into the unwritten half of the dispatch's own output: lane " + std::to_string(tid) + " is " + Hex(guest.OutputO()[tid]) + ", expected " + Hex(expected));
+    }
+
+    Reset(guest.OutputO());
+    Reset(guest.OutputP());
+    Run(device, compiled, guest.InputA(), Bytes, guest.OutputP(), false);
+    Run(device, compiled, guest.OutputP(), Bytes, guest.OutputO(), true);
+    Expect(guest.InputA(), guest.OutputO(), 2u * Constant, "an input moved onto a range a recorded dispatch still writes");
+    const auto refused = Since(refusals);
+    Print("two inputs moved onto written ranges", refused);
+    if (templates) {
+        Require(refused.refused == 2, "dispatch move: the inputs moved onto written ranges were not both refused");
+        Require(refused.rebuilt == 2 && refused.kept == 2 && refused.served == 0, "dispatch move: the two refusals were not two rebuilds inserted beside the template");
+    }
+
+    const auto afterRefusals = AgcDriver::VulkanDevice::DispatchMoveCounts();
+    Reset(guest.OutputO());
+    Run(device, compiled, guest.InputA(), Bytes, guest.OutputO(), true);
+    Expect(guest.InputA(), guest.OutputO(), Constant, "the input moved back after the refusals");
+    const auto back = Since(afterRefusals);
+    Print("the input back after the refusals", back);
+    if (templates) Require(back.checked == 1 && back.exact == 1 && back.moved == 0 && back.refused == 0 && back.rebuilt == 0, "dispatch move: the input back at the template's address was not an exact hit after the refusals");
+
+    for (int round = 0; round < 2; ++round) {
+        const auto alternating = AgcDriver::VulkanDevice::DispatchMoveCounts();
+        Reset(guest.OutputO());
+        Reset(guest.OutputP());
+        Run(device, compiled, guest.InputA(), Bytes, guest.OutputP(), false);
+        Run(device, compiled, guest.OutputP(), Bytes, guest.OutputO(), true);
+        Expect(guest.InputA(), guest.OutputO(), 2u * Constant, "an input refused again while A alternates");
+        Reset(guest.OutputO());
+        Run(device, compiled, guest.InputA(), Bytes, guest.OutputO(), true);
+        Expect(guest.InputA(), guest.OutputO(), Constant, "the input at A after the refused one");
+        const auto step = Since(alternating);
+        Print("A alternating with a refused input", step);
+        if (templates) Require(step.refused == 1 && step.served == 1 && step.rebuilt == 0 && step.exact == 2 && step.moved == 0, "dispatch move: a refused input rebuilt, or evicted the template of A, while A alternated with it");
+    }
+    const auto total = AgcDriver::VulkanDevice::DispatchMoveCounts();
+    std::printf("dispatch move: in total: checked %llu, exact %llu, moved %llu, refused %llu, dropped by stage A %llu\n", static_cast<unsigned long long>(total.checked), static_cast<unsigned long long>(total.exact), static_cast<unsigned long long>(total.moved), static_cast<unsigned long long>(total.refused), static_cast<unsigned long long>(total.filtered));
+    return templates;
+}
+
+}
+
+int main() {
+    try {
+        GuestBlock guest;
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (!RunTests(*device, guest)) return VulkanTestSkipped;
+        std::puts("dispatch move tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/DispatchMoveDepth.cpp

@@ -0,0 +1,460 @@
+#include "DepthFastClearHarness.hpp"
+#include "VulkanTestDevice.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+
+#ifndef _WIN32
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+#endif
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using DepthFastClearHarness::Covered;
+using DepthFastClearHarness::Draw;
+using DepthFastClearHarness::Height;
+using DepthFastClearHarness::Shaders;
+using DepthFastClearHarness::Surface;
+using DepthFastClearHarness::Width;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Bytes = Threads * 4u;
+constexpr std::uint32_t Format32Float = 22;
+constexpr std::uint32_t Type2D = 9;
+constexpr std::uint32_t Sentinel = 0xdeadbeefu;
+constexpr std::size_t Piece = 65536;
+
+alignas(256) constexpr std::array<std::uint32_t, 13> RowCode{
+    0x7e3c0300u,
+    0x7e3e0280u,
+    0x34060082u,
+    0xe0301000u, 0x80040b03u,
+    0xbf8c3f70u,
+    0x4a181681u,
+    0xe0701000u, 0x80000c03u,
+    0xf0201108u, 0x00010b1eu,
+    0xbf810000u, 0xbf810000u,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 10> ProbeCode{
+    0x7e3c0300u,
+    0x7e3e020cu,
+    0xf0001108u, 0x00010a1eu,
+    0x34060082u,
+    0xbf8c3f70u,
+    0xe0701000u, 0x80000a03u,
+    0xbf810000u, 0xbf810000u,
+};
+
+class GuestBlock {
+public:
+    GuestBlock(std::size_t pieces, bool inWatch) : bytes(pieces * Piece), watched(inWatch) {
+        if (inWatch) {
+#ifdef _WIN32
+            block = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, Piece));
+            if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes);
+#else
+            constexpr std::uintptr_t alignment = Piece;
+            void* mapped = mmap(nullptr, bytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            if (mapped != MAP_FAILED) {
+                const auto begin = reinterpret_cast<std::uintptr_t>(mapped);
+                const auto aligned = (begin + alignment - 1) & ~(alignment - 1);
+                if (aligned != begin) munmap(mapped, aligned - begin);
+                if (aligned + bytes != begin + bytes + alignment) munmap(reinterpret_cast<void*>(aligned + bytes), begin + alignment - aligned);
+                block = reinterpret_cast<std::uint8_t*>(aligned);
+                GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, bytes);
+            }
+#endif
+        } else {
+#ifdef _WIN32
+            block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+            block = static_cast<std::uint8_t*>(std::aligned_alloc(Piece, bytes));
+#endif
+            if (block != nullptr) std::memset(block, 0, bytes);
+        }
+        Require(block != nullptr, "dispatch move depth: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, bytes, true, true, true);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+        if (watched) {
+#ifdef _WIN32
+            GuestArena::GuestArenaReset_nid_postfix(block, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(block, bytes);
+#else
+            munmap(block, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, bytes);
+#endif
+        } else {
+#ifdef _WIN32
+            VirtualFree(block, 0, MEM_RELEASE);
+#else
+            std::free(block);
+#endif
+        }
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* PieceAt(std::size_t index) const { return block + index * Piece; }
+
+private:
+    std::size_t bytes = 0;
+    bool watched = false;
+    std::uint8_t* block = nullptr;
+};
+
+struct Buffers {
+    explicit Buffers(const GuestBlock& block) : block(block) {}
+    std::uint32_t* InputA() const { return reinterpret_cast<std::uint32_t*>(block.PieceAt(4)); }
+    std::uint32_t* InputB() const { return reinterpret_cast<std::uint32_t*>(block.PieceAt(5) + 12); }
+    std::uint32_t* InputC() const { return reinterpret_cast<std::uint32_t*>(block.PieceAt(6) + 32); }
+    std::uint32_t* Output() const { return reinterpret_cast<std::uint32_t*>(block.PieceAt(8)); }
+
+private:
+    const GuestBlock& block;
+};
+
+struct Env {
+    AgcDriver::VulkanDevice& device;
+    const Shaders& shaders;
+    const Buffers& buffers;
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+std::array<std::uint32_t, 8> TextureDescriptor(const void* data) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(data));
+    return {
+        static_cast<std::uint32_t>(address >> 8u),
+        static_cast<std::uint32_t>((address >> 40u) & 0xffu) | (Format32Float << 20u) | (((Width - 1u) & 3u) << 30u),
+        ((Width - 1u) >> 2u) | ((Height - 1u) << 14u),
+        0xfacu | (Type2D << 28u),
+        0u, 0u, 0u, 0u,
+    };
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData) {
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderRecompiler::ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    return ShaderRecompiler::Recompile(request);
+}
+
+ShaderRecompiler::RecompileResult RowProgram(AgcDriver::VulkanDevice& device, const void* plane, const std::uint32_t* input, const std::uint32_t* output) {
+    std::vector<std::uint32_t> userData(20, 0u);
+    const auto out = BufferDescriptor(output, Bytes);
+    const auto texture = TextureDescriptor(plane);
+    const auto in = BufferDescriptor(input, Bytes);
+    std::copy(out.begin(), out.end(), userData.begin());
+    std::copy(texture.begin(), texture.end(), userData.begin() + 4);
+    std::copy(in.begin(), in.end(), userData.begin() + 16);
+    return Compile(device, RowCode, userData);
+}
+
+void RunRow(AgcDriver::VulkanDevice& device, const ShaderRecompiler::RecompileResult& compiled, const std::uint32_t* input, const std::uint32_t* output) {
+    auto result = compiled;
+    std::size_t elements = 0;
+    for (auto& binding : result.bindings) {
+        if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue;
+        Require(binding.guestDescriptor.size() == static_cast<std::size_t>(binding.count) * 4u, "dispatch move depth: the program's buffer bindings have another shape");
+        for (std::uint32_t element = 0; element < binding.count; ++element, ++elements) {
+            auto* current = binding.guestDescriptor.data() + static_cast<std::size_t>(element) * 4u;
+            const auto built = static_cast<std::uint64_t>(current[0]) | (static_cast<std::uint64_t>(current[1] & 0xffffu) << 32u);
+            const auto words = built == static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(output)) ? BufferDescriptor(output, Bytes) : BufferDescriptor(input, Bytes);
+            std::copy(words.begin(), words.end(), current);
+        }
+    }
+    Require(elements == 2, "dispatch move depth: the program does not bind an input and an output");
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(RowCode.data()));
+    device.WaitIdle();
+}
+
+std::array<std::uint32_t, Threads> ReadRow(const Env& env, const void* plane, std::uint32_t y) {
+    std::vector<std::uint32_t> userData(16, 0u);
+    auto* output = env.buffers.Output();
+    const auto out = BufferDescriptor(output, Bytes);
+    const auto texture = TextureDescriptor(plane);
+    std::copy(out.begin(), out.end(), userData.begin());
+    std::copy(texture.begin(), texture.end(), userData.begin() + 4);
+    userData[12] = y;
+    const auto program = Compile(env.device, ProbeCode, userData);
+    std::fill(output, output + Threads, Sentinel);
+    env.device.Dispatch(program, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(ProbeCode.data()));
+    env.device.WaitIdle();
+    std::array<std::uint32_t, Threads> lanes{};
+    std::copy(output, output + Threads, lanes.begin());
+    return lanes;
+}
+
+void FillInput(std::uint32_t* where, float first) {
+    for (std::uint32_t lane = 0; lane < Threads; ++lane) where[lane] = std::bit_cast<std::uint32_t>(first + static_cast<float>(lane) / 1024.0f);
+}
+
+void FillPlane(std::uint8_t* plane, float value) {
+    auto* words = reinterpret_cast<std::uint32_t*>(plane);
+    std::fill(words, words + Piece / sizeof(std::uint32_t), std::bit_cast<std::uint32_t>(value));
+}
+
+void DrawDepth(const Env& env, const void* plane, float depth) {
+    const Surface surface{reinterpret_cast<std::uintptr_t>(plane), 0, 0, false, VK_FORMAT_D32_SFLOAT};
+    const float low = depth <= 0.5f ? 0.0f : 2.0f * depth - 1.0f;
+    const float high = depth <= 0.5f ? 2.0f * depth : 1.0f;
+    Require(Draw(env.device, env.shaders, surface, {.depthWrite = true, .depthCompare = VK_COMPARE_OP_ALWAYS, .minDepth = low, .maxDepth = high}) == Covered, "dispatch move depth: the depth producer did not cover the surface");
+}
+
+struct PlaneModel {
+    std::array<std::uint32_t, Threads> row0{};
+    std::uint32_t rest = 0;
+
+    void Set(float value) {
+        rest = std::bit_cast<std::uint32_t>(value);
+        row0.fill(rest);
+    }
+
+    std::array<std::uint32_t, Threads> Dispatch(const std::uint32_t* input) {
+        std::array<std::uint32_t, Threads> sums{};
+        for (std::uint32_t lane = 0; lane < Threads; ++lane) {
+            sums[lane] = input[lane] + 1u;
+            row0[lane] = input[lane];
+        }
+        return sums;
+    }
+};
+
+void Totals(const std::string& what);
+
+void Step(const Env& env, const ShaderRecompiler::RecompileResult& program, PlaneModel& model, const std::uint32_t* input, const std::string& what) {
+    const auto expected = model.Dispatch(input);
+    auto* output = env.buffers.Output();
+    std::fill(output, output + Threads, Sentinel);
+    RunRow(env.device, program, input, output);
+    for (std::uint32_t lane = 0; lane < Threads; ++lane) {
+        Require(output[lane] == expected[lane], "dispatch move depth: " + what + ": lane " + std::to_string(lane) + " is " + Hex(output[lane]) + ", expected " + Hex(expected[lane]));
+    }
+    Totals(what);
+}
+
+void Verify(const Env& env, const void* plane, const PlaneModel& model, const std::string& what) {
+    for (const std::uint32_t y : {0u, 1u, Height - 1u}) {
+        const auto lanes = ReadRow(env, plane, y);
+        for (std::uint32_t lane = 0; lane < Threads; ++lane) {
+            const auto expected = y == 0 ? model.row0[lane] : model.rest;
+            Require(lanes[lane] == expected, "dispatch move depth: " + what + ": row " + std::to_string(y) + ", texel " + std::to_string(lane) + " of the plane is " + Hex(lanes[lane]) + ", expected " + Hex(expected));
+        }
+    }
+}
+
+struct MoveSteps {
+    std::uint64_t checked, exact, moved, refused;
+};
+
+MoveSteps Since(const AgcDriver::VulkanDevice::DispatchMoveCount& before) {
+    const auto now = AgcDriver::VulkanDevice::DispatchMoveCounts();
+    return {now.checked - before.checked, now.exact - before.exact, now.moved - before.moved, now.refused - before.refused};
+}
+
+void Totals(const std::string& what) {
+    const auto now = AgcDriver::VulkanDevice::DispatchMoveCounts();
+    std::printf("dispatch move depth: %s: in total checked %llu, exact %llu, moved %llu, refused %llu, dropped by stage A %llu\n", what.c_str(), static_cast<unsigned long long>(now.checked), static_cast<unsigned long long>(now.exact), static_cast<unsigned long long>(now.moved), static_cast<unsigned long long>(now.refused), static_cast<unsigned long long>(now.filtered));
+}
+
+void DescribeBindings(const ShaderRecompiler::RecompileResult& program) {
+    for (const auto& binding : program.bindings) {
+        std::string written;
+        for (const bool bit : binding.bufferWritten) written += bit ? '1' : '0';
+        std::printf("dispatch move depth: binding role %d kind %d count %u, %zu descriptor words, buffers written [%s], images written %zu\n", static_cast<int>(binding.role), static_cast<int>(binding.kind), binding.count, binding.guestDescriptor.size(), written.c_str(), binding.imageWritten.size());
+    }
+}
+
+void Print(const char* what, const MoveSteps& step) {
+    std::printf("dispatch move depth: %s: checked %llu, exact %llu, moved %llu, refused %llu\n", what, static_cast<unsigned long long>(step.checked), static_cast<unsigned long long>(step.exact), static_cast<unsigned long long>(step.moved), static_cast<unsigned long long>(step.refused));
+}
+
+void Warm(const Env& env, std::uint8_t* plane) {
+    FillPlane(plane, 0.5f);
+    PlaneModel model;
+    model.Set(0.5f);
+    const auto program = RowProgram(env.device, plane, env.buffers.InputA(), env.buffers.Output());
+    Step(env, program, model, env.buffers.InputA(), "warm-up: the first input");
+    Step(env, program, model, env.buffers.InputB(), "warm-up: the second input");
+    Step(env, program, model, env.buffers.InputC(), "warm-up: the third input");
+}
+
+bool Control(const Env& env, std::uint8_t* plane) {
+    FillPlane(plane, 0.5f);
+    PlaneModel model;
+    model.Set(0.5f);
+    const auto refusals = AgcDriver::Graphics::ShaderResources::DepthGuardRefusals();
+    const auto program = RowProgram(env.device, plane, env.buffers.InputA(), env.buffers.Output());
+    DescribeBindings(program);
+    Step(env, program, model, env.buffers.InputA(), "control: the first dispatch");
+    Totals("control, after the first dispatch");
+    const auto start = AgcDriver::VulkanDevice::DispatchMoveCounts();
+    Step(env, program, model, env.buffers.InputB(), "control: an input moved");
+    Totals("control, after an input moved");
+    Step(env, program, model, env.buffers.InputC(), "control: the input moved again");
+    Totals("control, after the input moved again");
+    Step(env, program, model, env.buffers.InputC(), "control: an exact repeat");
+    Totals("control, after an exact repeat");
+    const auto steps = Since(start);
+    Print("control, a plane without a depth surface", steps);
+    Verify(env, plane, model, "control");
+    Require(AgcDriver::Graphics::ShaderResources::DepthGuardRefusals() == refusals, "dispatch move depth: the depth guard refused a proof with no depth surface in sight");
+    Require(steps.refused == 0, "dispatch move depth: the control's repeats were refused");
+    if (steps.checked == 0) return false;
+    Require(steps.checked == 3 && steps.moved == 3, "dispatch move depth: the control's repeats were not all served by the template of the first dispatch");
+    return true;
+}
+
+void Mirror(const Env& env, std::uint8_t* plane, const char* where) {
+    const std::string at = std::string(where) + ", the mirror: ";
+    FillPlane(plane, 0.5f);
+    PlaneModel model;
+    DrawDepth(env, plane, 0.25f);
+    model.Set(0.25f);
+    const auto program = RowProgram(env.device, plane, env.buffers.InputA(), env.buffers.Output());
+    Step(env, program, model, env.buffers.InputA(), at + "the first dispatch");
+    Verify(env, plane, model, at + "after the first dispatch");
+    DrawDepth(env, plane, 0.75f);
+    model.Set(0.75f);
+    const auto start = AgcDriver::VulkanDevice::DispatchMoveCounts();
+    Step(env, program, model, env.buffers.InputB(), at + "an input moved after the depth was drawn again");
+    Verify(env, plane, model, at + "after the moved dispatch");
+    Step(env, program, model, env.buffers.InputB(), at + "an exact repeat");
+    DrawDepth(env, plane, 0.25f);
+    model.Set(0.25f);
+    Step(env, program, model, env.buffers.InputC(), at + "the input moved again after another redraw");
+    Verify(env, plane, model, at + "after the last dispatch");
+    Print((at + "repeats and redraws").c_str(), Since(start));
+}
+
+void Appears(const Env& env, std::uint8_t* plane, const char* where, bool moved, bool templates) {
+    const std::string at = std::string(where) + (moved ? ", a surface appears, input moved: " : ", a surface appears, exact repeat: ");
+    FillPlane(plane, 0.125f);
+    PlaneModel model;
+    model.Set(0.125f);
+    const auto program = RowProgram(env.device, plane, env.buffers.InputA(), env.buffers.Output());
+    Step(env, program, model, env.buffers.InputA(), at + "the first dispatch, no depth surface yet");
+    DrawDepth(env, plane, 0.25f);
+    model.Set(0.25f);
+    const auto start = AgcDriver::VulkanDevice::DispatchMoveCounts();
+    const auto refusals = AgcDriver::Graphics::ShaderResources::DepthGuardRefusals();
+    Step(env, program, model, moved ? env.buffers.InputB() : env.buffers.InputA(), at + "the dispatch after the surface appeared");
+    std::printf("dispatch move depth: %sthe depth guard refused %llu proofs\n", at.c_str(), static_cast<unsigned long long>(AgcDriver::Graphics::ShaderResources::DepthGuardRefusals() - refusals));
+    if (templates) Require(AgcDriver::Graphics::ShaderResources::DepthGuardRefusals() > refusals, "dispatch move depth: " + at + "the template over the plain storage image was proved current although a depth surface covers its memory");
+    Print((at + "the dispatch after the surface appeared").c_str(), Since(start));
+    Totals(at + "after that dispatch");
+    Verify(env, plane, model, at + "after that dispatch");
+    DrawDepth(env, plane, 0.75f);
+    model.Set(0.75f);
+    Step(env, program, model, env.buffers.InputC(), at + "the input moved after the next redraw");
+    Verify(env, plane, model, at + "after the last dispatch");
+}
+
+bool RunTests(const Env& env, const GuestBlock& watched, const GuestBlock& unwatched, std::size_t pass) {
+    const auto inWatch = pass * 10;
+    const auto outside = pass * 3;
+    FillInput(env.buffers.InputA(), 0.0625f);
+    FillInput(env.buffers.InputB(), 0.109375f);
+    FillInput(env.buffers.InputC(), 0.15625f);
+    Warm(env, watched.PieceAt(9));
+    const bool templates = Control(env, watched.PieceAt(inWatch));
+    if (!templates) std::puts("dispatch move depth: the device kept no template (no host imports); the moved-buffer path is not exercised");
+    Mirror(env, watched.PieceAt(inWatch + 1), "inside the write watch");
+    Appears(env, watched.PieceAt(inWatch + 2), "inside the write watch", true, templates);
+    Appears(env, watched.PieceAt(inWatch + 3), "inside the write watch", false, templates);
+    Mirror(env, unwatched.PieceAt(outside), "outside the write watch");
+    Appears(env, unwatched.PieceAt(outside + 1), "outside the write watch", true, templates);
+    Appears(env, unwatched.PieceAt(outside + 2), "outside the write watch", false, templates);
+    return templates;
+}
+
+}
+
+int main() {
+    try {
+        const GuestBlock watched(20, true);
+        const GuestBlock unwatched(6, false);
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        const auto shaders = DepthFastClearHarness::Compile(*device);
+        const Buffers buffers(watched);
+        const Env env{*device, shaders, buffers};
+        bool templates = true;
+#ifndef _WIN32
+        const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(*device);
+        const auto decided = AgcDriver::Graphics::PrepareImportWatch(context);
+        struct Restore {
+            const AgcDriver::Graphics::Context& context;
+            AgcDriver::Graphics::ImportWatch decided;
+            ~Restore() { AgcDriver::Graphics::SetImportWatch(context, decided); }
+        } restore{context, decided};
+        std::size_t pass = 0;
+        for (const auto mode : {AgcDriver::Graphics::ImportWatch::Watch, AgcDriver::Graphics::ImportWatch::Unwatch}) {
+            AgcDriver::Graphics::SetImportWatch(context, mode);
+            std::printf("dispatch move depth: pass %zu, imports %s\n", pass, mode == AgcDriver::Graphics::ImportWatch::Watch ? "under the write watch" : "left unwatched");
+            templates = RunTests(env, watched, unwatched, pass++) && templates;
+        }
+#else
+        templates = RunTests(env, watched, unwatched, 0);
+#endif
+        if (!templates) return VulkanTestSkipped;
+        std::puts("dispatch move depth tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/DrawCacheRelocation.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
 #include "VulkanTestDevice.hpp"
 #include <array>
@@ -22,7 +23,18 @@ using AgcDriver::DriverDetail::DrawMiss;
 alignas(256) constexpr std::array<std::uint32_t, 8> VertexCode{0x7e000208u, 0x7e020280u, 0x7e040280u, 0x7e0602f2u, 0xf80008cfu, 0x03020100u, 0xbf810000u, 0xbf810000u};
 alignas(256) constexpr std::array<std::uint32_t, 8> SecondVertexCode = VertexCode;
 alignas(256) constexpr std::array<std::uint32_t, 8> PixelCode{0xf4000100u, 0xfa000000u, 0x7e000204u, 0x7e020280u, 0xf800180fu, 0x00010100u, 0xbf810000u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 16> TriangleVertexCode{
+    0x7e000d05u, 0x080200f4u, 0x10020300u, 0x100202f4u, 0x060202f3u, 0x060400f3u, 0x10040500u, 0x060404f3u,
+    0x7e060280u, 0x7e0802f2u, 0xf80008cfu, 0x04030201u, 0xbf810000u,
+};
+alignas(256) constexpr std::array<std::uint32_t, 16> ForwardedPixelCode{
+    0xf4040080u, 0xfa000000u, 0xbf8cc07fu, 0xf4000101u, 0xfa000000u, 0xbf8cc07fu, 0x7e000204u, 0xd5480001u,
+    0x02050800u, 0x7e020d01u, 0xf800180fu, 0x00010100u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u,
+};
 alignas(16) std::array<float, 4> Other{0.5f, 0.0f, 0.0f, 0.0f};
+alignas(16) std::array<float, 4> ForwardedData{};
+alignas(32) std::array<std::array<std::uint64_t, 4>, 6> AlignedTables{};
+alignas(32) std::array<std::uint64_t, 4> OffsetTable{};
 alignas(256) std::array<std::byte, 1024> ColorMemory{};
 
 struct Header {
@@ -33,6 +45,8 @@ struct Header {
 alignas(8) Header VertexHeader{};
 alignas(8) Header PixelHeader{};
 alignas(8) Header SecondVertexHeader{};
+alignas(8) Header TriangleVertexHeader{};
+alignas(8) Header ForwardedPixelHeader{};
 
 const Shader* Registered(Header& header, const std::uint32_t* code, std::size_t bytes, std::uint8_t type) {
     header.shader.file_header = 0x34333231u;
@@ -51,10 +65,10 @@ void Append(std::vector<std::uint32_t>& commands, std::uint32_t opcode, std::ini
     commands.insert(commands.end(), payload);
 }
 
-std::vector<std::uint32_t> DrawFrom(std::initializer_list<const float*> sources, const std::uint32_t* vertexCode = VertexCode.data()) {
+std::vector<std::uint32_t> DrawFrom(std::initializer_list<const void*> sources, const std::uint32_t* vertexCode = VertexCode.data(), const std::uint32_t* pixelCode = PixelCode.data()) {
     const auto color = reinterpret_cast<std::uintptr_t>(ColorMemory.data());
     const auto vertex = reinterpret_cast<std::uintptr_t>(vertexCode);
-    const auto pixel = reinterpret_cast<std::uintptr_t>(PixelCode.data());
+    const auto pixel = reinterpret_cast<std::uintptr_t>(pixelCode);
     const std::vector<std::pair<std::uint32_t, std::uint32_t>> context{
         {0x2d5, 0x2000},
         {0x1b6, 0}, {0x207, 0}, {0x200, 0}, {0x203, 0x800},
@@ -97,7 +111,7 @@ void MovedDataHitsTheCache(bool libraries) {
     Registered(VertexHeader, VertexCode.data(), sizeof(VertexCode), 2);
     Registered(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1);
     const auto mode = std::string(libraries ? "with" : "without") + " graphics pipeline libraries";
-    const auto draw = [&](std::initializer_list<const float*> sources) {
+    const auto draw = [&](std::initializer_list<const void*> sources) {
         auto commands = DrawFrom(sources);
         Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
         Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw submission failed");
@@ -145,6 +159,48 @@ void UnrelocatedAliasesStayOut(bool libraries) {
     Require(counters.hits - start.hits == twice.size(), mode + ": the exact key of a repeated draw lost its hits next to draws whose alias never relocates (" + std::to_string(counters.hits - start.hits) + " hits in " + std::to_string(twice.size()) + " draws)");
 }
 
+void MovedForwardedWordReachesStageDataHit(bool libraries) {
+    Registered(TriangleVertexHeader, TriangleVertexCode.data(), sizeof(TriangleVertexCode), 2);
+    Registered(ForwardedPixelHeader, ForwardedPixelCode.data(), sizeof(ForwardedPixelCode), 1);
+    const auto mode = std::string(libraries ? "with" : "without") + " graphics pipeline libraries";
+    constexpr std::byte Kept{0x40};
+    const auto draw = [&](const void* table) {
+        ColorMemory.fill(Kept);
+        auto commands = DrawFrom({table}, TriangleVertexCode.data(), ForwardedPixelCode.data());
+        Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+        Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw submission failed");
+        AgcDriverWaitIdle_nid_postfix();
+        AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast<std::uintptr_t>(ColorMemory.data()), ColorMemory.size());
+        return AgcDriver::DriverDetail::Driver::Get().DrawCacheCounters();
+    };
+    const auto pixels = [&](std::array<std::uint8_t, 4> expected) {
+        std::size_t drawn = 0;
+        for (std::size_t at = 0; at < ColorMemory.size(); at += 4) {
+            const std::array<std::uint8_t, 4> pixel{static_cast<std::uint8_t>(ColorMemory[at]), static_cast<std::uint8_t>(ColorMemory[at + 1]), static_cast<std::uint8_t>(ColorMemory[at + 2]), static_cast<std::uint8_t>(ColorMemory[at + 3])};
+            if (pixel == std::array<std::uint8_t, 4>{0x40, 0x40, 0x40, 0x40}) continue;
+            if (pixel != expected) return "pixel " + std::to_string(at / 4) + " is " + std::to_string(pixel[0]) + "," + std::to_string(pixel[1]) + "," + std::to_string(pixel[2]) + "," + std::to_string(pixel[3]) + ", expected " + std::to_string(expected[0]) + "," + std::to_string(expected[1]) + "," + std::to_string(expected[2]) + "," + std::to_string(expected[3]);
+            ++drawn;
+        }
+        return drawn == 0 ? std::string("the draw wrote no pixel") : std::string();
+    };
+    const auto data = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(ForwardedData.data()));
+    for (auto& table : AlignedTables) table[0] = data;
+    OffsetTable[2] = data;
+    ForwardedData = {1.0f, 0.0f, 0.0f, 0.0f};
+    const auto start = AgcDriver::DriverDetail::Driver::Get().DrawCacheCounters();
+    AgcDriver::DriverDetail::DrawEntryCounters counters{};
+    for (const auto& table : AlignedTables) counters = draw(table.data());
+    Require(counters.hits != start.hits, mode + ": draws whose pointer table moved to identical contents never reused the cached draw (" + std::to_string(counters.lookups - start.lookups) + " lookups, " + std::to_string(counters.inserts - start.inserts) + " inserts)");
+    const auto taught = pixels({255, 0, 0, 255});
+    Require(taught.empty(), mode + ": a draw through a pointer table with bit 4 of its address clear: " + taught);
+    ForwardedData[0] = 0.25f;
+    const auto before = counters;
+    counters = draw(&OffsetTable[2]);
+    Require(counters.lookups == before.lookups + 1 && counters.hits == before.hits + 1 && counters.stageDataHits == before.stageDataHits + 1, mode + ": a draw whose pointer table moved and whose data changed in place was not a relocated stage data hit (" + std::to_string(counters.hits - before.hits) + " hits, " + std::to_string(counters.stageDataHits - before.stageDataHits) + " stage data hits)");
+    const auto moved = pixels({64, 255, 255, 64});
+    Require(moved.empty(), mode + ": a relocated stage data hit did not draw with the moved user word: " + moved);
+}
+
 }
 
 int main() {
@@ -158,6 +214,7 @@ int main() {
         if (!libraries && std::getenv("APS5_NO_GPL") == nullptr) return VulkanTestSkipped;
         MovedDataHitsTheCache(libraries);
         UnrelocatedAliasesStayOut(libraries);
+        MovedForwardedWordReachesStageDataHit(libraries);
         AgcDriverShutdown_nid_postfix();
         std::cout << "A draw whose data moved reused the cached draw " << (libraries ? "with" : "without") << " graphics pipeline libraries\n";
         return 0;

core/libs/prx/libSceAgcDriver/tests/execution/DrawCacheReuse.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp"
 #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
 #include "VulkanTestDevice.hpp"
 #include <algorithm>
@@ -243,6 +244,7 @@ int main() {
         StageDataHitsRefreshTheirWords(libraries);
         RetainedPixelCacheTransitions();
         AgcDriverShutdown_nid_postfix();
+        Require(AgcDriver::Graphics::PipelineLibraryWorkers() == 0, "the driver's shutdown left the pipeline library worker running");
         std::cout << "A repeated draw reused the cached draw " << (libraries ? "with" : "without") << " graphics pipeline libraries\n";
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/execution/F16FmaDenormalModes.cpp

@@ -205,13 +205,18 @@ void Run(AgcDriver::VulkanDevice& device, const Mode& mode) {
     device.WaitIdle();
 }
 
+bool Matches(std::uint32_t actual, std::uint32_t expected) {
+    const auto nan = [](std::uint32_t half) { return (half & 0x7fffu) > 0x7c00u; };
+    return actual == expected || (nan(expected) && actual <= 0xffffu && nan(actual));
+}
+
 void Check(const Mode& mode) {
     constexpr const char* Names[Results] = {"v_fma_f16", "v_fmac_f16"};
     for (std::uint32_t tid = 0; tid < Threads; ++tid) {
         const std::uint32_t expected = Expected[tid][mode.column];
         for (std::uint32_t form = 0; form < Results; ++form) {
             const std::uint32_t actual = Output[tid * Results + form];
-            Require(actual == expected, std::string(Names[form]) + " " + mode.name + " of " + Hex(Rows[tid][0]) + ", " + Hex(Rows[tid][1]) + ", " + Hex(Rows[tid][2]) + " is " + Hex(actual) + ", expected " + Hex(expected));
+            Require(Matches(actual, expected), std::string(Names[form]) + " " + mode.name + " of " + Hex(Rows[tid][0]) + ", " + Hex(Rows[tid][1]) + ", " + Hex(Rows[tid][2]) + " is " + Hex(actual) + ", expected " + Hex(expected));
         }
     }
 }

core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp

@@ -31,7 +31,9 @@ constexpr std::uint32_t MaskedThreads = 16;
 constexpr std::size_t BlockBytes = 65536;
 constexpr std::uint32_t CheckedBytes = 0x2000;
 constexpr std::uint8_t Fill = 0xcd;
-#ifdef __linux__
+#if defined(__linux__) && defined(ANYPS5_ENABLE_ASAN)
+constexpr std::array<std::uintptr_t, 2> GuestWindowAddresses{0x60000000, 0x200000000000};
+#elif defined(__linux__)
 constexpr std::array<std::uintptr_t, 2> GuestWindowAddresses{0x80000000000, 0x200000000000};
 #endif
 

core/libs/prx/libSceAgcDriver/tests/execution/FmaMixF16DenormalModes.cpp

@@ -166,6 +166,11 @@ void Run(AgcDriver::VulkanDevice& device, const Mode& mode) {
     device.WaitIdle();
 }
 
+bool Matches(std::uint32_t actual, std::uint32_t expected) {
+    const auto nan = [](std::uint32_t single) { return (single & 0x7fffffffu) > 0x7f800000u; };
+    return actual == expected || (nan(expected) && nan(actual));
+}
+
 void Check(const Mode& mode) {
     constexpr const char* Names[Results] = {"v_fma_mix_f32 of three f16 sources", "v_fma_mix_f32 of an f16, an f32 and an f16 source"};
     const std::uint32_t f32Kept = (mode.value.floatMode >> 4u) & 1u;
@@ -174,7 +179,7 @@ void Check(const Mode& mode) {
         for (std::uint32_t form = 0; form < Results; ++form) {
             const std::uint32_t actual = Output[tid * Results + form];
             const std::uint32_t expected = Expected[f32Kept][f16InputsKept][tid][form];
-            Require(actual == expected, std::string(Names[form]) + " " + mode.name + " of " + Hex(Rows[tid][0]) + ", " + Hex(Rows[tid][1]) + ", " + Hex(Rows[tid][2]) + " is " + Hex(actual) + ", expected " + Hex(expected));
+            Require(Matches(actual, expected), std::string(Names[form]) + " " + mode.name + " of " + Hex(Rows[tid][0]) + ", " + Hex(Rows[tid][1]) + ", " + Hex(Rows[tid][2]) + " is " + Hex(actual) + ", expected " + Hex(expected));
         }
     }
 }

core/libs/prx/libSceAgcDriver/tests/execution/GpuMapOnFiber.cpp

@@ -34,7 +34,11 @@ using AgcDriver::Graphics::Require;
 constexpr std::size_t HostStackContextBytes = 16 * 1024;
 constexpr std::size_t ContextBytes = 64 * 1024;
 constexpr std::size_t MappingBytes = 4 * 65536;
+#ifdef ANYPS5_ENABLE_ASAN
+constexpr std::size_t FiberStackLimit = 16 * 1024;
+#else
 constexpr std::size_t FiberStackLimit = 8 * 1024;
+#endif
 constexpr std::uint64_t Fill = 0x5aa5c33c5aa5c33cull;
 
 alignas(16) std::array<std::uint64_t, ContextBytes / sizeof(std::uint64_t)> FiberContext;

core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp

@@ -110,6 +110,20 @@ alignas(256) constexpr std::array<std::uint32_t, 32> TableWave32Code{
     0x01090401u, 0x7e120202u, 0x34101287u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u,
 };
 
+alignas(256) constexpr std::array<std::uint32_t, 29> SelectedTableCode{
+    0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4000400u, 0xfa000060u,
+    0xf4200444u, 0x20000000u, 0x9312b011u, 0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u,
+    0x100202ffu, 0x3c800000u, 0x7e0402ffu, 0x3e000000u, 0xf09c8f08u, 0x01090401u, 0x7e120202u, 0x34101286u,
+    0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u,
+};
+
+alignas(256) constexpr std::array<std::uint32_t, 31> SelectedTableCutCode{
+    0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4000400u, 0xfa000060u,
+    0xf4200444u, 0x20000000u, 0x9312b011u, 0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u,
+    0x100202ffu, 0x3c800000u, 0x7e0402ffu, 0x3e000000u, 0x7da400ffu, 0x00001000u, 0xf09c8f08u, 0x01090401u,
+    0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u,
+};
+
 alignas(256) constexpr std::array<std::uint32_t, 22> DirectCode{
     0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, 0xf4080900u, 0xfa000040u, 0x360200bfu, 0x7e020d01u,
     0x060202f0u, 0x100202ffu, 0x3c800000u, 0x7e0402ffu, 0x3e000000u, 0xf09c8f08u, 0x01090401u, 0x7e120202u,
@@ -184,12 +198,25 @@ alignas(256) constexpr std::array<std::uint32_t, 15> StoreTableCode{
     0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e040280u, 0xf0208f08u, 0x00090401u, 0xbf810000u,
 };
 
+alignas(256) constexpr std::array<std::uint32_t, 16> StoreLaneTableCode{
+    0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0x8f108202u, 0xf4200444u, 0x20000000u, 0x9312b011u,
+    0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e040280u, 0x7e080301u, 0xf0208f08u, 0x00090401u, 0xbf810000u,
+};
+
 alignas(256) constexpr std::array<std::uint32_t, 17> StoreTableCutCode{
     0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0x8f108202u, 0xf4200444u, 0x20000000u, 0x9312b011u,
     0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e040280u, 0x7da400ffu, 0x00001000u, 0xf0208f08u, 0x00090401u,
     0xbf810000u,
 };
 
+constexpr std::uint32_t SelectedStoreValue = 0xc0de5eedu;
+
+alignas(256) constexpr std::array<std::uint32_t, 18> SelectedStoreCode{
+    0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4000400u, 0xfa000060u, 0xf4200444u, 0x20000000u,
+    0x9312b011u, 0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e040280u, 0x7e0802ffu, SelectedStoreValue, 0xf0208f08u,
+    0x00090401u, 0xbf810000u,
+};
+
 alignas(256) constexpr std::array<std::uint32_t, 30> PointerCode{
     0xf4080200u, 0xfa000000u, 0xf4080700u, 0xfa000010u, 0xf4080800u, 0xfa000020u, 0x8f108202u, 0xf4200444u,
     0x20000000u, 0x8711ff11u, 0x000000ffu, 0x8f128511u, 0xf40c0900u, 0x24000040u, 0x360200bfu, 0x7e020d01u,
@@ -368,7 +395,7 @@ Outcome Execute(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t>
     outcome.cacheable = AgcDriver::DriverDetail::CacheableResult(result);
     outcome.imageTableReadRanges = result.imageTableReadRanges;
     for (const auto& binding : result.bindings) {
-        if (binding.role == ShaderRecompiler::DescriptorRole::ImageTable) outcome.imageTableCount = binding.count;
+        if (binding.role == ShaderRecompiler::DescriptorRole::ImageTable || binding.role == ShaderRecompiler::DescriptorRole::ImageStorageTable) outcome.imageTableCount = binding.count;
         if (binding.role == ShaderRecompiler::DescriptorRole::GuestSamplers) outcome.samplers = SamplerWords(binding.guestDescriptor);
         if (binding.role == ShaderRecompiler::DescriptorRole::SamplerTable) outcome.samplerTable = SamplerWords(binding.guestDescriptor);
     }
@@ -934,6 +961,43 @@ void RunGuardedTableTests(AgcDriver::VulkanDevice& device) {
     SetClassRecords();
 }
 
+void RunSelectedRecordTests(AgcDriver::VulkanDevice& device) {
+    Fill();
+    SetClassRecords();
+    Keys = {};
+    Srt[24] = 12u;
+    Outcome first;
+    for (const auto record : {2u, 0u, 1u}) {
+        Keys[3] = record;
+        const auto selected = Run(device, SelectedTableCode, 1, Lanes, 64);
+        Require(!Faulted(selected) && selected.imageTableCount == 1u, "image table selected record: record " + std::to_string(record) + " faulted or bound other entries:\n" + selected.log);
+        const auto* words = Table.data() + record * (Stride / 4u);
+        const auto control = RunDirect(device, {words[0], words[1], words[2], words[3]}, {words[4], words[5], words[6], words[7]});
+        Require(Group(selected, 0) == Group(control, 0), "image table selected record: record " + std::to_string(record) + " differs from the direct binding of the same words");
+        if (record == 2u) first = selected;
+        Require(selected.artifactId == first.artifactId && selected.variantId == first.variantId, "image table selected record: another key changed the module");
+    }
+
+    Keys[3] = 1000u;
+    const auto beyond = Run(device, SelectedTableCode, 1, Lanes, 64);
+    Require(!Faulted(beyond) && Zeros(beyond), "image table selected record: a key past the table faulted or did not sample zeros:\n" + beyond.log);
+
+    Keys[3] = 7u;
+    const auto cut = Run(device, SelectedTableCutCode, 1, Lanes, 64);
+    Require(!Faulted(cut), "image table selected record: an invalid record selected with EXEC clear faulted:\n" + cut.log);
+    const auto invalid = Run(device, SelectedTableCode, 1, Lanes, 64);
+    const auto tableAddress = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(Table.data())) + 7u * Stride + 16u;
+    char address[32];
+    std::snprintf(address, sizeof(address), "0x%llx", static_cast<unsigned long long>(tableAddress));
+    Require(Faulted(invalid) && invalid.log.find("not an image") != std::string::npos && invalid.log.find(address) != std::string::npos && invalid.log.find("00001234") != std::string::npos && Zeros(invalid), "image table selected record: an invalid record selected with EXEC set did not report its address, words and reason:\n" + invalid.log);
+
+    Keys[3] = 0u;
+    const auto aliased = Run(device, SelectedTableCode, 1, Lanes, 64, true);
+    Require(Faulted(aliased) && aliased.log.find("is written by its own dispatch") != std::string::npos, "image table selected record: a dispatch storing over its selected record did not fault on use:\n" + aliased.log);
+    SetClassRecords();
+    Srt[24] = 0u;
+}
+
 template <std::size_t TableWords, std::size_t DirectWords>
 void RequireModeEntry(AgcDriver::VulkanDevice& device, const std::array<std::uint32_t, TableWords>& tableCode, const std::array<std::uint32_t, DirectWords>& directCode, std::uint32_t record, std::span<const std::uint32_t> image, const std::string& what) {
     Keys = {};
@@ -988,15 +1052,61 @@ void RunModeTests(AgcDriver::VulkanDevice& device) {
 }
 
 void RunStoreTests(AgcDriver::VulkanDevice& device) {
+    struct Registration {
+        void* data;
+        Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, true); }
+        ~Registration() { GuestAllocations::Mutation().Remove(data); }
+    } registeredUint(UintTexels.data(), sizeof(UintTexels)), registeredFloat(FloatTexels.data(), sizeof(FloatTexels)), registeredSint(SintTexels.data(), sizeof(SintTexels));
     SetClassRecords();
-    const auto saved = UintTexels;
+    const auto savedUint = UintTexels;
+    const auto savedFloat = FloatTexels;
+    const auto savedSint = SintTexels;
+    const auto untouched = [&] { return UintTexels == savedUint && FloatTexels == savedFloat && SintTexels == savedSint; };
     Keys = {};
     Keys[0] = 1;
     const auto cut = Run(device, StoreTableCutCode, 1, Lanes, 64);
+    AgcDriver::Graphics::StorageTexture::FlushAllPending("image table store test");
     Require(!Faulted(cut), "image table store: a store with EXEC clear faulted:\n" + cut.log);
-    const auto stored = Run(device, StoreTableCode, 1, Lanes, 64);
-    Require(Faulted(stored) && stored.log.find("unsupported operation") != std::string::npos, "image table store: an image_store through a table did not fault as an unsupported operation:\n" + stored.log);
-    Require(UintTexels == saved, "image table store: an unsupported store reached the image");
+    Require(untouched(), "image table store: a store with EXEC clear reached an image");
+    const auto stored = Run(device, StoreLaneTableCode, 1, Lanes, 64);
+    AgcDriver::Graphics::StorageTexture::FlushAllPending("image table store test");
+    Require(!Faulted(stored), "image table store: a store through a table faulted:\n" + stored.log);
+    Require(stored.imageTableCount == 2u, "image table store: the storage table does not hold one element per distinct T# (" + std::to_string(stored.imageTableCount) + ")");
+    for (std::uint32_t x = 0; x < Width; ++x) Require(UintTexels[x] == x, "image table store: texel " + std::to_string(x) + " of the selected image is " + Hex(UintTexels[x]) + ", expected " + Hex(x));
+    Require(std::equal(UintTexels.begin() + Width, UintTexels.end(), savedUint.begin() + Width), "image table store: a store to row 0 reached another row");
+    Require(FloatTexels == savedFloat && SintTexels == savedSint, "image table store: a store through key 1 reached another entry's image");
+    UintTexels = savedUint;
+    const auto restored = Run(device, StoreTableCutCode, 1, Lanes, 64);
+    AgcDriver::Graphics::StorageTexture::FlushAllPending("image table store test");
+    Require(!Faulted(restored) && untouched(), "image table store: restoring the image failed:\n" + restored.log);
+    Keys[0] = 6;
+    const auto null = Run(device, StoreLaneTableCode, 1, Lanes, 64);
+    AgcDriver::Graphics::StorageTexture::FlushAllPending("image table store test");
+    Require(untouched(), "image table store: a store through a null entry reached an image:\n" + null.log);
+}
+
+void RunSelectedStoreTests(AgcDriver::VulkanDevice& device) {
+    alignas(256) static std::array<std::uint32_t, Width * Height> storeTexels{};
+    struct Registration {
+        void* data;
+        Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, true); }
+        ~Registration() { GuestAllocations::Mutation().Remove(data); }
+    } registered(storeTexels.data(), sizeof(storeTexels));
+    for (std::uint32_t texel = 0; texel < storeTexels.size(); ++texel) storeTexels[texel] = 0x5000u + texel;
+    const auto saved = storeTexels;
+    SetClassRecords();
+    SetRecord(9, PointClamp, Image(storeTexels.data(), Format32UInt));
+    Keys = {};
+    Keys[3] = 9u;
+    Srt[24] = 12u;
+    const auto stored = Run(device, SelectedStoreCode, 1, Lanes, 64);
+    AgcDriver::Graphics::StorageTexture::FlushPending(reinterpret_cast<std::uintptr_t>(storeTexels.data()), sizeof(storeTexels), nullptr, "test");
+    device.WaitIdle();
+    Require(!Faulted(stored) && stored.imageTableCount == 0u, "image table selected store: an image_store through a T# whose record the CPU evaluates faulted or was bound through an image table:\n" + stored.log);
+    for (std::uint32_t texel = 0; texel < storeTexels.size(); ++texel) {
+        const auto expected = texel < Lanes ? SelectedStoreValue : saved[texel];
+        Require(storeTexels[texel] == expected, "image table selected store: texel " + std::to_string(texel) + " holds " + std::to_string(storeTexels[texel]) + " after an image_store through a T# whose record the CPU evaluates, expected " + std::to_string(expected));
+    }
 }
 
 void RunPointerTests(AgcDriver::VulkanDevice& device) {
@@ -1325,8 +1435,10 @@ int main() {
         RunReadTests(*device);
         RunLoopKeyTests(*device);
         RunGuardedTableTests(*device);
+        RunSelectedRecordTests(*device);
         RunModeTests(*device);
         RunStoreTests(*device);
+        RunSelectedStoreTests(*device);
         RunPointerTests(*device);
         RunPointerRecordTests(*device);
         RunSplitRecordTests(*device);

core/libs/prx/libSceAgcDriver/tests/execution/ImageXorSwizzleVolume.cpp

@@ -27,6 +27,7 @@ constexpr std::uint32_t Height = 24;
 constexpr std::uint32_t Depth = 20;
 constexpr std::uint32_t Format8888UNorm = 56;
 constexpr std::uint32_t Type3D = 10;
+constexpr std::uint32_t TileS4KBX = 0x15;
 constexpr std::uint32_t TileZ64KBX = 0x18;
 constexpr std::uint32_t TileS64KBX = 0x19;
 constexpr std::uint32_t TileD64KBX = 0x1a;
@@ -79,6 +80,7 @@ void FillTexels(std::uint32_t tileMode) {
     Require(geometry.guestBytes <= Texels.size(), "volume does not fit the test allocation");
     const auto* equation = FindTextureSwizzleEquation(geometry.thick ? 0x100u | XorSwizzleMode(descriptor.tileMode) : XorSwizzleMode(descriptor.tileMode), 4u);
     Require(equation != nullptr, "volume swizzle equation is missing");
+    const auto blockBytes = ThinBlockLayout(descriptor.tileMode, 4u)[0];
     const auto& mip = geometry.mips.at(0);
     const auto block = geometry.thick ? ThickBlockExtent(descriptor.tileMode, 4u) : std::array<std::uint32_t, 3>{ThinBlockLayout(descriptor.tileMode, 4u)[1], ThinBlockLayout(descriptor.tileMode, 4u)[2], 1u};
     for (std::uint32_t z = 0; z < Depth; ++z) {
@@ -86,7 +88,7 @@ void FillTexels(std::uint32_t tileMode) {
             for (std::uint32_t x = 0; x < Width; ++x) {
                 auto offset = geometry.GuestLayerOffset(z) + mip.tiledOffset;
                 if (mip.tail) offset += EquationOffset(*equation, x + mip.tailX, y + mip.tailY, z);
-                else offset += (static_cast<std::uint64_t>(y / block[1]) * mip.blocksPerRow + x / block[0]) * 65536u + EquationOffset(*equation, x, y, z);
+                else offset += (static_cast<std::uint64_t>(y / block[1]) * mip.blocksPerRow + x / block[0]) * blockBytes + EquationOffset(*equation, x, y, z);
                 const auto texel = Texel(x, y, z);
                 std::copy(texel.begin(), texel.end(), Texels.begin() + static_cast<std::ptrdiff_t>(offset));
             }
@@ -145,6 +147,7 @@ int main() {
         const auto device = OpenVulkanTestDevice();
         if (!device) return VulkanTestSkipped;
         FillInput();
+        Run(*device, TileS4KBX, "thick SW_4KB_S_X volume");
         Run(*device, TileS64KBX, "thick SW_64KB_S_X volume");
         Run(*device, TileD64KBX, "thick SW_64KB_D_X volume");
         Run(*device, TileZ64KBX, "thin SW_64KB_Z_X volume");

core/libs/prx/libSceAgcDriver/tests/execution/InterpolationF16.cpp

@@ -160,10 +160,14 @@ void Draw(AgcDriver::VulkanDevice& device, std::uint32_t waveSize) {
     const auto vertexResult = ShaderRecompiler::Recompile(vertex);
     const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
     const auto pixelResult = RecompilePixel(PixelCode, waveSize, Fp16Input, target, vertexPush);
-    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
-        {ShaderStage::Vertex, &vertexResult, 0},
-        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
-    }};
+    Require(pixelResult.barycentricEmulation.active != target.fragmentShaderBarycentricEnabled, "16-bit interpolation must emulate barycentrics exactly when the device lacks them");
+    std::vector<AgcDriver::Graphics::CompiledShader> shaders{{ShaderStage::Vertex, &vertexResult, 0}};
+    ShaderRecompiler::RecompileResult geometry;
+    if (pixelResult.barycentricEmulation.active) {
+        geometry = ShaderRecompiler::BuildBarycentricGeometryShader(vertexResult, pixelResult, target, device.GeometryLimits());
+        shaders.push_back({ShaderStage::Geometry, &geometry, 0});
+    }
+    shaders.push_back({ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)});
 
     AgcDriver::Graphics::State state{};
     state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
@@ -259,6 +263,8 @@ ShaderRecompiler::SpirvTarget OfflineTarget(bool barycentric) {
     target.spirvVersion = 0x00010300u;
     target.subgroupSize = 64;
     target.fragmentShaderBarycentricEnabled = barycentric;
+    static constexpr std::array<std::uint32_t, 1> capabilities{spv::CapabilityFloat64};
+    target.supportedCapabilities = capabilities;
     return target;
 }
 
@@ -281,7 +287,8 @@ void ExpectRejected(std::uint32_t word0, std::uint32_t word1, const char* expect
 
 int main() {
     try {
-        ExpectFailure(PixelCode, Fp16Input, false, "requires fragmentShaderBarycentric", "16-bit interpolation was accepted without barycentrics");
+        Require(RecompilePixel(PixelCode, 64u, Fp16Input, OfflineTarget(false), 0u).barycentricEmulation.active, "16-bit interpolation without barycentrics does not emulate them");
+        Require(!RecompilePixel(PixelCode, 64u, Fp16Input, OfflineTarget(true), 0u).barycentricEmulation.active, "16-bit interpolation emulates barycentrics on a device that has them");
         ExpectFailure(PixelCode, 0x03000000u, true, "without FP16_INTERP_MODE", "16-bit interpolation of a 32-bit input was accepted");
         ExpectFailure(PixelCode, 0x03080420u, true, "passes its vertices through unchanged", "16-bit interpolation of a pass-through input was accepted");
         ExpectRejected(0xd75a001bu, 0x0c6a0200u, "interpolation modifiers", "v_interp_p2_f16 accepted an output modifier");
@@ -291,12 +298,18 @@ int main() {
         ExpectRejected(0xd743001du, 0x003e0140u, "not a vector register", "v_interp_p1lv_f16 accepted a scalar P0");
         const auto device = OpenVulkanTestDevice();
         if (!device) return VulkanTestSkipped;
-        if (!device->Target().fragmentShaderBarycentricEnabled) {
-            std::puts("skipped, the device has no fragmentShaderBarycentric");
+        if (!device->Target().fragmentShaderBarycentricEnabled && !device->GeometryLimits()) {
+            std::puts("skipped, the device has neither fragmentShaderBarycentric nor geometryShader");
             return VulkanTestSkipped;
         }
         for (const auto waveSize : {64u, 32u}) {
-            Draw(*device, waveSize);
+            try {
+                Draw(*device, waveSize);
+            } catch (const std::exception& error) {
+                if (std::string(error.what()).find("need the Float64 capability") == std::string::npos) throw;
+                std::puts("skipped, the device has no Float64 capability for the exact interpolation sums");
+                return VulkanTestSkipped;
+            }
             Check(waveSize);
         }
         std::puts("16-bit interpolation tests passed");

core/libs/prx/libSceAgcDriver/tests/execution/JobManagerCompute.cpp

@@ -0,0 +1,254 @@
+#include "VulkanTestDevice.hpp"
+#include "SceTypes.hpp"
+#include <array>
+#include <atomic>
+#include <chrono>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <new>
+#include <stdexcept>
+#include <string>
+#include <thread>
+
+extern "C" void AgcDriverShutdown_nid_postfix();
+
+struct MemorySizeQueryParams {
+    std::uint32_t version;
+    std::uint32_t workers;
+    std::uint32_t sequences;
+    std::uint32_t invocations;
+    std::uint32_t dispatches;
+    std::uint32_t syncPoints;
+    std::uint32_t waits;
+    std::uint32_t computeSequences;
+    std::uint64_t computeQueueMask;
+};
+
+struct ConfigParams {
+    std::uint32_t version;
+    std::uint32_t threadFlags;
+    std::uint32_t stackSize;
+    std::uint32_t priority;
+    const char* threadName;
+    std::uint32_t spinMicroseconds;
+    std::uint32_t computeAffinity;
+    std::uint32_t computePriority;
+    std::uint32_t reserved;
+};
+
+using JobEntry = void (APS5_VABI*)(void*, std::uint32_t);
+
+struct Sequence {
+    APS5_VABI virtual int Dispatch(JobEntry, void*, std::uint32_t, const char*) = 0;
+    APS5_VABI virtual int SyncPoint() = 0;
+    APS5_VABI virtual int Wait() = 0;
+    APS5_VABI virtual void CompleteDestructor() = 0;
+    APS5_VABI virtual void DeletingDestructor() = 0;
+    APS5_VABI virtual int DispatchCompute(const void*, void*, std::int32_t, std::int32_t, std::int32_t, const char*) = 0;
+    APS5_VABI virtual int DispatchComputeIndirect(const void*, void*, const void*, const char*) = 0;
+};
+
+struct SequenceFactory {
+    APS5_VABI virtual int Create(Sequence**, std::uint32_t, std::uint32_t, std::uint32_t, std::uint32_t, const char*) = 0;
+    APS5_VABI virtual int Destroy(Sequence*) = 0;
+    APS5_VABI virtual void CompleteDestructor() = 0;
+    APS5_VABI virtual void DeletingDestructor() = 0;
+    APS5_VABI virtual int CreateCompute(Sequence**, std::uint32_t, std::uint32_t, std::uint32_t, std::uint32_t, const char*) = 0;
+    APS5_VABI virtual int DestroyCompute(Sequence*) = 0;
+};
+
+extern "C" {
+void APS5_VABI _ZN3sce3Job10JobManagerC1Ev(void*);
+std::uint64_t APS5_VABI _ZN3sce3Job10JobManager27calculateRequiredMemorySizeEPKNS1_21MemorySizeQueryParamsE(const MemorySizeQueryParams*);
+int APS5_VABI _ZN3sce3Job10JobManager10initializeEPvmjjjmPKNS1_12ConfigParamsE(void*, void*, std::uint64_t, std::uint32_t, std::uint32_t, std::uint32_t, std::uint64_t, const ConfigParams*);
+int APS5_VABI _ZN3sce3Job10JobManager8shutdownEv(void*);
+SequenceFactory* APS5_VABI _ZN3sce3Job10JobManager34getUnifiedSequenceFactoryInterfaceENS1_8PriorityE(void*, std::uint32_t);
+}
+
+#define Construct _ZN3sce3Job10JobManagerC1Ev
+#define RequiredSize _ZN3sce3Job10JobManager27calculateRequiredMemorySizeEPKNS1_21MemorySizeQueryParamsE
+#define Initialize _ZN3sce3Job10JobManager10initializeEPvmjjjmPKNS1_12ConfigParamsE
+#define Shutdown _ZN3sce3Job10JobManager8shutdownEv
+#define Factories _ZN3sce3Job10JobManager34getUnifiedSequenceFactoryInterfaceENS1_8PriorityE
+
+namespace {
+
+constexpr int InvalidArgument = static_cast<int>(0x81581000u);
+constexpr int OutOfRange = static_cast<int>(0x81580000u);
+constexpr int NullArgument = static_cast<int>(0x81580001u);
+constexpr int NoNodes = static_cast<int>(0x81580002u);
+constexpr int LimitExceeded = static_cast<int>(0x81580003u);
+constexpr int NoneFree = static_cast<int>(0x81580006u);
+constexpr int Invalid = static_cast<int>(0x81580007u);
+
+void Require(bool condition, const std::string& message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+alignas(256) std::array<std::uint32_t, 12> ReaderCode{0xf4040100u, 0xfa000000u, 0xf4080201u, 0xfa000000u, 0xbf8cc07fu, 0xf4000182u, 0xfa000000u, 0xbf8cc07fu, 0x7e000206u, 0xe0700000u, 0x80020000u, 0xbf810000u};
+alignas(256) std::array<std::uint32_t, 4> Value{};
+alignas(256) std::array<std::uint32_t, 2> Table{};
+alignas(256) volatile std::uint32_t Output[4]{};
+alignas(256) std::array<std::uint32_t, 4> OutputDescriptor{};
+alignas(32) std::array<std::uint32_t, 8> IndirectArguments{1, 1, 1, 0, 0, 0, 0, 0};
+alignas(16) std::array<std::uint32_t, 8> Specials{};
+alignas(16) std::array<std::uint32_t, 8> SpecialsWithoutDispatch{};
+std::array<ShaderRegister, 9> Registers{};
+alignas(16) std::array<unsigned char, 0x80> Shader{};
+alignas(16) std::array<unsigned char, 0x80> ShaderWithoutDispatch{};
+
+std::uint32_t Low(const volatile void* pointer) {
+    return static_cast<std::uint32_t>(reinterpret_cast<std::uintptr_t>(pointer));
+}
+
+std::uint32_t High(const volatile void* pointer) {
+    return static_cast<std::uint32_t>(static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(pointer)) >> 32u);
+}
+
+void BuildShader() {
+    Table = {Low(Value.data()), High(Value.data())};
+    OutputDescriptor = {Low(Output), High(Output) & 0xffffu, sizeof(Output), 0x31016facu};
+    const auto code = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(ReaderCode.data()));
+    Registers = {{{0x20c, static_cast<std::uint32_t>(code >> 8u)}, {0x20d, static_cast<std::uint32_t>(code >> 40u) & 0xffu}, {0x207, 1}, {0x208, 1}, {0x209, 1}, {0x212, 0}, {0x213, 4u << 1u},
+                  {0x242, Low(OutputDescriptor.data())}, {0x243, High(OutputDescriptor.data())}}};
+    Specials[4] = 1;
+    const ShaderRegister* registers = Registers.data();
+    for (auto* shader : {&Shader, &ShaderWithoutDispatch}) {
+        const std::uint32_t* specials = shader == &Shader ? Specials.data() : SpecialsWithoutDispatch.data();
+        std::memcpy(shader->data() + 0x20, &registers, sizeof(registers));
+        std::memcpy(shader->data() + 0x28, &specials, sizeof(specials));
+        (*shader)[0x5c] = static_cast<unsigned char>(Registers.size());
+    }
+}
+
+std::uint64_t SizeFor(std::uint32_t computeSequences, std::uint64_t mask, bool limits) {
+    const MemorySizeQueryParams query{1, 2, 1, limits ? 100u : 0u, limits ? 8u : 0u, limits ? 4u : 0u, limits ? 2u : 0u, computeSequences, mask};
+    return RequiredSize(&query);
+}
+
+std::atomic<bool> Released{false};
+std::atomic<std::uint32_t> SeenByJob{0};
+
+void APS5_VABI BlockingJob(void*, std::uint32_t) {
+    while (!Released) std::this_thread::yield();
+}
+
+void APS5_VABI ReadOutputJob(void*, std::uint32_t) {
+    SeenByJob = Output[0];
+}
+
+void APS5_VABI Nothing(void*, std::uint32_t) {
+}
+
+void SizeCase() {
+    Require(SizeFor(0, 0, false) == 0x10d0 && SizeFor(0, 0, true) == 0x14c0, "size without compute sequences");
+    Require(SizeFor(1, 1, false) == 0x24fd8 && SizeFor(1, 1, true) == 0x253c8, "size with one compute sequence");
+    Require(SizeFor(2, 3, false) == 0x48e20 && SizeFor(2, 3, true) == 0x49210, "size with two compute sequences");
+    Require(SizeFor(3, 7, false) == 0x6cb68, "size with three compute sequences");
+    Require(SizeFor(1, 3, true) == 0x491c8, "size with two queues for one compute sequence");
+    Require(SizeFor(2, 1, true) == 0, "size with fewer queues than compute sequences");
+}
+
+void RunCase() {
+    const std::uint64_t size = SizeFor(1, 2, true);
+    void* memory = ::operator new((size + 255) & ~std::uint64_t{255}, std::align_val_t{256});
+    Require(memory != nullptr, "allocation failed");
+    alignas(64) static unsigned char manager[0x100];
+    Construct(manager);
+    const ConfigParams badHandler{1, 0x00021fff, 0x00010000, 0x2bc, nullptr, 0, 0x1fff, 0x50, 0};
+    const ConfigParams noAffinity{1, 0x00021fff, 0x00010000, 0x2bc, nullptr, 0, 0, 0x100, 0};
+    const ConfigParams wideAffinity{1, 0x00021fff, 0x00010000, 0x2bc, nullptr, 0, 0x2000, 0x100, 0};
+    Require(Initialize(manager, memory, size, 2, 1, 1, 2, &badHandler) == InvalidArgument, "a handler priority below 0x100 was accepted");
+    Require(Initialize(manager, memory, size, 2, 1, 1, 2, &noAffinity) == InvalidArgument, "an empty handler affinity was accepted");
+    Require(Initialize(manager, memory, size, 2, 1, 1, 2, &wideAffinity) == InvalidArgument, "a handler affinity beyond 13 cores was accepted");
+    Require(Initialize(manager, memory, size, 2, 1, 0xffffffffu, 2, nullptr) == InvalidArgument, "a sequence count that wraps was accepted");
+    Require(Initialize(manager, memory, size, 2, 1, 1, 2, nullptr) == 0, "initialize with one compute sequence failed");
+    SequenceFactory* factory = Factories(manager, 0);
+    Sequence* sequence = nullptr;
+    Sequence* other = nullptr;
+    Require(factory->CreateCompute(&sequence, 100, 4, 4, 2, "compute") == 0, "the compute sequence was not created");
+    Require(factory->CreateCompute(&other, 1, 1, 1, 1, "second") == NoneFree, "a second compute sequence was created");
+    Require(factory->Create(&other, 1, 1, 1, 1, "cpu") == 0, "the CPU sequence was not created");
+    Require(factory->DestroyCompute(other) == Invalid, "a CPU sequence was destroyed as a compute sequence");
+    Require(factory->Destroy(sequence) == Invalid, "a compute sequence was destroyed as a CPU sequence");
+    Require(factory->Destroy(other) == 0, "the CPU sequence was not destroyed");
+
+    Require(sequence->DispatchCompute(nullptr, Table.data(), 1, 1, 1, nullptr) == NullArgument, "a null shader was accepted");
+    Require(sequence->DispatchCompute(Shader.data(), Table.data(), 0, 1, 1, nullptr) == OutOfRange, "a zero group count was accepted");
+    Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, -1, nullptr) == OutOfRange, "a negative group count was accepted");
+    Require(sequence->DispatchComputeIndirect(Shader.data(), Table.data(), nullptr, nullptr) == NullArgument, "null indirect arguments were accepted");
+    Require(sequence->DispatchComputeIndirect(Shader.data(), Table.data(), reinterpret_cast<const char*>(IndirectArguments.data()) + 4, nullptr) == NullArgument, "misaligned indirect arguments were accepted");
+
+    Value[0] = 0x11110001u;
+    Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, "direct") == 0, "the compute dispatch failed");
+    Require(sequence->Wait() == 0, "the wait after a compute dispatch failed");
+    Require(Output[0] == 0x11110001u, "the compute dispatch had not run when the wait returned: " + std::to_string(Output[0]));
+
+    Value[0] = 0x22220002u;
+    Require(sequence->DispatchComputeIndirect(Shader.data(), Table.data(), IndirectArguments.data(), nullptr) == 0, "the indirect compute dispatch failed");
+    Require(sequence->Wait() == 0, "the wait after an indirect compute dispatch failed");
+    Require(Output[0] == 0x22220002u, "the indirect compute dispatch had not run when the wait returned: " + std::to_string(Output[0]));
+
+    Value[0] = 0x33330003u;
+    Require(sequence->DispatchCompute(ShaderWithoutDispatch.data(), Table.data(), 1, 1, 1, nullptr) == 0, "the dispatch of a shader without the dispatch bit failed");
+    Require(sequence->Wait() == 0, "the wait after a shader without the dispatch bit failed");
+    Require(Output[0] == 0x22220002u, "a shader without the dispatch bit ran");
+
+    Require(sequence->Dispatch(BlockingJob, nullptr, 1, "blocking") == 0, "the blocking job was not dispatched");
+    Require(sequence->SyncPoint() == 0, "the sync point failed");
+    Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, nullptr) == 0, "the compute dispatch behind a sync point failed");
+    std::this_thread::sleep_for(std::chrono::milliseconds(200));
+    Require(Output[0] == 0x22220002u, "a compute dispatch ran before the CPU job ahead of its sync point finished");
+    Released = true;
+    Require(sequence->SyncPoint() == 0, "the second sync point failed");
+    Require(sequence->Dispatch(ReadOutputJob, nullptr, 1, "reader") == 0, "the reading job was not dispatched");
+    Require(sequence->Wait() == 0, "the wait after the ordered dispatches failed");
+    Require(Output[0] == 0x33330003u, "the compute dispatch behind a sync point did not run");
+    Require(SeenByJob == 0x33330003u, "a CPU job behind a sync point ran before the compute dispatch ahead of it finished");
+
+    Value[0] = 0x44440004u;
+    Require(sequence->DispatchCompute(ShaderWithoutDispatch.data(), Table.data(), 1, 1, 1, nullptr) == 0, "the leading dispatch failed");
+    Require(sequence->SyncPoint() == 0, "the sync point between compute dispatches failed");
+    Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, "behind") == 0, "the compute dispatch behind a compute dispatch failed");
+    Require(sequence->Wait() == 0, "the wait after chained compute dispatches failed");
+    Require(Output[0] == 0x44440004u, "a compute dispatch behind a sync point and a compute dispatch did not run");
+
+    for (int index = 0; index < 4; ++index) Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, nullptr) == 0, "a compute dispatch within the limit failed");
+    Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, nullptr) == LimitExceeded, "a compute dispatch past the dispatch limit was accepted");
+    Require(sequence->DispatchCompute(Shader.data(), Table.data(), 1, 1, 1, nullptr) == Invalid, "a latched compute sequence accepted a dispatch");
+    Require(factory->DestroyCompute(sequence) == 0, "the compute sequence was not destroyed");
+
+    Require(factory->Create(&other, 0x7fff, 0x3ff, 0xff, 1, "nodes") == 0, "the node counting sequence was not created");
+    int accepted = 0;
+    int result = 0;
+    while (accepted < 0x3ff && (result = other->Dispatch(Nothing, nullptr, 1, nullptr)) == 0) ++accepted;
+    Require(result == NoNodes && accepted == 43, "the node pool of a compute configuration holds " + std::to_string(accepted) + " dispatches");
+    Require(Shutdown(manager) == 0, "shutdown failed");
+    ::operator delete(memory, std::align_val_t{256});
+}
+
+}
+
+int main() {
+    try {
+        auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        device.reset();
+        BuildShader();
+        SizeCase();
+        RunCase();
+        AgcDriverShutdown_nid_postfix();
+        std::cout << "job manager compute test passed\n";
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        try {
+            AgcDriverShutdown_nid_postfix();
+        } catch (const std::exception& shutdownError) {
+            std::cerr << shutdownError.what() << '\n';
+        }
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/LayeredVolumeDraw.cpp

@@ -97,9 +97,9 @@ void BuildBands() {
     }
 }
 
-AgcDriver::Graphics::ColorTarget VolumeTarget(std::byte* volume) {
+AgcDriver::Graphics::ColorTarget VolumeTarget(std::byte* volume, ColorTileMode tileMode = ColorTileMode::RenderTarget) {
     AgcDriver::Graphics::ColorTarget color{reinterpret_cast<std::uintptr_t>(volume), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, 0, 0xe4u};
-    color.tileMode = ColorTileMode::RenderTarget;
+    color.tileMode = tileMode;
     color.depth = Slices;
     color.depthSlice = FirstSlice;
     color.layers = Layers;
@@ -239,19 +239,23 @@ std::uint64_t BlockOffset(const AgcDriver::Graphics::TextureSwizzleEquation& equ
 }
 
 void Check(const std::byte* volume, const AgcDriver::Graphics::ColorTarget& color, const std::string& what, const std::array<std::uint8_t, Layers>& bands) {
+    using namespace AgcDriver::Graphics;
     const auto geometry = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::SurfaceForTarget(color));
-    Require(!geometry.thick, "a 64 KiB render-target volume is not stored as thin slices");
-    const auto block = AgcDriver::Graphics::ThinBlockLayout(AgcDriver::Graphics::TextureTileMode::kR64KBX, 4);
-    Require(Width <= block[1] && Height <= block[2], "the volume slices span several tile blocks");
-    const auto* equation = AgcDriver::Graphics::FindTextureSwizzleEquation(static_cast<std::uint32_t>(ColorTileMode::RenderTarget), 4);
-    Require(equation != nullptr, "missing the 64 KiB R_X swizzle equation");
+    Require(geometry.thick == (color.tileMode == ColorTileMode::S4KBX), "unexpected volume addressing family");
+    const auto tileMode = ColorTextureTileMode(color.tileMode);
+    const auto thin = ThinBlockLayout(tileMode, 4);
+    const auto block = geometry.thick ? ThickBlockExtent(tileMode, 4) : std::array<std::uint32_t, 3>{thin[1], thin[2], 1u};
+    const auto& mip = geometry.mips.at(0);
+    const auto* equation = FindTextureSwizzleEquation((geometry.thick ? 0x100u : 0u) | static_cast<std::uint32_t>(color.tileMode), 4);
+    Require(equation != nullptr, "missing the volume's swizzle equation");
     for (std::uint32_t slice = 0; slice < Slices; ++slice) {
         const auto* base = volume + geometry.GuestLayerOffset(slice);
         for (std::uint32_t y = 0; y < Height; ++y) {
             for (std::uint32_t x = 0; x < Width; ++x) {
                 const bool drawn = slice >= FirstSlice && x / BandWidth == slice - FirstSlice;
                 const auto expected = drawn ? bands[slice - FirstSlice] : Kept;
-                const auto* texel = base + BlockOffset(*equation, x, y, slice);
+                const auto offset = (static_cast<std::uint64_t>(y / block[1]) * mip.blocksPerRow + x / block[0]) * thin[0] + BlockOffset(*equation, x, y, slice);
+                const auto* texel = base + offset;
                 for (std::uint32_t channel = 0; channel < 4; ++channel) {
                     const auto actual = std::to_integer<std::uint8_t>(texel[channel]);
                     Require(actual == expected || (drawn && expected == 128 && actual == 127), what + ": slice " + std::to_string(slice) + " texel (" + std::to_string(x) + ", " + std::to_string(y) + ") channel " + std::to_string(channel) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected) + (drawn ? " (the band of layer " + std::to_string(slice - FirstSlice) + ")" : slice < FirstSlice ? " (outside the color view)" : " (another layer's band)"));
@@ -283,6 +287,10 @@ int main(int argc, char** argv) {
         const auto fieldColor = VolumeTarget(fieldVolume.Data());
         Run(*device, fieldColor, false, LayerFieldPixelCode, true);
         Check(fieldVolume.Data(), fieldColor, "a pixel shader reading the render target index of its packed ancillary word", LayerFieldBands);
+        const GuestVolume thickVolume((VolumeTarget(nullptr, ColorTileMode::S4KBX).bytes + 65535u) & ~std::size_t{65535u});
+        const auto thickColor = VolumeTarget(thickVolume.Data(), ColorTileMode::S4KBX);
+        Run(*device, thickColor, false, PixelCode, false);
+        Check(thickVolume.Data(), thickColor, "thick SW_4KB_S_X vertex draw and writeback", WhiteBands);
         if (vertexOnly) std::puts("layered volume vertex draw tests passed");
         else if (target.mesh.has_value()) Run(*device, color, true, PixelCode, false);
         else std::puts("mesh case skipped, the device has no VK_EXT_mesh_shader");

core/libs/prx/libSceAgcDriver/tests/execution/LdsMixedWidthAtomics.cpp

@@ -6,9 +6,13 @@
 #include <array>
 #include <cstdint>
 #include <cstdio>
+#include <cstdlib>
 #include <iostream>
+#include <optional>
 #include <span>
 #include <string>
+#include <string_view>
+#include <unordered_map>
 #include <vector>
 
 namespace {
@@ -182,14 +186,41 @@ void Check(const char* name, std::uint64_t operations, std::uint32_t run) {
     }
 }
 
-void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const char* name, std::uint64_t operations, std::uint32_t repeats) {
+std::uint64_t WorkgroupBytes(std::span<const std::uint32_t> words) {
+    std::unordered_map<std::uint32_t, std::span<const std::uint32_t>> types;
+    std::unordered_map<std::uint32_t, std::uint32_t> constants;
+    std::vector<std::uint32_t> pointers;
+    for (std::size_t at = 5; at < words.size();) {
+        const auto count = words[at] >> 16u;
+        Require(count != 0u && count <= words.size() - at, "lds lock budget: malformed SPIR-V");
+        const auto instruction = words.subspan(at, count);
+        const auto opcode = instruction[0] & 0xffffu;
+        if (opcode == spv::OpTypeInt || opcode == spv::OpTypeArray || opcode == spv::OpTypePointer) types[instruction[1]] = instruction;
+        if (opcode == spv::OpConstant) constants[instruction[2]] = instruction[3];
+        if (opcode == spv::OpVariable && instruction[3] == spv::StorageClassWorkgroup) pointers.push_back(instruction[1]);
+        at += count;
+    }
+    std::uint64_t total = 0;
+    for (const auto pointer : pointers) {
+        const auto definition = types.at(pointer);
+        Require((definition[0] & 0xffffu) == spv::OpTypePointer, "lds lock budget: Workgroup storage has no pointer type");
+        const auto array = types.at(definition[3]);
+        Require((array[0] & 0xffffu) == spv::OpTypeArray, "lds lock budget: unexpected Workgroup allocation shape");
+        const auto element = types.at(array[2]);
+        Require((element[0] & 0xffffu) == spv::OpTypeInt && element[2] == 32u, "lds lock budget: Workgroup array is not dwords");
+        total += static_cast<std::uint64_t>(constants.at(array[3])) * sizeof(std::uint32_t);
+    }
+    return total;
+}
+
+void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const char* name, std::uint64_t operations, std::uint32_t repeats, std::uint32_t ldsDwords = LdsDwords, std::optional<std::uint32_t> expectedDeviceDwords = std::nullopt) {
     std::vector<std::uint32_t> userData(8, 0u);
     const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
     const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
     std::copy(input.begin(), input.end(), userData.begin());
     std::copy(output.begin(), output.end(), userData.begin() + 4);
     const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
-    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, LdsDwords, {false, false, false}, false, 1};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, ldsDwords, {false, false, false}, false, 1};
     ShaderRecompiler::RecompileRequest request{
         {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
         {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
@@ -198,20 +229,46 @@ void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, c
     };
     request.useCache = false;
     const auto result = ShaderRecompiler::Recompile(request);
+    if (expectedDeviceDwords) {
+        const auto bytes = WorkgroupBytes(result.spirv.Words());
+        const auto limit = request.target.maxWorkgroupSharedMemoryBytes;
+        Require(bytes <= limit, "lds lock budget: guest LDS and lock exceed the device limit before dispatch");
+        Require(result.workgroupMemoryDwords == *expectedDeviceDwords, "lds lock budget: unexpected device-backed LDS allocation before dispatch");
+        if (*expectedDeviceDwords == 0u) Require(bytes == limit, "lds lock budget: exact-fit case did not retain its lock");
+        std::printf("lds lock budget: %s declarations verified; guest=%u; workgroup=%llu; device=%u\n", name, ldsDwords * 4u, static_cast<unsigned long long>(bytes), result.workgroupMemoryDwords);
+    }
     for (std::uint32_t run = 0; run < repeats; ++run) {
         Output.fill(0xdeadbeefu);
         device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
         device.WaitIdle();
         Check(name, operations, run);
     }
+    if (expectedDeviceDwords) std::printf("lds lock budget: %s GPU readback verified across 256 lanes\n", name);
+}
+
+void LockBudget(AgcDriver::VulkanDevice& device) {
+    const auto target = device.Target();
+    const auto limit = target.maxWorkgroupSharedMemoryBytes;
+    std::printf("lds lock budget: robustBufferAccess2=%u; maxWorkgroupSharedMemoryBytes=%u\n", static_cast<unsigned>(target.robustBufferAccess2), limit);
+    if (std::getenv("ANYPS5_REQUIRE_ROBUST2") != nullptr) Require(target.robustBufferAccess2, "lds lock budget: robustBufferAccess2 was not enabled");
+    Require(limit >= 32u && limit % 4u == 0u, "lds lock budget: unexpected device shared-memory limit");
+    const auto operations = static_cast<std::uint64_t>(Threads) * Iterations;
+    Run(device, UniformCode, "limit", operations, 1, limit / 4u, limit / 4u + 1u);
+    Run(device, UniformCode, "limit-4", operations, 1, (limit - 4u) / 4u, 0u);
+    std::puts("lds lock budget tests passed");
 }
 
 }
 
-int main() {
+int main(int argc, char** argv) {
     try {
         const auto device = OpenVulkanTestDevice();
         if (!device) return VulkanTestSkipped;
+        if (argc == 2 && std::string_view(argv[1]) == "--lock-budget") {
+            LockBudget(*device);
+            return 0;
+        }
+        Require(argc == 1, "lds mixed width atomics: unexpected argument");
         Run(*device, UniformCode, "uniform", static_cast<std::uint64_t>(Threads) * Iterations, 1);
         Run(*device, DivergentCode, "divergent", static_cast<std::uint64_t>(Threads) / 2u * Iterations, 1);
         Run(*device, ParityCode, "split lanes", Threads / 2u, Repeats);

core/libs/prx/libSceAgcDriver/tests/execution/MeshDrawRelocation.cpp

@@ -0,0 +1,152 @@
+#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
+#include "../MeshGeometryFixture.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <exception>
+#include <iostream>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using AgcDriver::DriverDetail::DrawMiss;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 64;
+alignas(256) std::array<std::byte, Width * Height * 4> Pixels{};
+alignas(256) constexpr std::array<std::uint16_t, 4> Indices{0, 1, 2, 0};
+alignas(256) constexpr std::array<std::array<float, 8>, 3> Vertices{{
+    {-0.9f, -0.8f, 0, 1, 1, 1, 1, 1}, {-0.1f, -0.8f, 0, 1, 1, 1, 1, 1}, {-0.5f, 0.8f, 0, 1, 1, 1, 1, 1}
+}};
+alignas(256) constexpr std::array<std::uint32_t, 4> PixelCode{0x7e0e02f2u, 0xf800180fu, 0x07070707u, 0xbf810000u};
+
+constexpr std::array<std::uint32_t, 3> LoadBufferDescriptor{0xf4080204u, 0xfa000000u, 0xbf8cc07fu};
+alignas(256) constexpr auto GeometryCode = [] {
+    std::array<std::uint32_t, LoadBufferDescriptor.size() + AgcDriver::Test::MeshGeometryCode.size()> code{};
+    std::copy(LoadBufferDescriptor.begin(), LoadBufferDescriptor.end(), code.begin());
+    std::copy(AgcDriver::Test::MeshGeometryCode.begin(), AgcDriver::Test::MeshGeometryCode.end(), code.begin() + LoadBufferDescriptor.size());
+    return code;
+}();
+
+alignas(16) std::array<std::array<std::uint32_t, 4>, 8> Descriptors{};
+
+struct Header {
+    Shader shader{};
+    ShaderUserData users{};
+};
+
+Header GeometryHeader{};
+Header PixelHeader{};
+
+void Register(Header& header, const std::uint32_t* code, std::size_t size, std::uint8_t type) {
+    header.shader.file_header = 0x34333231u;
+    header.shader.version = 0x18u;
+    header.shader.header_size = sizeof(header);
+    header.shader.shader_size = static_cast<std::uint32_t>(size);
+    header.shader.code = code;
+    header.shader.type = type;
+    header.shader.user_data = &header.users;
+    AgcDriverRegisterShader_nid_postfix(&header.shader);
+}
+
+void Write(std::vector<std::uint32_t>& commands, std::uint32_t opcode, std::uint32_t offset, std::uint32_t value) {
+    commands.insert(commands.end(), {0xc0010000u | (opcode << 8u), offset, value});
+}
+
+std::vector<std::uint32_t> Setup(const std::array<std::uint32_t, 4>& descriptor) {
+    const auto color = reinterpret_cast<std::uintptr_t>(Pixels.data());
+    const auto geometry = reinterpret_cast<std::uintptr_t>(GeometryCode.data());
+    const auto pixel = reinterpret_cast<std::uintptr_t>(PixelCode.data());
+    const auto table = reinterpret_cast<std::uintptr_t>(descriptor.data());
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> context{
+        {0x2d5, 0x20}, {0x1ff, 12}, {0x2ce, 3}, {0x29b, 2}, {0x2ab, 4}, {0x260, 0},
+        {0x1b6, 0}, {0x207, 0}, {0x200, 0}, {0x203, 0x800},
+        {0x2dc, 0xaa00}, {0x2f8, 0}, {0x292, 2}, {0x293, 0},
+        {0x80, 0}, {0x8d, 0}, {0x83, 0xffff}, {0x8c, 0xa},
+        {0x2f9, 0x2d}, {0x313, 0x6000}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff},
+        {0x206, 0x43f}, {0x204, 0x80000}, {0x205, 0x240},
+        {0x8e, 0xf}, {0x8f, 0xf}, {0x202, 0xcc0010},
+        {0x1c4, 0}, {0x1c5, 9}, {0x1c3, 4}, {0x31c, 0x28028},
+        {0x31b, 0}, {0x31d, 0}, {0x3b0, ((Height - 1u) << 14u) | (Width - 1u)},
+        {0x3b8, 0x9000000}, {0x1e0, 0}, {0xc, 0}, {0xd, (Height << 16u) | Width},
+        {0x81, 0x80000000}, {0x82, (Height << 16u) | Width},
+        {0x90, 0x80000000}, {0x91, (Height << 16u) | Width},
+        {0x94, 0x80000000}, {0x95, (Height << 16u) | Width},
+        {0x318, static_cast<std::uint32_t>(color >> 8u)}, {0x390, static_cast<std::uint32_t>(color >> 40u)},
+        {0x10f, std::bit_cast<std::uint32_t>(32.0f)}, {0x110, std::bit_cast<std::uint32_t>(32.0f)},
+        {0x111, std::bit_cast<std::uint32_t>(32.0f)}, {0x112, std::bit_cast<std::uint32_t>(32.0f)},
+        {0x113, std::bit_cast<std::uint32_t>(1.0f)}, {0x114, 0},
+        {0xb4, 0}, {0xb5, std::bit_cast<std::uint32_t>(1.0f)}
+    };
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> shaders{
+        {0xc8, static_cast<std::uint32_t>(geometry >> 8u)}, {0xc9, static_cast<std::uint32_t>(geometry >> 40u)},
+        {0x8a, 0x600c0000u}, {0x8b, (8u << 19u) | (3u << 16u) | (2u << 1u)},
+        {0x8c, static_cast<std::uint32_t>(table)}, {0x8d, static_cast<std::uint32_t>(table >> 32u)},
+        {0x8, static_cast<std::uint32_t>(pixel >> 8u)}, {0x9, static_cast<std::uint32_t>(pixel >> 40u)}, {0xb, 0}
+    };
+    std::vector<std::uint32_t> commands;
+    Write(commands, 0x79, 0x242, 4);
+    Write(commands, 0x79, 0x25b, (12u << 9u) | 4u);
+    for (const auto& [offset, value] : context) Write(commands, 0x69, offset, value);
+    for (const auto& [offset, value] : shaders) Write(commands, 0x76, offset, value);
+    commands.insert(commands.end(), {0xc0002a00u, 0u});
+    return commands;
+}
+
+AgcDriver::DriverDetail::DrawEntryCounters Draw(const std::array<std::uint32_t, 4>& descriptor) {
+    Pixels.fill(std::byte{0});
+    auto commands = Setup(descriptor);
+    const auto address = reinterpret_cast<std::uintptr_t>(Indices.data());
+    commands.insert(commands.end(), {0xc0042700u, 3u, static_cast<std::uint32_t>(address), static_cast<std::uint32_t>(address >> 32u), 3u, 0u});
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    Require(sceAgcDriverSubmitDcb(&packet) == 0, "mesh submission failed");
+    AgcDriverWaitIdle_nid_postfix();
+    Require(std::to_integer<unsigned>(Pixels[(24u * Width + 16u) * 4u]) > 200u, "the mesh draw did not cover its triangle");
+    return AgcDriver::DriverDetail::Driver::Get().DrawCacheCounters();
+}
+
+void MovedDescriptorTableHitsTheCache() {
+    const auto vertices = reinterpret_cast<std::uintptr_t>(Vertices.data());
+    for (auto& descriptor : Descriptors) descriptor = {static_cast<std::uint32_t>(vertices), static_cast<std::uint32_t>(vertices >> 32u) | (32u << 16u), static_cast<std::uint32_t>(Vertices.size()), 0x01016facu};
+    const auto front = static_cast<std::size_t>(DrawMiss::FrontDiffering);
+    AgcDriver::DriverDetail::DrawEntryCounters counters{};
+    for (std::size_t i = 0; i < 4; ++i) counters = Draw(Descriptors[i]);
+    Require(counters.lookups == 4 && counters.hits == 0, "the draws that teach the moved table made " + std::to_string(counters.hits) + " hits in " + std::to_string(counters.lookups) + " lookups, expected 0 in 4");
+    const auto learned = counters;
+    for (std::size_t i = 4; i < Descriptors.size(); ++i) counters = Draw(Descriptors[i]);
+    Require(counters.misses[front] == learned.misses[front], "a relocated mesh stage was refused as a differing front stage (" + std::to_string(counters.misses[front] - learned.misses[front]) + " times)");
+    Require(counters.hits - learned.hits == Descriptors.size() - 4, "mesh draws whose table moved again did not reuse the cached draw (" + std::to_string(counters.hits - learned.hits) + " hits in " + std::to_string(Descriptors.size() - 4) + " draws)");
+}
+
+}
+
+int main() {
+    try {
+        {
+            const auto device = OpenVulkanTestDevice();
+            if (!device || !TargetHasCapability(device->Target(), spv::CapabilityMeshShadingEXT)) return VulkanTestSkipped;
+            if (device->DeviceName().starts_with("llvmpipe")) {
+                std::puts("skipped, the device has no mesh shader compiler that handles these programs (llvmpipe)");
+                return VulkanTestSkipped;
+            }
+        }
+        Register(GeometryHeader, GeometryCode.data(), sizeof(GeometryCode), 2);
+        Register(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1);
+        MovedDescriptorTableHitsTheCache();
+        AgcDriverShutdown_nid_postfix();
+        std::cout << "A mesh draw whose descriptor table moved reused the cached draw\n";
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        AgcDriverShutdown_nid_postfix();
+        return 1;
+    }
+}