Merge branch 'main' of https://github.com/boykopovar/AnyPS5 into fix/recvmsg-after-shut-rd

# Conflicts:
#	core/libs/prx/libkernel/Socket/src/Udp.cpp
#	docs/dev/debt/libkernel.md

.github/workflows/build.yml

@@ -421,7 +421,7 @@ jobs:
 
   linux:
     needs: eligibility
-    if: ${{ (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.linux)) && needs.eligibility.outputs.full == 'true' }}
+    if: ${{ !github.event.repository.fork && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.linux)) && needs.eligibility.outputs.full == 'true' }}
     runs-on: [self-hosted, linux, x64, anyps5]
     timeout-minutes: 60
     env:

.github/workflows/input-mapper.yml

@@ -0,0 +1,29 @@
+name: Input mapper
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+    paths:
+      - '.github/workflows/input-mapper.yml'
+      - 'tools/input-mapper/**'
+      - 'core/libs/prx/libScePad/include/**'
+      - 'core/libs/prx/libSceVideoOut/src/InputMapping.cpp'
+
+permissions:
+  contents: read
+
+jobs:
+  input-mapper:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@v7
+        with:
+          node-version: '22'
+          package-manager-cache: false
+      - run: npm --prefix tools/input-mapper test

.github/workflows/pages-progress.yml

@@ -18,6 +18,7 @@ concurrency:
 
 jobs:
   progress:
+    if: ${{ !github.event.repository.fork }}
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:

.github/workflows/pages.yml

@@ -33,7 +33,7 @@ concurrency:
 
 jobs:
   pages:
-    if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
+    if: ${{ !github.event.repository.fork && (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success') }}
     runs-on: ubuntu-latest
     environment:
       name: github-pages

.gitmodules

@@ -1,3 +1,6 @@
+[submodule "3rdparty/jxrlib"]
+	path = 3rdparty/jxrlib
+	url = https://github.com/4creators/jxrlib.git
 [submodule "3rdparty/SDL2"]
 	path = 3rdparty/SDL2
 	url = https://github.com/libsdl-org/SDL.git
@@ -61,3 +64,6 @@
 [submodule "3rdparty/vorbis"]
 	path = 3rdparty/vorbis
 	url = https://github.com/xiph/vorbis.git
+[submodule "3rdparty/nghttp2"]
+	path = 3rdparty/nghttp2
+	url = https://github.com/nghttp2/nghttp2.git

CMakeLists.txt

@@ -55,6 +55,7 @@ endif()
 
 option(ANYPS5_ENABLE_SPIRV_TOOLS "Enable SPIRV-Tools validation and optimization" OFF)
 option(ANYPS5_ENABLE_ASAN "Build Linux targets with AddressSanitizer and use an ASan-compatible guest arena range" OFF)
+option(ANYPS5_ENABLE_TSAN "Build Linux targets with ThreadSanitizer" OFF)
 option(APS5_ENABLE_TIMING_LOG "Write frame timings asynchronously to frame-timing.log" OFF)
 option(APS5_ENABLE_TRACY "Build the Tracy profiler client and report performance timer scopes as Tracy zones" OFF)
 option(ANYPS5_USE_SYSTEM_DEPS "Resolve SDL2, freetype, Vulkan-Headers, SPIRV-Headers, glslang and SPIRV-Tools from the system instead of the bundled submodules" OFF)
@@ -76,6 +77,18 @@ if(NOT ANYPS5_CXX_LINKS_STANDARD_LIBRARY)
         "Supported toolchains are listed in docs/dev/BUILD.md.")
 endif()
 
+if(ANYPS5_ENABLE_ASAN AND ANYPS5_ENABLE_TSAN)
+    message(FATAL_ERROR "ANYPS5_ENABLE_ASAN and ANYPS5_ENABLE_TSAN cannot be combined")
+endif()
+if(ANYPS5_ENABLE_TSAN)
+    if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        message(FATAL_ERROR "ANYPS5_ENABLE_TSAN is supported only on Linux")
+    endif()
+    if(NOT CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang)$")
+        message(FATAL_ERROR "ANYPS5_ENABLE_TSAN requires GCC or Clang")
+    endif()
+endif()
+
 if(ANYPS5_ENABLE_ASAN)
     if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
         message(FATAL_ERROR "ANYPS5_ENABLE_ASAN is supported only on Linux")
@@ -86,6 +99,10 @@ if(ANYPS5_ENABLE_ASAN)
     add_compile_options(-fsanitize=address -fno-omit-frame-pointer)
     add_link_options(-fsanitize=address)
     add_compile_definitions(ANYPS5_ENABLE_ASAN=1)
+elseif(ANYPS5_ENABLE_TSAN)
+    add_compile_options(-fsanitize=thread -fno-omit-frame-pointer)
+    add_link_options(-fsanitize=thread)
+    add_compile_definitions(ANYPS5_ENABLE_TSAN=1)
 elseif(NOT APPLE)
     set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -static-libgcc -static-libstdc++")
 endif()
@@ -169,8 +186,15 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress_read_encoding.py)
         add_test(NAME import_audit COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_import_audit.py)
+        if(NOT APPLE)
+            add_test(NAME annotate_module COMMAND ${Python3_EXECUTABLE}
+                ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_annotate_module.py)
+            set_tests_properties(annotate_module PROPERTIES TIMEOUT 20)
+        endif()
         add_test(NAME unwrap_self COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_unwrap_self.py)
+        add_test(NAME audit_dump COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_audit_dump.py)
         foreach(shard RANGE 0 11)
             if(shard EQUAL 0)
                 set(conventions_test_name conventions_function_bodies)
@@ -213,6 +237,12 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_configure_seed.py ConfigureSeedRoundTripTests)
         add_test(NAME configure_seed_parsing COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_configure_seed.py ChecksParsingTests)
+        add_test(NAME vulkan_validation_gate_reports COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_vulkan_validation_gate.py ReportTests)
+        add_test(NAME vulkan_validation_gate_integration COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_vulkan_validation_gate.py IntegrationTests)
+        add_test(NAME vulkan_validation_gate_attribution COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_vulkan_validation_gate.py AttributionTests)
         add_test(NAME stubgen COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_stubgen.py)
         add_test(NAME debt_split COMMAND ${Python3_EXECUTABLE}
@@ -255,7 +285,7 @@ if(BUILD_TESTING)
             string(REGEX REPLACE "^    def (test_[A-Za-z0-9_]+).*" "\\1" test_case "${test_case}")
             add_test(NAME check_compile_modes_${test_case} COMMAND ${Python3_EXECUTABLE}
                 "${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile_modes.py" CheckCompileModeTests.${test_case})
-            set_tests_properties(check_compile_modes_${test_case} PROPERTIES TIMEOUT 30)
+            set_tests_properties(check_compile_modes_${test_case} PROPERTIES TIMEOUT 60)
         endforeach()
         add_test(NAME trace_digest COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_trace_digest.py)
@@ -290,7 +320,9 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile_preprocessor.py)
         set_tests_properties(check_compile_preprocessor PROPERTIES TIMEOUT 20)
         add_test(NAME check_compile COMMAND ${Python3_EXECUTABLE}
-            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile.py)
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile.py CheckCompileTests)
+        add_test(NAME check_compile_outputs COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile.py CheckCompileOutputTests)
         add_test(NAME performance_compare COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_performance_compare.py)
         set_tests_properties(performance_compare PROPERTIES TIMEOUT 20)
@@ -371,6 +403,8 @@ if(NOT ANYPS5_USE_SYSTEM_DEPS)
     endif()
 endif()
 
+include(cmake/Jxr.cmake)
+
 set(LIBUSB_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
 set(LIBUSB_INSTALL_TARGETS OFF CACHE BOOL "" FORCE)
 set(LIBUSB_BUILD_TESTING OFF CACHE BOOL "" FORCE)

cmake/Jxr.cmake

@@ -0,0 +1,44 @@
+enable_language(C)
+set(JXR_SOURCE_DIR ${CMAKE_SOURCE_DIR}/3rdparty/jxrlib)
+
+function(configure_jxr_native target)
+    target_include_directories(${target} SYSTEM PRIVATE
+        ${JXR_SOURCE_DIR}/common/include ${JXR_SOURCE_DIR}/image/sys ${JXR_SOURCE_DIR}/jxrgluelib)
+    target_compile_definitions(${target} PRIVATE DISABLE_PERF_MEASUREMENT)
+    if(WIN32)
+        target_compile_definitions(${target} PRIVATE WIN32 __in=)
+        target_compile_options(${target} PRIVATE -include ${JXR_SOURCE_DIR}/image/sys/strcodec.h)
+    else()
+        target_compile_definitions(${target} PRIVATE __ANSI__)
+        target_compile_options(${target} PRIVATE -include ${CMAKE_SOURCE_DIR}/cmake/JxrCompat.h)
+    endif()
+    set_target_properties(${target} PROPERTIES C_STANDARD 99 C_STANDARD_REQUIRED ON
+        POSITION_INDEPENDENT_CODE ON C_VISIBILITY_PRESET hidden)
+endfunction()
+
+add_library(jxr STATIC EXCLUDE_FROM_ALL
+    ${JXR_SOURCE_DIR}/image/sys/adapthuff.c
+    ${JXR_SOURCE_DIR}/image/sys/image.c
+    ${JXR_SOURCE_DIR}/image/sys/strcodec.c
+    ${JXR_SOURCE_DIR}/image/sys/strPredQuant.c
+    ${JXR_SOURCE_DIR}/image/sys/strTransform.c
+    ${JXR_SOURCE_DIR}/image/decode/decode.c
+    ${JXR_SOURCE_DIR}/image/decode/JXRTranscode.c
+    ${JXR_SOURCE_DIR}/image/decode/postprocess.c
+    ${JXR_SOURCE_DIR}/image/decode/segdec.c
+    ${JXR_SOURCE_DIR}/image/decode/strdec.c
+    ${JXR_SOURCE_DIR}/image/decode/strInvTransform.c
+    ${JXR_SOURCE_DIR}/image/decode/strPredQuantDec.c
+    ${JXR_SOURCE_DIR}/image/encode/encode.c
+    ${JXR_SOURCE_DIR}/image/encode/segenc.c
+    ${JXR_SOURCE_DIR}/image/encode/strenc.c
+    ${JXR_SOURCE_DIR}/image/encode/strFwdTransform.c
+    ${JXR_SOURCE_DIR}/image/encode/strPredQuantEnc.c
+    ${JXR_SOURCE_DIR}/jxrgluelib/JXRGlue.c
+    ${JXR_SOURCE_DIR}/jxrgluelib/JXRGlueJxr.c
+    ${JXR_SOURCE_DIR}/jxrgluelib/JXRGluePFC.c
+    ${JXR_SOURCE_DIR}/jxrgluelib/JXRMeta.c)
+configure_jxr_native(jxr)
+if(UNIX)
+    target_link_libraries(jxr PRIVATE m)
+endif()

cmake/JxrCompat.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#include <wchar.h>
+#include "strcodec.h"
+
+#if defined(__ANSI__) && !defined(WIN32) && !defined(_BIG__ENDIAN_)
+U32 _byteswap_ulong(U32 bits);
+#endif

core/libs/CMakeLists.txt

@@ -63,6 +63,13 @@ function(configure_windows_unwind target)
     endif()
 endfunction()
 
+function(configure_macos_unwind target)
+    if(APPLE)
+        target_compile_options(${target} PRIVATE -femit-dwarf-unwind=always)
+        target_link_options(${target} PRIVATE LINKER:-no_compact_unwind)
+    endif()
+endfunction()
+
 if(MINGW)
     foreach(runtimeLinkFlag IN ITEMS -static-libgcc -static-libstdc++ -static)
         string(REPLACE "${runtimeLinkFlag}" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
@@ -105,6 +112,7 @@ if(WIN32)
     target_compile_options(libc PRIVATE "SHELL:-include ${CMAKE_CURRENT_SOURCE_DIR}/prx/libc/include/HostStandardStreams.hpp")
 endif()
 configure_windows_unwind(libc)
+configure_macos_unwind(libc)
 if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
     target_link_options(libc PRIVATE LINKER:-z,now)
     target_link_options(libc INTERFACE LINKER:--no-as-needed)
@@ -269,6 +277,15 @@ if(BUILD_TESTING)
         set_tests_properties(guest_kernel_sys_sdk_version PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
     endif()
+    add_test_executable(guest_kernel_sys_console_identity_tests tests/GuestKernelSysConsoleIdentity.cpp)
+    target_include_directories(guest_kernel_sys_console_identity_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_console_identity_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_console_identity_tests)
+    add_test(NAME guest_kernel_sys_console_identity COMMAND guest_kernel_sys_console_identity_tests)
+    if(WIN32)
+        set_tests_properties(guest_kernel_sys_console_identity PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+    endif()
     add_test_executable(nid_buffer_validation_tests nid/tests/BufferValidation.cpp)
     target_include_directories(nid_buffer_validation_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/nid/include)
     target_compile_features(nid_buffer_validation_tests PRIVATE cxx_std_20)
@@ -278,6 +295,32 @@ if(BUILD_TESTING)
     add_test(NAME nid_buffer_validation COMMAND nid_buffer_validation_tests)
     set_tests_properties(nid_buffer_validation PROPERTIES TIMEOUT 10)
 
+    add_test_executable(guest_dinkumware_fetch_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/GuestDinkumwareFetch.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/nid/src/NidCompute.cpp ${CMAKE_CURRENT_SOURCE_DIR}/nid/src/Sha1.cpp)
+    target_include_directories(guest_dinkumware_fetch_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/nid/include)
+    target_link_libraries(guest_dinkumware_fetch_tests PRIVATE ${CMAKE_DL_LIBS})
+    configure_windows_unwind(guest_dinkumware_fetch_tests)
+    if(MINGW)
+        target_link_libraries(guest_dinkumware_fetch_tests PRIVATE libc)
+    endif()
+    add_custom_target(guest_dinkumware_fetch_libraries DEPENDS ${LIBC_PATCHED})
+    add_dependencies(guest_dinkumware_fetch_tests guest_dinkumware_fetch_libraries)
+    foreach(operation IN ITEMS and or xor)
+        foreach(width IN ITEMS 1 2 4)
+            add_test(NAME guest_dinkumware_fetch_${operation}_${width} COMMAND guest_dinkumware_fetch_tests
+                $<TARGET_FILE_DIR:libc> unpatched ${operation} ${width})
+            add_test(NAME guest_dinkumware_fetch_${operation}_${width}_patched COMMAND guest_dinkumware_fetch_tests
+                ${PATCHED_DIR} patched ${operation} ${width})
+            set_tests_properties(guest_dinkumware_fetch_${operation}_${width}
+                guest_dinkumware_fetch_${operation}_${width}_patched PROPERTIES TIMEOUT 20)
+            if(WIN32)
+                set_tests_properties(guest_dinkumware_fetch_${operation}_${width} PROPERTIES ENVIRONMENT_MODIFICATION
+                    "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+                set_tests_properties(guest_dinkumware_fetch_${operation}_${width}_patched PROPERTIES ENVIRONMENT_MODIFICATION
+                    "PATH=path_list_prepend:${PATCHED_DIR}")
+            endif()
+        endforeach()
+    endforeach()
     add_test_executable(pe_import_first_tests allocator/tests/PeImportFirstTests.cpp allocator/PeImportFirst.cpp)
     target_include_directories(pe_import_first_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_compile_features(pe_import_first_tests PRIVATE cxx_std_20)
@@ -301,7 +344,7 @@ if(BUILD_TESTING)
         set(public_header_manifest "${CMAKE_CURRENT_BINARY_DIR}/public_headers.json")
         add_test(NAME public_headers COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_public_headers.py --audit-only --manifest "${public_header_manifest}")
-        set_tests_properties(public_headers PROPERTIES TIMEOUT 30 RUN_SERIAL TRUE FIXTURES_SETUP public_headers_manifest)
+        set_tests_properties(public_headers PROPERTIES TIMEOUT 90 RUN_SERIAL TRUE FIXTURES_SETUP public_headers_manifest)
         foreach(batch RANGE 0 63)
             add_test(NAME public_headers_compile_${batch} COMMAND ${Python3_EXECUTABLE}
                 ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_public_headers.py --compiler ${CMAKE_CXX_COMPILER} ${public_header_compiler_args}
@@ -329,6 +372,11 @@ if(BUILD_TESTING)
         target_link_options(patched_libraries_tests PRIVATE -static -static-libgcc -static-libstdc++)
     endif()
     add_test(NAME patched_libraries COMMAND patched_libraries_tests ${PATCHED_DIR})
+    if(MINGW)
+        get_filename_component(patchedLibrariesRuntimeDirectory "${CMAKE_CXX_COMPILER}" DIRECTORY)
+        set_tests_properties(patched_libraries PROPERTIES
+            ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:${patchedLibrariesRuntimeDirectory}")
+    endif()
     if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
         add_library(host_libstdcxx_exception_module SHARED tests/HostLibstdcxxExceptionModule.cpp)
         target_link_libraries(host_libstdcxx_exception_module PRIVATE ${PATCHED_DIR}/libc.prx)
@@ -426,6 +474,18 @@ if(BUILD_TESTING)
         add_subdirectory(tests/write_watch)
         add_subdirectory(tests/guest_arena)
     endif()
+    add_test_executable(guest_kernel_sys_pthread_np_tests tests/GuestKernelSysPthreadNp.cpp)
+    target_include_directories(guest_kernel_sys_pthread_np_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_pthread_np_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_pthread_np_tests)
+    foreach(pthreadNpMode IN ITEMS rwlockattr kind)
+        add_test(NAME guest_kernel_sys_pthread_np_${pthreadNpMode} COMMAND guest_kernel_sys_pthread_np_tests ${pthreadNpMode})
+        set_tests_properties(guest_kernel_sys_pthread_np_${pthreadNpMode} PROPERTIES TIMEOUT 20)
+        if(WIN32)
+            set_tests_properties(guest_kernel_sys_pthread_np_${pthreadNpMode} PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+        endif()
+    endforeach()
     add_test_executable(guest_kernel_sys_module_internal_tests tests/GuestKernelSysModuleInternal.cpp)
     target_include_directories(guest_kernel_sys_module_internal_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_kernel_sys_module_internal_tests PRIVATE libkernel_sys libkernel libc)
@@ -475,6 +535,17 @@ if(BUILD_TESTING)
             PASS_REGULAR_EXPRESSION "C\\+\\+ exception St9bad_alloc raised on thread [0-9]+ cannot be delivered: 0x[0-9a-f]+ undeliverable_host_exception_tests\\.exe\\+0x[0-9a-f]+ has no unwind data")
     endif()
 
+    add_test_executable(guest_kernel_sys_uuid_tests tests/GuestKernelSysUuid.cpp)
+    target_include_directories(guest_kernel_sys_uuid_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_uuid_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_uuid_tests)
+    add_test(NAME guest_kernel_sys_uuid COMMAND guest_kernel_sys_uuid_tests)
+    set_tests_properties(guest_kernel_sys_uuid PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_kernel_sys_uuid PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+    endif()
+
     add_test_executable(crash_text_tests tests/CrashText.cpp)
     target_include_directories(crash_text_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     if(MINGW)

core/libs/prx/libSceAgc/Acb/include/Sync.hpp

@@ -9,7 +9,7 @@ extern "C" {
 uint32_t* APS5_VABI sceAgcAcbCondExec(CommandBuffer* buf, const volatile uint32_t* address, uint32_t numDwords);
 uint32_t APS5_VABI sceAgcAcbCondExecGetSize(void);
 std::uint32_t* APS5_VABI sceAgcAcbWaitRegMem(CommandBuffer* buf, std::uint8_t size, std::uint8_t compareFunction, std::uint8_t cachePolicy, const volatile void* address, std::uint64_t reference, std::uint64_t mask, std::uint32_t pollCycles);
-std::uint64_t APS5_VABI sceAgcAcbWaitOnAddressGetSize(std::uint8_t size);
+std::uint64_t APS5_VABI sceAgcAcbWaitOnAddressGetSize(std::uint32_t size);
 std::uint32_t* APS5_VABI sceAgcAcbEventWrite(CommandBuffer* buf, std::uint8_t eventType, const volatile void* address);
 std::uint64_t APS5_VABI sceAgcAcbEventWriteGetSize(std::uint8_t eventType);
 std::uint32_t APS5_VABI sceAgcAcbQueueEndOfShaderActionGetSize();

core/libs/prx/libSceAgc/Acb/src/Sync.cpp

@@ -25,8 +25,8 @@ std::uint32_t* APS5_VABI sceAgcAcbWaitRegMem(CommandBuffer* buf, std::uint8_t si
     return Agc::Command::WriteWait(buf, size, compareFunction, 0, cachePolicy, address, reference, mask, pollCycles, __func__);
 }
 
-std::uint64_t APS5_VABI sceAgcAcbWaitOnAddressGetSize(std::uint8_t size) {
-    return size == 0 ? 56 : 64;
+std::uint64_t APS5_VABI sceAgcAcbWaitOnAddressGetSize(std::uint32_t size) {
+    return size == 0 ? 56 : size == 1 ? 64 : 0;
 }
 
 std::uint32_t* APS5_VABI sceAgcAcbEventWrite(CommandBuffer* buf, std::uint8_t eventType, const volatile void* address) {

core/libs/prx/libSceAgc/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_library(libSceAgc SHARED EXCLUDE_FROM_ALL
+        ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Unimplemented.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Acb/src/Control.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Acb/src/Dispatch.cpp
@@ -293,6 +294,19 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_wait_on_address_size_tests EXCLUDE_FROM_ALL tests/WaitOnAddressSize.cpp)
+target_include_directories(agc_wait_on_address_size_tests PRIVATE ${LIBS_INCLUDE_DIR})
+target_link_libraries(agc_wait_on_address_size_tests PRIVATE libSceAgc libc)
+configure_windows_unwind(agc_wait_on_address_size_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_wait_on_address_size_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
+    add_test(NAME agc_wait_on_address_size COMMAND agc_wait_on_address_size_tests)
+    if(WIN32)
+        set_tests_properties(agc_wait_on_address_size PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_interpolant_mapping_tests EXCLUDE_FROM_ALL
         tests/InterpolantMapping.cpp
         Shader/src/InterpolantMapping.cpp

core/libs/prx/libSceAgc/DcbFlow/src/Sync.cpp

@@ -26,7 +26,7 @@ std::uint32_t* APS5_VABI sceAgcDcbWaitRegMem(CommandBuffer* buf, std::uint8_t si
 }
 
 uint32_t APS5_VABI sceAgcDcbWaitOnAddressGetSize(uint32_t size) {
-    return size == 0 ? 56 : 64;
+    return size == 0 ? 56 : size == 1 ? 64 : 0;
 }
 
 std::uint32_t* APS5_VABI sceAgcDcbEventWrite(CommandBuffer* buf, std::uint8_t eventType, const volatile void* address) {

core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp

@@ -13,7 +13,9 @@ uint32_t* APS5_VABI sceAgcDcbContextStateOp(CommandBuffer* buf, uint32_t operati
 }
 
 uint64_t APS5_VABI sceAgcDcbContextStateOpGetSize(uint32_t operation) {
-    Agc::Command::Require(operation <= 3, __func__, "invalid context state operation");
+    if (operation > 3) {
+        return 0;
+    }
     if (operation == 0) {
         return 5 * sizeof(std::uint32_t);
     }

core/libs/prx/libSceAgc/tests/ContextStateSize.cpp

@@ -47,9 +47,13 @@ void testSizes() {
     }
 }
 
-void testRejections() {
-    expectFailure([] { sceAgcDcbContextStateOpGetSize(4); });
-    expectFailure([] { sceAgcDcbContextStateOpGetSize(0xffffffffu); });
+void testInvalidOperations() {
+    for (const std::uint32_t operation : {4u, 15u, 16u, 0xffu, 0x100u, 0x103u, 0xffffffffu}) {
+        check(sceAgcDcbContextStateOpGetSize(operation) == 0, "an invalid context state operation did not return size 0");
+    }
+    std::array<std::uint32_t, 64> words{};
+    CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
+    expectFailure([&] { sceAgcDcbContextStateOp_0100(&buffer, 4); });
 }
 
 }
@@ -57,7 +61,7 @@ void testRejections() {
 int main() {
     try {
         testSizes();
-        testRejections();
+        testInvalidOperations();
         LibcRunShutdown_nid_postfix();
         std::puts("AGC context state size tests passed");
         return 0;

core/libs/prx/libSceAgc/tests/Platform.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgc/Misc/include/Platform.hpp"
+#include "prx/libSceAgc/include/Agc.hpp"
 #include "prx/libc/include/general/VabiMacros.hpp"
 #include "prx/libc/include/Shutdown.hpp"
 
@@ -34,12 +35,18 @@ void testRejections() {
     expectFailure([] { sceAgcGetIsTrinityMode(nullptr); });
 }
 
+void testExportLinked() {
+    auto* volatile linked = &sceAgcSuspendPointAndCheckStatus;
+    check(linked != nullptr, "sceAgcSuspendPointAndCheckStatus is not linked");
+}
+
 }
 
 int main() {
     try {
         testTrinityMode();
         testRejections();
+        testExportLinked();
         LibcRunShutdown_nid_postfix();
         std::puts("AGC platform tests passed");
         return 0;

core/libs/prx/libSceAgc/tests/WaitOnAddressSize.cpp

@@ -0,0 +1,41 @@
+#include "prx/libSceAgc/Acb/include/Sync.hpp"
+#include "prx/libSceAgc/DcbFlow/include/Sync.hpp"
+#include "prx/libSceAgc/Command/include/Packet.hpp"
+
+#include <array>
+#include <cstdint>
+#include <cstdio>
+
+namespace {
+
+struct Storage {
+    std::array<std::uint32_t, 64> words{};
+    CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
+
+    std::uint64_t written() const { return static_cast<std::uint64_t>(buffer.cursor_up - words.data()) * sizeof(std::uint32_t); }
+};
+
+int fail(const char* message) {
+    std::fprintf(stderr, "%s\n", message);
+    return 1;
+}
+
+}
+
+int main() {
+    alignas(8) std::uint64_t label = 0;
+    for (const std::uint32_t size : {0u, 1u}) {
+        const std::uint64_t expected = size == 0 ? 56 : 64;
+        if (sceAgcAcbWaitOnAddressGetSize(size) != expected || sceAgcDcbWaitOnAddressGetSize(size) != expected) return fail("wait size mismatch");
+        Storage acb;
+        Storage dcb;
+        sceAgcAcbWaitRegMem(&acb.buffer, static_cast<std::uint8_t>(size), 3, 0, &label, 1, ~0ull, 0x40);
+        sceAgcDcbWaitRegMem(&dcb.buffer, static_cast<std::uint8_t>(size), 3, 0, 0, &label, 1, ~0ull, 0x40);
+        if (acb.written() != expected || dcb.written() != expected) return fail("wait size differs from the packets the wait builders write");
+    }
+    for (const std::uint32_t size : {2u, 3u, 4u, 8u, 0xffu, 0x100u, 0x101u, 0xffffffffu}) {
+        if (sceAgcAcbWaitOnAddressGetSize(size) != 0 || sceAgcDcbWaitOnAddressGetSize(size) != 0) return fail("an invalid wait size did not return 0");
+    }
+    std::puts("AGC wait size tests passed");
+    return 0;
+}

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -238,6 +238,7 @@ foreach(agcOverlayStage IN ITEMS Vert Frag)
 endforeach()
 
 set(agcDriverSources
+        ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Unimplemented.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/ComputePipelineCapture.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/DisplayBuffer.cpp
@@ -327,6 +328,7 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineCache.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineJournal.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Resources.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DeferredDestruction.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PlaceholderImages.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Recorder.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ParallelCompare.cpp
@@ -1279,6 +1281,20 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_deferred_destruction_tests EXCLUDE_FROM_ALL tests/DeferredDestruction.cpp)
+target_include_directories(agc_deferred_destruction_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
+target_link_libraries(agc_deferred_destruction_tests PRIVATE libSceAgcDriver libc)
+set_target_properties(agc_deferred_destruction_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_deferred_destruction_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_deferred_destruction_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_deferred_destruction COMMAND agc_deferred_destruction_tests)
+    if(WIN32)
+        set_tests_properties(agc_deferred_destruction PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_device_memory_fallback_tests EXCLUDE_FROM_ALL tests/DeviceMemoryFallback.cpp)
 target_include_directories(agc_device_memory_fallback_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
 target_link_libraries(agc_device_memory_fallback_tests PRIVATE libSceAgcDriver libc)
@@ -1572,8 +1588,12 @@ if(BUILD_TESTING)
     set_tests_properties(agc_pipeline_library_off PROPERTIES ENVIRONMENT "APS5_NO_GPL=1")
     add_test(NAME agc_pipeline_library_fast_only COMMAND agc_driver_recorder_tests --pipeline-library-only)
     set_tests_properties(agc_pipeline_library_fast_only PROPERTIES ENVIRONMENT "APS5_NO_PIPELINE_LTO=1" SKIP_RETURN_CODE 77)
+    add_test(NAME agc_pipeline_library_lto_info COMMAND agc_driver_recorder_tests --pipeline-library-lto-info-only)
+    add_test(NAME agc_pipeline_library_lto_info_fast_only COMMAND agc_driver_recorder_tests --pipeline-library-lto-info-only)
+    set_tests_properties(agc_pipeline_library_lto_info agc_pipeline_library_lto_info_fast_only PROPERTIES SKIP_RETURN_CODE 77)
+    set_tests_properties(agc_pipeline_library_lto_info_fast_only PROPERTIES ENVIRONMENT "APS5_NO_PIPELINE_LTO=1")
     if(WIN32)
-        set_tests_properties(agc_pipeline_library agc_pipeline_library_off agc_pipeline_library_fast_only PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_pipeline_library agc_pipeline_library_off agc_pipeline_library_fast_only agc_pipeline_library_lto_info agc_pipeline_library_lto_info_fast_only PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     set_tests_properties(agc_pipeline_cache agc_pipeline_cache_file PROPERTIES SKIP_RETURN_CODE 77)
@@ -1832,10 +1852,14 @@ if(BUILD_TESTING)
     set_tests_properties(agc_recorder_cmask PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
     add_test(NAME agc_recorder_cube COMMAND agc_driver_recorder_tests --cube-only)
     set_tests_properties(agc_recorder_cube PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    add_test(NAME agc_recorder_failed_check COMMAND agc_driver_recorder_tests --failed-check-only)
+    set_tests_properties(agc_recorder_failed_check PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30
+        PASS_REGULAR_EXPRESSION "a failed check stopped the test with a texture upload in its open batch"
+        FAIL_REGULAR_EXPRESSION "recorder teardown")
     if(WIN32)
         set_tests_properties(agc_driver_host_import_lifetime agc_driver_host_import_mutation_race agc_driver_unit_shadow agc_driver_unit_shadow_budget PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
-        set_tests_properties(agc_recorder_completion_labels agc_recorder_cmask agc_recorder_cube PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_recorder_completion_labels agc_recorder_cmask agc_recorder_cube agc_recorder_failed_check PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     set_property(TEST agc_driver_host_import_lifetime APPEND PROPERTY ENVIRONMENT_MODIFICATION
@@ -2172,6 +2196,18 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources)
     target_link_libraries(${agcExecutionTest} PRIVATE libSceAgcDriver SDL2-static libc)
     set_target_properties(${agcExecutionTest} PROPERTIES CXX_EXTENSIONS OFF)
     configure_windows_unwind(${agcExecutionTest})
+    if(agcExecutionTests)
+        list(GET agcExecutionTests 0 agcExecutionPchTarget)
+        target_precompile_headers(${agcExecutionTest} REUSE_FROM ${agcExecutionPchTarget})
+    else()
+        target_precompile_headers(${agcExecutionTest} PRIVATE
+                ${CMAKE_CURRENT_SOURCE_DIR}/tests/execution/VulkanTestDevice.hpp
+                <array> <iostream> <span> <string> <vector>)
+    endif()
+    if(CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache")
+        set_target_properties(${agcExecutionTest} PROPERTIES CXX_COMPILER_LAUNCHER
+                "${CMAKE_COMMAND};-E;env;CCACHE_SLOPPINESS=pch_defines,time_macros;${CMAKE_CXX_COMPILER_LAUNCHER}")
+    endif()
     list(APPEND agcExecutionTests ${agcExecutionTest})
 endforeach()
 if(BUILD_TESTING)
@@ -2317,6 +2353,9 @@ if(BUILD_TESTING)
         set(testName "agc_driver_msaa_dcc_clear_${samples}")
         add_test(NAME ${testName} COMMAND agc_driver_alpha_to_coverage_tests --dcc-fast-clear ${samples})
         list(APPEND multisampledDccClearTests ${testName})
+        set(testName "agc_driver_msaa_dcc_compared_import_${samples}")
+        add_test(NAME ${testName} COMMAND agc_driver_alpha_to_coverage_tests --dcc-compared-import ${samples})
+        list(APPEND multisampledDccClearTests ${testName})
     endforeach()
     set_tests_properties(agc_driver_msaa_dcc_state ${multisampledDccClearTests} PROPERTIES
             ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2367,6 +2406,7 @@ target_include_directories(agc_driver_reversed_order_blend_tests PRIVATE $<TARGE
 target_include_directories(agc_driver_swap_alt_blend_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
 target_include_directories(agc_driver_packed_float_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+target_include_directories(agc_driver_uint_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
 target_link_libraries(agc_driver_job_manager_compute_tests PRIVATE libSceJobManager libSceAgc libkernel)
 
@@ -2386,6 +2426,8 @@ target_include_directories(agc_driver_color565_target_tests PRIVATE $<TARGET_PRO
 
 target_include_directories(agc_driver_depth_range_remap_draw_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_depth_bounds_emulation_draw_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
 foreach(stage IN ITEMS vert frag)
     set(agcProvokingShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/ProvokingVertex.${stage})
     set(agcProvokingShaderSpv ${agcTestShaderGeneratedRoot}/ProvokingVertex_${stage}.spv)
@@ -2524,6 +2566,12 @@ if(BUILD_TESTING)
     add_test(NAME agc_shader_frontend_pair_off COMMAND agc_shader_memory_tests --frontend-pair)
     set_tests_properties(agc_shader_frontend_pair PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     set_tests_properties(agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_FRONTEND_PAIR=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_shader_result_memo_budget COMMAND agc_shader_memory_tests --result-memo-budget)
+    set_tests_properties(agc_shader_result_memo_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_MASKED_RESULT_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_shader_result_memo_budget_threads COMMAND agc_shader_memory_tests --result-memo-budget-threads)
+    set_tests_properties(agc_shader_result_memo_budget_threads PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_MASKED_RESULT_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_shader_result_memo_budget_masked COMMAND agc_shader_memory_tests --result-memo-budget-masked)
+    set_tests_properties(agc_shader_result_memo_budget_masked PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_shader_layout_pair COMMAND agc_shader_memory_tests --layout-pair)
     add_test(NAME agc_shader_layout_pair_off COMMAND agc_shader_memory_tests --layout-pair)
     set_tests_properties(agc_shader_layout_pair PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2531,7 +2579,7 @@ if(BUILD_TESTING)
     add_test(NAME agc_shader_entry_lane_proof_budget COMMAND agc_shader_memory_tests --entry-lane-proof-budget)
     set_tests_properties(agc_shader_entry_lane_proof_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     if(WIN32)
-        set_tests_properties(agc_shader_bindless agc_shader_selected_records_image_limit agc_shader_selected_records_pair_limit agc_shader_selected_records_direct_image_limit agc_shader_selected_records_direct_pair_limit agc_shader_srt_read_patching agc_shader_type_zero_image_materialization agc_shader_frontend_pair agc_shader_frontend_pair_off agc_shader_layout_pair agc_shader_layout_pair_off agc_shader_entry_lane_proof_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_shader_bindless agc_shader_selected_records_image_limit agc_shader_selected_records_pair_limit agc_shader_selected_records_direct_image_limit agc_shader_selected_records_direct_pair_limit agc_shader_srt_read_patching agc_shader_type_zero_image_materialization agc_shader_frontend_pair agc_shader_frontend_pair_off agc_shader_layout_pair agc_shader_layout_pair_off agc_shader_result_memo_budget agc_shader_result_memo_budget_threads agc_shader_result_memo_budget_masked agc_shader_entry_lane_proof_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     foreach(agcTest IN ITEMS agc_vertex_input_tests agc_mesh_output_budget_tests agc_shared_write_tests agc_locks_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vertex_input_validation_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_barrier_batch_tests agc_shader_disk_cache_tests agc_screenshot_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests agc_formatted_buffer_byte_stores_tests ${agcExecutionTests} agc_fma_mix_once_tests agc_f16_single_convert_tests)
         if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests" OR agcTest STREQUAL "agc_driver_pipeline_optimized_readback_tests" OR agcTest STREQUAL "agc_driver_rcp_hardware_tests")
@@ -2860,12 +2908,12 @@ if(BUILD_TESTING)
                 ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_driver_f64_denorm_preserve_spirv COMMAND agc_driver_f64_fma_subnormals_tests --verify-spv-denorm-mode)
-    set_tests_properties(agc_driver_f64_denorm_preserve_spirv PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    set_tests_properties(agc_driver_f64_denorm_preserve_spirv PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     if(WIN32)
         set_tests_properties(agc_driver_f64_denorm_preserve_spirv PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_driver_f16_to_f32_unpack_spirv COMMAND agc_driver_f16_to_f32_unpack_tests --verify-spirv)
-    set_tests_properties(agc_driver_f16_to_f32_unpack_spirv PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    set_tests_properties(agc_driver_f16_to_f32_unpack_spirv PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     if(WIN32)
         set_tests_properties(agc_driver_f16_to_f32_unpack_spirv PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
@@ -2897,12 +2945,16 @@ if(BUILD_TESTING)
     set_tests_properties(agc_structurizer_iteration_selection_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_ITERATION_SELECTION=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_shader_control_flow_iteration_selection_off COMMAND agc_shader_control_flow_tests)
     set_tests_properties(agc_shader_control_flow_iteration_selection_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_ITERATION_SELECTION=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_structurizer_dominator_tree_off COMMAND agc_structurizer_tests)
+    set_tests_properties(agc_structurizer_dominator_tree_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_DOMINATOR_TREE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_shader_control_flow_dominator_tree_off COMMAND agc_shader_control_flow_tests)
+    set_tests_properties(agc_shader_control_flow_dominator_tree_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_DOMINATOR_TREE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_structurizer_forced_fallback COMMAND agc_structurizer_tests)
     set_tests_properties(agc_structurizer_forced_fallback PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FORCE_STRUCTURIZER_FALLBACK=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_shader_control_flow_forced_fallback COMMAND agc_shader_control_flow_tests)
     set_tests_properties(agc_shader_control_flow_forced_fallback PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FORCE_STRUCTURIZER_FALLBACK=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     if(WIN32)
-        set_tests_properties(agc_structurizer_intersection_off agc_structurizer_iteration_selection_off agc_shader_control_flow_iteration_selection_off agc_structurizer_forced_fallback agc_shader_control_flow_forced_fallback PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_structurizer_intersection_off agc_structurizer_iteration_selection_off agc_shader_control_flow_iteration_selection_off agc_structurizer_dominator_tree_off agc_shader_control_flow_dominator_tree_off agc_structurizer_forced_fallback agc_shader_control_flow_forced_fallback PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_driver_gated_dispatch_capture_short_timeout COMMAND agc_driver_gated_dispatch_capture_tests)
     set_tests_properties(agc_driver_gated_dispatch_capture_short_timeout PROPERTIES
@@ -3465,8 +3517,7 @@ if(BUILD_TESTING)
         endif()
     endforeach()
 
-    option(ANYPS5_RUN_QUARANTINED_FRAME_REPLAY "Build and register agc_driver_frame_replay on Windows" OFF)
-    if(WIN32 AND ANYPS5_RUN_QUARANTINED_FRAME_REPLAY)
+    if(WIN32)
         add_test_executable(agc_driver_frame_replay_tests tests/FrameReplay.cpp Execution/src/FrameCapture.cpp)
         target_include_directories(agc_driver_frame_replay_tests PRIVATE ${LIBS_INCLUDE_DIR})
         target_link_libraries(agc_driver_frame_replay_tests PRIVATE libSceAgc libSceAgcDriver libkernel libc libzstd_static)
@@ -3479,6 +3530,12 @@ if(BUILD_TESTING)
                 TIMEOUT 120)
         set_tests_properties(agc_driver_frame_replay PROPERTIES ENVIRONMENT_MODIFICATION
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        add_test(NAME agc_driver_frame_replay_concurrent COMMAND agc_driver_frame_replay_tests $<TARGET_FILE:agc_frame_replay> concurrent)
+        set_tests_properties(agc_driver_frame_replay_concurrent PROPERTIES
+                ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FRAME_CAPTURE=${CMAKE_CURRENT_BINARY_DIR}/frame-replay-concurrent-capture;APS5_FRAME_CAPTURE_AT_FLIP=1;APS5_FRAME_CAPTURE_FRAMES=8"
+                TIMEOUT 120)
+        set_tests_properties(agc_driver_frame_replay_concurrent PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 endif()
 

core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp

@@ -33,6 +33,8 @@ extern "C" void AgcDriverSuspendPoint_nid_postfix();
 extern "C" void AgcDriverSerialProgress_nid_postfix(std::uint64_t* accepted, std::uint64_t* completed);
 extern "C" std::size_t AgcDriverAwaitedWaits_nid_postfix(AgcDriver::AwaitedWait* waits, std::size_t capacity);
 extern "C" std::uint64_t AgcDriverDispatchesAdoptedAhead_nid_postfix();
+extern "C" std::uint64_t AgcDriverDispatchesRejectedAhead_nid_postfix();
+extern "C" void AgcDriverSetAheadAdoptHook_nid_postfix(void (*hook)());
 extern "C" void AgcDriverRegisterShader_nid_postfix(const Shader* shader);
 extern "C" void AgcDriverResolveShaderAbi_nid_postfix(const Shader* shader, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);
 extern "C" void AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span<const Shader* const> stages, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp

@@ -2,7 +2,9 @@
 #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DIAGNOSTICS_HPP
 
 #include "prx/libSceAgcDriver/Execution/include/QueueState.hpp"
+#include <cstddef>
 #include <cstdint>
+#include <optional>
 #include <stdexcept>
 #include <string>
 
@@ -21,6 +23,7 @@ public:
 bool SkipRefusedDraws();
 
 void ReportRefusedDraw(const RefusedDraw& refused);
+std::optional<std::size_t> RefusedShaderCount();
 
 void require(bool condition, const char* reason);
 

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -45,6 +45,7 @@
 
 namespace AgcDriver {
 class PerformanceTimer;
+struct OverlayDriverCounters;
 }
 
 namespace AgcDriver::DriverDetail {
@@ -59,6 +60,8 @@ public:
     void Shutdown();
     void WaitIdle();
     std::uint64_t DispatchesAdoptedAhead() const;
+    std::uint64_t DispatchesRejectedAhead() const;
+    void SetAheadAdoptHook(void (*hook)());
     void CheckFailure();
     void ReportFailure(std::exception_ptr error);
     void Submit(const Packet* packet, std::uint32_t queue, bool cpuFlip = false);
@@ -74,7 +77,7 @@ public:
     void RegisterVideoOutput(std::uint32_t handle, const std::shared_ptr<IVideoOutput>& output);
     void UnregisterVideoOutput(std::uint32_t handle, const std::shared_ptr<IVideoOutput>& output);
     void AttachWindow(const PresentationWindow& window);
-    void Present(const PresentationWindow& window, const DisplayBuffer* buffer, bool opaque, void (*gpuReady)(void*), void* context);
+    void Present(const PresentationWindow& window, const DisplayBuffer* buffer, const DisplayBuffer* plane, bool opaque, void (*gpuReady)(void*), void* context);
     void ReleaseWindow(void* window);
     void RegisterShader(const Shader* shader);
     void ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);
@@ -97,6 +100,7 @@ public:
     static void tagVariantRuns(DispatchVariant& variant, std::span<const ShaderRecompiler::MemoryRegion> regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords, std::span<const std::shared_ptr<const ShaderRecompiler::SrtGuestWords>> supplied = {});
     static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request, std::string_view suffix = {});
     DrawEntryCounters DrawCacheCounters();
+    OverlayDriverCounters OverlayCounters();
     EntryCounters DispatchCacheCounters();
     struct DispatchCacheFootprint {
         std::size_t entries = 0;
@@ -317,7 +321,7 @@ private:
         std::chrono::steady_clock::time_point resolvedAt{};
     };
     struct GroupCaptureStats {
-        std::uint64_t runs = 0, resolved = 0, resolvedHits = 0, adopted = 0, addressMismatches = 0, keyMismatches = 0, failures = 0, stale = 0, labelOverlaps = 0, writtenBefore = 0, deviceRejects = 0, argumentMismatches = 0;
+        std::uint64_t runs = 0, resolved = 0, resolvedHits = 0, adopted = 0, addressMismatches = 0, keyMismatches = 0, memoryMismatches = 0, failures = 0, stale = 0, labelOverlaps = 0, writtenBefore = 0, deviceRejects = 0, argumentMismatches = 0;
         double resolveMs = 0, resolveMaxMs = 0, ageMs = 0, ageMaxMs = 0;
         std::chrono::steady_clock::time_point reported = std::chrono::steady_clock::now();
     };
@@ -474,6 +478,8 @@ private:
     std::atomic<std::uint64_t> evidenceReads{0};
     std::atomic<std::uint64_t> evidenceValidations{0};
     std::atomic<std::uint64_t> dispatchesAdoptedAhead{0};
+    std::atomic<std::uint64_t> dispatchesRejectedAhead{0};
+    std::atomic<void (*)()> aheadAdoptHook{nullptr};
 
     AgcDriver::Mutex writtenBuffersMutex;
     std::deque<WrittenBuffer> writtenBuffers;

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp

@@ -227,6 +227,7 @@ void SubmitLibraryPrebuild(std::function<void()> task);
 void AwaitLibraryPrebuilds();
 void CloseRegistrationPreparation();
 unsigned RegistrationPreparationThreads();
+std::optional<std::size_t> RegistrationPreparationsPending();
 
 void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target, bool deferUnprepared = false);
 

core/libs/prx/libSceAgcDriver/Execution/include/FrameRecorder.hpp

@@ -10,6 +10,7 @@
 #include <cstdint>
 #include <deque>
 #include <filesystem>
+#include <functional>
 #include <map>
 #include <memory>
 #include <mutex>
@@ -35,10 +36,10 @@ public:
     void ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType);
     void RegisterVideoOutput(std::uint32_t handle);
     void UnregisterVideoOutput(std::uint32_t handle);
-    void Submit(const Packet& packet, std::uint32_t queue);
-    void Flip(const FlipInfo& info);
+    void Submit(const Packet& packet, std::uint32_t queue, const std::function<void()>& accept);
+    void Flip(const FlipInfo& info, const std::function<void()>& accept);
     void ReserveEopFlip(const FlipInfo& info, std::uint32_t token);
-    void Suspend();
+    void Suspend(const std::function<void()>& accept);
     void PresentBuffer(const DisplayBuffer& buffer);
     void FrameBoundary();
 
@@ -62,7 +63,8 @@ private:
     };
 
     FrameCall make(FrameCallKind kind);
-    void record(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders = {});
+    void record(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders = {}, const std::function<void()>& accept = {});
+    void recordLocked(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders);
     void emitDelta();
     void emitShaderDelta(std::span<const Shader* const> shaders);
     void writeDelta(std::span<const Span> spans, std::span<const std::uint64_t> hostPages, std::span<const Range> added = {});
@@ -72,6 +74,7 @@ private:
     void snapshot(const Range& range);
     void writeReferenceFrames();
     bool abandonIfDiskLow();
+    void stop(const char* reason);
     std::string header() const;
     std::vector<Range> committedRanges();
     std::vector<Range> newRanges();

core/libs/prx/libSceAgcDriver/Execution/include/PerformanceOverlay.hpp

@@ -4,6 +4,7 @@
 #define VK_NO_PROTOTYPES
 #include <vulkan/vulkan.h>
 #include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
+#include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp"
 #include <array>
 #include <atomic>
 #include <chrono>
@@ -13,7 +14,9 @@
 #include <map>
 #include <memory>
 #include <optional>
+#include <span>
 #include <string>
+#include <string_view>
 #include <tuple>
 #include <utility>
 #include <vector>
@@ -26,12 +29,15 @@ class Buffer;
 
 enum class OverlayCorner : std::uint8_t { TopLeft, TopRight, BottomLeft, BottomRight };
 
+inline constexpr std::uint32_t OverlayPageCount = 3;
+
 struct OverlaySettings {
-    bool visible = false;
+    std::uint32_t page = 0;
     OverlayCorner corner = OverlayCorner::TopLeft;
 };
 
 OverlaySettings ParseOverlaySettings(const char* enabled, const char* position);
+std::uint32_t OverlayPageAfter(std::uint32_t start, std::uint64_t presses);
 
 class PerformanceCounters {
 public:
@@ -123,10 +129,26 @@ struct OverlayRect {
 enum class OverlayColor : std::uint32_t { Background = 0, Text = 1, Graph = 2 };
 inline constexpr std::size_t OverlayColorCount = 3;
 
+struct OverlayMetrics {
+    static constexpr std::int32_t GlyphWidth = 5;
+    static constexpr std::int32_t GlyphHeight = 7;
+    static constexpr std::int32_t Advance = 6;
+    static constexpr std::int32_t LinePitch = 9;
+    static constexpr std::int32_t Columns = 27;
+    static constexpr std::int32_t Padding = 3;
+    static constexpr std::int32_t Margin = 4;
+    static constexpr std::int32_t GraphHeight = 20;
+    static constexpr std::int32_t GraphGap = 2;
+    static constexpr std::int32_t InnerWidth = Columns * Advance - 1;
+    static constexpr std::int32_t DetailColumns = 46;
+};
+
 struct OverlayContent {
     std::vector<std::string> above;
     std::vector<double> frameTimes;
     std::vector<std::string> below;
+    std::int32_t columns = OverlayMetrics::Columns;
+    bool graph = true;
 };
 
 struct OverlayMemory {
@@ -141,35 +163,100 @@ struct OverlayDetails {
     VkExtent2D window{};
 };
 
+struct OverlayThreadBusy {
+    std::string name;
+    double percent = 0;
+};
+
+struct OverlayPhase {
+    std::string name;
+    double msPerFrame = 0;
+};
+
+struct OverlayQueueWaits {
+    double memoryMs = 0;
+    double lockMs = 0;
+    double gpuMs = 0;
+};
+
+struct OverlayDetailSample {
+    double fps = 0;
+    double frameMs = 0;
+    std::vector<OverlayThreadBusy> busy;
+    std::vector<OverlayPhase> phases;
+    std::optional<OverlayQueueWaits> queueWaits;
+    std::optional<double> renderPassesPerFrame;
+    std::optional<std::uint64_t> pipelinesCompiling;
+    std::optional<double> pipelinesPerSecond;
+    std::optional<double> drawCacheHitPercent;
+    std::optional<double> pipelineCacheHitPercent;
+    std::optional<double> flipsPerSecond;
+    std::optional<double> presentsPerSecond;
+    std::optional<std::uint64_t> droppedFrames;
+    std::optional<std::uint64_t> repeatedPresents;
+    std::optional<OverlayMemory> commit;
+    std::optional<double> pageFaultsPerSecond;
+    std::optional<std::uint64_t> hostImports;
+    std::optional<OverlayMemory> hostImportBytes;
+    std::optional<double> cpuCopyMegabytesPerSecond;
+    std::optional<double> addressCopyCapPercent;
+    std::optional<double> unimplementedPerSecond;
+    std::optional<std::uint64_t> unimplementedTotal;
+    std::optional<std::uint64_t> refusedShaders;
+    std::string title;
+    std::string presentMode;
+};
+
+struct OverlayHits {
+    std::uint64_t lookups = 0;
+    std::uint64_t hits = 0;
+};
+
+struct OverlayDriverCounters {
+    std::uint64_t flips = 0;
+    std::optional<OverlayHits> drawCache;
+};
+
+struct OverlayPresentInfo {
+    std::uint64_t displayAddress = 0;
+    VkPresentModeKHR presentMode = VK_PRESENT_MODE_MAX_ENUM_KHR;
+    std::uint64_t acquireDrops = 0;
+};
+
+struct OverlayTraceFigures {
+    std::vector<OverlayThreadBusy> busy;
+    std::vector<OverlayPhase> phases;
+    std::optional<OverlayQueueWaits> queueWaits;
+    std::optional<double> renderPassesPerFrame;
+};
+
+inline constexpr std::size_t OverlayTopPhases = 5;
+
 struct OverlayLayout {
     OverlayRect bounds;
     std::vector<OverlayRect> rects;
     std::int32_t scale = 1;
 };
 
-struct OverlayMetrics {
-    static constexpr std::int32_t GlyphWidth = 5;
-    static constexpr std::int32_t GlyphHeight = 7;
-    static constexpr std::int32_t Advance = 6;
-    static constexpr std::int32_t LinePitch = 9;
-    static constexpr std::int32_t Columns = 27;
-    static constexpr std::int32_t Padding = 3;
-    static constexpr std::int32_t Margin = 4;
-    static constexpr std::int32_t GraphHeight = 20;
-    static constexpr std::int32_t GraphGap = 2;
-    static constexpr std::int32_t InnerWidth = Columns * Advance - 1;
-};
-
 const std::array<std::uint8_t, OverlayMetrics::GlyphHeight>& OverlayGlyph(char character);
+std::string_view OverlayFontCharacters();
 std::string FormatOverlayValue(double value);
 std::string FormatOverlayRate(double value);
 std::string FormatOverlayCount(double value);
 std::string FormatOverlayGigabytes(std::uint64_t bytes);
 OverlayContent ComposeOverlay(const OverlayStatistics& statistics, bool gpu, bool driver);
 OverlayContent ComposeOverlay(const OverlayStatistics& statistics, bool gpu, bool driver, const OverlayDetails& details);
+std::string OverlayText(std::string_view text);
+OverlayContent ComposeOverlayDetails(const OverlayDetailSample& sample);
+OverlayTraceFigures OverlayTraceDeltas(std::span<const TraceThreadTotals> before, std::span<const TraceThreadTotals> now, OverlayStatistics::Clock::duration elapsed, std::uint64_t presents, bool passes);
+std::optional<double> OverlayHitPercent(const OverlayHits& before, const OverlayHits& now);
+std::string_view OverlayPresentModeName(VkPresentModeKHR mode);
 bool OverlaySampleDue(std::optional<OverlayStatistics::Clock::time_point> sampledAt, OverlayStatistics::Clock::time_point now);
 std::optional<OverlayMemory> ReadVramBudget(const Graphics::Context& context);
 std::optional<std::uint64_t> ReadProcessPrivateBytes();
+std::optional<OverlayMemory> ParseCommitCharge(std::string_view meminfo);
+std::optional<OverlayMemory> ReadCommitCharge();
+std::optional<std::uint64_t> ReadPageFaults();
 std::int32_t OverlayScale(VkExtent2D extent, const OverlayContent& content);
 OverlayLayout LayOutOverlay(const OverlayContent& content, VkExtent2D extent, OverlayCorner corner);
 std::array<std::array<float, 4>, OverlayColorCount> OverlayPalette(VkFormat format, VkColorSpaceKHR colorSpace);
@@ -182,12 +269,16 @@ public:
     PerformanceOverlay& operator=(const PerformanceOverlay&) = delete;
 
     static bool Visible();
+    static std::uint32_t Page();
     static void Toggle();
     static OverlayCorner Corner();
+    static void SetDriverCounters(OverlayDriverCounters (*source)());
 
-    void NotePresent(OverlayStatistics::Clock::time_point at, VkExtent2D display);
+    void NotePresent(OverlayStatistics::Clock::time_point at, VkExtent2D display, const OverlayPresentInfo& info = {});
     const OverlayStatistics& Statistics() const { return statistics; }
     const OverlayDetails& Details() const { return details; }
+    const OverlayDetailSample& DetailSample() const { return detailSample; }
+    std::uint32_t ShownPage() const { return page; }
     OverlayRect Record(VkCommandBuffer commands, std::size_t slot, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout);
     OverlayRect Record(VkCommandBuffer commands, std::size_t slot, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout, const OverlayContent& content, OverlayCorner corner);
     void Release(std::size_t slot) noexcept;
@@ -203,7 +294,29 @@ private:
     VkPipeline pipeline(VkFormat format, VkRenderPass compatible);
     void releaseTransient(Slot& slot) noexcept;
     void release() noexcept;
+    struct DetailCounters {
+        std::optional<OverlayDriverCounters> driver;
+        OverlayHits pipelines;
+        std::uint64_t copiedBytes = 0;
+        std::optional<std::uint64_t> pageFaults;
+        std::uint64_t unimplemented = 0;
+        std::uint64_t acquireDrops = 0;
+    };
+
+    struct DetailState {
+        std::optional<OverlayStatistics::Clock::time_point> sampledAt;
+        DetailCounters base;
+        std::uint64_t repeats = 0;
+        std::uint64_t lastDisplayAddress = 0;
+        std::optional<OverlayStatistics::Clock::time_point> traceSampledAt;
+        std::vector<TraceThreadTotals> traceBase;
+        std::uint64_t tracePresents = 0;
+    };
+
     void restartIfToggled();
+    void noteDetailPresent(OverlayStatistics::Clock::time_point at, const OverlayPresentInfo& info);
+    void sampleDetails(OverlayStatistics::Clock::time_point at, const OverlayPresentInfo& info);
+    void sampleTrace(OverlayStatistics::Clock::time_point at);
 
     Graphics::Context context;
     VkPipelineLayout pipelineLayout = VK_NULL_HANDLE;
@@ -214,8 +327,11 @@ private:
     std::vector<Slot> slots;
     OverlayStatistics statistics;
     OverlayDetails details;
+    OverlayDetailSample detailSample;
+    DetailState detail;
     std::optional<OverlayStatistics::Clock::time_point> sampledAt;
     std::uint64_t toggles = 0;
+    std::uint32_t page = 0;
 };
 
 }

core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp

@@ -32,6 +32,7 @@ class TraceThread;
 
 extern "C" bool AgcDriverTraceRecording_nid_postfix();
 extern "C" AgcDriver::TraceThread* AgcDriverTraceThread_nid_postfix();
+extern "C" void AgcDriverTraceCount_nid_postfix(const char* scope);
 
 namespace AgcDriver {
 
@@ -769,8 +770,15 @@ private:
     }
 };
 
+struct TraceThreadTotals {
+    std::string name;
+    std::vector<TraceThread::Total> totals;
+};
+
 class PerformanceTrace {
 public:
+    static constexpr const char* RenderPassScope = "Graphics.RenderPass";
+
     static bool Configured() {
         static const bool configured = std::getenv("APS5_TRACE_FRAMES") != nullptr || std::getenv("APS5_TRACE_SECONDS") != nullptr || std::getenv("APS5_TRACE_SUMMARY_EVERY") != nullptr || std::getenv("APS5_TRACE_BENCH") != nullptr;
         return configured;
@@ -797,11 +805,21 @@ public:
         return AgcDriverTraceThread_nid_postfix();
     }
 
+    static void Count(const char* scope) {
+        if (Recording()) AgcDriverTraceCount_nid_postfix(scope);
+    }
+
+    static void SetOverlayTotals(bool on);
+    static bool OverlayTotals();
+    static std::optional<std::vector<TraceThreadTotals>> ThreadTotals();
     static void StopOutput();
 
 private:
     friend bool ::AgcDriverTraceRecording_nid_postfix();
     friend TraceThread* ::AgcDriverTraceThread_nid_postfix();
+    friend void ::AgcDriverTraceCount_nid_postfix(const char* scope);
+
+    static TraceThread* bindNamed();
 
     static void OpenInterval(std::chrono::steady_clock::time_point now, std::uint64_t present);
     static void FlushInterval(std::chrono::steady_clock::time_point now, std::uint32_t width, std::uint32_t height, const VulkanDevice* device);
@@ -810,6 +828,8 @@ private:
     inline static std::atomic<bool> buffering{false};
     inline static std::atomic<std::uint64_t> presentCount{0};
     static thread_local TraceThread* local;
+    static thread_local const char* localName;
+    static std::atomic<TraceThread* (*)()> binder;
     inline static std::mutex gpuMutex;
     inline static std::vector<TraceGpuRange> gpuRanges;
     inline static std::uint64_t gpuLost = 0;

core/libs/prx/libSceAgcDriver/Execution/include/Presentation.hpp

@@ -26,8 +26,8 @@ struct PresentationWindow {
 }
 
 extern "C" void AgcDriverAttachWindow_nid_postfix(const AgcDriver::PresentationWindow& window);
-extern "C" void AgcDriverPresentClear_nid_postfix(const AgcDriver::PresentationWindow& window, bool opaque, void (*gpuReady)(void*), void* context);
-extern "C" void AgcDriverPresentBuffer_nid_postfix(const AgcDriver::PresentationWindow& window, const AgcDriver::DisplayBuffer& buffer, void (*gpuReady)(void*), void* context);
+extern "C" void AgcDriverPresentClear_nid_postfix(const AgcDriver::PresentationWindow& window, bool opaque, const AgcDriver::DisplayBuffer* plane, void (*gpuReady)(void*), void* context);
+extern "C" void AgcDriverPresentBuffer_nid_postfix(const AgcDriver::PresentationWindow& window, const AgcDriver::DisplayBuffer& buffer, const AgcDriver::DisplayBuffer* plane, void (*gpuReady)(void*), void* context);
 extern "C" void AgcDriverReleaseWindow_nid_postfix(void* window);
 extern "C" void AgcDriverTogglePerformanceOverlay_nid_postfix();
 extern "C" void AgcDriverReportFailure_nid_postfix(std::exception_ptr error);

core/libs/prx/libSceAgcDriver/Execution/include/PresentationPass.hpp

@@ -4,9 +4,12 @@
 #define VK_NO_PROTOTYPES
 #include <vulkan/vulkan.h>
 #include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
+#include <cstddef>
 #include <cstdint>
 #include <map>
 #include <memory>
+#include <span>
+#include <tuple>
 #include <utility>
 #include <vector>
 
@@ -24,8 +27,14 @@ public:
     PresentationPass& operator=(const PresentationPass&) = delete;
 
     void Bind(std::size_t slot, VkImage image, std::uint32_t width, std::uint32_t height, std::uint64_t pixelFormat, float whiteNits, std::uint32_t layer = 0);
+    void BindTexels(std::size_t slot, VkCommandBuffer commands, std::span<const std::byte> texels, std::uint32_t width, std::uint32_t height, std::uint64_t pixelFormat, float whiteNits);
     void RecordPresent(VkCommandBuffer commands, std::size_t slot, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout finalLayout);
+    void RecordOverlay(VkCommandBuffer commands, std::size_t slot, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout);
     void RecordDump(VkCommandBuffer commands, std::size_t slot, std::uint32_t stride, VkBuffer destination);
+    void RecordDumpImage(VkCommandBuffer commands, std::size_t slot, std::uint32_t stride);
+    void RecordDumpReadback(VkCommandBuffer commands, std::size_t slot, VkBuffer destination);
+    VkImage DumpImage(std::size_t slot) const;
+    VkExtent2D DumpImageExtent(std::size_t slot) const;
     void Release(std::size_t slot) noexcept;
     static bool TenBitFormat(VkFormat format);
     static VkExtent2D DumpExtent(std::uint32_t width, std::uint32_t height, std::uint32_t stride);
@@ -48,14 +57,21 @@ private:
         VkImageView dumpView = VK_NULL_HANDLE;
         VkFramebuffer dumpFramebuffer = VK_NULL_HANDLE;
         VkExtent2D dumpExtent{};
+        VkImage uploadImage = VK_NULL_HANDLE;
+        VkDeviceMemory uploadMemory = VK_NULL_HANDLE;
+        VkExtent2D uploadExtent{};
+        std::unique_ptr<Graphics::Buffer> uploadTexels;
     };
 
-    const Target& target(VkFormat format, VkImageLayout finalLayout);
+    const Target& target(VkFormat format, VkImageLayout finalLayout, VkImageLayout loadFrom = VK_IMAGE_LAYOUT_UNDEFINED);
     VkImageView createView(VkImage image, VkFormat format, VkImageUsageFlags usage, std::uint32_t layer = 0) const;
     VkFramebuffer createFramebuffer(VkRenderPass renderPass, VkImageView view, VkExtent2D extent) const;
+    VkFramebuffer destinationFramebuffer(Slot& slot, const Target& target, VkImage destination, VkFormat format, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout);
     void ensureDumpTarget(Slot& slot, VkExtent2D extent);
     void releaseDumpTarget(Slot& slot) noexcept;
-    void draw(VkCommandBuffer commands, const Slot& slot, const Target& target, VkFramebuffer framebuffer, VkExtent2D extent, const std::int32_t (&parameters)[9]) const;
+    void ensureUploadImage(Slot& slot, VkExtent2D extent);
+    void releaseUploadImage(Slot& slot) noexcept;
+    void draw(VkCommandBuffer commands, const Slot& slot, const Target& target, VkFramebuffer framebuffer, VkExtent2D extent, VkRect2D scissor, const std::int32_t (&parameters)[9]) const;
     void release() noexcept;
 
     Graphics::Context context;
@@ -66,7 +82,7 @@ private:
     VkShaderModule vertex = VK_NULL_HANDLE;
     VkShaderModule fragment = VK_NULL_HANDLE;
     std::unique_ptr<Graphics::Buffer> pqNits;
-    std::map<std::pair<VkFormat, VkImageLayout>, Target> targets;
+    std::map<std::tuple<VkFormat, VkImageLayout, VkImageLayout>, Target> targets;
     std::vector<Slot> slots;
 };
 

core/libs/prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp

@@ -36,6 +36,8 @@ public:
     static std::vector<AspectFitRect> LetterboxBands(std::uint32_t width, std::uint32_t height, std::uint32_t destinationWidth, std::uint32_t destinationHeight);
     std::uint32_t SourceWidth() const { return sourceWidth; }
     std::uint32_t SourceHeight() const { return sourceHeight; }
+    VkImage SourceImage() const { return sourceImage; }
+    VkFormat SourceFormat() const { return sourceFormat; }
 
 private:
     void release() noexcept;
@@ -46,6 +48,8 @@ private:
     std::uint32_t sourceHeight = 0;
     VkImage sourceImage = VK_NULL_HANDLE;
     VkDeviceMemory sourceMemory = VK_NULL_HANDLE;
+    VkDeviceSize sourceBytes = 0;
+    std::uint32_t sourceMemoryType = 0;
     VkImage blackImage = VK_NULL_HANDLE;
     VkDeviceMemory blackMemory = VK_NULL_HANDLE;
 };

core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp

@@ -259,9 +259,9 @@ public:
     // mutex (the same wait under it is only the fallback when the answer changes in between).
     bool PresentWaitsForSlots(const DisplayBuffer* buffer) const;
     bool AcquireImage();
-    bool PresentClear(std::uint32_t width, std::uint32_t height, bool opaque);
+    bool PresentClear(std::uint32_t width, std::uint32_t height, bool opaque, const DisplayBuffer* plane = nullptr);
     void PresentPixels(std::uint32_t width, std::uint32_t height, std::span<const std::byte> pixels);
-    bool PresentDisplayBuffer(const DisplayBuffer& buffer);
+    bool PresentDisplayBuffer(const DisplayBuffer& buffer, const DisplayBuffer* plane = nullptr);
     double FinishPresent();
     void QueuePresent();
     bool PresentsOnOwnQueue() const;
@@ -275,8 +275,8 @@ public:
     // blit-submit times after the flip packet; noted in-place reads checked
     // (APS5_FLIP_READ_CHECK=1) and found overwritten before the blit's fence.
     struct PresentStatistics {
-        std::uint64_t presents, gpuUnread, batchesAheadOfBlit, batchesAfterFlip, readsChecked, readsOverwritten, acquireDrops;
-        double gpuBusyMs, gpuGapMs, lastSubmitAfterFlipMs, blitSubmitAfterFlipMs;
+        std::uint64_t presents, gpuUnread, batchesAheadOfBlit, batchesAfterFlip, readsChecked, readsOverwritten, acquireDrops, planes, keyWriterWaits;
+        double gpuBusyMs, gpuGapMs, lastSubmitAfterFlipMs, blitSubmitAfterFlipMs, keyWriterWaitMs;
     };
     static PresentStatistics PresentCounts();
     struct PresyncStatistics {
@@ -392,7 +392,7 @@ private:
     // The tail of a dispatch's device call from the open batch's command buffer to the completion
     // registration: keeps, the template data refresh, barriers, bind, push, dispatch, marks.
     void recordDispatch(RecordedDispatch& record);
-    bool present(std::uint32_t width, std::uint32_t height, bool opaque, std::span<const std::byte> pixels, const DisplayBuffer* display = nullptr, const std::shared_ptr<Graphics::StorageTexture>& resident = nullptr, VkFilter residentFilter = VK_FILTER_LINEAR, bool dumpFrame = false, bool residentConvert = false, const VkClearColorValue* uniform = nullptr, std::uint32_t residentLayer = 0, bool passthrough = false);
+    bool present(std::uint32_t width, std::uint32_t height, bool opaque, std::span<const std::byte> pixels, const DisplayBuffer* display = nullptr, const std::shared_ptr<Graphics::StorageTexture>& resident = nullptr, VkFilter residentFilter = VK_FILTER_LINEAR, bool dumpFrame = false, bool residentConvert = false, const VkClearColorValue* uniform = nullptr, std::uint32_t residentLayer = 0, bool passthrough = false, const DisplayBuffer* plane = nullptr);
     bool takeScreenshot(bool dumpFrame);
     void recreateSwapchain(std::uint32_t width, std::uint32_t height, bool force);
     void refreshSurfaceFormats();

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Diagnostics.cpp

@@ -88,18 +88,38 @@ bool SkipRefusedDraws() {
     return skip;
 }
 
+namespace {
+
+struct RefusedShaders {
+    std::mutex mutex;
+    std::set<std::uint64_t> reported;
+};
+
+RefusedShaders& refusedShaders() {
+    static RefusedShaders shaders;
+    return shaders;
+}
+
+}
+
 void ReportRefusedDraw(const RefusedDraw& refused) {
-    static std::mutex reportedMutex;
-    static std::set<std::uint64_t> reported;
+    auto& shaders = refusedShaders();
     {
-        std::lock_guard lock(reportedMutex);
-        if (!reported.insert(refused.address).second) return;
+        std::lock_guard lock(shaders.mutex);
+        if (!shaders.reported.insert(refused.address).second) return;
     }
     const std::string_view what = refused.what();
     const auto line = what.substr(0, what.find('\n'));
     std::fprintf(stderr, "[gpu] APS5_SKIP_REFUSED_DRAWS: skipping the draws that use shader 0x%llx: %.*s\n", static_cast<unsigned long long>(refused.address), static_cast<int>(line.size()), line.data());
 }
 
+std::optional<std::size_t> RefusedShaderCount() {
+    if (!SkipRefusedDraws()) return std::nullopt;
+    auto& shaders = refusedShaders();
+    std::lock_guard lock(shaders.mutex);
+    return shaders.reported.size();
+}
+
 void require(bool condition, const char* reason) {
     if (!condition) {
         throw std::runtime_error(std::string("AGC driver: ") + reason);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp

@@ -16,6 +16,7 @@
 #include "Optimization/ResourceProgram.hpp"
 #include <algorithm>
 #include <cstdlib>
+#include <cstring>
 #include <optional>
 #include <stdexcept>
 #include <utility>
@@ -57,6 +58,19 @@ void ReportDispatchItems(bool profile) {
     std::fprintf(stderr, "[dispatch-item] %llu commit items (%llu recipes rebuilt on the committer); drained instead, cumulative:%s\n", static_cast<unsigned long long>(itemCommits.load(std::memory_order_relaxed)), static_cast<unsigned long long>(itemRebuilds.load(std::memory_order_relaxed)), drained.c_str());
 }
 
+bool CapturedCurrent(std::span<const ShaderRecompiler::MemoryRegion> captured, const ShaderSnapshot& code) {
+    for (const auto& region : captured) {
+        const auto address = region.guestAddress;
+        const auto bytes = region.bytes.size();
+        if (bytes == 0 || (address >= code.codeAddress && address < code.codeAddress + code.code.size() * sizeof(std::uint32_t)) || (!code.header.empty() && address >= code.headerAddress && address < code.headerAddress + code.header.size())) continue;
+        if (Driver::QueryPendingSpan(address, bytes, {}) != ShaderMemory::PendingWrite::None || !GuestMemory::Accessible(reinterpret_cast<const void*>(address), bytes)) return false;
+        GuestMemory::GuardRead(address, bytes);
+        GuestMemory::FlushGpuWrites(address, bytes);
+        if (std::memcmp(region.bytes.data(), reinterpret_cast<const void*>(address), bytes) != 0) return false;
+    }
+    return true;
+}
+
 }
 
 bool Driver::pipelineDispatch() {
@@ -281,6 +295,14 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
         fromAhead = false;
         ++groupStats.keyMismatches;
     }
+    if (fromAhead) {
+        if (const auto hook = aheadAdoptHook.load(std::memory_order_acquire)) hook();
+        if (!CapturedCurrent(ahead.captured, snapshot)) {
+            fromAhead = false;
+            ++dispatchesRejectedAhead;
+            ++groupStats.memoryMismatches;
+        }
+    }
     phaseTiming.Phase(PhaseKey);
     timing.Mark("decode_key_device_setup");
     bool readsClear = false;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp

@@ -42,6 +42,14 @@ std::uint64_t Driver::DispatchesAdoptedAhead() const {
     return dispatchesAdoptedAhead.load(std::memory_order_relaxed);
 }
 
+std::uint64_t Driver::DispatchesRejectedAhead() const {
+    return dispatchesRejectedAhead.load(std::memory_order_relaxed);
+}
+
+void Driver::SetAheadAdoptHook(void (*hook)()) {
+    aheadAdoptHook.store(hook, std::memory_order_release);
+}
+
 namespace {
 
 std::size_t& PreviewedUntil() {
@@ -326,9 +334,9 @@ void Driver::reportGroupCapture(std::uint32_t queue) {
     auto& stats = groupCaptureStats();
     const auto now = std::chrono::steady_clock::now();
     if (now - stats.reported < std::chrono::seconds(10)) return;
-    AgcDriver::ProfilePrint_nid_no_patch("[groupcap] queue 0x%x (10 s): %llu runs resolved %llu dispatches (%llu cache hits) in %.1f ms (max %.2f ms); %llu adopted %.2f ms after the resolve on average, max %.2f ms; mismatches: address %llu key %llu; %llu resolve failures, %llu stale, %llu label overlaps, %llu read an earlier dispatch's output, %llu rejected by the device, %llu dropped because the argument re-read differed\n",
+    AgcDriver::ProfilePrint_nid_no_patch("[groupcap] queue 0x%x (10 s): %llu runs resolved %llu dispatches (%llu cache hits) in %.1f ms (max %.2f ms); %llu adopted %.2f ms after the resolve on average, max %.2f ms; mismatches: address %llu key %llu memory %llu; %llu resolve failures, %llu stale, %llu label overlaps, %llu read an earlier dispatch's output, %llu rejected by the device, %llu dropped because the argument re-read differed\n",
                                          queue, static_cast<unsigned long long>(stats.runs), static_cast<unsigned long long>(stats.resolved), static_cast<unsigned long long>(stats.resolvedHits), stats.resolveMs, stats.resolveMaxMs,
-                                         static_cast<unsigned long long>(stats.adopted), stats.adopted != 0 ? stats.ageMs / static_cast<double>(stats.adopted) : 0.0, stats.ageMaxMs, static_cast<unsigned long long>(stats.addressMismatches), static_cast<unsigned long long>(stats.keyMismatches),
+                                         static_cast<unsigned long long>(stats.adopted), stats.adopted != 0 ? stats.ageMs / static_cast<double>(stats.adopted) : 0.0, stats.ageMaxMs, static_cast<unsigned long long>(stats.addressMismatches), static_cast<unsigned long long>(stats.keyMismatches), static_cast<unsigned long long>(stats.memoryMismatches),
                                          static_cast<unsigned long long>(stats.failures), static_cast<unsigned long long>(stats.stale), static_cast<unsigned long long>(stats.labelOverlaps), static_cast<unsigned long long>(stats.writtenBefore), static_cast<unsigned long long>(stats.deviceRejects), static_cast<unsigned long long>(stats.argumentMismatches));
     stats = GroupCaptureStats{};
     stats.reported = now;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCacheStatistics.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/PerformanceOverlay.hpp"
 #include <cstdlib>
 
 namespace AgcDriver::DriverDetail {
@@ -46,6 +47,13 @@ DrawEntryCounters Driver::DrawCacheCounters() {
     return drawEntryCounters;
 }
 
+OverlayDriverCounters Driver::OverlayCounters() {
+    OverlayDriverCounters counters{flipsCounted.load(std::memory_order_relaxed), std::nullopt};
+    std::unique_lock cacheLock(drawCacheMutex, std::try_to_lock);
+    if (cacheLock.owns_lock()) counters.drawCache = OverlayHits{drawEntryCounters.lookups, drawEntryCounters.hits};
+    return counters;
+}
+
 std::string Driver::profileDrawCacheLines(const DrawEntryCounters& counters) {
     std::string text;
     const auto append = [&](const char* format, auto... values) {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp

@@ -183,6 +183,7 @@ std::shared_ptr<DrawDecode> Driver::decodeDraw(const QueueState& queue, const Su
     product->pixel = Graphics::DecodePixelStageInfo(queue.context, Graphics::ExportMappings(product->state), Graphics::PixelProgramSkipped(queue));
     Graphics::NormalizeUnusedViewportDepth(product->state, product->pixel);
     Graphics::ApplyDepthRemap(context, product->state);
+    Graphics::ApplyDepthBoundsEmulation(context, product->state, product->pixel);
     product->pixel.targetExportPacking = Graphics::ExportPackings(product->state);
     product->pixel.dualSourceBlend = product->state.dualSourceBlend;
     product->pixel.reversedBlend = product->state.reversedBlend;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp

@@ -19,9 +19,8 @@ void Submit(const Packet* packet, std::uint32_t queue) try {
         return;
     }
     GuestMemory::CheckRange(packet, sizeof(Packet), alignof(Packet));
-    recorder->Submit(*packet, queue);
     const auto flips = driver.FlipSubmissions();
-    driver.Submit(packet, queue);
+    recorder->Submit(*packet, queue, [&] { driver.Submit(packet, queue); });
     if (driver.FlipSubmissions() != flips) recorder->FrameBoundary();
 } catch (const ProcessShutdown&) {
     LibcAwaitExit_nid_postfix();
@@ -41,8 +40,12 @@ void RegisterShader(const Shader* shader) {
 }
 
 void SuspendPoint() {
-    if (auto* const recorder = FrameRecorder::Active()) recorder->Suspend();
-    DriverDetail::Driver::Get().SuspendPoint();
+    auto* const recorder = FrameRecorder::Active();
+    if (recorder == nullptr) {
+        DriverDetail::Driver::Get().SuspendPoint();
+        return;
+    }
+    recorder->Suspend([] { DriverDetail::Driver::Get().SuspendPoint(); });
 }
 
 void RegisterVideoOutput(std::uint32_t handle, const std::shared_ptr<IVideoOutput>& output) {
@@ -59,13 +62,13 @@ void AttachWindow(const PresentationWindow& window) {
     DriverDetail::Driver::Get().AttachWindow(window);
 }
 
-void PresentClear(const PresentationWindow& window, bool opaque, void (*gpuReady)(void*), void* context) {
-    DriverDetail::Driver::Get().Present(window, nullptr, opaque, gpuReady, context);
+void PresentClear(const PresentationWindow& window, bool opaque, const DisplayBuffer* plane, void (*gpuReady)(void*), void* context) {
+    DriverDetail::Driver::Get().Present(window, nullptr, plane, opaque, gpuReady, context);
 }
 
-void PresentBuffer(const PresentationWindow& window, const DisplayBuffer& buffer, void (*gpuReady)(void*), void* context) {
+void PresentBuffer(const PresentationWindow& window, const DisplayBuffer& buffer, const DisplayBuffer* plane, void (*gpuReady)(void*), void* context) {
     if (auto* const recorder = FrameRecorder::Active()) recorder->PresentBuffer(buffer);
-    DriverDetail::Driver::Get().Present(window, &buffer, true, gpuReady, context);
+    DriverDetail::Driver::Get().Present(window, &buffer, plane, true, gpuReady, context);
 }
 
 void ReleaseWindow(void* window) {
@@ -146,9 +149,12 @@ extern "C" void AgcDriverSerialProgress_nid_postfix(std::uint64_t* accepted, std
 
 extern "C" void AgcDriverSubmitFlip_nid_postfix(const AgcDriver::FlipInfo& info) try {
     auto* const recorder = AgcDriver::FrameRecorder::Active();
-    if (recorder != nullptr) recorder->Flip(info);
-    AgcDriver::DriverDetail::Driver::Get().SubmitFlip(info);
-    if (recorder != nullptr) recorder->FrameBoundary();
+    if (recorder == nullptr) {
+        AgcDriver::DriverDetail::Driver::Get().SubmitFlip(info);
+        return;
+    }
+    recorder->Flip(info, [&] { AgcDriver::DriverDetail::Driver::Get().SubmitFlip(info); });
+    recorder->FrameBoundary();
 } catch (const ProcessShutdown&) {
     LibcAwaitExit_nid_postfix();
 }
@@ -157,6 +163,14 @@ extern "C" std::uint64_t AgcDriverDispatchesAdoptedAhead_nid_postfix() {
     return AgcDriver::DriverDetail::Driver::Get().DispatchesAdoptedAhead();
 }
 
+extern "C" std::uint64_t AgcDriverDispatchesRejectedAhead_nid_postfix() {
+    return AgcDriver::DriverDetail::Driver::Get().DispatchesRejectedAhead();
+}
+
+extern "C" void AgcDriverSetAheadAdoptHook_nid_postfix(void (*hook)()) {
+    AgcDriver::DriverDetail::Driver::Get().SetAheadAdoptHook(hook);
+}
+
 extern "C" void AgcDriverRegisterVideoOutput_nid_postfix(std::uint32_t handle, const std::shared_ptr<AgcDriver::IVideoOutput>& output) {
     AgcDriver::RegisterVideoOutput(handle, output);
 }
@@ -169,12 +183,12 @@ extern "C" void AgcDriverAttachWindow_nid_postfix(const AgcDriver::PresentationW
     AgcDriver::AttachWindow(window);
 }
 
-extern "C" void AgcDriverPresentClear_nid_postfix(const AgcDriver::PresentationWindow& window, bool opaque, void (*gpuReady)(void*), void* context) {
-    AgcDriver::PresentClear(window, opaque, gpuReady, context);
+extern "C" void AgcDriverPresentClear_nid_postfix(const AgcDriver::PresentationWindow& window, bool opaque, const AgcDriver::DisplayBuffer* plane, void (*gpuReady)(void*), void* context) {
+    AgcDriver::PresentClear(window, opaque, plane, gpuReady, context);
 }
 
-extern "C" void AgcDriverPresentBuffer_nid_postfix(const AgcDriver::PresentationWindow& window, const AgcDriver::DisplayBuffer& buffer, void (*gpuReady)(void*), void* context) {
-    AgcDriver::PresentBuffer(window, buffer, gpuReady, context);
+extern "C" void AgcDriverPresentBuffer_nid_postfix(const AgcDriver::PresentationWindow& window, const AgcDriver::DisplayBuffer& buffer, const AgcDriver::DisplayBuffer* plane, void (*gpuReady)(void*), void* context) {
+    AgcDriver::PresentBuffer(window, buffer, plane, gpuReady, context);
 }
 
 extern "C" void AgcDriverReleaseWindow_nid_postfix(void* window) {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp

@@ -6,6 +6,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp"
+#include "prx/libSceAgcDriver/Execution/include/PerformanceOverlay.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
@@ -230,6 +231,7 @@ Driver::Driver() {
         gpuMapImportThreads.fetch_add(1, std::memory_order_relaxed);
         host.join();
     });
+    PerformanceOverlay::SetDriverCounters([] { return Driver::Get().OverlayCounters(); });
 }
 
 void Driver::WaitIdle() {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp

@@ -71,8 +71,8 @@ bool keepsDrawPipeline(std::span<const std::uint32_t> packet, std::uint32_t head
         }
         case 0x3c: case 0x93: {
             if (packet.size() < 5) return false;
-            const std::uint64_t awaited = packet[2] | (static_cast<std::uint64_t>(packet[3]) << 32u);
             const auto bytes = Pm4::WaitAwaitedBytes(packet);
+            const std::uint64_t awaited = (packet[2] & (bytes == 8 ? ~7u : ~3u)) | (static_cast<std::uint64_t>(packet[3]) << 32u);
             if (DrawPipeline::Queue0().Overlaps(awaited, bytes)) return false;
             GuestMemory::CheckRange(reinterpret_cast<const void*>(awaited), bytes, bytes);
             return Pm4::WaitSatisfiedUnchecked(packet);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Presentation.cpp

@@ -70,7 +70,7 @@ void Driver::AttachWindow(const PresentationWindow& window) {
     device = std::make_shared<VulkanDevice>(&window);
 }
 
-void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buffer, bool opaque, void (*gpuReady)(void*), void* context) {
+void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buffer, const DisplayBuffer* plane, bool opaque, void (*gpuReady)(void*), void* context) {
     if (PerformanceTrace::Configured()) PerformanceTrace::Present(window.width, window.height, device.Load().get());
     PerformanceContext timingContext(window.timing.get());
     PerformanceTimer timing("Driver.Present");
@@ -130,10 +130,10 @@ void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buff
                     presenting->WaitIdle();
                     timing.Mark("device_idle_wait");
                 }
-                submitted = presenting->PresentDisplayBuffer(*buffer);
+                submitted = presenting->PresentDisplayBuffer(*buffer, plane);
                 timing.Mark("present_display_buffer");
             } else {
-                submitted = presenting->PresentClear(window.width, window.height, opaque);
+                submitted = presenting->PresentClear(window.width, window.height, opaque, plane);
                 timing.Mark("present_clear");
             }
             if (submitted && syncFlip) {
@@ -206,7 +206,7 @@ void Driver::reportPresents(double waitedMs, std::size_t inFlight) {
     const auto flips = flipsCounted.load() - previousFlips;
     const double perFlip = flips != 0 ? 1.0 / static_cast<double>(flips) : 0.0;
     const auto serial = flipSerial.load();
-    AgcDriver::ProfilePrint_nid_no_patch( "[present] %llu presents over 10 s (%llu dropped: no free swapchain image; %zu may trail on the GPU); %s p50 %.2f ms, p90 %.2f ms, max %.2f ms over %zu; per frame, presented or dropped: GPU busy %.1f ms, idle gaps %.1f ms (%llu batches without a completion record); per flip: %.1f batches recorded (%.1f unsignaled at the flip); per present: %.1f batches ahead of the blit, %.1f after it; after the flip packet: last submit %.1f ms, blit submit %.1f ms; in-place reads overwritten before execution: %llu of %llu checked\n", static_cast<unsigned long long>(presents), static_cast<unsigned long long>(dropped), inFlight, inFlight != 0 ? "inflight_wait" : "render_fence_wait", percentile(0.5), percentile(0.9), waits.empty() ? 0.0 : waits.back(), waits.size(), (counts.gpuBusyMs - previous.gpuBusyMs) * perFrame, (counts.gpuGapMs - previous.gpuGapMs) * perFrame, static_cast<unsigned long long>(counts.gpuUnread - previous.gpuUnread), static_cast<double>(serial - windowSerial) * perFlip, static_cast<double>(flipBatchesUnsignaled.load() - previousUnsignaled) * perFlip, static_cast<double>(counts.batchesAheadOfBlit - previous.batchesAheadOfBlit) * perPresent, static_cast<double>(counts.batchesAfterFlip - previous.batchesAfterFlip) * perPresent, (counts.lastSubmitAfterFlipMs - previous.lastSubmitAfterFlipMs) * perPresent, (counts.blitSubmitAfterFlipMs - previous.blitSubmitAfterFlipMs) * perPresent, static_cast<unsigned long long>(counts.readsOverwritten - previous.readsOverwritten), static_cast<unsigned long long>(counts.readsChecked - previous.readsChecked));
+    AgcDriver::ProfilePrint_nid_no_patch( "[present] %llu presents over 10 s (%llu dropped: no free swapchain image; %zu may trail on the GPU); %s p50 %.2f ms, p90 %.2f ms, max %.2f ms over %zu; per frame, presented or dropped: GPU busy %.1f ms, idle gaps %.1f ms (%llu batches without a completion record); per flip: %.1f batches recorded (%.1f unsignaled at the flip); per present: %.1f batches ahead of the blit, %.1f after it; after the flip packet: last submit %.1f ms, blit submit %.1f ms; in-place reads overwritten before execution: %llu of %llu checked; waits for the display keys' writer outside the GPU mutex: %llu, %.1f ms\n", static_cast<unsigned long long>(presents), static_cast<unsigned long long>(dropped), inFlight, inFlight != 0 ? "inflight_wait" : "render_fence_wait", percentile(0.5), percentile(0.9), waits.empty() ? 0.0 : waits.back(), waits.size(), (counts.gpuBusyMs - previous.gpuBusyMs) * perFrame, (counts.gpuGapMs - previous.gpuGapMs) * perFrame, static_cast<unsigned long long>(counts.gpuUnread - previous.gpuUnread), static_cast<double>(serial - windowSerial) * perFlip, static_cast<double>(flipBatchesUnsignaled.load() - previousUnsignaled) * perFlip, static_cast<double>(counts.batchesAheadOfBlit - previous.batchesAheadOfBlit) * perPresent, static_cast<double>(counts.batchesAfterFlip - previous.batchesAfterFlip) * perPresent, (counts.lastSubmitAfterFlipMs - previous.lastSubmitAfterFlipMs) * perPresent, (counts.blitSubmitAfterFlipMs - previous.blitSubmitAfterFlipMs) * perPresent, static_cast<unsigned long long>(counts.readsOverwritten - previous.readsOverwritten), static_cast<unsigned long long>(counts.readsChecked - previous.readsChecked), static_cast<unsigned long long>(counts.keyWriterWaits - previous.keyWriterWaits), counts.keyWriterWaitMs - previous.keyWriterWaitMs);
     previous = counts;
     previousFlips = flipsCounted.load();
     previousUnsignaled = flipBatchesUnsignaled.load();

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp

@@ -146,7 +146,7 @@ void Driver::run(std::uint32_t id) noexcept {
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
     auto& costs = submissionCosts(id);
     try {
-        if (PerformanceTrace::Configured()) PerformanceTrace::BindThread(role);
+        PerformanceTrace::BindThread(role);
         for (;;) {
             submission = Submission{};
             struct Running {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp

@@ -138,7 +138,7 @@ std::shared_ptr<const ShaderSnapshot> ReadRawShader(std::uint64_t address) {
                 cache.erase(found);
             }
             const auto bytes = result->code.size() * sizeof(std::uint32_t);
-            while (!cache.empty() && (cache.size() >= 64 || cacheBytes + bytes > 8 * 1024 * 1024)) {
+            while (!cache.empty() && (cache.size() >= 4096 || cacheBytes + bytes > 64 * 1024 * 1024)) {
                 cacheBytes -= cache.back()->code.size() * sizeof(std::uint32_t);
                 cache.pop_back();
             }
@@ -1498,6 +1498,17 @@ public:
         return liveWorkers.load(std::memory_order_acquire);
     }
 
+    static std::optional<std::size_t> Pending() {
+        std::size_t pending = 0;
+        for (auto* pool : {Existing(), ExistingPrebuild()}) {
+            if (pool == nullptr) continue;
+            std::unique_lock lock(pool->mutex, std::try_to_lock);
+            if (!lock.owns_lock()) return std::nullopt;
+            pending += pool->tasks.size() + pool->running;
+        }
+        return pending;
+    }
+
     static void CloseAll() {
         std::array<RegistrationPreparePool*, 2> pools{};
         {
@@ -1637,7 +1648,7 @@ private:
 
     static void* Main(void* argument) {
         auto& pool = *static_cast<RegistrationPreparePool*>(argument);
-        if (PerformanceTrace::Configured()) PerformanceTrace::BindThread("shader prepare", TraceThread::DefaultCapacity / 8);
+        PerformanceTrace::BindThread("shader prepare", TraceThread::DefaultCapacity / 8);
         for (;;) {
             Task task;
             {
@@ -1750,6 +1761,10 @@ unsigned RegistrationPreparationThreads() {
     return RegistrationPreparePool::LiveWorkers();
 }
 
+std::optional<std::size_t> RegistrationPreparationsPending() {
+    return RegistrationPreparePool::Pending();
+}
+
 std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target, bool deferUnprepared) {
     ShaderPreparationTransaction transaction;
     require(decoded.programs.size() == decoded.roles.size(), "graphics ABI program roles are incomplete");

core/libs/prx/libSceAgcDriver/Execution/src/FrameRecorder.cpp

@@ -163,20 +163,20 @@ void FrameRecorder::UnregisterVideoOutput(std::uint32_t handle) {
     record(std::move(call), true);
 }
 
-void FrameRecorder::Submit(const Packet& packet, std::uint32_t queue) {
+void FrameRecorder::Submit(const Packet& packet, std::uint32_t queue, const std::function<void()>& accept) {
     auto call = make(FrameCallKind::Submit);
     call.queue = queue;
     call.address = pointer(packet.addr);
     call.size = packet.dw_num;
     call.flags = packet.flags;
-    record(std::move(call), false);
+    record(std::move(call), false, {}, accept);
 }
 
-void FrameRecorder::Flip(const FlipInfo& info) {
+void FrameRecorder::Flip(const FlipInfo& info, const std::function<void()>& accept) {
     auto call = make(FrameCallKind::Flip);
     call.cpuFlip = true;
     call.values = {info.handle, flipValue(info.index), info.mode, flipValue(info.argument)};
-    record(std::move(call), false);
+    record(std::move(call), false, {}, accept);
 }
 
 void FrameRecorder::ReserveEopFlip(const FlipInfo& info, std::uint32_t token) {
@@ -198,8 +198,8 @@ void FrameRecorder::ReserveEopFlip(const FlipInfo& info, std::uint32_t token) {
     ++recordedCalls;
 }
 
-void FrameRecorder::Suspend() {
-    record(make(FrameCallKind::Suspend), false);
+void FrameRecorder::Suspend(const std::function<void()>& accept) {
+    record(make(FrameCallKind::Suspend), false, {}, accept);
 }
 
 void FrameRecorder::PresentBuffer(const DisplayBuffer& buffer) {
@@ -236,8 +236,13 @@ void FrameRecorder::writeReferenceFrames() {
     }
 }
 
-void FrameRecorder::record(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders) {
+void FrameRecorder::record(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders, const std::function<void()>& accept) {
     std::lock_guard lock(mutex);
+    recordLocked(std::move(call), prologueCall, shaders);
+    if (accept) accept();
+}
+
+void FrameRecorder::recordLocked(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders) {
     if (state == State::Waiting) {
         if (!prologueCall) return;
         call.ordinal = ++ordinal;
@@ -360,10 +365,23 @@ void FrameRecorder::emitShaderDelta(std::span<const Shader* const> shaders) {
 
 void FrameRecorder::writeDelta(std::span<const Span> spans, std::span<const std::uint64_t> hostPages, std::span<const Range> added) {
     if (abandonIfDiskLow()) return;
-    const auto [accepted, completed] = DriverDetail::Driver::Get().SerialProgress();
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> blocks;
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> written;
+    for (const auto& span : spans) {
+        GuestMemory::CollectWrittenBlocks(span.address, static_cast<std::size_t>(span.bytes), span.since, written);
+        blocks.insert(blocks.end(), written.begin(), written.end());
+    }
+    auto& driver = DriverDetail::Driver::Get();
+    auto [accepted, completed] = driver.SerialProgress();
+    if (completed < accepted && (!added.empty() || !blocks.empty())) {
+        if (!driver.WaitIdleFor(std::chrono::seconds(10))) {
+            stop("stopped: the GPU work submitted before a memory delta did not finish within 10 s");
+            return;
+        }
+        std::tie(accepted, completed) = driver.SerialProgress();
+    }
     const auto group = deltaGroups + 1;
     bool any = false;
-    std::vector<std::pair<std::uint64_t, std::uint64_t>> written;
     std::vector<std::byte> bytes;
     for (const auto& range : added) {
         writer->WriteRange(range.address, range.bytes, range.kind, range.protection, range.access);
@@ -397,15 +415,12 @@ void FrameRecorder::writeDelta(std::span<const Span> spans, std::span<const std:
         deltaBytes += bytes.size();
         any = true;
     }
-    for (const auto& span : spans) {
-        GuestMemory::CollectWrittenBlocks(span.address, static_cast<std::size_t>(span.bytes), span.since, written);
-        for (const auto& [begin, end] : written) {
-            bytes.resize(static_cast<std::size_t>(end - begin));
-            GuestMemory::ReadCommittedUnsynced(begin, bytes);
-            writer->WriteDelta(group, completed, begin, bytes);
-            deltaBytes += bytes.size();
-            any = true;
-        }
+    for (const auto& [begin, end] : blocks) {
+        bytes.resize(static_cast<std::size_t>(end - begin));
+        GuestMemory::ReadCommittedUnsynced(begin, bytes);
+        writer->WriteDelta(group, completed, begin, bytes);
+        deltaBytes += bytes.size();
+        any = true;
     }
     if (!any) return;
     deltaGroups = group;
@@ -436,8 +451,13 @@ bool FrameRecorder::abandonIfDiskLow() {
     std::error_code error;
     const auto space = std::filesystem::space(directory, error);
     if (error || space.available >= MinimumFreeBytes) return false;
-    warnings.push_back("stopped: less than 4 GiB free on the capture volume");
-    std::fprintf(stderr, "[frame-capture] stopped: less than 4 GiB free on the capture volume\n");
+    stop("stopped: less than 4 GiB free on the capture volume");
+    return true;
+}
+
+void FrameRecorder::stop(const char* reason) {
+    warnings.push_back(reason);
+    std::fprintf(stderr, "[frame-capture] %s\n", reason);
     try {
         writer->Finish(header());
     } catch (const std::exception& failure) {
@@ -445,7 +465,6 @@ bool FrameRecorder::abandonIfDiskLow() {
     }
     writer.reset();
     state = State::Done;
-    return true;
 }
 
 void FrameRecorder::start() {

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -1825,12 +1825,26 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function
             hashes[static_cast<std::size_t>((cursor - first) / page)] = HashPage(reinterpret_cast<const std::byte*>(cursor));
         }
     }
-    std::erase_if(spans, [&](const auto& span) { return !walkWrites(tracker, span.first, span.second, StampKind::Cpu, true); });
+    const auto saveBytes = [&](std::uint64_t begin, std::uint64_t end) {
+        lock.Suspend();
+        beforeUnwatch(begin, static_cast<std::size_t>(end - begin));
+        lock.Resume();
+    };
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> leaving;
+    std::erase_if(spans, [&](const auto& span) {
+        if (walkWrites(tracker, span.first, span.second, StampKind::Cpu, true)) return false;
+        leaving.push_back(span);
+        return true;
+    });
+    for (const auto& [begin, end] : leaving) saveBytes(begin, end);
     if (!runImport()) return false;
     const WindowCopy window{first, hashes};
     auto unwatchedFrom = first;
     for (const auto& [begin, end] : spans) {
-        if (!walkWrites(tracker, begin, end, StampKind::Cpu, true, &window)) continue;
+        if (!walkWrites(tracker, begin, end, StampKind::Cpu, true, &window)) {
+            saveBytes(begin, end);
+            continue;
+        }
 #ifdef _WIN32
         GuestArena::GuestArenaCompareResidentWhenShared_nid_no_patch(static_cast<std::uintptr_t>(begin), static_cast<std::size_t>(end - begin));
 #endif

core/libs/prx/libSceAgcDriver/Execution/src/PerformanceOverlay.cpp

@@ -1,19 +1,27 @@
 #include "prx/libSceAgcDriver/Execution/include/PerformanceOverlay.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Presentation.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PqToSdr.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
 #include "prx/libSceAgcDriver/Graphics/shaders/PerformanceOverlayFrag_spv.h"
 #include "prx/libSceAgcDriver/Graphics/shaders/PerformanceOverlayVert_spv.h"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
+#include "prx/libc/include/General.hpp"
 #include "prx/libc/include/HostThreadLocal.hpp"
+#include "prx/libkernel/AppMetadata/include/AppMetadata.hpp"
 #include <algorithm>
 #include <atomic>
 #include <cmath>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <charconv>
 #include <initializer_list>
+#include <map>
 #include <mutex>
 #include <stdexcept>
 #include <string_view>
@@ -24,6 +32,8 @@
 #endif
 #include <windows.h>
 #include <psapi.h>
+#else
+#include <sys/resource.h>
 #endif
 
 namespace AgcDriver {
@@ -37,9 +47,11 @@ struct OverlayParameters {
 };
 
 std::atomic<std::uint64_t> toggleCount{0};
+std::atomic<std::uint32_t> currentPage{0};
 std::atomic<std::uint64_t> gpuTotal{0};
 std::atomic<std::uint64_t> driverTotal{0};
 std::atomic<std::uint64_t> pipelineTotal{0};
+std::atomic<OverlayDriverCounters (*)()> driverCounters{nullptr};
 
 struct PacketCounterRegistry {
     std::mutex mutex;
@@ -65,23 +77,45 @@ struct PacketCounterLease {
 constexpr double GigabyteBytes = 1024.0 * 1024.0 * 1024.0;
 
 const OverlaySettings& settings() {
-    static const OverlaySettings parsed = ParseOverlaySettings(std::getenv("ANYPS5_OVERLAY"), std::getenv("ANYPS5_OVERLAY_POS"));
+    static const OverlaySettings parsed = [] {
+        const auto read = ParseOverlaySettings(std::getenv("ANYPS5_OVERLAY"), std::getenv("ANYPS5_OVERLAY_POS"));
+        currentPage.store(read.page, std::memory_order_relaxed);
+        if (read.page == 2) PerformanceTrace::SetOverlayTotals(true);
+        return read;
+    }();
     return parsed;
 }
 
-std::int32_t panelWidth(std::int32_t scale) {
-    return (OverlayMetrics::InnerWidth + 2 * OverlayMetrics::Padding) * scale;
+std::string busyName(std::string_view thread) {
+    auto queue = thread.substr(TraceFormat::WorkerPrefix.size());
+    if (queue.starts_with("0x")) queue.remove_prefix(2);
+    return "Q" + std::string(queue);
+}
+
+std::optional<std::uint64_t> meminfoKilobytes(std::string_view line, std::string_view name) {
+    if (!line.starts_with(name)) return std::nullopt;
+    auto rest = line.substr(name.size());
+    while (!rest.empty() && rest.front() == ' ') rest.remove_prefix(1);
+    std::uint64_t value = 0;
+    const auto [last, error] = std::from_chars(rest.data(), rest.data() + rest.size(), value);
+    if (error != std::errc{} || std::string_view(last, static_cast<std::size_t>(rest.data() + rest.size() - last)) != " kB") return std::nullopt;
+    return value * 1024u;
+}
+
+std::int32_t panelWidth(std::int32_t columns, std::int32_t scale) {
+    return (columns * OverlayMetrics::Advance - 1 + 2 * OverlayMetrics::Padding) * scale;
 }
 
 std::int32_t panelHeight(const OverlayContent& content, std::int32_t scale) {
     const auto lines = static_cast<std::int32_t>(content.above.size() + content.below.size());
     const auto glyphGap = OverlayMetrics::LinePitch - OverlayMetrics::GlyphHeight;
-    return (2 * OverlayMetrics::Padding + lines * OverlayMetrics::LinePitch + OverlayMetrics::GraphHeight + OverlayMetrics::GraphGap - glyphGap) * scale;
+    const auto graph = content.graph ? OverlayMetrics::GraphHeight + OverlayMetrics::GraphGap : 0;
+    return (2 * OverlayMetrics::Padding + lines * OverlayMetrics::LinePitch + graph - glyphGap) * scale;
 }
 
 bool fits(VkExtent2D extent, const OverlayContent& content, std::int32_t scale) {
     const auto margins = 2 * OverlayMetrics::Margin * scale;
-    return panelWidth(scale) + margins <= static_cast<std::int32_t>(extent.width) && panelHeight(content, scale) + margins <= static_cast<std::int32_t>(extent.height);
+    return panelWidth(content.columns, scale) + margins <= static_cast<std::int32_t>(extent.width) && panelHeight(content, scale) + margins <= static_cast<std::int32_t>(extent.height);
 }
 
 float srgbDecode(float value) {
@@ -106,9 +140,10 @@ bool srgbFormat(VkFormat format) {
 
 OverlaySettings ParseOverlaySettings(const char* enabled, const char* position) {
     OverlaySettings parsed;
-    const std::string_view visible = enabled != nullptr ? enabled : "";
-    if (visible == "1") parsed.visible = true;
-    else if (!visible.empty() && visible != "0") throw std::runtime_error("ANYPS5_OVERLAY must be 1 or 0, not '" + std::string(visible) + "'");
+    const std::string_view page = enabled != nullptr ? enabled : "";
+    if (page == "1") parsed.page = 1;
+    else if (page == "2") parsed.page = 2;
+    else if (!page.empty() && page != "0") throw std::runtime_error("ANYPS5_OVERLAY must be 0, 1 or 2, not '" + std::string(page) + "'");
     const std::string_view corner = position != nullptr ? position : "";
     if (corner.empty() || corner == "tl") parsed.corner = OverlayCorner::TopLeft;
     else if (corner == "tr") parsed.corner = OverlayCorner::TopRight;
@@ -118,6 +153,11 @@ OverlaySettings ParseOverlaySettings(const char* enabled, const char* position)
     return parsed;
 }
 
+std::uint32_t OverlayPageAfter(std::uint32_t start, std::uint64_t presses) {
+    Graphics::Require(start < OverlayPageCount, "the performance overlay has no such page");
+    return static_cast<std::uint32_t>((start + presses % OverlayPageCount) % OverlayPageCount);
+}
+
 void PerformanceCounters::AddGpuNanoseconds(std::uint64_t nanoseconds) {
     gpuTotal.fetch_add(nanoseconds, std::memory_order_relaxed);
 }
@@ -289,6 +329,10 @@ const std::array<std::uint8_t, OverlayMetrics::GlyphHeight>& OverlayGlyph(char c
     static constexpr Glyph period{0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x0c};
     static constexpr Glyph colon{0x00, 0x0c, 0x0c, 0x00, 0x0c, 0x0c, 0x00};
     static constexpr Glyph slash{0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x00};
+    static constexpr Glyph percent{0x18, 0x19, 0x02, 0x04, 0x08, 0x13, 0x03};
+    static constexpr Glyph plus{0x00, 0x04, 0x04, 0x1f, 0x04, 0x04, 0x00};
+    static constexpr Glyph minus{0x00, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00};
+    static constexpr Glyph underscore{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f};
     static constexpr Glyph a{0x0e, 0x11, 0x11, 0x11, 0x1f, 0x11, 0x11};
     static constexpr Glyph b{0x1e, 0x11, 0x11, 0x1e, 0x11, 0x11, 0x1e};
     static constexpr Glyph c{0x0e, 0x11, 0x10, 0x10, 0x10, 0x11, 0x0e};
@@ -298,12 +342,14 @@ const std::array<std::uint8_t, OverlayMetrics::GlyphHeight>& OverlayGlyph(char c
     static constexpr Glyph g{0x0e, 0x11, 0x10, 0x17, 0x11, 0x11, 0x0f};
     static constexpr Glyph h{0x11, 0x11, 0x11, 0x1f, 0x11, 0x11, 0x11};
     static constexpr Glyph i{0x0e, 0x04, 0x04, 0x04, 0x04, 0x04, 0x0e};
+    static constexpr Glyph j{0x07, 0x02, 0x02, 0x02, 0x02, 0x12, 0x0c};
     static constexpr Glyph k{0x11, 0x12, 0x14, 0x18, 0x14, 0x12, 0x11};
     static constexpr Glyph l{0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x1f};
     static constexpr Glyph m{0x11, 0x1b, 0x15, 0x15, 0x11, 0x11, 0x11};
     static constexpr Glyph n{0x11, 0x11, 0x19, 0x15, 0x13, 0x11, 0x11};
     static constexpr Glyph o{0x0e, 0x11, 0x11, 0x11, 0x11, 0x11, 0x0e};
     static constexpr Glyph p{0x1e, 0x11, 0x11, 0x1e, 0x10, 0x10, 0x10};
+    static constexpr Glyph q{0x0e, 0x11, 0x11, 0x11, 0x15, 0x12, 0x0d};
     static constexpr Glyph r{0x1e, 0x11, 0x11, 0x1e, 0x14, 0x12, 0x11};
     static constexpr Glyph s{0x0f, 0x10, 0x10, 0x0e, 0x01, 0x01, 0x1e};
     static constexpr Glyph t{0x1f, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04};
@@ -311,12 +357,18 @@ const std::array<std::uint8_t, OverlayMetrics::GlyphHeight>& OverlayGlyph(char c
     static constexpr Glyph v{0x11, 0x11, 0x11, 0x11, 0x11, 0x0a, 0x04};
     static constexpr Glyph w{0x11, 0x11, 0x11, 0x15, 0x15, 0x15, 0x0a};
     static constexpr Glyph x{0x11, 0x11, 0x0a, 0x04, 0x0a, 0x11, 0x11};
+    static constexpr Glyph y{0x11, 0x11, 0x11, 0x0a, 0x04, 0x04, 0x04};
+    static constexpr Glyph z{0x1f, 0x01, 0x02, 0x04, 0x08, 0x10, 0x1f};
     if (character >= '0' && character <= '9') return digits[static_cast<std::size_t>(character - '0')];
     switch (character) {
         case ' ': return space;
         case '.': return period;
         case ':': return colon;
         case '/': return slash;
+        case '%': return percent;
+        case '+': return plus;
+        case '-': return minus;
+        case '_': return underscore;
         case 'A': return a;
         case 'B': return b;
         case 'C': return c;
@@ -326,12 +378,14 @@ const std::array<std::uint8_t, OverlayMetrics::GlyphHeight>& OverlayGlyph(char c
         case 'G': return g;
         case 'H': return h;
         case 'I': return i;
+        case 'J': return j;
         case 'K': return k;
         case 'L': return l;
         case 'M': return m;
         case 'N': return n;
         case 'O': return o;
         case 'P': return p;
+        case 'Q': return q;
         case 'R': return r;
         case 'S': return s;
         case 'T': return t;
@@ -339,10 +393,16 @@ const std::array<std::uint8_t, OverlayMetrics::GlyphHeight>& OverlayGlyph(char c
         case 'V': return v;
         case 'W': return w;
         case 'X': return x;
+        case 'Y': return y;
+        case 'Z': return z;
         default: throw std::runtime_error(std::string("the performance overlay font has no glyph for '") + character + "'");
     }
 }
 
+std::string_view OverlayFontCharacters() {
+    return " .:/%+-_0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+}
+
 std::string FormatOverlayValue(double value) {
     const double clamped = std::isfinite(value) ? std::clamp(value, 0.0, 9999.0) : 0.0;
     char text[16];
@@ -393,7 +453,7 @@ OverlayContent ComposeOverlay(const OverlayStatistics& statistics, bool gpu, boo
         const auto size = [](VkExtent2D extent) { return std::to_string(extent.width) + "X" + std::to_string(extent.height); };
         for (const std::string_view separator : {" / ", "/"}) {
             auto line = "RES: " + size(details.display) + std::string(separator) + size(details.window);
-            if (line.size() > static_cast<std::size_t>(OverlayMetrics::Columns)) continue;
+            if (line.size() > static_cast<std::size_t>(content.columns)) continue;
             content.below.push_back(std::move(line));
             break;
         }
@@ -401,6 +461,140 @@ OverlayContent ComposeOverlay(const OverlayStatistics& statistics, bool gpu, boo
     return content;
 }
 
+std::string OverlayText(std::string_view text) {
+    const auto font = OverlayFontCharacters();
+    std::string shown;
+    shown.reserve(text.size());
+    for (const char character : text) {
+        const char upper = character >= 'a' && character <= 'z' ? static_cast<char>(character - 'a' + 'A') : character;
+        shown.push_back(font.find(upper) != std::string_view::npos ? upper : ' ');
+    }
+    return shown;
+}
+
+OverlayContent ComposeOverlayDetails(const OverlayDetailSample& sample) {
+    OverlayContent content;
+    content.columns = OverlayMetrics::DetailColumns;
+    content.graph = false;
+    const auto width = static_cast<std::size_t>(content.columns);
+    const auto add = [&](std::string line) {
+        if (line.size() > width) line.resize(width);
+        content.below.push_back(std::move(line));
+    };
+    const auto joined = [&](const std::string& label, std::initializer_list<std::optional<std::string>> parts) {
+        std::string line = label;
+        bool any = false;
+        for (const auto& part : parts) {
+            if (!part) continue;
+            line += (line.empty() ? "" : any ? "  " : " ") + *part;
+            any = true;
+        }
+        if (any) add(std::move(line));
+    };
+    const auto shown = [](const auto& value, auto format) -> std::optional<std::string> {
+        if (!value) return std::nullopt;
+        return format(*value);
+    };
+    const auto value = [](double number) { return FormatOverlayValue(number); };
+    const auto count = [](double number) { return FormatOverlayCount(number); };
+    const auto rate = [](double number) { return FormatOverlayRate(number); };
+    add("FPS: " + rate(sample.fps) + "  FRAME: " + value(sample.frameMs) + " MS");
+    if (!sample.busy.empty()) {
+        const std::string label = "BUSY %:";
+        std::string line = label;
+        for (const auto& thread : sample.busy) {
+            const auto item = " " + OverlayText(thread.name) + " " + count(thread.percent);
+            if (line.size() > label.size() && line.size() + item.size() > width) {
+                add(std::move(line));
+                line = std::string(label.size(), ' ');
+            }
+            line += item;
+        }
+        add(std::move(line));
+    }
+    for (const auto& phase : sample.phases) {
+        const auto amount = " " + value(phase.msPerFrame) + " MS/FRAME";
+        auto name = OverlayText(phase.name);
+        name.resize(std::min(name.size(), width - amount.size()));
+        add(name + std::string(width - name.size() - amount.size(), ' ') + amount);
+    }
+    if (sample.queueWaits) add("Q0 WAIT MS/FRAME: MEM " + value(sample.queueWaits->memoryMs) + " LOCK " + value(sample.queueWaits->lockMs) + " GPU " + value(sample.queueWaits->gpuMs));
+    joined("RENDER PASSES/FRAME:", {shown(sample.renderPassesPerFrame, value)});
+    joined("PIPELINES:", {shown(sample.pipelinesCompiling, [&](std::uint64_t pending) { return "COMPILING " + count(static_cast<double>(pending)); }), shown(sample.pipelinesPerSecond, [&](double made) { return "NEW/S " + count(made); })});
+    joined("HIT %:", {shown(sample.drawCacheHitPercent, [&](double percent) { return "DRAW CACHE " + value(percent); }), shown(sample.pipelineCacheHitPercent, [&](double percent) { return "PIPELINES " + value(percent); })});
+    joined("", {shown(sample.flipsPerSecond, [&](double flips) { return "FLIPS/S: " + rate(flips); }), shown(sample.presentsPerSecond, [&](double presents) { return "PRESENTS/S: " + rate(presents); })});
+    joined("", {shown(sample.droppedFrames, [&](std::uint64_t dropped) { return "DROPPED/S: " + count(static_cast<double>(dropped)); }), shown(sample.repeatedPresents, [&](std::uint64_t repeated) { return "REPEATED/S: " + count(static_cast<double>(repeated)); })});
+    joined("", {shown(sample.commit, [](const OverlayMemory& memory) { return "COMMIT: " + FormatOverlayGigabytes(memory.used) + "/" + FormatOverlayGigabytes(memory.budget) + " GB"; }), shown(sample.pageFaultsPerSecond, [&](double faults) { return "FAULTS/S: " + count(faults); })});
+    joined("HOST IMPORTS:", {shown(sample.hostImports, [&](std::uint64_t imports) { return count(static_cast<double>(imports)); }), shown(sample.hostImportBytes, [](const OverlayMemory& memory) { return FormatOverlayGigabytes(memory.used) + "/" + FormatOverlayGigabytes(memory.budget) + " GB"; })});
+    joined("CPU COPIES:", {shown(sample.cpuCopyMegabytesPerSecond, [&](double megabytes) { return value(megabytes) + " MB/S"; }), shown(sample.addressCopyCapPercent, [&](double percent) { return "CAP: " + value(percent) + "%"; })});
+    joined("UNIMPLEMENTED:", {shown(sample.unimplementedPerSecond, [&](double calls) { return count(calls) + "/S"; }), shown(sample.unimplementedTotal, [&](std::uint64_t calls) { return count(static_cast<double>(calls)) + " TOTAL"; })});
+    joined("REFUSED SHADERS:", {shown(sample.refusedShaders, [&](std::uint64_t refused) { return count(static_cast<double>(refused)); })});
+    joined("", {sample.title.empty() ? std::nullopt : std::optional<std::string>("TITLE: " + OverlayText(sample.title)), sample.presentMode.empty() ? std::nullopt : std::optional<std::string>("PRESENT: " + OverlayText(sample.presentMode))});
+    return content;
+}
+
+OverlayTraceFigures OverlayTraceDeltas(std::span<const TraceThreadTotals> before, std::span<const TraceThreadTotals> now, OverlayStatistics::Clock::duration elapsed, std::uint64_t presents, bool passes) {
+    using Duration = OverlayStatistics::Clock::duration;
+    const auto milliseconds = [](Duration length) { return std::chrono::duration<double, std::milli>(length).count(); };
+    std::map<std::string, Duration> busy;
+    std::map<std::string, Duration> phases;
+    std::optional<OverlayQueueWaits> waits;
+    std::uint64_t renderPasses = 0;
+    for (std::size_t index = 0; index < now.size(); ++index) {
+        const auto& thread = now[index];
+        const auto totals = TraceFormat::Since(thread.totals, index < before.size() ? std::span<const TraceThread::Total>(before[index].totals) : std::span<const TraceThread::Total>());
+        const bool worker = std::string_view(thread.name).starts_with(TraceFormat::WorkerPrefix);
+        const bool firstQueue = thread.name == std::string(TraceFormat::WorkerPrefix) + "0x0";
+        if (firstQueue && !waits) waits = OverlayQueueWaits{};
+        for (const auto& total : totals) {
+            const std::string_view scope = total.scope;
+            const std::string_view stage = total.stage != nullptr ? total.stage : "";
+            if (scope == PerformanceTrace::RenderPassScope) {
+                renderPasses += total.count;
+                continue;
+            }
+            if (worker && total.stage == nullptr && std::find(TraceFormat::WorkerBusyScopes.begin(), TraceFormat::WorkerBusyScopes.end(), scope) != TraceFormat::WorkerBusyScopes.end()) busy[thread.name] += total.length;
+            if (scope != TraceFormat::WorkerWaitScope) phases[TraceFormat::Name(total.scope, total.stage)] += total.length;
+            if (!firstQueue) continue;
+            if (scope == "Driver.Packet" && (stage == "memory_wait" || stage == "rendering_wait")) waits->memoryMs += milliseconds(total.length);
+            else if (scope == "GpuMutex.wait") waits->lockMs += milliseconds(total.length);
+            else if ((total.stage == nullptr && (scope == "Recorder.Sync" || scope == "Recorder.WaitSerial")) || (scope == "Graphics.Wait" && stage == "fence_wait")) waits->gpuMs += milliseconds(total.length);
+        }
+        if (worker && !busy.contains(thread.name)) busy[thread.name] = Duration{};
+    }
+    OverlayTraceFigures figures;
+    if (elapsed > Duration{}) {
+        for (const auto& [name, length] : busy) figures.busy.push_back({busyName(name), 100.0 * static_cast<double>(length.count()) / static_cast<double>(elapsed.count())});
+        std::stable_sort(figures.busy.begin(), figures.busy.end(), [](const OverlayThreadBusy& left, const OverlayThreadBusy& right) { return left.name.size() < right.name.size(); });
+    }
+    if (presents == 0) return figures;
+    const double perFrame = 1.0 / static_cast<double>(presents);
+    std::vector<std::pair<std::string, Duration>> top(phases.begin(), phases.end());
+    std::stable_sort(top.begin(), top.end(), [](const auto& left, const auto& right) { return left.second > right.second; });
+    for (std::size_t index = 0; index < top.size() && index < OverlayTopPhases; ++index) figures.phases.push_back({top[index].first, milliseconds(top[index].second) * perFrame});
+    if (waits) figures.queueWaits = OverlayQueueWaits{waits->memoryMs * perFrame, waits->lockMs * perFrame, waits->gpuMs * perFrame};
+    if (passes) figures.renderPassesPerFrame = static_cast<double>(renderPasses) * perFrame;
+    return figures;
+}
+
+std::optional<double> OverlayHitPercent(const OverlayHits& before, const OverlayHits& now) {
+    const bool reset = now.lookups < before.lookups || now.hits < before.hits;
+    const auto lookups = reset ? now.lookups : now.lookups - before.lookups;
+    const auto hits = reset ? now.hits : now.hits - before.hits;
+    if (lookups == 0 || hits > lookups) return std::nullopt;
+    return 100.0 * static_cast<double>(hits) / static_cast<double>(lookups);
+}
+
+std::string_view OverlayPresentModeName(VkPresentModeKHR mode) {
+    switch (mode) {
+        case VK_PRESENT_MODE_FIFO_KHR: return "fifo";
+        case VK_PRESENT_MODE_FIFO_LATEST_READY_KHR: return "fifo_latest_ready";
+        case VK_PRESENT_MODE_MAILBOX_KHR: return "mailbox";
+        case VK_PRESENT_MODE_IMMEDIATE_KHR: return "immediate";
+        default: return {};
+    }
+}
+
 bool OverlaySampleDue(std::optional<OverlayStatistics::Clock::time_point> sampledAt, OverlayStatistics::Clock::time_point now) {
     return !sampledAt || now - *sampledAt >= std::chrono::seconds(1);
 }
@@ -439,6 +633,51 @@ std::optional<std::uint64_t> ReadProcessPrivateBytes() {
 #endif
 }
 
+std::optional<OverlayMemory> ParseCommitCharge(std::string_view meminfo) {
+    std::optional<std::uint64_t> committed;
+    std::optional<std::uint64_t> limit;
+    while (!meminfo.empty()) {
+        const auto end = meminfo.find('\n');
+        const auto line = meminfo.substr(0, end);
+        meminfo = end == std::string_view::npos ? std::string_view{} : meminfo.substr(end + 1);
+        if (const auto value = meminfoKilobytes(line, "Committed_AS:")) committed = value;
+        else if (const auto value = meminfoKilobytes(line, "CommitLimit:")) limit = value;
+    }
+    if (!committed || !limit) return std::nullopt;
+    return OverlayMemory{*committed, *limit};
+}
+
+std::optional<OverlayMemory> ReadCommitCharge() {
+#ifdef _WIN32
+    PERFORMANCE_INFORMATION information{};
+    information.cb = sizeof(information);
+    if (K32GetPerformanceInfo(&information, sizeof(information)) == 0) return std::nullopt;
+    const auto page = static_cast<std::uint64_t>(information.PageSize);
+    return OverlayMemory{static_cast<std::uint64_t>(information.CommitTotal) * page, static_cast<std::uint64_t>(information.CommitLimit) * page};
+#else
+    std::FILE* meminfo = std::fopen("/proc/meminfo", "r");
+    if (meminfo == nullptr) return std::nullopt;
+    std::string text;
+    char block[1024];
+    for (std::size_t read = 0; (read = std::fread(block, 1, sizeof(block), meminfo)) != 0;) text.append(block, read);
+    std::fclose(meminfo);
+    return ParseCommitCharge(text);
+#endif
+}
+
+std::optional<std::uint64_t> ReadPageFaults() {
+#ifdef _WIN32
+    PROCESS_MEMORY_COUNTERS counters{};
+    counters.cb = sizeof(counters);
+    if (K32GetProcessMemoryInfo(GetCurrentProcess(), &counters, sizeof(counters)) == 0) return std::nullopt;
+    return static_cast<std::uint64_t>(counters.PageFaultCount);
+#else
+    rusage usage{};
+    if (getrusage(RUSAGE_SELF, &usage) != 0) return std::nullopt;
+    return static_cast<std::uint64_t>(usage.ru_minflt) + static_cast<std::uint64_t>(usage.ru_majflt);
+#endif
+}
+
 std::int32_t OverlayScale(VkExtent2D extent, const OverlayContent& content) {
     auto scale = std::max<std::int32_t>(1, static_cast<std::int32_t>((extent.height + 240) / 480));
     while (scale > 1 && !fits(extent, content, scale)) --scale;
@@ -448,10 +687,12 @@ std::int32_t OverlayScale(VkExtent2D extent, const OverlayContent& content) {
 OverlayLayout LayOutOverlay(const OverlayContent& content, VkExtent2D extent, OverlayCorner corner) {
     Graphics::Require(extent.width != 0 && extent.height != 0, "the performance overlay needs a destination");
     Graphics::Require(content.frameTimes.size() <= OverlayStatistics::FrameHistory, "the performance overlay graph holds at most 120 frames");
+    Graphics::Require(content.columns > 0, "the performance overlay panel needs at least one column");
+    Graphics::Require(content.graph ? content.columns * OverlayMetrics::Advance - 1 >= static_cast<std::int32_t>(OverlayStatistics::FrameHistory) : content.frameTimes.empty(), "the performance overlay graph needs a graph row as wide as its 120 frames");
     OverlayLayout layout;
     const auto scale = OverlayScale(extent, content);
     layout.scale = scale;
-    const auto width = panelWidth(scale);
+    const auto width = panelWidth(content.columns, scale);
     const auto height = panelHeight(content, scale);
     const auto margin = OverlayMetrics::Margin * scale;
     const bool right = corner == OverlayCorner::TopRight || corner == OverlayCorner::BottomRight;
@@ -461,7 +702,7 @@ OverlayLayout LayOutOverlay(const OverlayContent& content, VkExtent2D extent, Ov
     layout.bounds = {left, top, width, height, static_cast<std::uint32_t>(OverlayColor::Background)};
     layout.rects.push_back(layout.bounds);
     const auto text = [&](const std::string& line, std::int32_t row) {
-        Graphics::Require(line.size() <= static_cast<std::size_t>(OverlayMetrics::Columns), "a performance overlay line is wider than the panel");
+        Graphics::Require(line.size() <= static_cast<std::size_t>(content.columns), "a performance overlay line is wider than the panel");
         for (std::size_t column = 0; column < line.size(); ++column) {
             const auto& glyph = OverlayGlyph(line[column]);
             const auto glyphLeft = left + (OverlayMetrics::Padding + static_cast<std::int32_t>(column) * OverlayMetrics::Advance) * scale;
@@ -487,7 +728,7 @@ OverlayLayout LayOutOverlay(const OverlayContent& content, VkExtent2D extent, Ov
         row += OverlayMetrics::LinePitch;
     }
     const auto graphBottom = top + (row + OverlayMetrics::GraphHeight) * scale;
-    if (!content.frameTimes.empty()) {
+    if (content.graph && !content.frameTimes.empty()) {
         double maximum = 0;
         double sum = 0;
         for (const auto time : content.frameTimes) {
@@ -501,7 +742,7 @@ OverlayLayout LayOutOverlay(const OverlayContent& content, VkExtent2D extent, Ov
             layout.rects.push_back({left + (OverlayMetrics::Padding + first + static_cast<std::int32_t>(i)) * scale, graphBottom - pixels, scale, pixels, static_cast<std::uint32_t>(OverlayColor::Graph)});
         }
     }
-    row += OverlayMetrics::GraphHeight + OverlayMetrics::GraphGap;
+    if (content.graph) row += OverlayMetrics::GraphHeight + OverlayMetrics::GraphGap;
     for (const auto& line : content.below) {
         text(line, row);
         row += OverlayMetrics::LinePitch;
@@ -580,19 +821,39 @@ void PerformanceOverlay::release() noexcept {
 }
 
 bool PerformanceOverlay::Visible() {
-    return settings().visible != ((toggleCount.load(std::memory_order_relaxed) & 1u) != 0);
+    static_cast<void>(settings());
+    return currentPage.load(std::memory_order_relaxed) != 0;
+}
+
+std::uint32_t PerformanceOverlay::Page() {
+    static_cast<void>(settings());
+    return currentPage.load(std::memory_order_relaxed);
 }
 
 void PerformanceOverlay::Toggle() {
-    toggleCount.fetch_add(1, std::memory_order_relaxed);
+    static std::mutex toggleMutex;
+    std::lock_guard lock(toggleMutex);
+    const auto presses = toggleCount.load(std::memory_order_relaxed) + 1;
+    const auto shown = OverlayPageAfter(settings().page, presses);
+    currentPage.store(shown, std::memory_order_relaxed);
+    toggleCount.store(presses, std::memory_order_relaxed);
+    PerformanceTrace::SetOverlayTotals(shown == 2);
+}
+
+void PerformanceOverlay::SetDriverCounters(OverlayDriverCounters (*source)()) {
+    driverCounters.store(source, std::memory_order_release);
 }
 
 OverlayCorner PerformanceOverlay::Corner() {
     return settings().corner;
 }
 
-void PerformanceOverlay::NotePresent(OverlayStatistics::Clock::time_point at, VkExtent2D display) {
+void PerformanceOverlay::NotePresent(OverlayStatistics::Clock::time_point at, VkExtent2D display, const OverlayPresentInfo& info) {
     restartIfToggled();
+    if (page == 2) {
+        noteDetailPresent(at, info);
+        return;
+    }
     if (OverlaySampleDue(sampledAt, at)) {
         details.vram = ReadVramBudget(context);
         details.ram = ReadProcessPrivateBytes();
@@ -605,13 +866,100 @@ void PerformanceOverlay::NotePresent(OverlayStatistics::Clock::time_point at, Vk
 
 void PerformanceOverlay::restartIfToggled() {
     const auto current = toggleCount.load(std::memory_order_relaxed);
+    page = Page();
     if (current == toggles) return;
     statistics.Reset();
     details = {};
+    detailSample = {};
+    detail = {};
     sampledAt.reset();
     toggles = current;
 }
 
+void PerformanceOverlay::noteDetailPresent(OverlayStatistics::Clock::time_point at, const OverlayPresentInfo& info) {
+    PerformanceTrace::BindThread("presenter");
+    statistics.AddPresent(at, OverlayCounters{0, 0, 0, 0, PerformanceCounters::Pipelines()});
+    if (info.displayAddress != 0 && info.displayAddress == detail.lastDisplayAddress) ++detail.repeats;
+    detail.lastDisplayAddress = info.displayAddress;
+    ++detail.tracePresents;
+    detailSample.fps = statistics.Fps();
+    detailSample.frameMs = statistics.LastFrameMs();
+    detailSample.presentsPerSecond = statistics.Fps();
+    detailSample.pipelinesPerSecond = static_cast<double>(statistics.PipelinesInWindow());
+    if (OverlaySampleDue(detail.sampledAt, at)) sampleDetails(at, info);
+}
+
+void PerformanceOverlay::sampleDetails(OverlayStatistics::Clock::time_point at, const OverlayPresentInfo& info) {
+    DetailCounters now;
+    if (const auto source = driverCounters.load(std::memory_order_acquire)) now.driver = source();
+    const auto pipelines = Graphics::PipelineStoreTotals();
+    now.pipelines = OverlayHits{pipelines.hits + pipelines.misses, pipelines.hits};
+    const auto copies = Graphics::AddressCopyCounters::Fold();
+    now.copiedBytes = copies.bytes;
+    now.pageFaults = ReadPageFaults();
+    now.unimplemented = NotImplementedCalls_nid_no_patch();
+    now.acquireDrops = info.acquireDrops;
+    const auto& before = detail.base;
+    if (detail.sampledAt) {
+        const double seconds = std::chrono::duration<double>(at - *detail.sampledAt).count();
+        const auto perSecond = [&](std::uint64_t count) { return static_cast<double>(count) / seconds; };
+        if (now.driver && before.driver) {
+            detailSample.flipsPerSecond = perSecond(now.driver->flips - before.driver->flips);
+            if (now.driver->drawCache && before.driver->drawCache) detailSample.drawCacheHitPercent = OverlayHitPercent(*before.driver->drawCache, *now.driver->drawCache);
+        }
+        detailSample.pipelineCacheHitPercent = OverlayHitPercent(before.pipelines, now.pipelines);
+        detailSample.cpuCopyMegabytesPerSecond = perSecond(now.copiedBytes - before.copiedBytes) / (1024.0 * 1024.0);
+        if (const auto limit = Graphics::AddressCopyCounters::Limit(); limit != 0) detailSample.addressCopyCapPercent = 100.0 * static_cast<double>(copies.largest) / static_cast<double>(limit);
+        if (now.pageFaults && before.pageFaults) detailSample.pageFaultsPerSecond = perSecond(*now.pageFaults - *before.pageFaults);
+        detailSample.unimplementedPerSecond = perSecond(now.unimplemented - before.unimplemented);
+        detailSample.droppedFrames = static_cast<std::uint64_t>(std::llround(perSecond(now.acquireDrops - before.acquireDrops)));
+        detailSample.repeatedPresents = static_cast<std::uint64_t>(std::llround(perSecond(detail.repeats)));
+    } else if (HasAppParamJson_nid_no_patch()) {
+        const auto id = GetAppTitleId_nid_postfix();
+        detailSample.title.assign(id.value, strnlen(id.value, sizeof(id.value)));
+    }
+    if (now.driver && before.driver && !now.driver->drawCache) now.driver->drawCache = before.driver->drawCache;
+    detailSample.unimplementedTotal = now.unimplemented;
+    if (const auto pending = DriverDetail::RegistrationPreparationsPending()) detailSample.pipelinesCompiling = *pending;
+    if (const auto refused = DriverDetail::RefusedShaderCount()) detailSample.refusedShaders = *refused;
+    else detailSample.refusedShaders.reset();
+    detailSample.commit = ReadCommitCharge();
+    if (context.hostImportAlignment != 0) {
+        const auto imports = Graphics::HostImportCounters();
+        detailSample.hostImports = imports.live;
+        detailSample.hostImportBytes = OverlayMemory{imports.liveBytes, imports.liveBytes + Graphics::HostImportBudgetRemaining(context)};
+    }
+    detailSample.presentMode = std::string(OverlayPresentModeName(info.presentMode));
+    sampleTrace(at);
+    detail.base = std::move(now);
+    detail.repeats = 0;
+    detail.sampledAt = at;
+}
+
+void PerformanceOverlay::sampleTrace(OverlayStatistics::Clock::time_point at) {
+    if (!PerformanceTrace::Recording()) {
+        detail.traceSampledAt.reset();
+        detail.traceBase.clear();
+        detailSample.busy.clear();
+        detailSample.phases.clear();
+        detailSample.queueWaits.reset();
+        detailSample.renderPassesPerFrame.reset();
+        return;
+    }
+    auto threads = PerformanceTrace::ThreadTotals();
+    if (!threads) return;
+    if (detail.traceSampledAt) {
+        auto figures = OverlayTraceDeltas(detail.traceBase, *threads, at - *detail.traceSampledAt, detail.tracePresents, PerformanceTrace::OverlayTotals());
+        detailSample.busy = std::move(figures.busy);
+        detailSample.phases = std::move(figures.phases);
+        detailSample.queueWaits = figures.queueWaits;
+        detailSample.renderPassesPerFrame = figures.renderPassesPerFrame;
+    }
+    detail.traceBase = std::move(*threads);
+    detail.traceSampledAt = at;
+    detail.tracePresents = 0;
+}
+
 VkRenderPass PerformanceOverlay::renderPass(VkFormat format, VkImageLayout initialLayout, VkImageLayout finalLayout) {
     const auto key = std::tuple{format, initialLayout, finalLayout};
     if (const auto found = renderPasses.find(key); found != renderPasses.end()) return found->second;
@@ -711,6 +1059,7 @@ VkPipeline PerformanceOverlay::pipeline(VkFormat format, VkRenderPass compatible
 }
 
 OverlayRect PerformanceOverlay::Record(VkCommandBuffer commands, std::size_t slot, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout) {
+    if (page == 2) return Record(commands, slot, destination, format, colorSpace, extent, initialLayout, finalLayout, ComposeOverlayDetails(detailSample), Corner());
     auto shown = details;
     shown.window = extent;
     return Record(commands, slot, destination, format, colorSpace, extent, initialLayout, finalLayout, ComposeOverlay(statistics, PerformanceCounters::GpuEnabled(), PerformanceCounters::DriverEnabled(), shown), Corner());

core/libs/prx/libSceAgcDriver/Execution/src/PerformanceTrace.cpp

@@ -214,7 +214,11 @@ double seconds(Clock::duration elapsed) {
 }
 
 void PerformanceTrace::BindThread(const char* name, std::size_t capacity) {
-    if (!Configured() || local != nullptr) return;
+    if (local != nullptr) return;
+    if (!Configured()) {
+        localName = name;
+        return;
+    }
     const bool windowed = std::getenv("APS5_TRACE_FRAMES") != nullptr || std::getenv("APS5_TRACE_SECONDS") != nullptr;
     if (windowed && capacity == TraceThread::DefaultCapacity) capacity = TraceThread::ParseCapacity(std::getenv("APS5_TRACE_EVENTS"));
     auto bound = std::make_unique<TraceThread>(name, presentCount, windowed ? capacity : 0, buffering);
@@ -224,6 +228,39 @@ void PerformanceTrace::BindThread(const char* name, std::size_t capacity) {
     state.threads.push_back(std::move(bound));
 }
 
+TraceThread* PerformanceTrace::bindNamed() {
+    if (localName == nullptr) return nullptr;
+    auto bound = std::make_unique<TraceThread>(localName, presentCount, 0, buffering);
+    auto& state = session();
+    std::lock_guard lock(state.mutex);
+    local = bound.get();
+    state.threads.push_back(std::move(bound));
+    return local;
+}
+
+void PerformanceTrace::SetOverlayTotals(bool on) {
+    if (Configured()) return;
+    binder.store(on ? &bindNamed : nullptr, std::memory_order_release);
+    recording.store(on, std::memory_order_relaxed);
+}
+
+bool PerformanceTrace::OverlayTotals() {
+    return binder.load(std::memory_order_relaxed) != nullptr;
+}
+
+std::optional<std::vector<TraceThreadTotals>> PerformanceTrace::ThreadTotals() {
+    auto& state = session();
+    std::unique_lock lock(state.mutex, std::try_to_lock);
+    if (!lock.owns_lock()) return std::nullopt;
+    std::vector<TraceThreadTotals> threads;
+    threads.reserve(state.threads.size());
+    for (const auto& thread : state.threads) {
+        auto snapshot = thread->Collect();
+        threads.push_back({std::move(snapshot.name), std::move(snapshot.totals)});
+    }
+    return threads;
+}
+
 void PerformanceTrace::OpenInterval(Clock::time_point now, std::uint64_t present) {
     auto& state = session();
     state.interval->Restart(now - state.first);

core/libs/prx/libSceAgcDriver/Execution/src/PerformanceTraceThread.cpp

@@ -4,6 +4,8 @@ namespace AgcDriver {
 
 std::atomic<bool> PerformanceTrace::recording{false};
 thread_local TraceThread* PerformanceTrace::local = nullptr;
+thread_local const char* PerformanceTrace::localName = nullptr;
+std::atomic<TraceThread* (*)()> PerformanceTrace::binder{nullptr};
 
 }
 
@@ -12,5 +14,12 @@ extern "C" bool AgcDriverTraceRecording_nid_postfix() {
 }
 
 extern "C" AgcDriver::TraceThread* AgcDriverTraceThread_nid_postfix() {
-    return AgcDriver::PerformanceTrace::local;
+    if (auto* thread = AgcDriver::PerformanceTrace::local) return thread;
+    const auto bind = AgcDriver::PerformanceTrace::binder.load(std::memory_order_acquire);
+    return bind != nullptr ? bind() : nullptr;
+}
+
+extern "C" void AgcDriverTraceCount_nid_postfix(const char* scope) {
+    if (AgcDriver::PerformanceTrace::binder.load(std::memory_order_relaxed) == nullptr) return;
+    if (auto* thread = AgcDriverTraceThread_nid_postfix()) thread->Add(scope, nullptr, {}, {});
 }

core/libs/prx/libSceAgcDriver/Execution/src/PresentationPass.cpp

@@ -86,6 +86,7 @@ void PresentationPass::release() noexcept {
     for (std::size_t i = 0; i < slots.size(); ++i) {
         Release(i);
         releaseDumpTarget(slots[i]);
+        releaseUploadImage(slots[i]);
     }
     for (const auto& [key, entry] : targets) {
         if (entry.pipeline) context.Function<PFN_vkDestroyPipeline>("vkDestroyPipeline")(context.device, entry.pipeline, nullptr);
@@ -155,22 +156,24 @@ VkFramebuffer PresentationPass::createFramebuffer(VkRenderPass renderPass, VkIma
     return framebuffer;
 }
 
-const PresentationPass::Target& PresentationPass::target(VkFormat format, VkImageLayout finalLayout) {
-    const auto key = std::pair{format, finalLayout};
+const PresentationPass::Target& PresentationPass::target(VkFormat format, VkImageLayout finalLayout, VkImageLayout loadFrom) {
+    const auto key = std::tuple{format, finalLayout, loadFrom};
     if (const auto found = targets.find(key); found != targets.end()) return found->second;
+    const bool blended = loadFrom != VK_IMAGE_LAYOUT_UNDEFINED;
     VkFormatProperties properties{};
     context.formatProperties(context.physical, format, &properties);
     Graphics::Require((properties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT) != 0, "presentation pass format is not renderable");
+    Graphics::Require(!blended || (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT) != 0, "presentation pass cannot blend an overlay into this format");
     Target created;
     try {
         VkAttachmentDescription attachment{};
         attachment.format = format;
         attachment.samples = VK_SAMPLE_COUNT_1_BIT;
-        attachment.loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
+        attachment.loadOp = blended ? VK_ATTACHMENT_LOAD_OP_LOAD : VK_ATTACHMENT_LOAD_OP_DONT_CARE;
         attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
         attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
         attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
-        attachment.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+        attachment.initialLayout = loadFrom;
         attachment.finalLayout = finalLayout;
         const VkAttachmentReference reference{0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL};
         VkSubpassDescription subpass{};
@@ -179,7 +182,7 @@ const PresentationPass::Target& PresentationPass::target(VkFormat format, VkImag
         subpass.pColorAttachments = &reference;
         const bool transfer = finalLayout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
         const VkSubpassDependency dependencies[2] = {
-            {VK_SUBPASS_EXTERNAL, 0, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, 0},
+            {VK_SUBPASS_EXTERNAL, 0, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, blended ? VkAccessFlags{VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT} : VkAccessFlags{0}, blended ? VkAccessFlags{VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT} : VkAccessFlags{VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT}, 0},
             {0, VK_SUBPASS_EXTERNAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, transfer ? VK_PIPELINE_STAGE_TRANSFER_BIT : VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, transfer ? VK_ACCESS_TRANSFER_READ_BIT : VkAccessFlags{0}, 0},
         };
         VkRenderPassCreateInfo renderPassInfo{VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO};
@@ -212,7 +215,16 @@ const PresentationPass::Target& PresentationPass::target(VkFormat format, VkImag
         VkPipelineMultisampleStateCreateInfo multisample{VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO};
         multisample.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT;
         VkPipelineColorBlendAttachmentState blendAttachment{};
-        blendAttachment.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT;
+        blendAttachment.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | (blended ? 0u : VK_COLOR_COMPONENT_A_BIT);
+        if (blended) {
+            blendAttachment.blendEnable = VK_TRUE;
+            blendAttachment.srcColorBlendFactor = VK_BLEND_FACTOR_ONE;
+            blendAttachment.dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA;
+            blendAttachment.colorBlendOp = VK_BLEND_OP_ADD;
+            blendAttachment.srcAlphaBlendFactor = VK_BLEND_FACTOR_ZERO;
+            blendAttachment.dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE;
+            blendAttachment.alphaBlendOp = VK_BLEND_OP_ADD;
+        }
         VkPipelineColorBlendStateCreateInfo blend{VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO};
         blend.attachmentCount = 1;
         blend.pAttachments = &blendAttachment;
@@ -262,7 +274,75 @@ void PresentationPass::Bind(std::size_t index, VkImage image, std::uint32_t widt
     context.Function<PFN_vkUpdateDescriptorSets>("vkUpdateDescriptorSets")(context.device, 1, &write, 0, nullptr);
 }
 
-void PresentationPass::draw(VkCommandBuffer commands, const Slot& slot, const Target& entry, VkFramebuffer framebuffer, VkExtent2D extent, const std::int32_t (&parameters)[9]) const {
+void PresentationPass::releaseUploadImage(Slot& slot) noexcept {
+    if (slot.uploadImage) context.Function<PFN_vkDestroyImage>("vkDestroyImage")(context.device, slot.uploadImage, nullptr);
+    if (slot.uploadMemory) context.Function<PFN_vkFreeMemory>("vkFreeMemory")(context.device, slot.uploadMemory, nullptr);
+    slot.uploadImage = VK_NULL_HANDLE;
+    slot.uploadMemory = VK_NULL_HANDLE;
+    slot.uploadExtent = {};
+    slot.uploadTexels.reset();
+}
+
+void PresentationPass::ensureUploadImage(Slot& slot, VkExtent2D extent) {
+    if (slot.uploadImage != VK_NULL_HANDLE && slot.uploadExtent.width == extent.width && slot.uploadExtent.height == extent.height) return;
+    releaseUploadImage(slot);
+    try {
+        VkImageCreateInfo imageInfo{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
+        imageInfo.imageType = VK_IMAGE_TYPE_2D;
+        imageInfo.format = VK_FORMAT_R32_UINT;
+        imageInfo.extent = {extent.width, extent.height, 1u};
+        imageInfo.mipLevels = 1;
+        imageInfo.arrayLayers = 1;
+        imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
+        imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
+        imageInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
+        imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
+        imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+        Graphics::Check(context.Function<PFN_vkCreateImage>("vkCreateImage")(context.device, &imageInfo, nullptr, &slot.uploadImage), "vkCreateImage presentation upload");
+        VkMemoryRequirements requirements{};
+        context.Function<PFN_vkGetImageMemoryRequirements>("vkGetImageMemoryRequirements")(context.device, slot.uploadImage, &requirements);
+        VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
+        allocation.allocationSize = requirements.size;
+        allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
+        Graphics::Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &slot.uploadMemory), "vkAllocateMemory presentation upload");
+        Graphics::Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, slot.uploadImage, slot.uploadMemory, 0), "vkBindImageMemory presentation upload");
+        slot.uploadTexels = std::make_unique<Graphics::Buffer>(context, static_cast<std::size_t>(extent.width) * extent.height * 4, VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
+        slot.uploadExtent = extent;
+    } catch (...) {
+        releaseUploadImage(slot);
+        throw;
+    }
+}
+
+void PresentationPass::BindTexels(std::size_t index, VkCommandBuffer commands, std::span<const std::byte> texels, std::uint32_t width, std::uint32_t height, std::uint64_t pixelFormat, float whiteNits) {
+    Graphics::Require(index < slots.size(), "presentation pass slot is out of range");
+    Graphics::Require(width != 0 && height != 0 && texels.size() == static_cast<std::size_t>(width) * height * 4, "presentation pass texels do not cover their extent");
+    auto& slot = slots[index];
+    ensureUploadImage(slot, {width, height});
+    std::memcpy(slot.uploadTexels->Bytes().data(), texels.data(), texels.size());
+    VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
+    barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
+    barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+    barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+    barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+    barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+    barrier.image = slot.uploadImage;
+    barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
+    const auto pipelineBarrier = context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier");
+    pipelineBarrier(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
+    VkBufferImageCopy copy{};
+    copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
+    copy.imageExtent = {width, height, 1u};
+    context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, slot.uploadTexels->Handle(), slot.uploadImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &copy);
+    barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
+    barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
+    barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+    barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL;
+    pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
+    Bind(index, slot.uploadImage, width, height, pixelFormat, whiteNits);
+}
+
+void PresentationPass::draw(VkCommandBuffer commands, const Slot& slot, const Target& entry, VkFramebuffer framebuffer, VkExtent2D extent, VkRect2D scissor, const std::int32_t (&parameters)[9]) const {
     VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO};
     begin.renderPass = entry.renderPass;
     begin.framebuffer = framebuffer;
@@ -270,7 +350,6 @@ void PresentationPass::draw(VkCommandBuffer commands, const Slot& slot, const Ta
     context.Function<PFN_vkCmdBeginRenderPass>("vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE);
     context.Function<PFN_vkCmdBindPipeline>("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, entry.pipeline);
     const VkViewport viewport{0.0f, 0.0f, static_cast<float>(extent.width), static_cast<float>(extent.height), 0.0f, 1.0f};
-    const VkRect2D scissor{{0, 0}, extent};
     context.Function<PFN_vkCmdSetViewport>("vkCmdSetViewport")(commands, 0, 1, &viewport);
     context.Function<PFN_vkCmdSetScissor>("vkCmdSetScissor")(commands, 0, 1, &scissor);
     context.Function<PFN_vkCmdBindDescriptorSets>("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &slot.set, 0, nullptr);
@@ -279,12 +358,13 @@ void PresentationPass::draw(VkCommandBuffer commands, const Slot& slot, const Ta
     context.Function<PFN_vkCmdEndRenderPass>("vkCmdEndRenderPass")(commands);
 }
 
-void PresentationPass::RecordPresent(VkCommandBuffer commands, std::size_t index, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout finalLayout) {
-    Graphics::Require(index < slots.size() && slots[index].source != VK_NULL_HANDLE, "presentation pass slot has no source");
-    Graphics::Require(destination != VK_NULL_HANDLE && extent.width != 0 && extent.height != 0, "presentation pass destination is unavailable");
-    auto& slot = slots[index];
-    const auto& entry = target(format, finalLayout);
-    const auto view = createView(destination, format, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | (finalLayout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL ? VK_IMAGE_USAGE_TRANSFER_SRC_BIT : 0u));
+VkFramebuffer PresentationPass::destinationFramebuffer(Slot& slot, const Target& entry, VkImage destination, VkFormat format, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout) {
+    VkImageUsageFlags usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
+    for (const auto layout : {initialLayout, finalLayout}) {
+        if (layout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
+        if (layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) usage |= VK_IMAGE_USAGE_TRANSFER_DST_BIT;
+    }
+    const auto view = createView(destination, format, usage);
     VkFramebuffer framebuffer = VK_NULL_HANDLE;
     try {
         framebuffer = createFramebuffer(entry.renderPass, view, extent);
@@ -293,12 +373,35 @@ void PresentationPass::RecordPresent(VkCommandBuffer commands, std::size_t index
         throw;
     }
     slot.transient.emplace_back(view, framebuffer);
+    return framebuffer;
+}
+
+void PresentationPass::RecordPresent(VkCommandBuffer commands, std::size_t index, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout finalLayout) {
+    Graphics::Require(index < slots.size() && slots[index].source != VK_NULL_HANDLE, "presentation pass slot has no source");
+    Graphics::Require(destination != VK_NULL_HANDLE && extent.width != 0 && extent.height != 0, "presentation pass destination is unavailable");
+    auto& slot = slots[index];
+    const auto& entry = target(format, finalLayout);
+    const auto framebuffer = destinationFramebuffer(slot, entry, destination, format, extent, VK_IMAGE_LAYOUT_UNDEFINED, finalLayout);
     const auto rect = ComputeContainRect_nid_postfix(slot.width, slot.height, extent.width, extent.height);
     const bool hdr10 = colorSpace == VK_COLOR_SPACE_HDR10_ST2084_EXT;
     Graphics::Require(colorSpace == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR || (hdr10 && (slot.mode & ModePq) != 0 && TenBitFormat(format)), "presentation pass presents only PQ display buffers to a 10-bit HDR10 destination");
     const std::uint32_t mode = (hdr10 ? slot.mode & ~ModePq : slot.mode) | (TenBitFormat(format) ? ModeKeepTenBit : 0u);
     const std::int32_t parameters[9] = {rect.x, rect.y, static_cast<std::int32_t>(rect.width), static_cast<std::int32_t>(rect.height), static_cast<std::int32_t>(slot.width), static_cast<std::int32_t>(slot.height), 1, static_cast<std::int32_t>(mode), std::bit_cast<std::int32_t>(slot.whiteNits)};
-    draw(commands, slot, entry, framebuffer, extent, parameters);
+    draw(commands, slot, entry, framebuffer, extent, {{0, 0}, extent}, parameters);
+}
+
+void PresentationPass::RecordOverlay(VkCommandBuffer commands, std::size_t index, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout) {
+    Graphics::Require(index < slots.size() && slots[index].source != VK_NULL_HANDLE, "presentation pass slot has no source");
+    Graphics::Require(destination != VK_NULL_HANDLE && extent.width != 0 && extent.height != 0, "presentation pass destination is unavailable");
+    Graphics::Require(initialLayout != VK_IMAGE_LAYOUT_UNDEFINED, "an overlay is blended over a destination whose contents are kept");
+    auto& slot = slots[index];
+    Graphics::Require((slot.mode & (ModeTenBit | ModePq)) == 0, "an overlay plane is blended only from an 8-bit SDR display buffer");
+    Graphics::Require(colorSpace == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, "an overlay plane is blended only into an SDR destination");
+    const auto& entry = target(format, finalLayout, initialLayout);
+    const auto framebuffer = destinationFramebuffer(slot, entry, destination, format, extent, initialLayout, finalLayout);
+    const auto rect = ComputeContainRect_nid_postfix(slot.width, slot.height, extent.width, extent.height);
+    const std::int32_t parameters[9] = {rect.x, rect.y, static_cast<std::int32_t>(rect.width), static_cast<std::int32_t>(rect.height), static_cast<std::int32_t>(slot.width), static_cast<std::int32_t>(slot.height), 1, static_cast<std::int32_t>(slot.mode), std::bit_cast<std::int32_t>(slot.whiteNits)};
+    draw(commands, slot, entry, framebuffer, extent, {{rect.x, rect.y}, {rect.width, rect.height}}, parameters);
 }
 
 void PresentationPass::releaseDumpTarget(Slot& slot) noexcept {
@@ -346,18 +449,38 @@ void PresentationPass::ensureDumpTarget(Slot& slot, VkExtent2D extent) {
 }
 
 void PresentationPass::RecordDump(VkCommandBuffer commands, std::size_t index, std::uint32_t stride, VkBuffer destination) {
+    RecordDumpImage(commands, index, stride);
+    RecordDumpReadback(commands, index, destination);
+}
+
+void PresentationPass::RecordDumpImage(VkCommandBuffer commands, std::size_t index, std::uint32_t stride) {
     Graphics::Require(index < slots.size() && slots[index].source != VK_NULL_HANDLE, "presentation pass slot has no source");
-    Graphics::Require(destination != VK_NULL_HANDLE, "presentation dump buffer is unavailable");
     auto& slot = slots[index];
     const auto extent = DumpExtent(slot.width, slot.height, stride);
     ensureDumpTarget(slot, extent);
     const auto& entry = target(VK_FORMAT_B8G8R8A8_UNORM, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
     const std::int32_t parameters[9] = {0, 0, static_cast<std::int32_t>(extent.width), static_cast<std::int32_t>(extent.height), static_cast<std::int32_t>(slot.width), static_cast<std::int32_t>(slot.height), static_cast<std::int32_t>(stride), static_cast<std::int32_t>(slot.mode | ModeSubsample), std::bit_cast<std::int32_t>(slot.whiteNits)};
-    draw(commands, slot, entry, slot.dumpFramebuffer, extent, parameters);
+    draw(commands, slot, entry, slot.dumpFramebuffer, extent, {{0, 0}, extent}, parameters);
+}
+
+VkImage PresentationPass::DumpImage(std::size_t index) const {
+    Graphics::Require(index < slots.size() && slots[index].dumpImage != VK_NULL_HANDLE, "presentation pass slot has no dump image");
+    return slots[index].dumpImage;
+}
+
+VkExtent2D PresentationPass::DumpImageExtent(std::size_t index) const {
+    Graphics::Require(index < slots.size() && slots[index].dumpImage != VK_NULL_HANDLE, "presentation pass slot has no dump image");
+    return slots[index].dumpExtent;
+}
+
+void PresentationPass::RecordDumpReadback(VkCommandBuffer commands, std::size_t index, VkBuffer destination) {
+    Graphics::Require(destination != VK_NULL_HANDLE, "presentation dump buffer is unavailable");
+    const auto image = DumpImage(index);
+    const auto extent = DumpImageExtent(index);
     VkBufferImageCopy copy{};
     copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
     copy.imageExtent = {extent.width, extent.height, 1u};
-    context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, slot.dumpImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destination, 1, &copy);
+    context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destination, 1, &copy);
     Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT);
 }
 

core/libs/prx/libSceAgcDriver/Execution/src/PresentationScaler.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp"
 #include "prx/libSceAgcDriver/Execution/include/AspectFit.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
 
 namespace AgcDriver {
@@ -23,7 +24,7 @@ PresentationScaler::~PresentationScaler() {
 
 void PresentationScaler::release() noexcept {
     if (sourceImage != VK_NULL_HANDLE) context.Function<PFN_vkDestroyImage>("vkDestroyImage")(context.device, sourceImage, nullptr);
-    if (sourceMemory != VK_NULL_HANDLE) context.Function<PFN_vkFreeMemory>("vkFreeMemory")(context.device, sourceMemory, nullptr);
+    if (sourceMemory != VK_NULL_HANDLE) Graphics::ReleaseImageMemory(context, sourceMemory, sourceBytes, sourceMemoryType);
     sourceImage = VK_NULL_HANDLE;
     sourceMemory = VK_NULL_HANDLE;
     sourceWidth = 0;
@@ -42,7 +43,9 @@ void PresentationScaler::EnsureSourceImage(std::uint32_t width, std::uint32_t he
     imageInfo.arrayLayers = 1;
     imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
     imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
-    imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
+    VkFormatProperties properties{};
+    context.formatProperties(context.physical, sourceFormat, &properties);
+    imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | ((properties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT) != 0 ? VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT : 0u);
     imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
     imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
     Graphics::Check(context.Function<PFN_vkCreateImage>("vkCreateImage")(context.device, &imageInfo, nullptr, &sourceImage), "vkCreateImage presentation source");
@@ -50,8 +53,9 @@ void PresentationScaler::EnsureSourceImage(std::uint32_t width, std::uint32_t he
     context.Function<PFN_vkGetImageMemoryRequirements>("vkGetImageMemoryRequirements")(context.device, sourceImage, &requirements);
     VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
     allocation.allocationSize = requirements.size;
-    allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
-    Graphics::Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &sourceMemory), "vkAllocateMemory presentation source");
+    allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "presentation source");
+    sourceMemory = Graphics::AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory presentation source", &sourceMemoryType, true);
+    sourceBytes = requirements.size;
     Graphics::Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, sourceImage, sourceMemory, 0), "vkBindImageMemory presentation source");
     sourceWidth = width;
     sourceHeight = height;

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -8,6 +8,7 @@
 #include "SpirvBackend/SpirvOptimizer.hpp"
 #endif
 #include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
 #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
 #include "prx/libSceAgcDriver/Execution/include/FloatFlushProbe.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
@@ -344,6 +345,7 @@ struct VulkanDevice::State {
     std::vector<VkImage> images;
     VkFence acquireFence = VK_NULL_HANDLE;
     VkPresentModeKHR presentMode = VK_PRESENT_MODE_FIFO_KHR;
+    std::uint64_t presentedAddress = 0;
     struct RetiredSwapchain {
         VkSwapchainKHR swapchain = VK_NULL_HANDLE;
         std::vector<VkSemaphore> rendered;
@@ -369,6 +371,8 @@ struct VulkanDevice::State {
     void* markerMapping = nullptr;
     BufferMarkerLog markerLog;
     bool deviceLostHook = false;
+    bool deferDestruction = false;
+    PFN_vkGetDeviceProcAddr contextDeviceProc = nullptr;
     VkTimeDomainKHR hostTimeDomain = VK_TIME_DOMAIN_DEVICE_KHR;
     VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES};
     std::vector<std::uint32_t> capabilities{1};
@@ -518,6 +522,7 @@ struct VulkanDevice::State {
     VkPhysicalDeviceMeshShaderPropertiesEXT meshLimits{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MESH_SHADER_PROPERTIES_EXT};
     std::unique_ptr<PresentationScaler> scaler;
     std::unique_ptr<PresentationPass> presentation;
+    std::unique_ptr<PresentationPass> planePass;
     std::unique_ptr<PerformanceOverlay> overlay;
     VkFormat surfaceFormat = VK_FORMAT_B8G8R8A8_UNORM;
     VkColorSpaceKHR surfaceColorSpace = VK_COLOR_SPACE_SRGB_NONLINEAR_KHR;
@@ -540,6 +545,7 @@ struct VulkanDevice::State {
         VkFence fence = VK_NULL_HANDLE;
         VkCommandBuffer commands = VK_NULL_HANDLE;
         std::shared_ptr<void> kept;
+        std::shared_ptr<void> keptPlane;
         std::unique_ptr<Graphics::Buffer> dumpBuffer;
         std::unique_ptr<PresentationScaler> dumpScaler;
         std::uint32_t dumpWidth = 0;
@@ -741,7 +747,9 @@ struct VulkanDevice::State {
     void RetireSlot(PresentSlot& slot) {
         slot.inFlight = false;
         if (presentation) presentation->Release(static_cast<std::size_t>(&slot - presentSlots.data()));
+        if (planePass) planePass->Release(static_cast<std::size_t>(&slot - presentSlots.data()));
         slot.kept.reset();
+        slot.keptPlane.reset();
         if (slot.dumpRecorded) {
             slot.dumpRecorded = false;
             WriteFrameDump(slot);
@@ -914,6 +922,7 @@ struct VulkanDevice::State {
             detiler.reset();
             colorTransfer.reset();
             overlay.reset();
+            planePass.reset();
             presentation.reset();
             scaler.reset();
             hdrScaler.reset();
@@ -945,6 +954,7 @@ struct VulkanDevice::State {
                 destroyPool(device, pool, nullptr);
             }
             if (settlePool != VK_NULL_HANDLE) destroyPool(device, settlePool, nullptr);
+            Graphics::DestroyDeferredObjects(device);
             destroyDevice(device, nullptr);
         }
         if (instance != VK_NULL_HANDLE) {
@@ -1283,6 +1293,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         VkPhysicalDeviceProperties2 driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &driver};
         state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &driverProperties);
         driverId = driver.driverID;
+        state->deferDestruction = driver.driverID == VK_DRIVER_ID_MOLTENVK && std::getenv("APS5_NO_DEFERRED_DESTRUCTION") == nullptr;
         state->driverInfo = std::string(driver.driverName) + " " + driver.driverInfo;
         state->driverId = driver.driverID;
         state->individuallyFreeDescriptorSets = driver.driverID == VK_DRIVER_ID_MESA_LLVMPIPE;
@@ -2095,6 +2106,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     check(state->InstanceFunction<PFN_vkCreateDevice>("vkCreateDevice")(selected, &deviceInfo, nullptr, &state->device), "vkCreateDevice");
     state->shaderProfile = std::move(shaderProfile);
     if (compileOnly) return;
+    if (state->deferDestruction) state->contextDeviceProc = Graphics::DeferDestructionToIdle(state->device, state->deviceProc);
     if (calibratedTimestamps != nullptr) {
         const bool khr = std::strcmp(calibratedTimestamps, VK_KHR_CALIBRATED_TIMESTAMPS_EXTENSION_NAME) == 0;
         const auto domains = reinterpret_cast<PFN_vkGetPhysicalDeviceCalibrateableTimeDomainsKHR>(state->instanceProc(state->instance, khr ? "vkGetPhysicalDeviceCalibrateableTimeDomainsKHR" : "vkGetPhysicalDeviceCalibrateableTimeDomainsEXT"));
@@ -3199,9 +3211,9 @@ bool VulkanDevice::Presentable() const {
     return state->extent.width != 0 && state->extent.height != 0;
 }
 
-bool VulkanDevice::PresentClear(std::uint32_t width, std::uint32_t height, bool opaque) {
+bool VulkanDevice::PresentClear(std::uint32_t width, std::uint32_t height, bool opaque, const DisplayBuffer* plane) {
     APS5_LOG_OUT_DEBUG("PresentClear width=%u height=%u opaque=%u", width, height, static_cast<unsigned>(opaque));
-    return present(width, height, opaque, {});
+    return present(width, height, opaque, {}, nullptr, nullptr, VK_FILTER_LINEAR, false, false, nullptr, 0, false, plane);
 }
 
 void VulkanDevice::PresentPixels(std::uint32_t width, std::uint32_t height, std::span<const std::byte> pixels) {
@@ -3332,6 +3344,56 @@ bool NoResidentPresent() {
     return no;
 }
 
+// 64 KiB blocks the CPU wrote since the image last matched guest memory would be shown stale (the
+// write-back keeps the CPU's bytes for them): Refresh merges them into the image first, as a sampled
+// texture of the image would (see cachedTexture). The uncached walk: the collect memo is per worker
+// packet, so on this thread a memoized answer could miss a CPU write that landed after a worker's
+// walk of the same range.
+bool RefreshPresented(Graphics::StorageTexture& resident, const DisplayBuffer& buffer, std::uint32_t layer) {
+    static const bool noAdopt = std::getenv("APS5_NO_PRESENT_EPOCH") != nullptr;
+    const auto bytes = DisplayBufferSize(buffer);
+    if (!noAdopt) GuestMemory::AdoptLatestCollectEpoch();
+    GuestMemory::CollectWritesUncached(buffer.address, bytes);
+    if (GuestMemory::UnchangedSince(buffer.address, bytes, resident.GenerationFor(buffer.address, bytes)) && (buffer.dccAddress == 0 || !ResidentKeysMoved(resident))) return false;
+    if (resident.ImageLayers() > 1) resident.RefreshLayers(layer, 1);
+    else resident.Refresh();
+    return true;
+}
+
+struct PlaneSource {
+    std::shared_ptr<Graphics::StorageTexture> resident;
+    std::uint32_t layer = 0;
+    std::vector<std::byte> texels;
+};
+
+PlaneSource OverlayPlaneSource(const Graphics::Context& context, const DisplayBuffer& buffer) {
+    require(DisplayTexelBytes(buffer.pixelFormat) == 4u && !DisplayTenBit(buffer.pixelFormat) && !DisplayPq(buffer.pixelFormat), "VideoOut: only an 8-bit SDR overlay plane is composited over the main output");
+    PlaneSource source;
+    if (!NoResidentPresent()) {
+        VkFilter filter = VK_FILTER_LINEAR;
+        bool pending = false;
+        bool convert = false;
+        source.resident = PresentableResident(context, buffer, filter, pending, convert, source.layer);
+        if (source.resident != nullptr && buffer.dccAddress != 0 && !ResidentServesDisplay(*source.resident, buffer)) source.resident.reset();
+        if (source.resident != nullptr) {
+            static_cast<void>(RefreshPresented(*source.resident, buffer, source.layer));
+            return source;
+        }
+    }
+    const auto bytes = DisplayBufferSize(buffer);
+    GuestMemory::FlushGpuWrites(buffer.address, bytes);
+    const auto keys = buffer.dccAddress != 0 ? DisplayBufferKeys(buffer) : Graphics::DccKeys::Uncompressed;
+    if (keys == Graphics::DccKeys::Uncompressed) {
+        GuestMemory::CheckRange(reinterpret_cast<const void*>(buffer.address), bytes, buffer.alignment);
+        source.texels = DisplayBufferTexels(buffer, {reinterpret_cast<const std::byte*>(buffer.address), bytes});
+        return source;
+    }
+    const auto texel = DisplayBufferClearTexel(buffer, keys);
+    source.texels.resize(static_cast<std::size_t>(buffer.width) * buffer.height * texel.size());
+    for (std::size_t offset = 0; offset < source.texels.size(); offset += texel.size()) std::memcpy(source.texels.data() + offset, texel.data(), texel.size());
+    return source;
+}
+
 // Debug aid: APS5_DUMP_FRAMES=<n> saves the first n presented display buffers as frame_<index>.bmp,
 // read back by the blit's own submission and written after its fence (RetireSlot), or with
 // APS5_NO_GPU_DUMP=1 decoded from the tiled guest buffer on the CPU, as before.
@@ -3356,7 +3418,17 @@ FrameDumps& Dumps() {
 
 }
 
-bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) {
+bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer, const DisplayBuffer* plane) {
+    state->presentedAddress = buffer.address;
+    if (buffer.dccAddress != 0 && buffer.tilingMode == 0) {
+        static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
+        const auto before = profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0;
+        state->recorder->SyncThrough(buffer.dccAddress, DisplayBufferKeyBytes(buffer), true);
+        if (const auto waited = profile ? Graphics::Recorder::ThreadWaitedMs() - before : 0.0; waited > 0.0) {
+            ++presentCounters.keyWriterWaits;
+            presentCounters.keyWriterWaitMs += waited;
+        }
+    }
     static const bool storeAtFlip = Graphics::StorageTexture::StoreAtFlipRequested(std::getenv("APS5_STORE_AT_FLIP"));
     if (storeAtFlip) Graphics::StorageTexture::FlushAllPending("flip");
     else Graphics::StorageTexture::FlushDumpSelected("flip");
@@ -3379,7 +3451,7 @@ bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) {
         }
         const bool screenshot = takeScreenshot(dumpFrame);
         auto shot = screenshot && passthrough ? ReadDisplayBuffer(buffer) : std::vector<std::byte>{};
-        if (!present(buffer.width, buffer.height, true, pixels, nullptr, nullptr, VK_FILTER_LINEAR, dumpFrame || (screenshot && !passthrough), false, nullptr, 0, passthrough)) {
+        if (!present(buffer.width, buffer.height, true, pixels, nullptr, nullptr, VK_FILTER_LINEAR, dumpFrame || (screenshot && !passthrough), false, nullptr, 0, passthrough, plane)) {
             if (dumpFrame) --dumps.dumped;
             if (screenshot) Screenshot::Request();
             return false;
@@ -3411,19 +3483,7 @@ bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) {
         } else if (resident == nullptr) {
             ++unsuitable;
         } else {
-            // 64 KiB blocks the CPU wrote since the image last matched guest memory would be shown
-            // stale (the write-back keeps the CPU's bytes for them): Refresh merges them into the
-            // image first, as a sampled texture of the image would (see cachedTexture). The uncached
-            // walk: the collect memo is per worker packet, so on this thread a memoized answer could
-            // miss a CPU write that landed after a worker's walk of the same range.
-            static const bool noAdopt = std::getenv("APS5_NO_PRESENT_EPOCH") != nullptr;
-            if (!noAdopt) GuestMemory::AdoptLatestCollectEpoch();
-            GuestMemory::CollectWritesUncached(buffer.address, bytes);
-            if (!GuestMemory::UnchangedSince(buffer.address, bytes, resident->GenerationFor(buffer.address, bytes)) || (buffer.dccAddress != 0 && ResidentKeysMoved(*resident))) {
-                if (resident->ImageLayers() > 1) resident->RefreshLayers(layer, 1);
-                else resident->Refresh();
-                ++refreshedPresents;
-            }
+            if (RefreshPresented(*resident, buffer, layer)) ++refreshedPresents;
             ++residentPresents;
         }
     }
@@ -3454,7 +3514,7 @@ bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) {
         const auto channel = [&](std::size_t index) { return static_cast<float>(std::to_integer<unsigned>((*cleared)[index])) / 255.0f; };
         uniform = {{channel(2), channel(1), channel(0), channel(3)}};
     }
-    if (!present(buffer.width, buffer.height, true, {}, cleared ? nullptr : &buffer, resident, filter, dumpFrame || (screenshot && !cpuScreenshot), convert, cleared ? &uniform : nullptr, layer, passthrough)) {
+    if (!present(buffer.width, buffer.height, true, {}, cleared ? nullptr : &buffer, resident, filter, dumpFrame || (screenshot && !cpuScreenshot), convert, cleared ? &uniform : nullptr, layer, passthrough, plane)) {
         // A dropped frame (swapchain out of date) keeps the dump numbering contiguous.
         if (dumpFrame) --dumps.dumped;
         if (screenshot) Screenshot::Request();
@@ -3510,7 +3570,7 @@ bool VulkanDevice::AcquireImage() {
     return true;
 }
 
-bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaque, std::span<const std::byte> pixels, const DisplayBuffer* display, const std::shared_ptr<Graphics::StorageTexture>& resident, VkFilter residentFilter, bool dumpFrame, bool residentConvert, const VkClearColorValue* uniform, std::uint32_t residentLayer, bool passthrough) {
+bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaque, std::span<const std::byte> pixels, const DisplayBuffer* display, const std::shared_ptr<Graphics::StorageTexture>& resident, VkFilter residentFilter, bool dumpFrame, bool residentConvert, const VkClearColorValue* uniform, std::uint32_t residentLayer, bool passthrough, const DisplayBuffer* plane) {
     PerformanceTimer timing("Vulkan.Present");
     APS5_LOG_OUT_DEBUG("present begin width=%u height=%u opaque=%u pixels=%zu", width, height, static_cast<unsigned>(opaque), pixels.size());
     require(state->swapchain != VK_NULL_HANDLE, "device has no swapchain");
@@ -3544,6 +3604,12 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
     auto& slot = state->presentSlots[state->presentCursor];
     require(!slot.inFlight, "presentation slot is still in flight");
     const bool overlay = PerformanceOverlay::Visible();
+    if (plane != nullptr && state->surfaceColorSpace == VK_COLOR_SPACE_HDR10_ST2084_EXT) {
+        static std::once_flag reported;
+        std::call_once(reported, [] { std::fprintf(stderr, "[present] the overlay plane is not composited over an HDR10 presentation\n"); });
+        plane = nullptr;
+    }
+    const auto planeSource = plane != nullptr ? OverlayPlaneSource(graphicsContext(), *plane) : PlaneSource{};
     if (display != nullptr && resident == nullptr) {
         static_cast<void>(DisplayBufferSize(*display));
         state->colorTransfer->Upload(display->address, width, height, Graphics::ColorTileMode::RenderTarget, DisplayTexelBytes(display->pixelFormat));
@@ -3588,6 +3654,34 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
     barrier.image = state->images[index];
     barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
     auto pipelineBarrier = state->DeviceFunction<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier");
+    const auto planeSlot = static_cast<std::size_t>(&slot - state->presentSlots.data());
+    VkImageMemoryBarrier planeBarrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
+    bool planeBound = false;
+    const auto bindPlane = [&] {
+        if (planeBound) return;
+        planeBound = true;
+        if (!state->planePass) state->planePass = std::make_unique<PresentationPass>(graphicsContext(), state->presentSlots.size());
+        if (planeSource.resident == nullptr) {
+            state->planePass->BindTexels(planeSlot, commands, planeSource.texels, plane->width, plane->height, plane->pixelFormat, whiteNits);
+            return;
+        }
+        planeBarrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
+        planeBarrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
+        planeBarrier.oldLayout = VK_IMAGE_LAYOUT_GENERAL;
+        planeBarrier.newLayout = VK_IMAGE_LAYOUT_GENERAL;
+        planeBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+        planeBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+        planeBarrier.image = planeSource.resident->Image();
+        planeBarrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS};
+        pipelineBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, nullptr, 0, nullptr, 1, &planeBarrier);
+        Graphics::Recorder::CountBarriers(CommandClass::PresentBlit);
+        state->planePass->Bind(planeSlot, planeSource.resident->Image(), plane->width, plane->height, plane->pixelFormat, whiteNits, planeSource.layer);
+    };
+    const auto blendPlaneInto = [&](VkImage image, VkFormat format, VkExtent2D extent) {
+        if (plane == nullptr) return;
+        bindPlane();
+        state->planePass->RecordOverlay(commands, planeSlot, image, format, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, extent, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
+    };
     if (!drawn) {
         pipelineBarrier(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
         Graphics::Recorder::CountBarriers(CommandClass::PresentBlit);
@@ -3635,7 +3729,7 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
         }
         if (drawn) {
             state->presentation->Bind(slotIndex, resident->Image(), width, height, display->pixelFormat, whiteNits, residentLayer);
-            state->presentation->RecordPresent(commands, slotIndex, barrier.image, state->surfaceFormat, state->surfaceColorSpace, state->extent, overlay ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
+            state->presentation->RecordPresent(commands, slotIndex, barrier.image, state->surfaceFormat, state->surfaceColorSpace, state->extent, overlay || plane != nullptr ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
         } else {
             scaler->RecordLetterbox(commands, width, height, barrier.image, state->extent.width, state->extent.height);
             if (direct) PresentationScaler::RecordBlitFrom(graphicsContext(), commands, resident->Image(), VK_IMAGE_LAYOUT_GENERAL, width, height, residentFilter, barrier.image, state->extent.width, state->extent.height, residentLayer);
@@ -3656,13 +3750,17 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
             const auto dumpBytes = static_cast<std::size_t>(slot.dumpWidth) * slot.dumpHeight * 4;
             if (!slot.dumpBuffer || slot.dumpBuffer->Bytes().size() != dumpBytes) slot.dumpBuffer = std::make_unique<Graphics::Buffer>(graphicsContext(), dumpBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT);
             if (drawn) {
-                state->presentation->RecordDump(commands, slotIndex, scale, slot.dumpBuffer->Handle());
+                state->presentation->RecordDumpImage(commands, slotIndex, scale);
+                blendPlaneInto(state->presentation->DumpImage(slotIndex), VK_FORMAT_B8G8R8A8_UNORM, state->presentation->DumpImageExtent(slotIndex));
+                state->presentation->RecordDumpReadback(commands, slotIndex, slot.dumpBuffer->Handle());
             } else if (direct) {
                 if (!slot.dumpScaler) slot.dumpScaler = std::make_unique<PresentationScaler>(graphicsContext(), VK_FORMAT_B8G8R8A8_UNORM, VK_FORMAT_B8G8R8A8_UNORM);
                 slot.dumpScaler->EnsureSourceImage(slot.dumpWidth, slot.dumpHeight);
                 slot.dumpScaler->RecordBlitInto(commands, resident->Image(), VK_IMAGE_LAYOUT_GENERAL, VK_FILTER_NEAREST, width, height, residentLayer);
+                blendPlaneInto(slot.dumpScaler->SourceImage(), VK_FORMAT_B8G8R8A8_UNORM, {slot.dumpWidth, slot.dumpHeight});
                 slot.dumpScaler->RecordReadback(commands, slot.dumpBuffer->Handle());
             } else {
+                blendPlaneInto(scaler->SourceImage(), scaler->SourceFormat(), {scaler->SourceWidth(), scaler->SourceHeight()});
                 scaler->RecordReadback(commands, slot.dumpBuffer->Handle());
             }
         }
@@ -3673,11 +3771,21 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
             Graphics::Recorder::CountBarriers(CommandClass::PresentBlit);
         }
     }
+    if (plane != nullptr) {
+        bindPlane();
+        state->planePass->RecordOverlay(commands, planeSlot, barrier.image, state->surfaceFormat, state->surfaceColorSpace, state->extent, drawn ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, overlay ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
+        if (planeSource.resident != nullptr) {
+            planeBarrier.srcAccessMask = VK_ACCESS_SHADER_READ_BIT;
+            planeBarrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
+            pipelineBarrier(commands, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &planeBarrier);
+            Graphics::Recorder::CountBarriers(CommandClass::PresentBlit);
+        }
+    }
     if (overlay) {
         if (!state->overlay) state->overlay = std::make_unique<PerformanceOverlay>(graphicsContext(), state->presentSlots.size());
-        state->overlay->NotePresent(std::chrono::steady_clock::now(), clearOnly ? VkExtent2D{} : VkExtent2D{width, height});
-        state->overlay->Record(commands, static_cast<std::size_t>(&slot - state->presentSlots.data()), barrier.image, state->surfaceFormat, state->surfaceColorSpace, state->extent, drawn ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
-    } else if (!drawn) {
+        state->overlay->NotePresent(std::chrono::steady_clock::now(), clearOnly ? VkExtent2D{} : VkExtent2D{width, height}, OverlayPresentInfo{std::exchange(state->presentedAddress, 0), state->presentMode, presentCounters.acquireDrops});
+        state->overlay->Record(commands, static_cast<std::size_t>(&slot - state->presentSlots.data()), barrier.image, state->surfaceFormat, state->surfaceColorSpace, state->extent, drawn || plane != nullptr ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
+    } else if (!drawn && plane == nullptr) {
         barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
         barrier.dstAccessMask = 0;
         barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
@@ -3705,6 +3813,7 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
         // The image stays alive until the slot is retired (storage-cache eviction only drops the
         // cache's reference).
         slot.kept = resident;
+        slot.keptPlane = planeSource.resident;
         slot.imageIndex = index;
         slot.inFlight = true;
         slot.shared = shared;
@@ -3720,6 +3829,7 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
         state->queuePending = true;
         state->queueIndex = index;
         ++presentCounters.presents;
+        if (plane != nullptr) ++presentCounters.planes;
         if (auto* frame = PerformanceContext::Current()) {
             frame->NoteBlit(batchesAtBlit, lastSubmittedAt, submittedAt);
             // The serials are cumulative: only their difference (the batches other queues put
@@ -3977,7 +4087,7 @@ Graphics::Context VulkanDevice::buildContext() const {
         state->physical,
         state->queue,
         state->pool,
-        state->deviceProc,
+        state->contextDeviceProc != nullptr ? state->contextDeviceProc : state->deviceProc,
         state->InstanceFunction<PFN_vkGetPhysicalDeviceFormatProperties>("vkGetPhysicalDeviceFormatProperties"),
         state->InstanceFunction<PFN_vkGetPhysicalDeviceImageFormatProperties>("vkGetPhysicalDeviceImageFormatProperties"),
         state->memoryProperties,

core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp

@@ -3,6 +3,7 @@
 
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
 #include "Recompiler.hpp"
+#include <memory>
 #include <string>
 #include <utility>
 #include <vector>
@@ -22,7 +23,7 @@ public:
         std::uint32_t stage = 0;
         std::uint64_t shader = 0;
         std::vector<ShaderRecompiler::ImageTableEntryPoison> poison;
-        std::vector<std::string> messages;
+        std::vector<std::shared_ptr<const std::string>> messages;
     };
     void SetImageTables(std::vector<ImageTableFaults> tables, std::vector<std::pair<std::uint64_t, std::uint64_t>> ranges);
     struct SrtGuardFaults {

core/libs/prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp

@@ -0,0 +1,17 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DEFERREDDESTRUCTION_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DEFERREDDESTRUCTION_HPP
+
+#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
+#include <chrono>
+#include <cstddef>
+
+namespace AgcDriver::Graphics {
+
+PFN_vkGetDeviceProcAddr DeferDestructionToIdle(VkDevice device, PFN_vkGetDeviceProcAddr resolver);
+void DestroyDeferredAtIdle(const Context& context, std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now());
+void DestroyDeferredObjects(VkDevice device) noexcept;
+std::size_t DeferredObjects(VkDevice device);
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp

@@ -6,6 +6,7 @@
 #include "prx/libc/include/GuestAllocations.hpp"
 #include <algorithm>
 #include <array>
+#include <atomic>
 #include <cstddef>
 #include <cstdint>
 #include <limits>
@@ -75,8 +76,8 @@ ImportProbe ProbeDmaBufImportWriteProtection(const Context& context);
 ImportWatch PrepareImportWatch(const Context& context);
 void SetImportWatch(const Context& context, ImportWatch watch);
 
-enum class ImportRequester : std::uint8_t { Map, Build, Access, Join, Segment };
-inline constexpr std::size_t ImportRequesterCount = 5;
+enum class ImportRequester : std::uint8_t { Map, Build, Access, Join, Segment, ColorTarget };
+inline constexpr std::size_t ImportRequesterCount = 6;
 
 struct HostImportRequesterCounts {
     std::uint64_t made = 0;
@@ -135,6 +136,7 @@ bool ImportMappedRanges(const Context& context, const GuestAllocations::Mapped&
 // made outside the device lock (a sampled texture's path, a dispatch's pre-sync); the path taken
 // re-checks with HostImportFor when it binds the import.
 bool HostImportCovers(const Context& context, std::uint64_t address, std::size_t bytes);
+bool ComparedHostImportCovers(const Context& context, std::uint64_t address, std::size_t bytes);
 struct RegisteredPiece {
     std::uint64_t address;
     std::uint64_t bytes;
@@ -271,6 +273,24 @@ struct AddressCopy {
 };
 std::string AddressCopyOverflow(std::vector<AddressCopy> copies, std::uint64_t limit);
 
+class AddressCopyCounters {
+public:
+    struct Totals {
+        std::uint64_t bytes = 0;
+        std::uint64_t largest = 0;
+    };
+
+    static AddressCopyCounters& ThisThread();
+    static Totals Fold();
+    static std::uint64_t Limit();
+
+    void Count(std::uint64_t copied);
+
+private:
+    alignas(64) std::atomic<std::uint64_t> bytes{0};
+    std::atomic<std::uint64_t> largest{0};
+};
+
 class GuestBufferMemory {
 public:
     explicit GuestBufferMemory(const Context& context);

core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp

@@ -401,7 +401,7 @@ private:
         std::size_t mapAllocation = std::numeric_limits<std::size_t>::max();
         std::vector<std::uint32_t> map;
         std::vector<ShaderRecompiler::ImageTableEntryPoison> poison;
-        std::vector<std::string> messages;
+        std::vector<std::shared_ptr<const std::string>> messages;
         std::uint32_t elementBase = 0;
         std::uint32_t set = 0;
         std::size_t firstSampler = 0;

core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp

@@ -181,6 +181,7 @@ ShaderStages DecodeShaderStages(const QueueState& queue, bool vertexWorkgroup =
 void ApplyPointSize(const State& state, ShaderRecompiler::ShaderVertexStageInfo& vertex);
 State DecodeState(const QueueState& queue, bool vertexWorkgroup = false);
 void ApplyDepthRemap(const Context& context, State& state);
+void ApplyDepthBoundsEmulation(const Context& context, State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel);
 void NormalizeUnusedViewportDepth(State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel);
 std::array<VkSampleLocationEXT, 32> SampleLocations(const State& state);
 std::array<std::uint8_t, 32> PixelSampleLocations(const State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel);

core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp

@@ -65,7 +65,7 @@ public:
     // compute pass writing it and the next pass sampling it share one image and copy nothing.
     // CanCopyFrom says whether the two descriptors address the same surface compatibly.
     Texture(const Context& context, const std::shared_ptr<StorageTexture>& source, const GuestTextureResource& descriptor, VkComponentMapping components);
-    Texture(const Context& context, VkImage depthImage, VkFormat depthFormat, VkImageAspectFlags aspect, VkComponentMapping components, VkImageViewType viewType = VK_IMAGE_VIEW_TYPE_2D, std::uint32_t baseLayer = 0, std::shared_ptr<OwnedImage> owner = {}, VkImageUsageFlags usage = 0);
+    Texture(const Context& context, VkImage depthImage, VkFormat depthFormat, VkImageAspectFlags aspect, VkComponentMapping components, VkImageViewType viewType = VK_IMAGE_VIEW_TYPE_2D, std::uint32_t baseLayer = 0, std::shared_ptr<OwnedImage> owner = {}, VkImageUsageFlags usage = 0, std::uint32_t layerCount = VK_REMAINING_ARRAY_LAYERS);
     struct DepthCopyTag {};
     Texture(const Context& context, const std::shared_ptr<StorageTexture>& source, const GuestTextureResource& descriptor, VkComponentMapping components, DepthCopyTag);
     struct SrgbExpandTag {};

core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp

@@ -13,6 +13,7 @@ namespace AgcDriver::Graphics {
 struct Context;
 
 VkFormat ResolveTextureFormat(std::uint32_t guestFormat);
+VkFormat DccClearTextureFormat(std::uint32_t guestFormat);
 VkComponentSwizzle TextureComponentChannel(std::uint32_t guestFormat, VkComponentSwizzle component);
 std::uint32_t SrgbDecodeFormats(PFN_vkGetPhysicalDeviceFormatProperties formatProperties, VkPhysicalDevice physical);
 VkFormat SampledTextureFormat(const Context& context, std::uint32_t guestFormat);

core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp

@@ -272,7 +272,7 @@ void BdaResources::CheckFault() const {
                 message << std::dec << ")";
             }
             message << ": " << ShaderRecompiler::ImageTableAbi::PoisonReasonName(static_cast<ShaderRecompiler::ImageTableAbi::PoisonReason>(poison.reason));
-            if (report.bytes < tables->messages.size() && !tables->messages[report.bytes].empty()) message << ": " << tables->messages[report.bytes];
+            if (report.bytes < tables->messages.size() && tables->messages[report.bytes] != nullptr && !tables->messages[report.bytes]->empty()) message << ": " << *tables->messages[report.bytes];
         } else {
             message << ": poison entry " << report.bytes;
         }

core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp

@@ -521,9 +521,9 @@ DccKeys textureClearKeys(const GuestTextureResource& resource, std::uint64_t gue
     if (const auto known = WaitForKeyWriters(resource, guestBytes)) keys = *known;
     else keys = currentDccKeys(resource.dccAddress, DccKeyBytes(guestBytes), memoized);
     if (keys == DccKeys::Uncompressed || (keys == DccKeys::ClearSingle && SingleKeysReadable(resource))) return keys;
-    if (IsConvertedTextureFormat(resource.format) && (keys == DccKeys::Clear0001 || keys == DccKeys::Clear1110)) throw std::runtime_error(std::string("AGC graphics: DCC clear code ") + DccKeysName(keys) + " of converted texture format " + std::to_string(resource.format) + " is not implemented");
+    if (DccClearTextureFormat(resource.format) == ResolveTextureFormat(resource.format) && IsConvertedTextureFormat(resource.format) && (keys == DccKeys::Clear0001 || keys == DccKeys::Clear1110)) throw std::runtime_error(std::string("AGC graphics: DCC clear code ") + DccKeysName(keys) + " of converted texture format " + std::to_string(resource.format) + " is not implemented");
     std::byte probe[16]{};
-    if (!IsDccClear(keys) || !FillDccClear(ResolveTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, std::span(probe, std::min<std::size_t>(sizeof(probe), BytesPerElement(resource.format))))) {
+    if (!IsDccClear(keys) || !FillDccClear(DccClearTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, std::span(probe, std::min<std::size_t>(sizeof(probe), BytesPerElement(resource.format))))) {
         static AgcDriver::Mutex reportedMutex;
         static std::set<std::pair<std::uint64_t, int>> reported;
         std::lock_guard lock(reportedMutex);
@@ -776,7 +776,7 @@ void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std:
     // Named for the [hooksync] attribution: the read goes through the flush hook.
     const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::TextureRead);
     if (keys == DccKeys::Uncompressed || keys == DccKeys::ClearSingle) GuestMemory::ReadCommitted(resource.baseAddress, bytes);
-    else FillDccClear(ResolveTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, bytes);
+    else FillDccClear(DccClearTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, bytes);
     if (keys == DccKeys::ClearSingle) ExpandSingleBlocks(bytes, BytesPerElement(resource.format));
 }
 
@@ -823,7 +823,7 @@ namespace {
 DccKeys EncodedKeys(const GuestTextureResource& resource, DccKeys keys) {
     if (keys == DccKeys::ClearSingle && SingleKeysReadable(resource)) return keys;
     std::byte probe[16]{};
-    if (!IsDccClear(keys) || !FillDccClear(ResolveTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, std::span(probe, std::min<std::size_t>(sizeof(probe), BytesPerElement(resource.format))))) return DccKeys::Uncompressed;
+    if (!IsDccClear(keys) || !FillDccClear(DccClearTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, std::span(probe, std::min<std::size_t>(sizeof(probe), BytesPerElement(resource.format))))) return DccKeys::Uncompressed;
     return keys;
 }
 

core/libs/prx/libSceAgcDriver/Graphics/src/DeferredDestruction.cpp

@@ -0,0 +1,134 @@
+#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
+#include <atomic>
+#include <cstring>
+#include <mutex>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace AgcDriver::Graphics {
+
+namespace {
+
+constexpr std::size_t DrainObjects = 512;
+constexpr auto DrainAge = std::chrono::milliseconds(100);
+
+struct Deferred {
+    PFN_vkDestroyBuffer destroyBuffer = nullptr;
+    PFN_vkFreeMemory freeMemory = nullptr;
+    std::vector<VkBuffer> buffers;
+    std::vector<VkDeviceMemory> memory;
+    std::chrono::steady_clock::time_point oldest{};
+};
+
+struct Registry {
+    std::mutex mutex;
+    std::unordered_map<VkDevice, Deferred> devices;
+    std::atomic<PFN_vkGetDeviceProcAddr> resolver{nullptr};
+};
+
+Registry& registry() {
+    static auto& state = *new Registry();
+    return state;
+}
+
+void destroyAll(VkDevice device, const Deferred& deferred) noexcept {
+    for (const auto buffer : deferred.buffers) deferred.destroyBuffer(device, buffer, nullptr);
+    for (const auto memory : deferred.memory) deferred.freeMemory(device, memory, nullptr);
+}
+
+template<typename THandle>
+bool defer(VkDevice device, THandle handle, std::vector<THandle> Deferred::*list) noexcept {
+    try {
+        auto& state = registry();
+        std::lock_guard lock(state.mutex);
+        const auto found = state.devices.find(device);
+        if (found == state.devices.end()) return false;
+        auto& deferred = found->second;
+        if (deferred.buffers.empty() && deferred.memory.empty()) deferred.oldest = std::chrono::steady_clock::now();
+        (deferred.*list).push_back(handle);
+        return true;
+    } catch (...) {
+        return false;
+    }
+}
+
+VKAPI_ATTR void VKAPI_CALL deferredDestroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks* allocator) {
+    if (buffer == VK_NULL_HANDLE || defer(device, buffer, &Deferred::buffers)) return;
+    reinterpret_cast<PFN_vkDestroyBuffer>(registry().resolver.load(std::memory_order_acquire)(device, "vkDestroyBuffer"))(device, buffer, allocator);
+}
+
+VKAPI_ATTR void VKAPI_CALL deferredFreeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks* allocator) {
+    if (memory == VK_NULL_HANDLE || defer(device, memory, &Deferred::memory)) return;
+    reinterpret_cast<PFN_vkFreeMemory>(registry().resolver.load(std::memory_order_acquire)(device, "vkFreeMemory"))(device, memory, allocator);
+}
+
+VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL deferringDeviceProc(VkDevice device, const char* name) {
+    if (std::strcmp(name, "vkDestroyBuffer") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&deferredDestroyBuffer);
+    if (std::strcmp(name, "vkFreeMemory") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&deferredFreeMemory);
+    return registry().resolver.load(std::memory_order_acquire)(device, name);
+}
+
+}
+
+PFN_vkGetDeviceProcAddr DeferDestructionToIdle(VkDevice device, PFN_vkGetDeviceProcAddr resolver) {
+    if (device == VK_NULL_HANDLE || resolver == nullptr) throw std::invalid_argument("AGC graphics: deferred destruction needs a device and its resolver");
+    Deferred deferred;
+    deferred.destroyBuffer = reinterpret_cast<PFN_vkDestroyBuffer>(resolver(device, "vkDestroyBuffer"));
+    deferred.freeMemory = reinterpret_cast<PFN_vkFreeMemory>(resolver(device, "vkFreeMemory"));
+    if (deferred.destroyBuffer == nullptr || deferred.freeMemory == nullptr) throw std::runtime_error("AGC graphics: missing Vulkan function: vkDestroyBuffer or vkFreeMemory");
+    auto& state = registry();
+    std::lock_guard lock(state.mutex);
+    const auto current = state.resolver.load(std::memory_order_acquire);
+    if (current != nullptr && current != resolver) throw std::runtime_error("AGC graphics: deferred destruction found two different device resolvers");
+    if (!state.devices.emplace(device, std::move(deferred)).second) throw std::runtime_error("AGC graphics: deferred destruction was set up twice for one device");
+    state.resolver.store(resolver, std::memory_order_release);
+    return &deferringDeviceProc;
+}
+
+void DestroyDeferredAtIdle(const Context& context, std::chrono::steady_clock::time_point now) {
+    auto& state = registry();
+    {
+        std::lock_guard lock(state.mutex);
+        const auto found = state.devices.find(context.device);
+        if (found == state.devices.end()) return;
+        const auto objects = found->second.buffers.size() + found->second.memory.size();
+        if (objects == 0 || (objects < DrainObjects && now - found->second.oldest < DrainAge)) return;
+    }
+    const auto idle = context.Function<PFN_vkQueueWaitIdle>("vkQueueWaitIdle")(context.queue);
+    if (idle != VK_SUCCESS && idle != VK_ERROR_DEVICE_LOST) throw std::runtime_error("AGC graphics: vkQueueWaitIdle before destroying deferred objects: Vulkan result " + std::to_string(idle));
+    Deferred taken;
+    {
+        std::lock_guard lock(state.mutex);
+        const auto found = state.devices.find(context.device);
+        if (found == state.devices.end()) return;
+        taken.destroyBuffer = found->second.destroyBuffer;
+        taken.freeMemory = found->second.freeMemory;
+        taken.buffers.swap(found->second.buffers);
+        taken.memory.swap(found->second.memory);
+    }
+    destroyAll(context.device, taken);
+}
+
+void DestroyDeferredObjects(VkDevice device) noexcept {
+    Deferred taken;
+    {
+        auto& state = registry();
+        std::lock_guard lock(state.mutex);
+        const auto found = state.devices.find(device);
+        if (found == state.devices.end()) return;
+        taken = std::move(found->second);
+        state.devices.erase(found);
+    }
+    destroyAll(device, taken);
+}
+
+std::size_t DeferredObjects(VkDevice device) {
+    auto& state = registry();
+    std::lock_guard lock(state.mutex);
+    const auto found = state.devices.find(device);
+    return found == state.devices.end() ? 0 : found->second.buffers.size() + found->second.memory.size();
+}
+
+}

core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp

@@ -719,10 +719,30 @@ public:
         RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_MEMORY_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
         context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, storage->Image(), VK_IMAGE_LAYOUT_GENERAL, depthPlane.staging->Handle(), 1, &colorRegion);
         RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
-        context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, depthPlane.staging->Handle(), image, VK_IMAGE_LAYOUT_GENERAL, 1, &depthRegion);
-        RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT);
+        if (target.stencilAddress == 0) {
+            VkImageMemoryBarrier discard{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
+            discard.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
+            discard.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+            discard.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+            discard.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+            discard.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+            discard.image = image;
+            discard.subresourceRange = {VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, 1};
+            const auto barrier = context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier");
+            barrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &discard);
+            context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, depthPlane.staging->Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &depthRegion);
+            VkImageMemoryBarrier toGeneral = discard;
+            toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
+            toGeneral.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
+            toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+            toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL;
+            barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral);
+        } else {
+            context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, depthPlane.staging->Handle(), image, VK_IMAGE_LAYOUT_GENERAL, 1, &depthRegion);
+            RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT);
+        }
         if (batch) batch->SubmitAndWait();
-        else Recorder::CountBarriers(Recorder::CommandClass::Copy, 3);
+        else Recorder::CountBarriers(Recorder::CommandClass::Copy, target.stencilAddress == 0 ? 4 : 3);
     }
 
     std::shared_ptr<StorageTexture> Mirror(const GuestTextureResource& resource) {
@@ -1170,7 +1190,6 @@ struct DepthPlaneSlice {
 class DepthPlaneCopy {
 public:
     DepthPlaneCopy(const Context& context, VkExtent2D extent, VkFormat format, std::uint32_t layers) : context(context), extent(extent), format(format), layers(layers), guestLayers(layers), surfaceLayers(layers) {
-        this->context.bufferPool.reset();
         VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
         info.flags = layers % 6u == 0 && extent.width == extent.height ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT : 0u;
         info.imageType = VK_IMAGE_TYPE_2D;
@@ -1185,15 +1204,16 @@ public:
         info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
         try {
             Check(context.Function<PFN_vkCreateImage>("vkCreateImage")(context.device, &info, nullptr, &image), "vkCreateImage depth plane copy");
+            owned = std::make_shared<OwnedImage>(context, image, VK_NULL_HANDLE);
             VkMemoryRequirements requirements{};
             context.Function<PFN_vkGetImageMemoryRequirements>("vkGetImageMemoryRequirements")(context.device, image, &requirements);
             VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
             allocation.allocationSize = requirements.size;
             allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
-            memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory depth plane copy");
+            owned->Allocate(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory depth plane copy");
+            memory = owned->memory;
             Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory depth plane copy");
             imageBytes = requirements.size;
-            owned = std::make_shared<OwnedImage>(context, image, memory);
         } catch (...) {
             release();
             throw;
@@ -1204,19 +1224,21 @@ public:
     DepthPlaneCopy& operator=(const DepthPlaneCopy&) = delete;
 
     std::shared_ptr<Texture> Refresh(std::span<const DepthPlaneSlice> slices, const GuestTextureResource& resource, VkComponentMapping components, VkImageViewType viewType, std::uint32_t baseLayer) {
-        Require(slices.size() == layers, "depth plane copy slices do not match its layers");
+        const auto requested = static_cast<std::uint32_t>(slices.size());
+        Require(requested != 0 && requested <= layers, "depth plane copy slices exceed its layers");
+        Require(baseLayer < requested, "a depth plane copy view starts past the slices it was given");
         const auto geometry = DescribeSurface(resource);
         const bool guestFill = std::any_of(slices.begin(), slices.end(), [](const auto& slice) { return slice.image == VK_NULL_HANDLE; });
-        if (guestFill && (geometry.layers != layers || geometry.sliceLinearBytes < sliceBytes() || geometry.mips.empty())) {
+        if (guestFill && (geometry.layers != requested || geometry.sliceLinearBytes < sliceBytes() || geometry.mips.empty())) {
             char text[320];
             std::snprintf(text, sizeof(text), "AGC graphics: depth plane copy geometry does not match its layers (copy %ux%u, %u layers, %llu bytes a slice; texture %ux%u guest format %u, dimension %d, tile mode %u, %u layers, %llu bytes a slice, %zu mips)", extent.width, extent.height, layers, static_cast<unsigned long long>(sliceBytes()), resource.width, resource.height, resource.format, static_cast<int>(resource.dimension), resource.tileMode, geometry.layers, static_cast<unsigned long long>(geometry.sliceLinearBytes), geometry.mips.size());
             throw std::runtime_error(text);
         }
-        const auto stagingBytes = std::max<std::size_t>(static_cast<std::size_t>(geometry.sliceLinearBytes), static_cast<std::size_t>(sliceBytes())) * layers;
+        const auto stagingBytes = std::max<std::size_t>(static_cast<std::size_t>(geometry.sliceLinearBytes), static_cast<std::size_t>(sliceBytes())) * requested;
         std::vector<std::shared_ptr<Buffer>> uploads;
         std::vector<std::uint32_t> uploadLayers;
         std::vector<std::uint32_t> copiedLayers;
-        for (std::uint32_t layer = 0; layer < layers; ++layer) {
+        for (std::uint32_t layer = 0; layer < requested; ++layer) {
             auto& held = guestLayers[layer];
             auto& surface = surfaceLayers[layer];
             if (slices[layer].image != VK_NULL_HANDLE) {
@@ -1244,10 +1266,11 @@ public:
             if (staging == nullptr || stagingBytes > staging->Size()) staging = std::make_shared<DeviceBuffer>(context, stagingBytes, VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
             record(slices, resource, geometry, copiedLayers, uploads, uploadLayers, staging);
         }
-        const std::array<std::uint32_t, 6> key{static_cast<std::uint32_t>(components.r), static_cast<std::uint32_t>(components.g), static_cast<std::uint32_t>(components.b), static_cast<std::uint32_t>(components.a), static_cast<std::uint32_t>(viewType), baseLayer};
+        const auto viewLayers = viewType == VK_IMAGE_VIEW_TYPE_2D ? 1u : requested - baseLayer;
+        const std::array<std::uint32_t, 7> key{static_cast<std::uint32_t>(components.r), static_cast<std::uint32_t>(components.g), static_cast<std::uint32_t>(components.b), static_cast<std::uint32_t>(components.a), static_cast<std::uint32_t>(viewType), baseLayer, viewLayers};
         auto& texture = textures[key];
         if (texture == nullptr) {
-            texture = std::make_shared<Texture>(context, image, format, aspect(), components, viewType, baseLayer, owned);
+            texture = std::make_shared<Texture>(context, image, format, aspect(), components, viewType, baseLayer, owned, 0, viewLayers);
             texture->MarkRefreshedPerUse();
         }
         return texture;
@@ -1275,7 +1298,7 @@ private:
     VkDeviceMemory memory = VK_NULL_HANDLE;
     std::shared_ptr<OwnedImage> owned;
     std::shared_ptr<DeviceBuffer> staging;
-    std::map<std::array<std::uint32_t, 6>, std::shared_ptr<Texture>> textures;
+    std::map<std::array<std::uint32_t, 7>, std::shared_ptr<Texture>> textures;
     std::vector<GuestLayer> guestLayers;
     std::vector<DepthPlaneSlice> surfaceLayers;
     bool filled = false;
@@ -1365,8 +1388,8 @@ private:
     }
 };
 
-std::map<std::tuple<VkDevice, std::uint64_t, std::uint32_t, std::uint32_t, VkFormat, std::uint32_t>, std::unique_ptr<DepthPlaneCopy>>& planeCopies() {
-    static auto* copies = new std::map<std::tuple<VkDevice, std::uint64_t, std::uint32_t, std::uint32_t, VkFormat, std::uint32_t>, std::unique_ptr<DepthPlaneCopy>>();
+std::map<std::tuple<VkDevice, std::uint64_t, std::uint32_t, std::uint32_t, VkFormat>, std::unique_ptr<DepthPlaneCopy>>& planeCopies() {
+    static auto* copies = new std::map<std::tuple<VkDevice, std::uint64_t, std::uint32_t, std::uint32_t, VkFormat>, std::unique_ptr<DepthPlaneCopy>>();
     return *copies;
 }
 
@@ -1375,7 +1398,7 @@ std::uint64_t& planeCopyClock() {
     return clock;
 }
 
-void evictPlaneCopies(const std::tuple<VkDevice, std::uint64_t, std::uint32_t, std::uint32_t, VkFormat, std::uint32_t>& keep) {
+void evictPlaneCopies(const std::tuple<VkDevice, std::uint64_t, std::uint32_t, std::uint32_t, VkFormat>& keep) {
     static const VkDeviceSize budget = [] {
         const char* text = std::getenv("APS5_DEPTH_PLANE_COPY_MIB");
         return static_cast<VkDeviceSize>(text != nullptr ? std::strtoull(text, nullptr, 10) : 256ull) << 20u;
@@ -2399,9 +2422,9 @@ std::shared_ptr<Texture> DepthSurfaceTexture(const Context& context, std::span<c
     }
     const auto imageFormat = integer ? VK_FORMAT_R16_UINT : d16 ? VK_FORMAT_D16_UNORM : VK_FORMAT_D32_SFLOAT;
     auto& copies = planeCopies();
-    const std::tuple<VkDevice, std::uint64_t, std::uint32_t, std::uint32_t, VkFormat, std::uint32_t> copyKey{context.device, base.address, base.extent.width, base.extent.height, imageFormat, layers};
+    const std::tuple<VkDevice, std::uint64_t, std::uint32_t, std::uint32_t, VkFormat> copyKey{context.device, base.address, base.extent.width, base.extent.height, imageFormat};
     auto& copy = copies[copyKey];
-    if (copy == nullptr) copy = std::make_unique<DepthPlaneCopy>(context, base.extent, imageFormat, layers);
+    if (copy == nullptr || copy->layers < layers) copy = std::make_unique<DepthPlaneCopy>(context, base.extent, imageFormat, layers);
     copy->lastUse = ++planeCopyClock();
     auto texture = copy->Refresh(slices, resource, components, cube || resource.dimension == TextureDimension::k2DArray ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D, layered && !cube ? resource.baseArray : 0u);
     evictPlaneCopies(copyKey);

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -2253,7 +2253,10 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
     outcome.passContinued = continued;
     outcome.passBegun = !continued;
     const auto commands = start.commands;
-    if (!continued) recorder->PrepareSampleSlot();
+    if (!continued) {
+        recorder->PrepareSampleSlot();
+        PerformanceTrace::Count(PerformanceTrace::RenderPassScope);
+    }
     if (capture) captureInputs(context, *recorder, commands, resources, drawBindings, record.targets.empty() || record.targets.front() == nullptr ? 0 : record.targets.front()->Descriptor().baseAddress);
     // The draw's [gputime] class range: from its first barrier to the pass's trailing barrier (a
     // continued draw lies inside its pass's range).
@@ -2566,7 +2569,7 @@ bool prepareDraw(const Context& context, const State& state, const Pm4::DrawPara
         // Debug aid: APS5_NO_RESIDENT_TARGETS=1 copies every target in and out again.
         static const bool residentTargets = std::getenv("APS5_NO_RESIDENT_TARGETS") == nullptr;
         static const bool residentLinear = std::getenv("APS5_NO_RESIDENT_LINEAR") == nullptr;
-        const bool linearResident = residentLinear && color.tileMode == ColorTileMode::Linear && ExistingHostImportFor(context, color.address, colorLayout.Bytes()) != nullptr;
+        const bool linearResident = residentLinear && residentTargets && context.detiler != nullptr && color.tileMode == ColorTileMode::Linear && (ExistingHostImportFor(context, color.address, colorLayout.Bytes()) != nullptr || (GuestMemory::Watched(color.address, colorLayout.Bytes()) && HostImportFor(context, color.address, colorLayout.Bytes(), ImportRequester::ColorTarget) != nullptr));
         if ((binding.gpuTiling || ((color.tileMode != ColorTileMode::Linear || linearResident) && context.detiler != nullptr)) && residentTargets) {
             // The lookup refreshes the image on every draw (StorageTexture::Refresh: FlushPending,
             // CollectWrites over the target's pages, the DCC key scan of TextureClearKeys, then

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -34,6 +34,7 @@
 #include <functional>
 #include <bit>
 #include <condition_variable>
+#include <deque>
 #include <iterator>
 #include <list>
 #include <map>
@@ -179,6 +180,14 @@ struct ImportRefusal {
     std::uint64_t budget;
 };
 
+struct GateRefusal {
+    std::uint64_t base;
+    std::uint64_t bytes;
+    GuestAllocations::Mapped ranges;
+    std::uint64_t commits;
+    std::uint64_t watches;
+};
+
 struct EvictingRefusal {
     std::uint64_t base;
     std::uint64_t bytes;
@@ -250,6 +259,7 @@ struct HostImports {
     std::map<std::pair<std::uint64_t, std::uint64_t>, ImportFailure> failedBlocks;
     std::map<std::pair<std::uint64_t, std::uint64_t>, ImportRefusal> refusals;
     std::map<std::pair<std::uint64_t, std::uint64_t>, EvictingRefusal> evictingRefusals;
+    std::map<std::pair<std::uint64_t, std::uint64_t>, GateRefusal> gateRefusals;
     // Registry generation the imports were last reconciled with.
     std::uint64_t refreshedGeneration = 0;
     // Bumped whenever an import is dropped, so HostImport pointers taken under the lock earlier can be
@@ -287,6 +297,10 @@ struct HostImports {
     std::uint8_t reportNoted = 0;
     std::map<std::uint64_t, std::uint64_t> spaceUses;
     std::uint64_t budgetRefusals = 0;
+    std::array<const HostImport*, 4> recentImports{};
+    std::uint64_t recentEpoch = std::numeric_limits<std::uint64_t>::max();
+    std::size_t recentSize = 0;
+    std::size_t recentNext = 0;
 };
 
 HostImports& Imports() {
@@ -294,6 +308,31 @@ HostImports& Imports() {
     return imports;
 }
 
+bool recentImportsEnabled() {
+    static const bool enabled = std::getenv("APS5_NO_RECENT_IMPORTS") == nullptr;
+    return enabled;
+}
+
+const HostImport* recentImport(HostImports& state, std::uint64_t begin, std::uint64_t end) {
+    if (state.recentEpoch != state.epoch || state.recentSize != state.imports.size()) return nullptr;
+    for (const auto* entry : state.recentImports) {
+        if (entry != nullptr && begin >= entry->base && end <= entry->base + entry->bytes) {
+            entry->lastUse = ++state.useCounter;
+            return entry;
+        }
+    }
+    return nullptr;
+}
+
+void rememberImport(HostImports& state, const HostImport* entry) {
+    if (state.recentEpoch != state.epoch || state.recentSize != state.imports.size()) {
+        state.recentImports = {};
+        state.recentEpoch = state.epoch;
+        state.recentSize = state.imports.size();
+    }
+    state.recentImports[state.recentNext++ % state.recentImports.size()] = entry;
+}
+
 #ifdef _WIN32
 void waitSettleBatch(const ImportSettle& settle, VkDevice device, const SettleBatch& batch) {
     if (batch.busy && settle.waitForFences != nullptr) Check(settle.waitForFences(device, 1, &batch.fence, VK_TRUE, 60'000'000'000ull), "vkWaitForFences (settle batch)");
@@ -828,8 +867,12 @@ struct BackingFile {
     std::uint64_t offset = 0;
 };
 
+bool sharedBacking(const Context& context, std::uint64_t base, std::uint64_t bytes, BackingFile& backing) {
+    return context.dmaBufImport && GuestArena::GuestArenaSharedBacking_nid_postfix(static_cast<std::uintptr_t>(base), static_cast<std::size_t>(bytes), &backing.file, &backing.offset);
+}
+
 bool sharedBacking(const Context& context, const HostImport& entry, BackingFile& backing) {
-    return context.dmaBufImport && GuestArena::GuestArenaSharedBacking_nid_postfix(static_cast<std::uintptr_t>(entry.base), static_cast<std::size_t>(entry.bytes), &backing.file, &backing.offset);
+    return sharedBacking(context, entry.base, entry.bytes, backing);
 }
 
 const char* createBackingImport(const Context& context, HostImport& entry, BackingFile& backing, VkResult& failure) {
@@ -1252,6 +1295,62 @@ void decideImportWatch(const Context& context, HostImports& state) {
 #endif
 }
 
+struct ImportWatching {
+    bool settled = false;
+    bool hostPointer = false;
+    bool dmaBuf = false;
+};
+
+ImportWatching importWatching(const Context& context, HostImports& state, bool alias, bool writable, std::uint64_t bytes) {
+    decideImportWatch(context, state);
+#ifdef _WIN32
+    const bool settled = !alias && state.watchPrivateImports && writable && bytes >= settleFloorBytes;
+    const bool watched = alias || !state.unwatchImports || settled;
+    return {settled, watched, watched};
+#else
+    static_cast<void>(alias);
+    static_cast<void>(writable);
+    static_cast<void>(bytes);
+    return {false, !state.unwatchImports, !state.unwatchDmaBufImports};
+#endif
+}
+
+#ifdef _WIN32
+struct ImportPages {
+    bool committed = true;
+    bool writable = true;
+    bool readOnly = true;
+    bool shared = true;
+    MEMORY_BASIC_INFORMATION refused{};
+};
+
+bool writableProtection(std::uint32_t protection) {
+    protection &= 0xffu;
+    return protection == PAGE_READWRITE || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_WRITECOPY;
+}
+
+ImportPages importPages(std::uint64_t base, std::uint64_t bytes, bool opening = false) {
+    ImportPages pages;
+    for (std::uint64_t cursor = base; cursor < base + bytes;) {
+        MEMORY_BASIC_INFORMATION info{};
+        if (VirtualQuery(reinterpret_cast<const void*>(cursor), &info, sizeof(info)) == 0 || info.State != MEM_COMMIT || (info.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) {
+            pages.committed = false;
+            return pages;
+        }
+        const auto protection = info.Protect & 0xffu;
+        const bool pageWritable = (info.Type == MEM_PRIVATE || info.Type == MEM_IMAGE) && writableProtection(protection);
+        const bool pageReadOnly = info.Type != MEM_MAPPED && (protection == PAGE_READONLY || protection == PAGE_EXECUTE_READ);
+        if (!pageReadOnly) pages.readOnly = false;
+        if (info.Type != MEM_MAPPED) pages.shared = false;
+        if (info.Type != MEM_MAPPED && !pageWritable && !pageReadOnly && pages.refused.BaseAddress == nullptr) pages.refused = info;
+        if (!pageWritable) pages.writable = false;
+        if (opening && !pages.writable && !pages.shared) return pages;
+        cursor = reinterpret_cast<std::uint64_t>(info.BaseAddress) + info.RegionSize;
+    }
+    return pages;
+}
+#endif
+
 std::shared_ptr<const GuestAllocations::Range> leasedRangeOwner(const GuestAllocations::Lease& lease, std::uint64_t base) {
     const auto found = std::lower_bound(lease.begin(), lease.end(), base, [](const auto& range, std::uint64_t value) { return range->address < value; });
     return found != lease.end() && (*found)->address == base ? *found : nullptr;
@@ -1270,6 +1369,22 @@ struct ImportRun {
     std::vector<std::shared_ptr<const GuestAllocations::Range>> ranges;
 };
 
+bool importStaysWatched(const Context& context, HostImports& state, const ImportRun& run) {
+    if (std::any_of(run.ranges.begin(), run.ranges.end(), [](const auto& range) { return !range->writable; })) return false;
+    if (!GuestMemory::Watched(run.base, static_cast<std::size_t>(run.bytes))) return false;
+#ifdef _WIN32
+    MEMORY_BASIC_INFORMATION first{};
+    if (!importWatching(context, state, false, true, run.bytes).hostPointer && (VirtualQuery(reinterpret_cast<const void*>(run.base), &first, sizeof(first)) == 0 || first.Type != MEM_MAPPED)) return false;
+    const auto pages = importPages(run.base, run.bytes, true);
+    if (!pages.committed || (!pages.writable && !pages.shared)) return false;
+    return importWatching(context, state, !pages.writable, pages.writable, run.bytes).hostPointer;
+#else
+    const auto watching = importWatching(context, state, false, true, run.bytes);
+    BackingFile backing;
+    return watching.hostPointer && (watching.dmaBuf || !sharedBacking(context, run.base, run.bytes, backing));
+#endif
+}
+
 bool continuesRun(const GuestAllocations::Range& before, const GuestAllocations::Range& after) {
     return before.address + before.bytes == after.address && before.readable && after.readable && before.writable == after.writable && before.gpu == after.gpu;
 }
@@ -1493,8 +1608,32 @@ bool evictingRefusalCurrent(const HostImports& state, const EvictingRefusal& ref
     return refusal.epoch == state.epoch && refusal.liveBytes == state.liveBytes && budget <= refusal.budget && refusal.window == importWindow.load(std::memory_order_relaxed) && state.useCounter < refusal.ready && (state.buildFloor <= refusal.floor || state.useCounter < refusal.readyHeld);
 }
 
+void noteRetiredUnwatched(std::vector<std::pair<std::uint64_t, std::uint64_t>>& spans, const HostImport& entry, const GuestAllocations::Lease& lease) {
+    if (!entry.unwatched) return;
+    for (const auto& range : entry.ranges) {
+        if (leasedRangeOwner(lease, range->address) != range) continue;
+        const auto begin = std::max(range->address, entry.base);
+        const auto end = std::min(range->address + range->bytes, entry.base + entry.bytes);
+        if (begin < end) spans.emplace_back(begin, end);
+    }
+}
+
+void rewatchRetired(const HostImports& state, std::vector<std::pair<std::uint64_t, std::uint64_t>> spans) {
+    std::erase_if(spans, [&](const auto& span) {
+        auto it = state.imports.upper_bound(span.first);
+        if (it != state.imports.begin()) --it;
+        for (; it != state.imports.end() && it->first < span.second; ++it) {
+            if (it->second.unwatched && span.first < it->first + it->second.bytes) return true;
+        }
+        const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> one{{span}};
+        return StorageTexture::AnyLiveOverlaps(one);
+    });
+    if (!spans.empty()) GuestMemory::Rewatch(spans);
+}
+
 std::uint64_t retireLeastRecent(const Context& context, HostImports& state, const GuestAllocations::Lease& lease, std::uint64_t wanted) {
     std::uint64_t freed = 0;
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> unwatched;
     while (freed < wanted) {
         auto oldest = state.imports.end();
         for (auto it = state.imports.begin(); it != state.imports.end(); ++it) {
@@ -1502,8 +1641,10 @@ std::uint64_t retireLeastRecent(const Context& context, HostImports& state, cons
         }
         if (oldest == state.imports.end()) break;
         freed += oldest->second.pinnedBytes;
+        noteRetiredUnwatched(unwatched, oldest->second, lease);
         retireImport(context.device, state, oldest, lease);
     }
+    rewatchRetired(state, std::move(unwatched));
     return freed;
 }
 
@@ -1522,6 +1663,41 @@ std::uint64_t physicalMemoryBytes() {
 std::atomic<std::uint64_t> budgetOverride{0};
 std::atomic<std::uint64_t> addressCopyLimitOverride{0};
 
+std::uint64_t addressCopyLimit() {
+    static const std::uint64_t configured = [] {
+        const char* value = std::getenv("APS5_ADDRESS_COPY_MAX_MIB");
+        return (value ? std::strtoull(value, nullptr, 10) : 64ull) << 20u;
+    }();
+    const auto limit = addressCopyLimitOverride.load(std::memory_order_relaxed);
+    return limit != 0 ? limit : configured;
+}
+
+constexpr unsigned AddressCopyEpochShift = 48;
+constexpr std::uint64_t AddressCopyEpochMask = 0xffff;
+constexpr std::uint64_t AddressCopyLargestMask = (std::uint64_t{1} << AddressCopyEpochShift) - 1;
+std::atomic<std::uint64_t> addressCopyEpoch{0};
+
+struct AddressCopyRegistry {
+    std::mutex mutex;
+    std::deque<AddressCopyCounters> counters;
+    std::vector<AddressCopyCounters*> unused;
+};
+
+AddressCopyRegistry& addressCopyRegistry() {
+    static auto* registry = new AddressCopyRegistry;
+    return *registry;
+}
+
+struct AddressCopyLease {
+    AddressCopyCounters* counters = nullptr;
+    ~AddressCopyLease() {
+        if (counters == nullptr) return;
+        auto& registry = addressCopyRegistry();
+        std::lock_guard lock(registry.mutex);
+        registry.unused.push_back(counters);
+    }
+};
+
 // Pinned imports count against the driver's system memory heap, which imports, type-3 mirrors, buffers and textures
 // placed in host memory share; past it ordinary host allocations fail. The budget is APS5_HOST_IMPORT_MIB, else
 // HostImportBudget.
@@ -1817,6 +1993,21 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
         }
         state.evictingRefusals.erase(remembered);
     }
+    if (requester == ImportRequester::ColorTarget) {
+#ifdef _WIN32
+        const auto commits = GuestArena::GuestArenaCommitGeneration_nid_postfix();
+#else
+        const std::uint64_t commits = 0;
+#endif
+        const auto watches = GuestMemory::UnwatchSerial();
+        const std::pair key{base, bytes};
+        if (const auto refused = state.gateRefusals.find(key); refused != state.gateRefusals.end() && refused->second.commits == commits && refused->second.watches == watches && sameRun(refused->second, run)) return nullptr;
+        if (!importStaysWatched(context, state, run)) {
+            state.gateRefusals.insert_or_assign(key, GateRefusal{base, bytes, GuestAllocations::Mapped(run.ranges.begin(), run.ranges.end()), commits, watches});
+            return nullptr;
+        }
+        state.gateRefusals.erase(key);
+    }
     auto first = state.imports.lower_bound(base);
     if (first != state.imports.begin() && std::prev(first)->second.base + std::prev(first)->second.bytes > base) --first;
     std::uint64_t replaced = 0;
@@ -1891,27 +2082,17 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
 #ifdef _WIN32
     // Drivers pin imported pages, so every page must be committed and accessible.
     const GuestArena::HostImportAccess importAccess(reinterpret_cast<void*>(base), static_cast<std::size_t>(bytes));
-    bool writable = true;
-    bool readOnly = true;
-    MEMORY_BASIC_INFORMATION refused{};
-    for (std::uint64_t cursor = base; cursor < base + bytes;) {
-        MEMORY_BASIC_INFORMATION info{};
-        if (VirtualQuery(reinterpret_cast<const void*>(cursor), &info, sizeof(info)) == 0 || info.State != MEM_COMMIT || (info.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) return failedImport();
-        const auto protection = info.Protect & 0xffu;
-        const bool pageWritable = (info.Type == MEM_PRIVATE || info.Type == MEM_IMAGE) && (protection == PAGE_READWRITE || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_WRITECOPY);
-        const bool pageReadOnly = info.Type != MEM_MAPPED && (protection == PAGE_READONLY || protection == PAGE_EXECUTE_READ);
-        if (!pageReadOnly) readOnly = false;
-        if (info.Type != MEM_MAPPED && !pageWritable && !pageReadOnly && refused.BaseAddress == nullptr) refused = info;
-        if (!pageWritable) writable = false;
-        cursor = reinterpret_cast<std::uint64_t>(info.BaseAddress) + info.RegionSize;
-    }
-    if (!writable && readOnly) return failedImport();
+    const auto pages = importPages(base, bytes);
+    if (!pages.committed) return failedImport();
+    const bool writable = pages.writable;
+    if (!writable && pages.readOnly) return failedImport();
     if (!writable) {
-        if (refused.BaseAddress != nullptr) {
+        if (pages.refused.BaseAddress != nullptr) {
             char text[256];
-            std::snprintf(text, sizeof(text), "AGC graphics: host import of 0x%llx+0x%llx: memory at 0x%llx (type 0x%lx, protection 0x%lx) is neither read-write, read-only nor a shared mapping", static_cast<unsigned long long>(base), static_cast<unsigned long long>(bytes), reinterpret_cast<unsigned long long>(refused.BaseAddress), refused.Type, refused.Protect);
+            std::snprintf(text, sizeof(text), "AGC graphics: host import of 0x%llx+0x%llx: memory at 0x%llx (type 0x%lx, protection 0x%lx) is neither read-write, read-only nor a shared mapping", static_cast<unsigned long long>(base), static_cast<unsigned long long>(bytes), reinterpret_cast<unsigned long long>(pages.refused.BaseAddress), pages.refused.Type, pages.refused.Protect);
             throw std::runtime_error(text);
         }
+        std::erase_if(state.gateRefusals, [&](const auto& refusal) { return refusal.first.first < base + bytes && base < refusal.first.first + refusal.first.second; });
         try {
             entry.alias = GuestArena::GuestArenaMapAlias_nid_postfix(static_cast<std::uintptr_t>(base), static_cast<std::size_t>(bytes));
         } catch (const std::runtime_error&) {
@@ -1923,17 +2104,16 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
 #else
     if (!GuestMemory::Accessible(reinterpret_cast<const void*>(base), static_cast<std::size_t>(bytes), true)) return failedImport();
 #endif
-    decideImportWatch(context, state);
     VkResult result = VK_SUCCESS;
     const char* step = nullptr;
 #ifdef _WIN32
-    const bool settled = entry.alias == nullptr && state.watchPrivateImports && writable && bytes >= settleFloorBytes;
-    const bool watchedHostPointer = entry.alias != nullptr || !state.unwatchImports || settled;
-    const bool watchedDmaBuf = watchedHostPointer;
+    const auto watching = importWatching(context, state, entry.alias != nullptr, writable, bytes);
+    const bool settled = watching.settled;
 #else
-    const bool watchedHostPointer = !state.unwatchImports;
-    const bool watchedDmaBuf = !state.unwatchDmaBufImports;
+    const auto watching = importWatching(context, state, false, true, bytes);
 #endif
+    const bool watchedHostPointer = watching.hostPointer;
+    const bool watchedDmaBuf = watching.dmaBuf;
     GuestMemory::ImportWatched(base, bytes, [&] {
         step = watchedDmaBuf == watchedHostPointer ? createImport(context, entry, result) : createHostPointerImport(context, entry, result);
 #ifdef _WIN32
@@ -2063,6 +2243,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
         state.failedBlocks.clear();
         state.refusals.clear();
         state.evictingRefusals.clear();
+        state.gateRefusals.clear();
         state.ceiling = 0;
         state.allocationFailures = 0;
         state.allocationFailed = false;
@@ -2081,6 +2262,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
     const auto generation = GuestAllocations::GuestAllocationsGeneration_nid_postfix();
     if (generation == state.refreshedGeneration) return;
     state.refreshedGeneration = generation;
+    state.gateRefusals.clear();
     const auto starts = [&](std::uint64_t address) {
         const auto index = leasedIndexContaining(lease, address);
         return index != lease.size() && lease[index]->address == address;
@@ -2088,6 +2270,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
     std::erase_if(state.sparseRunEnds, [&](const auto& entry) { return !starts(entry.first); });
     std::erase_if(state.sparseRunJoins, [&](const auto& entry) { return !starts(entry.first); });
     const auto budget = effectiveBudget(context, state);
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> unwatched;
     for (auto it = state.imports.begin(); it != state.imports.end();) {
         if (sameRange(it->second, lease, budget)) {
             if (it->second.unwatched) GuestMemory::Unwatch(it->first, it->second.bytes);
@@ -2097,9 +2280,11 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
         static const bool trace = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr;
         if (trace && leasedRangeOwner(lease, it->first, it->first + it->second.bytes) != nullptr) std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx retired: the range was mapped again\n", static_cast<unsigned long long>(it->first), static_cast<unsigned long long>(it->second.bytes));
         const auto next = std::next(it);
+        noteRetiredUnwatched(unwatched, it->second, lease);
         retireImport(context.device, state, it, lease);
         it = next;
     }
+    rewatchRetired(state, std::move(unwatched));
     for (auto it = state.failed.begin(); it != state.failed.end();) it = sameRange(it->second, lease, budget) ? std::next(it) : state.failed.erase(it);
     for (auto it = state.failedJoins.begin(); it != state.failedJoins.end();) it = sameRun(it->second, joinedRun(lease, it->second.base, it->second.base + it->second.bytes)) ? std::next(it) : state.failedJoins.erase(it);
     for (auto it = state.failedBlocks.begin(); it != state.failedBlocks.end();) it = sameRun(it->second, exactRun(lease, it->first.first, it->first.second)) ? std::next(it) : state.failedBlocks.erase(it);
@@ -3459,6 +3644,7 @@ void ClearHostImports(VkDevice device) {
     state.failedBlocks.clear();
     state.refusals.clear();
     state.evictingRefusals.clear();
+    state.gateRefusals.clear();
     state.ceiling = 0;
     state.allocationFailures = 0;
     state.allocationFailed = false;
@@ -3682,6 +3868,7 @@ void SetImportWatch(const Context& context, ImportWatch watch) {
     state.watchDevice = context.device;
     state.unwatchImports = watch == ImportWatch::Unwatch;
     state.unwatchDmaBufImports = state.unwatchImports;
+    state.gateRefusals.clear();
 }
 
 void SetHostImportWindow(std::uint64_t bytes) {
@@ -3725,6 +3912,47 @@ void SetAddressCopyLimit(std::uint64_t bytes) {
     addressCopyLimitOverride.store(bytes, std::memory_order_relaxed);
 }
 
+AddressCopyCounters& AddressCopyCounters::ThisThread() {
+    auto& lease = HostThreadLocal<AddressCopyLease, AddressCopyLease>();
+    if (lease.counters == nullptr) {
+        auto& registry = addressCopyRegistry();
+        std::lock_guard lock(registry.mutex);
+        if (registry.unused.empty()) {
+            lease.counters = &registry.counters.emplace_back();
+        } else {
+            lease.counters = registry.unused.back();
+            registry.unused.pop_back();
+        }
+    }
+    return *lease.counters;
+}
+
+void AddressCopyCounters::Count(std::uint64_t copied) {
+    bytes.store(bytes.load(std::memory_order_relaxed) + copied, std::memory_order_relaxed);
+    const auto epoch = addressCopyEpoch.load(std::memory_order_relaxed) & AddressCopyEpochMask;
+    const auto kept = largest.load(std::memory_order_relaxed);
+    const auto clamped = std::min(copied, AddressCopyLargestMask);
+    if ((kept >> AddressCopyEpochShift) != epoch || (kept & AddressCopyLargestMask) < clamped) largest.store(epoch << AddressCopyEpochShift | clamped, std::memory_order_relaxed);
+}
+
+AddressCopyCounters::Totals AddressCopyCounters::Fold() {
+    auto& registry = addressCopyRegistry();
+    std::lock_guard lock(registry.mutex);
+    const auto epoch = addressCopyEpoch.load(std::memory_order_relaxed);
+    Totals totals;
+    for (const auto& counters : registry.counters) {
+        totals.bytes += counters.bytes.load(std::memory_order_relaxed);
+        const auto kept = counters.largest.load(std::memory_order_relaxed);
+        if ((kept >> AddressCopyEpochShift) == (epoch & AddressCopyEpochMask)) totals.largest = std::max(totals.largest, kept & AddressCopyLargestMask);
+    }
+    addressCopyEpoch.store(epoch + 1, std::memory_order_relaxed);
+    return totals;
+}
+
+std::uint64_t AddressCopyCounters::Limit() {
+    return addressCopyLimit();
+}
+
 void SetHostImportDeferredSettles(bool enabled) {
 #ifdef _WIN32
     deferredSettles.store(enabled, std::memory_order_relaxed);
@@ -3763,6 +3991,7 @@ const char* ImportRequesterName(ImportRequester requester) {
         case ImportRequester::Access: return "access";
         case ImportRequester::Join: return "join";
         case ImportRequester::Segment: return "segment";
+        case ImportRequester::ColorTarget: return "color target";
     }
     return "unknown";
 }
@@ -3823,8 +4052,15 @@ const HostImport* HostImportFor(const Context& context, std::uint64_t address, s
 #endif
     // A hit is only valid while the registry has not changed since the imports were reconciled.
     if (!importsStale(context, state)) {
+        if (recentImportsEnabled()) {
+            if (const auto* entry = recentImport(state, address, address + bytes)) {
+                entry->lastUse = ++state.useCounter;
+                return entry;
+            }
+        }
         if (const auto* entry = findImport(state, address, address + bytes)) {
             entry->lastUse = ++state.useCounter;
+            if (recentImportsEnabled()) rememberImport(state, entry);
             return entry;
         }
     }
@@ -3921,6 +4157,21 @@ bool HostImportCovers(const Context& context, std::uint64_t address, std::size_t
     return state.device == context.device && peekImport(state, address, address + bytes) != nullptr;
 }
 
+bool ComparedHostImportCovers(const Context& context, std::uint64_t address, std::size_t bytes) {
+#ifdef _WIN32
+    static_cast<void>(context);
+    static_cast<void>(address);
+    static_cast<void>(bytes);
+    return false;
+#else
+    if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return false;
+    auto& state = Imports();
+    std::lock_guard lock(state.mutex);
+    const auto* import = state.device == context.device ? peekImport(state, address, address + bytes) : nullptr;
+    return import != nullptr && import->unwatched;
+#endif
+}
+
 GuestBufferMemory::GuestBufferMemory(const Context& context) : context(context) {}
 
 bool GuestBufferMemory::WritesOverlap(std::uint64_t address, std::size_t bytes) const {
@@ -4192,6 +4443,11 @@ void GuestBufferMemory::AcquireRegistered() {
         }
         releaseImportedMirrors(context, imported);
         regionsSorted = sortedLookup;
+        if (!copies.empty()) {
+            std::uint64_t copied = 0;
+            for (const auto& copy : copies) copied += copy.committed;
+            AddressCopyCounters::ThisThread().Count(copied);
+        }
         if (const auto overflow = AddressCopyOverflow(std::move(copies), cap); !overflow.empty()) {
             std::fprintf(stderr, "FATAL: %s\n", overflow.c_str());
             std::fflush(stderr);

core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp

@@ -332,7 +332,9 @@ public:
         colorFill = MultisampledColorFill(use);
         fromTexture = false;
         dccRanges.clear();
+        dccCopies.clear();
         fmaskRange.reset();
+        fmaskCopy = {};
         metadataValid = std::make_shared<std::atomic<bool>>(true);
         const auto initial = initialDccClear(use);
         recordCommands(context, [&](VkCommandBuffer commands, Recorder* recorder) {
@@ -535,9 +537,35 @@ private:
     std::unique_ptr<FirstSampleCopy> firstSampleCopy;
     std::map<std::array<std::uint32_t, 4>, std::shared_ptr<Texture>> firstSampleTextures;
     std::vector<GuestMemory::UnchangedQuery> dccRanges;
+    struct DccCopy {
+        bool compared = false;
+        std::vector<std::byte> bytes;
+        std::uint64_t stamp = 0;
+        std::array<std::uint64_t, 2> checked{};
+    };
+    mutable std::vector<DccCopy> dccCopies;
+    mutable DccCopy fmaskCopy;
     std::optional<std::uint32_t> dccFill;
     std::optional<std::uint32_t> cmaskFill;
 
+    bool compared(std::size_t index) const {
+        return index < dccCopies.size() && dccCopies[index].compared;
+    }
+
+    static DccCopy comparedCopy(const GuestMemory::UnchangedQuery& range, std::vector<std::byte> bytes) {
+        return {true, std::move(bytes), GuestMemory::NewestStamp(range.address, range.bytes), {}};
+    }
+
+    static bool copyCurrent(const GuestMemory::UnchangedQuery& range, DccCopy& copy) {
+        if (GuestMemory::NewestStamp(range.address, range.bytes) != copy.stamp) return false;
+        const auto epoch = GuestMemory::CollectEpoch();
+        const std::array<std::uint64_t, 2> seen{epoch, GuestMemory::ForgetSerial()};
+        if (epoch != 0 && (seen[1] & 1u) == 0 && copy.checked == seen) return true;
+        if (!GuestMemory::EqualsCommittedUnsynced(range.address, copy.bytes)) return false;
+        copy.checked = seen;
+        return true;
+    }
+
     bool absorbDccFill(const ColorTarget& use, VkClearColorValue& clear) {
         const auto dcc = std::exchange(dccFill, std::nullopt);
         const auto cmaskPattern = std::exchange(cmaskFill, std::nullopt);
@@ -548,19 +576,30 @@ private:
         if (dccRanges.size() > 2) {
             const auto& range = dccRanges[2];
             auto* recorder = Recorder::Active();
-            const bool written = !GuestMemory::UnchangedSince(range.address, range.bytes, range.generation) || (recorder != nullptr && recorder->PendingWriteOverlaps(range.address, range.bytes));
+            const bool unchanged = compared(2) ? copyCurrent(range, dccCopies[2]) : GuestMemory::UnchangedSince(range.address, range.bytes, range.generation);
+            const bool written = !unchanged || (recorder != nullptr && recorder->PendingWriteOverlaps(range.address, range.bytes));
             const auto landed = LandedUniformMetadataByte(range.address, range.bytes);
             if (!landed || (written && (!cmaskPattern || *cmaskPattern != *landed * 0x01010101u))) return false;
             cmaskKeys = *landed;
         }
         std::vector<GuestMemory::UnchangedQuery> metadata(dccRanges.begin() + 1, dccRanges.end());
-        for (auto& range : metadata) {
-            if (!GuestMemory::Watched(range.address, range.bytes)) return false;
+        std::vector<bool> comparedMetadata(metadata.size(), false);
+        for (std::size_t index = 0; index < metadata.size(); ++index) {
+            auto& range = metadata[index];
+            if (!GuestMemory::Watched(range.address, range.bytes)) {
+                if (!compared(index + 1) && !ComparedHostImportCovers(context, range.address, range.bytes)) return false;
+                comparedMetadata[index] = true;
+                range.generation = 0;
+                continue;
+            }
             range.generation = collectBlocks(range.address, range.bytes);
             if (range.generation == 0) return false;
         }
-        clear = MultisampledDccClear(use, std::vector<std::byte>(dccRanges[1].bytes, std::byte{*keys}), std::vector<std::byte>(use.cmaskBytes, std::byte{cmaskKeys}));
+        std::array<std::vector<std::byte>, 2> landedBytes{std::vector<std::byte>(dccRanges[1].bytes, std::byte{*keys}), std::vector<std::byte>(use.cmaskBytes, std::byte{cmaskKeys})};
+        clear = MultisampledDccClear(use, landedBytes[0], landedBytes[1]);
         std::copy(metadata.begin(), metadata.end(), dccRanges.begin() + 1);
+        dccCopies.resize(dccRanges.size());
+        for (std::size_t index = 0; index < metadata.size(); ++index) dccCopies[index + 1] = comparedMetadata[index] ? comparedCopy(dccRanges[index + 1], std::move(landedBytes[index])) : DccCopy{};
         return true;
     }
 
@@ -574,11 +613,19 @@ private:
         dccRanges.push_back({use.address, static_cast<std::size_t>(colorBytes), 0});
         dccRanges.push_back({use.dccAddress, dccBytes, 0});
         if (use.cmaskAddress != 0) dccRanges.push_back({use.cmaskAddress, use.cmaskBytes, 0});
-        for (auto& range : dccRanges) {
+        dccCopies.assign(dccRanges.size(), {});
+        std::vector<GuestMemory::UnchangedQuery> watched;
+        for (std::size_t index = 0; index < dccRanges.size(); ++index) {
+            auto& range = dccRanges[index];
             GuestMemory::CheckRange(reinterpret_cast<const void*>(range.address), range.bytes, 256);
-            Require(WatchedAfterSettle(context, range.address, range.bytes), "initial multisampled DCC contents require watched guest memory");
+            if (!WatchedAfterSettle(context, range.address, range.bytes) && ComparedHostImportCovers(context, range.address, range.bytes)) {
+                dccCopies[index].compared = true;
+                continue;
+            }
+            Require(GuestMemory::Watched(range.address, range.bytes), "initial multisampled DCC contents require watched guest memory");
             range.generation = collectBlocks(range.address, range.bytes);
             Require(range.generation != 0, "initial multisampled DCC contents have no write generation");
+            watched.push_back(range);
         }
         for (std::size_t first = 0; first < dccRanges.size(); ++first) {
             for (std::size_t second = first + 1; second < dccRanges.size(); ++second) {
@@ -593,17 +640,28 @@ private:
             fmaskRange = GuestMemory::UnchangedQuery{use.fmaskAddress, static_cast<std::size_t>(DescribeSurface(resource).guestBytes), 0};
             auto& range = *fmaskRange;
             GuestMemory::CheckRange(reinterpret_cast<const void*>(range.address), range.bytes, 65536u);
-            Require(WatchedAfterSettle(context, range.address, range.bytes), "native FMASK contents require watched guest memory");
-            range.generation = GuestMemory::CollectWritesUncached(range.address, range.bytes);
-            Require(range.generation != 0, "native FMASK contents have no write generation");
+            fmaskCopy.compared = !WatchedAfterSettle(context, range.address, range.bytes) && ComparedHostImportCovers(context, range.address, range.bytes);
+            Require(fmaskCopy.compared || GuestMemory::Watched(range.address, range.bytes), "native FMASK contents require watched guest memory");
+            if (!fmaskCopy.compared) {
+                range.generation = GuestMemory::CollectWritesUncached(range.address, range.bytes);
+                Require(range.generation != 0, "native FMASK contents have no write generation");
+            }
             for (const auto& other : dccRanges) Require(range.address >= other.address + other.bytes || other.address >= range.address + range.bytes, "native FMASK overlaps its color or metadata planes");
         }
-        std::vector<std::byte> dcc(dccRanges[1].bytes);
-        GuestMemory::Read(use.dccAddress, dcc);
-        std::vector<std::byte> cmask(use.cmaskBytes);
-        if (!cmask.empty()) GuestMemory::Read(use.cmaskAddress, cmask);
-        const auto clear = MultisampledDccClear(use, dcc, cmask);
-        Require(GuestMemory::UnchangedSinceAll(dccRanges) && (!fmaskRange || GuestMemory::UnchangedSinceAll(std::span(&*fmaskRange, 1u))), "initial multisampled DCC contents changed during import");
+        std::array<std::vector<std::byte>, 3> bytes{};
+        for (std::size_t index = 0; index < dccRanges.size(); ++index) {
+            if (index == 0 && !compared(0)) continue;
+            bytes[index].resize(dccRanges[index].bytes);
+            GuestMemory::Read(dccRanges[index].address, bytes[index]);
+        }
+        std::vector<std::byte> fmaskBytes(fmaskCopy.compared ? fmaskRange->bytes : 0u);
+        if (fmaskCopy.compared) GuestMemory::Read(fmaskRange->address, fmaskBytes);
+        const auto clear = MultisampledDccClear(use, bytes[1], bytes[2]);
+        Require((watched.empty() || GuestMemory::UnchangedSinceAll(watched)) && (!fmaskRange || fmaskCopy.compared || GuestMemory::UnchangedSinceAll(std::span(&*fmaskRange, 1u))), "initial multisampled DCC contents changed during import");
+        for (std::size_t index = 0; index < dccRanges.size(); ++index) {
+            if (compared(index)) dccCopies[index] = comparedCopy(dccRanges[index], std::move(bytes[index]));
+        }
+        if (fmaskCopy.compared) fmaskCopy = comparedCopy(*fmaskRange, std::move(fmaskBytes));
         return clear;
     }
 
@@ -623,19 +681,28 @@ private:
         Require(metadataValid->load(std::memory_order_acquire), "multisampled DCC metadata was released while its color target remains resident");
         static_cast<void>(MultisampledDccBytes(use));
         Require(use.address == target.address && use.dccAddress == target.dccAddress && use.dccControl == target.dccControl && use.dccAlphaOnMsb == target.dccAlphaOnMsb && use.dccPipeAligned == target.dccPipeAligned && use.cmaskAddress == target.cmaskAddress && use.cmaskBytes == target.cmaskBytes && use.tileMode == target.tileMode && use.elementBytes == target.elementBytes && use.pipeBankXor == target.pipeBankXor, "multisampled DCC target layout or metadata changed");
-        const auto ranges = std::span(dccRanges).first(metadataRefilled ? 1u : dccRanges.size());
-        for (const auto& range : ranges) requireDccRange(GuestMemory::Watched(range.address, range.bytes), range, "multisampled DCC target reuse requires watched guest memory");
-        for (const auto& range : ranges) {
-            GuestMemory::CollectWritesUncached(range.address, range.bytes);
-            requireDccRange(GuestMemory::UnchangedSince(range.address, range.bytes, range.generation) || !GuestMemory::StoredOver(range.address, range.bytes, range.generation), range, "writes to a resident multisampled DCC target or its metadata are not implemented");
+        const auto ranges = metadataRefilled ? std::size_t{1} : dccRanges.size();
+        for (std::size_t index = 0; index < ranges; ++index) requireDccRange(compared(index) || GuestMemory::Watched(dccRanges[index].address, dccRanges[index].bytes), dccRanges[index], "multisampled DCC target reuse requires watched guest memory");
+        for (std::size_t index = 0; index < ranges; ++index) {
+            const auto& range = dccRanges[index];
+            if (compared(index)) {
+                requireDccRange(copyCurrent(range, dccCopies[index]), range, "writes to a resident multisampled DCC target or its metadata are not implemented");
+            } else {
+                GuestMemory::CollectWritesUncached(range.address, range.bytes);
+                requireDccRange(GuestMemory::UnchangedSince(range.address, range.bytes, range.generation) || !GuestMemory::StoredOver(range.address, range.bytes, range.generation), range, "writes to a resident multisampled DCC target or its metadata are not implemented");
+            }
             if (auto* recorder = Recorder::Active(); recorder != nullptr) requireDccRange(!recorder->PendingWriteOverlaps(range.address, range.bytes), range, "pending writes to a resident multisampled DCC target or its metadata are not implemented");
         }
         if (fmaskRange) {
             Require(use.fmaskAddress == target.fmaskAddress && use.fmaskPipeBankXor == target.fmaskPipeBankXor, "native FMASK target layout changed");
             const auto& range = *fmaskRange;
-            requireDccRange(WatchedAfterSettle(context, range.address, range.bytes), range, "native FMASK sampling requires watched guest memory");
-            GuestMemory::CollectWritesUncached(range.address, range.bytes);
-            requireDccRange(GuestMemory::UnchangedSince(range.address, range.bytes, range.generation) || !GuestMemory::StoredOver(range.address, range.bytes, range.generation), range, "writes to a resident native FMASK are not implemented");
+            requireDccRange(fmaskCopy.compared || WatchedAfterSettle(context, range.address, range.bytes), range, "native FMASK sampling requires watched guest memory");
+            if (fmaskCopy.compared) {
+                requireDccRange(copyCurrent(range, fmaskCopy), range, "writes to a resident native FMASK are not implemented");
+            } else {
+                GuestMemory::CollectWritesUncached(range.address, range.bytes);
+                requireDccRange(GuestMemory::UnchangedSince(range.address, range.bytes, range.generation) || !GuestMemory::StoredOver(range.address, range.bytes, range.generation), range, "writes to a resident native FMASK are not implemented");
+            }
             if (auto* recorder = Recorder::Active(); recorder != nullptr) requireDccRange(!recorder->PendingWriteOverlaps(range.address, range.bytes), range, "pending writes to a resident native FMASK are not implemented");
         }
     }

core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp

@@ -185,7 +185,8 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
     std::vector<VkPipeline> libraries;
     for (std::size_t part = 0; part < flags.size(); ++part) {
         if (part == 0 && mesh) continue;
-        const auto key = Combined(*partKeys[part], *contextKeys[part]);
+        auto key = Combined(*partKeys[part], *contextKeys[part]);
+        key.push_back(static_cast<std::byte>(optimize));
         auto& cache = device.parts[part];
         if (const auto found = cache.find(key); found != cache.end()) {
             libraries.push_back(found->second);
@@ -195,7 +196,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
         VkGraphicsPipelineLibraryCreateInfoEXT library{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT, flags[part] == VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT ? &rendering : &shaderRendering};
         library.flags = flags[part];
         VkGraphicsPipelineCreateInfo create{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, &library};
-        create.flags = VK_PIPELINE_CREATE_LIBRARY_BIT_KHR | VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT;
+        create.flags = optimize ? VK_PIPELINE_CREATE_LIBRARY_BIT_KHR | VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT : VK_PIPELINE_CREATE_LIBRARY_BIT_KHR;
         create.pDynamicState = info.pDynamicState;
         create.layout = partLayouts[part];
         switch (flags[part]) {

core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp

@@ -8,6 +8,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
 #include "prx/libSceAgcDriver/Graphics/shaders/MeshArguments_spv.h"
 #include "prx/libSceAgcDriver/Graphics/shaders/SampleCounter_spv.h"
 #include "prx/libSceAgcDriver/Graphics/shaders/SampleDumps_spv.h"
@@ -3679,6 +3680,7 @@ void Recorder::Submit() {
         submission.pSignalSemaphores = &timeline;
     }
     if (hold != VK_NULL_HANDLE || timeline != VK_NULL_HANDLE) submission.pNext = &timelineInfo;
+    DestroyDeferredAtIdle(context);
     const auto submitStart = std::chrono::steady_clock::now();
     const auto result = function(queueSubmit, "vkQueueSubmit")(context.queue, 1, &submission, batch->fence);
     if (result == VK_ERROR_DEVICE_LOST) {

core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
@@ -381,6 +382,7 @@ void CommandBatch::Submit() {
     submission.commandBufferCount = 1;
     submission.pCommandBuffers = &commands;
     timing.Mark("command_end");
+    DestroyDeferredAtIdle(context);
     Check(context.Function<PFN_vkQueueSubmit>("vkQueueSubmit")(context.queue, 1, &submission, fence), "vkQueueSubmit graphics");
     pending = true;
     submitted = true;

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp

@@ -66,7 +66,7 @@ VkComponentSwizzle ComponentSwizzleFor(std::uint8_t dstSel) {
 }
 
 VkComponentMapping ViewComponents(const GuestTextureResource& resource) {
-    if (IsConvertedTextureFormat(resource.format) && !IsRepackedTextureFormat(resource.format)) return {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_A};
+    if (IsConvertedTextureFormat(resource.format) && !IsRepackedTextureFormat(resource.format) && resource.format != static_cast<std::uint32_t>(ShaderRecompiler::IrBufferFormat::Format11_11_10Float)) return {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_A};
     const auto channel = [&](std::uint8_t dstSel) { return TextureComponentChannel(resource.format, ComponentSwizzleFor(dstSel)); };
     return {channel(resource.dstSelX), channel(resource.dstSelY), channel(resource.dstSelZ), channel(resource.dstSelW)};
 }
@@ -803,7 +803,7 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
         keys = scanKeys();
         if (*keys == source->FilledKeys()) {
             std::array<std::byte, 16> probe{};
-            if (!FillDccClear(ResolveTextureFormat(resource.format), *keys, resource.dccAlphaOnMsb, probe)) {
+            if (!FillDccClear(DccClearTextureFormat(resource.format), *keys, resource.dccAlphaOnMsb, probe)) {
                 char text[256];
                 std::snprintf(text, sizeof(text), "AGC graphics: sampled texture 0x%llx (%ux%u format %u, dcc 0x%llx) reads %s DCC keys filled over its pending image, a clear value the format has no encoding for", static_cast<unsigned long long>(resource.baseAddress), resource.width, resource.height, resource.format, static_cast<unsigned long long>(resource.dccAddress), DccKeysName(*keys));
                 throw std::runtime_error(text);
@@ -1189,9 +1189,9 @@ std::shared_ptr<StorageTexture> cachedStorageTexture(const Context& context, std
     const auto viewed = ArrayLayerWindow(requested, context.limits.maxImageArrayLayers);
     if (viewed.baseAddress != requested.baseAddress) guestBytes = 0;
     Require(viewed.dimension != TextureDimension::k2DMsaa, "storage image access to a multisampled texture is not implemented");
-    RequireNoMultisampleTarget(viewed.baseAddress, guestBytes != 0 ? static_cast<std::size_t>(guestBytes) : 1u, "a storage image access");
-    if (auto depth = DepthSurfaceStorage(context, words, viewed, mip)) return depth;
     const auto viewedBytes = guestBytes != 0 ? guestBytes : DescribeSurface(viewed).guestBytes;
+    RequireNoMultisampleTarget(viewed.baseAddress, static_cast<std::size_t>(viewedBytes), "a storage image access");
+    if (auto depth = DepthSurfaceStorage(context, words, viewed, mip)) return depth;
     if (viewedBytes >= (1ull << 30u)) {
         static std::atomic<std::uint64_t> large{0};
         const auto count = large.fetch_add(1) + 1;
@@ -2084,9 +2084,10 @@ void ShaderResources::buildPrepare(std::span<const CompiledShader> shaders, cons
                         const bool classified = element < binding.bufferWritten.size() && element < binding.bufferRead.size() && element < binding.bufferAtomic.size();
                         allocations[index].writeOnly = classified && allocations[index].guest && allocations[index].written && binding.bufferWritten[element] && !read && !atomic;
                         const auto& push = shader.program->pushConstants;
-                        if (!push.empty()) {
+                        const auto pushDataBytes = shader.program->stencilReferences.empty() ? push.size() : shader.program->stencilReferenceDword * sizeof(std::uint32_t);
+                        if (pushDataBytes != 0u) {
                             const auto position = shader.program->memoryOffsetDword * 4u + element;
-                            Require(position < push.size(), "guest buffer offset lies outside the shader's push constants");
+                            Require(position < pushDataBytes, "guest buffer offset lies outside the shader's push constants");
                             allocations[index].pushByte = static_cast<std::int32_t>(shader.pushConstantOffset + position);
                         } else {
                             allocations[index].dataByte = shader.program->memoryOffsetDword * 4u + element;
@@ -5025,7 +5026,7 @@ ShaderResources::TableStage& ShaderResources::tableStage(const CompiledShader& s
     table.stage = shader.stage;
     table.program = shader.program;
     table.poison = shader.program->imageTablePoison;
-    table.messages.assign(table.poison.size(), std::string{});
+    table.messages.assign(table.poison.size(), nullptr);
     return table;
 }
 
@@ -5036,7 +5037,8 @@ std::uint32_t ShaderResources::rejectTableElement(TableStage& table, std::span<c
     poison.resource = ShaderRecompiler::NoTable;
     poison.reason = static_cast<std::uint32_t>(ShaderRecompiler::ImageTableAbi::PoisonReason::DriverRejected);
     table.poison.push_back(poison);
-    table.messages.push_back(message);
+    const bool repeated = !table.messages.empty() && table.messages.back() != nullptr && *table.messages.back() == message;
+    table.messages.push_back(repeated ? table.messages.back() : std::make_shared<const std::string>(message));
     faultChecks = true;
     return ShaderRecompiler::ImageTableAbi::PoisonCode(static_cast<std::uint32_t>(table.poison.size() - 1u));
 }
@@ -5331,7 +5333,7 @@ void ShaderResources::finishImageTables() {
             poison.resource = index;
             poison.reason = static_cast<std::uint32_t>(Abi::PoisonReason::HeapWritten);
             table.poison.push_back(poison);
-            table.messages.push_back("the table at 0x" + HexText(base) + " is written by its own dispatch");
+            table.messages.push_back(std::make_shared<const std::string>("the table at 0x" + HexText(base) + " is written by its own dispatch"));
             map[header + Abi::TableFault] = Abi::PoisonCode(static_cast<std::uint32_t>(table.poison.size() - 1u));
             faultChecks = true;
         }

core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp

@@ -6,6 +6,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libc/include/General.hpp"
 #include <algorithm>
@@ -314,13 +315,33 @@ VkStencilOpState stencilFace(std::uint32_t compare, std::uint32_t ops, std::uint
     auto fixed = compare == 0 || compare == 7 ? 0u : face.compareMask;
     std::uint32_t invertMask = 0;
     std::uint32_t inverted = 0;
+    std::uint32_t logicMask = 0;
+    const auto logic = [](std::uint32_t op) { return op == 10 || op == 11 || op == 12 || op == 15; };
     const auto convert = [&](std::uint32_t op, bool testFailed) {
         if (face.writeMask == 0) return VK_STENCIL_OP_KEEP;
         char detail[112];
-        if (opValExport && ((op == 4 && !opValReference) || op == 5 || op == 6 || op == 8 || op == 9)) {
+        if (opValExport && ((op == 4 && !opValReference) || (op >= 5 && op != 7))) {
             std::snprintf(detail, sizeof(detail), "AGC graphics: stencil operation %u with a pixel shader that exports the stencil operation value is unsupported", op);
             throw std::runtime_error(detail);
         }
+        if (logic(op)) {
+            const auto bits = (op == 11 || op == 12 ? opValue : ~opValue) & face.writeMask;
+            if (bits == 0) return VK_STENCIL_OP_KEEP;
+            if (logicMask != 0 && logicMask != bits) {
+                std::snprintf(detail, sizeof(detail), "AGC graphics: stencil logic operations on write masks 0x%02x and 0x%02x are unsupported", logicMask, bits);
+                throw std::runtime_error(detail);
+            }
+            logicMask = bits;
+            if (op != 11) return op == 10 ? VK_STENCIL_OP_ZERO : VK_STENCIL_OP_INVERT;
+            if (testValExport) throw std::runtime_error("AGC graphics: stencil OR is unsupported with a pixel shader that exports the stencil test value");
+            if ((~reference & fixed & bits) != 0) {
+                std::snprintf(detail, sizeof(detail), "AGC graphics: stencil OR of 0x%02x against test value 0x%02x (masks 0x%02x/0x%02x) is unsupported", opValue, test, face.compareMask, face.writeMask);
+                throw std::runtime_error(detail);
+            }
+            reference |= bits;
+            fixed |= bits;
+            return VK_STENCIL_OP_REPLACE;
+        }
         switch (op) {
             case 0: return VK_STENCIL_OP_KEEP;
             case 1: return VK_STENCIL_OP_ZERO;
@@ -377,6 +398,15 @@ VkStencilOpState stencilFace(std::uint32_t compare, std::uint32_t ops, std::uint
         if (inverted != active) throw std::runtime_error("AGC graphics: a stencil replacement drawn as an inversion beside another stencil operation is unsupported");
         face.writeMask = invertMask;
     }
+    if (logicMask != 0) {
+        const auto other = [&](std::uint32_t op) { return op != 0 && !logic(op); };
+        if ((other(ops & 0xfu) || other((ops >> 4u) & 0xfu) || other((ops >> 8u) & 0xfu)) && logicMask != face.writeMask) {
+            char detail[128];
+            std::snprintf(detail, sizeof(detail), "AGC graphics: stencil operations 0x%03x combine a logic operation on write mask 0x%02x with write mask 0x%02x", ops & 0xfffu, logicMask, face.writeMask);
+            throw std::runtime_error(detail);
+        }
+        face.writeMask = logicMask;
+    }
     face.reference = reference;
     Require(!opValReference || compare == 0 || compare == 7 || face.compareMask == 0, "a stencil compare against the test value is unsupported with a pixel shader that exports the stencil operation value");
     return face;
@@ -953,12 +983,17 @@ struct DecodedColorFormat {
 
 const char* PackedColorTargetName(ShaderRecompiler::ColorExportPacking packing) {
     if (packing == ShaderRecompiler::ColorExportPacking::Unorm4_4_4_4) return "4_4_4_4 unorm";
+    if (packing == ShaderRecompiler::ColorExportPacking::Uint) return "unsigned integer";
     return packing == ShaderRecompiler::ColorExportPacking::Float11_11_10 ? "11_11_10 float" : "10_11_11 unorm";
 }
 
+bool WordPacked(ShaderRecompiler::ColorExportPacking packing) {
+    return packing == ShaderRecompiler::ColorExportPacking::Unorm10_11_11 || packing == ShaderRecompiler::ColorExportPacking::Float11_11_10 || packing == ShaderRecompiler::ColorExportPacking::Unorm4_4_4_4;
+}
+
 // CB_COLOR_INFO FORMAT / NUMBER_TYPE / COMP_SWAP to a Vulkan attachment format. AMD formats list
 // components from the least significant bits, as Vulkan's non-packed formats do.
-DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number, std::uint32_t swap) {
+DecodedColorFormat DecodeColorFormatRaw(std::uint32_t format, std::uint32_t number, std::uint32_t swap) {
     constexpr std::uint32_t unorm = 0, snorm = 1, sint = 5, srgb = 6, floating = 7;
     constexpr std::uint32_t uint = 4;
     const auto fail = [&]() -> DecodedColorFormat {
@@ -1326,6 +1361,27 @@ ReversedBlend TranslateReversedBlend(std::uint32_t blendControl, std::uint8_t co
     return {blend, routing->code};
 }
 
+void ApplyDepthBoundsEmulation(const Context& context, State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel) {
+    if (!state.depthBoundsTest || !state.depth || context.depthBounds) return;
+    ValidateDepthBounds(context, state);
+    Require(!context.depthRangeUnrestricted, "the depth bounds test without the depthBounds feature is emulated only where stored depth stays in [0, 1]");
+    Require(state.depth->format == VK_FORMAT_D32_SFLOAT || state.depth->format == VK_FORMAT_D32_SFLOAT_S8_UINT, "the depth bounds test on a normalized depth target needs the depthBounds feature");
+    Require(!state.depthTest && !state.depthWrite, "the depth bounds test with depth testing needs the depthBounds feature");
+    Require(!pixel.posZ && !pixel.depthExportEnable, "the depth bounds test from a pixel program that reads or exports depth needs the depthBounds feature");
+    const auto keepsOnFail = [](const VkStencilOpState& face) { return (face.writeMask & 0xffu) == 0 || (face.failOp == VK_STENCIL_OP_KEEP && face.depthFailOp == VK_STENCIL_OP_KEEP); };
+    Require(!state.stencilTest || (keepsOnFail(state.stencilFront) && keepsOnFail(state.stencilBack)), "the depth bounds test with a stencil test that writes on a failed test needs the depthBounds feature");
+    const bool upper = state.minDepthBounds <= 0.0f;
+    Require(upper || state.maxDepthBounds >= 1.0f, "the depth bounds test with both bounds inside (0, 1) needs the depthBounds feature");
+    const float bound = upper ? state.maxDepthBounds : state.minDepthBounds;
+    state.depthBoundsTest = false;
+    state.depthTest = true;
+    state.depthCompare = upper ? VK_COMPARE_OP_GREATER_OR_EQUAL : VK_COMPARE_OP_LESS_OR_EQUAL;
+    state.depthBias = false;
+    state.depthBiasPerFace = false;
+    state.viewport.minDepth = bound;
+    state.viewport.maxDepth = bound;
+}
+
 void ApplyDepthRemap(const Context& context, State& state) {
     const auto& viewport = state.viewport;
     const bool inRange = viewport.minDepth >= 0 && viewport.minDepth <= 1 && viewport.maxDepth >= 0 && viewport.maxDepth <= 1;
@@ -1533,6 +1589,7 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
         if (slotExport == 7 && ((read(cx, 0x31c + slot * 0xfu) >> 8u) & 7u) != 4u) throw std::runtime_error("AGC graphics: color export format 7 (UINT16_ABGR) into a target that is not unsigned integer is unsupported");
         color.exportIndex = index;
         color.uintExport = slotExport == 7;
+        if (color.uintExport && color.packing == ShaderRecompiler::ColorExportPacking::Uint) color.packing = ShaderRecompiler::ColorExportPacking::None;
         if (color.samples == 1 && result.samples > 1 && result.colors.empty() && !result.depth && (read(cx, 0x292) & 1u) == 0) {
             const auto sampleInputs = read(cx, 0x1b3) & read(cx, 0x1b4) & (ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::Ancillary) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::SampleCoverage));
             Require(sampleInputs == 0 && (read(cx, 0x203) & 0x100u) == 0, "a one-sample color target under a multisampled rasterizer without MSAA_ENABLE is unsupported with a sample index or coverage input, or a sample-mask export");
@@ -1690,7 +1747,8 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
                 if ((weighted(state.colorBlendOp) && (constantColor(state.srcColorBlendFactor) || constantColor(state.dstColorBlendFactor))) || (weighted(state.alphaBlendOp) && (constantColor(state.srcAlphaBlendFactor) || constantColor(state.dstAlphaBlendFactor)))) throw std::runtime_error("AGC graphics: blending into an 8_8 color target with the alternate component order with a constant color factor, which Vulkan applies to the second channel with the green blend constant instead of the alpha one, is not implemented");
             }
         }
-        if (color.packing != ShaderRecompiler::ColorExportPacking::None) {
+        if (color.packing == ShaderRecompiler::ColorExportPacking::Uint) state.blendEnable = VK_FALSE;
+        if (WordPacked(color.packing)) {
             const std::string packed = std::string(color.packing == ShaderRecompiler::ColorExportPacking::Float11_11_10 ? "an " : "a ") + PackedColorTargetName(color.packing);
             const auto slotExport = (exportFormat >> (4u * color.exportIndex)) & 0xfu;
             if (slotExport == 5 || slotExport == 6) throw std::runtime_error("AGC graphics: color export format " + std::to_string(slotExport) + " into " + packed + " target is unsupported");
@@ -1715,6 +1773,10 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
                 }
             }
         }
+        if (color.packing == ShaderRecompiler::ColorExportPacking::Uint) {
+            const auto slotExport = (exportFormat >> (4u * color.exportIndex)) & 0xfu;
+            if (slotExport != 1 && slotExport != 2 && slotExport != 3 && slotExport != 9) throw std::runtime_error("AGC graphics: color export format " + std::to_string(slotExport) + " into an unsigned integer target is unsupported");
+        }
         if (state.blendEnable && !reversed && (secondSource(state.srcColorBlendFactor) || secondSource(state.dstColorBlendFactor) || secondSource(state.srcAlphaBlendFactor) || secondSource(state.dstAlphaBlendFactor))) {
             const auto addition = [](VkBlendOp op) { return op == VK_BLEND_OP_ADD || op == VK_BLEND_OP_SUBTRACT || op == VK_BLEND_OP_REVERSE_SUBTRACT; };
             Require(addition(state.colorBlendOp) && addition(state.alphaBlendOp), "dual-source blending with a MIN or MAX operation is unsupported");
@@ -1917,6 +1979,12 @@ GuestTextureResource SurfaceForTarget(const ColorTarget& color) {
     surface.dccPipeAligned = color.dccPipeAligned;
     return surface;
 }
+DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number, std::uint32_t swap) {
+    auto decoded = DecodeColorFormatRaw(format, number, swap);
+    if (number == 4u && decoded.packing == ShaderRecompiler::ColorExportPacking::None) decoded.packing = ShaderRecompiler::ColorExportPacking::Uint;
+    return decoded;
+}
+
 
 ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot, bool allowMultisampleDcc) {
     const auto stride = slot * 0xfu;
@@ -2025,7 +2093,7 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot, bool allo
     color.format = decoded.format;
     color.componentMapping = decoded.componentMapping;
     color.packing = decoded.packing;
-    if (color.packing != ShaderRecompiler::ColorExportPacking::None && (info & 0x10000000u) != 0) throw std::runtime_error("AGC graphics: DCC-compressed " + std::string(PackedColorTargetName(color.packing)) + " color targets are unsupported");
+    if (WordPacked(color.packing) && (info & 0x10000000u) != 0) throw std::runtime_error("AGC graphics: DCC-compressed " + std::string(PackedColorTargetName(color.packing)) + " color targets are unsupported");
     Require(color.packing != ShaderRecompiler::ColorExportPacking::Unorm10_11_11 || (info & 0x00040000u) == 0, "truncating (ROUND_MODE) 10_11_11 unorm color targets are unsupported");
     Require(color.packing != ShaderRecompiler::ColorExportPacking::Unorm4_4_4_4 || (info & 0x00040000u) == 0, "truncating (ROUND_MODE) 4_4_4_4 unorm color targets are unsupported");
     Require(color.packing != ShaderRecompiler::ColorExportPacking::Float11_11_10 || (info & 0x00040000u) != 0, "rounding (ROUND_MODE 0) 11_11_10 float color targets are unsupported");
@@ -2197,7 +2265,7 @@ std::optional<ColorMetadataPass> DecodeColorMetadataPass(const QueueState& queue
         Require(destination.samples == 1, "CB resolve into a multisampled target is unsupported");
         Require(destination.cmaskAddress == 0, "CB resolve into a fast-clear color target is unsupported (the resolve does not update its CMASK)");
         Require(source.format == destination.format && source.extent.width == destination.extent.width && source.extent.height == destination.extent.height, "CB resolve between different formats or extents is unsupported");
-        if (source.packing != ShaderRecompiler::ColorExportPacking::None) throw std::runtime_error("AGC graphics: CB resolves of " + std::string(PackedColorTargetName(source.packing)) + " color targets are unsupported");
+        if (WordPacked(source.packing)) throw std::runtime_error("AGC graphics: CB resolves of " + std::string(PackedColorTargetName(source.packing)) + " color targets are unsupported");
         requireCovered(destination);
         pass.source = source;
         pass.targets.push_back(destination);

core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp

@@ -1087,7 +1087,7 @@ Texture::Texture(const Context& context, const std::shared_ptr<StorageTexture>&
     }
 }
 
-Texture::Texture(const Context& context, VkImage depthImage, VkFormat depthFormat, VkImageAspectFlags aspect, VkComponentMapping components, VkImageViewType viewType, std::uint32_t baseLayer, std::shared_ptr<OwnedImage> owner, VkImageUsageFlags usage) : context(context), owned(std::move(owner)) {
+Texture::Texture(const Context& context, VkImage depthImage, VkFormat depthFormat, VkImageAspectFlags aspect, VkComponentMapping components, VkImageViewType viewType, std::uint32_t baseLayer, std::shared_ptr<OwnedImage> owner, VkImageUsageFlags usage, std::uint32_t layerCount) : context(context), owned(std::move(owner)) {
     Require(owned == nullptr || (owned->image == depthImage && owned->context.device == context.device), "a depth sampled view must retain its own device and image");
     layout = VK_IMAGE_LAYOUT_GENERAL;
     VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
@@ -1099,7 +1099,7 @@ Texture::Texture(const Context& context, VkImage depthImage, VkFormat depthForma
     viewInfo.format = depthFormat;
     viewFormat = depthFormat;
     viewInfo.components = components;
-    viewInfo.subresourceRange = {aspect, 0, 1, baseLayer, viewType == VK_IMAGE_VIEW_TYPE_2D ? 1u : VK_REMAINING_ARRAY_LAYERS};
+    viewInfo.subresourceRange = {aspect, 0, 1, baseLayer, viewType == VK_IMAGE_VIEW_TYPE_2D ? 1u : layerCount};
     Check(context.Function<PFN_vkCreateImageView>("vkCreateImageView")(context.device, &viewInfo, nullptr, &view), "vkCreateImageView depth plane");
     viewRange = {viewInfo.viewType, viewInfo.subresourceRange.levelCount, viewInfo.subresourceRange.layerCount};
     if (viewType != VK_IMAGE_VIEW_TYPE_2D_ARRAY) return;

core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp

@@ -99,7 +99,7 @@ constexpr auto MakeFormatLookupTable() {
 constexpr auto kFormatLookupTable = MakeFormatLookupTable();
 
 std::uint32_t remapGuestFormat(std::uint32_t guestFormat) {
-    return guestFormat == 30 || guestFormat == 34 || guestFormat == 43 ? 20 : guestFormat;
+    return guestFormat == 30 || guestFormat == 34 || guestFormat == 36 || guestFormat == 43 ? 20 : guestFormat;
 }
 
 const FormatEntry& findFormatEntry(std::uint32_t guestFormat) {
@@ -115,6 +115,11 @@ VkFormat ResolveTextureFormat(std::uint32_t guestFormat) {
     return findFormatEntry(guestFormat).vkFormat;
 }
 
+VkFormat DccClearTextureFormat(std::uint32_t guestFormat) {
+    if (IsRepackedTextureFormat(guestFormat) || guestFormat == static_cast<std::uint32_t>(ShaderRecompiler::IrBufferFormat::Format11_11_10Float)) return VK_FORMAT_B10G11R11_UFLOAT_PACK32;
+    return ResolveTextureFormat(guestFormat);
+}
+
 std::uint32_t SrgbDecodeFormats(PFN_vkGetPhysicalDeviceFormatProperties formatProperties, VkPhysicalDevice physical) {
     const char* forced = std::getenv("APS5_SRGB_SHADER_DECODE");
     const bool always = forced != nullptr && (std::strcmp(forced, "1") == 0 || std::strcmp(forced, "copy") == 0);
@@ -131,7 +136,7 @@ std::uint32_t SrgbDecodeFormats(PFN_vkGetPhysicalDeviceFormatProperties formatPr
 }
 
 VkFormat SampledTextureFormat(const Context& context, std::uint32_t guestFormat) {
-    if (IsRepackedTextureFormat(guestFormat)) return VK_FORMAT_B10G11R11_UFLOAT_PACK32;
+    if (IsRepackedTextureFormat(guestFormat) || guestFormat == static_cast<std::uint32_t>(ShaderRecompiler::IrBufferFormat::Format11_11_10Float)) return VK_FORMAT_B10G11R11_UFLOAT_PACK32;
     const auto format = static_cast<ShaderRecompiler::IrBufferFormat>(guestFormat);
     if ((context.srgbDecodeFormats & ShaderRecompiler::SrgbDecodeBit(format)) == 0u) return ResolveTextureFormat(guestFormat);
     return ResolveTextureFormat(static_cast<std::uint32_t>(ShaderRecompiler::SrgbUnormFormat(format)));

core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp

@@ -25,6 +25,7 @@
 #include <mutex>
 #include <new>
 #include <string>
+#include <utility>
 #include <vector>
 
 #ifndef _WIN32
@@ -851,7 +852,7 @@ private:
             const auto* imported = reinterpret_cast<const VkImportMemoryHostPointerInfoEXT*>(next);
             active->calls.push_back({imported->pHostPointer, info->allocationSize});
             if (active->onImport) {
-                auto callback = std::move(active->onImport);
+                const auto callback = std::exchange(active->onImport, nullptr);
                 callback();
             }
             if (active->failure != VK_SUCCESS) return active->failure;
@@ -1130,7 +1131,7 @@ void importAtMapOverBudgetTests(const Context& context) {
 void importRequesterTests(const Context& context, const BdaTestAccess& access) {
     ClearHostImports(context.device);
     constexpr std::size_t piece = 65536;
-    constexpr std::size_t pieces = 8;
+    constexpr std::size_t pieces = 11;
     auto* block = static_cast<std::uint8_t*>(::operator new(pieces * piece, std::align_val_t{piece}));
     std::memset(block, 0x2d, pieces * piece);
     auto importing = context;
@@ -1144,7 +1145,7 @@ void importRequesterTests(const Context& context, const BdaTestAccess& access) {
     };
     const auto counted = [](ImportRequester requester) { return HostImportCounters().requesters[static_cast<std::size_t>(requester)]; };
     const auto made = [&](ImportRequester requester) { return counted(requester).made; };
-    Require(std::string(ImportRequesterName(ImportRequester::Map)) == "map" && std::string(ImportRequesterName(ImportRequester::Build)) == "build" && std::string(ImportRequesterName(ImportRequester::Access)) == "access" && std::string(ImportRequesterName(ImportRequester::Join)) == "join" && std::string(ImportRequesterName(ImportRequester::Segment)) == "segment", "an import requester has the wrong name");
+    Require(std::string(ImportRequesterName(ImportRequester::Map)) == "map" && std::string(ImportRequesterName(ImportRequester::Build)) == "build" && std::string(ImportRequesterName(ImportRequester::Access)) == "access" && std::string(ImportRequesterName(ImportRequester::Join)) == "join" && std::string(ImportRequesterName(ImportRequester::Segment)) == "segment" && std::string(ImportRequesterName(ImportRequester::ColorTarget)) == "color target", "an import requester has the wrong name");
     const auto mappedRange = [&](std::size_t index) {
         GuestAllocations::Mapped ranges;
         const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
@@ -1245,9 +1246,27 @@ void importRequesterTests(const Context& context, const BdaTestAccess& access) {
         SetHostImportWindow(0);
         ::operator delete(joined, std::align_val_t{window});
     }
+    change(10, true);
+    change(9, true);
+    {
+        const auto decided = PrepareImportWatch(importing);
+        const auto before = counted(ImportRequester::ColorTarget);
+        SetImportWatch(importing, ImportWatch::Unwatch);
+        const auto* compared = HostImportFor(importing, address(10), 16, ImportRequester::ColorTarget);
+        const auto refused = counted(ImportRequester::ColorTarget);
+        SetImportWatch(importing, ImportWatch::Watch);
+        const auto* readOnly = HostImportFor(importing, address(9), 16, ImportRequester::ColorTarget);
+        const auto* outside = HostImportFor(importing, address(10), 16, ImportRequester::ColorTarget);
+        SetImportWatch(importing, decided);
+        Require(compared == nullptr && refused.made == before.made && refused.refused == before.refused && refused.failed == before.failed, "a color target's import was made where imports of its memory are compared");
+        Require(readOnly == nullptr && !HostImportCovers(importing, address(9), piece), "a color target's import was made of registered memory the guest cannot write");
+        Require(outside == nullptr && !HostImportCovers(importing, address(10), piece) && made(ImportRequester::ColorTarget) == before.made, "a color target's import was made of memory outside the write watch");
+    }
     const auto counts = HostImportCounters();
     Require(counts.live >= 4 && counts.liveBytes == HostImportLiveBytes(), "the live import counts disagree with the live bytes");
     for (std::size_t index = 0; index < 6; ++index) change(index, false);
+    change(10, false);
+    change(9, false);
     {
         GuestAllocations::Mutation mutation;
         mutation.Remove(block + 6 * piece);
@@ -1263,8 +1282,9 @@ void hostImportReportTests() {
     counts.requesters[static_cast<std::size_t>(ImportRequester::Map)] = {2, 1u << 20u, 0, 0, 0};
     counts.requesters[static_cast<std::size_t>(ImportRequester::Build)] = {5, 4u << 20u, 7, 7u << 19u, 1, 4};
     counts.requesters[static_cast<std::size_t>(ImportRequester::Segment)] = {1, 1u << 16u, 0, 0, 2};
+    counts.requesters[static_cast<std::size_t>(ImportRequester::ColorTarget)] = {1, 1u << 20u, 0, 0, 0};
     const auto text = FormatHostImportCounts(counts);
-    Require(text == "3 live (3.0 MiB); made 8 (5.1 MiB), refused 7 (3.5 MiB, 4 remembered), failed 3; by requester (made/refused/failed): map 2/0/0, build 5/7/1, access 0/0/0, join 0/0/0, segment 1/0/2", "the host import report is wrong: " + text);
+    Require(text == "3 live (3.0 MiB); made 9 (6.1 MiB), refused 7 (3.5 MiB, 4 remembered), failed 3; by requester (made/refused/failed): map 2/0/0, build 5/7/1, access 0/0/0, join 0/0/0, segment 1/0/2, color target 1/0/0", "the host import report is wrong: " + text);
 }
 
 class PieceWindow {
@@ -2360,6 +2380,64 @@ void importedHeapMirrorTests(const Context& context, const BdaTestAccess& access
     GuestArena::GuestArenaRelease_nid_postfix(block, bytes);
 }
 
+void retiredImportWatchTests(const Context& context) {
+#ifdef _WIN32
+    if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) {
+        std::cout << "guest arena unavailable or not write-watched: the write watch of retired imports not tested\n";
+        return;
+    }
+    constexpr std::size_t piece = 65536;
+    constexpr std::size_t bytes = 2 * piece;
+    void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, bytes);
+    GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes);
+    std::memset(block, 0x5a, bytes);
+    auto* guest = static_cast<std::uint8_t*>(block);
+    const auto address = reinterpret_cast<std::uintptr_t>(block);
+    auto importing = context;
+    importing.hostImportAlignment = piece;
+    SetHostImportWindow(bytes);
+    SetHostImportDeferredSettles(false);
+    ClearHostImports(importing.device);
+    const auto decided = PrepareImportWatch(importing);
+    SetImportWatch(importing, ImportWatch::Unwatch);
+    const auto registry = [&](std::size_t offset, bool add) {
+        auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
+        if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, guest + offset, piece, true, true, true);
+        else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, guest + offset);
+        GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
+    };
+    const auto build = [&] {
+        GuestBufferMemory leased(importing);
+        leased.AcquireRegistered();
+        leased.Upload(true);
+        leased.WriteBack();
+    };
+    registry(0, true);
+    build();
+    Require(HostImportCovers(importing, address, piece), "the first 64 KiB piece was not imported on its own");
+    Require(!AgcDriver::GuestMemory::Watched(address, piece), "an unwatched 64 KiB import kept its piece in the write watch");
+    registry(piece, true);
+    SetHostImportBudget(4096);
+    const auto before = MirrorCounters();
+    build();
+    Require(!HostImportCovers(importing, address, piece), "the piece's import outlived the run that grew past it, or a 128 KiB block fit a 4 KiB budget");
+    Require(AgcDriver::GuestMemory::Watched(address, piece), "a 64 KiB piece whose import retired when its run grew into a block stayed outside the write watch");
+    Require(MirrorCounters().heapMirrors == before.heapMirrors + 2, "a 64 KiB piece whose import retired when its run grew into a block was copied, not heap mirrored, once no import fit");
+    registry(piece, false);
+    registry(0, false);
+    build();
+    SetImportWatch(importing, decided);
+    ClearHostImports(importing.device);
+    SetHostImportBudget(0);
+    SetHostImportWindow(0);
+    SetHostImportDeferredSettles(true);
+    GuestArena::GuestArenaReset_nid_postfix(block, bytes);
+    GuestArena::GuestArenaRelease_nid_postfix(block, bytes);
+#else
+    static_cast<void>(context);
+#endif
+}
+
 void importedFreshTests(const Context& context) {
 #ifndef _WIN32
     if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) {
@@ -4168,6 +4246,7 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
     heapMirrorTests(context, access);
     partialHeapImportBudgetTests(context, access);
     importedHeapMirrorTests(context, access);
+    retiredImportWatchTests(context);
     importWatchTests();
     importedFreshTests(context);
     gpuMappingTests(context);

core/libs/prx/libSceAgcDriver/tests/DeferredDestruction.cpp

@@ -0,0 +1,131 @@
+#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
+#include <chrono>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <exception>
+#include <string>
+#include <vector>
+
+using namespace AgcDriver::Graphics;
+
+namespace {
+
+int failures = 0;
+
+void Expect(bool condition, const std::string& what) {
+    if (condition) return;
+    std::fprintf(stderr, "FAIL: %s\n", what.c_str());
+    ++failures;
+}
+
+std::vector<std::string> events;
+VkResult idleResult = VK_SUCCESS;
+
+template<typename THandle>
+THandle Handle(std::uintptr_t value) {
+    return reinterpret_cast<THandle>(value);
+}
+
+std::string Name(const char* kind, const void* handle) {
+    return std::string(kind) + std::to_string(reinterpret_cast<std::uintptr_t>(handle));
+}
+
+VKAPI_ATTR void VKAPI_CALL mockDestroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks*) {
+    events.push_back(Name("buffer ", reinterpret_cast<const void*>(buffer)) + Name(" on ", device));
+}
+
+VKAPI_ATTR void VKAPI_CALL mockFreeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks*) {
+    events.push_back(Name("memory ", reinterpret_cast<const void*>(memory)) + Name(" on ", device));
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL mockQueueWaitIdle(VkQueue queue) {
+    events.push_back(Name("idle ", queue));
+    return idleResult;
+}
+
+VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL mockProc(VkDevice, const char* name) {
+    if (std::strcmp(name, "vkDestroyBuffer") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&mockDestroyBuffer);
+    if (std::strcmp(name, "vkFreeMemory") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&mockFreeMemory);
+    if (std::strcmp(name, "vkQueueWaitIdle") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&mockQueueWaitIdle);
+    return nullptr;
+}
+
+Context ContextFor(VkDevice device, PFN_vkGetDeviceProcAddr proc) {
+    Context context{};
+    context.device = device;
+    context.queue = Handle<VkQueue>(0x900);
+    context.deviceProc = proc;
+    return context;
+}
+
+void DeferredUntilTheQueueIsIdle() {
+    events.clear();
+    const auto device = Handle<VkDevice>(0x100);
+    const auto proc = DeferDestructionToIdle(device, mockProc);
+    const auto context = ContextFor(device, proc);
+    Expect(context.Function<PFN_vkQueueWaitIdle>("vkQueueWaitIdle") == &mockQueueWaitIdle, "a function other than vkDestroyBuffer and vkFreeMemory was not passed through");
+    context.Function<PFN_vkFreeMemory>("vkFreeMemory")(device, Handle<VkDeviceMemory>(0x21), nullptr);
+    context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(device, Handle<VkBuffer>(0x11), nullptr);
+    context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(device, VK_NULL_HANDLE, nullptr);
+    Expect(events.empty() && DeferredObjects(device) == 2, "a destroy or free was not deferred, or a null handle was queued");
+    const auto start = std::chrono::steady_clock::now();
+    DestroyDeferredAtIdle(context, start);
+    Expect(events.empty() && DeferredObjects(device) == 2, "deferred objects younger than the drain age were destroyed, or the queue was waited for");
+    DestroyDeferredAtIdle(context, start + std::chrono::seconds(1));
+    const std::vector<std::string> drained{"idle 2304", "buffer 17 on 256", "memory 33 on 256"};
+    Expect(events == drained && DeferredObjects(device) == 0, "old deferred objects were not destroyed after a queue wait, buffers before memory");
+    events.clear();
+    for (std::uintptr_t index = 0; index < 512; ++index) context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(device, Handle<VkBuffer>(0x1000 + index), nullptr);
+    DestroyDeferredAtIdle(context, std::chrono::steady_clock::now());
+    Expect(events.size() == 513 && events.front() == "idle 2304" && DeferredObjects(device) == 0, "512 young deferred objects were not destroyed after a queue wait");
+    events.clear();
+    idleResult = VK_ERROR_OUT_OF_HOST_MEMORY;
+    context.Function<PFN_vkFreeMemory>("vkFreeMemory")(device, Handle<VkDeviceMemory>(0x22), nullptr);
+    bool threw = false;
+    try {
+        DestroyDeferredAtIdle(context, std::chrono::steady_clock::now() + std::chrono::seconds(1));
+    } catch (const std::exception&) {
+        threw = true;
+    }
+    Expect(threw && DeferredObjects(device) == 1 && events.size() == 1, "a failed queue wait did not throw, or destroyed the deferred objects");
+    idleResult = VK_SUCCESS;
+    events.clear();
+    DestroyDeferredObjects(device);
+    Expect(events == std::vector<std::string>{"memory 34 on 256"} && DeferredObjects(device) == 0, "the deferred objects were not destroyed when the device went, or the queue was waited for");
+    events.clear();
+    context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(device, Handle<VkBuffer>(0x12), nullptr);
+    Expect(events == std::vector<std::string>{"buffer 18 on 256"}, "a destroy after the device went was deferred");
+}
+
+void OtherDevicesDestroyAtOnce() {
+    events.clear();
+    const auto device = Handle<VkDevice>(0x101);
+    const auto proc = DeferDestructionToIdle(device, mockProc);
+    const auto other = Handle<VkDevice>(0x102);
+    reinterpret_cast<PFN_vkFreeMemory>(proc(other, "vkFreeMemory"))(other, Handle<VkDeviceMemory>(0x23), nullptr);
+    Expect(events == std::vector<std::string>{"memory 35 on 258"} && DeferredObjects(other) == 0, "a free on a device without deferred destruction was deferred");
+    bool threw = false;
+    try {
+        DeferDestructionToIdle(device, mockProc);
+    } catch (const std::exception&) {
+        threw = true;
+    }
+    Expect(threw, "deferred destruction was set up twice for one device");
+    DestroyDeferredObjects(device);
+}
+
+}
+
+int main() {
+    try {
+        DeferredUntilTheQueueIsIdle();
+        OtherDevicesDestroyAtOnce();
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "FAIL: %s\n", error.what());
+        return 1;
+    }
+    if (failures != 0) return 1;
+    std::puts("deferred destruction tests passed");
+    return 0;
+}

core/libs/prx/libSceAgcDriver/tests/FmaMixOnce.cpp

@@ -82,7 +82,7 @@ int main() {
         const auto one = Compile(1);
         const auto three = Compile(3);
         Check(three.words - one.words < one.words - none.words, "fma mix once: 2 more v_fma_mixlo/mixhi_f16 added " + std::to_string(three.words - one.words) + " SPIR-V words, as many as the first one with its shared emulation (" + std::to_string(one.words - none.words) + ")");
-        Check(one.functions == none.functions + 1u, "fma mix once: one v_fma_mixlo_f16 emitted " + std::to_string(one.functions - none.functions) + " functions more than none, expected one shared emulation function");
+        Check(one.functions == none.functions + 2u, "fma mix once: one v_fma_mixlo_f16 emitted " + std::to_string(one.functions - none.functions) + " functions more than none, expected one shared f32 path and its shared exact emulation");
         Check(three.functions == one.functions, "fma mix once: two more v_fma_mixlo/mixhi_f16 in the same float mode emitted " + std::to_string(three.functions - one.functions) + " more functions, expected calls to the shared one");
         std::printf("fma mix once tests passed (%zu words without v_fma_mix, %zu for 1, %zu for 3)\n", none.words, one.words, three.words);
         return 0;

core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp

@@ -17,6 +17,7 @@
 #include <memory>
 #include <stdexcept>
 #include <string>
+#include <thread>
 #include <vector>
 #ifdef _WIN32
 #include <windows.h>
@@ -100,14 +101,18 @@ std::vector<std::byte> pattern(std::uint8_t seed) {
     return bytes;
 }
 
-void copyFrame(std::uint32_t* words, const Layout& layout, std::byte* target) {
-    CommandBuffer buffer{words, words + CommandBytes / sizeof(std::uint32_t), words, words + CommandBytes / sizeof(std::uint32_t), nullptr, nullptr, 0};
-    sceAgcDcbDmaData(&buffer, 0, DmaMemory, 0, at(target), DmaMemory, 0, at(layout.source), DataBytes, 0, 1, 0);
-    sceAgcDcbWriteData(&buffer, GraphicsMemory, 0, at(layout.label), &Done, 1, 0, 1);
+void copy(std::uint32_t* words, std::size_t commandBytes, const std::byte* source, std::byte* target, std::uint32_t* label) {
+    CommandBuffer buffer{words, words + commandBytes / sizeof(std::uint32_t), words, words + commandBytes / sizeof(std::uint32_t), nullptr, nullptr, 0};
+    sceAgcDcbDmaData(&buffer, 0, DmaMemory, 0, at(target), DmaMemory, 0, at(source), DataBytes, 0, 1, 0);
+    sceAgcDcbWriteData(&buffer, GraphicsMemory, 0, at(label), &Done, 1, 0, 1);
     Packet packet{words, static_cast<std::uint32_t>(buffer.cursor_up - words), 0, {}};
     check(sceAgcDriverSubmitDcb(&packet) == 0, "submission failed");
 }
 
+void copyFrame(std::uint32_t* words, const Layout& layout, std::byte* target) {
+    copy(words, CommandBytes, layout.source, target, layout.label);
+}
+
 void flip() {
     AgcDriverSubmitFlip_nid_postfix(AgcDriver::FlipInfo{Handle, -1, 1, 0});
 }
@@ -187,15 +192,75 @@ void testCaptureAndReplay(const std::filesystem::path& tool) {
     check(readFile(secondDump) == second, "the replay did not reproduce the second copy");
 }
 
+void testConcurrentCopies(const std::filesystem::path& tool) {
+    constexpr std::size_t Copies = 64;
+    constexpr std::size_t Frames = 8;
+    constexpr std::size_t CopyCommandBytes = 1024;
+    const char* directory = std::getenv("APS5_FRAME_CAPTURE");
+    check(directory != nullptr && *directory != '\0', "APS5_FRAME_CAPTURE is not set");
+    const std::filesystem::path capture(directory);
+    const auto layout = allocate();
+    const auto other = allocate();
+    auto* const otherSource = other.base + 128 * 1024;
+    auto* const targets = otherSource + DataBytes;
+    auto* const otherLabel = reinterpret_cast<std::uint32_t*>(other.base + 512 * 1024);
+    const auto copied = pattern(57);
+    std::memcpy(otherSource, copied.data(), copied.size());
+    auto output = std::make_shared<Output>();
+    AgcDriverRegisterVideoOutput_nid_postfix(Handle, output);
+
+    flip();
+    std::thread copier([&] {
+        for (std::size_t index = 0; index < Copies; ++index) {
+            copy(reinterpret_cast<std::uint32_t*>(other.base + index * CopyCommandBytes), CopyCommandBytes, otherSource, targets + index * DataBytes, otherLabel);
+        }
+    });
+    for (std::size_t frame = 0; frame + 1 < Frames; ++frame) {
+        const auto bytes = pattern(static_cast<std::uint8_t>(frame));
+        std::memcpy(layout.source, bytes.data(), bytes.size());
+        copyFrame(layout.firstCommands, layout, layout.firstTarget);
+        flip();
+    }
+    copier.join();
+    flip();
+    AgcDriverWaitIdle_nid_postfix();
+
+    const FrameCaptureReader reader(capture);
+    check(!reader.Truncated(), "the capture is truncated");
+    std::uint64_t base = 0;
+    std::size_t submits = 0, memory = 0;
+    for (const auto& call : reader.Calls()) {
+        if (call.kind == FrameCallKind::FrameStart) base = call.completed;
+        if (call.kind == FrameCallKind::Memory) {
+            ++memory;
+            check(call.completed >= base + submits, "a memory delta was read with " + std::to_string(base + submits - call.completed) + " of the " + std::to_string(submits) + " submissions recorded before it unfinished");
+        }
+        submits += call.kind == FrameCallKind::Submit;
+    }
+    check(submits == Copies + Frames - 1, "the capture holds " + std::to_string(submits) + " submits, not " + std::to_string(Copies + Frames - 1));
+    check(memory != 0, "the capture holds no memory deltas");
+
+    const auto dump = capture / "targets.bin";
+    const auto command = "\"\"" + tool.string() + "\" \"" + capture.string() + "\" --strict --dump " + std::to_string(at(targets)) + ":" + std::to_string(Copies * DataBytes) + ":\"" + dump.string() + "\"\"";
+    const auto code = std::system(command.c_str());
+    check(code == 0, "agc_frame_replay exited with " + std::to_string(code));
+    const auto replayed = readFile(dump);
+    check(replayed.size() == Copies * DataBytes, "the replay dumped " + std::to_string(replayed.size()) + " bytes of the copy targets");
+    for (std::size_t index = 0; index < Copies; ++index) {
+        check(std::memcmp(replayed.data() + index * DataBytes, copied.data(), DataBytes) == 0, "the replay did not reproduce copy " + std::to_string(index) + " of the second thread");
+    }
+}
+
 }
 
 int main(int argc, char** argv) {
-    if (argc != 2) {
-        std::fprintf(stderr, "usage: agc_driver_frame_replay_tests <agc_frame_replay>\n");
+    if (argc != 2 && !(argc == 3 && std::string(argv[2]) == "concurrent")) {
+        std::fprintf(stderr, "usage: agc_driver_frame_replay_tests <agc_frame_replay> [concurrent]\n");
         return 2;
     }
     try {
-        testCaptureAndReplay(argv[1]);
+        if (argc == 3) testConcurrentCopies(argv[1]);
+        else testCaptureAndReplay(argv[1]);
     } catch (const std::exception& error) {
         std::fprintf(stderr, "FAILED: %s\n", error.what());
         return 1;

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -3973,6 +3973,10 @@ void multisampleTests() {
         for (const auto offset : {0x31cu, 0x32bu}) packed.context[offset] = (packed.context[offset] & ~0x41f7cu) | packedInfo;
         expectFailure([&] { DecodeColorMetadataPass(packed); }, std::string("CB resolves of ") + name + " color targets");
     }
+    auto uintSource = rasterizedResolve;
+    for (const auto offset : {0x31cu, 0x32bu}) uintSource.context[offset] = (uintSource.context[offset] & ~0x700u) | (4u << 8u);
+    const auto uintPass = DecodeColorMetadataPass(uintSource);
+    Require(uintPass.has_value() && uintPass->mode == ColorMetadataPass::Mode::Resolve && uintPass->source.has_value() && uintPass->source->packing == ShaderRecompiler::ColorExportPacking::Uint && uintPass->targets.size() == 1, "a CB resolve of an unsigned integer target was refused");
     queue = rasterizedResolve;
     queue.context[0x31c] |= 0x10000000u;
     queue.context[0x325] = 0;
@@ -4674,6 +4678,72 @@ void DepthRemapTests() {
     Require(floating.depthRemap.mode == 1 && floating.viewport.minDepth == 0 && floating.viewport.maxDepth == 1 && floating.depthClamp, "a written float depth target refused clipped clamp bounds");
 }
 
+void DepthBoundsEmulationTests() {
+    auto queue = makeState();
+    queue.context[0x204] = 0;
+    queue.context[0x113] = std::bit_cast<std::uint32_t>(0.5f);
+    queue.context[0x114] = std::bit_cast<std::uint32_t>(0.5f);
+    auto base = AgcDriver::Graphics::DecodeState(queue);
+    base.depth = AgcDriver::Graphics::DepthTarget{0x10000, 0, {64, 64}, VK_FORMAT_D32_SFLOAT_S8_UINT, 0.0f, 0};
+    base.depthTest = base.depthWrite = base.stencilTest = false;
+    base.depthBoundsTest = true;
+    base.depthBias = true;
+    base.depthBiasConstant = 4.0f;
+    const ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    const AgcDriver::Graphics::Context emulating{};
+    const auto emulated = [&](float minBound, float maxBound) {
+        auto state = base;
+        state.minDepthBounds = minBound;
+        state.maxDepthBounds = maxBound;
+        AgcDriver::Graphics::ApplyDepthBoundsEmulation(emulating, state, pixel);
+        Require(!state.depthBoundsTest && state.depthTest && !state.depthWrite && !state.depthBias && state.viewport.x == base.viewport.x && state.viewport.width == base.viewport.width && state.viewport.height == base.viewport.height, "an emulated depth bounds test did not become a read-only depth test over the same coverage");
+        return state;
+    };
+    const auto upper = emulated(0.0f, 0.000125f);
+    Require(upper.depthCompare == VK_COMPARE_OP_GREATER_OR_EQUAL && upper.viewport.minDepth == 0.000125f && upper.viewport.maxDepth == 0.000125f, "an upper depth bound was not tested as the bound at or above the stored depth");
+    const auto lower = emulated(0.000125f, 1.0f);
+    Require(lower.depthCompare == VK_COMPARE_OP_LESS_OR_EQUAL && lower.viewport.minDepth == 0.000125f && lower.viewport.maxDepth == 0.000125f, "a lower depth bound was not tested as the bound at or below the stored depth");
+    const auto whole = emulated(0.0f, 1.0f);
+    Require(whole.depthCompare == VK_COMPARE_OP_GREATER_OR_EQUAL && whole.viewport.minDepth == 1.0f && whole.viewport.maxDepth == 1.0f, "bounds covering [0, 1] were not tested against the upper bound");
+    AgcDriver::Graphics::Context native{};
+    native.depthBounds = true;
+    auto kept = base;
+    AgcDriver::Graphics::ApplyDepthBoundsEmulation(native, kept, pixel);
+    Require(kept.depthBoundsTest && !kept.depthTest && kept.depthBias && kept.viewport.minDepth == base.viewport.minDepth, "a device with the depth bounds test had it emulated");
+    auto absent = base;
+    absent.depth.reset();
+    AgcDriver::Graphics::ApplyDepthBoundsEmulation(emulating, absent, pixel);
+    Require(absent.depthBoundsTest && !absent.depthTest, "a draw without a depth target had its depth bounds emulated");
+    const auto refuses = [&](const char* expected, auto change) {
+        auto state = base;
+        state.minDepthBounds = 0.0f;
+        state.maxDepthBounds = 0.5f;
+        auto context = emulating;
+        auto program = pixel;
+        change(state, context, program);
+        expectFailure([&] { AgcDriver::Graphics::ApplyDepthBoundsEmulation(context, state, program); }, expected);
+    };
+    refuses("both bounds inside", [](auto& state, auto&, auto&) { state.minDepthBounds = 0.25f; });
+    refuses("with depth testing", [](auto& state, auto&, auto&) { state.depthTest = true; });
+    refuses("reads or exports depth", [](auto&, auto&, auto& program) { program.posZ = true; });
+    refuses("reads or exports depth", [](auto&, auto&, auto& program) { program.depthExportEnable = true; });
+    refuses("normalized depth target", [](auto& state, auto&, auto&) { state.depth->format = VK_FORMAT_D16_UNORM; });
+    refuses("stays in [0, 1]", [](auto&, auto& context, auto&) { context.depthRangeUnrestricted = true; });
+    refuses("depth bounds outside [0, 1]", [](auto& state, auto&, auto&) { state.maxDepthBounds = 1.5f; });
+    refuses("writes on a failed test", [](auto& state, auto&, auto&) {
+        state.stencilTest = true;
+        state.stencilFront.writeMask = 0xffu;
+        state.stencilFront.failOp = VK_STENCIL_OP_REPLACE;
+    });
+    auto stencil = base;
+    stencil.stencilTest = true;
+    stencil.stencilFront.writeMask = stencil.stencilBack.writeMask = 0xffu;
+    stencil.stencilFront.passOp = stencil.stencilBack.passOp = VK_STENCIL_OP_REPLACE;
+    stencil.maxDepthBounds = 0.5f;
+    AgcDriver::Graphics::ApplyDepthBoundsEmulation(emulating, stencil, pixel);
+    Require(stencil.depthTest && stencil.stencilTest, "a stencil test that writes only on a pass refused the depth bounds emulation");
+}
+
 void UnusedViewportDepthTests() {
     auto queue = makeState();
     queue.context[0x204] = 0;
@@ -9169,7 +9239,7 @@ void stencilRefExportTests() {
             Require(AgcDriver::Graphics::DrawRejection(unused, false).empty(), "the precheck rejected STENCIL_OP_VAL_EXPORT_ENABLE without a stencil test in DB_SHADER_CONTROL " + std::to_string(shaderControl));
             Require(AgcDriver::Graphics::DecodePixelStageInfo(unused.context, IdentityExports).stencilRefExportEnable == ((shaderControl & 2u) != 0), "STENCIL_OP_VAL_EXPORT_ENABLE without a stencil test exported the stencil reference");
             const bool replaces = (shaderControl & 6u) == 4u;
-            for (const auto [ops, writeMask, compare, accepted] : {std::tuple{0x000u, 0xffu, 7u, true}, std::tuple{0x737u, 0xffu, 7u, true}, std::tuple{0x110u, 0xffu, 7u, true}, std::tuple{0x050u, 0x00u, 7u, true}, std::tuple{0x040u, 0xffu, 7u, replaces}, std::tuple{0x40040u, 0xffu, 7u, replaces}, std::tuple{0x404u, 0x0fu, 0u, replaces}, std::tuple{0x040u, 0xffu, 2u, false}, std::tuple{0x340u, 0xffu, 7u, false}, std::tuple{0x042u, 0xffu, 7u, false}, std::tuple{0x050u, 0xffu, 7u, false}, std::tuple{0x600u, 0xffu, 7u, false}, std::tuple{0x008u, 0xffu, 7u, false}, std::tuple{0x900u, 0xffu, 7u, false}, std::tuple{0x5000u, 0xffu, 7u, false}}) {
+            for (const auto [ops, writeMask, compare, accepted] : {std::tuple{0x000u, 0xffu, 7u, true}, std::tuple{0x737u, 0xffu, 7u, true}, std::tuple{0x110u, 0xffu, 7u, true}, std::tuple{0x050u, 0x00u, 7u, true}, std::tuple{0x040u, 0xffu, 7u, replaces}, std::tuple{0x40040u, 0xffu, 7u, replaces}, std::tuple{0x404u, 0x0fu, 0u, replaces}, std::tuple{0x040u, 0xffu, 2u, false}, std::tuple{0x340u, 0xffu, 7u, false}, std::tuple{0x042u, 0xffu, 7u, false}, std::tuple{0x050u, 0xffu, 7u, false}, std::tuple{0x600u, 0xffu, 7u, false}, std::tuple{0x008u, 0xffu, 7u, false}, std::tuple{0x900u, 0xffu, 7u, false}, std::tuple{0x5000u, 0xffu, 7u, false}, std::tuple{0x0c0u, 0xffu, 7u, false}, std::tuple{0x00bu, 0xffu, 7u, false}}) {
                 auto stenciled = stencilRefExportQueue(shaderControl, zFormat);
                 for (const auto [offset, word] : {std::pair{0x010u, 0x80000180u}, std::pair{0x011u, 0x20000181u}, std::pair{0x012u, 0x00001000u}, std::pair{0x013u, 0x00002000u}, std::pair{0x015u, 0x00002000u}, std::pair{0x007u, 0x003f003fu}, std::pair{0x000u, 0u}, std::pair{0x002u, 0u}, std::pair{0x00au, 0u}, std::pair{0x00bu, 0x3f800000u}, std::pair{0x200u, 0x007007b3u}}) stenciled.context[offset] = word;
                 stenciled.context[0x10b] = ops;
@@ -9479,6 +9549,99 @@ void stencilReferenceUnusedTests() {
     std::cout << "Unused stencil reference sample equivalence passed: " << accepted << " accepted states, " << countermodels << " dependent countermodels\n";
 }
 
+void stencilLogicOperationTests() {
+    const auto queueFor = [](std::uint32_t ops, std::uint32_t front, std::uint32_t back, std::uint32_t depthControl) {
+        auto queue = makeState();
+        for (const auto [offset, word] : {std::pair{0x000u, 0u}, std::pair{0x002u, 0u}, std::pair{0x007u, (1u << 16u) | 3u}, std::pair{0x00au, 0u}, std::pair{0x00bu, 0u}, std::pair{0x010u, 0x22900983u}, std::pair{0x011u, 0x20000181u}, std::pair{0x012u, 0x100u}, std::pair{0x014u, 0x100u}, std::pair{0x013u, 0x200u}, std::pair{0x015u, 0x200u}, std::pair{0x005u, 0x300u}, std::pair{0x01eu, 0x1u}}) queue.context[offset] = word;
+        queue.context[0x10b] = ops;
+        queue.context[0x10c] = front;
+        queue.context[0x10d] = back;
+        queue.context[0x200] = depthControl;
+        return queue;
+    };
+    const auto refMaskOf = [](std::uint32_t test, std::uint32_t writeMask, std::uint32_t value) { return test | 0xff00u | (writeMask << 16u) | (value << 24u); };
+    const auto apply = [](std::uint32_t op, std::uint32_t stencil, std::uint32_t value) {
+        switch (op) {
+            case 1: return 0u;
+            case 10: return stencil & value;
+            case 11: return stencil | value;
+            case 12: return stencil ^ value;
+            case 15: return ~(stencil ^ value);
+            default: return stencil;
+        }
+    };
+    const auto matches = [&](const AgcDriver::Graphics::State& state, const VkStencilOpState& face, std::uint32_t ops, std::uint32_t compare, std::uint32_t refMask, const std::string& name) {
+        const auto test = refMask & 0xffu;
+        const auto compareMask = (refMask >> 8u) & 0xffu;
+        const auto writeMask = (refMask >> 16u) & 0xffu;
+        const auto value = refMask >> 24u;
+        for (std::uint32_t stencil = 0; stencil < 256u; ++stencil) {
+            for (const bool depthPass : {false, true}) {
+                const bool stencilPass = stencilComparison(static_cast<VkCompareOp>(compare), test & compareMask, stencil & compareMask);
+                const auto op = (ops >> (!stencilPass ? 0u : !depthPass ? 8u : 4u)) & 0xfu;
+                const auto expected = ((stencil & ~writeMask) | (apply(op, stencil, value) & writeMask)) & 0xffu;
+                const auto [actual, passed] = stencilReferenceOutcome(state, face, face.reference, stencil, depthPass, true);
+                Require(actual == expected && passed == (stencilPass && depthPass), name + ": stencil " + std::to_string(stencil) + (depthPass ? " passing" : " failing") + " the depth test became " + std::to_string(actual) + ", expected " + std::to_string(expected));
+            }
+        }
+    };
+    std::size_t accepted = 0;
+    for (const auto op : {10u, 11u, 12u, 15u}) {
+        for (const auto placement : {0x010u, 0x001u, 0x100u, 0x111u}) {
+            for (const auto compare : {7u, 0u, 2u, 3u}) {
+                for (const auto [test, writeMask, value] : {std::tuple{0x00u, 0xffu, 0x5au}, std::tuple{0xa5u, 0xffu, 0x21u}, std::tuple{0xa5u, 0x3cu, 0xffu}, std::tuple{0xffu, 0xf0u, 0x0fu}, std::tuple{0x5au, 0x81u, 0x00u}}) {
+                    const auto ops = op * placement;
+                    const auto refMask = refMaskOf(test, writeMask, value);
+                    const auto queue = queueFor(ops, refMask, refMask, 0x13u | (compare << 8u));
+                    const auto name = "stencil operation " + std::to_string(op) + " in DB_STENCIL_CONTROL " + std::to_string(ops) + " with compare " + std::to_string(compare) + " and DB_STENCILREFMASK " + std::to_string(refMask);
+                    const auto bits = (op == 11u || op == 12u ? value : ~value) & writeMask;
+                    if (op == 11u && compare != 0u && compare != 7u && (~test & bits) != 0u) {
+                        expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeState(queue)); }, "stencil OR of");
+                        continue;
+                    }
+                    const auto state = AgcDriver::Graphics::DecodeState(queue);
+                    Require(state.stencilTest && state.depthTest && (state.stencilFront.writeMask & 0xffu) == (bits != 0u ? bits : writeMask), "the write mask is not narrowed to the bits changed by " + name);
+                    matches(state, state.stencilFront, ops, compare, refMask, name);
+                    ++accepted;
+                }
+            }
+        }
+    }
+    for (const auto [ops, refMask, name] : {std::tuple{0x0c1u, refMaskOf(0x80u, 0xffu, 0xffu), "XOR of every written bit beside ZERO"}, std::tuple{0x0cbu, refMaskOf(0x8fu, 0xffu, 0x0fu), "OR and XOR of the same bits"}, std::tuple{0x0fau, refMaskOf(0x80u, 0xffu, 0xf0u), "AND and XNOR of the same bits"}, std::tuple{0xb0bu, refMaskOf(0x30u, 0x3cu, 0xf0u), "OR on stencil fail and depth fail"}}) {
+        const auto queue = queueFor(ops, refMask, refMask, 0x13u | (3u << 8u));
+        const auto state = AgcDriver::Graphics::DecodeState(queue);
+        matches(state, state.stencilFront, ops, 3u, refMask, name);
+        ++accepted;
+    }
+    {
+        const auto front = refMaskOf(0x11u, 0xffu, 0x0fu);
+        const auto back = refMaskOf(0x22u, 0x3cu, 0x30u);
+        const auto queue = queueFor(0x0c0u | (0x0a0u << 12u), front, back, 0x93u | (7u << 8u) | (5u << 20u));
+        const auto state = AgcDriver::Graphics::DecodeState(queue);
+        Require(state.stencilFront.passOp == VK_STENCIL_OP_INVERT && state.stencilFront.writeMask == 0x0fu && state.stencilBack.passOp == VK_STENCIL_OP_ZERO && state.stencilBack.writeMask == 0x0cu, "front XOR and back AND did not keep their own write masks");
+        matches(state, state.stencilFront, 0x0c0u, 7u, front, "front XOR beside a back AND");
+        matches(state, state.stencilBack, 0x0a0u, 5u, back, "back AND beside a front XOR");
+        accepted += 2;
+    }
+    for (const auto [ops, refMask, reason] : {std::tuple{0x0d0u, refMaskOf(0u, 0xffu, 0x0fu), "stencil operation 13 is unsupported"}, std::tuple{0x0e0u, refMaskOf(0u, 0xffu, 0x0fu), "stencil operation 14 is unsupported"}, std::tuple{0x0cau, refMaskOf(0u, 0xffu, 0x0fu), "stencil logic operations on write masks"}, std::tuple{0x0fcu, refMaskOf(0u, 0xffu, 0x0fu), "stencil logic operations on write masks"},std::tuple{0x0c1u, refMaskOf(0u, 0xffu, 0x0fu), "combine a logic operation"}, std::tuple{0x5c0u, refMaskOf(0u, 0xffu, 0x01u), "combine a logic operation"}}) {
+        const auto queue = queueFor(ops, refMask, refMask, 0x13u | (7u << 8u));
+        expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeState(queue)); }, reason);
+    }
+    for (const auto ops : {0x0c0u, 0x0b0u}) {
+        auto exported = stencilRefExportQueue(0x12u, 9u);
+        for (const auto [offset, word] : {std::pair{0x010u, 0x80000180u}, std::pair{0x011u, 0x20000181u}, std::pair{0x012u, 0x00001000u}, std::pair{0x013u, 0x00002000u}, std::pair{0x015u, 0x00002000u}, std::pair{0x007u, 0x003f003fu}, std::pair{0x000u, 0u}, std::pair{0x002u, 0u}, std::pair{0x00au, 0u}, std::pair{0x00bu, 0x3f800000u}, std::pair{0x200u, 0x007007b3u}}) exported.context[offset] = word;
+        exported.context[0x10b] = ops;
+        exported.context[0x10c] = exported.context[0x10d] = refMaskOf(0u, 0xffu, 0x0fu);
+        if (ops == 0x0c0u) {
+            const auto state = AgcDriver::Graphics::DecodeState(exported);
+            Require(state.stencilFront.passOp == VK_STENCIL_OP_INVERT && state.stencilFront.writeMask == 0x0fu, "a stencil test value export changed a stencil XOR");
+        } else {
+            expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeState(exported)); }, "stencil OR is unsupported with a pixel shader that exports the stencil test value");
+        }
+    }
+    std::cout << "Stencil logic operations matched AND, OR, XOR and XNOR under the write mask for " << accepted << " register states\n";
+}
+
 std::set<std::string> bufferViews(std::span<const std::uint32_t> words) {
     std::set<std::string> views;
     for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
@@ -9993,6 +10156,34 @@ ShaderRecompiler::RecompileRequest packedExportRequest(const AgcDriver::QueueSta
     return request;
 }
 
+void uintTargetTests() {
+    using ShaderRecompiler::ColorExportPacking;
+    auto queue = makeState();
+    queue.context[0x31c] = (queue.context.at(0x31c) & ~0x1f7cu) | (4u << 2u) | (4u << 8u);
+    queue.context[0x1e0] = 1u << 30u;
+    for (std::uint32_t i = 0; i < 4; ++i) queue.context[0x105 + i] = 0;
+    const auto state = AgcDriver::Graphics::DecodeState(queue);
+    Require(state.color.format == VK_FORMAT_R32_UINT && state.color.elementBytes == 4 && state.color.bytes == colorMemory.size() && state.color.packing == ColorExportPacking::Uint, "a 32-bit UINT target did not get the unsigned integer export packing");
+    const auto packings = AgcDriver::Graphics::ExportPackings(state);
+    Require(packings[0] == ColorExportPacking::Uint && std::all_of(packings.begin() + 1, packings.end(), [](ColorExportPacking packing) { return packing == ColorExportPacking::None; }), "export 0 did not carry the unsigned integer packing");
+    Require(!state.blends.at(0).blendEnable && state.blends.at(0).colorWriteMask != 0, "blending into a UINT target was not dropped");
+    auto bytes = queue;
+    bytes.context[0x31c] = (bytes.context.at(0x31c) & ~0x1f7cu) | (1u << 2u) | (4u << 8u);
+    const auto eight = AgcDriver::Graphics::DecodeState(bytes);
+    Require(eight.color.format == VK_FORMAT_R8_UINT && eight.color.packing == ColorExportPacking::Uint && !eight.blends.at(0).blendEnable, "an 8-bit UINT target did not get the unsigned integer export packing");
+    auto words = queue;
+    words.context[0x1c5] = 7u;
+    words.context[0x1e0] = 0;
+    const auto uint16 = AgcDriver::Graphics::DecodeState(words);
+    Require(uint16.color.packing == ColorExportPacking::None && uint16.color.uintExport, "a UINT16_ABGR export into a UINT target was retyped twice");
+    auto halves = queue;
+    halves.context[0x1c5] = 4u;
+    expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeState(halves)); }, "color export format 4 into an unsigned integer target is unsupported");
+    auto unorm = queue;
+    unorm.context[0x31c] = (unorm.context.at(0x31c) & ~0x1f7cu) | (4u << 2u) | (7u << 8u);
+    Require(AgcDriver::Graphics::DecodeState(unorm).color.packing == ColorExportPacking::None, "a 32-bit float target got the unsigned integer export packing");
+}
+
 void unorm10_11_11TargetTests() {
     using ShaderRecompiler::ColorExportPacking;
     auto queue = makeState();
@@ -10470,7 +10661,7 @@ void float11_11_10TargetTests() {
     const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &packed, 0}}};
     Require(AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false) == std::set<std::uint32_t>{0u}, "the packed float export did not reach attachment 0");
     auto invalid = request;
-    invalid.context.pixel->targetExportPacking[0] = static_cast<ColorExportPacking>(4);
+    invalid.context.pixel->targetExportPacking[0] = static_cast<ColorExportPacking>(5);
     expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(invalid)); }, "invalid prepared fragment export packing");
 }
 
@@ -12294,6 +12485,7 @@ int main(int argc, char** argv) {
         srgb8TargetTests();
         DepthClipTests();
         DepthRemapTests();
+        DepthBoundsEmulationTests();
         UnusedViewportDepthTests();
         DepthStencilTests();
         OneDimensionalColorTests();
@@ -12304,12 +12496,14 @@ int main(int argc, char** argv) {
         unorm10_11_11TargetTests();
         float11_11_10TargetTests();
         unorm4_4_4_4TargetTests();
+        uintTargetTests();
         DepthBoundsBiasTests();
         conservativeZExportTests();
         earlyFragmentTestsTests();
         orderedPixelShaderTests();
         stencilRefExportTests();
         stencilReferenceUnusedTests();
+        stencilLogicOperationTests();
         aliasedBufferViewTests();
         LineStripTests();
         ShaderModuleKeyTests();

core/libs/prx/libSceAgcDriver/tests/GuardedMemoryLoads.cpp

@@ -32,7 +32,7 @@ struct LoadCase {
     std::uint32_t components = 1;
 };
 
-std::vector<std::uint32_t> emitLoad(const LoadCase& test, bool active, std::uint32_t offset, std::uint32_t records, bool robust) {
+std::vector<std::uint32_t> emitLoad(const LoadCase& test, bool active, std::uint32_t offset, std::uint32_t records, bool robust, bool valid = true) {
     IrProgram program;
     program.SetWaveSize(32);
     program.Resources().stage = IrShaderStage::Compute;
@@ -76,7 +76,8 @@ std::vector<std::uint32_t> emitLoad(const LoadCase& test, bool active, std::uint
         program.Metadata().bindings.pushDataStartDword = 0;
         program.Metadata().bindings.memoryOffsetCount = 1;
         auto& size = constant(records);
-        auto& handle = emit(IrOpcode::GetBufferResource, IrType::BufferResource, {&zero, &zero, &size, &zero});
+        auto& base = constant(valid ? 0x100u : 0u);
+        auto& handle = emit(IrOpcode::GetBufferResource, IrType::BufferResource, {&base, &zero, &size, &zero});
         emit(test.opcode, test.type, {&handle, &zero, &address, &zero, &enabled});
     } else {
         if (test.kind == ResourceKind::Gds) program.Metadata().bindings.descriptors.push_back({DescriptorBindingKind::Gds, {0}});
@@ -104,8 +105,8 @@ std::vector<std::uint32_t> emitLoad(const LoadCase& test, bool active, std::uint
 
 class AccessTrace {
 public:
-    AccessTrace(const std::vector<std::uint32_t>& module, std::uint32_t misalignment = 0, bool valid = true)
-        : words(module), misalignment(misalignment), valid(valid) {
+    AccessTrace(const std::vector<std::uint32_t>& module, std::uint32_t misalignment = 0)
+        : words(module), misalignment(misalignment) {
         require(words.size() >= 5, "SPIR-V module header is missing");
         for (std::size_t offset = 5; offset < words.size();) {
             const auto count = words[offset] >> 16;
@@ -206,7 +207,6 @@ private:
             const auto spec = specializations.at(id);
             if (spec == PipelineSpecialization::BufferBase) return 0;
             if (spec == PipelineSpecialization::BufferBase + 1) return 0x30001000;
-            if (spec == PipelineSpecialization::BufferBase + 2) return valid;
             if (spec == PipelineSpecialization::BufferBase + 3) return misalignment;
             if (spec == PipelineSpecialization::PushDataOffset) return 0;
             throw std::runtime_error("unexpected specialization in access fixture");
@@ -250,7 +250,6 @@ private:
 
     const std::vector<std::uint32_t>& words;
     std::uint32_t misalignment;
-    bool valid;
     std::uint32_t entryFunction = 0;
     std::map<std::uint32_t, std::span<const std::uint32_t>> definitions;
     std::map<std::uint32_t, std::string> names;
@@ -281,8 +280,8 @@ std::uint32_t ldsLength(const std::vector<std::uint32_t>& words) {
 }
 
 void expectReads(const LoadCase& test, bool robust, bool active, std::uint32_t offset, std::vector<std::uint32_t> expected, std::uint32_t records = 64, std::uint32_t misalignment = 0, bool valid = true) {
-    const auto module = emitLoad(test, active, offset, records, robust);
-    const auto actual = AccessTrace(module, misalignment, valid).Run();
+    const auto module = emitLoad(test, active, offset, records, robust, valid);
+    const auto actual = AccessTrace(module, misalignment).Run();
     if (actual != expected) {
         std::string detail = std::string(robust ? "robust" : "plain") + " guest reads:";
         for (const auto index : actual) detail += " " + std::to_string(index);

core/libs/prx/libSceAgcDriver/tests/HostImportRetireHold.cpp

@@ -360,6 +360,23 @@ void BusyRetireOutlivesALaterBatch() {
     Expect(!Live(retired), "the dropped import outlived every batch of its hold");
 }
 
+void RecentImportFollowsReplacement() {
+    Heap heap;
+    ActiveRecorder active;
+    const auto context = mockContext();
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    const auto* first = HostImportFor(context, heap.Address(), 4096);
+    Expect(first != nullptr, "the first range was not imported");
+    if (first == nullptr) return;
+    Expect(HostImportFor(context, heap.Address(), 4096) == first, "a repeated lookup did not find the same import");
+    const auto retired = first->buffer;
+    heap.Grow();
+    const auto* grown = HostImportFor(context, heap.Address(RangeBytes), 4096);
+    Expect(grown != nullptr && grown->buffer != retired, "the grown run was not imported as a new import");
+    if (grown == nullptr) return;
+    Expect(HostImportFor(context, heap.Address(), 4096) == grown, "a lookup after the replacement served the retired import instead of the one that replaced it");
+}
+
 }
 
 int main() {
@@ -374,6 +391,7 @@ int main() {
         IdleRetireOutlivesTheHold();
         BusyRetireOutlivesALaterBatch();
         AgcDriver::Tests::RunHostImportFailureTests(mockContext(), false);
+        RecentImportFollowsReplacement();
     } catch (const std::exception& error) {
         std::fprintf(stderr, "%s\n", error.what());
         return 1;

core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp

@@ -1074,6 +1074,54 @@ int main() {
         return !StencilReplaySafe(b.program);
     });
 
+    const auto wholeQuadEval = [](Builder& b, IrValue& live) -> IrValue& {
+        auto& ballot = b.emit(IrOpcode::Ballot, IrType::U32x4, {&live});
+        auto& allLow = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()});
+        auto& allHigh = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()});
+        auto& low = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(0u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&allLow, &b.constant(0u)})});
+        auto& high = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(1u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&allHigh, &b.constant(1u)})});
+        return b.emit(IrOpcode::WqmU64, IrType::U64, {&b.emit(IrOpcode::CompositeConstructU64, IrType::U64, {&low, &high})});
+    };
+
+    const auto splitQuads = [&](Builder& b, IrValue& live) -> IrValue& {
+        auto& wqmLow = wholeQuadEval(b, live);
+        auto& wqmHigh = wholeQuadEval(b, live);
+        return b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmLow, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmHigh, &b.constant(1u)}));
+    };
+
+    passed &= run("stencil replay pairs structurally equivalent whole-quad evaluations for split low and high words", [&] {
+        Builder b;
+        b.program.Resources().stage = IrShaderStage::Pixel;
+        exportUnder(b, splitQuads(b, b.logicalAnd(pixelPredicate(b), helperPredicate(b, false))), true, true);
+        return StencilReplaySafe(b.program);
+    });
+
+    passed &= run("stencil replay refuses to pair whole-quad evaluations separated by an execution-mask change", [&] {
+        Builder b;
+        b.program.Resources().stage = IrShaderStage::Pixel;
+        auto& live = b.logicalAnd(pixelPredicate(b), helperPredicate(b, false));
+        auto& wqmLow = wholeQuadEval(b, live);
+        b.emit(IrOpcode::SetExecLo, IrType::Void, {&b.constant(0xffffffffu)});
+        auto& wqmHigh = wholeQuadEval(b, live);
+        auto& guard = b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmLow, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmHigh, &b.constant(1u)}));
+        exportUnder(b, guard, true, true);
+        return !StencilReplaySafe(b.program);
+    });
+
+    passed &= run("stencil replay refuses to pair whole-quad evaluations whose predicates differ only in a constant", [&] {
+        Builder b;
+        b.program.Resources().stage = IrShaderStage::Pixel;
+        auto& helperLow = b.emit(IrOpcode::GetBuiltin, IrType::U32, {&b.constant(static_cast<std::uint32_t>(StageInputKind::HelperInvocation)), &b.constant(0u)});
+        auto& lowLive = b.logicalAnd(pixelPredicate(b), b.emit(IrOpcode::IEqual32, IrType::Bool, {&helperLow, &b.constant(0u)}));
+        auto& wqmLow = wholeQuadEval(b, lowLive);
+        auto& helperHigh = b.emit(IrOpcode::GetBuiltin, IrType::U32, {&b.constant(static_cast<std::uint32_t>(StageInputKind::HelperInvocation)), &b.constant(0u)});
+        auto& highLive = b.logicalAnd(pixelPredicate(b), b.emit(IrOpcode::IEqual32, IrType::Bool, {&helperHigh, &b.constant(1u)}));
+        auto& wqmHigh = wholeQuadEval(b, highLive);
+        auto& guard = b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmLow, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmHigh, &b.constant(1u)}));
+        exportUnder(b, guard, true, true);
+        return !StencilReplaySafe(b.program);
+    });
+
     const auto execCopy = [](Builder& b, IrValue& low, IrValue& high) -> IrValue& {
         auto& active = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()});
         auto& lanes = b.emit(IrOpcode::CompositeConstructU64, IrType::U64, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&active, &b.constant(0u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&active, &b.constant(1u)})});
@@ -1188,10 +1236,205 @@ int main() {
         return StencilReplaySafe(b.program);
     };
 
+    const auto optionalPredicateBody = [&](bool negated) {
+        Builder b;
+        b.program.Resources().stage = IrShaderStage::Pixel;
+        auto& entry = *b.block;
+        auto& predicate = pixelPredicate(b);
+        auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate});
+        auto& body = b.newBlock();
+        auto& merge = b.newBlock();
+        entry.AddBranch(&body);
+        entry.AddBranch(&merge);
+        body.AddBranch(&merge);
+        auto& entryInfo = b.program.Metadata().blockInfo[0u];
+        entryInfo.terminator.kind = TerminatorKind::ConditionalBranch;
+        entryInfo.terminator.condition = BranchCondition::ExecZero;
+        entryInfo.terminator.trueBlock = merge.Id();
+        entryInfo.terminator.falseBlock = body.Id();
+        entryInfo.condition = &condition;
+        auto& bodyInfo = b.program.Metadata().blockInfo[1u];
+        bodyInfo.terminator.kind = TerminatorKind::Branch;
+        bodyInfo.terminator.trueBlock = merge.Id();
+        b.block = &body;
+        auto& written = negated ? b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate}) : b.ballotBit(predicate);
+        b.block = &merge;
+        auto& merged = b.phi(IrType::Bool, entry, predicate);
+        merged.AddPhiOperand(&body, &written);
+        b.keep(b.select(merged, b.constant(1u), b.constant(0u)));
+        exportUnder(b, b.truth(), true, true);
+        return StencilReplaySafe(b.program);
+    };
+
+    passed &= run("an optional wave body may merge a lane's own bit of the predicate's ballot with the skipped predicate", [&] { return optionalPredicateBody(false); });
+    passed &= run("an optional wave body cannot merge a value that differs from the skipped predicate on inactive lanes", [&] { return !optionalPredicateBody(true); });
     passed &= run("an optional wave body preserves inactive values consumed only within a lane", [&] { return optionalPixelBody(false); });
     passed &= run("an optional wave body cannot change helper coordinates for later implicit samples", [&] { return !optionalPixelBody(true); });
     passed &= run("an optional wave body cannot change helper coordinates for LOD queries", [&] { return !optionalPixelBody(true, IrOpcode::ImageQueryLod); });
 
+    const auto optionalPixelRegion = [&](bool unmaskedArm) {
+        Builder b;
+        b.program.Resources().stage = IrShaderStage::Pixel;
+        auto& entry = *b.block;
+        auto& predicate = pixelPredicate(b);
+        auto& zero = b.constant(0u);
+        auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate});
+        auto& body = b.newBlock();
+        auto& left = b.newBlock();
+        auto& right = b.newBlock();
+        auto& join = b.newBlock();
+        auto& merge = b.newBlock();
+        entry.AddBranch(&body);
+        entry.AddBranch(&merge);
+        body.AddBranch(&left);
+        body.AddBranch(&right);
+        left.AddBranch(&join);
+        right.AddBranch(&join);
+        join.AddBranch(&merge);
+        auto& entryInfo = b.program.Metadata().blockInfo[0u];
+        entryInfo.terminator.kind = TerminatorKind::ConditionalBranch;
+        entryInfo.terminator.condition = BranchCondition::ExecZero;
+        entryInfo.terminator.trueBlock = merge.Id();
+        entryInfo.terminator.falseBlock = body.Id();
+        entryInfo.condition = &condition;
+        b.block = &body;
+        auto& uniformBit = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.emit(IrOpcode::GetUserData, IrType::U32, {}), &zero});
+        auto& bodyInfo = b.program.Metadata().blockInfo[1u];
+        bodyInfo.terminator.kind = TerminatorKind::ConditionalBranch;
+        bodyInfo.terminator.condition = BranchCondition::SccNonZero;
+        bodyInfo.terminator.trueBlock = left.Id();
+        bodyInfo.terminator.falseBlock = right.Id();
+        bodyInfo.condition = &uniformBit;
+        for (const unsigned index : {2u, 3u, 4u}) {
+            b.program.Metadata().blockInfo[index].terminator.kind = TerminatorKind::Branch;
+            b.program.Metadata().blockInfo[index].terminator.trueBlock = index == 4u ? merge.Id() : join.Id();
+        }
+        b.block = &left;
+        auto& leftValue = b.select(predicate, b.constant(42u), zero);
+        b.block = &right;
+        auto& rightValue = unmaskedArm ? b.constant(7u) : b.select(predicate, b.constant(43u), zero);
+        b.block = &join;
+        auto& joined = b.phi(IrType::U32, left, leftValue);
+        joined.AddPhiOperand(&right, &rightValue);
+        b.block = &merge;
+        auto& merged = b.phi(IrType::U32, entry, zero);
+        merged.AddPhiOperand(&join, &joined);
+        b.keep(merged);
+        exportUnder(b, b.truth(), true, true);
+        return StencilReplaySafe(b.program);
+    };
+
+    passed &= run("an optional wave body spanning several blocks preserves inactive values through its own phis", [&] { return optionalPixelRegion(false); });
+    passed &= run("an optional wave body spanning several blocks cannot write an inactive lane on any of its paths", [&] { return !optionalPixelRegion(true); });
+
+    const auto nestedOptionalGuard = [&](bool unsafeInner, bool ballotEscape) {
+        Builder b;
+        b.program.Resources().stage = IrShaderStage::Pixel;
+        auto& entry = *b.block;
+        auto& predicate = pixelPredicate(b);
+        auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate});
+        auto& base = helperPredicate(b, false);
+        auto& body = b.newBlock();
+        auto& merge = b.newBlock();
+        auto& inner = b.newBlock();
+        auto& join = b.newBlock();
+        entry.AddBranch(&body);
+        entry.AddBranch(&merge);
+        body.AddBranch(&merge);
+        merge.AddBranch(&inner);
+        merge.AddBranch(&join);
+        inner.AddBranch(&join);
+        auto& entryInfo = b.program.Metadata().blockInfo[0u];
+        entryInfo.terminator.kind = TerminatorKind::ConditionalBranch;
+        entryInfo.terminator.condition = BranchCondition::ExecZero;
+        entryInfo.terminator.trueBlock = merge.Id();
+        entryInfo.terminator.falseBlock = body.Id();
+        entryInfo.condition = &condition;
+        b.program.Metadata().blockInfo[1u].terminator.kind = TerminatorKind::Branch;
+        b.program.Metadata().blockInfo[1u].terminator.trueBlock = merge.Id();
+        b.block = &body;
+        auto& taken = b.emit(IrOpcode::SelectU1, IrType::Bool, {&predicate, &b.truth(), &base});
+        b.block = &merge;
+        auto& merged = b.phi(IrType::Bool, entry, base);
+        merged.AddPhiOperand(&body, &taken);
+        auto& innerCondition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&merged});
+        auto& mergeInfo = b.program.Metadata().blockInfo[2u];
+        mergeInfo.terminator.kind = TerminatorKind::ConditionalBranch;
+        mergeInfo.terminator.condition = BranchCondition::ExecZero;
+        mergeInfo.terminator.trueBlock = join.Id();
+        mergeInfo.terminator.falseBlock = inner.Id();
+        mergeInfo.condition = &innerCondition;
+        b.program.Metadata().blockInfo[3u].terminator.kind = TerminatorKind::Branch;
+        b.program.Metadata().blockInfo[3u].terminator.trueBlock = join.Id();
+        b.block = &inner;
+        auto& written = b.select(merged, b.constant(42u), b.constant(0u));
+        if (unsafeInner) b.emit(IrOpcode::Discard, IrType::Void, {});
+        b.block = &join;
+        auto& joined = b.phi(IrType::U32, merge, b.constant(0u));
+        joined.AddPhiOperand(&inner, &written);
+        if (ballotEscape) {
+            const auto [low, high] = b.ballotWords(merged);
+            b.keep(b.threadBit(*low, *high));
+            b.keep(b.add(*low, 1u));
+        }
+        b.keep(joined);
+        exportUnder(b, b.truth(), true, true);
+        return StencilReplaySafe(b.program);
+    };
+
+    passed &= run("an optional wave body merge may guard another optional wave body", [&] { return nestedOptionalGuard(false, false); });
+    passed &= run("an optional wave body merge cannot guard a body with side effects", [&] { return !nestedOptionalGuard(true, false); });
+    passed &= run("an optional wave body merge cannot feed a ballot escaping the lane's own bit", [&] { return !nestedOptionalGuard(false, true); });
+
+    passed &= run("an optional wave body merge may feed an own-lane ballot read", [&] {
+        Builder b;
+        b.program.Resources().stage = IrShaderStage::Pixel;
+        auto& entry = *b.block;
+        auto& predicate = pixelPredicate(b);
+        auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate});
+        auto& base = helperPredicate(b, false);
+        auto& body = b.newBlock();
+        auto& merge = b.newBlock();
+        entry.AddBranch(&body);
+        entry.AddBranch(&merge);
+        body.AddBranch(&merge);
+        auto& entryInfo = b.program.Metadata().blockInfo[0u];
+        entryInfo.terminator.kind = TerminatorKind::ConditionalBranch;
+        entryInfo.terminator.condition = BranchCondition::ExecZero;
+        entryInfo.terminator.trueBlock = merge.Id();
+        entryInfo.terminator.falseBlock = body.Id();
+        entryInfo.condition = &condition;
+        b.program.Metadata().blockInfo[1u].terminator.kind = TerminatorKind::Branch;
+        b.program.Metadata().blockInfo[1u].terminator.trueBlock = merge.Id();
+        b.block = &body;
+        auto& taken = b.emit(IrOpcode::SelectU1, IrType::Bool, {&predicate, &b.truth(), &base});
+        b.block = &merge;
+        auto& merged = b.phi(IrType::Bool, entry, base);
+        merged.AddPhiOperand(&body, &taken);
+        b.keep(b.ballotBit(merged));
+        exportUnder(b, b.truth(), true, true);
+        return StencilReplaySafe(b.program);
+    });
+
+    passed &= run("an active mask may flag a sample with an explicit lod", [&] {
+        Builder b;
+        b.program.Resources().stage = IrShaderStage::Pixel;
+        MemoryInfo memory{};
+        memory.kind = ResourceKind::Image;
+        memory.imageSampleFlags = RdnaImageSampleFlagLevelZero;
+        b.program.Resources().memoryInfo.push_back(memory);
+        auto& active = b.ballotBit(b.truth());
+        auto& address = b.emit(IrOpcode::MakeImageAddress, IrType::ImageAddress, {&b.constant(0u)});
+        auto& image = b.emit(IrOpcode::GetImageResource, IrType::ImageResource, {});
+        auto& sampler = b.emit(IrOpcode::GetSamplerResource, IrType::SamplerResource, {});
+        MemoryFlags flags{0u, 0u};
+        std::uint64_t bits = 0;
+        std::memcpy(&bits, &flags, sizeof(flags));
+        b.emit(IrOpcode::ImageSampleRaw, IrType::U32x4, {&image, &sampler, &address, &active}, bits);
+        exportUnder(b, b.truth(), true, true);
+        return StencilReplaySafe(b.program);
+    });
+
     passed &= run("varying attribute sources cannot be treated as uniform any-active predicates", [&] {
         for (const auto opcode : {IrOpcode::GetAttribute, IrOpcode::GetInterpolationParameter, IrOpcode::GetInterpolationParameterF16, IrOpcode::GetTessellationAttribute}) {
             Builder b;

core/libs/prx/libSceAgcDriver/tests/PerformanceOverlay.cpp

@@ -2,7 +2,9 @@
 #include "prx/libSceAgcDriver/Execution/include/PresentationPass.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ToneMapLuminance.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
+#include "prx/libc/include/General.hpp"
 #include <SDL_loadso.h>
 #include <algorithm>
 #include <array>
@@ -18,6 +20,7 @@
 #include <span>
 #include <string>
 #include <string_view>
+#include <thread>
 #include <vector>
 
 namespace {
@@ -533,6 +536,206 @@ void fullPanelTest(const Device& device, PerformanceOverlay& overlay) {
     for (const auto& line : content.below) std::printf("  %s\n", line.c_str());
 }
 
+AgcDriver::OverlayDetailSample pageTwoSample() {
+    AgcDriver::OverlayDetailSample sample;
+    sample.fps = 59.94;
+    sample.frameMs = 16.7;
+    sample.busy = {{"Q0", 63}, {"Q20", 4}, {"COMMIT", 12}, {"FINISH", 3}, {"PRESENT", 6}};
+    sample.phases = {{"Driver.Packet", 3.2}, {"Vulkan.GraphicsPipeline.pipeline_create", 1.5}, {"Recorder.Sync", 0.4}};
+    sample.queueWaits = AgcDriver::OverlayQueueWaits{0.4, 0.2, 1.3};
+    sample.renderPassesPerFrame = 12;
+    sample.pipelinesCompiling = 3;
+    sample.pipelinesPerSecond = 12;
+    sample.drawCacheHitPercent = 97.1;
+    sample.pipelineCacheHitPercent = 99.9;
+    sample.flipsPerSecond = 60;
+    sample.presentsPerSecond = 59.94;
+    sample.droppedFrames = 0;
+    sample.repeatedPresents = 0;
+    sample.commit = AgcDriver::OverlayMemory{12ull << 30u, 31ull << 30u};
+    sample.pageFaultsPerSecond = 1234;
+    sample.hostImports = 123;
+    sample.hostImportBytes = AgcDriver::OverlayMemory{1ull << 30u, 6ull << 30u};
+    sample.cpuCopyMegabytesPerSecond = 12.3;
+    sample.addressCopyCapPercent = 3;
+    sample.unimplementedPerSecond = 0;
+    sample.unimplementedTotal = 4;
+    sample.refusedShaders = 2;
+    sample.title = "PPSA01234";
+    sample.presentMode = "fifo";
+    return sample;
+}
+
+void pageTwoTest(const Context& context, PerformanceOverlay& overlay) {
+    const VkExtent2D extent{1280, 720};
+    VkClearColorValue clear{};
+    clear.float32[0] = 0.25f;
+    clear.float32[1] = 0.5f;
+    clear.float32[2] = 0.75f;
+    clear.float32[3] = 1.0f;
+    const auto content = AgcDriver::ComposeOverlayDetails(pageTwoSample());
+    OverlayRect bounds{};
+    const auto reference = clearAndOverlay(context, nullptr, VK_FORMAT_B8G8R8A8_UNORM, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, extent, clear, content, OverlayCorner::TopRight, nullptr);
+    const auto drawn = clearAndOverlay(context, &overlay, VK_FORMAT_B8G8R8A8_UNORM, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, extent, clear, content, OverlayCorner::TopRight, &bounds);
+    const auto layout = AgcDriver::LayOutOverlay(content, extent, OverlayCorner::TopRight);
+    Require(bounds.x == layout.bounds.x && bounds.y == layout.bounds.y && bounds.width == layout.bounds.width && bounds.height == layout.bounds.height, "page 2 is not drawn where its layout puts it");
+    onlyInside(reference, drawn, extent, bounds, "page 2 at 1280x720");
+    std::int32_t row = OverlayMetrics::Padding;
+    for (const auto& line : content.below) {
+        requireTextDrawn(drawn, extent, bounds, layout.scale, reference.front(), line, row);
+        row += OverlayMetrics::LinePitch;
+    }
+    std::printf("page 2 at 1280x720: %dx%d at %d,%d (scale %d), every line drawn bit for bit, nothing changed outside it:\n", bounds.width, bounds.height, bounds.x, bounds.y, layout.scale);
+    for (const auto& line : content.below) std::printf("  |%-46s|\n", line.c_str());
+}
+
+void pageCycleTest(const Device& device) {
+    const auto& context = device.GetContext();
+    const VkExtent2D extent{1280, 720};
+    Require(PerformanceOverlay::Page() == 0 && !PerformanceOverlay::Visible(), "the performance overlay does not start hidden without ANYPS5_OVERLAY");
+    PerformanceOverlay overlay(context, 1);
+    const auto start = AgcDriver::OverlayStatistics::Clock::now();
+    const auto present = [&](int frame) { overlay.NotePresent(start + std::chrono::milliseconds(16) * frame, VkExtent2D{640, 360}); };
+    const auto record = [&] {
+        Image destination(context, VK_FORMAT_B8G8R8A8_UNORM, extent, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT);
+        OverlayRect bounds{};
+        CommandBatch batch(context);
+        const auto commands = batch.Handle();
+        transition(context, commands, destination.Handle(), VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT);
+        bounds = overlay.Record(commands, 0, destination.Handle(), VK_FORMAT_B8G8R8A8_UNORM, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, extent, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
+        batch.SubmitAndWait();
+        overlay.Release(0);
+        return bounds;
+    };
+    PerformanceOverlay::Toggle();
+    Require(PerformanceOverlay::Page() == 1 && PerformanceOverlay::Visible(), "the first F10 does not show page 1");
+    for (int frame = 0; frame < 5; ++frame) present(frame);
+    Require(overlay.ShownPage() == 1 && overlay.Statistics().FrameTimes().size() == 4, "page 1 did not keep its frames");
+    const auto pageOne = record();
+    PerformanceOverlay::Toggle();
+    Require(PerformanceOverlay::Page() == 2 && PerformanceOverlay::Visible(), "the second F10 does not show page 2");
+    present(5);
+    Require(overlay.ShownPage() == 2 && overlay.Statistics().FrameTimes().empty(), "changing to page 2 kept page 1's statistics");
+    for (int frame = 6; frame < 10; ++frame) present(frame);
+    const auto& sample = overlay.DetailSample();
+    Require(std::abs(sample.frameMs - 16.0) < 1e-6 && sample.presentsPerSecond.has_value() && sample.pipelinesPerSecond.has_value(), "page 2 did not sample its frames");
+    const auto pageTwo = record();
+    const auto expected = AgcDriver::LayOutOverlay(AgcDriver::ComposeOverlayDetails(sample), extent, PerformanceOverlay::Corner()).bounds;
+    Require(pageTwo.width == expected.width && pageTwo.height == expected.height && pageTwo.width > pageOne.width, "page 2 is not drawn from its own composer: " + std::to_string(pageTwo.width) + "x" + std::to_string(pageTwo.height));
+    PerformanceOverlay::Toggle();
+    Require(PerformanceOverlay::Page() == 0 && !PerformanceOverlay::Visible(), "the third F10 does not hide the overlay");
+    present(10);
+    Require(overlay.ShownPage() == 0 && overlay.Statistics().FrameTimes().empty() && overlay.DetailSample().fps == 0, "hiding the overlay kept page 2's state");
+    std::printf("F10 cycle: off, page 1 (%dx%d), page 2 (%dx%d), off; each change of page resets the statistics\n", pageOne.width, pageOne.height, pageTwo.width, pageTwo.height);
+}
+
+AgcDriver::OverlayDriverCounters fakeDriverCounters() {
+    static std::uint64_t calls = 0;
+    ++calls;
+    return AgcDriver::OverlayDriverCounters{calls * 66, AgcDriver::OverlayHits{calls * 1000, calls * 900}};
+}
+
+void pageTwoSamplerTest(const Device& device) {
+    using AgcDriver::PerformanceTrace;
+    using namespace std::chrono_literals;
+    const auto& context = device.GetContext();
+    const VkExtent2D extent{1280, 720};
+    Require(PerformanceOverlay::Page() == 0 && !PerformanceTrace::Recording(), "the trace records before page 2 is shown");
+    PerformanceOverlay overlay(context, 1);
+    PerformanceOverlay::SetDriverCounters(&fakeDriverCounters);
+    PerformanceOverlay::Toggle();
+    Require(!PerformanceTrace::Recording(), "page 1 turns the trace on");
+    PerformanceOverlay::Toggle();
+    Require(PerformanceOverlay::Page() == 2 && PerformanceTrace::Recording() && PerformanceTrace::OverlayTotals(), "page 2 does not turn the trace totals on");
+    const auto start = AgcDriver::OverlayStatistics::Clock::now();
+    const auto present = [&](std::chrono::milliseconds at, std::uint64_t address, std::uint64_t drops) { overlay.NotePresent(start + at, VkExtent2D{640, 360}, AgcDriver::OverlayPresentInfo{address, VK_PRESENT_MODE_FIFO_KHR, drops}); };
+    present(0ms, 0x1000, 7);
+    const auto& sample = overlay.DetailSample();
+    Require(sample.commit.has_value() && sample.unimplementedTotal.has_value() && sample.presentMode == "fifo" && sample.pipelinesCompiling == std::optional<std::uint64_t>{0}, "page 2's first sample does not read the commit charge, the unimplemented calls, the compile backlog and the present mode");
+    Require(!sample.flipsPerSecond && !sample.pageFaultsPerSecond && !sample.cpuCopyMegabytesPerSecond && sample.busy.empty() && sample.phases.empty(), "page 2's first sample shows rates before a second has passed");
+    Require(sample.hostImports.has_value() == (context.hostImportAlignment != 0), "the host import line does not follow the device's host import support");
+    std::thread([] {
+        PerformanceTrace::BindThread("queue worker 0x0");
+        if (auto* trace = PerformanceTrace::Thread()) {
+            const auto begin = std::chrono::steady_clock::now();
+            trace->Add("Driver.Packet", nullptr, begin, begin + 110ms);
+        }
+        for (int pass = 0; pass < 6; ++pass) PerformanceTrace::Count(PerformanceTrace::RenderPassScope);
+        AgcDriver::Graphics::AddressCopyCounters::ThisThread().Count(std::uint64_t{11} << 20u);
+    }).join();
+    try {
+        NotImplemented_nid_no_patch("page 2 sampler test");
+    } catch (const std::runtime_error&) {
+    }
+    present(16ms, 0x1000, 7);
+    present(33ms, 0x2000, 8);
+    present(1100ms, 0x2000, 9);
+    const auto near = [](const std::optional<double>& value, double expected) { return value.has_value() && std::abs(*value - expected) < 1e-6; };
+    Require(near(sample.flipsPerSecond, 60) && near(sample.drawCacheHitPercent, 90), "66 flips and 900 of 1000 draw-cache lookups over 1.1 s are not 60 flips/s and 90 %");
+    Require(sample.droppedFrames == std::optional<std::uint64_t>{2} && sample.repeatedPresents == std::optional<std::uint64_t>{2}, "2 dropped and 2 repeated presents over 1.1 s are not shown as 2/s each");
+    Require(sample.pageFaultsPerSecond.has_value() && near(sample.unimplementedPerSecond, 1 / 1.1), "the page faults or the unimplemented calls per second are missing");
+    Require(near(sample.cpuCopyMegabytesPerSecond, 10), "11 MiB of address copies over 1.1 s are not 10 MB/s");
+    if (std::getenv("APS5_ADDRESS_COPY_MAX_MIB") == nullptr) Require(near(sample.addressCopyCapPercent, 100.0 * 11 / 64), "an 11 MiB build is not 17.2 % of the 64 MiB cap");
+    Require(near(sample.renderPassesPerFrame, 2), "6 render passes over 3 frames are not 2 per frame");
+    const auto queue = std::find_if(sample.busy.begin(), sample.busy.end(), [](const AgcDriver::OverlayThreadBusy& busy) { return busy.name == "Q0"; });
+    Require(queue != sample.busy.end() && std::abs(queue->percent - 10) < 1e-6 && sample.queueWaits.has_value(), "110 ms of packets over 1.1 s do not make queue 0 10 % busy");
+    Require(!sample.phases.empty() && sample.phases.front().name == "Driver.Packet" && std::abs(sample.phases.front().msPerFrame - 110.0 / 3) < 1e-6, "the top phase is not 110 ms of packets over 3 frames");
+    Image destination(context, VK_FORMAT_B8G8R8A8_UNORM, extent, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT);
+    CommandBatch batch(context);
+    const auto commands = batch.Handle();
+    transition(context, commands, destination.Handle(), VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT);
+    const auto bounds = overlay.Record(commands, 0, destination.Handle(), VK_FORMAT_B8G8R8A8_UNORM, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, extent, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
+    batch.SubmitAndWait();
+    overlay.Release(0);
+    const auto content = AgcDriver::ComposeOverlayDetails(sample);
+    const auto expected = AgcDriver::LayOutOverlay(content, extent, PerformanceOverlay::Corner()).bounds;
+    Require(bounds.width == expected.width && bounds.height == expected.height, "page 2 is not drawn from its sample");
+    std::printf("page 2 from its sampler (fake driver counters, a 'queue worker 0x0' thread, this process's OS counters):\n");
+    for (const auto& line : content.below) std::printf("  |%-46s|\n", line.c_str());
+    PerformanceOverlay::Toggle();
+    Require(PerformanceOverlay::Page() == 0 && !PerformanceTrace::Recording() && !PerformanceTrace::OverlayTotals(), "hiding the overlay leaves the trace totals on");
+    present(1200ms, 0x3000, 9);
+    Require(overlay.ShownPage() == 0 && overlay.DetailSample().fps == 0 && overlay.DetailSample().busy.empty() && !overlay.DetailSample().commit, "hiding the overlay kept page 2's sample");
+    PerformanceOverlay::SetDriverCounters(nullptr);
+}
+
+void presentCostReport(const Device& device) {
+    using AgcDriver::PerformanceCounters;
+    using AgcDriver::PerformanceTrace;
+    using Clock = AgcDriver::OverlayStatistics::Clock;
+    const auto& context = device.GetContext();
+    const VkExtent2D extent{1920, 1080};
+    Require(PerformanceOverlay::Page() == 0, "the cost report does not start hidden");
+    PerformanceOverlay overlay(context, 1);
+    const auto start = Clock::now();
+    const auto perCall = [](int calls, auto&& body) {
+        const auto began = std::chrono::steady_clock::now();
+        for (int call = 0; call < calls; ++call) body(call);
+        return std::chrono::duration<double, std::nano>(std::chrono::steady_clock::now() - began).count() / calls;
+    };
+    volatile bool shown = false;
+    const auto hiddenNs = perCall(10'000'000, [&](int) { shown = PerformanceOverlay::Visible(); });
+    const auto passNs = perCall(10'000'000, [](int) { PerformanceTrace::Count(PerformanceTrace::RenderPassScope); });
+    auto& copies = AgcDriver::Graphics::AddressCopyCounters::ThisThread();
+    const auto copyNs = perCall(10'000'000, [&](int) { copies.Count(4096); });
+    PerformanceOverlay::Toggle();
+    const auto pageOneNs = perCall(20'000, [&](int frame) {
+        overlay.NotePresent(start + std::chrono::microseconds(16667) * frame, extent);
+        static_cast<void>(AgcDriver::LayOutOverlay(AgcDriver::ComposeOverlay(overlay.Statistics(), PerformanceCounters::GpuEnabled(), PerformanceCounters::DriverEnabled(), overlay.Details()), extent, PerformanceOverlay::Corner()));
+    });
+    PerformanceOverlay::Toggle();
+    const auto later = start + std::chrono::hours(1);
+    const auto pageTwoNs = perCall(20'000, [&](int frame) {
+        overlay.NotePresent(later + std::chrono::microseconds(16667) * frame, extent, AgcDriver::OverlayPresentInfo{0x1000u + static_cast<std::uint64_t>(frame % 2), VK_PRESENT_MODE_FIFO_KHR, 0});
+        static_cast<void>(AgcDriver::LayOutOverlay(AgcDriver::ComposeOverlayDetails(overlay.DetailSample()), extent, PerformanceOverlay::Corner()));
+    });
+    const auto sampled = later + std::chrono::hours(1);
+    const auto sampleNs = perCall(200, [&](int second) { overlay.NotePresent(sampled + std::chrono::seconds(second), extent, AgcDriver::OverlayPresentInfo{0x1000, VK_PRESENT_MODE_FIFO_KHR, 0}); });
+    PerformanceOverlay::Toggle();
+    Require(!PerformanceOverlay::Visible() && !PerformanceTrace::Recording(), "the cost report does not end hidden");
+    std::printf("present cost on this host: hidden %.1f ns per present (Visible), %.1f ns per render pass begun (the trace check), %.1f ns per address-copying build (the per-thread counter); page 1 %.2f us per present (NotePresent, compose, layout); page 2 %.2f us per present at 60 fps (NotePresent with its once-a-second sample, compose, layout), %.1f us per present that samples\n", hiddenNs, passNs, copyNs, pageOneNs / 1000, pageTwoNs / 1000, sampleNs / 1000);
+}
+
 void notePresentTest(const Device& device) {
     const auto& context = device.GetContext();
     const VkExtent2D extent{1280, 720};
@@ -646,7 +849,10 @@ void disabledIsUnchangedTest(const Context& context) {
     onlyInside(drawnBefore, presenter.Drawn(&bounds), Presenter::Extent, bounds, "presentation pass with the overlay", false);
     onlyInside(blittedBefore, presenter.Blitted(&bounds), Presenter::Extent, bounds, "blit with the overlay", false);
     PerformanceOverlay::Toggle();
-    Require(!PerformanceOverlay::Visible(), "F10 does not hide the performance overlay again");
+    Require(PerformanceOverlay::Visible() && PerformanceOverlay::Page() == 2, "a second F10 does not show page 2");
+    onlyInside(drawnBefore, presenter.Drawn(&bounds), Presenter::Extent, bounds, "presentation pass with page 2", false);
+    PerformanceOverlay::Toggle();
+    Require(!PerformanceOverlay::Visible(), "a third F10 does not hide the performance overlay again");
     Require(presenter.Drawn(&unused) == drawnBefore, "the presentation pass output differs after the overlay was hidden");
     Require(presenter.Blitted(&unused) == blittedBefore, "the blit output differs after the overlay was hidden");
     std::printf("overlay hidden: presentation pass and blit output bit-identical before and after showing it\n");
@@ -725,9 +931,13 @@ int main(int argc, char** argv) {
             hdrOverlayTest(context, overlay);
             cornerTest(context, overlay);
             fullPanelTest(*device, overlay);
+            pageTwoTest(context, overlay);
             costReport(*device, overlay);
         }
         notePresentTest(*device);
+        pageCycleTest(*device);
+        pageTwoSamplerTest(*device);
+        presentCostReport(*device);
         disabledIsUnchangedTest(context);
         std::cout << "performance overlay tests passed\n";
         return 0;

core/libs/prx/libSceAgcDriver/tests/PerformanceOverlayStatistics.cpp

@@ -1,4 +1,8 @@
 #include "prx/libSceAgcDriver/Execution/include/PerformanceOverlay.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libc/include/General.hpp"
 #include <algorithm>
 #include <chrono>
 #include <cmath>
@@ -10,6 +14,7 @@
 #include <optional>
 #include <stdexcept>
 #include <string>
+#include <string_view>
 #include <thread>
 #include <vector>
 
@@ -161,9 +166,10 @@ void lowRateTests() {
 
 void settingsTests() {
     const auto hidden = AgcDriver::ParseOverlaySettings(nullptr, nullptr);
-    require(!hidden.visible && hidden.corner == OverlayCorner::TopLeft, "the overlay is not off and top-left by default");
-    require(!AgcDriver::ParseOverlaySettings("", "").visible && !AgcDriver::ParseOverlaySettings("0", nullptr).visible, "an empty or 0 ANYPS5_OVERLAY shows the overlay");
-    require(AgcDriver::ParseOverlaySettings("1", nullptr).visible, "ANYPS5_OVERLAY=1 does not show the overlay");
+    require(hidden.page == 0 && hidden.corner == OverlayCorner::TopLeft, "the overlay is not off and top-left by default");
+    require(AgcDriver::ParseOverlaySettings("", "").page == 0 && AgcDriver::ParseOverlaySettings("0", nullptr).page == 0, "an empty or 0 ANYPS5_OVERLAY shows the overlay");
+    require(AgcDriver::ParseOverlaySettings("1", nullptr).page == 1, "ANYPS5_OVERLAY=1 does not start on page 1");
+    require(AgcDriver::ParseOverlaySettings("2", "br").page == 2 && AgcDriver::ParseOverlaySettings("2", "br").corner == OverlayCorner::BottomRight, "ANYPS5_OVERLAY=2 does not start on page 2 in the chosen corner");
     const std::pair<const char*, OverlayCorner> corners[] = {{"tl", OverlayCorner::TopLeft}, {"tr", OverlayCorner::TopRight}, {"bl", OverlayCorner::BottomLeft}, {"br", OverlayCorner::BottomRight}};
     for (const auto& [name, corner] : corners) require(AgcDriver::ParseOverlaySettings("1", name).corner == corner, std::string("ANYPS5_OVERLAY_POS=") + name + " picks another corner");
     const auto refused = [](const char* enabled, const char* position, const char* variable) {
@@ -174,7 +180,12 @@ void settingsTests() {
         }
         return false;
     };
-    require(refused("yes", nullptr, "ANYPS5_OVERLAY") && refused("2", nullptr, "ANYPS5_OVERLAY"), "an ANYPS5_OVERLAY value other than 1 or 0 was accepted or not named");
+    require(refused("yes", nullptr, "ANYPS5_OVERLAY") && refused("3", nullptr, "ANYPS5_OVERLAY") && refused("02", nullptr, "ANYPS5_OVERLAY") && refused("-1", nullptr, "ANYPS5_OVERLAY"), "an ANYPS5_OVERLAY value other than 0, 1 or 2 was accepted or not named");
+    try {
+        static_cast<void>(AgcDriver::ParseOverlaySettings("3", nullptr));
+    } catch (const std::exception& error) {
+        require(std::string(error.what()).find("0, 1 or 2") != std::string::npos, std::string("the ANYPS5_OVERLAY error does not list 0, 1 and 2: ") + error.what());
+    }
     require(refused("1", "center", "ANYPS5_OVERLAY_POS") && refused(nullptr, "TL", "ANYPS5_OVERLAY_POS"), "an unknown ANYPS5_OVERLAY_POS was accepted or not named");
 }
 
@@ -431,6 +442,211 @@ void fullPanelLayoutTests() {
     }
 }
 
+void fontTests() {
+    const auto characters = AgcDriver::OverlayFontCharacters();
+    for (const char character : characters) {
+        const auto& glyph = AgcDriver::OverlayGlyph(character);
+        require(std::all_of(glyph.begin(), glyph.end(), [](std::uint8_t row) { return row <= 0x1f; }), std::string("the glyph of '") + character + "' is wider than 5 pixels");
+        require(character == ' ' || std::any_of(glyph.begin(), glyph.end(), [](std::uint8_t row) { return row != 0; }), std::string("the glyph of '") + character + "' is empty");
+        for (const char other : characters) {
+            if (other != character) require(glyph != AgcDriver::OverlayGlyph(other), std::string("'") + character + "' and '" + other + "' have the same glyph");
+        }
+    }
+    for (const char letter : std::string("JQYZ%+-_")) require(characters.find(letter) != std::string_view::npos, std::string("the font lists no '") + letter + "'");
+    for (const char missing : std::string("qjy(>,#")) {
+        bool refused = false;
+        try {
+            static_cast<void>(AgcDriver::OverlayGlyph(missing));
+        } catch (const std::exception&) {
+            refused = true;
+        }
+        require(refused && characters.find(missing) == std::string_view::npos, std::string("'") + missing + "' is drawn or listed although the font has no glyph for it");
+    }
+}
+
+void widePanelTests() {
+    OverlayContent wide;
+    wide.columns = 48;
+    wide.graph = false;
+    for (int i = 0; i < 20; ++i) wide.below.push_back(std::string(48, i % 2 == 0 ? 'W' : 'Z'));
+    const VkExtent2D small{1280, 720};
+    const auto layout = AgcDriver::LayOutOverlay(wide, small, OverlayCorner::TopLeft);
+    const auto s = layout.scale;
+    require(layout.bounds.width == (48 * OverlayMetrics::Advance - 1 + 2 * OverlayMetrics::Padding) * s, "a 48-column panel is " + std::to_string(layout.bounds.width) + " pixels wide at scale " + std::to_string(s));
+    require(layout.bounds.height == (2 * OverlayMetrics::Padding + 20 * OverlayMetrics::LinePitch - (OverlayMetrics::LinePitch - OverlayMetrics::GlyphHeight)) * s, "a panel without a graph keeps the graph's rows");
+    const auto text = coverage(layout, static_cast<std::uint32_t>(AgcDriver::OverlayColor::Text));
+    for (std::size_t line = 0; line < wide.below.size(); ++line) requireLineDrawn(layout, text, wide.below[line], OverlayMetrics::Padding + static_cast<std::int32_t>(line) * OverlayMetrics::LinePitch);
+    for (const auto& rect : layout.rects) require(rect.color != static_cast<std::uint32_t>(AgcDriver::OverlayColor::Graph), "a panel without a graph draws graph bars");
+    const std::pair<VkExtent2D, std::int32_t> scales[] = {{{1280, 720}, 2}, {{1920, 1080}, 2}, {{3840, 2160}, 5}, {{320, 180}, 1}};
+    for (const auto& [extent, scale] : scales) require(AgcDriver::OverlayScale(extent, wide) == scale, "a 48-column panel at " + std::to_string(extent.width) + "x" + std::to_string(extent.height) + " is drawn at scale " + std::to_string(AgcDriver::OverlayScale(extent, wide)) + ", not " + std::to_string(scale));
+    auto widest = wide;
+    widest.columns = 140;
+    require(AgcDriver::OverlayScale({3840, 2160}, widest) == 4, "a 140-column panel at 3840x2160 is not shrunk from scale 5 to 4");
+    const auto refused = [](const OverlayContent& content) {
+        try {
+            static_cast<void>(AgcDriver::LayOutOverlay(content, {1920, 1080}, OverlayCorner::TopLeft));
+        } catch (const std::exception&) {
+            return true;
+        }
+        return false;
+    };
+    auto longer = wide;
+    longer.below.push_back(std::string(49, 'W'));
+    require(refused(longer), "a line longer than the panel's columns was drawn");
+    auto graphless = wide;
+    graphless.frameTimes = {16.0};
+    require(refused(graphless), "frame times were laid out on a panel without a graph");
+    OverlayContent cramped{{"FPS: 60.0"}, {16.0}, {}};
+    cramped.columns = 20;
+    require(refused(cramped), "a graph of 120 frames was laid out on a 20-column panel");
+    cramped.columns = 21;
+    require(!refused(cramped), "a 21-column panel cannot hold the graph");
+}
+
+void pageCycleTests() {
+    for (std::uint32_t start = 0; start < AgcDriver::OverlayPageCount; ++start) {
+        for (std::uint64_t presses = 0; presses <= 3; ++presses) {
+            const auto page = AgcDriver::OverlayPageAfter(start, presses);
+            require(page == (start + presses) % 3, "starting on page " + std::to_string(start) + ", " + std::to_string(presses) + " presses of F10 show page " + std::to_string(page));
+        }
+    }
+    require(AgcDriver::OverlayPageAfter(2, ~std::uint64_t{0}) == 2 && AgcDriver::OverlayPageAfter(1, ~std::uint64_t{0} - 1) == 0, "the page wraps wrongly after 2^64 presses");
+    bool refused = false;
+    try {
+        static_cast<void>(AgcDriver::OverlayPageAfter(3, 0));
+    } catch (const std::exception&) {
+        refused = true;
+    }
+    require(refused, "a starting page 3 was accepted");
+}
+
+AgcDriver::OverlayDetailSample fullDetailSample() {
+    AgcDriver::OverlayDetailSample sample;
+    sample.fps = 59.94;
+    sample.frameMs = 16.7;
+    sample.busy = {{"Q0", 63.4}, {"Q20", 4.2}, {"COMMIT", 12}, {"FINISH", 3}, {"PRESENT", 5.5}, {"Q21", 100}, {"Q22", 0}};
+    sample.phases = {{"Driver.Packet", 3.21}, {"Vulkan.GraphicsPipelineLibrary.shader_compile_and_link", 12.5}, {"Recorder.Sync", 0.4}, {"GpuMutex.wait.draw", 0.06}, {"Driver.Draw.capture_resources", 0.9}};
+    sample.queueWaits = AgcDriver::OverlayQueueWaits{0.4, 0.2, 1.3};
+    sample.renderPassesPerFrame = 12;
+    sample.pipelinesCompiling = 3;
+    sample.pipelinesPerSecond = 12;
+    sample.drawCacheHitPercent = 97.1;
+    sample.pipelineCacheHitPercent = 99.9;
+    sample.flipsPerSecond = 60;
+    sample.presentsPerSecond = 59.94;
+    sample.droppedFrames = 1;
+    sample.repeatedPresents = 2;
+    sample.commit = AgcDriver::OverlayMemory{static_cast<std::uint64_t>(12.3 * GiB), static_cast<std::uint64_t>(31.9 * GiB)};
+    sample.pageFaultsPerSecond = 1234;
+    sample.hostImports = 123;
+    sample.hostImportBytes = AgcDriver::OverlayMemory{static_cast<std::uint64_t>(1.2 * GiB), 6 * GiB};
+    sample.cpuCopyMegabytesPerSecond = 12.3;
+    sample.addressCopyCapPercent = 3;
+    sample.unimplementedPerSecond = 0;
+    sample.unimplementedTotal = 4;
+    sample.refusedShaders = 2;
+    sample.title = "PPSA01234";
+    sample.presentMode = "fifo_latest_ready";
+    return sample;
+}
+
+void requireDrawable(const OverlayContent& content, const std::string& what) {
+    const auto font = AgcDriver::OverlayFontCharacters();
+    for (const auto* lines : {&content.above, &content.below}) {
+        for (const auto& line : *lines) {
+            require(line.size() <= static_cast<std::size_t>(content.columns), what + ": '" + line + "' is wider than its " + std::to_string(content.columns) + " columns");
+            for (const char character : line) {
+                require(font.find(character) != std::string_view::npos, what + ": '" + line + "' has '" + std::string(1, character) + "', which the font cannot draw");
+                static_cast<void>(AgcDriver::OverlayGlyph(character));
+            }
+        }
+    }
+}
+
+void pageTwoComposeTests() {
+    const auto full = AgcDriver::ComposeOverlayDetails(fullDetailSample());
+    require(full.columns == OverlayMetrics::DetailColumns && !full.graph && full.above.empty() && full.frameTimes.empty(), "page 2 is not a 46-column panel without a graph");
+    const auto phase = [](const std::string& name, const std::string& amount) { return name + std::string(46 - name.size() - amount.size(), ' ') + amount; };
+    const std::vector<std::string> expected{
+        "FPS: 59.9  FRAME: 16.7 MS",
+        "BUSY %: Q0 63 Q20 4 COMMIT 12 FINISH 3",
+        "        PRESENT 6 Q21 100 Q22 0",
+        phase("DRIVER.PACKET", " 3.2 MS/FRAME"),
+        "VULKAN.GRAPHICSPIPELINELIBRARY.S 12.5 MS/FRAME",
+        phase("RECORDER.SYNC", " 0.4 MS/FRAME"),
+        phase("GPUMUTEX.WAIT.DRAW", " 0.1 MS/FRAME"),
+        phase("DRIVER.DRAW.CAPTURE_RESOURCES", " 0.9 MS/FRAME"),
+        "Q0 WAIT MS/FRAME: MEM 0.4 LOCK 0.2 GPU 1.3",
+        "RENDER PASSES/FRAME: 12.0",
+        "PIPELINES: COMPILING 3  NEW/S 12",
+        "HIT %: DRAW CACHE 97.1  PIPELINES 99.9",
+        "FLIPS/S: 60.0  PRESENTS/S: 59.9",
+        "DROPPED/S: 1  REPEATED/S: 2",
+        "COMMIT: 12.3/31.9 GB  FAULTS/S: 1234",
+        "HOST IMPORTS: 123  1.2/6.0 GB",
+        "CPU COPIES: 12.3 MB/S  CAP: 3.0%",
+        "UNIMPLEMENTED: 0/S  4 TOTAL",
+        "REFUSED SHADERS: 2",
+        "TITLE: PPSA01234  PRESENT: FIFO_LATEST_READY",
+    };
+    require(full.below.size() == expected.size(), "page 2 of a full sample has " + std::to_string(full.below.size()) + " lines, not " + std::to_string(expected.size()));
+    for (std::size_t line = 0; line < expected.size(); ++line) require(full.below[line] == expected[line], "page 2 line " + std::to_string(line) + " reads '" + full.below[line] + "', not '" + expected[line] + "'");
+    requireDrawable(full, "page 2 of a full sample");
+
+    const auto empty = AgcDriver::ComposeOverlayDetails({});
+    require(empty.below == std::vector<std::string>{"FPS: 0.0  FRAME: 0.0 MS"}, "page 2 of an empty sample shows " + std::to_string(empty.below.size()) + " lines, the last '" + empty.below.back() + "'");
+
+    auto partial = fullDetailSample();
+    partial.flipsPerSecond.reset();
+    partial.commit.reset();
+    partial.title.clear();
+    partial.repeatedPresents.reset();
+    partial.cpuCopyMegabytesPerSecond.reset();
+    const auto some = AgcDriver::ComposeOverlayDetails(partial);
+    const auto has = [&](const std::string& line) { return std::find(some.below.begin(), some.below.end(), line) != some.below.end(); };
+    require(has("PRESENTS/S: 59.9") && has("DROPPED/S: 1") && has("FAULTS/S: 1234") && has("PRESENT: FIFO_LATEST_READY") && has("CPU COPIES: CAP: 3.0%"), "a line with a missing source does not show the rest of its values alone");
+
+    AgcDriver::OverlayDetailSample odd;
+    odd.busy = {{"queue worker 0x57", 1e9}, {"q(1)", 1}};
+    odd.phases = {{"Weird<name>#with,symbols/and lower case and a very long tail beyond the panel", 1e9}};
+    odd.title = "ab-cd_12";
+    odd.presentMode = "mailbox";
+    const auto sanitized = AgcDriver::ComposeOverlayDetails(odd);
+    requireDrawable(sanitized, "page 2 of names the font cannot draw");
+    require(AgcDriver::OverlayText("abc-XYZ_09(%)é") == "ABC-XYZ_09 %   ", "OverlayText maps '" + AgcDriver::OverlayText("abc-XYZ_09(%)é") + "'");
+    for (const auto& line : sanitized.below) require(line.size() <= 46, "'" + line + "' is wider than page 2");
+
+    OverlayStatistics statistics = countedStatistics();
+    auto details = fullDetails();
+    details.display = {12345, 12345};
+    details.window = {16384, 16384};
+    requireDrawable(AgcDriver::ComposeOverlay(statistics, true, true, details), "page 1 of a full panel");
+}
+
+void pageTwoLayoutTests() {
+    const auto content = AgcDriver::ComposeOverlayDetails(fullDetailSample());
+    const auto lines = static_cast<std::int32_t>(content.below.size());
+    const auto width = 46 * OverlayMetrics::Advance - 1 + 2 * OverlayMetrics::Padding;
+    const auto height = 2 * OverlayMetrics::Padding + lines * OverlayMetrics::LinePitch - (OverlayMetrics::LinePitch - OverlayMetrics::GlyphHeight);
+    require(width + 2 * OverlayMetrics::Margin <= 1280 && height + 2 * OverlayMetrics::Margin <= 720, "page 2 does not fit 1280x720 at scale 1");
+    const std::pair<VkExtent2D, std::int32_t> scales[] = {{{1280, 720}, 2}, {{1920, 1080}, 2}, {{3840, 2160}, 5}, {{640, 360}, 1}};
+    for (const auto& [extent, scale] : scales) {
+        for (const auto corner : {OverlayCorner::TopLeft, OverlayCorner::BottomRight}) {
+            const auto layout = AgcDriver::LayOutOverlay(content, extent, corner);
+            require(layout.scale == scale, "page 2 at " + std::to_string(extent.width) + "x" + std::to_string(extent.height) + " is drawn at scale " + std::to_string(layout.scale) + ", not " + std::to_string(scale));
+            require(layout.bounds.width == width * scale && layout.bounds.height == height * scale, "page 2's panel is " + std::to_string(layout.bounds.width) + "x" + std::to_string(layout.bounds.height) + " at scale " + std::to_string(scale));
+            require(layout.bounds.x >= 0 && layout.bounds.y >= 0 && layout.bounds.x + layout.bounds.width <= static_cast<std::int32_t>(extent.width) && layout.bounds.y + layout.bounds.height <= static_cast<std::int32_t>(extent.height), "page 2 leaves " + std::to_string(extent.width) + "x" + std::to_string(extent.height));
+            for (const auto& rect : layout.rects) require(rect.color != static_cast<std::uint32_t>(AgcDriver::OverlayColor::Graph), "page 2 draws graph bars");
+            if (corner != OverlayCorner::TopLeft) continue;
+            const auto text = coverage(layout, static_cast<std::uint32_t>(AgcDriver::OverlayColor::Text));
+            for (std::size_t line = 0; line < content.below.size(); ++line) requireLineDrawn(layout, text, content.below[line], OverlayMetrics::Padding + static_cast<std::int32_t>(line) * OverlayMetrics::LinePitch);
+        }
+    }
+    auto stretched = content;
+    while (stretched.below.size() < 60) stretched.below.push_back("PIPELINES: COMPILING 3  NEW/S 12");
+    require(AgcDriver::OverlayScale({3840, 2160}, stretched) == 3, "a 60-line page 2 at 3840x2160 is not shrunk to scale 3");
+}
+
 void sampleTests() {
     require(AgcDriver::OverlaySampleDue(std::nullopt, Start), "the first frame is not sampled");
     require(!AgcDriver::OverlaySampleDue(Start, Start + 999ms) && AgcDriver::OverlaySampleDue(Start, Start + 1000ms), "memory is sampled more than once a second, or not after a second");
@@ -446,6 +662,187 @@ void sampleTests() {
     require(touched[Bytes - 1] == 0x5a, "the touched block was not written");
 }
 
+constexpr const char* PacketScope = "Driver.Packet";
+constexpr const char* WorkerWaitScope = "Driver.WorkerWait";
+constexpr const char* LockScope = "GpuMutex.wait";
+constexpr const char* SyncScope = "Recorder.Sync";
+constexpr const char* SerialScope = "Recorder.WaitSerial";
+constexpr const char* WaitScope = "Graphics.Wait";
+constexpr const char* FinishScope = "Driver.DrawFinish";
+constexpr const char* MemoryStage = "memory_wait";
+constexpr const char* RenderingStage = "rendering_wait";
+constexpr const char* DrawSite = "draw";
+constexpr const char* PresentSite = "present";
+constexpr const char* FenceStage = "fence_wait";
+
+AgcDriver::TraceThread::Total total(const char* scope, const char* stage, std::chrono::milliseconds length, std::uint64_t count) {
+    return AgcDriver::TraceThread::Total{scope, stage, std::chrono::duration_cast<Clock::duration>(length), count};
+}
+
+void traceDeltaTests() {
+    using AgcDriver::TraceThreadTotals;
+    const std::vector<TraceThreadTotals> before{
+        {"queue worker 0x0", {total(PacketScope, nullptr, 100ms, 10), total(PacketScope, MemoryStage, 5ms, 1), total(WorkerWaitScope, nullptr, 50ms, 2)}},
+        {"queue worker 0x1", {total(PacketScope, nullptr, 10ms, 1)}},
+    };
+    const std::vector<TraceThreadTotals> now{
+        {"queue worker 0x0", {total(PacketScope, nullptr, 500ms, 60), total(PacketScope, MemoryStage, 35ms, 3), total(PacketScope, RenderingStage, 10ms, 1), total(LockScope, DrawSite, 20ms, 4), total(LockScope, PresentSite, 5ms, 1), total(SyncScope, nullptr, 6ms, 2), total(SerialScope, nullptr, 4ms, 1), total(WaitScope, FenceStage, 5ms, 1), total(WaitScope, nullptr, 9ms, 1), total(WorkerWaitScope, nullptr, 450ms, 40)}},
+        {"queue worker 0x1", {total(PacketScope, nullptr, 110ms, 2)}},
+        {"queue worker 0xa", {total(PacketScope, nullptr, 50ms, 1)}},
+        {"draw committer", {total(AgcDriver::PerformanceTrace::RenderPassScope, nullptr, 0ms, 120), total(FinishScope, nullptr, 80ms, 40), total(SyncScope, nullptr, 7ms, 1)}},
+    };
+    const auto figures = AgcDriver::OverlayTraceDeltas(before, now, 1s, 40, true);
+    require(figures.busy.size() == 3 && figures.busy[0].name == "Q0" && figures.busy[1].name == "Q1" && figures.busy[2].name == "Qa", "the busy queues are not Q0, Q1 and Qa in that order");
+    require(within(figures.busy[0].percent, 40) && within(figures.busy[1].percent, 10) && within(figures.busy[2].percent, 5), "the busy shares over a second are not 40, 10 and 5 %");
+    const std::vector<std::pair<std::string, double>> phases{{"Driver.Packet", 13.75}, {"Driver.DrawFinish", 2}, {"Driver.Packet.memory_wait", 0.75}, {"GpuMutex.wait.draw", 0.5}, {"Recorder.Sync", 0.325}};
+    require(figures.phases.size() == phases.size(), "page 2 does not keep the top 5 phases");
+    for (std::size_t index = 0; index < phases.size(); ++index) require(figures.phases[index].name == phases[index].first && within(figures.phases[index].msPerFrame, phases[index].second), "phase " + std::to_string(index) + " is " + figures.phases[index].name + " at " + std::to_string(figures.phases[index].msPerFrame) + " ms per frame, not " + phases[index].first);
+    require(figures.queueWaits && within(figures.queueWaits->memoryMs, 1.0) && within(figures.queueWaits->lockMs, 0.625) && within(figures.queueWaits->gpuMs, 0.375), "queue 0's waits per frame are not 1 ms on memory, 0.625 ms on the lock and 0.375 ms on the GPU");
+    require(figures.renderPassesPerFrame && within(*figures.renderPassesPerFrame, 3), "120 render passes over 40 frames are not 3 per frame");
+    require(!AgcDriver::OverlayTraceDeltas(before, now, 1s, 40, false).renderPassesPerFrame, "render passes are shown while they are not counted");
+    const auto noFrames = AgcDriver::OverlayTraceDeltas(before, now, 1s, 0, true);
+    require(noFrames.busy.size() == 3 && noFrames.phases.empty() && !noFrames.queueWaits && !noFrames.renderPassesPerFrame, "per-frame figures are shown for a second without frames");
+    require(AgcDriver::OverlayTraceDeltas(before, now, Clock::duration{}, 40, true).busy.empty(), "busy shares are shown for no elapsed time");
+    const auto noQueue = AgcDriver::OverlayTraceDeltas({}, std::vector<TraceThreadTotals>{now[3]}, 1s, 40, true);
+    require(noQueue.busy.empty() && !noQueue.queueWaits && noQueue.renderPassesPerFrame, "a trace without queue workers shows busy shares or queue waits");
+    AgcDriver::OverlayDetailSample sample;
+    sample.busy = figures.busy;
+    sample.phases = figures.phases;
+    sample.queueWaits = figures.queueWaits;
+    sample.renderPassesPerFrame = figures.renderPassesPerFrame;
+    requireDrawable(AgcDriver::ComposeOverlayDetails(sample), "page 2 from trace totals");
+}
+
+void hitRateTests() {
+    require(within(AgcDriver::OverlayHitPercent({100, 80}, {300, 230}).value_or(-1), 75), "150 hits of 200 lookups are not 75 %");
+    require(within(AgcDriver::OverlayHitPercent({1000, 900}, {200, 150}).value_or(-1), 75), "counters that restarted (the draw cache's 10 s report) are not read from zero");
+    require(!AgcDriver::OverlayHitPercent({5, 5}, {5, 5}), "a hit rate is shown without lookups");
+    require(!AgcDriver::OverlayHitPercent({0, 0}, {4, 5}), "more hits than lookups give a hit rate");
+}
+
+void presentModeTests() {
+    require(AgcDriver::OverlayPresentModeName(VK_PRESENT_MODE_FIFO_KHR) == "fifo" && AgcDriver::OverlayPresentModeName(VK_PRESENT_MODE_FIFO_LATEST_READY_KHR) == "fifo_latest_ready" && AgcDriver::OverlayPresentModeName(VK_PRESENT_MODE_MAILBOX_KHR) == "mailbox" && AgcDriver::OverlayPresentModeName(VK_PRESENT_MODE_IMMEDIATE_KHR) == "immediate", "the present modes are not named as ANYPS5_PRESENT_MODE names them");
+    require(AgcDriver::OverlayPresentModeName(VK_PRESENT_MODE_FIFO_RELAXED_KHR).empty() && AgcDriver::OverlayPresentModeName(VK_PRESENT_MODE_MAX_ENUM_KHR).empty(), "a present mode the driver never selects has a name");
+    AgcDriver::OverlayDetailSample sample;
+    sample.title = "PPSA01234";
+    sample.presentMode = AgcDriver::OverlayPresentModeName(VK_PRESENT_MODE_FIFO_LATEST_READY_KHR);
+    const auto content = AgcDriver::ComposeOverlayDetails(sample);
+    require(content.below.back() == "TITLE: PPSA01234  PRESENT: FIFO_LATEST_READY", "the longest present mode is drawn as '" + content.below.back() + "'");
+}
+
+void operatingSystemTests() {
+    const auto parsed = AgcDriver::ParseCommitCharge("MemTotal:       16318412 kB\nCommitLimit:     8159204 kB\nCommitted_AS:   12345678 kB\nVmallocTotal:   34359738367 kB\n");
+    require(parsed && parsed->used == 12345678ull * 1024 && parsed->budget == 8159204ull * 1024, "Committed_AS and CommitLimit are not read from /proc/meminfo's text");
+    require(!AgcDriver::ParseCommitCharge("Committed_AS:   12345678 kB\n"), "a commit charge is shown without its limit");
+    require(!AgcDriver::ParseCommitCharge("CommitLimit:     8159204 kB\nCommitted_AS:   12 MB\n"), "a value in another unit is read as kB");
+    require(!AgcDriver::ParseCommitCharge(""), "an empty /proc/meminfo gives a commit charge");
+    const auto commit = AgcDriver::ReadCommitCharge();
+    require(commit && commit->used != 0 && commit->budget != 0, "the system's commit charge and limit cannot be read");
+    static std::uint8_t* volatile touched = nullptr;
+    const auto before = AgcDriver::ReadPageFaults();
+    require(before.has_value(), "the process's page faults cannot be read");
+    constexpr std::size_t Bytes = std::size_t{16} << 20u;
+    const auto block = std::make_unique<std::uint8_t[]>(Bytes);
+    touched = block.get();
+    std::memset(touched, 0x3c, Bytes);
+    const auto after = AgcDriver::ReadPageFaults();
+    require(after && *after > *before, "touching 16 MiB of new memory did not raise the page faults from " + std::to_string(*before));
+    require(touched[Bytes - 1] == 0x3c, "the touched block was not written");
+}
+
+void addressCopyFoldTests() {
+    using AgcDriver::Graphics::AddressCopyCounters;
+    const auto start = AddressCopyCounters::Fold();
+    auto& counters = AddressCopyCounters::ThisThread();
+    require(&counters == &AddressCopyCounters::ThisThread(), "one thread got two address copy counters");
+    counters.Count(std::uint64_t{3} << 20u);
+    counters.Count(std::uint64_t{1} << 20u);
+    bool separate = false;
+    std::thread worker([&] {
+        auto& own = AddressCopyCounters::ThisThread();
+        separate = &own != &counters;
+        own.Count(std::uint64_t{5} << 20u);
+    });
+    worker.join();
+    require(separate, "two builders share address copy counters");
+    const auto folded = AddressCopyCounters::Fold();
+    require(folded.bytes - start.bytes == std::uint64_t{9} << 20u && folded.largest == std::uint64_t{5} << 20u, "two builders' 9 MiB of copies with a largest build of 5 MiB fold to " + std::to_string(folded.bytes - start.bytes) + " bytes, largest " + std::to_string(folded.largest));
+    const auto idle = AddressCopyCounters::Fold();
+    require(idle.bytes == folded.bytes && idle.largest == 0, "a second without copies keeps the previous second's largest build");
+    std::thread next([] { AddressCopyCounters::ThisThread().Count(std::uint64_t{2} << 20u); });
+    next.join();
+    const auto reused = AddressCopyCounters::Fold();
+    require(reused.bytes == idle.bytes + (std::uint64_t{2} << 20u) && reused.largest == std::uint64_t{2} << 20u, "the copies of a builder that ended were lost, or a new builder restarted them");
+    if (std::getenv("APS5_ADDRESS_COPY_MAX_MIB") == nullptr) require(AddressCopyCounters::Limit() == std::uint64_t{64} << 20u, "the address copy cap is not 64 MiB by default");
+    AgcDriver::Graphics::SetAddressCopyLimit(std::uint64_t{8} << 20u);
+    require(AddressCopyCounters::Limit() == std::uint64_t{8} << 20u, "the cap in use is not the one set");
+    AgcDriver::Graphics::SetAddressCopyLimit(0);
+}
+
+void driverSourceTests() {
+    const auto before = NotImplementedCalls_nid_no_patch();
+    try {
+        NotImplemented_nid_no_patch("performance overlay test");
+    } catch (const std::runtime_error&) {
+    }
+    require(NotImplementedCalls_nid_no_patch() == before + 1, "an unimplemented-function call is not counted");
+    if (std::getenv("APS5_SKIP_REFUSED_DRAWS") == nullptr) require(!AgcDriver::DriverDetail::RefusedShaderCount(), "refused shaders are counted while a refused shader still stops the title");
+    require(AgcDriver::DriverDetail::RegistrationPreparationsPending() == std::optional<std::size_t>{0}, "a process that prepared no shaders has a compile backlog");
+}
+
+void traceSwitchTests() {
+    using AgcDriver::PerformanceOverlay;
+    using AgcDriver::PerformanceTrace;
+    require(!PerformanceTrace::Configured(), "the overlay statistics test runs with APS5_TRACE_* set");
+    require(PerformanceOverlay::Page() == 0 && !PerformanceTrace::Recording() && !PerformanceTrace::OverlayTotals(), "the trace records before page 2 is shown");
+    std::string failure;
+    std::thread([&] {
+        if (PerformanceTrace::Thread() != nullptr) failure = "a thread is bound to the trace while nothing records";
+    }).join();
+    require(failure.empty(), failure);
+    PerformanceOverlay::Toggle();
+    require(PerformanceOverlay::Page() == 1 && !PerformanceTrace::Recording() && !PerformanceTrace::OverlayTotals(), "page 1 turns the trace on");
+    PerformanceOverlay::Toggle();
+    require(PerformanceOverlay::Page() == 2 && PerformanceTrace::Recording() && PerformanceTrace::OverlayTotals(), "page 2 does not turn the trace totals on");
+    std::thread([&] {
+        PerformanceTrace::BindThread("queue worker 0x5");
+        auto* thread = PerformanceTrace::Thread();
+        if (thread == nullptr || thread != PerformanceTrace::Thread()) {
+            failure = "a named worker is not bound once while page 2 shows";
+            return;
+        }
+        const auto begin = Clock::now();
+        thread->Add(PacketScope, nullptr, begin, begin + 3ms);
+        PerformanceTrace::Count(AgcDriver::PerformanceTrace::RenderPassScope);
+        PerformanceTrace::Count(AgcDriver::PerformanceTrace::RenderPassScope);
+        const auto snapshot = thread->Collect();
+        if (!snapshot.slices.empty() || snapshot.dropped) failure = "page 2's trace totals buffer slices";
+    }).join();
+    require(failure.empty(), failure);
+    std::thread([&] {
+        if (PerformanceTrace::Thread() != nullptr) failure = "a thread that never named itself is bound while page 2 shows";
+    }).join();
+    require(failure.empty(), failure);
+    const auto threads = PerformanceTrace::ThreadTotals();
+    require(threads.has_value(), "the trace totals cannot be read");
+    const auto worker = std::find_if(threads->begin(), threads->end(), [](const AgcDriver::TraceThreadTotals& thread) { return thread.name == "queue worker 0x5"; });
+    require(worker != threads->end(), "the named worker's totals are missing");
+    std::uint64_t passes = 0;
+    Clock::duration packets{};
+    for (const auto& entry : worker->totals) {
+        if (std::string_view(entry.scope) == AgcDriver::PerformanceTrace::RenderPassScope) passes += entry.count;
+        if (std::string_view(entry.scope) == PacketScope && entry.stage == nullptr) packets += entry.length;
+    }
+    require(passes == 2 && packets == 3ms, "the worker's totals hold " + std::to_string(passes) + " render passes and " + std::to_string(std::chrono::duration<double, std::milli>(packets).count()) + " ms of packets, not 2 and 3 ms");
+    PerformanceOverlay::Toggle();
+    require(PerformanceOverlay::Page() == 0 && !PerformanceTrace::Recording() && !PerformanceTrace::OverlayTotals(), "leaving page 2 leaves the trace totals on");
+    std::thread([&] {
+        PerformanceTrace::Count(AgcDriver::PerformanceTrace::RenderPassScope);
+        if (PerformanceTrace::Thread() != nullptr) failure = "a thread is bound to the trace after page 2 was left";
+    }).join();
+    require(failure.empty(), failure);
+}
+
 void costReport() {
     auto& counters = AgcDriver::PacketCounters::ThisThread();
     constexpr int Increments = 20'000'000;
@@ -489,9 +886,21 @@ int main() {
         countFormatTests();
         detailComposeTests();
         fullPanelLayoutTests();
+        fontTests();
+        widePanelTests();
+        pageCycleTests();
+        pageTwoComposeTests();
+        pageTwoLayoutTests();
         sampleTests();
+        traceDeltaTests();
+        hitRateTests();
+        presentModeTests();
+        operatingSystemTests();
+        addressCopyFoldTests();
+        driverSourceTests();
+        traceSwitchTests();
         costReport();
-        std::cout << "performance overlay statistics, layout, palette, packet, pipeline, memory and resolution tests passed\n";
+        std::cout << "performance overlay statistics, layout, palette, packet, pipeline, memory, resolution, page 2 source and trace switch tests passed\n";
         return 0;
     } catch (const std::exception& error) {
         std::cerr << error.what() << '\n';

core/libs/prx/libSceAgcDriver/tests/Pm4.cpp

@@ -2010,6 +2010,41 @@ void testWaitRegMem64ZeroHighMaskDrain() {
     check(outstanding == 1, "WAIT_REG_MEM_64 with zero high mask at 4 mod 8 skipped the drain of a pending write to the dword it compares");
 }
 
+void testWaitRegMemMisalignedBehindBusyPipeline() {
+    alignas(8) static std::uint64_t target = 0;
+    alignas(8) static std::uint32_t other = 0;
+    static std::uint32_t done = 0;
+    const auto base = reinterpret_cast<std::uintptr_t>(&target);
+    const auto otherAddr = reinterpret_cast<std::uintptr_t>(&other);
+    const std::vector<std::uint32_t> waits[]{
+        makePacket(0x3c, {0x13, static_cast<std::uint32_t>(base + 1), static_cast<std::uint32_t>((base + 1) >> 32u), 42, 0xffffffffu, 0}),
+        makePacket(0x93, {0x13, static_cast<std::uint32_t>(base + 3), static_cast<std::uint32_t>((base + 3) >> 32u), 42, 0, 0xffffffffu, 0xffffffffu, 0})};
+    for (const auto& wait : waits) {
+        std::atomic_ref<std::uint64_t>(target).store(0);
+        std::atomic_ref<std::uint32_t>(done).store(0);
+        std::atomic<bool> hold{true};
+        std::atomic<bool> started{false};
+        AgcDriver::DriverDetail::DrawPipeline::Queue0().Enqueue([&] {
+            started.store(true);
+            while (hold.load()) std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }, { {otherAddr, otherAddr + 4} });
+        while (!started.load()) std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        const auto opcode = (wait[0] >> 8u) & 0xffu;
+        const auto drainsBefore = AgcDriver::DriverDetail::DrawPipeline::DrainRequestsByOpcode(opcode);
+        auto words = joinPackets({wait, writeWord(done, 1)});
+        submitWords(words, 0);
+        const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5);
+        while (AgcDriver::DriverDetail::DrawPipeline::DrainRequestsByOpcode(opcode) == drainsBefore && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        const bool drained = AgcDriver::DriverDetail::DrawPipeline::DrainRequestsByOpcode(opcode) != drainsBefore;
+        std::atomic_ref<std::uint64_t>(target).store(42);
+        hold.store(false);
+        AgcDriverWaitIdle_nid_postfix();
+        waitDrawPipelineIdle();
+        check(drained, "unsatisfied misaligned WAIT_REG_MEM behind a busy draw pipeline did not drain it");
+        check(std::atomic_ref<std::uint32_t>(done).load() == 1, "misaligned WAIT_REG_MEM behind a busy draw pipeline did not finish");
+    }
+}
+
 int main(int argc, char** argv) {
     try {
         if (argc == 2 && std::string(argv[1]) == "failure") {
@@ -2086,6 +2121,7 @@ int main(int argc, char** argv) {
         testWaitRegMemMasking();
         testWaitRegMemPipelineDrain();
         testWaitRegMem64ZeroHighMaskDrain();
+        testWaitRegMemMisalignedBehindBusyPipeline();
         LibcRunShutdown_nid_postfix();
         std::puts("PM4 catalog, registers, state, memory, conditional execution and submission tests passed");
         return 0;

core/libs/prx/libSceAgcDriver/tests/PresentationPass.cpp

@@ -219,13 +219,21 @@ struct Source {
     std::vector<std::byte> decoded;
 };
 
-Source makeSource(std::uint64_t pixelFormat, VkFormat storage, std::uint32_t width, std::uint32_t height, std::uint32_t salt = 0) {
+Source makeSource(std::uint64_t pixelFormat, VkFormat storage, std::uint32_t width, std::uint32_t height, std::uint32_t salt = 0, bool premultiplied = false) {
     Source source{pixelFormat, storage, width, height, std::vector<std::byte>(static_cast<std::size_t>(width) * height * 4), {}};
     std::uint32_t seed = 0x9e3779b9u ^ static_cast<std::uint32_t>(storage) ^ static_cast<std::uint32_t>(pixelFormat >> 32u) ^ salt;
     for (auto& byte : source.words) {
         seed = seed * 1664525u + 1013904223u;
         byte = static_cast<std::byte>(seed >> 24u);
     }
+    if (premultiplied) {
+        source.words[3] = std::byte{0};
+        source.words[7] = std::byte{255};
+        for (std::size_t texel = 0; texel < source.words.size(); texel += 4) {
+            const auto alpha = std::to_integer<unsigned>(source.words[texel + 3]);
+            for (std::size_t channel = 0; channel < 3; ++channel) source.words[texel + channel] = static_cast<std::byte>(std::to_integer<unsigned>(source.words[texel + channel]) * alpha / 255u);
+        }
+    }
     const ColorTargetLayout layout(width, height, ColorTileMode::RenderTarget);
     std::vector<std::byte> storageBytes(layout.Bytes() + 65536);
     auto* aligned = reinterpret_cast<std::byte*>((reinterpret_cast<std::uintptr_t>(storageBytes.data()) + 65535u) & ~std::uintptr_t{65535});
@@ -539,6 +547,182 @@ void passConvertsPq(const Context& context, PresentationPass& pass, const Source
     }
 }
 
+std::vector<std::byte> overlayPresent(const Context& context, PresentationPass& pass, PresentationPass& planes, const Source& main, const Image* mainImage, const Source& overlay, const Image* overlayImage, VkExtent2D extent) {
+    Image destination(context, VK_FORMAT_B8G8R8A8_UNORM, extent, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT);
+    const auto bytes = static_cast<std::size_t>(extent.width) * extent.height * 4;
+    Buffer readback(context, bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT);
+    {
+        CommandBatch batch(context);
+        const auto commands = batch.Handle();
+        VkImageLayout loaded = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+        if (mainImage != nullptr) {
+            pass.Bind(0, mainImage->Handle(), main.width, main.height, main.pixelFormat, AgcDriver::SdrToneMapLuminanceNits);
+            pass.RecordPresent(commands, 0, destination.Handle(), VK_FORMAT_B8G8R8A8_UNORM, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, extent, loaded);
+        } else {
+            loaded = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+            transition(context, commands, destination.Handle(), VK_IMAGE_LAYOUT_UNDEFINED, loaded, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT);
+            const VkClearColorValue color{{64.0f / 255.0f, 128.0f / 255.0f, 191.0f / 255.0f, 1.0f}};
+            const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
+            context.Function<PFN_vkCmdClearColorImage>("vkCmdClearColorImage")(commands, destination.Handle(), loaded, &color, 1, &range);
+        }
+        if (overlayImage != nullptr) planes.Bind(1, overlayImage->Handle(), overlay.width, overlay.height, overlay.pixelFormat, AgcDriver::SdrToneMapLuminanceNits);
+        else planes.BindTexels(1, commands, overlay.words, overlay.width, overlay.height, overlay.pixelFormat, AgcDriver::SdrToneMapLuminanceNits);
+        planes.RecordOverlay(commands, 1, destination.Handle(), VK_FORMAT_B8G8R8A8_UNORM, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, extent, loaded, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
+        readImage(context, commands, destination, readback);
+        batch.SubmitAndWait();
+    }
+    pass.Release(0);
+    planes.Release(1);
+    return bytesOf(readback, bytes);
+}
+
+std::vector<std::byte> clearedFrame(VkExtent2D extent) {
+    std::vector<std::byte> frame(static_cast<std::size_t>(extent.width) * extent.height * 4);
+    for (std::size_t offset = 0; offset < frame.size(); offset += 4) {
+        frame[offset] = std::byte{191};
+        frame[offset + 1] = std::byte{128};
+        frame[offset + 2] = std::byte{64};
+        frame[offset + 3] = std::byte{255};
+    }
+    return frame;
+}
+
+void overlayBlendsPremultiplied(const Context& context, PresentationPass& pass, PresentationPass& planes, const Source& main, const Image& mainImage, const Source& overlay, const Image& overlayImage) {
+    for (const auto extent : {VkExtent2D{overlay.width, overlay.height}, VkExtent2D{overlay.width + 60, overlay.height}, VkExtent2D{overlay.width, overlay.height + 51}, VkExtent2D{117, 76}, VkExtent2D{331, 215}}) {
+        const auto rect = AgcDriver::ComputeContainRect_nid_postfix(overlay.width, overlay.height, extent.width, extent.height);
+        const bool exact = rect.width == overlay.width && rect.height == overlay.height;
+        const auto alone = passPresent(context, pass, overlay, overlayImage, extent, VK_FORMAT_B8G8R8A8_UNORM);
+        for (const bool cleared : {false, true}) {
+            const auto base = cleared ? clearedFrame(extent) : passPresent(context, pass, main, mainImage, extent, VK_FORMAT_B8G8R8A8_UNORM);
+            for (const bool uploaded : {false, true}) {
+                const auto blended = overlayPresent(context, pass, planes, main, cleared ? nullptr : &mainImage, overlay, uploaded ? nullptr : &overlayImage, extent);
+                int worst = 0;
+                for (std::uint32_t y = 0; y < extent.height; ++y) {
+                    for (std::uint32_t x = 0; x < extent.width; ++x) {
+                        const auto offset = (static_cast<std::size_t>(y) * extent.width + x) * 4;
+                        const bool inside = static_cast<std::int32_t>(x) >= rect.x && static_cast<std::int32_t>(x) < rect.x + static_cast<std::int32_t>(rect.width) && static_cast<std::int32_t>(y) >= rect.y && static_cast<std::int32_t>(y) < rect.y + static_cast<std::int32_t>(rect.height);
+                        for (std::size_t channel = 0; channel < 4; ++channel) {
+                            int expected = std::to_integer<int>(base[offset + channel]);
+                            if (inside && channel < 3) {
+                                const double alpha = std::to_integer<int>(alone[offset + 3]) / 255.0;
+                                expected = static_cast<int>(std::min(255.0, std::to_integer<int>(alone[offset + channel]) + expected * (1.0 - alpha)) + 0.5);
+                            }
+                            const int difference = std::abs(expected - std::to_integer<int>(blended[offset + channel]));
+                            Require(inside || difference == 0, "the overlay plane changed the destination outside its rectangle: " + describe(overlay, extent));
+                            worst = std::max(worst, difference);
+                        }
+                    }
+                }
+                std::printf("  overlay %s over %s, %s: by at most %d\n", describe(overlay, extent).c_str(), cleared ? "a cleared destination" : "the main plane", uploaded ? "uploaded texels" : "resident image", worst);
+                Require(worst <= (exact ? 1 : 2), "the overlay plane is not blended with its premultiplied alpha: " + describe(overlay, extent));
+            }
+        }
+    }
+}
+
+int blendedDifference(const std::vector<std::byte>& base, const std::vector<std::byte>& alone, const std::vector<std::byte>& blended) {
+    Require(base.size() == alone.size() && base.size() == blended.size(), "compared frames differ in size");
+    int worst = 0;
+    for (std::size_t offset = 0; offset < base.size(); offset += 4) {
+        const double alpha = std::to_integer<int>(alone[offset + 3]) / 255.0;
+        for (std::size_t channel = 0; channel < 4; ++channel) {
+            int expected = std::to_integer<int>(base[offset + channel]);
+            if (channel < 3) expected = static_cast<int>(std::min(255.0, std::to_integer<int>(alone[offset + channel]) + expected * (1.0 - alpha)) + 0.5);
+            worst = std::max(worst, std::abs(expected - std::to_integer<int>(blended[offset + channel])));
+        }
+    }
+    return worst;
+}
+
+void overlayBlendsIntoDumps(const Context& context, PresentationPass& pass, PresentationPass& planes, const Source& main, const Image& mainImage, const Source& overlay, const Image& overlayImage) {
+    for (const std::uint32_t stride : {1u, 3u}) {
+        const auto extent = PresentationPass::DumpExtent(main.width, main.height, stride);
+        const auto base = passDump(context, pass, main, mainImage, stride);
+        const auto alone = passPresent(context, pass, overlay, overlayImage, extent, VK_FORMAT_B8G8R8A8_UNORM);
+        const auto bytes = static_cast<std::size_t>(extent.width) * extent.height * 4;
+        Buffer readback(context, bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT);
+        pass.Bind(1, mainImage.Handle(), main.width, main.height, main.pixelFormat, AgcDriver::SdrToneMapLuminanceNits);
+        planes.Bind(1, overlayImage.Handle(), overlay.width, overlay.height, overlay.pixelFormat, AgcDriver::SdrToneMapLuminanceNits);
+        {
+            CommandBatch batch(context);
+            const auto commands = batch.Handle();
+            pass.RecordDumpImage(commands, 1, stride);
+            Require(pass.DumpImageExtent(1).width == extent.width && pass.DumpImageExtent(1).height == extent.height, "the dump image extent differs from the dump extent");
+            planes.RecordOverlay(commands, 1, pass.DumpImage(1), VK_FORMAT_B8G8R8A8_UNORM, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, extent, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
+            pass.RecordDumpReadback(commands, 1, readback.Handle());
+            batch.SubmitAndWait();
+        }
+        pass.Release(1);
+        planes.Release(1);
+        const int worst = blendedDifference(base, alone, bytesOf(readback, bytes));
+        std::printf("  overlay %s blended into the presentation dump at stride %u: by at most %d\n", describe(overlay, extent).c_str(), stride, worst);
+        Require(worst <= (stride == 1 ? 1 : 2), "the overlay plane is not blended into the presentation dump: " + describe(overlay, extent));
+    }
+    const VkExtent2D extent{main.width, main.height};
+    PresentationScaler scaler(context, VK_FORMAT_B8G8R8A8_UNORM, VK_FORMAT_B8G8R8A8_UNORM);
+    scaler.EnsureSourceImage(main.width, main.height);
+    Buffer upload(context, main.decoded.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
+    std::memcpy(upload.Bytes().data(), main.decoded.data(), main.decoded.size());
+    const auto bytes = main.decoded.size();
+    Buffer readback(context, bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT);
+    const auto alone = passPresent(context, pass, overlay, overlayImage, extent, VK_FORMAT_B8G8R8A8_UNORM);
+    planes.Bind(1, overlayImage.Handle(), overlay.width, overlay.height, overlay.pixelFormat, AgcDriver::SdrToneMapLuminanceNits);
+    {
+        CommandBatch batch(context);
+        const auto commands = batch.Handle();
+        scaler.RecordUpload(commands, upload.Handle());
+        planes.RecordOverlay(commands, 1, scaler.SourceImage(), scaler.SourceFormat(), VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, extent, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
+        scaler.RecordReadback(commands, readback.Handle());
+        batch.SubmitAndWait();
+    }
+    planes.Release(1);
+    const int worst = blendedDifference(main.decoded, alone, bytesOf(readback, bytes));
+    std::printf("  overlay %s blended into the scaler source: by at most %d\n", describe(overlay, extent).c_str(), worst);
+    Require(worst <= 1, "the overlay plane is not blended into the scaler source the dump reads back: " + describe(overlay, extent));
+}
+
+void overlayRefusals(const Context& context, PresentationPass& planes, const Image& overlayImage, const Source& overlay) {
+    Image destination(context, VK_FORMAT_B8G8R8A8_UNORM, {overlay.width, overlay.height}, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT);
+    const auto refused = [&](std::uint64_t pixelFormat, VkColorSpaceKHR colorSpace, VkImageLayout initialLayout) {
+        bool threw = false;
+        try {
+            CommandBatch batch(context);
+            planes.Bind(1, overlayImage.Handle(), overlay.width, overlay.height, pixelFormat, AgcDriver::SdrToneMapLuminanceNits);
+            planes.RecordOverlay(batch.Handle(), 1, destination.Handle(), VK_FORMAT_B8G8R8A8_UNORM, colorSpace, {overlay.width, overlay.height}, initialLayout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
+        } catch (const std::exception&) {
+            threw = true;
+        }
+        planes.Release(1);
+        return threw;
+    };
+    Require(refused(Bgra8 | TenBit, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL), "a 10-bit overlay plane was blended without its 2-bit alpha");
+    Require(refused(Rgb10Pq, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL), "a PQ overlay plane was blended");
+    Require(refused(Bgra8, VK_COLOR_SPACE_HDR10_ST2084_EXT, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL), "an overlay plane was blended into an HDR10 destination");
+    Require(refused(Bgra8, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, VK_IMAGE_LAYOUT_UNDEFINED), "an overlay plane was blended over discarded contents");
+    bool truncated = false;
+    try {
+        CommandBatch batch(context);
+        planes.BindTexels(1, batch.Handle(), std::span(overlay.words).first(overlay.words.size() - 4), overlay.width, overlay.height, overlay.pixelFormat, AgcDriver::SdrToneMapLuminanceNits);
+    } catch (const std::exception&) {
+        truncated = true;
+    }
+    Require(truncated, "overlay texels that do not cover their extent were uploaded");
+}
+
+void overlayPlaneTests(const Context& context) {
+    PresentationPass pass(context, 2);
+    PresentationPass planes(context, 2);
+    const auto main = makeSource(Bgra8, VK_FORMAT_B8G8R8A8_UNORM, 200, 130);
+    const auto mainImage = residentImage(context, main);
+    for (const auto& [pixelFormat, storage] : {std::pair{Bgra8, VK_FORMAT_B8G8R8A8_UNORM}, std::pair{Rgba8, VK_FORMAT_R8G8B8A8_UNORM}}) {
+        const auto overlay = makeSource(pixelFormat, storage, 200, 130, 0x0f0f0f0fu, true);
+        const auto overlayImage = residentImage(context, overlay);
+        overlayBlendsPremultiplied(context, pass, planes, main, *mainImage, overlay, *overlayImage);
+        overlayBlendsIntoDumps(context, pass, planes, main, *mainImage, overlay, *overlayImage);
+        overlayRefusals(context, planes, *overlayImage, overlay);
+    }
+}
+
 void residentPresentTests(const Context& context) {
     PresentationPass pass(context, 2);
     const std::array<std::pair<std::uint64_t, VkFormat>, 5> cases{{
@@ -568,6 +752,64 @@ void residentPresentTests(const Context& context) {
     blitLetterboxMatchesClear(context);
 }
 
+PFN_vkGetDeviceProcAddr realDeviceProc = nullptr;
+PFN_vkAllocateMemory realAllocateMemory = nullptr;
+const VkPhysicalDeviceMemoryProperties* refusingMemory = nullptr;
+std::uint32_t refusedAllocations = 0;
+std::uint32_t lastAllocatedType = UINT32_MAX;
+
+VKAPI_ATTR VkResult VKAPI_CALL refuseDeviceLocalAllocation(VkDevice device, const VkMemoryAllocateInfo* info, const VkAllocationCallbacks* callbacks, VkDeviceMemory* memory) {
+    const auto heap = refusingMemory->memoryTypes[info->memoryTypeIndex].heapIndex;
+    if ((refusingMemory->memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0) {
+        ++refusedAllocations;
+        return VK_ERROR_OUT_OF_DEVICE_MEMORY;
+    }
+    lastAllocatedType = info->memoryTypeIndex;
+    return realAllocateMemory(device, info, callbacks, memory);
+}
+
+VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL refusingDeviceProc(VkDevice device, const char* name) {
+    if (std::strcmp(name, "vkAllocateMemory") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&refuseDeviceLocalAllocation);
+    return realDeviceProc(device, name);
+}
+
+void sourceImageOutsideDeviceMemoryTests(const Context& base) {
+    VkImageCreateInfo probeInfo{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
+    probeInfo.imageType = VK_IMAGE_TYPE_2D;
+    probeInfo.format = VK_FORMAT_B8G8R8A8_UNORM;
+    probeInfo.extent = {64u, 64u, 1u};
+    probeInfo.mipLevels = 1;
+    probeInfo.arrayLayers = 1;
+    probeInfo.samples = VK_SAMPLE_COUNT_1_BIT;
+    probeInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
+    probeInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
+    VkImage probe = VK_NULL_HANDLE;
+    Check(base.Function<PFN_vkCreateImage>("vkCreateImage")(base.device, &probeInfo, nullptr, &probe), "vkCreateImage probe");
+    VkMemoryRequirements requirements{};
+    base.Function<PFN_vkGetImageMemoryRequirements>("vkGetImageMemoryRequirements")(base.device, probe, &requirements);
+    base.Function<PFN_vkDestroyImage>("vkDestroyImage")(base.device, probe, nullptr);
+    if (!base.HostMemoryType(requirements.memoryTypeBits)) {
+        std::printf("presentation source in host memory: skipped, the device offers no host memory type for the image\n");
+        return;
+    }
+    auto context = base;
+    context.bufferPool.reset();
+    realDeviceProc = base.deviceProc;
+    realAllocateMemory = base.Function<PFN_vkAllocateMemory>("vkAllocateMemory");
+    refusingMemory = &context.memory;
+    refusedAllocations = 0;
+    lastAllocatedType = UINT32_MAX;
+    context.deviceProc = refusingDeviceProc;
+    {
+        PresentationScaler scaler(context, VK_FORMAT_B8G8R8A8_UNORM, VK_FORMAT_B8G8R8A8_UNORM);
+        scaler.EnsureSourceImage(64u, 64u);
+        Require(refusedAllocations != 0, "the presentation source never asked for device-local memory");
+        Require(lastAllocatedType != UINT32_MAX && (context.memory.memoryHeaps[context.memory.memoryTypes[lastAllocatedType].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0, "the presentation source was not placed in host memory when device-local memory was refused");
+        Require(scaler.SourceWidth() == 64u && scaler.SourceHeight() == 64u, "the presentation source in host memory lost its extent");
+    }
+    context.bufferPool.reset();
+}
+
 }
 
 int main() {
@@ -582,7 +824,9 @@ int main() {
             return VulkanTestSkipped;
         }
         residentPresentTests(device->GetContext());
-        std::cout << "resident present format, presentation pass and letterbox tests passed\n";
+        overlayPlaneTests(device->GetContext());
+        sourceImageOutsideDeviceMemoryTests(device->GetContext());
+        std::cout << "resident present format, presentation pass, overlay plane and letterbox tests passed\n";
         return 0;
     } catch (const std::exception& error) {
         std::cerr << error.what() << '\n';

core/libs/prx/libSceAgcDriver/tests/Recorder.cpp

@@ -25,9 +25,11 @@
 #include "prx/libSceAgcDriver/Execution/include/DisplayFormat.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp"
+#include "prx/libc/include/Api.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
 #include "prx/libc/include/GuestArena.hpp"
 #include "prx/libc/include/GuestWriteWatch.hpp"
+#include "prx/libc/include/Shutdown.hpp"
 #include "prx/libc/include/general/VabiMacros.hpp"
 #include "ResidentPresent.hpp"
 #include "ImageTableAbi.hpp"
@@ -66,6 +68,7 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <exception>
 #include <filesystem>
 #include <fstream>
 #include <iomanip>
@@ -3025,15 +3028,33 @@ void requireImportServesGuest(const Context& context, Recorder& recorder, const
     const auto bytes = static_cast<std::size_t>(end - begin);
     Require(begin >= import.base && end <= import.base + import.bytes, what + " is not inside its import");
     const auto collected = CollectWritesUncached(begin, bytes);
-    Require(collected != 0 && Watched(begin, bytes), what + " left the write watch");
-    Require(UnchangedSinceCollected(begin, page, collected) && UnchangedSinceCollected(end - page, page, collected), what + " reports writes nobody made");
+    const bool watched = Watched(begin, bytes);
+    Require(!watched || collected != 0, what + " is watched but cannot collect writes");
+    if (watched) {
+        Require(UnchangedSinceCollected(begin, page, collected) && UnchangedSinceCollected(end - page, page, collected), what + " reports writes nobody made");
+    } else {
+        Require(collected == 0, what + " collects writes without being watched");
+        const auto generation = WriteGeneration();
+        Require(UntrackedHint(begin, bytes) == 0 && !UnchangedSince(begin, bytes, generation), what + " provides an unchanged or untracked-hint proof without a write watch");
+        std::vector<void*> compared(bytes / page);
+        auto count = compared.size();
+        GuestArena::GuestArenaComparedPages_nid_no_patch(begin, bytes, compared.data(), &count);
+        Require(count == compared.size(), what + " left the write watch without compared shared pages");
+        for (std::size_t index = 0; index < count; ++index) {
+            Require(compared[index] == reinterpret_cast<void*>(begin + index * page), what + " has an unexpected compared shared page");
+            MEMORY_BASIC_INFORMATION memory{};
+            Require(VirtualQuery(compared[index], &memory, sizeof(memory)) == sizeof(memory) && memory.State == MEM_COMMIT, what + " has an inaccessible compared shared page");
+            const auto protection = memory.Protect & 0xff;
+            Require(protection == PAGE_READWRITE || protection == PAGE_EXECUTE_READWRITE, what + " has a compared shared page that is not writable");
+        }
+    }
     auto* first = reinterpret_cast<std::byte*>(begin);
     auto* last = reinterpret_cast<std::byte*>(end - page);
     for (std::size_t index = 0; index < probe; ++index) {
         first[index] = static_cast<std::byte>((index * 7u + salt) & 0xffu);
         last[index] = static_cast<std::byte>((index * 13u + salt * 5u + 1u) & 0xffu);
     }
-    Require(!UnchangedSinceCollected(begin, page, collected) && !UnchangedSinceCollected(end - page, page, collected), "a CPU store into " + what + " was not collected");
+    if (watched) Require(!UnchangedSinceCollected(begin, page, collected) && !UnchangedSinceCollected(end - page, page, collected), "a CPU store into " + what + " was not collected");
     Buffer readback(context, 2 * probe, VK_BUFFER_USAGE_TRANSFER_DST_BIT);
     const auto commands = recorder.Commands();
     RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
@@ -3163,7 +3184,7 @@ int reimportBench(const Device& device) {
         const auto* tail = HostImportFor(context, begin, static_cast<std::size_t>(end - begin));
         Require(tail != nullptr && tail->base == begin && tail->base + tail->bytes == end, "the re-import bench lost the import of its kept side");
         requireImportServesGuest(context, recorder, *tail, begin, end, ++salt, std::string("the kept side after the ") + pattern.name + " cuts");
-        std::printf("re-import bench, %s cuts: the kept side was imported again after %zu of %zu cuts, slowest import after a cut %.2f ms; CPU stores into it are collected and the GPU reads them through its buffer\n", pattern.name, again, moves, slowest);
+        std::printf("re-import bench, %s cuts: the kept side was imported again after %zu of %zu cuts, slowest import after a cut %.2f ms; write tracking sees its CPU stores and the GPU reads them through its buffer\n", pattern.name, again, moves, slowest);
         cleanup();
         gpu.unlock();
         return again;
@@ -3284,7 +3305,7 @@ int hostImportCutTests(const Device& device) {
         }
         cleanup();
         gpu.unlock();
-        std::cout << "host import after cut: " << name << ": the changed import was retired before the remaining range was imported, and the GPU and write watch see the guest's bytes" << (test.unmap ? ", also after the pages were mapped again" : "") << '\n';
+        std::cout << "host import after cut: " << name << ": the changed import was retired before the remaining range was imported, and the GPU and write tracking see the guest's bytes" << (test.unmap ? ", also after the pages were mapped again" : "") << '\n';
     };
     const auto remapWhileTrimmed = [&] {
         const std::string name = "an unmap and a new mapping of the retained side while the old import is retired";
@@ -3315,7 +3336,7 @@ int hostImportCutTests(const Device& device) {
         Require(remapped->memoryBase == begin && remapped->pinnedBytes == end - begin, "after " + name + " the pages mapped again are served from the import that pinned the old ones");
         cleanup();
         gpu.unlock();
-        std::cout << "host import cut: " << name << ": the pages mapped again were imported again and the GPU and the write watch see the guest's bytes\n";
+        std::cout << "host import cut: " << name << ": the pages mapped again were imported again and the GPU and write tracking see the guest's bytes\n";
     };
     gpu.unlock();
     try {
@@ -9609,6 +9630,29 @@ void minLodTests(const Device& device, Recorder& recorder) {
     expectRed(sample(0x100, 0.0f, 1), unorm(1), "minimum LOD clamp: MIN_LOD at the view's base level reads its base level");
 }
 
+void failedCheckReportTests(const Device& device, Recorder& recorder) {
+    const auto& context = device.GetContext();
+    Require(Recorder::Active() == &recorder, "the failed check test needs the active recorder");
+    TextureDetiler detiler(context);
+    GuestTextureResource resource{};
+    resource.baseAddress = 0x100000;
+    resource.width = 64;
+    resource.height = 64;
+    resource.mipCount = 1;
+    resource.tileMode = TextureTileMode::kLinear;
+    resource.dimension = TextureDimension::k2D;
+    resource.format = 56;
+    resource.dstSelX = 4;
+    resource.dstSelY = 5;
+    resource.dstSelZ = 6;
+    resource.dstSelW = 7;
+    const std::vector<std::byte> snapshot(static_cast<std::size_t>(DescribeSurface(resource).guestBytes), std::byte{0x40});
+    const VkComponentMapping identity{VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY};
+    Texture texture(context, detiler, resource, identity, snapshot);
+    Require(recorder.Recording(), "a texture upload was not recorded into the open batch");
+    Require(false, "a failed check stopped the test with a texture upload in its open batch");
+}
+
 void singleCubeTests(const Device& device, Recorder& recorder) {
     auto context = device.GetContext();
     TextureDetiler detiler(context);
@@ -11058,6 +11102,97 @@ void denormalClearTests(const Device& device, Recorder& recorder) {
     recorder.Sync();
 }
 
+class LibraryCreateFlags {
+public:
+    explicit LibraryCreateFlags(const Context& context) : resolver(context.deviceProc), device(context.device) {
+        create = function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines");
+        active = this;
+    }
+    ~LibraryCreateFlags() { active = nullptr; }
+    LibraryCreateFlags(const LibraryCreateFlags&) = delete;
+    LibraryCreateFlags& operator=(const LibraryCreateFlags&) = delete;
+    PFN_vkGetDeviceProcAddr Resolver() const { return resolve; }
+    std::uint32_t Libraries() const { return libraries.load(); }
+    std::uint32_t Retaining() const { return retaining.load(); }
+
+private:
+    template<typename TFunction>
+    TFunction function(const char* name) const { return reinterpret_cast<TFunction>(resolver(device, name)); }
+    static VKAPI_ATTR VkResult VKAPI_CALL createPipelines(VkDevice device, VkPipelineCache cache, std::uint32_t count, const VkGraphicsPipelineCreateInfo* infos, const VkAllocationCallbacks* callbacks, VkPipeline* pipelines) {
+        const auto result = active->create(device, cache, count, infos, callbacks, pipelines);
+        for (std::uint32_t index = 0; index < count; ++index) {
+            if (pipelines[index] == VK_NULL_HANDLE || (infos[index].flags & VK_PIPELINE_CREATE_LIBRARY_BIT_KHR) == 0) continue;
+            active->libraries.fetch_add(1);
+            if ((infos[index].flags & VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT) != 0) active->retaining.fetch_add(1);
+        }
+        return result;
+    }
+    static VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL resolve(VkDevice device, const char* name) {
+        if (std::strcmp(name, "vkCreateGraphicsPipelines") == 0) return reinterpret_cast<PFN_vkVoidFunction>(createPipelines);
+        return active->resolver(device, name);
+    }
+    PFN_vkGetDeviceProcAddr resolver;
+    VkDevice device;
+    PFN_vkCreateGraphicsPipelines create = nullptr;
+    std::atomic<std::uint32_t> libraries{0};
+    std::atomic<std::uint32_t> retaining{0};
+    static inline LibraryCreateFlags* active = nullptr;
+};
+
+void pipelineLibraryLtoInfoTests(const Device& device) {
+    std::lock_guard gpu(GpuMutex());
+    const auto& context = device.GetContext();
+    LibraryCreateFlags created(context);
+    struct Cleanup {
+        VkDevice device;
+        ~Cleanup() { ClearCachedPipelines(device); }
+    } cleanup{context.device};
+    ClearCachedPipelines(context.device);
+    if (!context.graphicsPipelineLibrary) {
+        std::cout << "graphics pipeline libraries unavailable or disabled: no library retains link-time optimization info\n";
+        return;
+    }
+    ShaderRecompiler::RecompileResult vertex, fragment;
+    vertex.variantId = 41;
+    fragment.variantId = 42;
+    vertex.spirv = std::vector<std::uint32_t>(std::begin(TRIANGLE_vert_SPV), std::end(TRIANGLE_vert_SPV));
+    fragment.spirv = std::vector<std::uint32_t>(std::begin(TRIANGLE_frag_SPV), std::end(TRIANGLE_frag_SPV));
+    const std::array shaders{CompiledShader{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, CompiledShader{ShaderRecompiler::ShaderStage::Fragment, &fragment, 0}};
+    State state{};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.hasColorTarget = true;
+    state.color.format = VK_FORMAT_R8G8B8A8_UNORM;
+    state.colors.push_back(state.color);
+    state.blend.colorWriteMask = 15;
+    state.blends.push_back(state.blend);
+    ShaderResources resources(context, shaders, state.color, static_cast<std::uint32_t>(state.colors.size()), 0, 0);
+    VertexInputLayout input;
+    auto fast = context;
+    fast.deviceProc = created.Resolver();
+    fast.pipelineLinkTimeOptimization = false;
+    auto optimizing = fast;
+    optimizing.pipelineLinkTimeOptimization = true;
+    const bool lto = std::getenv("APS5_NO_PIPELINE_LTO") == nullptr;
+    const auto fastPipeline = CachedPipeline(fast, state, input, resources, shaders, VK_IMAGE_LAYOUT_GENERAL);
+    const auto fastLibraries = created.Libraries();
+    Require(fastLibraries != 0, "the pipeline was not linked from libraries created through the traced device");
+    Require(created.Retaining() == 0, std::to_string(created.Retaining()) + " of the " + std::to_string(fastLibraries) + " libraries made for a context without optimized links were created to retain link-time optimization info");
+    WaitForOptimizedPipelines(context.device);
+    Require(!fastPipeline->Optimized(), "a pipeline of a context without optimized links was optimized in the background");
+    const auto optimizedPipeline = CachedPipeline(optimizing, state, input, resources, shaders, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+    Require(optimizedPipeline != fastPipeline, "the two attachment layouts shared one cached pipeline");
+    const auto libraries = created.Libraries() - fastLibraries;
+    if (lto) {
+        Require(libraries == fastLibraries, "a link that will be optimized made " + std::to_string(libraries) + " libraries, expected " + std::to_string(fastLibraries) + ": it reused libraries made without link-time optimization info");
+        Require(created.Retaining() == libraries, std::to_string(created.Retaining()) + " of the " + std::to_string(libraries) + " libraries of a link that will be optimized were created to retain link-time optimization info");
+    } else {
+        Require(libraries == 0 && created.Retaining() == 0, "with APS5_NO_PIPELINE_LTO set, forcing pipelineLinkTimeOptimization changed the libraries or their create flags");
+    }
+    WaitForOptimizedPipelines(context.device);
+    Require(optimizedPipeline->Optimized() == lto, lto ? "the pipeline linked from libraries that retain link-time optimization info was not optimized in the background" : "a pipeline was optimized with APS5_NO_PIPELINE_LTO set");
+    std::cout << "Pipeline library link-time optimization info tests passed: " << fastLibraries << " libraries without the info, " << created.Retaining() << " with it\n";
+}
+
 void pipelineLibraryTests(const Device& device) {
     std::lock_guard gpu(GpuMutex());
     const auto& context = device.GetContext();
@@ -11325,7 +11460,23 @@ void Run(Context context) {
 #endif
 
 int main(int argc, char** argv) {
-    try {
+    _ZSt13set_terminatePFvvE_nid_postfix([] {
+        std::fflush(nullptr);
+        std::cout.flush();
+        if (const auto failure = std::current_exception()) {
+            try {
+                std::rethrow_exception(failure);
+            } catch (const std::exception& error) {
+                std::cerr << error.what() << '\n';
+            } catch (...) {
+                std::cerr << "terminated by an exception that is not a std::exception\n";
+            }
+        } else {
+            std::cerr << "terminated without an active exception\n";
+        }
+        LibcTerminate_nid_no_patch(1);
+    });
+    {
         if (argc == 2 && std::string_view(argv[1]) == "--collect-mapping-history-only") {
             if (!collectMappingHistoryTests()) return 77;
             std::cout << "collect mapping history tests passed\n";
@@ -11438,6 +11589,10 @@ int main(int argc, char** argv) {
             }
             return sparseRunImportTests(device, true) ? 0 : 77;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--pipeline-library-lto-info-only") {
+            pipelineLibraryLtoInfoTests(device);
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--pipeline-library-only") {
             pipelineLibraryTests(device);
             return 0;
@@ -11508,6 +11663,10 @@ int main(int argc, char** argv) {
             std::cout << "CB resolves into tiled targets tests passed\n";
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--failed-check-only") {
+            failedCheckReportTests(device, recorder);
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--cube-only") {
             singleCubeTests(device, recorder);
             cubeExtraSliceTests(device, recorder);
@@ -11845,8 +12004,5 @@ int main(int argc, char** argv) {
         recorder.Activate();
         std::cout << "Recorder read tracking and label tests passed\n";
         return 0;
-    } catch (const std::exception& error) {
-        std::cerr << error.what() << '\n';
-        return 1;
     }
 }

core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp

@@ -47,6 +47,7 @@
 #include <atomic>
 #include <functional>
 #include <new>
+#include <chrono>
 #include <thread>
 #include <string>
 #include <string_view>
@@ -436,6 +437,90 @@ void verifySpecializedModuleRelease() {
     require(shared.spirv.data() == again.spirv.data(), "specialized module release: a second result of a live module copied its words");
 }
 
+const std::vector<std::uint32_t>& resultMemoPixelCode() {
+    static const std::vector<std::uint32_t> code{0xbf800000u, 0x7e000200u, 0xf800180fu, 0u, 0xbf810000u};
+    return code;
+}
+
+bool compileResultMemoPixel(std::uint32_t value) {
+    using namespace ShaderRecompiler;
+    std::array<std::uint32_t, 1> color{value};
+    auto request = FrontendRequest(resultMemoPixelCode(), color);
+    request.shader.stage = ShaderStage::Fragment;
+    request.context.compute.reset();
+    request.context.pixel = ShaderPixelStageInfo{};
+    request.context.pixel->wave32 = true;
+    request.context.pixel->targetOutputMode[0] = 9u;
+    request.context.waveSize = 32u;
+    request.useCache = true;
+    AgcDriver::ShaderMemory memory({});
+    const auto capture = memory.Capture(request);
+    const auto regions = memory.Regions();
+    request.context.memory = regions;
+    bool hit = false;
+    static_cast<void>(Recompile(request, *capture, &hit));
+    return hit;
+}
+
+void verifyResultMemoBudgetThreads() {
+    using namespace ShaderRecompiler;
+    require(!compileResultMemoPixel(1u) && compileResultMemoPixel(1u), "result memo budget threads: the memo did not keep a result");
+    const auto one = ResultMemoBytes();
+    const auto budget = 3u * one;
+    SetResultMemoBudget(budget);
+    static_cast<void>(compileResultMemoPixel(2u));
+    static_cast<void>(compileResultMemoPixel(3u));
+    require(ResultMemoBytes() == budget, "result memo budget threads: three results did not fill a budget of three");
+    std::vector<std::thread> late;
+    SetResultMemoTrimHook([&] {
+        const auto before = ResultMemoBytes();
+        for (const auto value : {5u, 6u}) late.emplace_back([value] { static_cast<void>(compileResultMemoPixel(value)); });
+        const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30);
+        while (ResultMemoBytes() < before + 2u * one && std::chrono::steady_clock::now() < deadline) std::this_thread::yield();
+    });
+    static_cast<void>(compileResultMemoPixel(4u));
+    for (auto& thread : late) thread.join();
+    require(late.size() == 2u, "result memo budget threads: the fourth result did not start a trim");
+    require(ResultMemoBytes() <= budget, "result memo budget threads: results inserted during a trim left the memo over its budget");
+    SetResultMemoBudget(0u);
+}
+
+void verifyResultMemoBudgetMasked() {
+    using namespace ShaderRecompiler;
+    require(ResultMemoBytes() == 0u, "result memo budget masked: the memo held results before the first compile");
+    require(!compileResultMemoPixel(1u), "result memo budget masked: the first compile hit the memo");
+    const auto first = ResultMemoBytes();
+    require(!compileResultMemoPixel(2u), "result memo budget masked: a new snapshot hit the memo");
+    const auto second = ResultMemoBytes() - first;
+    require(second != 0u && second < first, std::string("result memo budget masked: the masked table's entry was not counted (first result " + std::to_string(first) + " bytes, patched result " + std::to_string(second) + ")").c_str());
+    SetResultMemoBudget(1u);
+    require(ResultMemoBytes() == 0u, std::string("result memo budget masked: a trim to one byte left " + std::to_string(ResultMemoBytes()) + " bytes").c_str());
+    SetResultMemoBudget(0u);
+    require(!compileResultMemoPixel(3u), "result memo budget masked: a snapshot after the trim hit the memo");
+    require(ResultMemoBytes() == first, std::string("result memo budget masked: the trim left a masked entry to patch from (" + std::to_string(ResultMemoBytes()) + " bytes after one compile, " + std::to_string(first) + " for a full one)").c_str());
+}
+
+void verifyResultMemoBudget() {
+    using namespace ShaderRecompiler;
+    const auto compile = [](std::uint32_t value) { return compileResultMemoPixel(value); };
+    require(ResultMemoBytes() == 0u, "result memo budget: the memo held results before the first compile");
+    require(!compile(1u) && compile(1u), "result memo budget: the memo did not keep a result");
+    const auto one = ResultMemoBytes();
+    require(one != 0u, "result memo budget: a memo'd result counted no bytes");
+    const auto budget = 3u * one;
+    SetResultMemoBudget(budget);
+    for (const auto value : {2u, 3u, 4u}) {
+        require(!compile(value), "result memo budget: a new snapshot hit the memo");
+        require(ResultMemoBytes() <= budget, "result memo budget: the memo holds more than its budget");
+    }
+    require(ResultMemoBytes() < budget, "result memo budget: a fourth result over a budget of three was not trimmed");
+    require(compile(4u) && compile(3u), "result memo budget: the most recently used results were dropped");
+    require(!compile(1u), "result memo budget: the least recently used result outlived the trim");
+    SetResultMemoBudget(0u);
+    for (const auto value : {5u, 6u, 7u}) static_cast<void>(compile(value));
+    require(ResultMemoBytes() > budget, "result memo budget: clearing the budget kept bounding the memo");
+}
+
 ShaderRecompiler::RecompileRequest LayoutPairRequest(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, bool registration, std::uint32_t wordLimit) {
     using namespace ShaderRecompiler;
     auto request = FrontendRequest(code, userData);
@@ -8485,6 +8570,21 @@ int main(int argc, char** argv) {
             std::cout << "Prepared frontend reuse preserves full and partial-thread programs\n";
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--result-memo-budget") {
+            verifyResultMemoBudget();
+            std::cout << "The result memo keeps its most recently used results within its byte budget\n";
+            return 0;
+        }
+        if (argc == 2 && std::string_view(argv[1]) == "--result-memo-budget-masked") {
+            verifyResultMemoBudgetMasked();
+            std::cout << "The result memo budget counts and trims the masked table's entries\n";
+            return 0;
+        }
+        if (argc == 2 && std::string_view(argv[1]) == "--result-memo-budget-threads") {
+            verifyResultMemoBudgetThreads();
+            std::cout << "The result memo ends within its byte budget when results are inserted during a trim\n";
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--layout-pair") {
             verifyLayoutPair();
             std::cout << "Both push-constant layouts of a pixel shader compile from one program unchanged\n";

core/libs/prx/libSceAgcDriver/tests/Submit.cpp

@@ -952,7 +952,8 @@ int main(int argc, char** argv) {
         });
         for (auto& reader : readers) reader.join();
         for (const auto& snapshot : concurrent) check(snapshot == branched, "concurrent raw compute readers lost snapshot reuse");
-        alignas(256) std::array<std::array<std::uint32_t, 64>, 65> programs{};
+        struct alignas(256) RawProgram : std::array<std::uint32_t, 64> {};
+        std::vector<RawProgram> programs(4097);
         for (auto& program : programs) program[0] = 0xbf810000;
         const auto firstAddress = reinterpret_cast<std::uintptr_t>(programs.front().data());
         readers.clear();
@@ -962,6 +963,9 @@ int main(int argc, char** argv) {
         for (auto& reader : readers) reader.join();
         for (const auto& snapshot : concurrent) check(snapshot == concurrent.front(), "concurrent raw compute misses duplicated snapshots");
         const auto evicted = AgcDriver::DriverDetail::ReadRawShader(firstAddress);
+        for (std::size_t i = 1; i <= 400; ++i)
+            AgcDriver::DriverDetail::ReadRawShader(reinterpret_cast<std::uintptr_t>(programs[i].data()));
+        check(AgcDriver::DriverDetail::ReadRawShader(firstAddress) == evicted, "raw shader cache evicted a program while a title used only 401");
         for (std::size_t i = 1; i < programs.size(); ++i)
             AgcDriver::DriverDetail::ReadRawShader(reinterpret_cast<std::uintptr_t>(programs[i].data()));
         check(AgcDriver::DriverDetail::ReadRawShader(firstAddress) != evicted && evicted->code[0] == 0xbf810000,

core/libs/prx/libSceAgcDriver/tests/TextureFormat.cpp

@@ -117,6 +117,7 @@ void repackedFormatTests() {
     Require(IsRepackedTextureFormat(guest) && !IsRepackedTextureFormat(36u) && !IsRepackedTextureFormat(20u), "only texture format 43 is sampled through a repacked copy");
     Context context{};
     Require(SampledTextureFormat(context, guest) == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && ResolveTextureFormat(guest) == VK_FORMAT_R32_UINT, "format 43 must be sampled as B10G11R11_UFLOAT and stored as R32_UINT");
+    Require(SampledTextureFormat(context, 36u) == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && ResolveTextureFormat(36u) == VK_FORMAT_R32_UINT, "format 36 must be sampled as B10G11R11_UFLOAT and stored as R32_UINT");
     const auto field = [](std::uint32_t word, VkComponentSwizzle channel) -> std::pair<std::uint32_t, std::uint32_t> {
         switch (channel) {
             case VK_COMPONENT_SWIZZLE_R: return {word & 0x7ffu, 6u};
@@ -217,11 +218,22 @@ void packedFloatDccClearTests() {
             for (const auto texel : texels) Require(texel == expected, "packed float DCC clear did not encode RGB zero or one");
         }
     }
-    alignas(64) std::array<std::uint8_t, 4> keys{0xc0, 0xc0, 0xc0, 0xc0};
+    Require(DccClearTextureFormat(36u) == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && DccClearTextureFormat(43u) == VK_FORMAT_B10G11R11_UFLOAT_PACK32, "formats 36 and 43 DCC clears must use native packed float layout");
+    alignas(64) std::array<std::uint8_t, 4> keys{};
     GuestTextureResource resource{};
-    resource.format = 36;
     resource.dccAddress = reinterpret_cast<std::uint64_t>(keys.data());
-    Require(TextureClearKeys(resource, 1024) == DccKeys::Clear1111, "packed float texture discarded its white fast-clear keys");
+    for (const std::uint32_t format : {36u, 43u}) {
+        resource.format = format;
+        keys.fill(0xc0);
+        Require(TextureClearKeys(resource, 1024) == DccKeys::Clear1111, "packed float texture format " + std::to_string(format) + " discarded its white fast-clear keys");
+        keys.fill(0x80);
+        Require(TextureClearKeys(resource, 1024) == DccKeys::Clear1110, "packed float texture format " + std::to_string(format) + " discarded its 1110 fast-clear keys");
+        for (const auto [keysVal, expected] : {std::pair{DccKeys::Clear1111, 0x781e03c0u}, {DccKeys::Clear1110, 0x781e03c0u}, {DccKeys::Clear0000, 0u}}) {
+            std::array<std::uint32_t, 4> surface{};
+            ReadTextureSurface(resource, keysVal, std::as_writable_bytes(std::span(surface)));
+            for (const auto texel : surface) Require(texel == expected, "packed float format " + std::to_string(format) + " DCC clear surface read produced unexpected texel value");
+        }
+    }
 }
 
 void fillContractTests() {

core/libs/prx/libSceAgcDriver/tests/UnsupportedDiagnostics.cpp

@@ -4,5 +4,7 @@
 int main() {
     if (sceAgcDriverGetCoredumpAddress() != nullptr) return 1;
     if (sceAgcDriverSetTargetRingForDiag() != static_cast<int>(0x8A6D0001)) return 2;
+    auto* volatile linked = &sceAgcDriverSetFlip;
+    if (linked == nullptr) return 3;
     return 0;
 }

core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp

@@ -34,6 +34,7 @@ using namespace AgcDriver::GuestMemory;
 
 constexpr std::size_t Block = 65536;
 #ifdef _WIN32
+constexpr wchar_t MetadataCommitMode[] = L"--metadata-commit";
 constexpr wchar_t HostHoleMode[] = L"--host-hole";
 constexpr wchar_t ResidentRemapMode[] = L"--resident-direct-remap";
 constexpr wchar_t RewatchImportPagesMode[] = L"--rewatch-import-pages";
@@ -62,7 +63,9 @@ bool CheckMetadataCommit() {
     Require(committed < (64u << 20u), "write tracking committed dense generation tables for untouched guest ranges");
     const auto generation = TrackerGeneration();
     for (std::uint64_t offset = 0; offset + Block <= bytes; offset += std::uint64_t{1} << 32u) {
-        Require(NewestStamp(arena + offset, Block) == 0 && UnchangedSince(arena + offset, Block, generation), "an untouched leaf has a write stamp");
+        const bool host = GuestArena::GuestArenaHostRegionOverlaps_nid_postfix(arena + offset, Block);
+        Require(NewestStamp(arena + offset, Block) == 0, "an untouched leaf has a write stamp");
+        Require(UnchangedSince(arena + offset, Block, generation) != host, host ? "an arena host region reads as unchanged" : "an untouched leaf reads as changed");
     }
     PROCESS_MEMORY_COUNTERS_EX queried{};
     Require(K32GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PROCESS_MEMORY_COUNTERS*>(&queried), sizeof(queried)), "cannot measure private commit after empty-range queries");
@@ -1255,11 +1258,18 @@ std::uint64_t ChildHostHole() {
     return (arena + arenaBytes / 2) & ~(std::uint64_t{Block} - 1);
 }
 
-void CheckHostHoleInChild() {
+std::uint64_t ProbedHostHole() {
+    std::uintptr_t arena = 0;
+    std::size_t arenaBytes = 0;
+    GuestArena::GuestArenaRange_nid_postfix(&arena, &arenaBytes);
+    return arena + ((arenaBytes / 2) & ~((std::uint64_t{1} << 32u) - 1));
+}
+
+void CheckHostHoleInChild(const wchar_t* mode, std::uint64_t hole) {
     std::array<wchar_t, 32768> executable{};
     const DWORD length = GetModuleFileNameW(nullptr, executable.data(), static_cast<DWORD>(executable.size()));
     Require(length != 0 && length < executable.size(), "cannot name the test executable");
-    const std::wstring command = L"\"" + std::wstring(executable.data(), length) + L"\" " + HostHoleMode;
+    const std::wstring command = L"\"" + std::wstring(executable.data(), length) + L"\" " + mode;
     std::vector<wchar_t> commandLine(command.begin(), command.end());
     commandLine.push_back(L'\0');
     STARTUPINFOW startup{};
@@ -1270,7 +1280,7 @@ void CheckHostHoleInChild() {
     startup.hStdError = GetStdHandle(STD_ERROR_HANDLE);
     PROCESS_INFORMATION process{};
     Require(CreateProcessW(executable.data(), commandLine.data(), nullptr, nullptr, TRUE, CREATE_NO_WINDOW | CREATE_SUSPENDED, nullptr, nullptr, &startup, &process) != 0, "cannot create the host-hole child");
-    const bool reserved = VirtualAllocEx(process.hProcess, reinterpret_cast<void*>(ChildHostHole()), Block, MEM_RESERVE, PAGE_NOACCESS) != nullptr;
+    const bool reserved = VirtualAllocEx(process.hProcess, reinterpret_cast<void*>(hole), Block, MEM_RESERVE, PAGE_NOACCESS) != nullptr;
     const bool resumed = reserved && ResumeThread(process.hThread) != static_cast<DWORD>(-1);
     const bool exited = resumed && WaitForSingleObject(process.hProcess, 60000) == WAIT_OBJECT_0;
     if (!exited) {
@@ -1718,6 +1728,80 @@ void CheckUnwatchedImport() {
 #endif
 }
 
+struct HookCall {
+    std::uint64_t address;
+    std::size_t bytes;
+    bool watched;
+};
+
+std::vector<HookCall> hookCalls;
+std::uint64_t hookProbe = 0;
+std::size_t hookProbeBytes = 0;
+
+void RecordHook(std::uint64_t address, std::size_t bytes) {
+    hookCalls.push_back({address, bytes, Watched(hookProbe, hookProbeBytes)});
+}
+
+struct RecordedHook {
+    RecordedHook() {
+        hookCalls.clear();
+        SetUnwatchHook(&RecordHook);
+    }
+    ~RecordedHook() { SetUnwatchHook(nullptr); }
+    RecordedHook(const RecordedHook&) = delete;
+    RecordedHook& operator=(const RecordedHook&) = delete;
+};
+
+void RequireHookOver(std::uint64_t address, std::size_t bytes, const std::string& path) {
+    Require(hookCalls.size() == 1, path + " did not run the unwatch hook once");
+    const auto& call = hookCalls.front();
+    Require(call.address <= address && address + bytes <= call.address + call.bytes, path + " gave the unwatch hook a range that misses its own");
+    Require(call.watched, path + " ran the unwatch hook after its range left the write watch");
+}
+
+void CheckUnwatchHooks() {
+    void* memory = AllocateWatched(8 * Block);
+    const auto base = reinterpret_cast<std::uint64_t>(memory);
+    std::memset(memory, 0x11, 8 * Block);
+    Require(CollectWritesUncached(base, 8 * Block) != 0, "the hooked block is not collected");
+    const RecordedHook recorded;
+    hookProbe = base;
+    hookProbeBytes = Block;
+    Unwatch(base, Block);
+    RequireHookOver(base, Block, "Unwatch");
+    Require(!Watched(base, Block), "Unwatch left its range watched");
+
+    hookCalls.clear();
+    hookProbe = base + Block;
+    bool hookedBeforeImport = false;
+    Require(ImportWatched(base + Block, Block, [&] {
+        hookedBeforeImport = hookCalls.size() == 1;
+        return true;
+    }, false), "an import that does not stay watched failed");
+    RequireHookOver(base + Block, Block, "an import that does not stay watched");
+    Require(hookedBeforeImport, "an import that does not stay watched ran the unwatch hook after the import");
+    Require(!Watched(base + Block, Block), "an import that does not stay watched left its range watched");
+
+    hookCalls.clear();
+    Require(ImportWatched(base + 2 * Block, Block, [] { return true; }), "an import that stays watched failed");
+    Require(hookCalls.empty() && Watched(base + 2 * Block, Block), "an import whose page walk vouched for its pages ran the unwatch hook or gave up its range");
+
+#ifndef _WIN32
+    hookCalls.clear();
+    const auto late = base + 4 * Block;
+    hookProbe = late;
+    bool hookedBeforeLoss = false;
+    Require(ImportWatched(late, 3 * Block, [&] {
+        GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast<const void*>(late + 2 * Block), Block);
+        hookedBeforeLoss = !hookCalls.empty();
+        return true;
+    }), "an import whose range lost a block during the import failed");
+    RequireHookOver(late, 3 * Block, "an import whose page walk failed after the import");
+    Require(!hookedBeforeLoss, "an import whose page walk failed after the import ran the unwatch hook before it");
+    Require(!Watched(late, Block), "an import whose page walk failed after the import left its range watched");
+#endif
+}
+
 class PartlyCommitted {
 public:
     static constexpr std::size_t Bytes = 4 * Block;
@@ -1881,7 +1965,7 @@ void CheckAdoptLatestEpoch() {
 int main() {
     try {
 #ifdef _WIN32
-        if (std::wstring_view(GetCommandLineW()).ends_with(L"--metadata-commit")) {
+        if (std::wstring_view(GetCommandLineW()).ends_with(MetadataCommitMode)) {
             return CheckMetadataCommit() ? 0 : 77;
         }
 #endif
@@ -1936,7 +2020,8 @@ int main() {
         CheckPrivateMappingReuse();
         CheckRewatch();
         CheckUntrackedHint();
-        CheckHostHoleInChild();
+        CheckHostHoleInChild(HostHoleMode, ChildHostHole());
+        CheckHostHoleInChild(MetadataCommitMode, ProbedHostHole());
         CheckFailedCollectUnwatch();
 #endif
         CheckVersionRanges();
@@ -1948,6 +2033,7 @@ int main() {
         CheckStoreAcrossImports();
         CheckImportWindowFootprint();
         CheckUnwatchedImport();
+        CheckUnwatchHooks();
         CheckCommittedRanges();
         CheckAdoptLatestEpoch();
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/execution/AlphaToCoverage.cpp

@@ -808,11 +808,12 @@ void DccRefillTests(AgcDriver::VulkanDevice& device, std::uint32_t selectedSampl
                     };
                     record(metadata, dccBytes, 0xc0c0c0c0u);
                     if (cmask != nullptr) record(*cmask, target.cmaskBytes, 0xccccccccu);
-                    if (!AgcDriver::GuestMemory::Watched(metadata.Address(), dccBytes) || (cmask != nullptr && !AgcDriver::GuestMemory::Watched(cmask->Address(), target.cmaskBytes))) expected = "requires watched guest memory";
+                    const bool unwatched = !AgcDriver::GuestMemory::Watched(metadata.Address(), dccBytes) || (cmask != nullptr && !AgcDriver::GuestMemory::Watched(cmask->Address(), target.cmaskBytes));
 #ifdef _WIN32
-                    rewatchable = !expected.empty();
+                    if (unwatched) expected = "requires watched guest memory";
+                    rewatchable = unwatched;
 #else
-                    Require(mode == 6 || expected.empty(), "a GPU fill of multisampled DCC metadata left it unwatched although host imports stay watched");
+                    Require(unwatched == (mode == 6), "a GPU fill of multisampled DCC metadata did not take the write-watch mode of its host import");
 #endif
                 } else if (mode == 2) {
                     fill(metadata, dccBytes / 2u, 0xc0c0c0c0u);
@@ -914,6 +915,170 @@ void DccImportWindowTests(AgcDriver::VulkanDevice& device, std::uint32_t selecte
     }
 }
 
+#ifndef _WIN32
+const char* DccComparedImportTests(AgcDriver::VulkanDevice& device, std::uint32_t selectedSamples) {
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    if (context.hostImportAlignment == 0) return "skipped, the device imports no host memory, so no multisampled DCC plane lies in a compared host import";
+    Require(AgcDriver::GuestMemory::WriteWatched(), "the guest arena is not write-watched");
+    const auto decided = AgcDriver::Graphics::PrepareImportWatch(context);
+    if (decided != AgcDriver::Graphics::ImportWatch::Unwatch) return "skipped, the device reports host imports that keep write protection, so the write-watch probe compares none of them";
+    struct Restore {
+        const AgcDriver::Graphics::Context& context;
+        AgcDriver::Graphics::ImportWatch decided;
+        ~Restore() { AgcDriver::Graphics::SetImportWatch(context, decided); }
+    } restore{context, decided};
+    const auto probe = AgcDriver::Graphics::ProbeImportWriteProtection(context);
+    const bool watchedImportsHold = probe.failure == nullptr && probe.writtenAtImport == 0 && probe.writtenAfterSubmit == 0;
+    if (!watchedImportsHold) std::cout << "the watched-import control case is skipped: the device reports watched host memory written when it imports or reads it\n";
+    const ColorTargetLayout layout(Width, Height, ColorTileMode::RenderTarget, 8);
+    std::vector<std::unique_ptr<Block>> blocks;
+    struct ClearTargets {
+        AgcDriver::VulkanDevice& device;
+        ~ClearTargets() {
+            std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex());
+            device.SubmitRecorded(true);
+            device.WaitIdle();
+            AgcDriver::Graphics::ClearMultisampleTargets(device.Device());
+            AgcDriver::Graphics::ClearCachedTextures(device.Device());
+        }
+    } clearTargets{device};
+    const auto expectRange = [](std::string_view message, const char* plane, std::uint64_t address, std::size_t bytes, std::uint32_t samples) {
+        char detail[160];
+        std::snprintf(detail, sizeof(detail), "(%s range 0x%llx+0x%zx, %u samples)", plane, static_cast<unsigned long long>(address), bytes, samples);
+        Require(message.find(detail) != std::string_view::npos, "a refusal over a compared host import did not identify the affected plane, range and sample count");
+    };
+    const Color written{0.25f, 0.25f, 0.25f, 0.25f};
+    const Color initial{0.0f, 0.0f, 0.0f, 1.0f};
+    constexpr std::string_view deviceLimit = "AGC graphics: the device cannot render color with the target's sample count";
+    for (const auto samples : {2u, 4u, 8u}) {
+        if (selectedSamples != 0 && samples != selectedSamples) continue;
+        for (const auto mode : {7u, 8u, 9u, 10u, 11u, 0u, 1u, 2u, 3u, 4u, 5u, 6u}) {
+            if (mode >= 7 && samples != 8) continue;
+            if (mode == 5 && !watchedImportsHold) continue;
+            const bool watchedImports = mode == 5;
+            AgcDriver::Graphics::SetImportWatch(context, watchedImports ? AgcDriver::Graphics::ImportWatch::Watch : decided);
+            blocks.push_back(std::make_unique<Block>(layout.Bytes() * samples));
+            auto& source = *blocks.back();
+            std::memset(source.data, 0, source.bytes);
+            blocks.push_back(std::make_unique<Block>(BlockBytes));
+            auto& metadata = *blocks.back();
+            std::memset(metadata.data, 0x40, metadata.bytes);
+            blocks.push_back(std::make_unique<Block>(BlockBytes));
+            auto& cmask = *blocks.back();
+            std::memset(cmask.data, 0xcc, cmask.bytes);
+            auto target = DecodeTarget(source, samples);
+            target.dccAddress = metadata.Address();
+            target.dccPipeAligned = true;
+            target.dccAlphaOnMsb = true;
+            target.cmaskAddress = cmask.Address();
+            target.cmaskBytes = AgcDriver::Graphics::CmaskBytes(Width, Height);
+            target.cmaskFastClear = true;
+            Block* fmask = nullptr;
+            if (mode >= 7) {
+                blocks.push_back(std::make_unique<Block>(2u * BlockBytes));
+                fmask = blocks.back().get();
+                std::memset(fmask->data, 0x11, fmask->bytes);
+                target.fmaskAddress = fmask->Address();
+                target.fmaskPipeBankXor = 0;
+            }
+            const std::array<std::pair<const Block*, std::size_t>, 3> planes{{{&source, static_cast<std::size_t>(AgcDriver::Graphics::MultisampledColorBytes(target))}, {&metadata, AgcDriver::Graphics::MultisampledDccBytes(target)}, {&cmask, static_cast<std::size_t>(target.cmaskBytes)}}};
+            if (mode == 4) {
+                AgcDriver::GuestMemory::Unwatch(source.Address(), source.bytes);
+            } else if (mode == 11) {
+                AgcDriver::GuestMemory::Unwatch(fmask->Address(), fmask->bytes);
+            } else {
+                std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex());
+                for (std::size_t plane = 0; plane < planes.size(); ++plane) {
+                    if ((mode == 6 && plane != 1) || mode == 8) continue;
+                    const auto* import = AgcDriver::Graphics::HostImportFor(context, planes[plane].first->Address(), planes[plane].first->bytes);
+                    Require(import != nullptr, "the host import of a multisampled DCC plane failed");
+                    Require(import->unwatched != watchedImports && AgcDriver::GuestMemory::Watched(planes[plane].first->Address(), planes[plane].second) == watchedImports, "a host import of a multisampled DCC plane did not take the write-watch mode the test set");
+                }
+                if (fmask != nullptr) {
+                    const auto* import = AgcDriver::Graphics::HostImportFor(context, fmask->Address(), fmask->bytes);
+                    Require(import != nullptr && import->unwatched && !AgcDriver::GuestMemory::Watched(fmask->Address(), fmask->bytes), "the host import of the FMASK plane was not compared");
+                }
+                if (mode == 8) {
+                    for (const auto& [block, bytes] : planes) Require(AgcDriver::GuestMemory::Watched(block->Address(), bytes), "the FMASK plane's import took a color or metadata plane out of the write watch");
+                }
+            }
+            const char* fmaskCase = mode == 7 ? "color, DCC, CMASK and FMASK compared" : mode == 8 ? "only the FMASK compared" : mode == 9 ? "a CPU store of a new byte into a compared FMASK" : mode == 10 ? "a same-value driver store into a compared FMASK" : "an explicitly unwatched FMASK";
+            const std::string_view refusal = mode == 4 ? "initial multisampled DCC contents require watched" : mode == 11 ? "native FMASK contents require watched guest memory" : "";
+            bool refused = false;
+            try {
+                device.ColorMetadataPass({AgcDriver::Graphics::ColorMetadataPass::Mode::FmaskDecompress, {target}});
+                Draw(device, target, false, Float32Export, Program(written, Float32Export, 1u), false, true);
+            } catch (const std::runtime_error& error) {
+                if (mode >= 7 && mode != 11 && std::string_view(error.what()) == deviceLimit) {
+                    std::printf("8 samples, %s: the FMASK passed the initial import checks, then %s, so its reuse is not tested\n", fmaskCase, error.what());
+                    programCount = 0;
+                    continue;
+                }
+                if (refusal.empty()) throw;
+                refused = std::string_view(error.what()).find(refusal) != std::string_view::npos;
+            }
+            if (!refusal.empty()) {
+                Require(refused, mode == 11 ? "an explicitly unwatched FMASK range was not refused at its initial import" : "an explicitly unwatched multisampled DCC color range was not refused at its initial import");
+                if (mode == 11) std::printf("8 samples, %s: refused at the initial import\n", fmaskCase);
+                programCount = 0;
+                continue;
+            }
+            blocks.push_back(std::make_unique<Block>(layout.Bytes()));
+            auto& resolved = *blocks.back();
+            AgcDriver::Graphics::ColorMetadataPass pass{};
+            pass.mode = AgcDriver::Graphics::ColorMetadataPass::Mode::Resolve;
+            pass.source = target;
+            pass.targets = {DecodeTarget(resolved)};
+            const auto resolve = [&] {
+                std::memset(resolved.data, 0, resolved.bytes);
+                device.ColorMetadataPass(pass);
+                const auto stored = ReadBack(device, resolved);
+                for (std::uint32_t y = 0; y < Height; ++y) {
+                    for (std::uint32_t x = 0; x < Width; ++x) {
+                        for (std::uint32_t channel = 0; channel < 4; ++channel) {
+                            const auto expectedValue = x < ScissorWidth ? initial[channel] + (written[channel] - initial[channel]) / static_cast<float>(samples) : initial[channel];
+                            Require(FromHalf(stored[layout.Offset(x, y) / 2u + channel]) == expectedValue, "a multisampled DCC target over a compared host import did not start from the fast clear its metadata holds");
+                        }
+                    }
+                }
+            };
+            resolve();
+            if (mode == 0 || (mode >= 5 && mode <= 8)) {
+                Draw(device, target, false, Float32Export, Program(written, Float32Export, 1u), false, true);
+                resolve();
+                if (mode >= 7) std::printf("8 samples, %s: imported and reused\n", fmaskCase);
+                programCount = 0;
+                continue;
+            }
+            const char* plane = mode == 1 ? "color" : mode == 2 ? "DCC" : mode == 3 ? "CMASK" : "FMASK";
+            const auto& [block, bytes] = mode >= 9 ? std::pair<const Block*, std::size_t>{fmask, fmask->bytes} : planes[mode - 1u];
+            if (mode == 1) static_cast<volatile std::uint8_t*>(source.data)[0] = 0x5au;
+            if (mode == 2) static_cast<volatile std::uint8_t*>(metadata.data)[0] = 0x00u;
+            if (mode == 3) {
+                const std::array<std::byte, 1> same{std::byte{0xcc}};
+                AgcDriver::GuestMemory::Write(cmask.Address(), same);
+            }
+            if (mode == 9) static_cast<volatile std::uint8_t*>(fmask->data)[0] = 0x5au;
+            if (mode == 10) {
+                const std::array<std::byte, 1> same{std::byte{0x11}};
+                AgcDriver::GuestMemory::Write(fmask->Address(), same);
+            }
+            const std::string_view storeRefusal = mode >= 9 ? "writes to a resident native FMASK" : "writes to a resident multisampled DCC";
+            try {
+                Draw(device, target, false, Float32Export, Program(written, Float32Export, 1u), false, true);
+            } catch (const std::runtime_error& error) {
+                refused = std::string_view(error.what()).find(storeRefusal) != std::string_view::npos;
+                if (refused) expectRange(error.what(), plane, block->Address(), bytes, samples);
+            }
+            Require(refused, mode == 3 ? "a same-value driver store over a compared CMASK range did not invalidate multisampled DCC residency" : mode == 10 ? "a same-value driver store over a compared FMASK range did not invalidate its residency" : mode == 9 ? "a CPU store of a new byte over a compared FMASK range did not invalidate its residency" : "a CPU store of new bytes over a compared multisampled DCC plane did not invalidate its residency");
+            if (mode >= 9) std::printf("8 samples, %s: refused at reuse\n", fmaskCase);
+            programCount = 0;
+        }
+    }
+    return nullptr;
+}
+#endif
+
 int main(int argc, char** argv) {
     try {
         const auto device = OpenVulkanTestDevice();
@@ -933,6 +1098,29 @@ int main(int argc, char** argv) {
             std::cout << "Multisampled DCC fast clear tests passed\n";
             return 0;
         }
+        if ((argc == 2 || argc == 3) && std::string_view(argv[1]) == "--dcc-compared-import") {
+            const auto samples = argc == 2 ? 0u : std::string_view(argv[2]) == "2" ? 2u : std::string_view(argv[2]) == "4" ? 4u : std::string_view(argv[2]) == "8" ? 8u : 0u;
+            Require(argc == 2 || samples != 0, "unknown DCC test sample count");
+#ifdef _WIN32
+            std::puts("skipped, Linux comparison mode only: Windows host imports settle on a second queue or keep their write watch");
+            return VulkanTestSkipped;
+#else
+            const char* skipped = nullptr;
+            try {
+                skipped = DccComparedImportTests(*device, samples);
+            } catch (const std::runtime_error& error) {
+                if (std::string_view(error.what()) != "AGC graphics: the device cannot render color with the target's sample count") throw;
+                std::printf("skipped, the device lacks the requested multisample color attachments: %s\n", error.what());
+                return VulkanTestSkipped;
+            }
+            if (skipped != nullptr) {
+                std::puts(skipped);
+                return VulkanTestSkipped;
+            }
+            std::cout << "Multisampled DCC compared host import tests passed\n";
+            return 0;
+#endif
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--dithered8") {
             try {
                 DitheredCoverage(*device);

core/libs/prx/libSceAgcDriver/tests/execution/DepthBoundsEmulationDraw.cpp

@@ -0,0 +1,173 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "Recompiler.hpp"
+#include "DepthFastClearHarness.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <optional>
+#include <span>
+#include <stdexcept>
+#include <vector>
+
+namespace {
+
+namespace Harness = DepthFastClearHarness;
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+struct Bounds {
+    float min;
+    float max;
+};
+
+struct Case {
+    float stored;
+    Bounds bounds;
+    bool covered;
+    const char* what;
+};
+
+alignas(4096) std::array<std::array<float, Harness::Width * Harness::Height>, 16> Depths{};
+alignas(256) std::array<std::array<std::array<float, 4>, 3>, 64> VertexSets{};
+std::size_t vertexSets = 0;
+
+std::uint32_t Draw(AgcDriver::VulkanDevice& device, std::uint64_t depth, float z, std::optional<Bounds> bounds, bool emulate) {
+    Require(vertexSets < VertexSets.size(), "the test ran out of vertex sets");
+    auto& vertices = VertexSets[vertexSets++];
+    vertices = {{{-1.0f, -1.0f, z, 1.0f}, {3.0f, -1.0f, z, 1.0f}, {-1.0f, 3.0f, z, 1.0f}}};
+    Harness::Pixels.fill(std::byte{0});
+
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 32u, 32u, std::nullopt, std::nullopt};
+    state.color = {reinterpret_cast<std::uintptr_t>(Harness::Pixels.data()), {Harness::Width, Harness::Height}, VK_FORMAT_R8G8B8A8_UNORM, Harness::Pixels.size(), 0xe4u};
+    state.colors = {state.color};
+    state.hasColorTarget = true;
+    state.depth = AgcDriver::Graphics::DepthTarget{depth, 0, {Harness::Width, Harness::Height}, VK_FORMAT_D32_SFLOAT, 1.0f, 0};
+    state.depthTest = !bounds.has_value();
+    state.depthWrite = !bounds.has_value();
+    state.depthCompare = VK_COMPARE_OP_ALWAYS;
+    if (bounds) {
+        state.depthBoundsTest = true;
+        state.minDepthBounds = bounds->min;
+        state.maxDepthBounds = bounds->max;
+    }
+    state.renderExtent = {Harness::Width, Harness::Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Harness::Height), static_cast<float>(Harness::Width), -static_cast<float>(Harness::Height), 0.0f, 1.0f};
+    state.negativeOneToOne = false;
+    state.scissor = {{0, 0}, {Harness::Width, Harness::Height}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blend.colorWriteMask = 15;
+    state.blends = {state.blend};
+    state.blendConstants = {};
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.wave32 = true;
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY);
+    pixel.posX = true;
+    pixel.posY = true;
+    pixel.targetOutputMode[0] = 9;
+    pixel.targetExportMapping.fill(0xe4u);
+    if (bounds && emulate) {
+        auto context = device.graphicsContext();
+        context.depthBounds = false;
+        context.depthRangeUnrestricted = false;
+        AgcDriver::Graphics::ApplyDepthBoundsEmulation(context, state, pixel);
+        Require(!state.depthBoundsTest && state.depthTest && !state.depthWrite, "the depth bounds test was not emulated");
+    }
+
+    auto target = device.Target();
+    target.depthRangeRemap = false;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = Harness::BufferDescriptor(vertices.data(), 16u, static_cast<std::uint32_t>(vertices.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(Harness::VertexCode.data()), std::as_bytes(std::span(Harness::VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(Harness::VertexCode.data()), Harness::VertexCode, 0, {}},
+        {32, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+    std::vector<std::uint32_t> pixelUserData(4, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(Harness::PixelCode.data()), std::as_bytes(std::span(Harness::PixelCode))}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(Harness::PixelCode.data()), Harness::PixelCode, 0, {}},
+        {32, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(vertices.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+    device.WaitIdle();
+    std::uint32_t covered = 0;
+    for (std::size_t index = 0; index < Harness::Pixels.size() / 4u; ++index) {
+        if (std::to_integer<std::uint8_t>(Harness::Pixels[index * 4u]) == 255u) ++covered;
+    }
+    return covered;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        const auto context = device->graphicsContext();
+        const float below = std::nextafter(0.25f, 0.0f);
+        const float above = std::nextafter(0.25f, 1.0f);
+        const std::array<Case, 12> cases{{
+            {0.25f, {0.0f, 0.25f}, true, "stored 0.25, upper bound equal to it"},
+            {0.25f, {0.0f, below}, false, "stored 0.25, upper bound one step below it"},
+            {0.25f, {0.25f, 1.0f}, true, "stored 0.25, lower bound equal to it"},
+            {0.25f, {above, 1.0f}, false, "stored 0.25, lower bound one step above it"},
+            {0.000125f, {0.0f, 0.000125f}, true, "stored 0.000125 inside [0, 0.000125]"},
+            {0.5f, {0.0f, 0.000125f}, false, "stored 0.5 outside [0, 0.000125]"},
+            {0.5f, {0.000125f, 1.0f}, true, "stored 0.5 inside [0.000125, 1]"},
+            {0.0f, {0.000125f, 1.0f}, false, "stored 0 outside [0.000125, 1]"},
+            {0.0f, {0.0f, 1.0f}, true, "stored 0 inside [0, 1]"},
+            {1.0f, {0.0f, 1.0f}, true, "stored 1 inside [0, 1]"},
+            {1.0f, {0.0f, 0.5f}, false, "stored 1 outside [0, 0.5]"},
+            {0.75f, {0.5f, 1.0f}, true, "stored 0.75 inside [0.5, 1]"},
+        }};
+        std::size_t compared = 0;
+        for (std::size_t index = 0; index < cases.size(); ++index) {
+            const auto& test = cases[index];
+            const auto depth = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(Depths[index].data()));
+            Require(Draw(*device, depth, test.stored, std::nullopt, false) == Harness::Covered, "the draw that stores the depth did not cover the target");
+            const auto expected = test.covered ? Harness::Covered : 0u;
+            for (const bool emulate : {true, false}) {
+                if (!emulate && !context.depthBounds) continue;
+                const auto passed = Draw(*device, depth, 0.5f, test.bounds, emulate);
+                if (passed != expected) {
+                    char detail[256];
+                    std::snprintf(detail, sizeof(detail), "%s (%s): %u of %u pixels passed, expected %s", test.what, emulate ? "emulated" : "native", passed, Harness::Covered, test.covered ? "all" : "none");
+                    throw std::runtime_error(detail);
+                }
+                if (!emulate) ++compared;
+            }
+        }
+        std::printf("depth bounds emulation draw tests passed (%zu cases, %zu compared with the native test)\n", cases.size(), compared);
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/DepthPlaneGuestLayers.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Execution/include/BdaFeatures.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
@@ -46,6 +47,18 @@ PFN_vkCmdCopyImageToBuffer realCopyImageToBuffer = nullptr;
 PFN_vkCmdCopyBufferToImage realCopyBufferToImage = nullptr;
 std::atomic<std::uint32_t> imageToBufferCopies{0};
 std::atomic<std::uint32_t> bufferToImageCopies{0};
+std::atomic<VkImage> lastFilledImage{VK_NULL_HANDLE};
+PFN_vkAllocateMemory realAllocateMemory = nullptr;
+std::atomic<std::uint32_t> memoryAllocations{0};
+std::atomic<VkDeviceSize> lastAllocationBytes{0};
+std::atomic<std::uint32_t> lastAllocationType{0};
+
+VKAPI_ATTR VkResult VKAPI_CALL CountAllocateMemory(VkDevice device, const VkMemoryAllocateInfo* info, const VkAllocationCallbacks* allocator, VkDeviceMemory* memory) {
+    memoryAllocations.fetch_add(1);
+    lastAllocationBytes = info->allocationSize;
+    lastAllocationType = info->memoryTypeIndex;
+    return realAllocateMemory(device, info, allocator, memory);
+}
 
 VKAPI_ATTR void VKAPI_CALL CountImageToBuffer(VkCommandBuffer commands, VkImage image, VkImageLayout layout, VkBuffer buffer, std::uint32_t count, const VkBufferImageCopy* regions) {
     imageToBufferCopies.fetch_add(1);
@@ -54,6 +67,7 @@ VKAPI_ATTR void VKAPI_CALL CountImageToBuffer(VkCommandBuffer commands, VkImage
 
 VKAPI_ATTR void VKAPI_CALL CountBufferToImage(VkCommandBuffer commands, VkBuffer buffer, VkImage image, VkImageLayout layout, std::uint32_t count, const VkBufferImageCopy* regions) {
     bufferToImageCopies.fetch_add(1);
+    lastFilledImage = image;
     realCopyBufferToImage(commands, buffer, image, layout, count, regions);
 }
 
@@ -64,6 +78,10 @@ VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL CountingDeviceProc(VkDevice device, con
         realCopyImageToBuffer = reinterpret_cast<PFN_vkCmdCopyImageToBuffer>(function);
         return reinterpret_cast<PFN_vkVoidFunction>(&CountImageToBuffer);
     }
+    if (std::strcmp(name, "vkAllocateMemory") == 0) {
+        realAllocateMemory = reinterpret_cast<PFN_vkAllocateMemory>(function);
+        return reinterpret_cast<PFN_vkVoidFunction>(&CountAllocateMemory);
+    }
     if (std::strcmp(name, "vkCmdCopyBufferToImage") == 0) {
         realCopyBufferToImage = reinterpret_cast<PFN_vkCmdCopyBufferToImage>(function);
         return reinterpret_cast<PFN_vkVoidFunction>(&CountBufferToImage);
@@ -434,6 +452,76 @@ void RecordedUploads(const Context& context, const WatchedBlock& block) {
     recorder.Sync();
 }
 
+void PooledPlaneCopies(const Context& context, const WatchedBlock& first, const WatchedBlock& second) {
+    ClearDepthSurfaces(context.device);
+    FillLayer(first.Data() + LayerBytes, 0.25f);
+    FillLayer(second.Data() + LayerBytes, 0.75f);
+    DepthSurfaceView(context, DepthTarget{first.Address(), 0, {Side, Side}, VK_FORMAT_D32_SFLOAT, 1.0f, 0, 0});
+    DepthSurfaceView(context, DepthTarget{second.Address(), 0, {Side, Side}, VK_FORMAT_D32_SFLOAT, 1.0f, 0, 0});
+    const auto sample = [&](const WatchedBlock& block, int round) {
+        lastFilledImage = VK_NULL_HANDLE;
+        bufferToImageCopies = 0;
+        Require(DepthSurfaceTexture(context, words, ArrayOver(block.Address()), components) != nullptr, "the 2D array over a depth surface was not sampled through its depth planes in round " + std::to_string(round));
+        Require(DepthPlaneCopyCount() == 1, "the depth plane copies outgrew a budget that holds one in round " + std::to_string(round) + ", " + std::to_string(DepthPlaneCopyCount()) + " copies");
+        Require(lastFilledImage.load() != VK_NULL_HANDLE && bufferToImageCopies.load() == 1, "a plane copy made again over a full budget was not filled from its surface once in round " + std::to_string(round) + ", " + std::to_string(bufferToImageCopies.load()) + " fills");
+    };
+    sample(first, 0);
+    sample(second, 0);
+    const auto keepsCopies = [&] {
+        if (context.bufferPool == nullptr) return false;
+        const VkMemoryAllocateInfo probe{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, nullptr, lastAllocationBytes.load(), lastAllocationType.load()};
+        const auto memory = context.bufferPool->TakeImageMemory(probe, "vkAllocateMemory pool probe");
+        const auto before = context.bufferPool->RetainedImages();
+        context.bufferPool->PutImageMemory(memory, probe.allocationSize, probe.memoryTypeIndex);
+        const bool kept = context.bufferPool->RetainedImages() > before;
+        return kept;
+    }();
+    memoryAllocations = 0;
+    for (int round = 1; round <= 3; ++round) {
+        sample(first, round);
+        sample(second, round);
+    }
+    if (!keepsCopies) {
+        std::printf("note: the device memory pool does not keep a block of a plane copy's size and memory type here (not device-tiered, or its heap over budget); the reuse of evicted copies' memory is not checked\n");
+    } else {
+        Require(memoryAllocations.load() == 0, "plane copies evicted and made again over a full budget allocated device memory " + std::to_string(memoryAllocations.load()) + " times instead of taking the evicted copies' memory back");
+    }
+    ClearDepthSurfaces(context.device);
+}
+
+void SharedLayerRanges(const Context& context, const WatchedBlock& block) {
+    ClearDepthSurfaces(context.device);
+    auto resource = ArrayOver(block.Address());
+    resource.depthOrLastArray = 3;
+    const auto geometry = DescribeSurface(resource);
+    for (std::uint32_t layer = 0; layer < 4; ++layer) {
+        DepthSurfaceView(context, DepthTarget{block.Address() + geometry.GuestLayerOffset(layer), 0, {Side, Side}, VK_FORMAT_D32_SFLOAT, 1.0f, 0, 0});
+    }
+    DepthSurfaceView(context, DepthTarget{block.Address() + 16u * LayerBytes, 0, {Side, Side}, VK_FORMAT_D32_SFLOAT, 1.0f, 0, 0});
+    const auto sample = [&](std::uint32_t lastArray, std::uint32_t baseArray, std::uint32_t fromSlices, std::uint32_t intoCopy) {
+        auto range = resource;
+        range.depthOrLastArray = lastArray;
+        range.baseArray = baseArray;
+        imageToBufferCopies = 0;
+        bufferToImageCopies = 0;
+        const auto texture = DepthSurfaceTexture(context, words, range, components);
+        const auto when = "layers " + std::to_string(baseArray) + "-" + std::to_string(lastArray);
+        Require(texture != nullptr, "the 2D array over depth surfaces was not sampled through its depth planes for " + when);
+        if (imageToBufferCopies.load() != fromSlices || bufferToImageCopies.load() != intoCopy) {
+            throw std::runtime_error("sampling " + when + " of the depth surfaces recorded " + std::to_string(imageToBufferCopies.load()) + " depth surface copies and " + std::to_string(bufferToImageCopies.load()) + " plane copy fills, expected " + std::to_string(fromSlices) + " and " + std::to_string(intoCopy));
+        }
+        const auto view = texture->SampledViewRange(false);
+        Require(view.layers == lastArray + 1u - baseArray, "the view sampling " + when + " covers " + std::to_string(view.layers) + " layers, not the texture's own");
+        return texture;
+    };
+    sample(3, 0, 4, 1);
+    sample(2, 0, 0, 0);
+    sample(1, 0, 0, 0);
+    sample(2, 1, 0, 0);
+    sample(3, 0, 0, 0);
+    ClearDepthSurfaces(context.device);
+}
+
 void RunCubeGather(const Context& base, const WatchedBlock& block) {
     TextureDetiler detiler(base);
     auto context = base;
@@ -508,6 +596,11 @@ void RunCubeTiledSlice(const Context& base, const WatchedBlock& block) {
 
 int main() {
     try {
+#ifdef _WIN32
+        _putenv_s("APS5_DEPTH_PLANE_COPY_MIB", "1");
+#else
+        setenv("APS5_DEPTH_PLANE_COPY_MIB", "1", 1);
+#endif
         if (!AgcDriver::GuestMemory::WriteWatched()) {
             std::puts("skipped, guest memory has no write watch");
             return Skipped;
@@ -528,6 +621,11 @@ int main() {
             Require(AgcDriver::GuestMemory::Watched(block.Address(), BlockBytes), "depth plane guest layers: the guest block is not write-watched");
             Run(context, block);
             RecordedUploads(context, block);
+            {
+                WatchedBlock second;
+                PooledPlaneCopies(context, block, second);
+            }
+            SharedLayerRanges(context, block);
             RunCubeGather(context, block);
             RunCubeTiledSlice(context, block);
             SparseGuestLayers(context);

core/libs/prx/libSceAgcDriver/tests/execution/DispatchAheadAdoptCheck.cpp

@@ -0,0 +1,208 @@
+#include "VulkanTestDevice.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Submit/include/Acb.hpp"
+#include <array>
+#include <atomic>
+#include <chrono>
+#include <cstdint>
+#include <initializer_list>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace {
+
+void Require(bool condition, const std::string& message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+alignas(256) std::array<std::uint32_t, 7> CounterCode{0xf4080200u, 0xfa000000u, 0xbf8cc07fu, 0x7e000281u, 0xe0c80000u, 0x80020000u, 0xbf810000u};
+alignas(256) std::array<std::uint32_t, 7> StoreCode{0xf4080200u, 0xfa000000u, 0xbf8cc07fu, 0x7e000202u, 0xe0700000u, 0x80020000u, 0xbf810000u};
+alignas(256) std::array<std::uint32_t, 15> SpinCode{0xf4080200u, 0xfa000000u, 0xbe840381u, 0x9304ff04u, 0x0019660du, 0x8004ff04u, 0x3c6ef35fu, 0x80828102u, 0xbf078002u, 0xbf85fff9u, 0xbf8cc07fu, 0x7e000204u, 0xe0700000u, 0x80020000u, 0xbf810000u};
+
+constexpr std::uint32_t Compute = 0x20u;
+constexpr std::uint32_t OtherCompute = 0x28u;
+constexpr std::uint32_t SpinIterations = 1u << 20u;
+
+alignas(256) std::array<std::uint32_t, 4> Stale{};
+alignas(256) std::array<std::uint32_t, 4> Fresh{};
+alignas(256) std::array<std::uint32_t, 4> Target{};
+alignas(256) std::array<std::uint32_t, 4> TargetDescriptor{};
+alignas(256) std::array<std::uint32_t, 4> Spun{};
+alignas(256) std::array<std::uint32_t, 4> SpunDescriptor{};
+alignas(256) volatile std::uint32_t Gate = 1;
+alignas(256) volatile std::uint32_t Done = 0;
+alignas(256) volatile std::uint32_t OtherDone = 0;
+
+enum class Window { None, CpuStore, QueuedGpuStore };
+
+std::atomic<Window> Armed{Window::None};
+std::uint32_t SpinDispatches = 16;
+std::vector<std::uint32_t> OtherWords;
+bool SawPending = false;
+
+std::uint32_t Low(const volatile void* pointer) {
+    return static_cast<std::uint32_t>(reinterpret_cast<std::uintptr_t>(pointer));
+}
+
+std::uint32_t High(const volatile void* pointer) {
+    return static_cast<std::uint32_t>(static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(pointer)) >> 32u);
+}
+
+std::array<std::uint32_t, 4> Descriptor(const void* pointer, std::uint32_t bytes) {
+    return {Low(pointer), High(pointer) & 0xffffu, bytes, 0x31016facu};
+}
+
+void SetShaderRegister(std::vector<std::uint32_t>& words, std::uint32_t offset, std::uint32_t value) {
+    words.insert(words.end(), {0xc0017600u, offset, value});
+}
+
+void Program(std::vector<std::uint32_t>& words, const void* code, std::initializer_list<std::uint32_t> userData) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(code));
+    SetShaderRegister(words, 0x20c, static_cast<std::uint32_t>(address >> 8u));
+    SetShaderRegister(words, 0x20d, static_cast<std::uint32_t>(address >> 40u) & 0xffu);
+    for (const std::uint32_t offset : {0x207u, 0x208u, 0x209u}) SetShaderRegister(words, offset, 1);
+    SetShaderRegister(words, 0x212, 0);
+    SetShaderRegister(words, 0x213, static_cast<std::uint32_t>(userData.size()) << 1u);
+    std::uint32_t offset = 0x240;
+    for (const auto value : userData) SetShaderRegister(words, offset++, value);
+}
+
+void Dispatch(std::vector<std::uint32_t>& words, const void* code, std::initializer_list<std::uint32_t> userData) {
+    Program(words, code, userData);
+    words.insert(words.end(), {0xc0031500u, 1, 1, 1, 0x8041u});
+}
+
+void WriteData(std::vector<std::uint32_t>& words, const volatile std::uint32_t* target, std::uint32_t value) {
+    words.insert(words.end(), {0xc0033700u, 0x00100200u, Low(target), High(target), value});
+}
+
+void Submit(std::uint32_t queue, std::vector<std::uint32_t>& words) {
+    Packet packet{words.data(), static_cast<std::uint32_t>(words.size()), 0, {}};
+    Require(sceAgcDriverSubmitAcb(queue, &packet) == 0, "compute submit failed");
+}
+
+void QueueStoreAhead() {
+    OtherWords.clear();
+    for (std::uint32_t index = 0; index < SpinDispatches; ++index) Dispatch(OtherWords, SpinCode.data(), {Low(SpunDescriptor.data()), High(SpunDescriptor.data()), SpinIterations});
+    Dispatch(OtherWords, StoreCode.data(), {Low(TargetDescriptor.data()), High(TargetDescriptor.data()), Low(Fresh.data())});
+    WriteData(OtherWords, &OtherDone, 1);
+    Submit(OtherCompute, OtherWords);
+    const auto start = std::chrono::steady_clock::now();
+    while (std::chrono::steady_clock::now() - start < std::chrono::seconds(5) && OtherDone == 0) {
+        if (AgcDriver::DriverDetail::Driver::QueryPendingSpan(reinterpret_cast<std::uintptr_t>(Target.data()), sizeof(Target), {}) != AgcDriver::ShaderMemory::PendingWrite::None) {
+            SawPending = true;
+            return;
+        }
+        std::this_thread::yield();
+    }
+}
+
+void BeforeAdoption() {
+    const auto window = Armed.exchange(Window::None);
+    if (window == Window::CpuStore) reinterpret_cast<volatile std::uint32_t&>(Target[0]) = Low(Fresh.data());
+    else if (window == Window::QueuedGpuStore) QueueStoreAhead();
+}
+
+void WaitIdle() {
+    AgcDriverWaitIdle_nid_postfix();
+    for (const auto* buffer : {&Stale, &Fresh, &Target}) AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast<std::uintptr_t>(buffer->data()), sizeof(*buffer));
+}
+
+void Reset() {
+    WaitIdle();
+    Stale = {};
+    Fresh = {};
+    Target = Descriptor(Stale.data(), sizeof(Stale));
+    Done = 0;
+    OtherDone = 0;
+    SawPending = false;
+}
+
+void Gated(Window window) {
+    Reset();
+    std::vector<std::uint32_t> words;
+    words.insert(words.end(), {0xc0053c00u, 0x13u, Low(&Gate), High(&Gate), 1u, 0xffffffffu, 0x19u});
+    Dispatch(words, CounterCode.data(), {Low(Target.data()), High(Target.data())});
+    WriteData(words, &Done, 1);
+    Armed = window;
+    Submit(Compute, words);
+    WaitIdle();
+    Armed = Window::None;
+    Require(Done == 1, "the gated submission did not finish");
+}
+
+std::string Counts() {
+    return "(stale " + std::to_string(Stale[0]) + ", fresh " + std::to_string(Fresh[0]) + ")";
+}
+
+}
+
+int main() {
+    try {
+        auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        device.reset();
+        Require(High(Stale.data()) == High(Fresh.data()), "the stale and fresh buffers differ in their high address word");
+        TargetDescriptor = Descriptor(Target.data(), sizeof(Target));
+        SpunDescriptor = Descriptor(Spun.data(), sizeof(Spun));
+        AgcDriverSetAheadAdoptHook_nid_postfix(&BeforeAdoption);
+        Reset();
+        std::vector<std::uint32_t> warmUp;
+        Dispatch(warmUp, CounterCode.data(), {Low(Target.data()), High(Target.data())});
+        WriteData(warmUp, &Done, 1);
+        Submit(Compute, warmUp);
+        std::vector<std::uint32_t> otherWarmUp;
+        Dispatch(otherWarmUp, SpinCode.data(), {Low(SpunDescriptor.data()), High(SpunDescriptor.data()), SpinIterations});
+        Dispatch(otherWarmUp, StoreCode.data(), {Low(TargetDescriptor.data()), High(TargetDescriptor.data()), Low(Fresh.data())});
+        Submit(OtherCompute, otherWarmUp);
+        WaitIdle();
+        Require(Done == 1 && Target[0] == Low(Fresh.data()), "the warm-up dispatches did not run");
+
+        Gated(Window::None);
+        Require(Stale[0] == 1 && Fresh[0] == 0, "the gated dispatch without a store in the window did not count into the stale buffer " + Counts());
+        const auto adopted = AgcDriverDispatchesAdoptedAhead_nid_postfix();
+        Require(adopted != 0, "the gated dispatch was not resolved ahead and adopted");
+
+        auto rejected = AgcDriverDispatchesRejectedAhead_nid_postfix();
+        Gated(Window::CpuStore);
+        Require(Fresh[0] == 1 && Stale[0] == 0, "a dispatch resolved ahead whose descriptor the CPU replaced before its adoption used the old descriptor " + Counts());
+        Require(AgcDriverDispatchesRejectedAhead_nid_postfix() == rejected + 1, "the CPU store in the adoption window was not caught by the adoption check");
+        std::cout << "cpu store before adoption: passed\n";
+
+        bool conclusive = false;
+        for (int attempt = 0; attempt < 4 && !conclusive; ++attempt, SpinDispatches *= 4) {
+            rejected = AgcDriverDispatchesRejectedAhead_nid_postfix();
+            Gated(Window::QueuedGpuStore);
+            conclusive = SawPending;
+            if (!conclusive) {
+                std::cout << "queued gpu store before adoption: the store finished before the adoption with " << SpinDispatches << " spin dispatches, retrying\n";
+                continue;
+            }
+            Require(Fresh[0] == 1 && Stale[0] == 0, "a dispatch resolved ahead whose descriptor a GPU store queued on another queue replaces before its adoption used the old descriptor " + Counts());
+            Require(AgcDriverDispatchesRejectedAhead_nid_postfix() == rejected + 1, "the queued GPU store in the adoption window was not caught by the adoption check");
+        }
+        AgcDriverSetAheadAdoptHook_nid_postfix(nullptr);
+        if (!conclusive) {
+            std::cout << "queued gpu store before adoption: never observed the store pending at the adoption\n";
+            AgcDriverShutdown_nid_postfix();
+            return VulkanTestSkipped;
+        }
+        std::cout << "queued gpu store before adoption: passed\n";
+        std::cout << "dispatch ahead adopt check test passed\n";
+        AgcDriverShutdown_nid_postfix();
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        try {
+            AgcDriverShutdown_nid_postfix();
+        } catch (const std::exception& shutdownError) {
+            std::cerr << shutdownError.what() << '\n';
+        }
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/Float16Dot.cpp

@@ -31,6 +31,12 @@ alignas(256) constexpr std::array<std::uint32_t, 37> Code{
     0xe0781020, 0x80011203, 0xe0781030, 0x80011603, 0xbf810000,
 };
 
+alignas(256) constexpr std::array<std::uint32_t, 20> AccumulatorCode{
+    0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xcc13600a, 0x1c1a0b04, 0xcc13000b,
+    0x1c1a0b04, 0xcc13440c, 0x1c1a0b04, 0xcc13400d, 0x9c1a0b04, 0xcc13440e, 0x9c1a0b04, 0xe0781000,
+    0x80010a03, 0xe0701010, 0x80010e03, 0xbf810000,
+};
+
 constexpr std::uint32_t Rows[32][4] = {
     {0x0001bc00u, 0x81bfacbau, 0xc541aa2fu, 0x00000000u},
     {0x01f61a79u, 0x3c000381u, 0xc1729fffu, 0x00000000u},
@@ -101,6 +107,43 @@ constexpr std::uint32_t Expected[32][10] = {
     {0xbaf43c38u, 0xbaf43c38u, 0x3af43c38u, 0x3af43c38u, 0x3abe63ecu, 0xbaf43c38u, 0xc021e000u, 0x3fb50000u, 0xba90cd60u, 0x3abe63ecu}
 };
 
+constexpr std::uint32_t AccumulatorExpected[32][5] = {
+    {0xc541a901u, 0xc541a901u, 0x4541ab5du, 0x4541ab5du, 0xc541a901u},
+    {0xc1729fdfu, 0xc1729fdfu, 0x4172a01fu, 0x4172a01fu, 0xc1729fdfu},
+    {0x4187e315u, 0x4187e315u, 0x4187e315u, 0xc1882737u, 0xc1882737u},
+    {0x3f7fdb0fu, 0x3f7fdb0fu, 0x3f7fdb0fu, 0xbf801279u, 0xbf801279u},
+    {0xbe511687u, 0xbe511687u, 0xbe511687u, 0xbe5166f4u, 0xbe5166f4u},
+    {0xc1e56f13u, 0xc1e56f13u, 0x41e55ba5u, 0x41e55ba5u, 0xc1e56f13u},
+    {0x439a1228u, 0x439a1228u, 0x439a1228u, 0xc3a5d2acu, 0xc3a5d2acu},
+    {0x37f5fb14u, 0x37f5fb14u, 0x37f5fc0cu, 0x37f5fc0cu, 0x37f5fb14u},
+    {0x7fd0a000u, 0x7fd0a000u, 0x7fd0a000u, 0x7fd0a000u, 0x7fd0a000u},
+    {0xffc8ab86u, 0xffc8ab86u, 0x7fc8ab86u, 0x7fc8ab86u, 0xffc8ab86u},
+    {0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u},
+    {0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u},
+    {0xb974e480u, 0xb974e480u, 0xb974e480u, 0xb974e480u, 0xb974e480u},
+    {0x7f800000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0xff800000u},
+    {0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x80000000u},
+    {0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u},
+    {0x7fc42000u, 0x7fc42000u, 0x7fc42000u, 0x7fc42000u, 0x7fc42000u},
+    {0xbf052ae0u, 0xbf052ae0u, 0xbf052ae0u, 0xbf052ae0u, 0xbf052ae0u},
+    {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u},
+    {0x5f834da7u, 0x5f834da7u, 0x5f834da7u, 0xdf834da7u, 0xdf834da7u},
+    {0x7f7fffffu, 0x7f7fffffu, 0x7f7fffffu, 0xff7fffffu, 0xff7fffffu},
+    {0x42750eeeu, 0x42750eeeu, 0x42750eeeu, 0xc2750eeeu, 0xc2750eeeu},
+    {0x43380748u, 0x43380748u, 0x43380748u, 0xc3380746u, 0xc3380746u},
+    {0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u},
+    {0xf540cddbu, 0xf540cddbu, 0x7540cddbu, 0x7540cddbu, 0xf540cddbu},
+    {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u},
+    {0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0xffe00000u, 0xffe00000u},
+    {0x3cb7c720u, 0x3cb7c720u, 0x3cb7c720u, 0x3cb7c720u, 0x3cb7c720u},
+    {0xbf9bd158u, 0xbf9bd158u, 0xbf9bd158u, 0xbf9bd158u, 0xbf9bd158u},
+    {0x565f49ebu, 0x565f49ebu, 0x565f49ebu, 0xd65f49ebu, 0xd65f49ebu},
+    {0xbaf43c38u, 0xbaf43c38u, 0xbaf43c38u, 0xbaf43c38u, 0xbaf43c38u}
+};
+
+constexpr const char* AccumulatorNames[5] = {"v_dot2 op_sel src2", "v_dot2 op_sel_hi src2 cleared", "v_dot2 neg_hi src2", "v_dot2 neg_lo src2", "v_dot2 neg_lo and neg_hi src2"};
+
 constexpr const char* Names[10] = {"v_dot2c", "v_dot2", "v_dot2 neg_lo src0/src2 neg_hi src1", "v_dot2 neg_lo src1 neg_hi src0", "v_dot2 op_sel src0", "v_dot2 clamp", "v_dot2c inline 1.0", "v_dot2c literal", "v_dot2c dpp quad_perm", "v_dot2 op_sel src1"};
 
 void Fill(std::uint32_t tid, std::uint32_t* words) {
@@ -140,21 +183,9 @@ auto Compile(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> cod
     return ShaderRecompiler::Recompile(request);
 }
 
-void CheckRefused(AgcDriver::VulkanDevice& device, std::uint32_t word0, const std::string& what) {
-    alignas(256) const std::array<std::uint32_t, 3> code{word0, 0x1c1a0b04u, 0xbf810000u};
-    std::string refusal;
-    try {
-        static_cast<void>(Compile(device, code));
-    } catch (const std::exception& error) {
-        refusal = error.what();
-    }
-    Require(refusal.find("v_dot2_f32_f16 accumulator op_sel, op_sel_hi and neg_hi are not implemented") != std::string::npos, what + " was not refused");
-}
-
-void Run(AgcDriver::VulkanDevice& device) {
+void Run(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code) {
     for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
     Output.fill(0xdeadbeefu);
-    const std::span<const std::uint32_t> code(Code);
     const auto result = Compile(device, code);
     device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
     device.WaitIdle();
@@ -167,6 +198,13 @@ void Check() {
     }
 }
 
+void CheckAccumulator() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::uint32_t* out = &Output[tid * Results];
+        for (std::uint32_t i = 0; i < 5; ++i) Expect(tid, out[i], AccumulatorExpected[tid][i], AccumulatorNames[i]);
+    }
+}
+
 }
 
 int main() {
@@ -177,11 +215,10 @@ int main() {
             std::puts("skipped, v_dot2_f32_f16 is computed in f64 and the device has no shaderFloat64");
             return VulkanTestSkipped;
         }
-        Run(*device);
+        Run(*device, Code);
         Check();
-        CheckRefused(*device, 0xcc13600au, "op_sel on the accumulator");
-        CheckRefused(*device, 0xcc13440au, "neg_hi on the accumulator");
-        CheckRefused(*device, 0xcc13000au, "op_sel_hi cleared on the accumulator");
+        Run(*device, AccumulatorCode);
+        CheckAccumulator();
         std::puts("v_dot2_f32_f16 tests passed");
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp

@@ -37,7 +37,9 @@ constexpr std::uint32_t Type2D = 9;
 constexpr std::uint32_t UnormFormat = 30;
 constexpr std::uint32_t UintFormat = 34;
 constexpr std::uint32_t FloatFormat = 43;
+constexpr std::uint32_t NativeFloatFormat = 36;
 constexpr std::size_t FloatOffset = 40960;
+constexpr std::size_t Float36Offset = 45056;
 constexpr std::uint32_t SwizzleXYZ1 = 0x3acu;
 constexpr std::uint32_t SwizzleXZY1 = 0x374u;
 constexpr std::uint32_t SwizzleZYX1 = 0x32eu;
@@ -285,6 +287,16 @@ constexpr std::array<std::array<std::uint32_t, 4>, 7> FloatStoreEdges{{
     {0x7f800001u, 0x7f810000u, 0x7fa00000u, 0xfa1f07e1u},
 }};
 
+constexpr std::array<std::array<std::uint32_t, 4>, 7> Float36StoreEdges{{
+    {0xbf800000u, 0x80000000u, 0x7fc00000u, 0xfc000000u},
+    {0x7f800000u, 0x7149f2cau, 0x477fff00u, 0xf7fdffc0u},
+    {0x3f830000u, 0x3f818000u, 0x35800000u, 0x001e03c1u},
+    {0x00000001u, 0xff800000u, 0xffc00000u, 0xfc000000u},
+    {0x477c0000u, 0x4788b800u, 0x33000000u, 0x003dffbeu},
+    {0x36400000u, 0x477e0000u, 0x3dcccccdu, 0x5cfdf803u},
+    {0x7f800001u, 0x7f810000u, 0x7fa00000u, 0xfa3e0fc1u},
+}};
+
 constexpr std::array<std::array<std::uint32_t, 4>, 3> FloatLoadEdges{{
     {0x3e0u | (0x7c0u << 10u) | (0x03fu << 21u), 0x7f800000u, 0x7f800000u, 0x387c0000u},
     {0x001u | (0x041u << 10u) | (0x7bfu << 21u), 0x36000000u, 0x38820000u, 0x477e0000u},
@@ -302,6 +314,10 @@ std::uint32_t FloatTexel(std::uint32_t index) {
     return SmallFloat(FloatValues[index % 11u], 5u) | (SmallFloat(FloatValues[(index + 4u) % 11u], 6u) << 10u) | (SmallFloat(FloatValues[(index + 8u) % 11u], 6u) << 21u);
 }
 
+std::uint32_t Float36Texel(std::uint32_t index) {
+    return SmallFloat(FloatValues[index % 11u], 6u) | (SmallFloat(FloatValues[(index + 4u) % 11u], 6u) << 11u) | (SmallFloat(FloatValues[(index + 8u) % 11u], 5u) << 22u);
+}
+
 void CheckFloatLoad(AgcDriver::VulkanDevice& device, std::uint8_t* texels) {
     for (std::uint32_t index = 0; index < LoadWidth; ++index) {
         const auto texel = index < FloatLoadEdges.size() ? FloatLoadEdges[index][0] : FloatTexel(index);
@@ -344,8 +360,32 @@ void CheckFloatStore(AgcDriver::VulkanDevice& device, std::uint8_t* texels) {
     }
 }
 
-void CheckFloatSample(AgcDriver::VulkanDevice& device, const std::uint8_t* texels, const std::array<std::uint32_t, 4>& expected, const std::string& what) {
-    Run(device, SampleLz, TextureDescriptor(texels, FloatFormat, SwizzleXYZ1, Threads), PointSampler, 1);
+void CheckFloat36Store(AgcDriver::VulkanDevice& device, std::uint8_t* texels) {
+    std::fill(texels, texels + Threads * 4u, static_cast<std::uint8_t>(0xa5u));
+    Input.fill(0u);
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        Input[tid * 4u + 0u] = std::bit_cast<std::uint32_t>(FloatValues[tid % 11u]);
+        Input[tid * 4u + 1u] = std::bit_cast<std::uint32_t>(FloatValues[(tid + 4u) % 11u]);
+        Input[tid * 4u + 2u] = std::bit_cast<std::uint32_t>(FloatValues[(tid + 8u) % 11u]);
+        if (tid < Float36StoreEdges.size()) {
+            for (std::uint32_t component = 0; component < 3u; ++component) {
+                Input[tid * 4u + component] = Float36StoreEdges[tid][component];
+            }
+        }
+    }
+    Run(device, StoreXyz, TextureDescriptor(texels, NativeFloatFormat, SwizzleXYZ1, Threads), BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u)), 1);
+    AgcDriver::Graphics::StorageTexture::FlushPending(reinterpret_cast<std::uintptr_t>(texels), Threads * 4u, nullptr, "test");
+    device.WaitIdle();
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        std::uint32_t actual = 0;
+        std::memcpy(&actual, texels + tid * 4u, 4u);
+        const auto expected = tid < Float36StoreEdges.size() ? Float36StoreEdges[tid][3] : Float36Texel(tid);
+        Require(actual == expected, "image_store of R11_G11_B10_FLOAT: texel " + std::to_string(tid) + " is " + Hex(actual) + ", expected " + Hex(expected));
+    }
+}
+
+void CheckFloatSample(AgcDriver::VulkanDevice& device, const std::uint8_t* texels, const std::array<std::uint32_t, 4>& expected, const std::string& what, std::uint32_t format = FloatFormat) {
+    Run(device, SampleLz, TextureDescriptor(texels, format, SwizzleXYZ1, Threads), PointSampler, 1);
     for (std::uint32_t tid = 0; tid < Threads; ++tid) {
         for (std::uint32_t component = 0; component < 4u; ++component) {
             const auto actual = Output[tid * 4u + component];
@@ -545,6 +585,9 @@ int main(int argc, char** argv) {
         CheckFloatSample(*device, texels + FloatOffset, {0u, 0u, 0x7fc00000u, 0x3f800000u}, "image_sample_lz of R10_G11_B11_FLOAT results still on the GPU");
         CheckFloatLoad(*device, texels + FloatOffset);
         CheckFloatSample(*device, texels + FloatOffset, {0x7f800000u, 0x7f800000u, 0x387c0000u, 0x3f800000u}, "image_sample_lz of R10_G11_B11_FLOAT");
+        CheckFloat36Store(*device, texels + Float36Offset);
+        Run(*device, StoreXyz, TextureDescriptor(texels + Float36Offset, NativeFloatFormat, SwizzleXYZ1, Threads), BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u)), 1);
+        CheckFloatSample(*device, texels + Float36Offset, {0u, 0u, 0x7fc00000u, 0x3f800000u}, "image_sample_lz of R11_G11_B10_FLOAT results still on the GPU", NativeFloatFormat);
         CheckSample(*device, texels, SampleAt, SampleStep, SampleStart, SwizzleXYZ1, PointSampler, false, false, "point image_sample_lz X Y Z 1");
         CheckSample(*device, texels, SampleAt, SampleStep, SampleStart, SwizzleZYX1, BilinearSampler, true, false, "bilinear image_sample_lz Z Y X 1");
         CheckSample(*device, texels, SampleWrapped, SampleStep, WrappedStart, SwizzleX011, WrapPointSampler, false, true, "wrapped point image_sample_lz X 0 1 1");

core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreLineThroughPlaneArray.cpp

@@ -0,0 +1,185 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Width = 32;
+constexpr std::uint32_t Type1D = 8;
+constexpr std::uint32_t IdentitySwizzle = 0xfacu;
+constexpr std::size_t TexelBytes = 4096;
+constexpr std::size_t BlockBytes = 65536;
+
+alignas(256) std::array<std::uint32_t, Threads * 4> Input{};
+
+alignas(256) constexpr std::array<std::uint32_t, 17> StoreCode{
+    0x34020084, 0xe0381000, 0x80001401, 0xbf8c3f70, 0x7e3c0300, 0x7e3e0281, 0x7e400280, 0xf0201f28, 0x0001141e,
+    0x7e3e0280, 0x7e4002c1, 0xf0201528, 0x0001141e, 0x7e400281, 0xf0201528, 0x0001141e, 0xbf810000,
+};
+
+class GuestBlock {
+public:
+    GuestBlock() {
+#ifdef _WIN32
+        block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+#else
+        block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
+#endif
+        Require(block != nullptr, "image store line through plane array: cannot allocate the guest block");
+        GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true);
+    }
+
+    ~GuestBlock() {
+        GuestAllocations::Mutation().Remove(block);
+#ifdef _WIN32
+        VirtualFree(block, 0, MEM_RELEASE);
+#else
+        std::free(block);
+#endif
+    }
+
+    GuestBlock(const GuestBlock&) = delete;
+    GuestBlock& operator=(const GuestBlock&) = delete;
+
+    std::uint8_t* Data() { return block; }
+
+private:
+    std::uint8_t* block = nullptr;
+};
+
+struct Format {
+    const char* name;
+    std::uint32_t format;
+    std::uint32_t components;
+};
+
+constexpr std::array<Format, 2> Formats{{
+    {"32_SINT", 21u, 1u},
+    {"32_32_32_32_SINT", 76u, 4u},
+}};
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+std::uint32_t Value(std::uint32_t tid, std::uint32_t index) {
+    return (tid * 4u + index + 1u) * 0x9e3779b1u;
+}
+
+std::uint64_t TexelOffset(const Format& format, std::uint32_t x) {
+    const auto mip = AgcDriver::Graphics::ComputeMipLayout(AgcDriver::Graphics::TextureTileMode::kLinear, format.format, Width, 1u, 1u).at(0);
+    return mip.tiledOffset + static_cast<std::uint64_t>(x) * format.components * 4u;
+}
+
+std::vector<std::uint8_t> Initial() {
+    std::vector<std::uint8_t> bytes(TexelBytes);
+    for (std::size_t index = 0; index < bytes.size(); ++index) bytes[index] = static_cast<std::uint8_t>(0x80u + 37u * index + 11u * (index >> 4u));
+    return bytes;
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+std::array<std::uint32_t, 8> TextureDescriptor(const void* data, std::uint32_t format) {
+    const auto address = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(data));
+    return {
+        static_cast<std::uint32_t>(address >> 8u),
+        static_cast<std::uint32_t>((address >> 40u) & 0xffu) | (format << 20u) | (((Width - 1u) & 3u) << 30u),
+        (Width - 1u) >> 2u,
+        IdentitySwizzle | (Type1D << 28u),
+        0u,
+        0u,
+        0u,
+        0u,
+    };
+}
+
+void Dispatch(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span<const std::uint32_t> code, std::uint32_t format) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        for (std::uint32_t index = 0; index < 4u; ++index) Input[tid * 4u + index] = Value(tid, index);
+    }
+    const auto initial = Initial();
+    std::copy(initial.begin(), initial.end(), texels);
+    std::vector<std::uint32_t> userData(16, 0u);
+    const auto buffer = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto texture = TextureDescriptor(texels, format);
+    std::copy(buffer.begin(), buffer.end(), userData.begin());
+    std::copy(texture.begin(), texture.end(), userData.begin() + 4);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+    AgcDriver::Graphics::StorageTexture::FlushPending(reinterpret_cast<std::uintptr_t>(texels), TexelBytes, nullptr, "test");
+    device.WaitIdle();
+}
+
+void CheckTexels(const std::uint8_t* texels, const Format& format) {
+    const auto initial = Initial();
+    for (std::uint32_t x = 0; x < Width; ++x) {
+        const auto offset = TexelOffset(format, x);
+        for (std::uint32_t component = 0; component < format.components; ++component) {
+            std::uint32_t actual = 0;
+            std::memcpy(&actual, texels + offset + component * 4u, 4u);
+            const auto wanted = Value(x, component);
+            Require(actual == wanted, std::string("image_store 2d_array through a 1D ") + format.name + " T#: texel " + std::to_string(x) + " component " + std::to_string(component) + " is " + Hex(actual) + ", expected the y = 1, slice = 0 store " + Hex(wanted) + " and the stores to slices 0xffffffff and 1 dropped");
+        }
+    }
+    const auto end = TexelOffset(format, Width);
+    Require(std::equal(texels + end, texels + TexelBytes, initial.begin() + static_cast<std::ptrdiff_t>(end)), std::string("image_store 2d_array through a 1D ") + format.name + " T#: bytes past the texture changed");
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        GuestBlock block;
+        auto* texels = block.Data();
+        for (const auto& format : Formats) {
+            Dispatch(*device, texels, StoreCode, format.format);
+            CheckTexels(texels, format);
+        }
+        std::puts("image store line through plane array tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp

@@ -667,6 +667,19 @@ void RunBorderSwizzleTests(AgcDriver::VulkanDevice& device) {
             const auto activeSampler = tableImage ? RunDirectCode(device, TableCode, PointClamp, image) : RunDirectCode(device, TableSamplerCode, PointClamp, image);
             requireFault(activeSampler, "sampler table entry with an opaque black border", pair);
         }
+        auto secondOpaqueBlack = opaqueBlack;
+        secondOpaqueBlack[1] = (3u * 256u) << 12u;
+        SetRecord(0, opaqueBlack, image);
+        SetRecord(1, secondOpaqueBlack, image);
+        for (const std::uint32_t record : {0u, 1u}) {
+            Keys[0] = record;
+            Keys[1] = record;
+            requireFault(RunDirectCode(device, TableCode, PointClamp, image), "sampler table entry with an opaque black border", name + " rejected S# " + std::to_string(record) + " of two with the same reason");
+        }
+        SetRecord(0, PointClamp, image);
+        SetRecord(1, opaqueBlack, image);
+        Keys[0] = 1;
+        Keys[1] = 0;
     }
     const auto cut = Run(device, TableCutCode, 1, Lanes, 64);
     Require(Holds(cut.samplerTable, opaqueBlack), "image table border: the EXEC-clear opaque-black S# was not included in the snapshot");

core/libs/prx/libSceAgcDriver/tests/execution/InterpolationModes.cpp

@@ -7,6 +7,7 @@
 #include <algorithm>
 #include <array>
 #include <bit>
+#include <chrono>
 #include <cmath>
 #include <cstdint>
 #include <cstdio>
@@ -302,20 +303,187 @@ void RunRandom(AgcDriver::VulkanDevice& device, bool integer) {
     for (std::size_t index = RandomRows * RandomWords; index < Data.size(); ++index) Require(Data[index] == Sentinel, "random interpolation rows wrote beyond the rows");
 }
 
-int main() {
+constexpr std::uint32_t ChainWords = 8;
+constexpr std::uint32_t ChainGroup = 64;
+alignas(256) std::array<std::uint32_t, (1u << 16u) * ChainWords> ChainData{};
+
+RecompileResult CompileChain(std::span<const std::uint32_t> steps, bool half, bool integer, std::uint32_t iterations = 1u) {
+    IrProgram program;
+    const ShaderStageInputInfo inputs{};
+    SpirvEmitterState state(program, inputs);
+    state.integerInterpolation = integer;
+    auto& module = state.module;
+    module.EmitCapability(spv::CapabilityShader);
+    module.EmitCapability(integer ? spv::CapabilityInt64 : spv::CapabilityFloat64);
+    module.AddMemoryModel(spv::AddressingModelLogical, spv::MemoryModelGLSL450);
+    const auto u32 = TypeU32(state);
+    const auto array = module.DecoratedType(spv::OpTypeRuntimeArray, {{spv::OpDecorate, {spv::DecorationArrayStride, 4u}}}, u32);
+    const auto structure = module.Type(spv::OpTypeStruct, array);
+    module.AddAnnotation(spv::OpDecorate, structure, spv::DecorationBlock);
+    module.AddAnnotation(spv::OpMemberDecorate, structure, 0u, spv::DecorationOffset, 0u);
+    const auto buffer = module.DefineGlobalVariable(module.Type(spv::OpTypePointer, spv::StorageClassStorageBuffer, structure), spv::StorageClassStorageBuffer);
+    module.AddAnnotation(spv::OpDecorate, buffer, spv::DecorationDescriptorSet, 0u);
+    module.AddAnnotation(spv::OpDecorate, buffer, spv::DecorationBinding, 0u);
+    const auto invocation = module.DefineGlobalVariable(TypePointer(state, spv::StorageClassInput, TypeU32Vector(state, 3u)), spv::StorageClassInput);
+    module.AddAnnotation(spv::OpDecorate, invocation, spv::DecorationBuiltIn, spv::BuiltInGlobalInvocationId);
+    const auto element = module.Type(spv::OpTypePointer, spv::StorageClassStorageBuffer, u32);
+    const auto main = module.AllocateId();
+    const auto voidType = module.Type(spv::OpTypeVoid);
+    module.EmitEntryPoint(spv::ExecutionModelGLCompute, main, "main", {invocation});
+    module.AddExecutionMode(main, spv::ExecutionModeLocalSize, ChainGroup, 1u, 1u);
+    module.AddFunction(spv::OpFunction, voidType, main, spv::FunctionControlMaskNone, module.Type(spv::OpTypeFunction, voidType));
+    module.AddFunction(spv::OpLabel, module.AllocateId());
+    const auto id = module.AllocateId();
+    module.AddFunction(spv::OpLoad, TypeU32Vector(state, 3u), id, invocation);
+    const auto lane = module.AllocateId();
+    module.AddFunction(spv::OpCompositeExtract, u32, lane, id, 0u);
+    const auto rowBase = Binary(state, spv::OpIMul, u32, lane, ConstantU32(state, ChainWords));
+    const auto address = [&](std::uint32_t offset) {
+        const auto pointer = module.AllocateId();
+        module.AddFunction(spv::OpAccessChain, element, pointer, buffer, ConstantU32(state, 0u), Binary(state, spv::OpIAdd, u32, rowBase, ConstantU32(state, offset)));
+        return pointer;
+    };
+    const auto load = [&](std::uint32_t offset) {
+        const auto word = module.AllocateId();
+        module.AddFunction(spv::OpLoad, u32, word, address(offset));
+        return Unary(state, spv::OpBitcast, TypeF32(state), word);
+    };
+    IrProgram constants;
+    IrBuilder builder(constants);
+    builder.SetInsertionPoint(constants.CreateBlock());
+    const std::array deltas{load(0u), load(1u), load(2u), load(3u)};
+    const auto coordinate = load(4u);
+    const auto chain = [&](std::uint32_t value) {
+        for (std::uint32_t step = 0; step < steps.size(); ++step) {
+            const auto* flags = &builder.Constant(steps[step]);
+            value = half ? EmitFPInterpolateF16(state, deltas[step % 4u], coordinate, value, flags) : EmitFPInterpolateF32(state, deltas[step % 4u], coordinate, value, flags);
+        }
+        return value;
+    };
+    auto value = load(5u);
+    if (iterations == 1u) {
+        value = chain(value);
+    } else {
+        const auto entry = module.AllocateId();
+        const auto header = module.AllocateId();
+        const auto body = module.AllocateId();
+        const auto next = module.AllocateId();
+        const auto merge = module.AllocateId();
+        const auto counter = module.AllocateId();
+        const auto carried = module.AllocateId();
+        const auto counted = module.AllocateId();
+        const auto chained = module.AllocateId();
+        module.AddFunction(spv::OpBranch, entry);
+        EmitLabel(state, entry);
+        module.AddFunction(spv::OpBranch, header);
+        EmitLabel(state, header);
+        module.AddFunction(spv::OpPhi, u32, counter, ConstantU32(state, 0u), entry, counted, next);
+        module.AddFunction(spv::OpPhi, TypeF32(state), carried, value, entry, chained, next);
+        module.AddFunction(spv::OpLoopMerge, merge, next, spv::LoopControlMaskNone);
+        module.AddFunction(spv::OpBranchConditional, Binary(state, spv::OpULessThan, TypeBool(state), counter, ConstantU32(state, iterations)), body, merge);
+        EmitLabel(state, body);
+        module.AddFunction(spv::OpCopyObject, TypeF32(state), chained, chain(carried));
+        module.AddFunction(spv::OpBranch, next);
+        EmitLabel(state, next);
+        module.AddFunction(spv::OpIAdd, u32, counted, counter, ConstantU32(state, 1u));
+        module.AddFunction(spv::OpBranch, header);
+        EmitLabel(state, merge);
+        value = carried;
+    }
+    module.AddFunction(spv::OpStore, address(6u), Unary(state, spv::OpBitcast, u32, value));
+    module.AddFunction(spv::OpReturn);
+    module.AddFunction(spv::OpFunctionEnd);
+    RecompileResult result{};
+    result.spirv = module.Finalize();
+    const auto base = reinterpret_cast<std::uintptr_t>(ChainData.data());
+    result.bindings.push_back({DescriptorKind::StorageBuffer, DescriptorRole::GuestBuffers, 0u, 0u, 1u,
+        {static_cast<std::uint32_t>(base), static_cast<std::uint32_t>((base >> 32u) & 0xffffu), static_cast<std::uint32_t>(sizeof(ChainData)), 0x31016facu}, false});
+    return result;
+}
+
+void FillChains(std::uint32_t lanes, bool half, bool special) {
+    Generator generator{half ? 0x6b5f00000001ull : 0x6b5f00000002ull};
+    const auto mantissa = [&]() {
+        const auto bits = generator.Pick(3u) == 0u ? 1u + generator.Pick(23u) : 23u;
+        return static_cast<std::uint32_t>(generator.Next() & 0x7fffffu) & ~((1u << (23u - bits)) - 1u);
+    };
+    const auto value = [&](std::uint32_t lowest, std::uint32_t count) { return (generator.Pick(2u) << 31u) | ((lowest + generator.Pick(count)) << 23u) | mantissa(); };
+    ChainData.fill(Sentinel);
+    for (std::uint32_t lane = 0; lane < lanes; ++lane) {
+        auto* row = &ChainData[lane * ChainWords];
+        for (std::uint32_t index = 0; index < 4u; ++index) row[index] = special && generator.Pick(8u) == 0u ? generator.Half() : special && generator.Pick(8u) == 0u ? generator.Single() : value(112u, 16u);
+        row[4] = special && generator.Pick(8u) == 0u ? generator.Single() : value(117u, 10u) & 0x7fffffffu;
+        row[5] = special && generator.Pick(8u) == 0u ? generator.Single() : value(110u, 24u);
+    }
+}
+
+std::uint32_t ChainReference(const std::uint32_t* row, std::span<const std::uint32_t> steps, bool half) {
+    auto value = row[5];
+    for (std::uint32_t step = 0; step < steps.size(); ++step) value = Reference(row[step % 4u], row[4], value, half, steps[step]);
+    return value;
+}
+
+void RunChains(AgcDriver::VulkanDevice& device, bool integer) {
+    constexpr std::uint32_t lanes = 8192;
+    std::array<std::uint32_t, 2 * RandomModes> steps{};
+    for (std::uint32_t step = 0; step < steps.size(); ++step) steps[step] = ModeFlags[step % ModeFlags.size()] | (((step / ModeFlags.size()) % 4u) << InterpolationRound16Shift);
+    for (const bool half : {false, true}) {
+        FillChains(lanes, half, true);
+        device.Dispatch(CompileChain(steps, half, integer), lanes / ChainGroup, 1u, 1u);
+        device.WaitIdle();
+        for (std::uint32_t lane = 0; lane < lanes; ++lane) {
+            const auto* row = &ChainData[lane * ChainWords];
+            const auto expected = ChainReference(row, steps, half);
+            char message[200];
+            std::snprintf(message, sizeof(message), "%s chain (%s), lane %u: 0x%08x 0x%08x 0x%08x 0x%08x * 0x%08x from 0x%08x gave 0x%08x, reference 0x%08x",
+                half ? "f16" : "f32", integer ? "integer" : "Float64", lane, row[0], row[1], row[2], row[3], row[4], row[5], row[6], expected);
+            Require(row[6] == expected, message);
+        }
+    }
+}
+
+void Benchmark(AgcDriver::VulkanDevice& device, bool integer) {
+    constexpr std::uint32_t lanes = 1u << 16u;
+    constexpr std::uint32_t iterations = 1024;
+    const std::array<std::uint32_t, 4> steps{ModeFlags[0], ModeFlags[0], ModeFlags[0], ModeFlags[0]};
+    for (const bool half : {false, true}) {
+        FillChains(lanes, half, false);
+        const auto program = CompileChain(steps, half, integer, iterations);
+        device.Dispatch(program, lanes / ChainGroup, 1u, 1u);
+        device.WaitIdle();
+        std::array<double, 9> times{};
+        for (auto& milliseconds : times) {
+            const auto start = std::chrono::steady_clock::now();
+            device.Dispatch(program, lanes / ChainGroup, 1u, 1u);
+            device.WaitIdle();
+            milliseconds = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
+        }
+        std::sort(times.begin(), times.end());
+        std::printf("%s path, %u lanes x %u chained %s interpolations: median %.3f ms, min %.3f, max %.3f\n", integer ? "integer" : "Float64", lanes, iterations * static_cast<std::uint32_t>(steps.size()), half ? "f16" : "f32", times[4], times.front(), times.back());
+    }
+}
+
+int main(int argc, char** argv) {
     try {
         const auto device = OpenVulkanTestDevice();
         if (!device) return VulkanTestSkipped;
         const auto target = device->Target();
         const bool float64 = std::find(target.supportedCapabilities.begin(), target.supportedCapabilities.end(), spv::CapabilityFloat64) != target.supportedCapabilities.end();
+        if (argc == 2 && std::string(argv[1]) == "--benchmark") {
+            Benchmark(*device, true);
+            if (float64) Benchmark(*device, false);
+            return 0;
+        }
         for (std::uint32_t mode = 0; mode < ModeFlags.size(); ++mode) {
             Run(*device, mode, true);
             if (float64) Run(*device, mode, false);
         }
         RunRandom(*device, true);
         if (float64) RunRandom(*device, false);
-        std::puts(float64 ? "interpolation modes passed: 32 hardware rows and 2048 random rows against the reference in 32 float modes, 3 ops, integer and Float64 paths"
-                          : "interpolation modes passed: 32 hardware rows and 2048 random rows against the reference in 32 float modes, 3 ops, integer path (the device has no Float64)");
+        RunChains(*device, true);
+        if (float64) RunChains(*device, false);
+        std::puts(float64 ? "interpolation modes passed: 32 hardware rows and 2048 random rows against the reference in 32 float modes, 3 ops, 8192 chains of 64 f32 and f16 interpolations, integer and Float64 paths"
+                          : "interpolation modes passed: 32 hardware rows and 2048 random rows against the reference in 32 float modes, 3 ops, 8192 chains of 64 f32 and f16 interpolations, integer path (the device has no Float64)");
         return 0;
     } catch (const std::exception& error) {
         std::cerr << error.what() << '\n';