Merge remote-tracking branch 'origin/main' into fix/recompiler-cache-line-endings

# Conflicts:
#	core/shader/recompiler/ShaderCacheVersion.cmake

.github/DISCUSSION_TEMPLATE/game-bring-up.yml

@@ -6,7 +6,7 @@ body:
         One thread per title, for every platform: search this category for the title ID first. If a thread exists, reply there, also when your OS or GPU differs from the one it reports.
         Name the discussion `Game name (title ID)`, for example `DOOM Shareware (PPSA99666)`, without the platform.
         Bug reports and implementation work belong in issues and pull requests; link them here as the blocker.
-        Per-game progress is also tracked on the [blocker map](https://alxcbtw.github.io/anyps5-roadmap/), built from public GitHub data. Only share material you have the right to distribute.
+        Only share material you have the right to distribute.
   - type: checkboxes
     id: searched
     attributes:

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,59 @@
+name: Bug report
+description: A title, homebrew or test stops, crashes or renders wrongly.
+labels: [bug]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Search open and closed issues and pull requests for the error text first, and comment there if it is already reported.
+        Game bring-up progress belongs in [Discussions](https://github.com/boykopovar/AnyPS5/discussions/categories/game-bring-up), one thread per title.
+        A slow scene or a change in speed goes into the [Performance report](https://github.com/boykopovar/AnyPS5/issues/new?template=performance.yml) form, which asks for a [performance trace](https://github.com/boykopovar/AnyPS5/blob/main/docs/dev/BUILD.md#performance-trace) and its `tools/trace_digest.py` digest.
+  - type: input
+    id: summary
+    attributes:
+      label: What happens
+      description: One sentence, with the exact error text if there is one.
+    validations:
+      required: true
+  - type: input
+    id: title
+    attributes:
+      label: Title, test or build step
+      description: Title ID and content version (for example PPSAxxxxx 01.000.000), a homebrew with its source, the ctest name, or the build step that fails.
+    validations:
+      required: true
+  - type: input
+    id: commit
+    attributes:
+      label: AnyPS5 commit
+      description: The `main` commit that was built, or the release that was used, and any pull requests merged on top.
+    validations:
+      required: true
+  - type: input
+    id: system
+    attributes:
+      label: System
+      description: OS, GPU and driver version, and the compiler if you built it yourself.
+      placeholder: Arch Linux, GCC 16.2.1, RTX 5080 (NVIDIA 615.78.08)
+    validations:
+      required: true
+  - type: textarea
+    id: steps
+    attributes:
+      label: Steps to reproduce
+      description: The commands that were run, environment variables included, and how long it takes to stop.
+    validations:
+      required: true
+  - type: textarea
+    id: log
+    attributes:
+      label: Log
+      description: The lines around the stop.
+      render: text
+    validations:
+      required: true
+  - type: textarea
+    id: notes
+    attributes:
+      label: What was checked
+      description: Other devices or commits tried, and whether it is the same every run. Attach the full log and screenshots here.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Game bring-up
+    url: https://github.com/boykopovar/AnyPS5/discussions/categories/game-bring-up
+    about: Progress of one title, one thread per title ID
+  - name: Question
+    url: https://github.com/boykopovar/AnyPS5/discussions/categories/q-a
+    about: Building, converting and running

.github/workflows/build.yml

@@ -33,7 +33,7 @@ permissions:
 
 concurrency:
   group: build-${{ github.event_name }}-${{ github.event.pull_request.number || inputs.pr || github.ref }}
-  cancel-in-progress: true
+  cancel-in-progress: ${{ github.event_name != 'push' }}
 
 env:
   CONFIGURE: >-

.github/workflows/windows-msys2.yml

@@ -0,0 +1,117 @@
+name: Windows MSYS2
+
+on:
+  schedule:
+    - cron: '23 8 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: windows-msys2-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CONFIGURE: >-
+    -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON
+    -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++
+    -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+
+jobs:
+  windows-msys2:
+    runs-on: windows-latest
+    timeout-minutes: 45
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+      - id: submodules
+        name: Key the submodules on their commits
+        run: |
+          echo "key=$(git ls-files -s | awk '$1 == "160000" { print $2, $4 }' | cat - .gitmodules | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
+          echo "ffmpeg=$(git rev-parse HEAD:3rdparty/ffmpeg-core)" >> "$GITHUB_OUTPUT"
+      - id: submodule-cache
+        uses: actions/cache/restore@v6
+        with:
+          path: ${{ runner.temp }}/submodule-gitdirs.tar
+          key: submodule-gitdirs-${{ runner.os }}-${{ steps.submodules.outputs.key }}
+      - name: Check out the submodules
+        run: |
+          if [ -f "$RUNNER_TEMP/submodule-gitdirs.tar" ]; then tar --force-local -xf "$RUNNER_TEMP/submodule-gitdirs.tar"; fi
+          git submodule update --init --recursive --depth 1 --jobs 8
+      - if: steps.submodule-cache.outputs.cache-hit != 'true'
+        run: tar --force-local -cf "$RUNNER_TEMP/submodule-gitdirs.tar" .git/modules
+      - if: steps.submodule-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v6
+        continue-on-error: true
+        with:
+          path: ${{ runner.temp }}/submodule-gitdirs.tar
+          key: submodule-gitdirs-${{ runner.os }}-${{ steps.submodules.outputs.key }}
+      - name: Use the CA bundle of Git for Windows
+        run: |
+          for prefix in ucrt64 mingw64; do
+            bundle="C:/Program Files/Git/$prefix/etc/ssl/certs/ca-bundle.crt"
+            if [ -f "$bundle" ]; then echo "SSL_CERT_FILE=$bundle" >> "$GITHUB_ENV"; exit 0; fi
+          done
+          echo "::error::No CA bundle found in Git for Windows"; exit 1
+      - id: msys2
+        uses: msys2/setup-msys2@v2
+        with:
+          msystem: UCRT64
+          update: true
+          install: >-
+            mingw-w64-ucrt-x86_64-gcc
+            mingw-w64-ucrt-x86_64-cmake
+            mingw-w64-ucrt-x86_64-ninja
+      - shell: pwsh
+        env:
+          MSYS2_LOCATION: ${{ steps.msys2.outputs.msys2-location }}
+        run: Add-Content -LiteralPath $env:GITHUB_PATH -Value "$env:MSYS2_LOCATION\ucrt64\bin"
+      - uses: hendrikmuhs/[email protected]
+        with:
+          key: windows-msys2
+          save: ${{ github.ref == 'refs/heads/main' }}
+          evict-old-files: job
+      - id: ffmpeg-cache
+        uses: actions/cache/restore@v6
+        with:
+          path: build/externals
+          key: ffmpeg-${{ runner.os }}-${{ steps.submodules.outputs.ffmpeg }}
+      - shell: cmd
+        run: cmake -S . -B build %CONFIGURE% "-DANYPS5_AGC_EXECUTION_TESTS=F16ToF32Unpack;F64FmaSubnormals;RegistrationPrepareShutdown"
+      - if: steps.ffmpeg-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v6
+        continue-on-error: true
+        with:
+          path: build/externals
+          key: ${{ steps.ffmpeg-cache.outputs.cache-primary-key }}
+      - name: Check that every test executable is registered with ctest
+        shell: cmd
+        run: python tools/check_test_registration.py build
+      - shell: cmd
+        run: cmake --build build
+      - shell: cmd
+        run: cmake --build build --target libs
+      - shell: cmd
+        working-directory: build
+        run: ctest --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit ctest.xml
+      - run: python tools/check_skips.py build/ctest.xml --job windows
+      - name: Report tests slower than 30 s
+        if: ${{ !cancelled() }}
+        run: |
+          [ -f build/ctest.log ] || exit 0
+          awk '/Test +#[0-9]+: / && $NF == "sec" { for (i = 1; i <= NF; i++) if ($i ~ /^#[0-9]+:$/) name = $(i + 1); t = $(NF - 1) + 0; if (t > max[name]) max[name] = t }
+            END { for (name in max) if (max[name] > 30) printf "::warning title=Slow test::%s took %.1f s on %s; tests must finish in 30 s (CONTRIBUTING.md)\n", name, max[name], ENVIRON["RUNNER_OS"] }' build/ctest.log
+      - uses: actions/upload-artifact@v7
+        if: ${{ !cancelled() }}
+        with:
+          name: windows-msys2-tests
+          path: |
+            build/ctest.log
+            build/ctest.xml
+          if-no-files-found: ignore

.gitmodules

@@ -22,6 +22,9 @@
 [submodule "3rdparty/ffmpeg-core"]
 	path = 3rdparty/ffmpeg-core
 	url = https://github.com/KytyPS5/ext-ffmpeg-core.git
+[submodule "3rdparty/opus"]
+	path = 3rdparty/opus
+	url = https://github.com/xiph/opus.git
 [submodule "3rdparty/freetype"]
 	path = 3rdparty/freetype
 	url = https://github.com/freetype/freetype.git
@@ -52,3 +55,9 @@
 [submodule "3rdparty/vgmstream"]
 	path = 3rdparty/vgmstream
 	url = https://github.com/vgmstream/vgmstream.git
+[submodule "3rdparty/ogg"]
+	path = 3rdparty/ogg
+	url = https://github.com/xiph/ogg.git
+[submodule "3rdparty/vorbis"]
+	path = 3rdparty/vorbis
+	url = https://github.com/xiph/vorbis.git

CMakeLists.txt

@@ -107,6 +107,7 @@ if(BUILD_TESTING)
         ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/tests/RequestSerializerTests.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp)
     target_include_directories(request_serializer_tests PRIVATE
+        ${CMAKE_CURRENT_SOURCE_DIR}/core/libs
         ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler
         ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ControlFlow/include)
     add_test(NAME request_serializer COMMAND request_serializer_tests)
@@ -168,7 +169,7 @@ if(BUILD_TESTING)
             endif()
             add_test(NAME ${conventions_test_name} COMMAND ${Python3_EXECUTABLE}
                 ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_comments.py --shard ${shard}/12)
-            set_tests_properties(${conventions_test_name} PROPERTIES TIMEOUT 30)
+            set_tests_properties(${conventions_test_name} PROPERTIES TIMEOUT 120)
         endforeach()
         add_test(NAME conventions_wrapped_declarations COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_wrapped_declarations.py)
@@ -347,6 +348,7 @@ add_subdirectory(3rdparty/libusb-cmake EXCLUDE_FROM_ALL)
 set_target_properties(usb-1.0 PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
 include(cmake/ZArchive.cmake)
+include(cmake/Opus.cmake)
 if(WIN32)
     set(MI_BUILD_SHARED ON CACHE BOOL "" FORCE)
     set(MI_BUILD_STATIC OFF CACHE BOOL "" FORCE)

README.md

@@ -28,7 +28,7 @@ The [shader recompiler](core/shader/recompiler/Recompiler.cpp) successfully prod
 
 ## Compatibility
 
-Per-game progress (stage reached, open blockers, libraries and GPU coverage) is tracked on the [blocker map](https://alxcbtw.github.io/anyps5-roadmap/), a community project built from public GitHub data ([source](https://github.com/AlxcBTW/anyps5-roadmap)). Test reports go in the [Game bring-up](https://github.com/boykopovar/AnyPS5/discussions/categories/game-bring-up) discussions, one thread per title.
+Per-game status and test reports live in the [Game bring-up](https://github.com/boykopovar/AnyPS5/discussions/categories/game-bring-up) discussions, one thread per title.
 
 ## Input mapping
 

cmake/Opus.cmake

@@ -0,0 +1,19 @@
+function(add_bundled_opus)
+    set(BUILD_SHARED_LIBS OFF)
+    set(BUILD_TESTING OFF)
+    set(OPUS_BUILD_SHARED_LIBRARY OFF)
+    set(OPUS_BUILD_TESTING OFF)
+    set(OPUS_BUILD_PROGRAMS OFF)
+    set(OPUS_INSTALL_PKG_CONFIG_MODULE OFF)
+    set(OPUS_INSTALL_CMAKE_CONFIG_MODULE OFF)
+    set(OPUS_STACK_PROTECTOR OFF)
+    set(OPUS_FORTIFY_SOURCE OFF)
+    add_subdirectory(3rdparty/opus EXCLUDE_FROM_ALL)
+    set_target_properties(opus PROPERTIES POSITION_INDEPENDENT_CODE ON C_VISIBILITY_PRESET hidden)
+    target_compile_definitions(opus PRIVATE OPUS_EXPORT=)
+    if(MINGW OR CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        target_link_options(opus INTERFACE LINKER:--exclude-libs,libopus.a)
+    endif()
+endfunction()
+
+add_bundled_opus()

core/Decoder/Jpeg/include/Decoder/Jpeg.hpp

@@ -19,6 +19,10 @@ struct Header {
     std::uint32_t width;
     std::uint32_t height;
     std::uint32_t channels;
+    std::uint8_t componentIds[4]{};
+    std::uint8_t samplingFactors[4]{};
+    bool progressive = false;
+    bool app0BeforeFrame = false;
 };
 
 enum class Sampling {

core/Decoder/Jpeg/src/Jpeg.cpp

@@ -126,7 +126,33 @@ std::optional<Header> ParseHeader(std::span<const std::uint8_t> jpeg) {
     int channels = 0;
     if (!stbi_info_from_memory(jpeg.data(), static_cast<int>(jpeg.size()), &width, &height, &channels)) return std::nullopt;
     if (width <= 0 || height <= 0 || channels <= 0) return std::nullopt;
-    return Header{static_cast<std::uint32_t>(width), static_cast<std::uint32_t>(height), static_cast<std::uint32_t>(channels)};
+    Header header{static_cast<std::uint32_t>(width), static_cast<std::uint32_t>(height), static_cast<std::uint32_t>(channels)};
+    std::size_t offset = 2;
+    while (offset < jpeg.size()) {
+        if (jpeg[offset++] != 0xFF) return std::nullopt;
+        while (offset < jpeg.size() && jpeg[offset] == 0xFF) ++offset;
+        if (offset == jpeg.size()) return std::nullopt;
+        const std::uint8_t marker = jpeg[offset++];
+        if (marker == 0xD9 || marker == 0xDA) return std::nullopt;
+        if (marker == 0x01 || (marker >= 0xD0 && marker <= 0xD8)) continue;
+        if (jpeg.size() - offset < 2) return std::nullopt;
+        const std::size_t length = (static_cast<std::size_t>(jpeg[offset]) << 8) | jpeg[offset + 1];
+        if (length < 2 || length > jpeg.size() - offset) return std::nullopt;
+        if (marker == 0xE0) header.app0BeforeFrame = true;
+        if (marker == 0xC0 || marker == 0xC1 || marker == 0xC2) {
+            if (length < 8) return std::nullopt;
+            header.channels = jpeg[offset + 7];
+            if (header.channels == 0 || header.channels > 4 || length != 8 + 3 * header.channels) return std::nullopt;
+            header.progressive = marker == 0xC2;
+            for (std::uint32_t i = 0; i < header.channels; ++i) {
+                header.componentIds[i] = jpeg[offset + 8 + 3 * i];
+                header.samplingFactors[i] = jpeg[offset + 9 + 3 * i];
+            }
+            return header;
+        }
+        offset += length;
+    }
+    return std::nullopt;
 }
 
 std::optional<Image> Decode(std::span<const std::uint8_t> jpeg) {

core/Decoder/Png/src/Png.cpp

@@ -48,6 +48,37 @@ bool isChunkType(const std::uint8_t* bytes, const char* type) {
     return std::equal(bytes, bytes + 4, type);
 }
 
+std::optional<std::uint16_t> readGrayscaleTransparencyKey(std::span<const std::uint8_t> png) {
+    std::size_t offset = SIGNATURE.size();
+    if (png.size() < offset || !std::equal(SIGNATURE.begin(), SIGNATURE.end(), png.begin())) return std::nullopt;
+    std::optional<std::uint16_t> key;
+    while (png.size() - offset >= CHUNK_OVERHEAD) {
+        const std::uint32_t length = readBigEndian32(&png[offset]);
+        const std::uint8_t* type = &png[offset + 4];
+        if (length > png.size() - offset - CHUNK_OVERHEAD) return std::nullopt;
+        if (isChunkType(type, "tRNS")) {
+            if (length != 2) return std::nullopt;
+            key = static_cast<std::uint16_t>(png[offset + 8] << 8 | png[offset + 9]);
+        }
+        if (isChunkType(type, "IDAT") || isChunkType(type, "IEND")) return key;
+        offset += CHUNK_OVERHEAD + length;
+    }
+    return std::nullopt;
+}
+
+void repairGrayscaleTransparency(const Header& header, std::span<const std::uint8_t> png, Image& image) {
+    if (header.colorType != ColorType::Grayscale || !header.hasTransparency) return;
+    if (header.bitDepth != 1 && header.bitDepth != 2 && header.bitDepth != 4) return;
+    const std::optional<std::uint16_t> key = readGrayscaleTransparencyKey(png);
+    if (!key.has_value()) return;
+    const std::uint16_t mask = static_cast<std::uint16_t>((1u << header.bitDepth) - 1);
+    if ((*key & 0xFF) == (*key & mask)) return;
+    const std::uint8_t transparent = static_cast<std::uint8_t>((*key & mask) * (255u / mask));
+    for (std::size_t offset = 3; offset < image.pixels.size(); offset += 4) {
+        image.pixels[offset] = static_cast<std::uint8_t>(image.pixels[offset - 3] == transparent ? 0 : 255);
+    }
+}
+
 bool isValidFormat(std::uint8_t bitDepth, std::uint8_t colorType) {
     switch (colorType) {
     case 0:
@@ -158,6 +189,8 @@ std::optional<Image> Decode(std::span<const std::uint8_t> png) {
 
     Image image{static_cast<std::uint32_t>(width), static_cast<std::uint32_t>(height), {}};
     image.pixels.assign(decoded.get(), decoded.get() + static_cast<std::size_t>(image.width) * image.height * 4);
+    const std::optional<Header> header = ParseHeader(png);
+    if (header.has_value()) repairGrayscaleTransparency(*header, png, image);
     return image;
 }
 

core/Decoder/Png/tests/Png.cpp

@@ -3,6 +3,7 @@
 #include <algorithm>
 #include <cstdint>
 #include <cstdlib>
+#include <optional>
 #include <stdexcept>
 #include <vector>
 
@@ -50,6 +51,17 @@ static const std::uint8_t GRAY1[] = {
     0x42, 0x60, 0x82,
 };
 
+static const std::uint8_t GRAY1_TRNS[] = {
+    0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52,
+    0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0xDC, 0x59, 0x42,
+    0x27, 0x00, 0x00, 0x00, 0x02, 0x74, 0x52, 0x4E, 0x53, 0x00, 0x02, 0x98, 0x9D, 0xAC, 0x14, 0x00,
+    0x00, 0x00, 0x0A, 0x49, 0x44, 0x41, 0x54, 0x78, 0xDA, 0x63, 0x70, 0x00, 0x00, 0x00, 0x42, 0x00,
+    0x41, 0x84, 0xBF, 0x8E, 0x62, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE, 0x42, 0x60,
+    0x82,
+};
+
+static const std::uint8_t PNG_SIGNATURE[] = {0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A};
+
 static void AppendBigEndian32(std::vector<std::uint8_t>& bytes, std::uint32_t value) {
     for (int shift = 24; shift >= 0; shift -= 8) bytes.push_back(static_cast<std::uint8_t>(value >> shift));
 }
@@ -64,6 +76,130 @@ static std::vector<std::uint8_t> HeaderOnly(std::uint32_t width, std::uint32_t h
     return bytes;
 }
 
+static std::uint32_t Crc32(std::span<const std::uint8_t> bytes) {
+    std::uint32_t crc = 0xFFFFFFFFu;
+    for (const std::uint8_t byte : bytes) {
+        crc ^= byte;
+        for (int bit = 0; bit < 8; ++bit) crc = (crc >> 1) ^ ((crc & 1) != 0 ? 0xEDB88320u : 0u);
+    }
+    return ~crc;
+}
+
+static std::uint32_t Adler32(std::span<const std::uint8_t> bytes) {
+    std::uint32_t low = 1;
+    std::uint32_t high = 0;
+    for (const std::uint8_t byte : bytes) {
+        low = (low + byte) % 65521;
+        high = (high + low) % 65521;
+    }
+    return high << 16 | low;
+}
+
+static void AppendTestChunk(std::vector<std::uint8_t>& png, const char* type, std::span<const std::uint8_t> data) {
+    AppendBigEndian32(png, static_cast<std::uint32_t>(data.size()));
+    const std::size_t start = png.size();
+    png.insert(png.end(), type, type + 4);
+    png.insert(png.end(), data.begin(), data.end());
+    AppendBigEndian32(png, Crc32(std::span<const std::uint8_t>(png.data() + start, png.size() - start)));
+}
+
+static std::vector<std::uint8_t> ZlibStore(std::span<const std::uint8_t> bytes) {
+    Require(bytes.size() <= 0xFFFF);
+    const std::uint16_t length = static_cast<std::uint16_t>(bytes.size());
+    const std::uint16_t inverse = static_cast<std::uint16_t>(~length);
+    std::vector<std::uint8_t> output{0x78, 0x01, 0x01, static_cast<std::uint8_t>(length), static_cast<std::uint8_t>(length >> 8),
+                                     static_cast<std::uint8_t>(inverse), static_cast<std::uint8_t>(inverse >> 8)};
+    output.insert(output.end(), bytes.begin(), bytes.end());
+    AppendBigEndian32(output, Adler32(bytes));
+    return output;
+}
+
+static std::vector<std::uint8_t> GrayLowBitPng(std::uint8_t bitDepth, std::span<const std::uint8_t> samples,
+                                               std::optional<std::uint16_t> transparency,
+                                               std::optional<std::uint16_t> precedingTransparency = std::nullopt) {
+    Require(bitDepth == 1 || bitDepth == 2 || bitDepth == 4);
+    std::vector<std::uint8_t> rows(1 + (samples.size() * bitDepth + 7) / 8, 0);
+    std::size_t bit = 0;
+    for (const std::uint8_t sample : samples) {
+        Require(sample < (1u << bitDepth));
+        rows[1 + bit / 8] |= static_cast<std::uint8_t>(sample << (8 - bitDepth - bit % 8));
+        bit += bitDepth;
+    }
+
+    std::vector<std::uint8_t> ihdr;
+    AppendBigEndian32(ihdr, static_cast<std::uint32_t>(samples.size()));
+    AppendBigEndian32(ihdr, 1);
+    ihdr.insert(ihdr.end(), {bitDepth, 0, 0, 0, 0});
+
+    std::vector<std::uint8_t> png(PNG_SIGNATURE, PNG_SIGNATURE + sizeof(PNG_SIGNATURE));
+    AppendTestChunk(png, "IHDR", ihdr);
+    if (precedingTransparency.has_value()) {
+        const std::uint8_t key[2] = {static_cast<std::uint8_t>(*precedingTransparency >> 8), static_cast<std::uint8_t>(*precedingTransparency)};
+        AppendTestChunk(png, "tRNS", key);
+    }
+    if (transparency.has_value()) {
+        const std::uint8_t key[2] = {static_cast<std::uint8_t>(*transparency >> 8), static_cast<std::uint8_t>(*transparency)};
+        AppendTestChunk(png, "tRNS", key);
+    }
+    AppendTestChunk(png, "IDAT", ZlibStore(rows));
+    AppendTestChunk(png, "IEND", std::span<const std::uint8_t>());
+    return png;
+}
+
+static std::uint8_t GrayScale(std::uint8_t bitDepth) {
+    if (bitDepth == 1) return 255;
+    if (bitDepth == 2) return 85;
+    return 17;
+}
+
+static void RequireGrayTransparency(std::uint8_t bitDepth, std::span<const std::uint8_t> samples, std::uint16_t key,
+                                    std::uint8_t transparentSample, std::optional<std::uint16_t> precedingKey = std::nullopt) {
+    const std::vector<std::uint8_t> png = GrayLowBitPng(bitDepth, samples, key, precedingKey);
+    const std::vector<std::uint8_t> original = png;
+    const auto header = Decoder::Png::ParseHeader(png);
+    Require(header.has_value());
+    Require(header->width == static_cast<std::uint32_t>(samples.size()) && header->height == 1 && header->bitDepth == bitDepth);
+    Require(header->colorType == Decoder::Png::ColorType::Grayscale && header->hasTransparency);
+
+    const auto image = Decoder::Png::Decode(png);
+    Require(image.has_value() && image->pixels.size() == samples.size() * 4);
+    const std::uint8_t scale = GrayScale(bitDepth);
+    for (std::size_t i = 0; i < samples.size(); ++i) {
+        const std::uint8_t gray = static_cast<std::uint8_t>(samples[i] * scale);
+        Require(image->pixels[i * 4] == gray && image->pixels[i * 4 + 1] == gray && image->pixels[i * 4 + 2] == gray);
+        Require(image->pixels[i * 4 + 3] == (samples[i] == transparentSample ? 0 : 255));
+    }
+    Require(png == original);
+}
+
+static void RequireGrayOpaque(std::uint8_t bitDepth, std::span<const std::uint8_t> samples) {
+    const std::vector<std::uint8_t> png = GrayLowBitPng(bitDepth, samples, std::nullopt);
+    const std::vector<std::uint8_t> original = png;
+    const auto header = Decoder::Png::ParseHeader(png);
+    Require(header.has_value() && !header->hasTransparency);
+    const auto image = Decoder::Png::Decode(png);
+    Require(image.has_value() && image->pixels.size() == samples.size() * 4);
+    const std::uint8_t scale = GrayScale(bitDepth);
+    for (std::size_t i = 0; i < samples.size(); ++i) {
+        const std::uint8_t gray = static_cast<std::uint8_t>(samples[i] * scale);
+        Require(image->pixels[i * 4] == gray && image->pixels[i * 4 + 1] == gray && image->pixels[i * 4 + 2] == gray);
+        Require(image->pixels[i * 4 + 3] == 255);
+    }
+    Require(png == original);
+}
+
+static void RequireTruncatedTransparency() {
+    const std::uint8_t samples[] = {0, 1};
+    const std::vector<std::uint8_t> png = GrayLowBitPng(1, samples, 0x0002);
+    const std::size_t transparencyOffset = sizeof(PNG_SIGNATURE) + 25;
+    const std::vector<std::uint8_t> truncated(png.begin(), png.begin() + transparencyOffset + 9);
+    const std::vector<std::uint8_t> original = truncated;
+    const auto header = Decoder::Png::ParseHeader(truncated);
+    Require(header.has_value() && !header->hasTransparency);
+    (void)Decoder::Png::Decode(truncated);
+    Require(truncated == original);
+}
+
 static void RequireRoundTrip(std::uint32_t channels, Decoder::Png::ColorType colorType) {
     constexpr std::uint32_t width = 20;
     constexpr std::uint32_t height = 10;
@@ -176,6 +312,42 @@ int main() {
     const std::uint8_t levels[8] = {0, 255, 0, 255, 255, 0, 0, 255};
     for (int i = 0; i < 8; ++i) Require(gray1->pixels[i * 4] == levels[i] && gray1->pixels[i * 4 + 3] == 255);
 
+    const auto gray1TrnsHeader = Decoder::Png::ParseHeader(GRAY1_TRNS);
+    Require(gray1TrnsHeader.has_value());
+    Require(gray1TrnsHeader->width == 2 && gray1TrnsHeader->height == 1 && gray1TrnsHeader->bitDepth == 1);
+    Require(gray1TrnsHeader->colorType == Decoder::Png::ColorType::Grayscale && gray1TrnsHeader->hasTransparency);
+    const auto gray1Trns = Decoder::Png::Decode(GRAY1_TRNS);
+    Require(gray1Trns.has_value());
+    Require(gray1Trns->pixels == std::vector<std::uint8_t>({0, 0, 0, 0, 255, 255, 255, 255}));
+
+    const std::uint8_t gray1Samples[] = {0, 1, 0, 1, 1, 0, 0, 1};
+    RequireGrayTransparency(1, gray1Samples, 0x0000, 0);
+    RequireGrayTransparency(1, gray1Samples, 0x0002, 0);
+    RequireGrayTransparency(1, gray1Samples, 0x0100, 0);
+    RequireGrayTransparency(1, gray1Samples, 0xFF02, 0);
+    RequireGrayTransparency(1, gray1Samples, 0xFFFF, 1);
+    RequireGrayTransparency(1, gray1Samples, 0x0001, 1, 0x0002);
+    RequireGrayOpaque(1, gray1Samples);
+
+    const std::uint8_t gray2Samples[] = {0, 1, 2, 3};
+    RequireGrayTransparency(2, gray2Samples, 0x0001, 1);
+    RequireGrayTransparency(2, gray2Samples, 0x0005, 1);
+    RequireGrayTransparency(2, gray2Samples, 0x0100, 0);
+    RequireGrayTransparency(2, gray2Samples, 0x0101, 1);
+    RequireGrayTransparency(2, gray2Samples, 0xFF05, 1);
+    RequireGrayTransparency(2, gray2Samples, 0xFFFF, 3);
+    RequireGrayOpaque(2, gray2Samples);
+
+    const std::uint8_t gray4Samples[] = {0, 3, 10, 15, 1};
+    RequireGrayTransparency(4, gray4Samples, 0x0003, 3);
+    RequireGrayTransparency(4, gray4Samples, 0x0013, 3);
+    RequireGrayTransparency(4, gray4Samples, 0x0100, 0);
+    RequireGrayTransparency(4, gray4Samples, 0x0103, 3);
+    RequireGrayTransparency(4, gray4Samples, 0xFF13, 3);
+    RequireGrayTransparency(4, gray4Samples, 0xFFFF, 15);
+    RequireGrayOpaque(4, gray4Samples);
+    RequireTruncatedTransparency();
+
     const auto interlaced = Decoder::Png::ParseHeader(HeaderOnly(3, 4, 8, 6, 1));
     Require(interlaced.has_value() && interlaced->interlaced && !interlaced->hasTransparency);
     Require(!Decoder::Png::ParseHeader(HeaderOnly(0, 4, 8, 6, 0)).has_value());

core/libs/CMakeLists.txt

@@ -39,9 +39,16 @@ if(MINGW)
     check_cxx_source_compiles("#ifdef __USING_SJLJ_EXCEPTIONS__\n#error\n#endif\nint main() { return 0; }" ANYPS5_MINGW_DWARF_EXCEPTIONS)
     unset(CMAKE_REQUIRED_FLAGS)
     if(NOT ANYPS5_MINGW_DWARF_EXCEPTIONS)
-        message(FATAL_ERROR "${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} selects SjLj exceptions with -fno-asynchronous-unwind-tables; the libraries need DWARF unwinding. Use the MinGW-w64 toolchain listed in docs/dev/BUILD.md.")
+        check_cxx_source_compiles("#if !defined(__SEH__) || defined(__USING_SJLJ_EXCEPTIONS__)\n#error\n#endif\n[[gnu::sysv_abi, gnu::noinline]] void Throw() { throw 1; }\n[[gnu::sysv_abi, gnu::noinline]] int Catch() { try { Throw(); } catch (int value) { return value; } return 0; }\nint main() { return Catch() != 1; }" ANYPS5_MINGW_SYSV_SEH_EXCEPTIONS)
+        if(NOT ANYPS5_MINGW_SYSV_SEH_EXCEPTIONS)
+            message(FATAL_ERROR "The Windows libraries require DWARF exceptions or GCC with System V ABI SEH support. Use the toolchains listed in docs/dev/BUILD.md.")
+        endif()
+        add_library(windows_seh_personalities OBJECT ${CMAKE_CURRENT_SOURCE_DIR}/prx/libc/src/specifics/windows/SehPersonalities.cpp)
+        target_link_options(windows_seh_personalities INTERFACE LINKER:--undefined=LibcSehPersonalities_nid_no_patch)
+        link_libraries(windows_seh_personalities)
+    else()
+        add_compile_options(-fno-asynchronous-unwind-tables)
     endif()
-    add_compile_options(-fno-asynchronous-unwind-tables)
 endif()
 
 function(configure_windows_unwind target)
@@ -50,7 +57,7 @@ function(configure_windows_unwind target)
             message(FATAL_ERROR "Windows DWARF unwinding requires objcopy")
         endif()
         add_custom_command(TARGET ${target} POST_BUILD
-                COMMAND "${CMAKE_OBJCOPY}" --rename-section .eh_frame=.ehfram "$<TARGET_FILE:${target}>"
+                COMMAND "${CMAKE_OBJCOPY}" --rename-section .eh_frame=.ehfram --rename-section .eh_fram=.ehfram "$<TARGET_FILE:${target}>"
                 VERBATIM
         )
     endif()
@@ -61,6 +68,18 @@ if(MINGW)
         string(REPLACE "${runtimeLinkFlag}" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
     endforeach()
 
+    if(NOT EXISTS "${CMAKE_RANLIB}")
+        if(EXISTS "${CMAKE_CXX_COMPILER_RANLIB}")
+            set(CMAKE_RANLIB "${CMAKE_CXX_COMPILER_RANLIB}")
+        else()
+            get_filename_component(_compiler_dir "${CMAKE_CXX_COMPILER}" DIRECTORY)
+            if(EXISTS "${_compiler_dir}/gcc-ranlib${CMAKE_EXECUTABLE_SUFFIX}")
+                set(CMAKE_RANLIB "${_compiler_dir}/gcc-ranlib${CMAKE_EXECUTABLE_SUFFIX}")
+            elseif(EXISTS "${_compiler_dir}/ranlib${CMAKE_EXECUTABLE_SUFFIX}")
+                set(CMAKE_RANLIB "${_compiler_dir}/ranlib${CMAKE_EXECUTABLE_SUFFIX}")
+            endif()
+        endif()
+    endif()
     if(NOT EXISTS "${CMAKE_RANLIB}")
         message(FATAL_ERROR "Stable import libraries require ranlib")
     endif()
@@ -186,6 +205,8 @@ foreach(LIB_DIR ${LIBS_DIRS})
     endif()
 endforeach()
 
+target_link_libraries(libSceUsbStorageDialog PRIVATE libSceUserService)
+
 if(APS5_ENABLE_TRACY)
     set_target_properties(TracyClient PROPERTIES
             RUNTIME_OUTPUT_DIRECTORY ${UNPATCHED_DIR}
@@ -209,7 +230,7 @@ if(TARGET mimalloc)
     target_compile_features(pe_import_first PRIVATE cxx_std_20)
     if(MINGW)
         target_compile_options(pe_import_first PRIVATE -fasynchronous-unwind-tables)
-        target_link_options(pe_import_first PRIVATE -static -static-libgcc -static-libstdc++)
+        target_link_options(pe_import_first PRIVATE -municode -static -static-libgcc -static-libstdc++)
     endif()
     add_custom_command(
             OUTPUT ${PATCHED_DIR}/mimalloc.dll
@@ -239,6 +260,15 @@ if(BUILD_TESTING)
         set_tests_properties(guest_posix_scalar_io PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
     endif()
+    add_test_executable(guest_kernel_sys_sdk_version_tests tests/GuestKernelSysSdkVersion.cpp)
+    target_include_directories(guest_kernel_sys_sdk_version_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_sdk_version_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_sdk_version_tests)
+    add_test(NAME guest_kernel_sys_sdk_version COMMAND guest_kernel_sys_sdk_version_tests)
+    if(WIN32)
+        set_tests_properties(guest_kernel_sys_sdk_version PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+    endif()
     add_test_executable(nid_buffer_validation_tests nid/tests/BufferValidation.cpp)
     target_include_directories(nid_buffer_validation_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/nid/include)
     target_compile_features(nid_buffer_validation_tests PRIVATE cxx_std_20)
@@ -300,18 +330,55 @@ if(BUILD_TESTING)
     target_link_libraries(guest_kernel_sys_tests PRIVATE libkernel_sys libkernel libc)
     configure_windows_unwind(guest_kernel_sys_tests)
     add_test(NAME guest_kernel_sys COMMAND guest_kernel_sys_tests)
+    add_test_executable(guest_kernel_sys_thread_defaults_tests tests/GuestKernelSysThreadDefaults.cpp)
+    target_include_directories(guest_kernel_sys_thread_defaults_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_thread_defaults_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_thread_defaults_tests)
+    add_test(NAME guest_kernel_sys_thread_defaults COMMAND guest_kernel_sys_thread_defaults_tests)
+    set_tests_properties(guest_kernel_sys_thread_defaults PROPERTIES TIMEOUT 20)
+    add_test_executable(guest_kernel_sys_barrier_tests tests/GuestKernelSysBarrier.cpp)
+    target_include_directories(guest_kernel_sys_barrier_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_barrier_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_barrier_tests)
+    add_test(NAME guest_kernel_sys_barrier COMMAND guest_kernel_sys_barrier_tests)
+    set_tests_properties(guest_kernel_sys_barrier PROPERTIES TIMEOUT 20)
+    add_test_executable(guest_kernel_sys_pthread_tests tests/GuestKernelSysPthread.cpp)
+    target_include_directories(guest_kernel_sys_pthread_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_pthread_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_pthread_tests)
+    add_test(NAME guest_kernel_sys_pthread COMMAND guest_kernel_sys_pthread_tests)
     if(Python3_Interpreter_FOUND)
         add_test(NAME nid_verneed COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_verneed.py $<TARGET_FILE:nid_patcher>)
     endif()
     if(WIN32)
-        set_tests_properties(guest_kernel_sys PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(guest_kernel_sys guest_kernel_sys_thread_defaults guest_kernel_sys_barrier guest_kernel_sys_pthread PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
     endif()
+
+    add_test_executable(guest_gnm_neo_packets_tests tests/GuestGnmNeoPackets.cpp)
+    target_include_directories(guest_gnm_neo_packets_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_gnm_neo_packets_tests PRIVATE libSceGnmDriverForNeoMode libc)
+    configure_windows_unwind(guest_gnm_neo_packets_tests)
+    add_test(NAME guest_gnm_neo_packets COMMAND guest_gnm_neo_packets_tests)
+    if(WIN32)
+        set_tests_properties(guest_gnm_neo_packets PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceGnmDriverForNeoMode>")
+    endif()
     if(APPLE)
         add_subdirectory(tests/write_watch)
         add_subdirectory(tests/guest_arena)
     endif()
+    add_test_executable(guest_kernel_sys_module_internal_tests tests/GuestKernelSysModuleInternal.cpp)
+    target_include_directories(guest_kernel_sys_module_internal_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_module_internal_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_module_internal_tests)
+    add_test(NAME guest_kernel_sys_module_internal COMMAND guest_kernel_sys_module_internal_tests)
+    set_tests_properties(guest_kernel_sys_module_internal PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_kernel_sys_module_internal PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+    endif()
     if(WIN32)
         add_test_executable(windows_write_watch_tests tests/WindowsWriteWatch.cpp)
         target_include_directories(windows_write_watch_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
@@ -325,6 +392,9 @@ if(BUILD_TESTING)
 
     if(WIN32)
         add_test_executable(undeliverable_host_exception_tests tests/UndeliverableHostException.cpp)
+        if(MINGW)
+            target_compile_options(undeliverable_host_exception_tests PRIVATE -fno-asynchronous-unwind-tables -fno-exceptions)
+        endif()
         add_dependencies(undeliverable_host_exception_tests libc)
         add_test(NAME undeliverable_host_exception COMMAND undeliverable_host_exception_tests $<TARGET_FILE:libc>)
         set_tests_properties(undeliverable_host_exception PROPERTIES
@@ -337,6 +407,16 @@ if(BUILD_TESTING)
         target_compile_options(crash_text_tests PRIVATE -fasynchronous-unwind-tables)
     endif()
     add_test(NAME crash_text COMMAND crash_text_tests)
+    add_test_executable(guest_kernel_sys_pselect_tests tests/GuestKernelSysPselect.cpp)
+    target_include_directories(guest_kernel_sys_pselect_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_pselect_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_pselect_tests)
+    add_test(NAME guest_kernel_sys_pselect COMMAND guest_kernel_sys_pselect_tests)
+    set_tests_properties(guest_kernel_sys_pselect PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_kernel_sys_pselect PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+    endif()
 
     if(WIN32)
         add_test_executable(windows_crash_text_tests tests/WindowsCrashText.cpp)
@@ -360,6 +440,26 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceVoice>")
     endif()
 
+    add_test_executable(guest_gnm_neo_retail_tests tests/GuestGnmNeoRetail.cpp)
+    target_include_directories(guest_gnm_neo_retail_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_gnm_neo_retail_tests PRIVATE libSceGnmDriverForNeoMode libc)
+    configure_windows_unwind(guest_gnm_neo_retail_tests)
+    add_test(NAME guest_gnm_neo_retail COMMAND guest_gnm_neo_retail_tests)
+    if(WIN32)
+        set_tests_properties(guest_gnm_neo_retail PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceGnmDriverForNeoMode>")
+    endif()
+
+    add_test_executable(guest_vr_tracker_devices_tests tests/GuestVrTrackerDevices.cpp)
+    target_include_directories(guest_vr_tracker_devices_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_vr_tracker_devices_tests PRIVATE libSceVrTracker libc)
+    configure_windows_unwind(guest_vr_tracker_devices_tests)
+    add_test(NAME guest_vr_tracker_devices COMMAND guest_vr_tracker_devices_tests)
+    if(WIN32)
+        set_tests_properties(guest_vr_tracker_devices PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceVrTracker>")
+    endif()
+
     add_test_executable(guest_move_tests tests/GuestMove.cpp)
     target_include_directories(guest_move_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_move_tests PRIVATE libSceMove libc)
@@ -380,9 +480,43 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpParty>")
     endif()
 
+    add_test_executable(guest_np_ps4_imports_tests tests/GuestNpPs4Imports.cpp)
+    target_include_directories(guest_np_ps4_imports_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_np_ps4_imports_tests PRIVATE libSceNpMatching2 libSceNpSignaling libSceNpTrophy libc)
+    configure_windows_unwind(guest_np_ps4_imports_tests)
+    add_test(NAME guest_np_ps4_imports COMMAND guest_np_ps4_imports_tests)
+    if(WIN32)
+        set_tests_properties(guest_np_ps4_imports PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpMatching2>")
+    endif()
+
+    add_test_executable(guest_np_common_tests tests/GuestNpCommon.cpp)
+    target_include_directories(guest_np_common_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_np_common_tests PRIVATE libSceNpCommon libkernel libc)
+    configure_windows_unwind(guest_np_common_tests)
+    foreach(testCase IN ITEMS ids sync clock)
+        add_test(NAME guest_np_common_${testCase} COMMAND guest_np_common_tests ${testCase})
+        set_tests_properties(guest_np_common_${testCase} PROPERTIES TIMEOUT 20)
+        if(WIN32)
+            set_tests_properties(guest_np_common_${testCase} PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpCommon>")
+        endif()
+    endforeach()
+
+    add_test_executable(guest_np_common_runtime_tests tests/GuestNpCommonRuntime.cpp)
+    target_include_directories(guest_np_common_runtime_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_np_common_runtime_tests PRIVATE libSceNpCommon libkernel libc)
+    configure_windows_unwind(guest_np_common_runtime_tests)
+    add_test(NAME guest_np_common_runtime COMMAND guest_np_common_runtime_tests)
+    set_tests_properties(guest_np_common_runtime PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_np_common_runtime PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpCommon>")
+    endif()
+
     add_test_executable(guest_offline_imports_tests tests/GuestOfflineImports.cpp)
     target_include_directories(guest_offline_imports_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
-    target_link_libraries(guest_offline_imports_tests PRIVATE libSceHttp libSceNpEntitlementAccess libSceRudp libc)
+    target_link_libraries(guest_offline_imports_tests PRIVATE libSceHttp libSceNpEntitlementAccess libSceRudp libSceVideoRecordingP libc)
     configure_windows_unwind(guest_offline_imports_tests)
     add_test(NAME guest_offline_imports COMMAND guest_offline_imports_tests)
     if(Python3_Interpreter_FOUND)
@@ -394,6 +528,17 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceHttp>")
     endif()
 
+    add_test_executable(guest_entitlements_tests tests/GuestEntitlements.cpp)
+    target_include_directories(guest_entitlements_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_entitlements_tests PRIVATE libSceNpEntitlementAccess libc)
+    configure_windows_unwind(guest_entitlements_tests)
+    add_test(NAME guest_entitlements COMMAND guest_entitlements_tests)
+    add_test(NAME guest_entitlements_missing COMMAND guest_entitlements_tests missing)
+    if(WIN32)
+        set_tests_properties(guest_entitlements guest_entitlements_missing PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpEntitlementAccess>")
+    endif()
+
     add_test_executable(guest_signin_dialog_tests tests/GuestSigninDialog.cpp)
     target_include_directories(guest_signin_dialog_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_signin_dialog_tests PRIVATE libSceSigninDialog libc)
@@ -406,7 +551,7 @@ if(BUILD_TESTING)
 
     add_test_executable(guest_usb_storage_dialog_tests tests/GuestUsbStorageDialog.cpp)
     target_include_directories(guest_usb_storage_dialog_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
-    target_link_libraries(guest_usb_storage_dialog_tests PRIVATE libSceUsbStorageDialog libc)
+    target_link_libraries(guest_usb_storage_dialog_tests PRIVATE libSceUsbStorageDialog libSceUserService libc)
     configure_windows_unwind(guest_usb_storage_dialog_tests)
     add_test(NAME guest_usb_storage_dialog COMMAND guest_usb_storage_dialog_tests)
     if(WIN32)
@@ -593,6 +738,16 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpCppWebApi>")
     endif()
 
+    add_test_executable(guest_np_cpp_web_api_signed_out_tests tests/GuestNpCppWebApiSignedOut.cpp)
+    target_include_directories(guest_np_cpp_web_api_signed_out_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_np_cpp_web_api_signed_out_tests PRIVATE libSceNpCppWebApi libc)
+    configure_windows_unwind(guest_np_cpp_web_api_signed_out_tests)
+    add_test(NAME guest_np_cpp_web_api_signed_out COMMAND guest_np_cpp_web_api_signed_out_tests)
+    if(WIN32)
+        set_tests_properties(guest_np_cpp_web_api_signed_out PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpCppWebApi>")
+    endif()
+
     add_test_executable(savedata_core_write_replacement_tests tests/SaveDataWriteReplacement.cpp)
     target_include_directories(savedata_core_write_replacement_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(savedata_core_write_replacement_tests PRIVATE libSceSaveData)
@@ -611,6 +766,17 @@ if(BUILD_TESTING)
     configure_windows_unwind(savedata_replace_file_failure_tests)
     add_test(NAME savedata_replace_file_failure COMMAND savedata_replace_file_failure_tests)
 
+    add_test_executable(guest_kernel_sys_ktimer_tests tests/GuestKernelSysKtimer.cpp)
+    target_include_directories(guest_kernel_sys_ktimer_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_ktimer_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_ktimer_tests)
+    add_test(NAME guest_kernel_sys_ktimer COMMAND guest_kernel_sys_ktimer_tests)
+    set_tests_properties(guest_kernel_sys_ktimer PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_kernel_sys_ktimer PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+    endif()
+
     add_test_executable(guest_crt_tests tests/GuestCrt.cpp)
     target_include_directories(guest_crt_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_crt_tests PRIVATE libSceAmpr libc)
@@ -633,6 +799,17 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceSaveData.native>")
     endif()
 
+    add_test_executable(guest_kernel_sys_aio_tests tests/GuestKernelSysAio.cpp)
+    target_include_directories(guest_kernel_sys_aio_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_kernel_sys_aio_tests PRIVATE libkernel_sys libkernel libc)
+    configure_windows_unwind(guest_kernel_sys_aio_tests)
+    add_test(NAME guest_kernel_sys_aio COMMAND guest_kernel_sys_aio_tests)
+    set_tests_properties(guest_kernel_sys_aio PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_kernel_sys_aio PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
+    endif()
+
     add_test_executable(guest_posix_random_tests tests/GuestPosixRandom.cpp)
     target_include_directories(guest_posix_random_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_posix_random_tests PRIVATE libScePosixForWebKit)
@@ -685,11 +862,23 @@ if(BUILD_TESTING)
 
     add_test_executable(guest_vorbis_dec_tests tests/GuestVorbisDec.cpp)
     target_include_directories(guest_vorbis_dec_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
-    target_link_libraries(guest_vorbis_dec_tests PRIVATE libSceVorbisDec libc)
+    target_compile_definitions(guest_vorbis_dec_tests PRIVATE OV_EXCLUDE_STATIC_CALLBACKS)
+    target_link_libraries(guest_vorbis_dec_tests PRIVATE libSceVorbisDec vorbisenc vorbisfile vorbis ogg libc)
     configure_windows_unwind(guest_vorbis_dec_tests)
     add_test(NAME guest_vorbis_dec COMMAND guest_vorbis_dec_tests)
+    set_tests_properties(guest_vorbis_dec PROPERTIES TIMEOUT 30)
     if(WIN32)
         set_tests_properties(guest_vorbis_dec PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceVorbisDec>")
     endif()
+
+    add_test_executable(guest_gnm_neo_default_state_tests tests/GuestGnmNeoDefaultState.cpp)
+    target_include_directories(guest_gnm_neo_default_state_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_gnm_neo_default_state_tests PRIVATE libSceGnmDriverForNeoMode libc)
+    configure_windows_unwind(guest_gnm_neo_default_state_tests)
+    add_test(NAME guest_gnm_neo_default_state COMMAND guest_gnm_neo_default_state_tests)
+    if(WIN32)
+        set_tests_properties(guest_gnm_neo_default_state PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceGnmDriverForNeoMode>")
+    endif()
 endif()

core/libs/SceTypes.hpp

@@ -333,7 +333,7 @@ struct VaList {
 };
 #pragma pack(pop)
 
-using FiberEntry = void* (*)(std::uint64_t, void*);
+using FiberEntry = void (APS5_VABI*)(std::uint64_t argOnInitialize, std::uint64_t argOnRun);
 
 struct FiberOptParam {
     std::uint32_t magic;
@@ -432,6 +432,48 @@ struct AcmBatchError {
     std::uint32_t reserved[8];
 };
 
+struct AcmReverbBuffer {
+    void* samples;
+    const void* blockMetadata;
+};
+
+struct AcmReverbInput {
+    std::uint32_t blockSize;
+    std::uint32_t blockCount;
+    std::uint32_t channelCount;
+    std::uint32_t sampleFormat;
+    std::uint32_t currentBlock;
+    std::uint32_t availableBlocks;
+    const AcmReverbBuffer* const* history;
+    const void* const* inputChannels;
+    std::uint32_t pendingBlocks;
+};
+
+struct AcmReverbFilter {
+    std::uint32_t blockSize;
+    std::uint32_t blockCount;
+    std::uint32_t channelCount;
+    std::uint32_t sampleFormat;
+    const void* blockMetadata;
+    const AcmReverbBuffer* const* channels;
+};
+
+struct AcmReverbOutput {
+    std::uint32_t blockSize;
+    std::uint32_t channelCount;
+    void* const* samples;
+    void* const* workspace;
+    float gain;
+    float previousGain;
+    void* const* previousSamples;
+};
+
+static_assert(sizeof(AcmReverbBuffer) == 16);
+static_assert(offsetof(AcmReverbInput, history) == 24);
+static_assert(offsetof(AcmReverbInput, pendingBlocks) == 40);
+static_assert(sizeof(AcmReverbFilter) == 32);
+static_assert(offsetof(AcmReverbOutput, previousSamples) == 32);
+
 struct AcmBatchInfo {
     void* buffer;
     std::size_t offset;
@@ -613,6 +655,13 @@ struct AudioOutPortState {
     std::uint64_t activeState;
 };
 
+struct AudioOutSystemState {
+    float loudness;
+    std::uint8_t reserved8[4];
+    std::uint64_t reserved[3];
+};
+static_assert(sizeof(AudioOutSystemState) == 32);
+
 struct Audio3dOpenParameters {
     std::uint64_t size_this;
     std::uint32_t granularity;
@@ -933,13 +982,60 @@ struct AudiodecM4aacInfo {
     std::int32_t i_result;
 };
 
+struct VoiceEvent {
+    std::int32_t type;
+    std::uint32_t reserved;
+    void* userData;
+    union {
+        struct {
+            std::uint16_t count;
+            std::uint16_t ports[7];
+        } dataReady;
+        struct {
+            std::uint16_t port;
+            std::uint8_t state;
+            std::uint8_t reserved[13];
+        } deviceState;
+    };
+};
+
+using VoiceEventHandler = void (APS5_VABI*)(const VoiceEvent* event);
+
+struct VoiceThreadsParams {
+    std::uint32_t mode;
+    std::int32_t firstPriority;
+    std::uint64_t firstAffinity;
+    std::int32_t secondPriority;
+    std::uint32_t reserved20;
+    std::uint64_t secondAffinity;
+    std::int32_t thirdPriority;
+    std::uint32_t reserved36;
+    std::uint64_t thirdAffinity;
+};
+
+static_assert(sizeof(VoiceEvent) == 32);
+static_assert(offsetof(VoiceEvent, userData) == 8);
+static_assert(offsetof(VoiceEvent, dataReady) == 16);
+static_assert(offsetof(VoiceEvent, deviceState) == 16);
+static_assert(sizeof(VoiceThreadsParams) == 48);
+static_assert(offsetof(VoiceThreadsParams, firstAffinity) == 8);
+static_assert(offsetof(VoiceThreadsParams, secondPriority) == 16);
+static_assert(offsetof(VoiceThreadsParams, secondAffinity) == 24);
+static_assert(offsetof(VoiceThreadsParams, thirdPriority) == 32);
+static_assert(offsetof(VoiceThreadsParams, thirdAffinity) == 40);
+
 struct VoiceInitParam {
     std::int32_t app_type;
-    std::uint64_t on_event;
+    VoiceEventHandler on_event;
     void* user_data;
-    std::uint8_t reserved[32 - sizeof(std::int32_t) - sizeof(std::uint64_t) - sizeof(void*)];
+    std::uint8_t reserved[16];
 };
 
+static_assert(sizeof(VoiceInitParam) == 40);
+static_assert(offsetof(VoiceInitParam, on_event) == 8);
+static_assert(offsetof(VoiceInitParam, user_data) == 16);
+static_assert(offsetof(VoiceInitParam, reserved) == 24);
+
 struct VoicePortParam {
     std::int32_t port_type;
     std::uint16_t threshold;
@@ -962,14 +1058,30 @@ struct VoicePortInfo {
     std::uint16_t reserved;
 };
 
-struct VoiceResourceInfo {
-    std::uint16_t max_voice_input_ports;
-    std::uint16_t max_voice_output_ports;
-    std::uint16_t max_device_input_ports;
-    std::uint16_t max_device_output_ports;
-    std::uint16_t max_ports;
+struct NpAuthClientId;
+
+struct NpAuthAuthorizedAppCodeParameter {
+    std::uint64_t size;
+    std::int32_t userId;
+    const NpAuthClientId* clientId;
+    const char* scope;
+    std::uint32_t option;
 };
 
+static_assert(sizeof(NpAuthAuthorizedAppCodeParameter) == 40);
+static_assert(offsetof(NpAuthAuthorizedAppCodeParameter, clientId) == 16);
+static_assert(offsetof(NpAuthAuthorizedAppCodeParameter, scope) == 24);
+static_assert(offsetof(NpAuthAuthorizedAppCodeParameter, option) == 32);
+
+struct VoiceResourceInfo {
+    std::uint16_t maxVoiceInputPorts;
+    std::uint16_t maxVoiceOutputPorts;
+    std::uint16_t maxDeviceInputPorts;
+    std::uint16_t maxDeviceOutputPorts;
+};
+
+static_assert(sizeof(VoiceResourceInfo) == 8);
+
 struct VoiceStartParam {
     void* container;
     std::uint32_t mem_size;
@@ -1155,8 +1267,8 @@ struct ImeKeycode {
     std::uint64_t timestamp;
 };
 
-using ImeTextFilter = std::int32_t (APS5_VABI *)(char16_t* out_text, std::uint32_t* out_text_length, const char16_t* source_text, std::uint32_t source_text_length);
-using ImeExtKeyboardFilter = int (APS5_VABI *)(const ImeKeycode* source_keycode, std::uint16_t* out_keycode, std::uint32_t* out_status, void* reserved);
+using ImeTextFilter = std::int32_t (APS5_VABI*)(char16_t* outText, std::uint32_t* outTextLength, const char16_t* sourceText, std::uint32_t sourceTextLength);
+using ImeExtKeyboardFilter = int (APS5_VABI*)(const ImeKeycode* sourceKeycode, std::uint16_t* outKeycode, std::uint32_t* outStatus, void* reserved);
 
 struct ImeExtendedParam {
     std::uint32_t option;
@@ -1318,7 +1430,12 @@ struct NetEpollEvent {
     NetEpollData data;
 };
 
-struct NetCtlNatInfo { std::uint8_t opaque[128]; };
+struct NetCtlNatInfo {
+    std::uint32_t size;
+    std::uint8_t opaque[12];
+};
+
+static_assert(sizeof(NetCtlNatInfo) == 16);
 
 union NetCtlInfo { std::uint8_t opaque[256]; };
 
@@ -1333,10 +1450,10 @@ using NetCtlCallback = void (APS5_VABI *)(int, void*);
 
 struct HttpEpoll;
 using HttpEpollHandle = HttpEpoll*;
-using HttpsCallback = int (*)(int, unsigned int, void* const*, int, void*);
-using HttpRedirectCallback = int (*)(int, std::int32_t, std::int32_t*, const char*, void*);
-using HttpCookieRecvCallback = int (*)(int, const char*, const char*, std::uint64_t, void*);
-using HttpAuthInfoCallback = int (*)(int, int, const char*, char*, char*, int, std::uint8_t**, std::uint64_t*, int*, void*);
+using HttpsCallback = int (APS5_VABI*)(int, unsigned int, void* const*, int, void*);
+using HttpRedirectCallback = int (APS5_VABI*)(int, std::int32_t, std::int32_t*, const char*, void*);
+using HttpCookieRecvCallback = int (APS5_VABI*)(int, const char*, const char*, std::uint64_t, void*);
+using HttpAuthInfoCallback = int (APS5_VABI*)(int, int, const char*, char*, char*, int, std::uint8_t**, std::uint64_t*, int*, void*);
 
 struct HttpNBEvent { std::uint8_t opaque[64]; };
 
@@ -1395,6 +1512,24 @@ struct NpUnifiedEntitlementLabel {
     char padding[3];
 };
 
+struct NpServiceEntitlementLabel {
+    char data[7];
+    std::uint8_t reserved[13];
+};
+
+struct NpEntitlementAccessTransactionId {
+    char data[65];
+    std::uint8_t reserved[7];
+};
+
+struct NpEntitlementAccessListParameter;
+struct NpEntitlementAccessUnifiedInfo;
+struct NpEntitlementAccessServiceInfo;
+
+static_assert(sizeof(NpUnifiedEntitlementLabel) == 20);
+static_assert(sizeof(NpServiceEntitlementLabel) == 20);
+static_assert(sizeof(NpEntitlementAccessTransactionId) == 72);
+
 struct NpEntitlementAccessInitParam { char reserved[32]; };
 struct NpEntitlementAccessBootParam { char reserved[32]; };
 
@@ -1690,6 +1825,34 @@ struct AppContentBootParam {
 
 struct AppContentMountPoint { char data[16]; };
 
+using ContentExportMallocCallback = void* (APS5_VABI *)(std::size_t size, void* userData);
+using ContentExportFreeCallback = void (APS5_VABI *)(void* memory, void* userData);
+using ContentExportDataCallback = std::int32_t (APS5_VABI *)(const void** data, std::size_t* size, void* userData);
+
+struct ContentExportInitParam {
+    ContentExportMallocCallback mallocFunc;
+    ContentExportFreeCallback freeFunc;
+    void* userData;
+};
+
+static_assert(sizeof(ContentExportInitParam) == 24);
+static_assert(offsetof(ContentExportInitParam, freeFunc) == 8);
+static_assert(offsetof(ContentExportInitParam, userData) == 16);
+
+struct ContentExportSfParam {
+    std::size_t size;
+    const char* metadata;
+    const std::int64_t* contentIds;
+    std::uint32_t contentCount;
+    std::uint32_t option;
+};
+
+static_assert(sizeof(ContentExportSfParam) == 32);
+static_assert(offsetof(ContentExportSfParam, metadata) == 8);
+static_assert(offsetof(ContentExportSfParam, contentIds) == 16);
+static_assert(offsetof(ContentExportSfParam, contentCount) == 24);
+static_assert(offsetof(ContentExportSfParam, option) == 28);
+
 struct ContentExportInitParam2 {
     void* malloc_func;
     void* free_func;
@@ -1897,11 +2060,22 @@ union PlayGoOptionalChunk {
 };
 
 struct PlayGoDialogResult {
+    std::int32_t errorCode;
     std::int32_t result;
-    std::int32_t reserved[9];
+    std::uint8_t reserved[32];
 };
 
-using RudpEventHandler = void (*)(int ctx_id, int event_id, int error_code, void* arg);
+struct RudpPollEvent {
+    std::int32_t contextId;
+    std::uint16_t events;
+    std::uint16_t returnedEvents;
+};
+
+static_assert(sizeof(RudpPollEvent) == 8);
+static_assert(offsetof(RudpPollEvent, events) == 4);
+static_assert(offsetof(RudpPollEvent, returnedEvents) == 6);
+
+using RudpEventHandler = void (APS5_VABI*)(int ctxId, int eventId, int errorCode, void* arg);
 
 struct SystemServiceStatus {
     std::int32_t event_num = 0;
@@ -1998,12 +2172,16 @@ struct UserServiceGamePresets {
     std::uint32_t audio_language;
 };
 
-struct UltMutexOptParam {
-    std::uint32_t reserved_header[2];
+struct alignas(8) UltMutexOptParam {
+    std::uint32_t header[3];
+    std::uint32_t buildVersion;
     std::uint32_t attribute;
-    std::uint32_t reserved0;
+    std::uint32_t reserved[27];
 };
 
+static_assert(sizeof(UltMutexOptParam) == 128);
+static_assert(offsetof(UltMutexOptParam, attribute) == 16);
+
 struct UltUlthreadRuntimeOptParam { std::uint8_t bytes[128]; };
 
 using UltUlthreadEntry = std::int32_t (APS5_VABI *)(std::uint64_t);

core/libs/allocator/PeImportFirst.cpp

@@ -5,6 +5,7 @@
 #include <cctype>
 #include <cstdint>
 #include <cstring>
+#include <filesystem>
 #include <fstream>
 #include <iterator>
 #include <stdexcept>
@@ -78,16 +79,16 @@ void MoveImportFirst(std::vector<unsigned char>& bytes, const std::string& dll)
     for (std::size_t index = 0; index < descriptors.size(); ++index) std::memcpy(bytes.data() + directory + index * DescriptorBytes, descriptors[index].data(), DescriptorBytes);
 }
 
-void MoveImportFirstInFile(const std::string& path, const std::string& dll) {
+void MoveImportFirstInFile(const std::filesystem::path& path, const std::string& dll) {
     std::vector<unsigned char> bytes;
     {
         std::ifstream input(path, std::ios::binary);
-        if (!input) throw std::runtime_error("cannot open " + path);
+        if (!input) throw std::runtime_error("cannot open " + path.string());
         bytes.assign(std::istreambuf_iterator<char>(input), std::istreambuf_iterator<char>());
     }
     MoveImportFirst(bytes, dll);
     std::ofstream output(path, std::ios::binary | std::ios::trunc);
-    if (!output || !output.write(reinterpret_cast<const char*>(bytes.data()), static_cast<std::streamsize>(bytes.size()))) throw std::runtime_error("cannot write " + path);
+    if (!output || !output.write(reinterpret_cast<const char*>(bytes.data()), static_cast<std::streamsize>(bytes.size()))) throw std::runtime_error("cannot write " + path.string());
 }
 
 }

core/libs/allocator/PeImportFirst.hpp

@@ -1,13 +1,14 @@
 #ifndef CORE_LIBS_ALLOCATOR_PEIMPORTFIRST_HPP
 #define CORE_LIBS_ALLOCATOR_PEIMPORTFIRST_HPP
 
+#include <filesystem>
 #include <string>
 #include <vector>
 
 namespace Allocator {
 
 void MoveImportFirst(std::vector<unsigned char>& image, const std::string& dll);
-void MoveImportFirstInFile(const std::string& path, const std::string& dll);
+void MoveImportFirstInFile(const std::filesystem::path& path, const std::string& dll);
 
 }
 

core/libs/allocator/PeImportFirstMain.cpp

@@ -2,17 +2,37 @@
 
 #include <cstdio>
 #include <exception>
+#include <filesystem>
+#include <string>
 
-int main(int argc, char** argv) {
-    if (argc != 3) {
-        std::fprintf(stderr, "usage: pe_import_first <file> <dll>\n");
-        return 2;
-    }
+namespace {
+
+int Usage() {
+    std::fprintf(stderr, "usage: pe_import_first <file> <dll>\n");
+    return 2;
+}
+
+int Run(const std::filesystem::path& file, const std::string& dll) {
     try {
-        Allocator::MoveImportFirstInFile(argv[1], argv[2]);
+        Allocator::MoveImportFirstInFile(file, dll);
     } catch (const std::exception& error) {
-        std::fprintf(stderr, "pe_import_first: %s: %s\n", argv[1], error.what());
+        std::fprintf(stderr, "pe_import_first: %s: %s\n", file.string().c_str(), error.what());
         return 1;
     }
     return 0;
 }
+
+}
+
+#ifdef _WIN32
+int wmain(int argc, wchar_t** argv) {
+    if (argc != 3) return Usage();
+    const auto dll = std::filesystem::path(argv[2]).u8string();
+    return Run(argv[1], std::string(dll.begin(), dll.end()));
+}
+#else
+int main(int argc, char** argv) {
+    if (argc != 3) return Usage();
+    return Run(argv[1], argv[2]);
+}
+#endif

core/libs/allocator/tests/PeImportFirstTests.cpp

@@ -113,7 +113,7 @@ void movesTheNamedImportFirst() {
 void movesTheImportInAFile(const std::filesystem::path& directory) {
     const auto path = directory / "moved.dll";
     writeFile(path, image({"kernel32.dll", "mimalloc-redirect.dll"}));
-    Allocator::MoveImportFirstInFile(path.string(), "mimalloc-redirect.dll");
+    Allocator::MoveImportFirstInFile(path, "mimalloc-redirect.dll");
     const auto moved = readFile(path);
     require(importName(moved, 0) == "mimalloc-redirect.dll" && importName(moved, 1) == "kernel32.dll", "the file's import did not move first");
 }
@@ -122,7 +122,7 @@ void failsWithoutTouchingTheFile(const std::filesystem::path& directory, const s
     const auto path = directory / (what + ".dll");
     writeFile(path, bytes);
     try {
-        Allocator::MoveImportFirstInFile(path.string(), "mimalloc-redirect.dll");
+        Allocator::MoveImportFirstInFile(path, "mimalloc-redirect.dll");
     } catch (const std::exception& error) {
         require(std::string(error.what()).find(expected) != std::string::npos, what + ": failed with another error: " + error.what());
         require(readFile(path) == bytes, what + ": the file changed although the move failed");
@@ -134,7 +134,8 @@ void failsWithoutTouchingTheFile(const std::filesystem::path& directory, const s
 }
 
 int main() {
-    const auto directory = std::filesystem::temp_directory_path() / ("aps5-pe-import-first-" + std::to_string(std::random_device{}()));
+    auto directory = std::filesystem::temp_directory_path() / std::filesystem::path(u8"aps5-pe-import-first-a\u00e7\u00e3o-\u65e5\u672c-");
+    directory += std::to_string(std::random_device{}());
     try {
         std::filesystem::create_directories(directory);
         movesTheNamedImportFirst();

core/libs/nid/src/ExportExclusions.cpp

@@ -175,7 +175,7 @@ std::unordered_set<std::string> ReadExportExclusions(const std::string& path) {
 }
 
 std::unordered_set<std::string> ReadExportList(const std::string& path) {
-    std::ifstream input(path, std::ios::binary);
+    std::ifstream input(Internal::NativePath(path), std::ios::binary);
     if (!input) throw std::runtime_error("cannot open export list: " + path);
     const auto text = ReadText(path, input);
     if (text.empty() || text.back() != '\n') throw std::runtime_error("malformed export list: " + path);
@@ -192,8 +192,8 @@ std::unordered_set<std::string> ReadExportList(const std::string& path) {
 
 bool WriteExportList(const std::string& referencePath, const std::string& path) {
     const auto text = FormatExportList(ReadExportExclusions(referencePath));
-    if (std::ifstream existing(path, std::ios::binary); existing && ReadText(path, existing) == text) return false;
-    std::ofstream output(path, std::ios::binary | std::ios::trunc);
+    if (std::ifstream existing(Internal::NativePath(path), std::ios::binary); existing && ReadText(path, existing) == text) return false;
+    std::ofstream output(Internal::NativePath(path), std::ios::binary | std::ios::trunc);
     if (!output) throw std::runtime_error("cannot write export list: " + path);
     output.write(text.data(), static_cast<std::streamsize>(text.size()));
     output.close();

core/libs/nid/tests/test_macho_nids.py

@@ -271,7 +271,7 @@ def check_refusals(patcher, directory):
 
 def main():
     patcher = Path(sys.argv[1]).resolve()
-    with tempfile.TemporaryDirectory(prefix="anyps5-macho-nid-") as directory:
+    with tempfile.TemporaryDirectory(prefix="anyps5-macho-nid-a\u00e7\u00e3o-\u65e5\u672c-") as directory:
         check_shrinking(patcher, directory)
         check_growing(patcher, directory)
         check_preserved(patcher, directory)

core/libs/prx/libSceAcm/Export.cpp

@@ -79,19 +79,130 @@ int APS5_VABI sceAcmContextDestroy(AcmContextId context) {
     return 0;
 }
 
-int APS5_VABI sceAcm_ConvReverb_SharedInput(void) {
+int APS5_VABI sceAcm_ConvReverb_SharedInput(AcmBatchInfo* batch, std::uint32_t mode, AcmReverbInput* input, std::uint32_t outputCount, const AcmReverbFilter* const* filters, const float* gains, const AcmReverbOutput* const* outputs) {
     return 0;
 }
 
-int APS5_VABI sceAcm_FFT(void) {
+int APS5_VABI sceAcm_FFT(AcmBatchInfo* batch, std::int32_t transformSize, std::uint32_t transformCount, std::int32_t inputFormat, const void* const* inputs, std::int32_t outputFormat, void* const* outputs, std::int32_t flags) {
+    if (!batch || transformCount == 0 || transformCount > 0x7FFFFFFF || !inputs || !outputs || (flags & 3) == 3 || inputFormat < 0 || inputFormat > 1 || outputFormat < 0 || outputFormat > 1) return SCE_ACM_ERROR_BAD_POINTER;
+    if (transformSize != 512 && transformSize != 1024 && transformSize != 2048) return SCE_ACM_ERROR_BAD_POINTER;
+    const bool realInput = (flags & 1) != 0;
+    if (transformSize == 2048 && !realInput) return SCE_ACM_ERROR_BAD_POINTER;
+    const std::uint32_t inputBytes = transformSize * (inputFormat == 1 ? 2 : 4);
+    const std::uint32_t outputBytes = transformSize * (outputFormat == 1 ? 2 : 4);
+    const std::uint32_t inputBlocks = (inputBytes >> 9) >> (realInput ? 1 : 0);
+    const std::uint32_t outputBlocks = outputBytes >> 9;
+    std::uint32_t transformCommand;
+    std::uint32_t inputCommand;
+    std::uint32_t outputCommand;
+    std::uint32_t setupCommand;
+    std::uint32_t sizeCode;
+    if (transformSize == 512) {
+        sizeCode = 4;
+        transformCommand = realInput ? 7 : 11;
+        inputCommand = realInput ? 2 * inputFormat : 9 + inputFormat;
+        outputCommand = 4 | outputFormat;
+        setupCommand = realInput ? 6 : 12;
+    } else if (transformSize == 1024) {
+        sizeCode = 5;
+        transformCommand = realInput ? 8 : 13;
+        inputCommand = realInput ? 2 * inputFormat : 14 | inputFormat;
+        outputCommand = 5 | (2 * outputFormat);
+        setupCommand = realInput ? 10 : 12;
+    } else {
+        sizeCode = 6;
+        transformCommand = 6;
+        inputCommand = 2 * inputFormat;
+        outputCommand = 4 | outputFormat;
+        setupCommand = 8;
+    }
+    auto* buffer = static_cast<std::byte*>(batch->buffer);
+    auto offset = batch->offset;
+    const auto capacity = batch->buffer_size;
+    const auto appendWord = [&](std::uint64_t value) {
+        if (offset > capacity || capacity - offset < sizeof(value)) return false;
+        std::memcpy(buffer + offset, &value, sizeof(value));
+        offset += sizeof(value);
+        return true;
+    };
+    std::uint64_t header = static_cast<std::uint64_t>(sizeCode) << 8;
+    if (!(flags & 8)) {
+        const auto encodedCount = transformCount < 64000 ? transformCount : 64000;
+        header |= (std::uint64_t{1} << 16) | (static_cast<std::uint64_t>(transformSize) << 8) | (static_cast<std::uint64_t>(flags & 6) << 19) | (static_cast<std::uint64_t>(flags & 1) << 23) | (std::uint64_t{1} << (inputFormat == 1 ? 24 : 26)) | (std::uint64_t{1} << (outputFormat == 1 ? 25 : 27)) | (static_cast<std::uint64_t>(encodedCount) << 28);
+    }
+    constexpr int noSpace = static_cast<int>(0x81940009);
+    constexpr int badAlignment = static_cast<int>(0x8194000A);
+    if (!appendWord(header) || !appendWord(10 | (setupCommand << 8))) return noSpace;
+    for (std::uint32_t index = 0; index < transformCount; ++index) {
+        const auto inputIndex = static_cast<std::size_t>(index) * ((flags & 2) ? 2 : 1);
+        const auto input = reinterpret_cast<std::uintptr_t>(inputs[inputIndex]);
+        const auto secondInput = (flags & 2) ? reinterpret_cast<std::uintptr_t>(inputs[inputIndex + 1]) : 0;
+        if (!(flags & 2) && !input) return SCE_ACM_ERROR_BAD_POINTER;
+        if ((input | secondInput) & 127) return badAlignment;
+        const auto inputRecordSize = (flags & 2) ? 24u : 16u;
+        if (offset > capacity || capacity - offset < inputRecordSize) return noSpace;
+        appendWord(((flags & 2) ? 7 : 6) | (static_cast<std::uint64_t>(inputBlocks) << 32) | ((flags & 2) ? static_cast<std::uint64_t>(inputBytes / 2) << 48 : 0));
+        appendWord(input);
+        if (flags & 2) appendWord(secondInput);
+        if (!appendWord(10 | (inputCommand << 8)) || !appendWord(10 | (transformCommand << 8)) || !appendWord(10 | (outputCommand << 8))) return noSpace;
+        const auto outputIndex = static_cast<std::size_t>(index) * ((flags & 4) ? 2 : 1);
+        const auto output = reinterpret_cast<std::uintptr_t>(outputs[outputIndex]);
+        const auto secondOutput = (flags & 4) ? reinterpret_cast<std::uintptr_t>(outputs[outputIndex + 1]) : 0;
+        if ((output | secondOutput) & 127) return badAlignment;
+        const auto outputRecordSize = (flags & 4) ? 24u : 16u;
+        if (offset > capacity || capacity - offset < outputRecordSize) return noSpace;
+        appendWord(((flags & 4) ? 9 : 8) | (static_cast<std::uint64_t>(outputBlocks) << 32) | ((flags & 4) ? static_cast<std::uint64_t>(outputBytes / 2) << 48 : 0));
+        appendWord(output);
+        if (flags & 4) appendWord(secondOutput);
+    }
+    batch->offset = offset;
     return 0;
 }
 
-int APS5_VABI sceAcm_Panner(void) {
+int APS5_VABI sceAcm_Panner(AcmBatchInfo* batch, std::uint32_t sourceCount, const void* const* sources, std::uint32_t sourceMode, std::uint32_t outputMode, std::uint32_t channelCount, const void* const* sourceParameters, const void* const* sourceMatrices, const void* const* inputChannels, void* const* outputChannels) {
+    if (!batch || sourceCount == 0 || sourceCount > 65535 || sourceMode > 2 || outputMode > 1 || channelCount == 0 || channelCount > 36 || !sources || !sourceParameters || !sourceMatrices || !outputChannels) return SCE_ACM_ERROR_BAD_POINTER;
+    std::uintptr_t alignment = reinterpret_cast<std::uintptr_t>(sources) | reinterpret_cast<std::uintptr_t>(sourceParameters) | reinterpret_cast<std::uintptr_t>(sourceMatrices) | reinterpret_cast<std::uintptr_t>(inputChannels) | reinterpret_cast<std::uintptr_t>(outputChannels);
+    for (std::uint32_t index = 0; index < sourceCount; ++index) {
+        if (!sources[index] || !sourceParameters[index] || !sourceMatrices[index]) return SCE_ACM_ERROR_BAD_POINTER;
+        std::uint32_t parameterMode;
+        std::memcpy(&parameterMode, sourceParameters[index], sizeof(parameterMode));
+        if (parameterMode > 1) return SCE_ACM_ERROR_BAD_POINTER;
+        alignment |= reinterpret_cast<std::uintptr_t>(sources[index]) | reinterpret_cast<std::uintptr_t>(sourceParameters[index]) | reinterpret_cast<std::uintptr_t>(sourceMatrices[index]);
+    }
+    for (std::uint32_t index = 0; index < channelCount; ++index) {
+        if (!outputChannels[index]) return SCE_ACM_ERROR_BAD_POINTER;
+        alignment |= reinterpret_cast<std::uintptr_t>(outputChannels[index]);
+        if (inputChannels) alignment |= reinterpret_cast<std::uintptr_t>(inputChannels[index]);
+    }
+    if (alignment & 127) return static_cast<int>(0x8194000A);
+    const auto offset = batch->offset;
+    const auto capacity = batch->buffer_size;
+    if (offset > capacity || capacity - offset < 8) return static_cast<int>(0x81940009);
+    auto* buffer = static_cast<std::byte*>(batch->buffer);
+    const auto writeWord = [buffer](std::size_t position, std::uint64_t value) { std::memcpy(buffer + position, &value, sizeof(value)); };
+    const auto encodedCount = sourceCount < 255 ? sourceCount : 255;
+    writeWord(offset, (static_cast<std::uint64_t>(channelCount) << 24) | (static_cast<std::uint64_t>(encodedCount) << 16) | (std::uint64_t{1} << (sourceMode + 32)) | (std::uint64_t{1} << (outputMode + 35)) | 0x300);
+    if (capacity < 128 || capacity - offset < 152) return static_cast<int>(0x81940009);
+    const auto descriptorOffset = capacity - 128;
+    std::memset(buffer + descriptorOffset, 0, 128);
+    writeWord(descriptorOffset, sourceCount);
+    writeWord(descriptorOffset + 8, reinterpret_cast<std::uintptr_t>(inputChannels));
+    writeWord(descriptorOffset + 16, reinterpret_cast<std::uintptr_t>(outputChannels));
+    writeWord(descriptorOffset + 24, reinterpret_cast<std::uintptr_t>(sources));
+    writeWord(descriptorOffset + 32, sourceMode | (static_cast<std::uint64_t>(outputMode) << 32));
+    writeWord(descriptorOffset + 40, channelCount);
+    writeWord(descriptorOffset + 48, reinterpret_cast<std::uintptr_t>(sourceParameters));
+    writeWord(descriptorOffset + 56, reinterpret_cast<std::uintptr_t>(sourceMatrices));
+    writeWord(offset + 8, 0x8000000000000D);
+    writeWord(offset + 16, reinterpret_cast<std::uintptr_t>(buffer + descriptorOffset));
+    if (capacity - offset < 160) return static_cast<int>(0x81940009);
+    writeWord(offset + 24, 10);
+    batch->offset = offset + 32;
+    batch->buffer_size = descriptorOffset;
     return 0;
 }
 
-int APS5_VABI sceAcmBatchJobPriority(void) {
+int APS5_VABI sceAcmBatchJobPriority(AcmBatchInfo* batch, std::uint32_t priority) {
     NotImplemented_nid_no_patch(__func__);
     return 0;
 }

core/libs/prx/libSceAgc/Acb/src/Sync.cpp

@@ -44,10 +44,7 @@ std::uint32_t APS5_VABI sceAgcAcbQueueEndOfShaderActionGetSize() {
     return 32;
 }
 
-std::uint32_t* APS5_VABI sceAgcAcbMemSemaphore(CommandBuffer* buf, std::uint8_t action, std::uint8_t clientCode, const volatile void* address) {
-    const std::uint8_t operation = action != 0 ? 7u : 6u;
-    const std::uint8_t signalType = clientCode & 1u;
-    const std::uint8_t waitForMailbox = (clientCode >> 1u) & 1u;
+std::uint32_t* APS5_VABI sceAgcAcbMemSemaphore(CommandBuffer* buf, const volatile void* address, std::uint8_t waitForMailbox, std::uint8_t signalType, std::uint8_t operation) {
     return Agc::Command::WriteMemSemaphore(buf, address, waitForMailbox, signalType, operation, __func__);
 }
 

core/libs/prx/libSceAgc/CMakeLists.txt

@@ -157,7 +157,7 @@ add_test_executable(agc_register_command_tests EXCLUDE_FROM_ALL
         Patch/src/IndirectRegisters.cpp
 )
 target_include_directories(agc_register_command_tests PRIVATE ${LIBS_INCLUDE_DIR})
-target_link_libraries(agc_register_command_tests PRIVATE libc)
+target_link_libraries(agc_register_command_tests PRIVATE libSceAgcDriver libc)
 configure_windows_unwind(agc_register_command_tests)
 if(BUILD_TESTING)
     set_target_properties(agc_register_command_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -176,7 +176,7 @@ add_test_executable(agc_cb_cond_write_tests EXCLUDE_FROM_ALL
         Cb/src/Branch.cpp
 )
 target_include_directories(agc_cb_cond_write_tests PRIVATE ${LIBS_INCLUDE_DIR})
-target_link_libraries(agc_cb_cond_write_tests PRIVATE libc)
+target_link_libraries(agc_cb_cond_write_tests PRIVATE libSceAgcDriver libc)
 configure_windows_unwind(agc_cb_cond_write_tests)
 if(BUILD_TESTING)
     set_target_properties(agc_cb_cond_write_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -220,7 +220,7 @@ add_test_executable(agc_branch_patch_tests EXCLUDE_FROM_ALL
         Patch/src/Branch.cpp
 )
 target_include_directories(agc_branch_patch_tests PRIVATE ${LIBS_INCLUDE_DIR})
-target_link_libraries(agc_branch_patch_tests PRIVATE libc)
+target_link_libraries(agc_branch_patch_tests PRIVATE libSceAgcDriver libc)
 configure_windows_unwind(agc_branch_patch_tests)
 if(BUILD_TESTING)
     set_target_properties(agc_branch_patch_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -240,7 +240,7 @@ add_test_executable(agc_predication_tests EXCLUDE_FROM_ALL
         Predication/src/Predication.cpp
 )
 target_include_directories(agc_predication_tests PRIVATE ${LIBS_INCLUDE_DIR})
-target_link_libraries(agc_predication_tests PRIVATE libc)
+target_link_libraries(agc_predication_tests PRIVATE libSceAgcDriver libc)
 configure_windows_unwind(agc_predication_tests)
 if(BUILD_TESTING)
     set_target_properties(agc_predication_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -267,6 +267,19 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_instrumentation_query_size_tests EXCLUDE_FROM_ALL tests/InstrumentationQuerySize.cpp)
+target_include_directories(agc_instrumentation_query_size_tests PRIVATE ${LIBS_INCLUDE_DIR})
+target_link_libraries(agc_instrumentation_query_size_tests PRIVATE libSceAgc libc)
+configure_windows_unwind(agc_instrumentation_query_size_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_instrumentation_query_size_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
+    add_test(NAME agc_instrumentation_query_size_tests COMMAND agc_instrumentation_query_size_tests)
+    if(WIN32)
+        set_tests_properties(agc_instrumentation_query_size_tests PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_context_state_size_tests EXCLUDE_FROM_ALL tests/ContextStateSize.cpp)
 target_include_directories(agc_context_state_size_tests PRIVATE ${LIBS_INCLUDE_DIR})
 target_link_libraries(agc_context_state_size_tests PRIVATE libSceAgc libc)
@@ -357,7 +370,7 @@ add_test_executable(agc_dma_data_tests EXCLUDE_FROM_ALL
         DcbFlow/src/Memory.cpp
 )
 target_include_directories(agc_dma_data_tests PRIVATE ${LIBS_INCLUDE_DIR})
-target_link_libraries(agc_dma_data_tests PRIVATE libc)
+target_link_libraries(agc_dma_data_tests PRIVATE libSceAgcDriver libc)
 configure_windows_unwind(agc_dma_data_tests)
 if(BUILD_TESTING)
     set_target_properties(agc_dma_data_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -377,7 +390,7 @@ add_test_executable(agc_write_data_patch_tests EXCLUDE_FROM_ALL
         Patch/src/WaitRegMem.cpp
 )
 target_include_directories(agc_write_data_patch_tests PRIVATE ${LIBS_INCLUDE_DIR})
-target_link_libraries(agc_write_data_patch_tests PRIVATE libc)
+target_link_libraries(agc_write_data_patch_tests PRIVATE libSceAgcDriver libc)
 configure_windows_unwind(agc_write_data_patch_tests)
 if(BUILD_TESTING)
     set_target_properties(agc_write_data_patch_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -397,7 +410,7 @@ add_test_executable(agc_prime_utcl2_tests EXCLUDE_FROM_ALL
         DcbState/src/Display.cpp
 )
 target_include_directories(agc_prime_utcl2_tests PRIVATE ${LIBS_INCLUDE_DIR})
-target_link_libraries(agc_prime_utcl2_tests PRIVATE libc)
+target_link_libraries(agc_prime_utcl2_tests PRIVATE libSceAgcDriver libc)
 configure_windows_unwind(agc_prime_utcl2_tests)
 if(BUILD_TESTING)
     set_target_properties(agc_prime_utcl2_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -418,7 +431,7 @@ add_test_executable(agc_cond_exec_patch_tests EXCLUDE_FROM_ALL
         Patch/src/CondExec.cpp
 )
 target_include_directories(agc_cond_exec_patch_tests PRIVATE ${LIBS_INCLUDE_DIR})
-target_link_libraries(agc_cond_exec_patch_tests PRIVATE libc)
+target_link_libraries(agc_cond_exec_patch_tests PRIVATE libSceAgcDriver libc)
 configure_windows_unwind(agc_cond_exec_patch_tests)
 if(BUILD_TESTING)
     set_target_properties(agc_cond_exec_patch_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)

core/libs/prx/libSceAgc/Command/src/Packet.cpp

@@ -1,3 +1,4 @@
+#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
 #include "prx/libSceAgc/Command/include/Packet.hpp"
 
 #include <algorithm>
@@ -67,6 +68,7 @@ void Reserve(CommandBuffer* buffer, std::uint32_t count, const char* function) {
 std::uint32_t* Allocate(CommandBuffer* buffer, std::uint32_t count, const char* function) {
     Reserve(buffer, count, function);
     auto* result = buffer->cursor_up;
+    AgcDriverPrepareCommandWrite_nid_no_patch(reinterpret_cast<std::uint64_t>(result), static_cast<std::size_t>(count) * sizeof(std::uint32_t));
     buffer->cursor_up += count;
     return result;
 }

core/libs/prx/libSceAgc/DcbDraw/src/Stats.cpp

@@ -21,14 +21,13 @@ std::uint32_t* APS5_VABI sceAgcDcbGetLodStats(CommandBuffer* buf, std::uint8_t c
     return Agc::Command::Emit(buf, 0x10u, {bufferSizeInBytes, static_cast<std::uint32_t>(address), static_cast<std::uint32_t>(address >> 32u), control}, __func__);
 }
 
-std::uint32_t APS5_VABI sceAgcDcbBeginOcclusionQueryGetSize() {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
+std::uint32_t APS5_VABI sceAgcDcbBeginOcclusionQueryGetSize(std::int32_t queryType) {
+    if (queryType == 1) return 0x120u;
+    return queryType == 0 ? 0x10u : 0u;
 }
 
 std::uint32_t APS5_VABI sceAgcDcbEndOcclusionQueryGetSize() {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
+    return 0x10u;
 }
 
 }

core/libs/prx/libSceAgc/DcbFlow/src/Sync.cpp

@@ -52,10 +52,7 @@ std::uint32_t APS5_VABI sceAgcDcbStallCommandBufferParserGetSize() {
     return 8;
 }
 
-std::uint32_t* APS5_VABI sceAgcDcbMemSemaphore(CommandBuffer* buf, std::uint8_t action, std::uint8_t clientCode, const volatile void* address) {
-    const std::uint8_t operation = action != 0 ? 7u : 6u;
-    const std::uint8_t signalType = clientCode & 1u;
-    const std::uint8_t waitForMailbox = (clientCode >> 1u) & 1u;
+std::uint32_t* APS5_VABI sceAgcDcbMemSemaphore(CommandBuffer* buf, const volatile void* address, std::uint8_t waitForMailbox, std::uint8_t signalType, std::uint8_t operation) {
     return Agc::Command::WriteMemSemaphore(buf, address, waitForMailbox, signalType, operation, __func__);
 }
 

core/libs/prx/libSceAgc/Misc/src/Debug.cpp

@@ -1,9 +1,16 @@
 #include "prx/libSceAgc/Misc/include/Debug.hpp"
 
+#include <atomic>
 #include <cstdint>
 #include "SceTypes.hpp"
 #include "prx/libc/include/General.hpp"
 
+namespace {
+
+std::atomic<std::uint32_t> shaderInstrumentation{0};
+
+}
+
 extern "C" {
 
 int APS5_VABI sceAgcDebugRaiseException(std::uint32_t exceptionId) {
@@ -11,4 +18,13 @@ int APS5_VABI sceAgcDebugRaiseException(std::uint32_t exceptionId) {
     return 0;
 }
 
+int APS5_VABI sceAgcSetShaderInstrumentation(std::uint32_t flags) {
+    shaderInstrumentation.store(flags, std::memory_order_relaxed);
+    return 0;
+}
+
+std::uint32_t APS5_VABI sceAgcGetShaderInstrumentation() {
+    return shaderInstrumentation.load(std::memory_order_relaxed);
+}
+
 }

core/libs/prx/libSceAgc/Misc/src/Semaphore.cpp

@@ -13,20 +13,12 @@ namespace {
 
 constexpr std::uint64_t semaphoreMemoryAlignment = 0x4000u;
 constexpr std::uint64_t semaphoreLabelBytes = 32u;
-constexpr std::uint64_t semaphoreCounterBytes = 8u;
 
 std::mutex semaphoreMutex;
 std::uint8_t* semaphoreMemory = nullptr;
 std::uint64_t semaphoreBytes = 0;
 
-bool hasOnlyZeroLabelTails(const std::uint8_t* memory, std::uint64_t sizeInBytes) {
-    for (std::uint64_t label = 0; label < sizeInBytes; label += semaphoreLabelBytes) {
-        for (std::uint64_t byte = semaphoreCounterBytes; byte < semaphoreLabelBytes; ++byte) {
-            if (memory[label + byte] != 0) return false;
-        }
-    }
-    return true;
-}
+
 
 }
 
@@ -35,15 +27,12 @@ extern "C" {
 int APS5_VABI sceAgcSetAmmSemaphoreMemory(void* memory, std::uint64_t sizeInBytes) {
     const auto address = reinterpret_cast<std::uintptr_t>(memory);
     std::lock_guard lock(semaphoreMutex);
-    Agc::Command::Require(semaphoreMemory == nullptr, __func__, "the AMM semaphore memory is already registered; the console's error is unknown");
+    if (semaphoreBytes != 0) return static_cast<int>(0x8a6c0048u);
+    if (((address | sizeInBytes) & (semaphoreMemoryAlignment - 1u)) != 0) return static_cast<int>(0x8a6c0002u);
     Agc::Command::Require(memory != nullptr && sizeInBytes != 0, __func__, "null or empty AMM semaphore memory; the console's error is unknown");
-    Agc::Command::Require(((address | sizeInBytes) & (semaphoreMemoryAlignment - 1u)) == 0, __func__, "AMM semaphore memory or size is not 16 KiB aligned; the console's error is unknown");
     Agc::Command::Require(sizeInBytes - 1u <= std::numeric_limits<std::uintptr_t>::max() - address, __func__, "AMM semaphore memory wraps the address space");
     auto* labels = static_cast<std::uint8_t*>(memory);
-    Agc::Command::Require(hasOnlyZeroLabelTails(labels, sizeInBytes), __func__, "AMM semaphore labels hold data past their first 8 bytes; whether registration clears it is unknown");
-    for (std::uint64_t label = 0; label < sizeInBytes; label += semaphoreLabelBytes) {
-        std::memset(labels + label, 0, semaphoreCounterBytes);
-    }
+    std::memset(labels, 0, sizeInBytes);
     semaphoreMemory = labels;
     semaphoreBytes = sizeInBytes;
     return 0;
@@ -51,9 +40,9 @@ int APS5_VABI sceAgcSetAmmSemaphoreMemory(void* memory, std::uint64_t sizeInByte
 
 int APS5_VABI sceAgcGetSemaphoreLabel(std::uint32_t index, void** label) {
     std::lock_guard lock(semaphoreMutex);
-    Agc::Command::Require(semaphoreMemory != nullptr, __func__, "no AMM semaphore memory is registered; the console's error is unknown");
-    Agc::Command::Require(index < semaphoreBytes / semaphoreLabelBytes, __func__, "the semaphore label lies outside the AMM semaphore memory; the console's error is unknown");
-    Agc::Command::Require(label != nullptr, __func__, "null semaphore label output; the console's error is unknown");
+    if (semaphoreBytes == 0) return static_cast<int>(0x8a6c0049u);
+    if (index >= semaphoreBytes / semaphoreLabelBytes) return static_cast<int>(0x8a6c000bu);
+    Agc::Command::Require(label != nullptr, __func__, "null semaphore label output");
     *label = semaphoreMemory + static_cast<std::uint64_t>(index) * semaphoreLabelBytes;
     return 0;
 }

core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp

@@ -35,7 +35,6 @@ constexpr std::size_t FusedCodeAlignment = 256;
 constexpr char TrailerMagic[8] = {'b', 'a', 'r', 'e', 'f', 'o', 'o', 't'};
 constexpr std::size_t TrailerCodeBytesOffset = sizeof(TrailerMagic) + 3 * sizeof(std::uint32_t);
 constexpr std::uint32_t Rsrc1VgprMask = 0x3fu;
-constexpr std::uint32_t Rsrc1SgprMask = 0xfu << 6u;
 constexpr std::uint32_t Rsrc2OffchipLdsEnable = 1u << 18u;
 constexpr std::uint32_t Rsrc2UserSgprMask = (0x1fu << 1u) | (1u << 27u);
 
@@ -121,15 +120,19 @@ std::uint32_t FrontProgramBytes(const char* function, const Shader* front) {
     return static_cast<std::uint32_t>((end - 1) * 4);
 }
 
-std::uint32_t MergeRsrc1(std::uint32_t back, std::uint32_t front) {
-    const auto vgprs = std::max(back & Rsrc1VgprMask, front & Rsrc1VgprMask);
-    const auto sgprs = std::max(back & Rsrc1SgprMask, front & Rsrc1SgprMask);
-    return (back & ~(Rsrc1VgprMask | Rsrc1SgprMask)) | vgprs | sgprs;
-}
-
-std::uint32_t MergeRsrc2(const char* function, std::uint32_t back, std::uint32_t front) {
-    if ((front & ~(Rsrc2UserSgprMask | Rsrc2OffchipLdsEnable) & ~back) != 0) Fail(function, "front half requires hardware state the back half does not enable");
-    return (back & ~Rsrc2UserSgprMask) | (front & (Rsrc2UserSgprMask | Rsrc2OffchipLdsEnable));
+std::array<std::uint32_t, 2> mergeGeometryResources(std::uint32_t frontRsrc1, std::uint32_t frontRsrc2, std::uint32_t backRsrc1, std::uint32_t backRsrc2) {
+    const auto frontVgprs = ((frontRsrc1 & Rsrc1VgprMask) + 1u) * 4u;
+    const auto backVgprs = ((backRsrc1 & Rsrc1VgprMask) + 1u) * 4u;
+    const auto frontTotal = frontVgprs + (frontRsrc2 >> 28u) * 8u;
+    const auto backTotal = backVgprs + (backRsrc2 >> 28u) * 8u;
+    const auto maxTotal = std::max(frontTotal, backTotal);
+    const auto shared = std::max(frontVgprs, backVgprs) >= maxTotal ? 0u : (maxTotal - std::min(frontTotal, backTotal) + 7u) / 64u;
+    const auto vgprs = std::max(frontRsrc1 & Rsrc1VgprMask, backRsrc1 & Rsrc1VgprMask);
+    const auto rsrc1Field = std::max((frontRsrc1 >> 29u) & 3u, (backRsrc1 >> 29u) & 3u);
+    const auto rsrc2Field = std::max((frontRsrc2 >> 16u) & 3u, (backRsrc2 >> 16u) & 3u);
+    const auto rsrc1 = (backRsrc1 & ~(Rsrc1VgprMask | (3u << 29u))) | vgprs | (rsrc1Field << 29u);
+    const auto rsrc2 = (backRsrc2 & ~(0xf0000000u | (3u << 16u) | Rsrc2UserSgprMask | Rsrc2OffchipLdsEnable)) | ((shared & 15u) << 28u) | (rsrc2Field << 16u) | (frontRsrc2 & (Rsrc2UserSgprMask | Rsrc2OffchipLdsEnable));
+    return {rsrc1, rsrc2};
 }
 
 const ShaderRegister* FindHalfRegister(const Shader* shader, std::uint32_t offset) {
@@ -149,10 +152,14 @@ GeometryLayout ComputeGeometryLayout(const char* function, const Shader* front,
     const auto* frontRsrc1 = FindHalfRegister(front, SPI_SHADER_PGM_RSRC1_GS);
     const auto* frontRsrc2 = FindHalfRegister(front, SPI_SHADER_PGM_RSRC2_GS);
     if (frontRsrc1 == nullptr || frontRsrc2 == nullptr) Fail(function, "front half has no GS resource registers");
+    const auto* backRsrc1 = FindHalfRegister(back, SPI_SHADER_PGM_RSRC1_GS);
+    const auto* backRsrc2 = FindHalfRegister(back, SPI_SHADER_PGM_RSRC2_GS);
+    if (backRsrc1 == nullptr || backRsrc2 == nullptr) Fail(function, "back half has no GS resource registers");
+    const auto resources = mergeGeometryResources(frontRsrc1->value, frontRsrc2->value, backRsrc1->value, backRsrc2->value);
     layout.sh.assign(back->sh_registers, back->sh_registers + back->num_sh_registers);
     for (auto& reg : layout.sh) {
-        if (reg.offset == SPI_SHADER_PGM_RSRC1_GS) reg.value = MergeRsrc1(reg.value, frontRsrc1->value);
-        else if (reg.offset == SPI_SHADER_PGM_RSRC2_GS) reg.value = MergeRsrc2(function, reg.value, frontRsrc2->value);
+        if (reg.offset == SPI_SHADER_PGM_RSRC1_GS) reg.value = resources[0];
+        else if (reg.offset == SPI_SHADER_PGM_RSRC2_GS) reg.value = resources[1];
     }
     for (std::uint32_t i = 0; i < front->num_sh_registers; ++i) {
         const auto& reg = front->sh_registers[i];

core/libs/prx/libSceAgc/Patch/src/WaitRegMem.cpp

@@ -45,34 +45,29 @@ extern "C" {
 int APS5_VABI sceAgcWaitRegMemPatchAddress(std::uint32_t* cmd, const volatile void* address) {
     auto* wait = Agc::Command::ValidateWait(cmd, __func__);
     const auto guestAddress = reinterpret_cast<std::uintptr_t>(address);
-    const auto alignment = ((wait[0] >> 8u) & 0xffu) == 0x3cu ? 4u : 8u;
-    Agc::Command::CheckAddress(guestAddress, alignment, __func__);
-    Agc::Command::CheckBits(guestAddress, 0xffffffffffffull, __func__);
-    cmd[2] = (cmd[2] & 0xffff0000u) | static_cast<std::uint32_t>(guestAddress >> 32u);
+    const auto lowMask = ((wait[0] >> 8u) & 0xffu) == 0x3cu ? 3u : 7u;
+    cmd[2] = (cmd[2] & 0xffff0000u) | (static_cast<std::uint32_t>(guestAddress >> 32u) & 0xffffu);
     cmd[3] = static_cast<std::uint32_t>(guestAddress);
-    wait[2] = static_cast<std::uint32_t>(guestAddress);
-    wait[3] = static_cast<std::uint32_t>(guestAddress >> 32u);
+    wait[2] = (static_cast<std::uint32_t>(guestAddress) & ~lowMask) | (wait[2] & lowMask);
+    wait[3] = (static_cast<std::uint32_t>(guestAddress >> 32u) & 0x3ffffu) | (wait[3] & 0xfffc0000u);
     return 0;
 }
 
-int APS5_VABI sceAgcWaitRegMemPatchReference(std::uint32_t* cmd, std::uint64_t reference) {
+int APS5_VABI sceAgcWaitRegMemPatchReference(std::uint32_t* cmd, std::uint32_t reference) {
     auto* wait = Agc::Command::ValidateWait(cmd, __func__);
-    Agc::Command::CheckBits(reference, 0xffffffffu, __func__);
     wait[4] = static_cast<std::uint32_t>(reference);
     return 0;
 }
 
-int APS5_VABI sceAgcWaitRegMemPatchMask(std::uint32_t* cmd, std::uint64_t mask) {
+int APS5_VABI sceAgcWaitRegMemPatchMask(std::uint32_t* cmd, std::uint32_t mask) {
     auto* wait = Agc::Command::ValidateWait(cmd, __func__);
-    Agc::Command::CheckBits(mask, 0xffffffffu, __func__);
     wait[((wait[0] >> 8u) & 0xffu) == 0x3cu ? 5 : 6] = static_cast<std::uint32_t>(mask);
     return 0;
 }
 
 int APS5_VABI sceAgcWaitRegMemPatchCompareFunction(std::uint32_t* cmd, std::uint8_t compareFunction) {
     auto* wait = Agc::Command::ValidateWait(cmd, __func__);
-    Agc::Command::Require(compareFunction <= 6, __func__, "invalid wait comparison");
-    wait[1] = (wait[1] & ~7u) | compareFunction;
+    wait[1] = (wait[1] & ~7u) | (compareFunction & 7u);
     return 0;
 }
 

core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp

@@ -53,12 +53,9 @@ std::uint32_t CreateInterpolantValueSplitF16(std::uint32_t psWord, const ShaderS
         : CreateInterpolantValue(psWord, gsSemantic);
 }
 
-std::uint32_t UpdateInterpolantValue(std::uint32_t psWord, const ShaderSemantic* gsSemantic) {
-    const std::uint32_t f16 = (psWord >> 20u) & 0x3u;
-    if (f16 != 0 && f16 != InterpolantF16HiOnly && ((psWord >> 28u) & 0x3u) != ((psWord >> 30u) & 0x3u)) {
-        throw std::runtime_error(std::string(__func__) + ": DEFAULT_VAL of an f16 input with different low and high defaults is unknown");
-    }
-    return CreateInterpolantValueSplitF16(psWord, gsSemantic);
+std::uint32_t CreateInterpolantValueFlatDefault(std::uint32_t psWord, const ShaderSemantic* gsSemantic) {
+    const std::uint32_t value = CreateInterpolantValue(psWord, gsSemantic);
+    return ((psWord & 0x00700000u) == 0x00400000u) ? (value | 0x00000020u) : value;
 }
 
 int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader* gs, const Shader* ps, CreateInterpolantValueFn createValue, bool fillIdentityTail) {
@@ -132,7 +129,15 @@ int APS5_VABI sceAgcCreateInterpolantMapping_10000(ShaderRegister* regs, const S
 }
 
 int APS5_VABI sceAgcUpdateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
-    return CreateInterpolantMapping(__func__, regs, gs, ps, UpdateInterpolantValue, false);
+    return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueSplitF16, false);
+}
+
+int APS5_VABI sceAgcCreateInterpolantMapping_nid_postfix(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
+    return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueFlatDefault, true);
+}
+
+int APS5_VABI sceAgcUpdateInterpolantMapping_nid_postfix(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
+    return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueFlatDefault, false);
 }
 
 }

core/libs/prx/libSceAgc/Unimplemented.cpp

@@ -5,47 +5,37 @@
 
 extern "C" {
 
-int APS5_VABI sceAgcAcbPushMarkerSpan() {
+std::uint32_t* APS5_VABI sceAgcAcbPushMarkerSpan(CommandBuffer* buffer, const char* text, std::uint32_t length, std::uint32_t color) {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-int APS5_VABI sceAgcAcbSetMarkerSpan() {
+std::uint32_t* APS5_VABI sceAgcAcbSetMarkerSpan(CommandBuffer* buffer, const char* text, std::uint32_t length, std::uint32_t color) {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-int APS5_VABI sceAgcAcquireMemSetEngine() {
+int APS5_VABI sceAgcAcquireMemSetEngine(std::uint32_t* packet, std::uint8_t engine) {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-int APS5_VABI sceAgcDcbPushMarkerSpan() {
+std::uint32_t* APS5_VABI sceAgcDcbPushMarkerSpan(CommandBuffer* buffer, const char* text, std::uint32_t length, std::uint32_t color) {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-int APS5_VABI sceAgcDcbSetMarkerSpan() {
+std::uint32_t* APS5_VABI sceAgcDcbSetMarkerSpan(CommandBuffer* buffer, const char* text, std::uint32_t length, std::uint32_t color) {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-int APS5_VABI sceAgcSetSemaphoreMemory() {
+int APS5_VABI sceAgcSetSemaphoreMemory(void* memory, std::uint64_t size) {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }
 
-int APS5_VABI sceAgcCbMemsetExclusive() {
- NotImplemented_nid_no_patch(__func__);
- return 0;
-}
-
-int APS5_VABI sceAgcSetShaderInstrumentation() {
- NotImplemented_nid_no_patch(__func__);
- return 0;
-}
-
-int APS5_VABI sceAgcGetShaderInstrumentation() {
+std::uint32_t* APS5_VABI sceAgcCbMemsetExclusive(CommandBuffer* buffer, void* destination, const void* pattern, std::uint64_t size) {
  NotImplemented_nid_no_patch(__func__);
  return 0;
 }

core/libs/prx/libSceAgc/tests/Commands.cpp

@@ -28,8 +28,10 @@ extern "C" int APS5_VABI sceAgcInit_0090(std::uint32_t* state, std::uint32_t ver
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexAuto(CommandBuffer* buf, std::uint32_t indexCount, std::uint64_t modifier);
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexIndirect(CommandBuffer* buf, std::uint32_t dataOffsetInBytes, std::uint64_t modifier);
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexIndirectMulti(CommandBuffer* buf, std::uint32_t dataOffsetInBytes, std::uint32_t countIndirect, std::uint32_t maxCountOrCount, const volatile void* countAddress, std::uint32_t strideInBytes, std::uint64_t modifier);
-extern "C" int APS5_VABI sceAgcWaitRegMemPatchReference(std::uint32_t* cmd, std::uint64_t reference);
-extern "C" int APS5_VABI sceAgcWaitRegMemPatchMask(std::uint32_t* cmd, std::uint64_t mask);
+extern "C" int APS5_VABI sceAgcWaitRegMemPatchAddress(std::uint32_t* cmd, const volatile void* address);
+extern "C" int APS5_VABI sceAgcWaitRegMemPatchCompareFunction(std::uint32_t* cmd, std::uint8_t compareFunction);
+extern "C" int APS5_VABI sceAgcWaitRegMemPatchReference(std::uint32_t* cmd, std::uint32_t reference);
+extern "C" int APS5_VABI sceAgcWaitRegMemPatchMask(std::uint32_t* cmd, std::uint32_t mask);
 extern "C" int APS5_VABI sceAgcGetDataPacketPayloadAddress_0090(std::uint32_t** addr, std::uint32_t* cmd, int type);
 extern "C" std::uint32_t* APS5_VABI sceAgcCbSetShRegisterRangeDirect(CommandBuffer* buf, std::uint32_t offset, const std::uint32_t* values, std::uint32_t numValues);
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbContextStateOp_0100(CommandBuffer* buf, std::uint32_t operation);
@@ -531,18 +533,28 @@ void testMemory() {
     check(packet[8] == 0x55667788u && packet[9] == 0x11223344u && packet[12] == 2u, "wait reference or poll interval mismatch");
     sceAgcWaitRegMemPatchReference(packet, 7);
     check(packet[8] == 7 && packet[9] == 0x11223344u, "reference patch changed the high word");
-    const auto before = storage.words;
-    expectFailure([&] { sceAgcWaitRegMemPatchReference(packet, 0x100000000ull); });
-    check(storage.words == before, "invalid memory operation modified packet memory");
     auto* truncated = Agc::Command::WriteWait(&storage.buffer, 0, 3, 0, 0, &value, 0x100000000ull, 0xffffffff00000001ull, 32, __func__);
     check(truncated[8] == 0u && truncated[9] == 1u, "32-bit wait did not keep the low halves of the reference and mask");
     sceAgcWaitRegMemPatchMask(packet, 0x0f0f0f0fu);
     check(packet[10] == 0x0f0f0f0fu && packet[11] == 0xffffffffu && packet[8] == 7, "64-bit mask patch changed the wrong word");
     sceAgcWaitRegMemPatchMask(truncated, 0xff00u);
     check(truncated[9] == 0xff00u && truncated[8] == 0u && truncated[10] == 2u, "32-bit mask patch changed the wrong word");
+    for (auto* waitPacket : {packet, truncated}) {
+        const auto lowMask = waitPacket == packet ? 7u : 3u;
+        waitPacket[6] |= lowMask;
+        waitPacket[7] = 0xa5fc0000u;
+        const auto tag = waitPacket[2] & 0xffff0000u;
+        const std::uintptr_t address = 0x1236abcd12345679ull;
+        check(sceAgcWaitRegMemPatchAddress(waitPacket, reinterpret_cast<const volatile void*>(address)) == 0, "wait address patch failed");
+        check(waitPacket[2] == (tag | 0xabcdu) && waitPacket[3] == 0x12345679u, "wait wrapper address patch mismatch");
+        check(waitPacket[6] == ((0x12345679u & ~lowMask) | lowMask) && waitPacket[7] == 0xa5feabcdu, "wait address patch did not preserve control bits");
+        for (std::uint32_t comparison = 0; comparison <= 255; ++comparison) {
+            const auto control = waitPacket[5] & ~7u;
+            check(sceAgcWaitRegMemPatchCompareFunction(waitPacket, static_cast<std::uint8_t>(comparison)) == 0, "wait comparison patch failed");
+            check(waitPacket[5] == (control | (comparison & 7u)), "wait comparison patch changed control bits");
+        }
+    }
     const auto beforeMask = storage.words;
-    expectFailure([&] { sceAgcWaitRegMemPatchMask(packet, 0x100000000ull); });
-    expectFailure([&] { sceAgcWaitRegMemPatchMask(truncated, 0x100000000ull); });
     expectFailure([&] { sceAgcWaitRegMemPatchMask(packet + 4, 1); });
     check(storage.words == beforeMask, "invalid mask patch modified packet memory");
 }
@@ -691,15 +703,13 @@ void testDefaults() {
     expectFailure([] { Agc::Command::GetRegisterDefaults(0xffffffffu, true, __func__); });
 }
 
-struct DefaultsType {
-    std::uint32_t key;
-    std::uint32_t bankIndex;
-};
+constexpr std::size_t PairEntryBytes = 8;
+constexpr std::size_t CompactEntryBytes = 12;
 
-std::uint32_t colourTargetBlock(const void* root) {
+std::uint32_t colourTargetBlock(const void* root, std::size_t entryBytes) {
     const auto* bytes = static_cast<const unsigned char*>(root);
     ShaderRegister** contextBlocks = nullptr;
-    const DefaultsType* types = nullptr;
+    const unsigned char* types = nullptr;
     std::uint32_t count = 0;
     std::memcpy(&contextBlocks, bytes, sizeof(contextBlocks));
     std::memcpy(&types, bytes + 0x30, sizeof(types));
@@ -709,12 +719,17 @@ std::uint32_t colourTargetBlock(const void* root) {
     std::uint32_t matches = 0;
     std::uint32_t block = 0;
     for (std::uint32_t entry = 0; entry < count; ++entry) {
-        if ((types[entry].bankIndex & 3u) == 0)
+        std::uint32_t key = 0;
+        std::uint32_t bankIndex = 0;
+        std::memcpy(&key, types + entry * entryBytes, sizeof(key));
+        std::memcpy(&bankIndex, types + entry * entryBytes + 4, sizeof(bankIndex));
+        bankIndex &= 0x3ffu;
+        if ((bankIndex & 3u) == 0)
             ++contexts;
-        if (types[entry].key != 0x38e92c91u)
+        if (key != 0x38e92c91u)
             continue;
-        check((types[entry].bankIndex & 3u) == 0, "MRT0 defaults are not in the context bank");
-        block = types[entry].bankIndex >> 2;
+        check((bankIndex & 3u) == 0, "MRT0 defaults are not in the context bank");
+        block = bankIndex >> 2;
         ++matches;
     }
     check(matches == 1 && block < contexts && contextBlocks[block] != nullptr, "MRT0 defaults key is missing, repeated or outside the context blocks");
@@ -727,7 +742,11 @@ std::uint32_t colourTargetBlock(const void* root) {
 void testDefaultsTypes() {
     const auto* legacy = sceAgcGetRegisterDefaults();
     check(legacy != sceAgcGetRegisterDefaults2(13), "legacy register defaults reused the compact versioned table");
-    check(colourTargetBlock(legacy) == 78, "MRT0 defaults are not block 78 of the 137-entry table");
+    check(colourTargetBlock(legacy, PairEntryBytes) == 78, "MRT0 defaults are not block 78 of the 137-entry table");
+    check(colourTargetBlock(sceAgcGetRegisterDefaults2(13), CompactEntryBytes) == 78, "versioned MRT0 defaults are not block 78 of the compact 137-entry table");
+    check(colourTargetBlock(sceAgcGetRegisterDefaults2(10), CompactEntryBytes) == 73, "versioned MRT0 defaults are not block 73 of the compact 128-entry table");
+    for (std::uint32_t version = 4; version < 14; ++version)
+        colourTargetBlock(sceAgcGetRegisterDefaults2(version), CompactEntryBytes);
 
     const auto checkCompactTypes = [](const void* root, std::uint32_t count, std::uint32_t key,
                                       std::uint32_t packedType, std::uint32_t extra) {

core/libs/prx/libSceAgc/tests/InstrumentationQuerySize.cpp

@@ -0,0 +1,48 @@
+#include "prx/libc/include/general/VabiMacros.hpp"
+#include "prx/libc/include/Shutdown.hpp"
+
+#include <cstdint>
+#include <cstdio>
+#include <stdexcept>
+
+extern "C" {
+int APS5_VABI sceAgcSetShaderInstrumentation(std::uint32_t);
+std::uint32_t APS5_VABI sceAgcGetShaderInstrumentation();
+std::uint32_t APS5_VABI sceAgcDcbBeginOcclusionQueryGetSize(std::int32_t);
+std::uint32_t APS5_VABI sceAgcDcbEndOcclusionQueryGetSize();
+}
+
+namespace {
+
+void check(bool condition, const char* message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+void testShaderInstrumentation() {
+    check(sceAgcGetShaderInstrumentation() == 0, "shader instrumentation starts cleared");
+    check(sceAgcSetShaderInstrumentation(0x80000005u) == 0, "setting shader instrumentation failed");
+    check(sceAgcGetShaderInstrumentation() == 0x80000005u, "shader instrumentation flags were not kept");
+    check(sceAgcSetShaderInstrumentation(0) == 0 && sceAgcGetShaderInstrumentation() == 0, "shader instrumentation was not cleared");
+}
+
+void testOcclusionQuerySizes() {
+    check(sceAgcDcbBeginOcclusionQueryGetSize(0) == 0x10u, "begin occlusion query size for type 0");
+    check(sceAgcDcbBeginOcclusionQueryGetSize(1) == 0x120u, "begin occlusion query size for type 1");
+    check(sceAgcDcbBeginOcclusionQueryGetSize(2) == 0, "begin occlusion query size for an unknown type");
+    check(sceAgcDcbEndOcclusionQueryGetSize() == 0x10u, "end occlusion query size");
+}
+
+}
+
+int main() {
+    try {
+        testShaderInstrumentation();
+        testOcclusionQuerySizes();
+        LibcRunShutdown_nid_postfix();
+        std::puts("AGC shader instrumentation and occlusion query size tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "%s\n", error.what());
+        return 1;
+    }
+}

core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp

@@ -14,6 +14,8 @@
 extern "C" int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Shader* gs, const Shader* ps);
 extern "C" int APS5_VABI sceAgcCreateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps);
 extern "C" int APS5_VABI sceAgcUpdateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps);
+extern "C" int APS5_VABI sceAgcCreateInterpolantMapping_nid_postfix(ShaderRegister* regs, const Shader* gs, const Shader* ps);
+extern "C" int APS5_VABI sceAgcUpdateInterpolantMapping_nid_postfix(ShaderRegister* regs, const Shader* gs, const Shader* ps);
 
 namespace {
 
@@ -158,13 +160,46 @@ void testUpdateMapping() {
     check(std::memcmp(create.data(), update.data(), written) == 0, "update differs from the split f16 mapping");
     checkIdentity(create, static_cast<std::uint32_t>(inputs.size()), "unused interpolants are not the identity");
 
-    for (const std::uint32_t index : {1u, 3u}) {
-        inputs[index].default_value = 0;
-        update = filled();
-        expectFailure([&] { sceAgcUpdateInterpolantMapping_10000(update.data(), &gs, &ps); });
-        check(std::memcmp(update.data(), untouched.data(), sizeof(update)) == 0, "rejected update wrote registers");
-        inputs[index].default_value = inputs[index].default_value_hi;
+    inputs[1].default_value = 0;
+    inputs[3].default_value = 0;
+    update = filled();
+    check(sceAgcUpdateInterpolantMapping_10000(update.data(), &gs, &ps) == 0, "update with split f16 defaults failed");
+    check(update[1].value == 0x01380007u && update[3].value == 0x03680409u, "f16 DEFAULT_VAL does not come from the low default");
+}
+
+void testUnversionedMapping() {
+    std::array<ShaderSemantic, 5> inputs{
+        semantic(1, 0, 2), semantic(2, 0, 2), semantic(5, 0, 2), semantic(3, 0, 0), semantic(4, 0, 3)};
+    inputs[0].is_flat_shaded = 1;
+    inputs[0].default_value = 1;
+    inputs[0].default_value_hi = 3;
+    inputs[1].default_value_hi = 1;
+    inputs[2].is_flat_shaded = 1;
+    inputs[2].default_value_hi = 2;
+    inputs[3].is_flat_shaded = 1;
+    inputs[3].default_value = 2;
+    std::array<ShaderSemantic, 4> outputs{semantic(1, 7, 3), semantic(2, 4, 1), semantic(3, 9, 0), semantic(4, 2, 1)};
+    Shader gs{};
+    gs.output_semantics = outputs.data();
+    gs.num_output_semantics = static_cast<std::uint16_t>(outputs.size());
+    Shader ps{};
+    ps.input_semantics = inputs.data();
+    ps.num_input_semantics = static_cast<std::uint32_t>(inputs.size());
+    const std::array<std::uint32_t, 5> flatDefault{0x02680527u, 0x02380024u, 0x02580020u, 0x00000629u, 0x03180002u};
+    const std::size_t written = inputs.size() * sizeof(ShaderRegister);
+    const auto untouched = filled();
+
+    auto create = filled();
+    check(sceAgcCreateInterpolantMapping_nid_postfix(create.data(), &gs, &ps) == 0, "unversioned mapping failed");
+    for (std::uint32_t i = 0; i < inputs.size(); ++i) {
+        check(create[i].offset == ShaderRegs::SPI_PS_INPUT_CNTL_0 + i && create[i].value == flatDefault[i], "unversioned mapping changed");
     }
+    checkIdentity(create, static_cast<std::uint32_t>(inputs.size()), "unused interpolants are not the identity");
+
+    auto update = filled();
+    check(sceAgcUpdateInterpolantMapping_nid_postfix(update.data(), &gs, &ps) == 0, "unversioned update failed");
+    check(std::memcmp(update.data(), create.data(), written) == 0, "unversioned update differs from its mapping");
+    check(std::memcmp(update.data() + inputs.size(), untouched.data(), sizeof(update) - written) == 0, "unversioned update wrote past its inputs");
 }
 
 void testRejections() {
@@ -208,8 +243,9 @@ int main() {
         testIdentity();
         testMapping();
         testUpdateMapping();
+        testUnversionedMapping();
         testRejections();
-        check(mappings == 7u && links == 5u, "mapping preparation call count changed");
+        check(mappings == 10u && links == 8u, "mapping preparation call count changed");
         LibcRunShutdown_nid_postfix();
         std::puts("AGC interpolant mapping tests passed");
         return 0;

core/libs/prx/libSceAgc/tests/Semaphore.cpp

@@ -15,6 +15,8 @@
 extern "C" int APS5_VABI sceAgcSetAmmSemaphoreMemory(void* memory, std::uint64_t sizeInBytes);
 extern "C" int APS5_VABI sceAgcGetSemaphoreLabel(std::uint32_t index, void** label);
 extern "C" std::uint32_t* APS5_VABI sceAgcCbMemSemaphore(CommandBuffer* buf, const volatile void* address, std::uint32_t operation, std::uint32_t modifier, std::uint32_t value);
+extern "C" std::uint32_t* APS5_VABI sceAgcAcbMemSemaphore(CommandBuffer* buf, const volatile void* address, std::uint8_t waitForMailbox, std::uint8_t signalType, std::uint8_t operation);
+extern "C" std::uint32_t* APS5_VABI sceAgcDcbMemSemaphore(CommandBuffer* buf, const volatile void* address, std::uint8_t waitForMailbox, std::uint8_t signalType, std::uint8_t operation);
 
 namespace {
 
@@ -70,8 +72,8 @@ void fillStorage() {
 
 void testUnregistered() {
     std::array<void*, 3> labels{storage.get(), storage.get(), storage.get()};
-    expectFailure([&] { sceAgcGetSemaphoreLabel(0, &labels[1]); });
-    expectFailure([] { sceAgcGetSemaphoreLabel(0, nullptr); });
+    check(sceAgcGetSemaphoreLabel(0, &labels[1]) == static_cast<int>(0x8a6c0049u), "unregistered label error mismatch");
+    check(sceAgcGetSemaphoreLabel(0, nullptr) == static_cast<int>(0x8a6c0049u), "unregistered null label error mismatch");
     check(labels[0] == storage.get() && labels[1] == storage.get() && labels[2] == storage.get(), "a rejected semaphore label changed the output");
 }
 
@@ -80,10 +82,10 @@ void testRegistrationRejections() {
     expectFailure([] { sceAgcSetAmmSemaphoreMemory(nullptr, memoryBytes); });
     expectFailure([] { sceAgcSetAmmSemaphoreMemory(nullptr, 0); });
     expectFailure([] { sceAgcSetAmmSemaphoreMemory(memory(), 0); });
-    expectFailure([] { sceAgcSetAmmSemaphoreMemory(memory() + 1, memoryBytes); });
-    expectFailure([] { sceAgcSetAmmSemaphoreMemory(memory() + labelBytes, blockBytes); });
-    expectFailure([] { sceAgcSetAmmSemaphoreMemory(memory(), blockBytes - 1u); });
-    expectFailure([] { sceAgcSetAmmSemaphoreMemory(memory(), blockBytes + labelBytes); });
+    check(sceAgcSetAmmSemaphoreMemory(memory() + 1, memoryBytes) == static_cast<int>(0x8a6c0002u), "unaligned semaphore memory error mismatch");
+    check(sceAgcSetAmmSemaphoreMemory(memory() + labelBytes, blockBytes) == static_cast<int>(0x8a6c0002u), "unaligned semaphore memory error mismatch");
+    check(sceAgcSetAmmSemaphoreMemory(memory(), blockBytes - 1u) == static_cast<int>(0x8a6c0002u), "unaligned semaphore memory error mismatch");
+    check(sceAgcSetAmmSemaphoreMemory(memory(), blockBytes + labelBytes) == static_cast<int>(0x8a6c0002u), "unaligned semaphore memory error mismatch");
     expectFailure([] {
         const auto top = std::numeric_limits<std::uintptr_t>::max() & ~static_cast<std::uintptr_t>(blockBytes - 1u);
         sceAgcSetAmmSemaphoreMemory(reinterpret_cast<void*>(top), memoryBytes);
@@ -92,18 +94,8 @@ void testRegistrationRejections() {
     testUnregistered();
 }
 
-void testUnsettledClearing() {
-    for (const std::uint64_t offset : {5u * labelBytes + counterBytes, memoryBytes - 1u}) {
-        memory()[offset] = 0x77u;
-        const auto before = snapshot();
-        expectFailure([] { sceAgcSetAmmSemaphoreMemory(memory(), memoryBytes); });
-        checkUnchanged(before, "a registration with unsettled clearing changed memory");
-        testUnregistered();
-        memory()[offset] = 0;
-    }
-}
-
 void testRegistration() {
+    std::fill(memory(), memory() + memoryBytes, std::uint8_t{0x77});
     const auto before = snapshot();
     check(sceAgcSetAmmSemaphoreMemory(memory(), memoryBytes) == 0, "registering the AMM semaphore memory failed");
     check(std::all_of(memory(), memory() + memoryBytes, [](std::uint8_t byte) { return byte == 0; }), "registration did not clear the semaphore counters");
@@ -123,7 +115,7 @@ void testLabels() {
 void testLabelRejections() {
     for (const std::uint32_t index : {labelCount, labelCount + 1u, std::numeric_limits<std::uint32_t>::max()}) {
         void* label = storage.get();
-        expectFailure([&] { sceAgcGetSemaphoreLabel(index, &label); });
+        check(sceAgcGetSemaphoreLabel(index, &label) == static_cast<int>(0x8a6c000bu), "out-of-range label error mismatch");
         check(label == storage.get(), "a rejected semaphore label changed the output");
     }
     expectFailure([] { sceAgcGetSemaphoreLabel(0, nullptr); });
@@ -132,9 +124,9 @@ void testLabelRejections() {
 void testRepeatedRegistration() {
     std::fill(memory(), memory() + memoryBytes, std::uint8_t{0x3c});
     const auto before = snapshot();
-    expectFailure([] { sceAgcSetAmmSemaphoreMemory(memory(), memoryBytes); });
-    expectFailure([] { sceAgcSetAmmSemaphoreMemory(storage.get(), blockBytes); });
-    expectFailure([] { sceAgcSetAmmSemaphoreMemory(nullptr, 0); });
+    check(sceAgcSetAmmSemaphoreMemory(memory(), memoryBytes) == static_cast<int>(0x8a6c0048u), "repeated registration error mismatch");
+    check(sceAgcSetAmmSemaphoreMemory(storage.get(), blockBytes) == static_cast<int>(0x8a6c0048u), "repeated registration error mismatch");
+    check(sceAgcSetAmmSemaphoreMemory(nullptr, 0) == static_cast<int>(0x8a6c0048u), "repeated registration error mismatch");
     checkUnchanged(before, "a repeated registration changed memory");
     void* label = nullptr;
     check(sceAgcGetSemaphoreLabel(labelCount - 1u, &label) == 0 && label == memory() + memoryBytes - labelBytes, "a repeated registration replaced the semaphore memory");
@@ -190,6 +182,40 @@ void testMemSemaphore() {
     expectFailure([&] { sceAgcCbMemSemaphore(nullptr, label, 0, 0, 0); });
 }
 
+using QueueMemSemaphore = std::uint32_t* APS5_VABI (*)(CommandBuffer*, const volatile void*, std::uint8_t, std::uint8_t, std::uint8_t);
+
+void testQueueMemSemaphore(QueueMemSemaphore write) {
+    const auto* semaphore = reinterpret_cast<const volatile void*>(0x102f221d98ull);
+
+    {
+        std::array<std::uint32_t, 16> words{};
+        words.fill(0xdeadbeefu);
+        CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
+        const auto* packet = write(&buffer, semaphore, 0, 1, 6);
+        check(packet == words.data(), "packet address mismatch");
+        check(packet[0] == 0xc0023900u, "MEM_SEMAPHORE header opcode mismatch");
+        check(packet[1] == 0x2f221d98u, "MEM_SEMAPHORE address low mismatch");
+        check(packet[2] == 0x10u, "MEM_SEMAPHORE address high mismatch");
+        check(packet[3] == 0xc0100000u, "MEM_SEMAPHORE signal control word mismatch");
+        check(buffer.cursor_up == words.data() + 4, "cursor did not advance by 4 words");
+        check(words[4] == 0xdeadbeefu, "words past packet modified");
+    }
+
+    {
+        std::array<std::uint32_t, 16> words{};
+        words.fill(0xdeadbeefu);
+        CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
+        const auto* packet = write(&buffer, semaphore, 1, 0, 7);
+        check(packet[3] == 0xe0010000u, "MEM_SEMAPHORE wait control word mismatch");
+    }
+
+    std::array<std::uint32_t, 16> words{};
+    CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
+    expectFailure([&] { write(&buffer, reinterpret_cast<const volatile void*>(1ull), 0, 1, 6); });
+    expectFailure([&] { write(&buffer, semaphore, 0, 1, 5); });
+    expectFailure([&] { write(&buffer, semaphore, 2, 0, 7); });
+}
+
 }
 
 int main() {
@@ -198,10 +224,11 @@ int main() {
         fillStorage();
         testUnregistered();
         testRegistrationRejections();
-        testUnsettledClearing();
         testRegistration();
         testLabels();
         testMemSemaphore();
+        testQueueMemSemaphore(sceAgcAcbMemSemaphore);
+        testQueueMemSemaphore(sceAgcDcbMemSemaphore);
         testLabelRejections();
         testRepeatedRegistration();
         storage.reset();

core/libs/prx/libSceAgc/tests/ShaderFusion.cpp

@@ -147,7 +147,7 @@ void testFusion(std::size_t misalignment, std::uint32_t terminator = SSetpcS6) {
     const auto* frontLo = find(fused, 0x0C8u);
     check(backLo != nullptr && backLo->value == static_cast<std::uint32_t>((base + 256) >> 8u), "back program address was not patched");
     check(frontLo != nullptr && frontLo->value == static_cast<std::uint32_t>(base >> 8u), "fused program address was not patched");
-    check(find(fused, 0x08Au)->value == ((1u << 24u) | (2u << 6u) | 5u), "RSRC1 register counts were not merged");
+    check(find(fused, 0x08Au)->value == ((1u << 24u) | (1u << 6u) | 5u), "RSRC1 register counts were not merged");
     check(find(fused, 0x08Bu)->value == ((6u << 1u) | (1u << 7u) | (1u << 15u)), "RSRC2 user SGPRs were not taken from the front half");
     check(find(fused, 0x0B2u) != nullptr && find(fused, 0x0B2u)->value == 0x55u, "front-only SH register was lost");
     check(find(fused, 0x080u) == nullptr, "front program checksum was kept");
@@ -208,7 +208,7 @@ void testOffchipLdsEnable() {
             Shader fused{};
             check(sceAgcFuseShaderHalves_0200(&fused, &halves.front, &halves.back, aligned(storage, 256)) == 0, "offchip LDS enable rejected by fusion");
             const auto* resource = find(fused, 0x08bu);
-            check(resource != nullptr && resource->value == ((6u << 1u) | (1u << 7u) | (1u << 15u) | frontEnable | backEnable), "fusion lost offchip LDS enable or changed the back-half state or front user SGPRs");
+            check(resource != nullptr && resource->value == ((6u << 1u) | (1u << 7u) | (1u << 15u) | frontEnable), "fusion did not take offchip LDS enable and user SGPRs from the front half");
         }
     }
     Halves reported;
@@ -222,6 +222,38 @@ void testOffchipLdsEnable() {
     check(find(fused, 0x08bu)->value == 0x00af0004u, "reported shader resources did not retain the front offchip LDS enable");
 }
 
+void testResourceMerging() {
+    struct Case {
+        std::uint32_t front1;
+        std::uint32_t front2;
+        std::uint32_t back1;
+        std::uint32_t back2;
+        std::uint32_t expected1;
+        std::uint32_t expected2;
+    };
+    const Case cases[] = {
+        {0u, 0x80000000u, 0u, 0u, 0u, 0x10000000u},
+        {1u, 0xf0000000u, 3u, 0u, 3u, 0x10000000u},
+        {0u, 0xf0000000u, 0u, 0xf0000000u, 0u, 0u},
+        {0x400003c1u, 0x0006000cu, 0x61000043u, 0x00018001u, 0x61000043u, 0x0006800du},
+        {0x60000007u, 0x00010000u, 0x21000041u, 0x00020000u, 0x61000047u, 0x00020000u}
+    };
+    for (const auto& entry : cases) {
+        Halves halves;
+        halves.frontSh[1].value = entry.front1;
+        halves.frontSh[2].value = entry.front2;
+        halves.backSh[2].value = entry.back1;
+        halves.backSh[3].value = entry.back2;
+        SizeAlign size{};
+        check(sceAgcGetFusedShaderSize_0080(&size, &halves.front, &halves.back) == 0, "resource merge size query failed");
+        std::vector<std::uint8_t> scratch(size.m_size);
+        Shader fused{};
+        check(sceAgcFuseShaderHalves_0200(&fused, &halves.front, &halves.back, scratch.data()) == 0, "resource merge failed");
+        check(find(fused, 0x08au)->value == entry.expected1, "resource register 1 merge mismatch");
+        check(find(fused, 0x08bu)->value == entry.expected2, "resource register 2 merge mismatch");
+    }
+}
+
 void testRejections() {
     Halves halves;
     SizeAlign size{};
@@ -249,11 +281,6 @@ void testRejections() {
         noTrailer.frontCode[5] = 0;
         expectFailure([&] { sceAgcGetFusedShaderSize_0080(&size, &noTrailer.front, &noTrailer.back); });
     }
-    {
-        Halves hardware;
-        hardware.frontSh[2].value |= 1u << 16u;
-        expectFailure([&] { sceAgcGetFusedShaderSize_0080(&size, &hardware.front, &hardware.back); });
-    }
     {
         Halves constants;
         constants.back.embedded_constant_buffer_size_dqw = 1;
@@ -305,6 +332,7 @@ int main() {
         testFusion(0, SSwappcNullS6);
         testScratchReuse();
         testOffchipLdsEnable();
+        testResourceMerging();
         testRejections();
         testHullHalves();
         LibcRunShutdown_nid_postfix();

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -225,6 +225,22 @@ foreach(agcSampleOrderVariant IN ITEMS "R8;comp;-DTEXEL_FORMAT=r8ui" "R16;comp;-
     )
     list(APPEND agcSampleOrderShaderHeaders ${agcSampleOrderShaderHeader})
 endforeach()
+foreach(agcFirstSampleClass IN ITEMS Float Uint Sint)
+    string(TOUPPER "${agcFirstSampleClass}" agcFirstSampleSymbol)
+    set(agcFirstSampleShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/FirstSample.frag)
+    set(agcFirstSampleShaderSpv ${agcTextureDetileGeneratedDir}/FirstSample${agcFirstSampleClass}.spv)
+    set(agcFirstSampleShaderHeader ${agcTextureDetileGeneratedDir}/FirstSample${agcFirstSampleClass}_spv.h)
+    add_custom_command(
+            OUTPUT "${agcFirstSampleShaderHeader}"
+            COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}"
+            COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -D${agcFirstSampleSymbol}_SAMPLES=1 -o "${agcFirstSampleShaderSpv}" "${agcFirstSampleShaderSource}"
+            COMMAND ${CMAKE_COMMAND} -DINPUT=${agcFirstSampleShaderSpv} -DOUTPUT=${agcFirstSampleShaderHeader}
+            -DSYMBOL=FIRST_SAMPLE_${agcFirstSampleSymbol}_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
+            DEPENDS "${agcFirstSampleShaderSource}" glslang-standalone
+            VERBATIM
+    )
+    list(APPEND agcPresentationShaderHeaders ${agcFirstSampleShaderHeader})
+endforeach()
 foreach(agcOverlayStage IN ITEMS Vert Frag)
     string(TOLOWER "${agcOverlayStage}" agcOverlayExtension)
     string(TOUPPER "${agcOverlayStage}" agcOverlaySymbol)
@@ -323,6 +339,7 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Draw.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DepthSurface.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/MultisampleTarget.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/FirstSampleCopy.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ColorTargetLayout.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ColorTargetTransfer.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Pipeline.cpp
@@ -337,6 +354,7 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderResources.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderAddressResources.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/GuestBufferMemory.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ResidentMirror.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/BdaResources.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/BufferAddress.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderValidation.cpp
@@ -433,6 +451,16 @@ if(MINGW OR CMAKE_SYSTEM_NAME STREQUAL "Linux")
 endif()
 
 if(BUILD_TESTING)
+    add_test_executable(agc_driver_unsupported_diagnostics_tests tests/UnsupportedDiagnostics.cpp)
+    target_include_directories(agc_driver_unsupported_diagnostics_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(agc_driver_unsupported_diagnostics_tests PRIVATE libSceAgcDriver)
+    configure_windows_unwind(agc_driver_unsupported_diagnostics_tests)
+    add_test(NAME agc_driver_unsupported_diagnostics COMMAND agc_driver_unsupported_diagnostics_tests)
+    if(WIN32)
+        set_tests_properties(agc_driver_unsupported_diagnostics PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceAgcDriver>")
+    endif()
+
     add_test_executable(agc_driver_draw_program_trace_tests
             tests/DrawProgramTrace.cpp
             Execution/src/Driver/Draw/DrawProgramTrace.cpp
@@ -459,6 +487,16 @@ if(WIN32)
 endif()
 
 if(BUILD_TESTING)
+    add_test_executable(agc_driver_nop_data_packet_tests tests/NopDataWritePacket.cpp)
+    target_include_directories(agc_driver_nop_data_packet_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(agc_driver_nop_data_packet_tests PRIVATE libSceAgcDriver)
+    configure_windows_unwind(agc_driver_nop_data_packet_tests)
+    add_test(NAME agc_driver_nop_data_packet COMMAND agc_driver_nop_data_packet_tests)
+    if(WIN32)
+        set_tests_properties(agc_driver_nop_data_packet PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+
     add_test_executable(agc_frame_dump_io_tests tests/FrameDumpIo.cpp)
     target_include_directories(agc_frame_dump_io_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(agc_frame_dump_io_tests PRIVATE libc)
@@ -693,6 +731,28 @@ target_link_libraries(agc_shared_write_tests PRIVATE libSceAgcDriver libkernel l
 set_target_properties(agc_shared_write_tests PROPERTIES CXX_EXTENSIONS OFF)
 configure_windows_unwind(agc_shared_write_tests)
 
+if(BUILD_TESTING AND WIN32)
+    add_test_executable(agc_page_state_tests tests/PageStates.cpp)
+    target_include_directories(agc_page_state_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(agc_page_state_tests PRIVATE libSceAgcDriver libc)
+    configure_windows_unwind(agc_page_state_tests)
+    add_test(NAME agc_page_states COMMAND agc_page_state_tests)
+    add_test(NAME agc_page_state_metadata COMMAND agc_page_state_tests --metadata)
+    set_tests_properties(agc_page_states agc_page_state_metadata PROPERTIES
+        TIMEOUT 30 ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+endif()
+
+add_test_executable(agc_locks_tests EXCLUDE_FROM_ALL tests/Locks.cpp)
+target_include_directories(agc_locks_tests PRIVATE
+        ${LIBS_INCLUDE_DIR}
+        ${CMAKE_SOURCE_DIR}/core/shader/recompiler
+        ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}
+        ${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR}
+)
+target_link_libraries(agc_locks_tests PRIVATE libSceAgcDriver libc)
+set_target_properties(agc_locks_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_locks_tests)
+
 add_test_executable(agc_write_tracking_tests EXCLUDE_FROM_ALL
         tests/WriteTracking.cpp
         Execution/src/GuestMemory.cpp
@@ -716,6 +776,19 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+if(BUILD_TESTING AND NOT WIN32)
+    add_test_executable(agc_collect_memo_generation_tests
+            tests/CollectMemoGeneration.cpp
+            Execution/src/GuestMemory.cpp
+            Execution/src/PerformanceTraceThread.cpp
+    )
+    target_include_directories(agc_collect_memo_generation_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(agc_collect_memo_generation_tests PRIVATE shader_recompiler_agc_driver libc)
+    set_target_properties(agc_collect_memo_generation_tests PROPERTIES CXX_EXTENSIONS OFF)
+    add_test(NAME agc_collect_memo_generation COMMAND agc_collect_memo_generation_tests)
+    set_tests_properties(agc_collect_memo_generation PROPERTIES TIMEOUT 30)
+endif()
+
 add_test_executable(agc_thread_priority_tests EXCLUDE_FROM_ALL
         tests/ThreadPriority.cpp
         Execution/src/ThreadPriority.cpp
@@ -1130,6 +1203,17 @@ if(BUILD_TESTING)
 endif()
 
 add_test_executable(agc_buffer_pool_tests EXCLUDE_FROM_ALL tests/BufferPool.cpp)
+
+if(BUILD_TESTING)
+    add_test_executable(agc_snapshot_copy_failure_tests tests/SnapshotCopyFailure.cpp tests/AllocationFailure.cpp)
+    target_include_directories(agc_snapshot_copy_failure_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
+    if(MINGW)
+        target_compile_options(agc_snapshot_copy_failure_tests PRIVATE -fasynchronous-unwind-tables)
+    endif()
+    configure_windows_unwind(agc_snapshot_copy_failure_tests)
+    add_test(NAME agc_snapshot_copy_failure COMMAND agc_snapshot_copy_failure_tests)
+    set_tests_properties(agc_snapshot_copy_failure PROPERTIES TIMEOUT 10)
+endif()
 target_sources(agc_buffer_pool_tests PRIVATE Graphics/src/BufferPool.cpp tests/AllocationFailure.cpp)
 target_include_directories(agc_buffer_pool_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
 target_link_libraries(agc_buffer_pool_tests PRIVATE libSceAgcDriver libc)
@@ -1403,6 +1487,7 @@ target_include_directories(agc_driver_recorder_tests PRIVATE
 )
 
 target_link_libraries(agc_driver_recorder_tests PRIVATE
+        libSceAgc
         libkernel
         libSceAgcDriver
         SDL2-static
@@ -1422,8 +1507,20 @@ if(BUILD_TESTING)
         set_tests_properties(agc_recorder_device_selection agc_gpu_timing_complete agc_gpu_timing_incomplete PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_command_memory_reuse COMMAND agc_driver_recorder_tests --command-reuse-only)
+    set_tests_properties(agc_command_memory_reuse PROPERTIES SKIP_RETURN_CODE 77)
+    if(WIN32)
+        set_tests_properties(agc_command_memory_reuse PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_pipeline_cache COMMAND agc_driver_recorder_tests --pipeline-cache-only)
     add_test(NAME agc_pipeline_cache_file COMMAND agc_driver_recorder_tests --pipeline-cache-file-only)
+    add_test(NAME agc_pipeline_cache_header COMMAND agc_driver_recorder_tests --pipeline-cache-header-only)
+    set_tests_properties(agc_pipeline_cache_header PROPERTIES TIMEOUT 30 SKIP_RETURN_CODE 77)
+    if(WIN32)
+        set_tests_properties(agc_pipeline_cache_header PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_pipeline_library COMMAND agc_driver_recorder_tests --pipeline-library-only)
     add_test(NAME agc_pipeline_library_off COMMAND agc_driver_recorder_tests --pipeline-library-only)
     set_tests_properties(agc_pipeline_library agc_pipeline_library_off PROPERTIES SKIP_RETURN_CODE 77)
@@ -1446,6 +1543,13 @@ if(BUILD_TESTING)
         set_tests_properties(agc_import_watch_other_family PROPERTIES SKIP_RETURN_CODE 77)
         add_test(NAME agc_import_settle_deferred COMMAND agc_driver_recorder_tests --deferred-settle-only)
         add_test(NAME agc_import_settle_deferred_other_family COMMAND agc_driver_recorder_tests --deferred-settle-other-family-only)
+        add_test(NAME agc_import_settle_write_order COMMAND agc_driver_recorder_tests --settle-write-order-only)
+        add_test(NAME agc_import_settle_write_order_other_family COMMAND agc_driver_recorder_tests --settle-write-order-other-family-only)
+        add_test(NAME agc_import_settle_wait_failure COMMAND agc_driver_recorder_tests --settle-wait-failure-only)
+        set_tests_properties(agc_import_settle_write_order agc_import_settle_write_order_other_family agc_import_settle_wait_failure PROPERTIES
+            SKIP_RETURN_CODE 77 TIMEOUT 30
+            FAIL_REGULAR_EXPRESSION "SYNC-HAZARD-|VUID-"
+            ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
         add_test(NAME agc_import_settle_first_use COMMAND agc_driver_recorder_tests --settle-first-use-only)
         add_test(NAME agc_import_settle_earlier_import COMMAND agc_driver_recorder_tests --settle-earlier-import-only)
         add_test(NAME agc_import_settle_retire COMMAND agc_driver_recorder_tests --settle-retire-only)
@@ -1528,6 +1632,12 @@ if(BUILD_TESTING)
         set_tests_properties(agc_storage_baseline_settled PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_bda_staging COMMAND agc_driver_recorder_tests --bda-staging-only)
+    set_tests_properties(agc_bda_staging PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_bda_staging PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_read_written_staging COMMAND agc_driver_recorder_tests --read-written-staging-only)
     set_tests_properties(agc_read_written_staging PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
     if(WIN32)
@@ -1592,6 +1702,20 @@ if(BUILD_TESTING)
     set_tests_properties(agc_unit_shadow PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30 ENVIRONMENT "APS5_PIN_WAIT_MS=3000")
     add_test(NAME agc_idle_publish_wait COMMAND agc_driver_recorder_tests --idle-publish-wait-only)
     set_tests_properties(agc_idle_publish_wait PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30 ENVIRONMENT "APS5_PIN_WAIT_MS=3000;APS5_WRITE_WATCH_IMPORTS=watch")
+    add_test(NAME agc_driver_bda_reads_shadowed_results COMMAND agc_driver_recorder_tests --bda-shadowed-results-only)
+    add_test(NAME agc_driver_bda_reads_shadowed_results_per_build COMMAND agc_driver_recorder_tests --bda-shadowed-results-only)
+    set_tests_properties(agc_driver_bda_reads_shadowed_results agc_driver_bda_reads_shadowed_results_per_build PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    set_tests_properties(agc_driver_bda_reads_shadowed_results_per_build PROPERTIES ENVIRONMENT "APS5_NO_ADDRESS_SPACE_CACHE=1")
+    if(WIN32)
+        set_tests_properties(agc_driver_bda_reads_shadowed_results agc_driver_bda_reads_shadowed_results_per_build PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+    add_test(NAME agc_unit_shadow_completion_store COMMAND agc_driver_recorder_tests --completion-store-shadow-only)
+    set_tests_properties(agc_unit_shadow_completion_store PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_unit_shadow_completion_store PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     if(WIN32)
         add_test(NAME agc_storage_cpu_snapshot COMMAND agc_driver_recorder_tests --storage-cpu-snapshot-only)
         set_tests_properties(agc_storage_cpu_snapshot PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" TIMEOUT 30 SKIP_RETURN_CODE 77
@@ -1637,10 +1761,12 @@ if(BUILD_TESTING)
     set_tests_properties(agc_shadowed_key_stores PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
     add_test(NAME agc_sample_dumps COMMAND agc_driver_recorder_tests --sample-dumps-only)
     set_tests_properties(agc_sample_dumps PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    add_test(NAME agc_uncounted_passes COMMAND agc_driver_recorder_tests --uncounted-passes-only)
+    set_tests_properties(agc_uncounted_passes PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
     add_test(NAME agc_draw_descriptors COMMAND agc_driver_recorder_tests --draw-descriptors-only)
     set_tests_properties(agc_draw_descriptors PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 60)
     if(WIN32)
-        set_tests_properties(agc_pending_key_stores agc_shadowed_key_stores agc_sample_dumps agc_draw_descriptors PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_pending_key_stores agc_shadowed_key_stores agc_sample_dumps agc_uncounted_passes agc_draw_descriptors PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_driver_host_import_lifetime COMMAND agc_driver_recorder_tests --host-import-lifetime-only)
@@ -1653,9 +1779,17 @@ if(BUILD_TESTING)
         ENVIRONMENT "APS5_PIN_WAIT_MS=2000" SKIP_RETURN_CODE 77 TIMEOUT 30)
     set_tests_properties(agc_driver_unit_shadow_budget PROPERTIES
         ENVIRONMENT "APS5_PIN_WAIT_MS=2000;APS5_UNIT_SHADOW_MIB=8" SKIP_RETURN_CODE 77 TIMEOUT 30)
+    add_test(NAME agc_recorder_completion_labels COMMAND agc_driver_recorder_tests --completion-labels-only)
+    set_tests_properties(agc_recorder_completion_labels PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    add_test(NAME agc_recorder_cmask COMMAND agc_driver_recorder_tests --cmask-only)
+    set_tests_properties(agc_recorder_cmask PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    add_test(NAME agc_recorder_cube COMMAND agc_driver_recorder_tests --cube-only)
+    set_tests_properties(agc_recorder_cube PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
     if(WIN32)
         set_tests_properties(agc_driver_host_import_lifetime agc_driver_host_import_mutation_race agc_driver_unit_shadow agc_driver_unit_shadow_budget PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_recorder_completion_labels agc_recorder_cmask agc_recorder_cube PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     set_property(TEST agc_driver_host_import_lifetime APPEND PROPERTY ENVIRONMENT_MODIFICATION
         "APS5_RELEASE_UNDER_LOCK=unset:" "APS5_RELEASE_ON_UNLOCK=unset:")
@@ -1832,6 +1966,7 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL
         tests/GuestSamplerResource.cpp
         tests/TextureDetiler.cpp
         tests/DepthSurfaceReuse.cpp
+        tests/LastRecordIndices.cpp
 )
 
 if(BUILD_TESTING)
@@ -1982,6 +2117,7 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources)
     add_test_executable(${agcExecutionTest} EXCLUDE_FROM_ALL ${agcExecutionTestSource})
     target_include_directories(${agcExecutionTest} PRIVATE
             ${LIBS_INCLUDE_DIR}
+            $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>
             ${CMAKE_SOURCE_DIR}/core/shader/recompiler
             ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}
             ${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR}
@@ -1998,6 +2134,13 @@ if(BUILD_TESTING)
         set_tests_properties(agc_driver_address_passes_targets_split PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    set_target_properties(agc_driver_rcp_hardware_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_driver_rcp_hardware_exact COMMAND agc_driver_rcp_hardware_tests)
+    set_tests_properties(agc_driver_rcp_hardware_exact PROPERTIES ENVIRONMENT "APS5_EXACT_RCP=1;ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 60)
+    if(WIN32)
+        set_tests_properties(agc_driver_rcp_hardware_exact PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
 endif()
 foreach(agcExecutionTestName IN LISTS ANYPS5_AGC_EXECUTION_TESTS)
     if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/execution/${agcExecutionTestName}.cpp)
@@ -2043,13 +2186,23 @@ if(BUILD_TESTING)
     add_test(NAME agc_driver_store_pending_identity COMMAND agc_driver_store_over_untracked_tests --pending-identity)
     set_tests_properties(agc_driver_store_pending_identity PROPERTIES
             ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_driver_store_idle_results_wake COMMAND agc_driver_store_idle_results_tests --idle-epoch-wake)
+    set_tests_properties(agc_driver_store_idle_results_wake PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     if(WIN32)
         set_tests_properties(agc_driver_store_pending_identity PROPERTIES ENVIRONMENT_MODIFICATION
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_driver_store_idle_results_wake PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 endif()
 
 if(BUILD_TESTING)
+    add_test(NAME agc_driver_native_fmask COMMAND agc_driver_sample_multisample_target_tests --native-fmask)
+    set_tests_properties(agc_driver_native_fmask PROPERTIES TIMEOUT 30 ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_native_fmask PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_native_msaa_vertex_input COMMAND agc_driver_sample_multisample_target_tests --native-vertex-input)
     set_tests_properties(agc_driver_native_msaa_vertex_input PROPERTIES
             ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" TIMEOUT 30 SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2127,6 +2280,12 @@ target_link_libraries(agc_driver_job_manager_compute_tests PRIVATE libSceJobMana
 
 target_include_directories(agc_driver_packed_unorm_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_packed_unorm4444_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
+target_include_directories(agc_driver_packed_unorm4444_partial_write_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
+target_include_directories(agc_driver_packed_unorm4444_multisample_partial_write_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
 target_include_directories(agc_driver_color1555_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
 target_include_directories(agc_driver_color565_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
@@ -2198,6 +2357,14 @@ target_include_directories(agc_driver_prepared_shaders_tests PRIVATE
 
 if(BUILD_TESTING)
     set_target_properties(agc_shader_mode_register_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    foreach(packedStorageFormat IN ITEMS 133 134 136)
+        set(packedStorageTest agc_driver_image_packed16_storage_${packedStorageFormat})
+        add_test(NAME ${packedStorageTest} COMMAND agc_driver_image_converted_unorm_tests --packed16 ${packedStorageFormat})
+        set_tests_properties(${packedStorageTest} PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
+        if(WIN32)
+            set_tests_properties(${packedStorageTest} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        endif()
+    endforeach()
     foreach(prebuildPolicyMode IN ITEMS unset 0 1)
         set(prebuildPolicyTest agc_driver_prepared_graphics_prebuild_${prebuildPolicyMode})
         add_test(NAME ${prebuildPolicyTest} COMMAND agc_driver_prepared_graphics_tests --prebuild-${prebuildPolicyMode})
@@ -2270,8 +2437,8 @@ if(BUILD_TESTING)
     if(WIN32)
         set_tests_properties(agc_shader_bindless agc_shader_selected_records_image_limit agc_shader_selected_records_pair_limit agc_shader_selected_records_direct_image_limit agc_shader_selected_records_direct_pair_limit agc_shader_srt_read_patching agc_shader_type_zero_image_materialization agc_shader_frontend_pair agc_shader_frontend_pair_off agc_shader_layout_pair agc_shader_layout_pair_off agc_shader_entry_lane_proof_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
-    foreach(agcTest IN ITEMS agc_vertex_input_tests agc_mesh_output_budget_tests agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vertex_input_validation_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_barrier_batch_tests agc_shader_disk_cache_tests agc_screenshot_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests agc_formatted_buffer_byte_stores_tests ${agcExecutionTests} agc_fma_mix_once_tests agc_f16_single_convert_tests)
-        if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests" OR agcTest STREQUAL "agc_driver_pipeline_optimized_readback_tests")
+    foreach(agcTest IN ITEMS agc_vertex_input_tests agc_mesh_output_budget_tests agc_shared_write_tests agc_locks_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vertex_input_validation_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_barrier_batch_tests agc_shader_disk_cache_tests agc_screenshot_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests agc_formatted_buffer_byte_stores_tests ${agcExecutionTests} agc_fma_mix_once_tests agc_f16_single_convert_tests)
+        if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests" OR agcTest STREQUAL "agc_driver_pipeline_optimized_readback_tests" OR agcTest STREQUAL "agc_driver_rcp_hardware_tests")
             continue()
         endif()
         string(REGEX REPLACE "_tests$" "" agcTestName "${agcTest}")
@@ -2304,6 +2471,12 @@ if(BUILD_TESTING)
         set_tests_properties(agc_driver_draw_unsupported_buffer_type_skip_refused PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_driver_period_copy_no_hle COMMAND agc_driver_period_copy_tests)
+    set_tests_properties(agc_driver_period_copy_no_hle PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_COPY_HLE=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_period_copy_no_hle PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_sparse_indirect_indices_unreadable COMMAND agc_driver_sparse_indirect_indices_tests --unreadable)
     set_tests_properties(agc_driver_sparse_indirect_indices agc_driver_sparse_indirect_indices_unreadable PROPERTIES
         ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2313,6 +2486,7 @@ if(BUILD_TESTING)
     endif()
     set_tests_properties(agc_driver_pixel_implicit_derivatives PROPERTIES TIMEOUT 30)
     set_tests_properties(agc_driver_storage_file_write PROPERTIES TIMEOUT 30 ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_SUSPEND_DRAIN=1")
+    set_tests_properties(agc_driver_clamped_global_fill PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_LABEL_READBACK_STORE=1")
     set_tests_properties(agc_driver_draw_pipeline_trace PROPERTIES
             ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=8;APS5_TRACE_FRAMES=0:1;APS5_TRACE_EVENTS=64" TIMEOUT 30)
     set_property(TEST agc_driver_draw_pipeline_trace APPEND PROPERTY ENVIRONMENT_MODIFICATION
@@ -2321,11 +2495,38 @@ if(BUILD_TESTING)
             --unset=APS5_TRACE_FRAMES --unset=APS5_TRACE_SECONDS --unset=APS5_TRACE_SUMMARY_EVERY
             $<TARGET_FILE:agc_driver_draw_pipeline_trace_tests> disabled)
     set_tests_properties(agc_driver_draw_pipeline_trace_disabled PROPERTIES ENVIRONMENT "APS5_PIPELINED_DRAWS=8" SKIP_RETURN_CODE 77 TIMEOUT 30)
+    add_test(NAME agc_driver_draw_finisher_trace COMMAND agc_driver_draw_pipeline_trace_tests finisher)
+    set_tests_properties(agc_driver_draw_finisher_trace PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_DRAW_PIPELINE=1;APS5_PIPELINED_DRAWS=8;APS5_TRACE_FRAMES=0:1;APS5_TRACE_EVENTS=64" SKIP_RETURN_CODE 77 TIMEOUT 30)
+    set_property(TEST agc_driver_draw_finisher_trace APPEND PROPERTY ENVIRONMENT_MODIFICATION
+            "APS5_TRACE_SECONDS=unset:" "APS5_TRACE_SUMMARY_EVERY=unset:")
+    add_test(NAME agc_driver_draw_finisher_trace_disabled COMMAND ${CMAKE_COMMAND} -E env
+            --unset=APS5_TRACE_FRAMES --unset=APS5_TRACE_SECONDS --unset=APS5_TRACE_SUMMARY_EVERY
+            $<TARGET_FILE:agc_driver_draw_pipeline_trace_tests> finisher-disabled)
+    set_tests_properties(agc_driver_draw_finisher_trace_disabled PROPERTIES ENVIRONMENT "APS5_DRAW_PIPELINE=1;APS5_PIPELINED_DRAWS=8" SKIP_RETURN_CODE 77 TIMEOUT 30)
     if(WIN32)
-        set_tests_properties(agc_driver_draw_pipeline_trace_disabled PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_driver_draw_pipeline_trace_disabled agc_driver_draw_finisher_trace agc_driver_draw_finisher_trace_disabled PROPERTIES ENVIRONMENT_MODIFICATION
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_driver_copied_writer_label_order_drain COMMAND agc_driver_copied_writer_label_order_tests)
+    set_tests_properties(agc_driver_copied_writer_label_order_drain PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_DRAIN_COMPLETION_LABELS=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_driver_copied_writer_label_order_drain PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+    add_test(NAME agc_driver_lazy_resolved_abi_switch_set COMMAND agc_driver_lazy_resolved_abi_tests)
+    set_tests_properties(agc_driver_lazy_resolved_abi_switch_set PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_LAZY_SHADER_PREPARE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_lazy_resolved_abi_switch_set PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     set_tests_properties(agc_driver_queued_raw_shader PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=8" TIMEOUT 30)
+    set(agcDrawPipelineTests agc_driver_draw_write_ranges agc_driver_draw_unsupported_buffer_type agc_driver_bda_span_reads agc_driver_copied_writer_copy agc_driver_image_table_reuse agc_driver_depth_storage_reuse agc_driver_staged_clear_sampled_read agc_driver_storage_write_back_overhang agc_driver_write_data_written_ahead agc_driver_gated_dispatch_capture agc_driver_buffer_load_store agc_driver_pipelined_dma agc_driver_pipelined_fill agc_driver_pipelined_predication_read agc_driver_pipelined_predicated_label_order agc_driver_pipelined_metadata_pass agc_driver_draw_cache_reuse agc_driver_draw_cache_relocation agc_driver_draw_data_hit_user_words agc_driver_finisher_label_order agc_driver_same_queue_label_wait agc_driver_unimported_label_order)
+    foreach(agcTestName IN LISTS agcDrawPipelineTests)
+        add_test(NAME ${agcTestName}_draw_pipeline COMMAND ${agcTestName}_tests)
+        set_tests_properties(${agcTestName}_draw_pipeline PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_DRAW_PIPELINE=1;APS5_PIPELINED_DRAWS=32;APS5_PIPELINE_DISPATCH=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+        if(WIN32)
+            set_tests_properties(${agcTestName}_draw_pipeline PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        endif()
+    endforeach()
     add_test(NAME agc_driver_partially_resident_pending_stores COMMAND agc_driver_partially_resident_snapshot_tests --pending-stores)
     set_tests_properties(agc_driver_multisample_readback PROPERTIES TIMEOUT 20)
     foreach(agcReadbackCase IN ITEMS uint8 uint16 uint64 uint128 depth upload retile depth_retile)
@@ -2459,12 +2660,13 @@ if(BUILD_TESTING)
     add_test(NAME agc_heap_mirror_unwatch_no_space_cache COMMAND agc_driver_graphics_tests --heap-mirror-unwatch)
     add_test(NAME agc_sparse_heap_mirror_unwatch COMMAND agc_driver_graphics_tests --sparse-heap-mirror-unwatch)
     add_test(NAME agc_sparse_heap_mirror_unwatch_no_space_cache COMMAND agc_driver_graphics_tests --sparse-heap-mirror-unwatch)
+    add_test(NAME agc_sparse_heap_mirror_budget COMMAND agc_driver_graphics_tests --sparse-heap-mirror-budget)
     add_test(NAME agc_heap_mirror_run_unwatch COMMAND agc_driver_graphics_tests --heap-mirror-run-unwatch)
     add_test(NAME agc_heap_mirror_run_unwatch_no_space_cache COMMAND agc_driver_graphics_tests --heap-mirror-run-unwatch)
     set_tests_properties(agc_heap_mirror_unwatch_no_space_cache agc_sparse_heap_mirror_unwatch_no_space_cache agc_heap_mirror_run_unwatch_no_space_cache PROPERTIES ENVIRONMENT "APS5_NO_ADDRESS_SPACE_CACHE=1")
-    set_tests_properties(agc_heap_mirror_unwatch agc_heap_mirror_unwatch_no_space_cache agc_sparse_heap_mirror_unwatch agc_sparse_heap_mirror_unwatch_no_space_cache agc_heap_mirror_run_unwatch agc_heap_mirror_run_unwatch_no_space_cache PROPERTIES TIMEOUT 30)
+    set_tests_properties(agc_heap_mirror_unwatch agc_heap_mirror_unwatch_no_space_cache agc_sparse_heap_mirror_unwatch agc_sparse_heap_mirror_unwatch_no_space_cache agc_heap_mirror_run_unwatch agc_heap_mirror_run_unwatch_no_space_cache agc_sparse_heap_mirror_budget PROPERTIES TIMEOUT 30)
     if(WIN32)
-        set_tests_properties(agc_heap_mirror_unwatch agc_heap_mirror_unwatch_no_space_cache agc_sparse_heap_mirror_unwatch agc_sparse_heap_mirror_unwatch_no_space_cache agc_heap_mirror_run_unwatch agc_heap_mirror_run_unwatch_no_space_cache PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_heap_mirror_unwatch agc_heap_mirror_unwatch_no_space_cache agc_sparse_heap_mirror_unwatch agc_sparse_heap_mirror_unwatch_no_space_cache agc_heap_mirror_run_unwatch agc_heap_mirror_run_unwatch_no_space_cache agc_sparse_heap_mirror_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     else()
         add_test(NAME agc_heap_mirror_unwatched_import COMMAND agc_driver_graphics_tests --heap-mirror-unwatched-import)
         set_tests_properties(agc_heap_mirror_unwatched_import PROPERTIES TIMEOUT 30)
@@ -2651,6 +2853,15 @@ if(BUILD_TESTING)
         set_tests_properties(agc_sampled_cache_device_selection agc_driver_sampled_cache_device_selection PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    foreach(imageRecoveryMode IN ITEMS sampled storage srgb repack)
+        add_test(NAME agc_driver_recover_${imageRecoveryMode}_allocation COMMAND agc_driver_sampled_cache_budget_tests recover-${imageRecoveryMode})
+        set_tests_properties(agc_driver_recover_${imageRecoveryMode}_allocation PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 30)
+        if(WIN32)
+            set_tests_properties(agc_driver_recover_${imageRecoveryMode}_allocation PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        endif()
+    endforeach()
+    set_property(TEST agc_driver_recover_srgb_allocation APPEND PROPERTY ENVIRONMENT_MODIFICATION "APS5_SRGB_SHADER_DECODE=set:copy")
     add_test(NAME agc_driver_resource_cache_storage_admission COMMAND agc_driver_sampled_cache_budget_tests resource-admission)
     add_test(NAME agc_driver_resource_cache_oversized_storage_admission COMMAND agc_driver_sampled_cache_budget_tests resource-oversized)
     add_test(NAME agc_driver_resource_cache_storage_clear COMMAND agc_driver_sampled_cache_budget_tests resource-clear)
@@ -2720,18 +2931,32 @@ if(BUILD_TESTING)
         set_tests_properties(agc_driver_flip_compute agc_driver_present_mode agc_driver_worker_failure_log PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_driver_linear_readback_label_off COMMAND agc_driver_linear_readback_label_tests)
+    set_tests_properties(agc_driver_linear_readback_label_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_LABEL_READBACK_STORE=1" WILL_FAIL TRUE TIMEOUT 60)
+    if(WIN32)
+        set_tests_properties(agc_driver_linear_readback_label_off PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_draw_cache_reuse_off COMMAND agc_driver_draw_cache_reuse_tests)
     set_tests_properties(agc_driver_draw_cache_reuse_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_GPL=1" SKIP_RETURN_CODE 77)
     add_test(NAME agc_driver_draw_cache_reuse_verify COMMAND agc_driver_draw_cache_reuse_tests)
     set_tests_properties(agc_driver_draw_cache_reuse_verify PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_VERIFY_DRAW_DATA_HITS=1" SKIP_RETURN_CODE 77)
+    add_test(NAME agc_driver_draw_cache_reuse_entry_verify COMMAND agc_driver_draw_cache_reuse_tests)
+    set_tests_properties(agc_driver_draw_cache_reuse_entry_verify PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_VERIFY_DRAW_ENTRIES=1" SKIP_RETURN_CODE 77 FAIL_REGULAR_EXPRESSION "FATAL:")
+    if(WIN32)
+        set_tests_properties(agc_driver_draw_cache_reuse_entry_verify PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_draw_cache_reuse_inline COMMAND agc_driver_draw_cache_reuse_tests)
     set_tests_properties(agc_driver_draw_cache_reuse_inline PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=0;APS5_VERIFY_DRAW_RECIPE=1;APS5_VERIFY_REGISTER_KEYS=1" SKIP_RETURN_CODE 77)
     set_tests_properties(agc_driver_draw_cache_reuse agc_driver_draw_cache_reuse_off agc_driver_draw_cache_reuse_verify agc_driver_draw_cache_reuse_inline PROPERTIES FAIL_REGULAR_EXPRESSION "FATAL:")
     set_property(TEST agc_driver_draw_cache_reuse agc_driver_draw_cache_reuse_off APPEND PROPERTY ENVIRONMENT "APS5_VERIFY_DRAW_RECIPE=1" "APS5_VERIFY_REGISTER_KEYS=1")
     add_test(NAME agc_driver_draw_data_hit_user_words_verify COMMAND agc_driver_draw_data_hit_user_words_tests)
     set_tests_properties(agc_driver_draw_data_hit_user_words_verify PROPERTIES ENVIRONMENT "APS5_VERIFY_DRAW_DATA_HITS=1" SKIP_RETURN_CODE 77)
+    add_test(NAME agc_driver_draw_data_hit_user_words_refresh COMMAND agc_driver_draw_data_hit_user_words_tests)
+    set_tests_properties(agc_driver_draw_data_hit_user_words_refresh PROPERTIES ENVIRONMENT "APS5_DRAW_PIPELINE=1" SKIP_RETURN_CODE 77)
     if(WIN32)
-        set_tests_properties(agc_driver_draw_cache_reuse_off agc_driver_draw_cache_reuse_verify agc_driver_draw_cache_reuse_inline agc_driver_draw_data_hit_user_words_verify PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_driver_draw_cache_reuse_off agc_driver_draw_cache_reuse_verify agc_driver_draw_cache_reuse_inline agc_driver_draw_data_hit_user_words_verify agc_driver_draw_data_hit_user_words_refresh PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 
@@ -2827,15 +3052,26 @@ if(BUILD_TESTING)
         endif()
     endforeach()
     add_test(NAME agc_driver_shutdown_threads_joined COMMAND agc_driver_shutdown_threads_tests --joined)
+    add_test(NAME agc_driver_shutdown_threads_queued_prebuilds COMMAND agc_driver_shutdown_threads_tests --queued-prebuilds)
     add_test(NAME agc_driver_shutdown_threads_compare_helpers COMMAND agc_driver_shutdown_threads_tests --compare-helpers)
     add_test(NAME agc_driver_shutdown_threads_draw_committer COMMAND agc_driver_shutdown_threads_tests --draw-committer)
     add_test(NAME agc_driver_shutdown_threads_shader_cache_writer COMMAND agc_driver_shutdown_threads_tests --shader-cache-writer)
     add_test(NAME agc_driver_shutdown_threads_late_shader_cache COMMAND agc_driver_shutdown_threads_tests --late-shader-cache)
     add_test(NAME agc_driver_shutdown_threads_async_log_held COMMAND agc_driver_shutdown_threads_tests --async-log-held)
     add_test(NAME agc_driver_shutdown_threads_exit_after_start COMMAND agc_driver_shutdown_threads_tests --exit-after-start)
-    set_tests_properties(agc_driver_shutdown_threads_joined agc_driver_shutdown_threads_compare_helpers agc_driver_shutdown_threads_draw_committer agc_driver_shutdown_threads_shader_cache_writer agc_driver_shutdown_threads_late_shader_cache agc_driver_shutdown_threads_async_log_held agc_driver_shutdown_threads_exit_after_start PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
+    set_tests_properties(agc_driver_shutdown_threads_joined agc_driver_shutdown_threads_queued_prebuilds agc_driver_shutdown_threads_compare_helpers agc_driver_shutdown_threads_draw_committer agc_driver_shutdown_threads_shader_cache_writer agc_driver_shutdown_threads_late_shader_cache agc_driver_shutdown_threads_async_log_held agc_driver_shutdown_threads_exit_after_start PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
     if(WIN32)
-        set_tests_properties(agc_driver_shutdown_threads_joined agc_driver_shutdown_threads_compare_helpers agc_driver_shutdown_threads_draw_committer agc_driver_shutdown_threads_shader_cache_writer agc_driver_shutdown_threads_late_shader_cache agc_driver_shutdown_threads_async_log_held agc_driver_shutdown_threads_exit_after_start PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_driver_shutdown_threads_joined agc_driver_shutdown_threads_queued_prebuilds agc_driver_shutdown_threads_compare_helpers agc_driver_shutdown_threads_draw_committer agc_driver_shutdown_threads_shader_cache_writer agc_driver_shutdown_threads_late_shader_cache agc_driver_shutdown_threads_async_log_held agc_driver_shutdown_threads_exit_after_start PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+    add_test(NAME agc_driver_prepared_shaders_abi_deferred_concurrent_lazy COMMAND agc_driver_prepared_shaders_tests --abi-deferred-concurrent)
+    set_tests_properties(agc_driver_prepared_shaders_abi_deferred_concurrent_lazy PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_FAILURE_MEMO=1;APS5_LAZY_SHADER_PREPARE=1;APS5_SUSPEND_DRAIN=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_prepared_shaders_abi_deferred_concurrent_lazy PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+    add_test(NAME agc_driver_prepared_shaders_abi_deferred_compute_lazy COMMAND agc_driver_prepared_shaders_tests --abi-deferred-compute)
+    set_tests_properties(agc_driver_prepared_shaders_abi_deferred_compute_lazy PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_FAILURE_MEMO=1;APS5_LAZY_SHADER_PREPARE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_prepared_shaders_abi_deferred_compute_lazy PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_driver_compute_user_data_calls_registered COMMAND agc_driver_compute_user_data_calls_tests --registered)
     set_tests_properties(agc_driver_compute_user_data_calls_registered PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2953,6 +3189,16 @@ set_target_properties(agc_shader_replay PROPERTIES CXX_EXTENSIONS OFF)
 configure_windows_unwind(agc_shader_replay)
 
 if(BUILD_TESTING)
+    foreach(optimizer IN ITEMS none full)
+        if(optimizer STREQUAL "full" AND NOT ANYPS5_ENABLE_SPIRV_TOOLS)
+            continue()
+        endif()
+        add_test(NAME agc_shader_specialized_bda_abi_${optimizer} COMMAND agc_shader_memory_tests --specialized-bda-abi)
+        set_tests_properties(agc_shader_specialized_bda_abi_${optimizer} PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_SPIRV_OPT=${optimizer}" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
+        if(WIN32)
+            set_tests_properties(agc_shader_specialized_bda_abi_${optimizer} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        endif()
+    endforeach()
     add_test_executable(agc_shader_replay_requests_tests tests/ShaderReplayRequests.cpp)
     target_link_libraries(agc_shader_replay_requests_tests PRIVATE shader_recompiler_agc_driver libc)
     set_target_properties(agc_shader_replay_requests_tests PROPERTIES CXX_EXTENSIONS OFF)
@@ -3042,12 +3288,15 @@ if(BUILD_TESTING)
     set_tests_properties(agc_driver_covered_centroid_state PROPERTIES TIMEOUT 30)
     add_test(NAME agc_driver_sample_location_grid_fallback COMMAND agc_driver_graphics_tests --sample-location-grid-fallback)
     set_tests_properties(agc_driver_sample_location_grid_fallback PROPERTIES TIMEOUT 30)
+    add_test(NAME agc_driver_unobserved_provoking_vertex COMMAND agc_driver_graphics_tests --unobserved-provoking-vertex)
+    set_tests_properties(agc_driver_unobserved_provoking_vertex PROPERTIES TIMEOUT 30)
     add_test(NAME agc_driver_sample_order_planning COMMAND agc_driver_graphics_tests --sample-order)
     set_tests_properties(agc_driver_sample_order_planning PROPERTIES TIMEOUT 30)
     if(WIN32)
         set_tests_properties(agc_compute_descriptor_layout_flags PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
         set_tests_properties(agc_driver_covered_centroid_state PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
         set_tests_properties(agc_driver_sample_location_grid_fallback PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_driver_unobserved_provoking_vertex PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
         set_tests_properties(agc_driver_sample_order_planning PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_compute_pipeline_capture COMMAND agc_compute_pipeline_capture_tests)
@@ -3365,6 +3614,19 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_gcn_decode_tests EXCLUDE_FROM_ALL tests/GcnDecode.cpp)
+target_link_libraries(agc_gcn_decode_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_gcn_decode_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_gcn_decode_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_gcn_decode_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_gcn_decode COMMAND agc_gcn_decode_tests)
+    if(WIN32)
+        set_tests_properties(agc_gcn_decode PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_pixel_position_export_tests EXCLUDE_FROM_ALL tests/PixelPositionExport.cpp)
 target_link_libraries(agc_pixel_position_export_tests PRIVATE shader_recompiler_agc_driver libc)
 set_target_properties(agc_pixel_position_export_tests PROPERTIES CXX_EXTENSIONS OFF)
@@ -3490,6 +3752,16 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+if(BUILD_TESTING)
+    add_test_executable(agc_flat_scratch_tests tests/FlatScratch.cpp)
+    target_link_libraries(agc_flat_scratch_tests PRIVATE shader_recompiler_agc_driver libc)
+    configure_windows_unwind(agc_flat_scratch_tests)
+    add_test(NAME agc_flat_scratch COMMAND agc_flat_scratch_tests)
+    set_tests_properties(agc_flat_scratch PROPERTIES TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_flat_scratch PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
 add_test_executable(agc_constant_select_tests EXCLUDE_FROM_ALL tests/ConstantSelect.cpp)
 target_link_libraries(agc_constant_select_tests PRIVATE shader_recompiler_agc_driver libc)
 set_target_properties(agc_constant_select_tests PROPERTIES CXX_EXTENSIONS OFF)

core/libs/prx/libSceAgcDriver/Eq/src/Event.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Eq/include/Event.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 
 #include <algorithm>
 #include <cstdint>
@@ -20,7 +21,7 @@ struct Registration {
     int id;
 };
 
-std::mutex g_mutex;
+AgcDriver::Mutex g_mutex;
 std::vector<Registration> g_registrations;
 
 }

core/libs/prx/libSceAgcDriver/Execution/include/DeviceFault.hpp

@@ -6,6 +6,8 @@
 #endif
 #include <vulkan/vulkan.h>
 #include <algorithm>
+#include <array>
+#include <atomic>
 #include <cstdint>
 #include <cstdio>
 #include <cstring>
@@ -74,6 +76,62 @@ inline std::vector<std::string> DeviceFaultReport(VkDevice device, PFN_vkGetDevi
     return lines;
 }
 
+class BufferMarkerLog {
+public:
+    static constexpr std::uint32_t Capacity = 4096;
+    static constexpr std::uint32_t Listed = 8;
+
+    std::uint32_t Next(std::uint64_t program) {
+        std::uint32_t id = 0;
+        while (id == 0) id = next.fetch_add(1, std::memory_order_relaxed) + 1;
+        auto& slot = slots[id % Capacity];
+        slot.program.store(program, std::memory_order_relaxed);
+        slot.id.store(id, std::memory_order_release);
+        return id;
+    }
+
+    std::vector<std::string> Report(std::uint32_t reached, std::uint32_t finished) const {
+        std::vector<std::string> lines;
+        if (reached == 0 && finished == 0) {
+            lines.emplace_back("[device-fault] no buffer marker reached");
+            return lines;
+        }
+        lines.push_back(Line("last dispatch reached", reached));
+        if (finished == 0) lines.emplace_back("[device-fault] no dispatch finished");
+        else lines.push_back(Line("last dispatch finished", finished));
+        const std::uint32_t pending = reached - finished;
+        if (pending == 0 || pending > Capacity) return lines;
+        const auto listed = std::min(pending, Listed);
+        for (std::uint32_t i = 1; i <= listed; ++i) lines.push_back(Line("unfinished dispatch", finished + i));
+        if (pending > listed) {
+            char line[96];
+            std::snprintf(line, sizeof(line), "[device-fault] %u more unfinished dispatches", pending - listed);
+            lines.emplace_back(line);
+        }
+        return lines;
+    }
+
+private:
+    struct Slot {
+        std::atomic<std::uint32_t> id{0};
+        std::atomic<std::uint64_t> program{0};
+    };
+
+    std::string Line(const char* what, std::uint32_t id) const {
+        char line[128];
+        const auto& slot = slots[id % Capacity];
+        if (id != 0 && slot.id.load(std::memory_order_acquire) == id) {
+            std::snprintf(line, sizeof(line), "[device-fault] %s: marker %u, program 0x%llx", what, id, static_cast<unsigned long long>(slot.program.load(std::memory_order_relaxed)));
+        } else {
+            std::snprintf(line, sizeof(line), "[device-fault] %s: marker %u, program not in the log", what, id);
+        }
+        return line;
+    }
+
+    std::atomic<std::uint32_t> next{0};
+    std::array<Slot, Capacity> slots{};
+};
+
 }
 
 #endif

core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp

@@ -3,6 +3,7 @@
 
 #include <cstdint>
 #include <span>
+#include <cstddef>
 #include "SceTypes.hpp"
 #include "SceShaders.hpp"
 
@@ -12,6 +13,7 @@ void Submit(const Packet* packet, std::uint32_t queue);
 
 }
 
+extern "C" void AgcDriverPrepareCommandWrite_nid_no_patch(std::uint64_t address, std::size_t bytes);
 extern "C" void AgcDriverWaitIdle_nid_postfix();
 extern "C" void AgcDriverRaiseWorkerThreadPriority_nid_postfix(const char* role);
 extern "C" void AgcDriverLockVulkanLoader_nid_postfix();

core/libs/prx/libSceAgcDriver/Execution/include/Driver/DeviceAccess.hpp

@@ -3,10 +3,9 @@
 
 #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
 #include <atomic>
-#include <condition_variable>
 #include <cstddef>
+#include <cstdint>
 #include <memory>
-#include <mutex>
 #include "prx/libc/include/general/AtomicSharedPtr.hpp"
 
 namespace AgcDriver::DriverDetail {
@@ -33,10 +32,8 @@ public:
     void unlock();
 
 private:
-    std::mutex mutex;
-    std::condition_variable changed;
-    std::size_t users = 0;
-    bool replacing = false;
+    static constexpr std::uint32_t Replacing = 0x80000000u;
+    std::atomic<std::uint32_t> state{0};
 };
 
 }

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp

@@ -4,6 +4,7 @@
 #include "UserDataUse.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp"
 #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp"
 #include <array>
 #include <atomic>
 #include <chrono>
@@ -11,6 +12,7 @@
 #include <cstdint>
 #include <list>
 #include <memory>
+#include <span>
 #include <utility>
 #include <vector>
 #include "prx/libc/include/general/AtomicSharedPtr.hpp"
@@ -45,6 +47,7 @@ struct DrawDecode {
 void ReadUserWords(const QueueState& queue, DrawProgram& program, bool staticAbi);
 void InitializeMerged(const QueueState& queue, DrawProgram& program, std::uint32_t pointerBase, bool pointerRequired, bool staticAbi);
 void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const ShaderRegistry& registry, bool staticAbi, bool includeFragment);
+ShaderRecompiler::ShaderVertexStageInfo DecodeOrderedVertexStageInfo(std::span<const std::byte> header, std::uint64_t headerAddress, std::span<const std::uint32_t> userData, std::vector<Graphics::DecodeRead>& reads, std::uint32_t hiddenUserWords);
 
 void AppendProgramMemory(std::vector<ShaderRecompiler::MemoryRegion>& memory, const std::vector<DrawProgram>& programs);
 std::vector<std::pair<std::size_t, std::size_t>> UserDataWords(const std::vector<DrawProgram>& programs, const std::vector<ShaderRecompiler::ProgramRole>& roles, std::uint32_t location);
@@ -88,7 +91,7 @@ struct StageDataHit {
 enum class DrawMiss : std::size_t { FrontDiffering, FragmentDiffering, OtherDiffering, Layout, Gate, Stages, Count };
 
 struct DrawEntryCounters {
-    std::uint64_t lookups = 0, absent = 0, hits = 0, stageValidations = 0, stageEqual = 0, variantsCompared = 0, inserts = 0, variantsInserted = 0, variantsEvicted = 0, present = 0, unstable = 0, touches = 0, verifyHits = 0, verifyMismatches = 0;
+    std::uint64_t lookups = 0, absent = 0, hits = 0, stageValidations = 0, stageEqual = 0, variantsCompared = 0, inserts = 0, shapeUpdates = 0, variantsInserted = 0, variantsEvicted = 0, present = 0, unstable = 0, touches = 0, verifyHits = 0, verifyMismatches = 0;
     std::array<std::uint64_t, static_cast<std::size_t>(DrawMiss::Count)> misses{};
     std::array<std::uint64_t, MaxDispatchVariants> variantHitsByRank{};
     double validateUs = 0;

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp

@@ -13,6 +13,8 @@
 #include <mutex>
 #include <optional>
 #include <thread>
+#include <span>
+#include <string>
 #include <utility>
 #include <vector>
 
@@ -22,6 +24,11 @@ class DrawPipeline {
 public:
     using Commit = std::function<void()>;
     using Range = std::pair<std::uint64_t, std::uint64_t>;
+    struct Watch {
+        std::uint64_t retired = 0;
+        std::size_t items = 0;
+        std::vector<Range> writes;
+    };
     enum class DrainReason : std::uint8_t { Packet, Flush, Labels, Capture, Indirect, Submission, Count };
 
     static DrawPipeline& Queue0();
@@ -32,17 +39,27 @@ public:
     static void FollowEpoch(std::uint64_t token);
     static unsigned Committers();
 
-    void Enqueue(Commit commit, std::vector<Range> writes, std::uint64_t labelAddress = 0, std::vector<std::byte> labelBytes = {}, std::vector<Range> reads = {});
+    void Enqueue(Commit commit, std::vector<Range> writes, std::uint64_t labelAddress = 0, std::vector<std::byte> labelBytes = {}, std::vector<Range> reads = {}, bool draw = false);
     void Drain(DrainReason reason, std::uint32_t opcode = 0x100);
     void Stop();
+    void ReleaseRetired();
     static std::uint64_t DrainRequestsByOpcode(std::uint32_t opcode);
-    bool Busy() const { return outstanding.load(std::memory_order_acquire) != 0; }
+    bool Busy() const;
     std::size_t Outstanding() const { return outstanding.load(std::memory_order_acquire); }
     bool Overlaps(std::uint64_t address, std::size_t bytes);
+    bool OverlapsAny(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges);
+    Watch WatchWrites();
+    bool WrittenSince(const Watch& watch, const std::vector<Range>& reads);
+    std::uint64_t Retired() const { return retiredItems.load(std::memory_order_acquire); }
+    void NoteDecodeDrain() { decodeDrains.fetch_add(1, std::memory_order_relaxed); }
+    std::uint64_t DecodeDrains() const { return decodeDrains.load(std::memory_order_relaxed); }
     std::optional<std::uint64_t> PendingLabel(std::uint64_t address, std::size_t bytes);
     static bool HeldReads(std::uint64_t address, std::size_t bytes);
     void ReleaseRunningReads();
 
+    static bool MetadataPassesPipelined();
+    static void CountPipelinedMetadataPass();
+    static std::uint64_t PipelinedMetadataPasses();
 private:
     struct Item {
         Commit commit;
@@ -50,22 +67,32 @@ private:
         std::uint64_t labelAddress = 0;
         std::vector<std::byte> labelBytes;
         std::vector<Range> reads;
+        bool draw = false;
     };
     DrawPipeline() = default;
     void run();
     void rethrowFailure();
     void report(std::chrono::steady_clock::time_point now);
+    void countWrites(const std::vector<Range>& writes, bool added);
 
     std::mutex mutex;
     std::condition_variable wake;
     std::condition_variable idle;
     std::deque<Item> items;
+    std::vector<std::pair<Range, std::uint32_t>> queuedWrites;
+    std::vector<Item> retired;
     std::exception_ptr failure;
     std::atomic<std::size_t> outstanding{0};
+    std::atomic<std::uint64_t> retiredItems{0};
+    std::atomic<std::uint64_t> decodeDrains{0};
+    std::uint64_t reportedDecodeDrains = 0;
     std::uint32_t idleWaiters = 0;
+    std::size_t resumeAt = 0;
     bool committerWaiting = false;
     bool stopped = false;
     bool closing = false;
+    std::uint64_t wakes = 0;
+    std::uint64_t lockWaitNs = 0;
     std::thread thread;
     std::uint64_t commits = 0;
     std::uint64_t commitNs = 0;
@@ -80,6 +107,54 @@ private:
     std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now();
 };
 
+class DrawFinisher {
+public:
+    using Job = std::function<void()>;
+    using Range = std::pair<std::uint64_t, std::uint64_t>;
+    struct Pending {
+        std::uint64_t key = 0;
+        std::uint64_t shape = 0;
+        bool unknown = true;
+        std::vector<Range> writes;
+        std::uint64_t labelAddress = 0;
+        std::vector<std::byte> labelBytes;
+    };
+    enum class LabelLookup : std::uint8_t { NotHeld, Value, Unknown };
+    static bool Overlap();
+    bool PendingOverlaps(std::uint64_t begin, std::uint64_t end);
+    LabelLookup PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t& value);
+    bool PendingOverlapsAny(std::span<const Range> ranges);
+    bool PendingKey(std::uint64_t key, std::uint64_t shape);
+    enum class DrainSite : std::uint32_t { Lookup, Reads, Packet, Kept, Count };
+    static void NoteDrain(DrainSite site, bool drained);
+    static std::string DrainText();
+    static void NoteDrainedOpcode(std::uint32_t opcode, bool besideCommits);
+    static DrawFinisher& Get();
+    static bool Enabled();
+    static bool OnFinisherThread();
+    static bool Busy() { return outstanding.load(std::memory_order_acquire) != 0; }
+    void Enqueue(Job job);
+    void Enqueue(Job job, Pending held);
+    void Drain();
+    void Settle();
+    void ReleaseRan();
+
+private:
+    DrawFinisher() = default;
+    void run();
+    inline static std::atomic<std::size_t> outstanding{0};
+    std::mutex mutex;
+    std::condition_variable wake;
+    std::condition_variable idle;
+    std::deque<Job> jobs;
+    std::deque<Pending> pending;
+    std::vector<Job> ran;
+    std::exception_ptr failure;
+    bool running = false;
+    bool sleeping = false;
+    std::thread thread;
+};
+
 }
 
 #endif

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -17,6 +17,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 #include "prx/libc/include/Shutdown.hpp"
 #include <array>
 #include <atomic>
@@ -41,8 +42,14 @@
 #include <utility>
 #include <vector>
 
+namespace AgcDriver {
+class PerformanceTimer;
+}
+
 namespace AgcDriver::DriverDetail {
 
+struct DrawTail;
+
 class Driver {
 public:
     static Driver& Get();
@@ -67,6 +74,17 @@ public:
     void ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);
     void ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType);
     void SettleShaderPreparations(std::span<const Shader* const> stages);
+    [[nodiscard]] std::size_t PreparedEntries(const Shader* shader);
+    struct DeferredAbiUseCounts {
+        std::uint64_t firstUses;
+        std::uint64_t found;
+        std::uint64_t own;
+        std::uint64_t entries;
+        std::uint64_t unmatchedEntries;
+    };
+    [[nodiscard]] static DeferredAbiUseCounts DeferredAbiUses();
+    [[nodiscard]] std::vector<std::array<std::uint64_t, 2>> PreparedEntryHashes(const Shader* shader);
+    [[nodiscard]] std::array<std::uint64_t, 4> PrebuiltLibraries();
     static void FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, const DrawProgram& program, Pm4::DrawParameters& parameters);
     static std::optional<Graphics::IndirectDrawPath> ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr<VulkanDevice>& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect);
     static void tagVariantRuns(DispatchVariant& variant, std::span<const ShaderRecompiler::MemoryRegion> regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords, std::span<const std::shared_ptr<const ShaderRecompiler::SrtGuestWords>> supplied = {});
@@ -86,6 +104,7 @@ private:
     friend class SampledReadScope;
     friend struct PendingView;
     friend struct PacketTimer;
+    friend struct DriverTestAccess;
     void stop();
     Driver();
     void rethrowFailure() const;
@@ -140,7 +159,7 @@ private:
     static bool verifyDrawRecipe();
     static std::size_t drawCacheEntries();
     void accountDrawVariant(const DispatchVariant& variant, bool added);
-    void insertDrawEntry(std::uint64_t key, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::shared_ptr<const DrawDecode> decode, std::uint64_t shape = 0);
+    void insertDrawEntry(std::uint64_t key, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::shared_ptr<const DrawDecode> decode, std::uint64_t shape = 0, bool replaceShape = false);
     bool admitDrawKey(std::uint64_t key, std::uint64_t shape);
     void eraseDrawEntry(std::uint64_t key, const std::shared_ptr<DrawEntry>& expected);
     std::shared_ptr<const DrawRecipe> findDrawRecipe(std::uint64_t key, const std::vector<std::shared_ptr<DispatchVariant>>& stages);
@@ -166,11 +185,12 @@ private:
     void lookupDraw(const Submission& submission, const std::shared_ptr<VulkanDevice>& localDevice, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector<DrawProgram>& programs, const std::vector<ShaderRecompiler::ProgramRole>& roles, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, bool useDrawEntries, bool registerKey, bool profile, std::uint64_t& drawKey, std::shared_ptr<DrawEntry>& entry, std::vector<std::shared_ptr<DispatchVariant>>& matched, std::vector<std::vector<ShaderRecompiler::MemoryRegion>>& matchedRegions, bool& drawHit, bool& verifyHit, DrawPhaseTiming& phaseTiming, std::array<double, DrawDriverPhaseCount>& phaseMs, const std::shared_ptr<DrawEntry>& dataEntry, DrawDataCandidates& dataCandidates, std::vector<StageDataHit>& stageData);
     std::shared_ptr<const ShaderRecompiler::RecompileResult> materializeDrawStage(std::size_t i, std::uint32_t pushOffset, const QueueState& queue, const Submission& submission, const std::vector<DrawProgram>& programs, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, std::vector<ShaderRecompiler::MemoryRegion>& memory, const std::vector<ShaderRecompiler::LinkedProgram>& linked, const Pm4::DrawParameters& drawParameters, const std::shared_ptr<VulkanDevice>& localDevice, ShaderMemory& shaderMemory, std::vector<StageCapture>& stageCaptures, std::vector<bool>& recompiled, bool drawHit, const std::vector<std::shared_ptr<DispatchVariant>>& matched, const std::vector<std::vector<ShaderRecompiler::MemoryRegion>>& matchedRegions, bool profile, std::uint64_t dumpTarget, std::uint64_t dumpSlot1, std::uint64_t& captures, DrawPhaseTiming& phaseTiming, std::array<double, DrawDriverPhaseCount>& phaseMs, std::string& rejected, const std::optional<ShaderRecompiler::MeshBarycentricLayout>& meshBarycentrics);
     static void recordStageDataPositions(DispatchVariant& variant, const ShaderRecompiler::ResourceCapture& capture, const std::vector<Graphics::DecodeRead>& decodeReads);
-    void cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawParameters& drawParameters, const std::optional<Graphics::IndirectDrawPath>& indirectCpu, const std::vector<DrawProgram>& programs, const std::vector<StageCapture>& stageCaptures, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, const std::vector<std::vector<Graphics::DecodeRead>>& decodeReads, bool verifyHit, const std::vector<std::shared_ptr<DispatchVariant>>& matched, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::uint64_t drawKey, bool registerKey, const std::shared_ptr<const DrawDecode>& decode, DrawPhaseTiming& phaseTiming, const std::vector<bool>& reused, const std::shared_ptr<DrawEntry>& entry, std::uint64_t shapeKey);
+    void cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawParameters& drawParameters, const std::optional<Graphics::IndirectDrawPath>& indirectCpu, const std::vector<DrawProgram>& programs, const std::vector<StageCapture>& stageCaptures, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, const std::vector<std::vector<Graphics::DecodeRead>>& decodeReads, bool verifyHit, const std::vector<std::shared_ptr<DispatchVariant>>& matched, const std::vector<StageDataHit>& stageData, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::uint64_t drawKey, bool registerKey, const std::shared_ptr<const DrawDecode>& decode, DrawPhaseTiming& phaseTiming, const std::vector<bool>& reused, const std::shared_ptr<DrawEntry>& entry, std::uint64_t shapeKey, bool shapeNeedsEntry = false);
     static bool drawPrecheck();
     std::optional<DrawVerdict> precheckDraw(const QueueState& queue, const Submission& submission, std::span<const std::uint32_t> packet, const Pm4::DrawParameters& drawParameters, std::string& rejected, bool& traceIndirect);
     static std::uint32_t drawUserWord(const DrawProgram& program, std::int32_t sgpr);
     DrawVerdict draw(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission, std::string& rejected);
+    DrawVerdict finishDraw(DrawTail& tail, PerformanceTimer& timing);
     void addDriverPhases(DispatchClass which, const std::array<double, DriverPhaseCount>& ms, bool hit, bool validated);
     static PendingDispatchPhases& pendingDispatchPhases();
     static std::chrono::steady_clock::time_point& packetStartedAt();
@@ -243,6 +263,7 @@ private:
     void noteDrawWriters(std::span<const Graphics::CompiledShader> stages, std::uint32_t queue);
     static std::vector<std::pair<std::uint64_t, std::uint64_t>> drawWriteRanges(const Graphics::State& graphics, std::span<const Graphics::CompiledShader> stages);
     void commitDraw(std::shared_ptr<VulkanDevice> localDevice, std::uint32_t queue, const Graphics::State& graphics, const Pm4::DrawParameters& drawParameters, std::span<const Graphics::CompiledShader> stages, std::span<const Graphics::GuestMemorySnapshot> snapshots, const std::shared_ptr<const DrawRecipe>& recipe, const std::vector<std::shared_ptr<DispatchVariant>>& recipeStages, std::uint64_t drawKey);
+    bool labelPacketQueues(std::span<const std::uint32_t> packet, std::uint32_t opcode);
     bool enqueueLabelPacket(std::span<const std::uint32_t> packet, std::uint32_t opcode, std::uint32_t queue);
     bool enqueueDmaPacket(std::span<const std::uint32_t> packet, std::uint32_t opcode, std::uint32_t queue, const QueueState& state);
     void commitDmaStore(std::uint32_t queue, std::uint64_t address, std::span<const std::byte> bytes);
@@ -338,7 +359,7 @@ private:
     void run(std::uint32_t id) noexcept;
 
     std::mutex mutex;
-    std::mutex shutdownMutex;
+    AgcDriver::Mutex shutdownMutex;
     std::condition_variable changed;
 
     std::map<std::uint32_t, QueueWorker> workers;
@@ -361,7 +382,7 @@ private:
     std::unordered_map<std::uint64_t, std::shared_ptr<DispatchEntry>> dispatchCache;
 
     std::list<std::uint64_t> dispatchOrder;
-    std::mutex dispatchCacheMutex;
+    AgcDriver::Mutex dispatchCacheMutex;
     std::uint64_t dispatchCacheHits = 0;
     std::uint64_t dispatchCacheEvictions = 0;
 
@@ -383,18 +404,18 @@ private:
     DrawRelocationPending drawRelocationPending;
     std::unordered_map<std::uint64_t, DrawRelocationTrial> drawRelocationTrials;
     DrawRelocationCounters drawRelocationCounters;
-    std::mutex drawCacheMutex;
+    AgcDriver::Mutex drawCacheMutex;
     std::uint64_t drawCacheHits = 0, drawCacheEvictions = 0, drawCacheVariants = 0, drawCacheVariantBytes = 0;
     std::optional<DrawKeySightings> drawKeySightings;
     std::optional<DrawKeySightings> drawShapeSightings;
 
     DrawEntryCounters drawEntryCounters;
 
-    std::mutex driverPhasesMutex;
+    AgcDriver::Mutex driverPhasesMutex;
     std::array<DriverPhaseTotals, DispatchClassCount> driverPhaseTotals{};
     std::chrono::steady_clock::time_point driverPhasesReport = std::chrono::steady_clock::now();
 
-    std::mutex drawPhasesMutex;
+    AgcDriver::Mutex drawPhasesMutex;
     DrawPhaseTotals drawPhaseTotals;
 
     std::map<std::uint32_t, QueueState> queues;
@@ -407,7 +428,7 @@ private:
     std::uint32_t nextEopToken = EopFlipTokenBase;
     DevicePointer device;
 
-    std::mutex labelStoresMutex;
+    AgcDriver::Mutex labelStoresMutex;
     std::unordered_map<std::uint64_t, std::array<LabelStore, LabelStoreHistory>> labelStores;
 
     std::vector<std::shared_ptr<VulkanDevice>> replacedDevices;
@@ -438,7 +459,7 @@ private:
     std::atomic<std::uint64_t> evidenceValidations{0};
     std::atomic<std::uint64_t> dispatchesAdoptedAhead{0};
 
-    std::mutex writtenBuffersMutex;
+    AgcDriver::Mutex writtenBuffersMutex;
     std::deque<WrittenBuffer> writtenBuffers;
     std::uint64_t writtenBufferSerial = 0;
     std::unordered_map<std::uint64_t, DwordEvidence> dwordEvidence;
@@ -448,7 +469,7 @@ private:
     std::atomic<std::uint64_t> observationsNoWait{0};
     std::atomic<std::uint64_t> observationsNotReached{0};
 
-    std::mutex validateMutex;
+    AgcDriver::Mutex validateMutex;
     ValidateCounters validateCounters;
 
 };

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferCopy.hpp

@@ -28,6 +28,8 @@ struct StreamCopy {
 
 bool MatchesStreamCopyKernel(std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute);
 std::optional<StreamCopy> StreamCopyRange(const std::vector<std::uint32_t>& userData, std::uint32_t groups);
+bool MatchesIndexedCopyKernel(std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute);
+std::optional<StreamCopy> IndexedCopyRange(const std::vector<std::uint32_t>& userData, std::uint32_t groups, std::uint32_t count, std::uint32_t sourceOffset, std::uint32_t destinationOffset);
 
 }
 

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp

@@ -127,6 +127,8 @@ struct PreparedShaderState {
     struct Entry {
         std::size_t codeOffset;
         std::shared_ptr<const ShaderRecompiler::SourceHandle> handle;
+        bool fromDeferredAbi = false;
+        bool matched = false;
     };
     struct Rectangle {
         std::uint64_t vertexId;
@@ -153,6 +155,14 @@ struct PreparedShaderState {
     bool deferred = false;
 };
 struct PreparedShaders : PreparedShaderState {
+    struct DeferredAbi {
+        std::vector<std::uint64_t> key;
+        std::vector<std::pair<std::uint32_t, std::uint32_t>> context;
+        std::vector<std::pair<std::uint32_t, std::uint32_t>> primitive;
+        std::weak_ptr<VulkanDevice> device;
+    };
+    std::vector<DeferredAbi> deferredAbis;
+    bool deferredPreparing = false;
     std::mutex mutex;
     std::condition_variable settled;
     std::exception_ptr failure;
@@ -165,6 +175,8 @@ struct RegisteredShaderState {
     Registers shader;
     Registers context;
     Registers userConfig;
+
+    bool operator==(const RegisteredShaderState&) const = default;
 };
 
 struct ShaderSnapshot {
@@ -177,6 +189,7 @@ struct ShaderSnapshot {
     std::shared_ptr<const RegisteredShaderState> registeredState;
     std::shared_ptr<std::atomic<std::uint8_t>> workgroupUse = std::make_shared<std::atomic<std::uint8_t>>(0u);
     std::shared_ptr<std::atomic<std::uint8_t>> positionVectorUse = std::make_shared<std::atomic<std::uint8_t>>(0u);
+    std::shared_ptr<std::atomic<std::uint8_t>> memoryWrites = std::make_shared<std::atomic<std::uint8_t>>(0u);
     std::shared_ptr<std::atomic<bool>> rawGraphicsNoted = std::make_shared<std::atomic<bool>>(false);
     std::optional<std::uint32_t> userDataTailRegister;
 };
@@ -189,6 +202,7 @@ void NoteRawGraphicsDraw(const ShaderSnapshot& snapshot);
 bool RawGraphicsDrawn();
 bool RawGraphicsDrawnIn(std::uint64_t begin, std::uint64_t end);
 bool ProgramUsesWorkgroup(const ShaderSnapshot& snapshot, std::size_t codeOffset);
+bool ProgramWritesMemory(const ShaderSnapshot& snapshot, std::size_t codeOffset);
 // The PA_CL_VS_OUT_CNTL position export bits a vertex program is compiled with: none for a program
 // without pos1..pos3 exports, so its artifact does not depend on them.
 std::uint32_t PositionExportLayout(const ShaderSnapshot* snapshot, std::size_t codeOffset, std::span<const std::uint32_t> code, std::uint32_t paClVsOutCntl, const ShaderSnapshot* continuation = nullptr);

core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp

@@ -1,6 +1,7 @@
 #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_GUESTMEMORY_HPP
 #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_GUESTMEMORY_HPP
 
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 #include <atomic>
 #include <cstddef>
 #include <cstdint>
@@ -84,6 +85,7 @@ bool UnchangedCommittedSince(std::span<const std::pair<std::uint64_t, std::uint6
 // Returns the generation the blocks were stamped with (0 when the arena is not write-watched):
 // UnchangedSince(range, it) holds until the next store over the range.
 std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes);
+std::uint64_t MarkPossiblyWritten(std::uint64_t address, std::size_t bytes);
 std::uint64_t StoreOwnBytes(std::uint64_t address, std::size_t bytes, const std::function<void()>& store);
 std::uint64_t StoreOwnBytes(std::uint64_t address, std::span<const std::byte> source);
 bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generation);
@@ -105,6 +107,9 @@ bool StoredAllSince_nid_no_patch(std::uint64_t address, std::size_t bytes, std::
 // workers bump before every packet as before. CollectEpochBumps counts the bumps ([guestmem] line).
 void BumpCollectEpoch();
 void AdoptLatestCollectEpoch();
+std::uint64_t ProofSerial();
+void BreakProofs();
+void MarkProofThread();
 std::uint64_t CollectEpoch();
 std::uint64_t CollectEpochBumps();
 std::uint64_t ThreadCollectEpoch();
@@ -152,8 +157,10 @@ public:
     std::uint32_t DepthOnThisThread() const;
 
 private:
+    void enter();
+    bool tryEnter();
     void acquired();
-    std::recursive_mutex mutex;
+    Mutex mutex;
     // The holder's thread token and recursion depth, written by the holder only; another thread
     // reads the owner just to see that it is not itself.
     std::atomic<const void*> owner{nullptr};

core/libs/prx/libSceAgcDriver/Execution/include/Mutex.hpp

@@ -0,0 +1,65 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_MUTEX_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_MUTEX_HPP
+
+#include <atomic>
+#include <cstdint>
+
+namespace AgcDriver {
+
+inline void WaitWhile(const std::atomic<std::uint32_t>& word, std::uint32_t value) noexcept {
+    word.wait(value, std::memory_order_relaxed);
+}
+
+inline void WakeOne(std::atomic<std::uint32_t>& word) noexcept {
+    word.notify_one();
+}
+
+inline void WakeAll(std::atomic<std::uint32_t>& word) noexcept {
+    word.notify_all();
+}
+
+class Mutex {
+public:
+    constexpr Mutex() noexcept = default;
+    Mutex(const Mutex&) = delete;
+    Mutex& operator=(const Mutex&) = delete;
+
+    void lock() noexcept {
+        if (!try_lock()) lockContended();
+    }
+
+    bool try_lock() noexcept {
+        std::uint32_t expected = Unlocked;
+        return state.compare_exchange_strong(expected, Locked, std::memory_order_acquire, std::memory_order_relaxed);
+    }
+
+    void unlock() noexcept {
+        if (state.exchange(Unlocked, std::memory_order_release) == Contended) WakeOne(state);
+    }
+
+private:
+    static constexpr std::uint32_t Unlocked = 0;
+    static constexpr std::uint32_t Locked = 1;
+    static constexpr std::uint32_t Contended = 2;
+
+    [[gnu::noinline]] void lockContended() noexcept {
+        while (state.exchange(Contended, std::memory_order_acquire) != Unlocked) WaitWhile(state, Contended);
+    }
+
+    std::atomic<std::uint32_t> state{Unlocked};
+};
+
+inline const void* CurrentThreadToken() noexcept {
+#if defined(_WIN32) && defined(__x86_64__) && defined(__GNUC__)
+    const void* block;
+    __asm__("movq %%gs:0x30, %0" : "=r"(block));
+    return block;
+#else
+    thread_local char token;
+    return &token;
+#endif
+}
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp

@@ -3,6 +3,7 @@
 
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp"
 #include "prx/libSceAgcDriver/Execution/include/TracyZone.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 #include <algorithm>
 #include <chrono>
 #include <cstddef>
@@ -206,7 +207,7 @@ private:
         return std::chrono::duration<double, std::milli>(elapsed).count();
     }
 
-    std::mutex mutex;
+    AgcDriver::Mutex mutex;
     bool localMetrics = false;
     std::map<std::pair<std::string_view, std::string_view>, Metric> metrics;
     std::map<std::pair<std::string_view, std::string_view>, Metric> asyncMetrics;

core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp

@@ -154,6 +154,7 @@ inline bool CopyDataImmediate(std::uint32_t source) { return source >= 10 && sou
 inline bool CopyDataToRegister(std::span<const std::uint32_t> packet) { return ((packet[1] >> 8u) & 0xfu) == 0; }
 inline std::size_t CopyDataBytes(std::span<const std::uint32_t> packet) { return (packet[1] & 0x10000u) != 0 ? 8 : 4; }
 inline constexpr std::uint32_t RewindValid = 0x80000000u;
+inline constexpr std::uint32_t RewindOffloadEnable = 0x01000000u;
 struct WaitFields {
     std::uint32_t function;
     std::uint64_t address;

core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp

@@ -221,7 +221,7 @@ inline Registers BuildInitialContextRegisters() {
     Registers result{
         {0x200, 0}, {0x201, 0}, {0x202, 0xcc0010}, {0x203, 0},
         {0x204, 0}, {0x205, 0}, {0x206, 1087}, {0x207, 0},
-        {0x0, 0}, {0x2, 0}, {0x3, 0}, {0x4, 0}, {0x8, 0}, {0x9, 0x3f800000}, {0xa, 0}, {0xb, 0},
+        {0x0, 0}, {0x2, 0}, {0x3, 0}, {0x4, 0}, {0x8, 0}, {0x9, 0x3f800000}, {0xa, 0}, {0xb, 0}, {0x10, 0}, {0x11, 0},
         {0x80, 0}, {0x83, 0xffff}, {0x8c, 0xaa99aaaa}, {0x8d, 0}, {0x8e, 0}, {0x8f, 0},
         {0xc, 0}, {0xd, 0x40004000}, {0x81, 0x80000000}, {0x82, 0x40004000},
         {0x90, 0x80000000}, {0x91, 0x40004000},

core/libs/prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp

@@ -16,6 +16,7 @@ public:
     ShaderPreparationTransaction& operator=(const ShaderPreparationTransaction&) = delete;
     PreparedShaderState& Edit(const ShaderSnapshot& snapshot);
     const PreparedShaderState& Read(const ShaderSnapshot& snapshot) const;
+    bool Tracks(const ShaderSnapshot& snapshot) const;
     void Hold();
     void Commit();
 

core/libs/prx/libSceAgcDriver/Execution/include/SubgroupBarrier.hpp

@@ -9,6 +9,10 @@ inline bool SubgroupControlBarrierUsable(VkDriverId driver) {
     return driver != VK_DRIVER_ID_AMD_PROPRIETARY;
 }
 
+inline bool WorkgroupScopeFenceUsable(VkDriverId driver) {
+    return driver != VK_DRIVER_ID_AMD_PROPRIETARY;
+}
+
 }
 
 #endif

core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp

@@ -16,7 +16,9 @@
 #include <functional>
 #include <memory>
 #include <optional>
+#include <span>
 #include <string>
+#include <utility>
 
 namespace AgcDriver {
 
@@ -31,9 +33,22 @@ struct ComputePipelineObjects;
 struct RecipeHit;
 // The record tail shared by an ordinary dispatch and a recipe hit (VulkanDevice::recordDispatch).
 struct RecordedDispatch;
+struct DispatchCompletion;
 // The module, layout and pipeline a compute dispatch binds (VulkanDevice::createComputePipeline).
 struct ComputePipelineObjects;
 
+struct DeviceQueuePlan {
+    std::uint32_t mainCount = 1;
+    bool settle = false;
+    std::uint32_t settleFamily = 0;
+    std::uint32_t settleIndex = 0;
+    bool sparseBinding = false;
+    std::uint32_t sparseFamily = VK_QUEUE_FAMILY_IGNORED;
+    std::uint32_t sparseIndex = 0;
+};
+
+DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> families, std::uint32_t family, bool settles, bool sparseBinding);
+
 class VulkanDevice {
 public:
     explicit VulkanDevice(const PresentationWindow* window = nullptr);
@@ -96,10 +111,10 @@ public:
     // Records a store of `bytes` at a guest address behind the recorded work, so the value appears
     // once that work completed; the batch is submitted by the queue worker's flush rules (or at once
     // with APS5_LABEL_SUBMIT_NOW=1). `stamp` is the record-order stamp and `queue` the recording
-    // queue for the recorder's pending-label table. Returns 0 when recorded on the GPU, 5 when kept
-    // as a completion action behind pending write-backs, else why the CPU must write it:
-    // 1 nothing recorded (the write is already ordered), 2 write-backs pending and 3 memory not
-    // imported, 4 unsuitable size or alignment.
+    // queue for the recorder's pending-label table. Returns 0 when recorded on the GPU, 5 when stored
+    // by a completion behind pending CPU write-backs or completion stores, else why the CPU must
+    // write it: 1 nothing recorded (the write is already ordered), 2 write-backs pending (only with
+    // APS5_DRAIN_COMPLETION_LABELS=1) and 3 memory not imported, 4 unsuitable size or alignment.
     // `reapFirst` retires finished batches before the checks; a caller recording a group of labels
     // under one lock passes it for the first label only.
     int WriteLabelOnGpu(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst = true);
@@ -273,7 +288,7 @@ public:
     // `recipe`, when given, receives the Recipe a successful call built for its dispatch-cache
     // variant (design_cpu_final M4): only when the resource cache served or took the object
     // (reusable, cacheable) and recipes are on (APS5_NO_DISPATCH_RECIPE=1 builds none); else null.
-    void Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span<const Graphics::GuestMemorySnapshot> snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr<PreparedDispatch> prepared = nullptr, std::shared_ptr<const Recipe>* recipe = nullptr);
+    void Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span<const Graphics::GuestMemorySnapshot> snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr<PreparedDispatch> prepared = nullptr, std::shared_ptr<const Recipe>* recipe = nullptr, std::optional<std::pair<std::uint64_t, std::size_t>> htileClear = std::nullopt);
     struct IndirectOutcome {
         int cpuReason;
         double argumentReadMs;
@@ -357,7 +372,8 @@ private:
     ShaderRecompiler::SpirvTarget buildTarget() const;
     Graphics::Context buildContext() const;
     // Body of Dispatch and DispatchIndirect: `arguments` 0 dispatches x, y, z groups.
-    IndirectOutcome dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t programAddress, std::shared_ptr<PreparedDispatch> prepared, std::shared_ptr<const Recipe>* recipe, const std::array<std::uint32_t, 3>& threadGroup = {});
+    IndirectOutcome dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t programAddress, std::shared_ptr<PreparedDispatch> prepared, std::shared_ptr<const Recipe>* recipe, const std::array<std::uint32_t, 3>& threadGroup = {}, std::optional<std::pair<std::uint64_t, std::size_t>> htileClear = std::nullopt);
+    IndirectOutcome recordCompletedDispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::uint64_t programAddress, std::shared_ptr<const Recipe>* recipeOut, const std::array<std::uint32_t, 3>& threadGroup, std::optional<std::pair<std::uint64_t, std::size_t>> htileClear, DispatchCompletion& completion);
     // The stage-A pre-sync over `surfaces` (see PrepareDispatch): the serial waited for, 0 none.
     std::uint64_t presync(std::span<const Graphics::PresyncSurface> surfaces);
     std::shared_ptr<ComputePipelineObjects> computePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span<const std::uint32_t> layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/DeviceAccess.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/DeviceAccess.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 
 namespace AgcDriver::DriverDetail {
 
@@ -21,27 +22,37 @@ DevicePointer::operator bool() const { return Load() != nullptr; }
 bool DevicePointer::operator==(std::nullptr_t) const { return Load() == nullptr; }
 
 void DeviceUseGate::lock_shared() {
-    std::unique_lock lock(mutex);
-    changed.wait(lock, [&] { return !replacing; });
-    ++users;
+    auto value = state.load(std::memory_order_relaxed);
+    for (;;) {
+        if ((value & Replacing) != 0) {
+            WaitWhile(state, value);
+            value = state.load(std::memory_order_relaxed);
+        } else if (state.compare_exchange_weak(value, value + 1, std::memory_order_acquire, std::memory_order_relaxed)) {
+            return;
+        }
+    }
 }
 
 void DeviceUseGate::unlock_shared() {
-    std::lock_guard lock(mutex);
-    if (--users == 0) changed.notify_all();
+    if (state.fetch_sub(1, std::memory_order_release) == (Replacing | 1)) WakeAll(state);
 }
 
 void DeviceUseGate::lock() {
-    std::unique_lock lock(mutex);
-    changed.wait(lock, [&] { return !replacing; });
-    replacing = true;
-    changed.wait(lock, [&] { return users == 0; });
+    auto value = state.load(std::memory_order_relaxed);
+    for (;;) {
+        if ((value & Replacing) != 0) {
+            WaitWhile(state, value);
+            value = state.load(std::memory_order_relaxed);
+        } else if (state.compare_exchange_weak(value, value | Replacing, std::memory_order_acquire, std::memory_order_relaxed)) {
+            break;
+        }
+    }
+    for (value = state.load(std::memory_order_acquire); value != Replacing; value = state.load(std::memory_order_acquire)) WaitWhile(state, value);
 }
 
 void DeviceUseGate::unlock() {
-    std::lock_guard lock(mutex);
-    replacing = false;
-    changed.notify_all();
+    state.fetch_and(~Replacing, std::memory_order_release);
+    WakeAll(state);
 }
 
 }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp

@@ -16,7 +16,9 @@
 #include "Optimization/ResourceProgram.hpp"
 #include <algorithm>
 #include <cstdlib>
+#include <optional>
 #include <stdexcept>
+#include <utility>
 
 namespace AgcDriver::DriverDetail {
 
@@ -117,12 +119,13 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
     if (indirectArguments == 0 && (packet[4] & 0x2u) != 0) {
         partialGroups = Pm4::PartialGroupThreads({packet[1], packet[2], packet[3]}, {readRegister(queue.shader, 0x207), readRegister(queue.shader, 0x208), readRegister(queue.shader, 0x209)});
     }
-    const bool pipelineCandidate = !resolving && !fromAhead && DispatchItemsEnabled() && DrawPipeline::Active() && indirectArguments == 0 && deferredLabels().labels.empty() && !matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute) && !MatchesStreamCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute);
+    const bool pipelineCandidate = !resolving && !fromAhead && DispatchItemsEnabled() && DrawPipeline::Active() && indirectArguments == 0 && deferredLabels().labels.empty() && !matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute) && !MatchesStreamCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute) && !MatchesIndexedCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute);
     const bool partialLastGroup = partialGroups != std::array<std::uint32_t, 3>{};
+    std::optional<std::pair<std::uint64_t, std::size_t>> htileClearCompletion;
     if (resolving) {
         if (indirectArguments == 0 && Graphics::MatchMultisampledSingleClear(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads)) return;
         if (!partialLastGroup && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return;
-        if (!partialLastGroup && indirectArguments == 0 && (matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute) || MatchesStreamCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute))) return;
+        if (!partialLastGroup && indirectArguments == 0 && (matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute) || MatchesStreamCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute) || MatchesIndexedCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute))) return;
     } else {
         if (indirectArguments == 0 && Graphics::TakeMultisampledSingleClear(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads)) {
             pendingDispatchPhases().outcome = DispatchOutcome::FillHle;
@@ -139,7 +142,8 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
         if (const auto htile = Graphics::HtileDepthClearAddress(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads); htile != 0) {
             const auto threads = (packet[4] & 0x20u) != 0 ? static_cast<std::uint64_t>(packet[1]) : static_cast<std::uint64_t>(packet[1]) * compute.numThreads[0];
             const auto nativeBytes = Graphics::HtileDepthClearBytes_nid_no_patch(std::span(snapshot.code).subspan(codeOffset), userData, compute, packet, indirectArguments);
-            Graphics::NoteHtileDepthClear(htile, static_cast<std::size_t>(threads * 16u), nativeBytes);
+            const auto bytes = static_cast<std::size_t>(threads * 16u);
+            if (Graphics::NoteHtileDepthClear(htile, bytes, nativeBytes)) htileClearCompletion = std::pair{htile, bytes};
         }
         if (!partialLastGroup && indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) {
             pendingDispatchPhases().outcome = DispatchOutcome::CopyHle;
@@ -462,18 +466,18 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
         const auto epoch = DrawPipeline::EpochToken().load(std::memory_order_relaxed);
         const auto queueId = submission.queue;
         itemCommits.fetch_add(1, std::memory_order_relaxed);
-        DrawPipeline::Queue0().Enqueue([this, localDevice = std::move(localDevice), result = compiledResult, snapshots = std::move(snapshots), liveWords = std::move(liveWords), groups, address, queueId, recipe, keepVariant, dataHit, epoch] {
+        DrawPipeline::Queue0().Enqueue([this, localDevice = std::move(localDevice), result = compiledResult, snapshots = std::move(snapshots), liveWords = std::move(liveWords), groups, address, queueId, recipe, keepVariant, dataHit, epoch, htileClearCompletion] {
             DrawPipeline::FollowEpoch(epoch);
             GuestMemory::SetCurrentPacket(0x15, queueId);
             const auto& compiledCommit = *result;
             const bool noteWrites = writeEvidenceEnabled() || traceCapSync();
             try {
-                std::shared_ptr<RecipeHit> recipeHit = localDevice->PrepareRecipe(recipe, false);
+                std::shared_ptr<RecipeHit> recipeHit = htileClearCompletion.has_value() ? nullptr : localDevice->PrepareRecipe(recipe, false);
                 std::shared_ptr<const Recipe> builtRecipe;
                 bool writersNoted = false;
                 for (;;) {
                     std::shared_ptr<PreparedDispatch> prepared;
-                    if (recipeHit == nullptr || VulkanDevice::VerifyRecipes()) prepared = localDevice->PrepareDispatch(compiledCommit, snapshots);
+                    if (recipeHit == nullptr || VulkanDevice::VerifyRecipes() || htileClearCompletion.has_value()) prepared = localDevice->PrepareDispatch(compiledCommit, snapshots);
                     GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Dispatch);
                     std::lock_guard gpuLock(GuestMemory::GpuMutex());
                     recordLabelsForPacket(localDevice.get(), queueId);
@@ -491,7 +495,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
                             continue;
                         }
                     } else {
-                        localDevice->Dispatch(compiledCommit, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared), &builtRecipe);
+                        localDevice->Dispatch(compiledCommit, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared), htileClearCompletion.has_value() ? nullptr : &builtRecipe, htileClearCompletion);
                     }
                     break;
                 }
@@ -517,7 +521,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
 
     std::shared_ptr<RecipeHit> recipeHit;
     if (cached && keepVariant != nullptr && !stampValidate()) {
-        recipeHit = localDevice->PrepareRecipe(keepVariant->recipe.load(std::memory_order_acquire), indirectArguments != 0);
+        recipeHit = htileClearCompletion.has_value() ? nullptr : localDevice->PrepareRecipe(keepVariant->recipe.load(std::memory_order_acquire), indirectArguments != 0);
         phaseTiming.Phase(PhaseRecipePrecheck);
     }
 
@@ -595,7 +599,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
                 const auto outcome = localDevice->DispatchIndirect(compiled, indirectArguments, snapshots, address, std::move(prepared), attachTo != nullptr ? &builtRecipe : nullptr, threadGroup);
                 countIndirect(outcome.cpuReason, outcome.argumentReadMs);
             } else {
-                localDevice->Dispatch(compiled, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared), attachTo != nullptr ? &builtRecipe : nullptr);
+                localDevice->Dispatch(compiled, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared), htileClearCompletion.has_value() ? nullptr : (attachTo != nullptr ? &builtRecipe : nullptr), htileClearCompletion);
             }
         } catch (const std::exception& error) {
             if (rethrow(error)) return;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp

@@ -1,6 +1,7 @@
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
 #include "ThreadOwned.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp"
@@ -107,6 +108,7 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l
     if (!allQueues && submission.queue == 0) return;
     auto& resolved = resolvedAhead();
     if (resolved.contains(from) || device.Load() == nullptr) return;
+    if (submission.queue == 0 && DrawPipeline::Depth() != 0 && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, (submission.commands[from] >> 8u) & 0xffu);
     auto& stats = groupCaptureStats();
     const auto begin = std::chrono::steady_clock::now();
     ++stats.runs;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp

@@ -3,6 +3,7 @@
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 #include <cstdio>
 #include <cstdlib>
 
@@ -30,7 +31,7 @@ const char* IndirectPathName(int path) {
 void Driver::countIndirect(int path, double readMs) {
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
     if (!profile || path < 0 || path >= IndirectPaths) return;
-    static std::mutex countsMutex;
+    static AgcDriver::Mutex countsMutex;
     static std::uint64_t counts[IndirectPaths] = {};
     static double readWaitedMs = 0;
     static auto lastReport = std::chrono::steady_clock::now();

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp

@@ -93,7 +93,7 @@ EntryOutcome Driver::validateVariant(std::uint64_t program, std::uint32_t queue,
     if (live != nullptr) live->clear();
     if (DrawPipeline::Active()) {
         auto& pipeline = DrawPipeline::Queue0();
-        if (pipeline.Busy() && std::any_of(variant.runs.begin(), variant.runs.end(), [&](const auto& run) { return pipeline.Overlaps(run.first, static_cast<std::size_t>(run.second - run.first)); })) pipeline.Drain(DrawPipeline::DrainReason::Capture);
+        if (pipeline.Busy() && pipeline.OverlapsAny(variant.runs)) pipeline.Drain(DrawPipeline::DrainReason::Capture);
     }
     const bool masked = live != nullptr && !variant.dataPositions.empty();
     const DataMask mask{variant.dataPositions, live};

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp

@@ -1,6 +1,8 @@
+#include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawProgramTrace.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
@@ -8,21 +10,214 @@
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "Optimization/ResourceProgram.hpp"
+#include "prx/libc/include/HostThreadLocal.hpp"
+#include <atomic>
+#include <array>
 #include <algorithm>
 #include <bit>
+#include <chrono>
+#include <cstdio>
+#include <unordered_set>
 #include <cstdlib>
 #include <cstdio>
 #include <cstring>
+#include <unordered_map>
 
 namespace AgcDriver::DriverDetail {
 
+struct DrawTail {
+    using GpuLock = std::unique_lock<std::remove_reference_t<decltype(GuestMemory::GpuMutex())>>;
+    bool profile = false;
+    std::array<double, DrawDriverPhaseCount> phaseMs{};
+    std::uint64_t captures = 0;
+    std::chrono::steady_clock::time_point phaseLap{};
+    DrawPhaseTiming phaseTiming{false, phaseMs, phaseLap};
+    Pm4::DrawParameters drawParameters{};
+    bool pipelined = false;
+    bool traceIndirect = false;
+    std::uint64_t dumpTarget = 0;
+    std::uint64_t dumpSlot1 = 0;
+    std::shared_ptr<VulkanDevice> localDevice;
+    bool registerKey = false;
+    std::uint64_t drawKey = 0;
+    std::shared_ptr<DrawEntry> entry;
+    std::shared_ptr<const DrawDecode> decode;
+    std::shared_ptr<const DrawDecode> relocatedDecode;
+    std::shared_ptr<DrawEntry> dataEntry;
+    std::uint64_t shapeKey = 0;
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    std::vector<DrawProgram> programs;
+    std::vector<ShaderRecompiler::MemoryRegion> memory;
+    std::vector<ShaderRecompiler::LinkedProgram> linked;
+    std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>> vertexInfos;
+    std::vector<std::vector<Graphics::DecodeRead>> decodeReads;
+    std::unique_ptr<ShaderMemory> shaderMemory;
+    std::vector<ShaderRecompiler::RecompileResult> results;
+    std::vector<std::shared_ptr<const ShaderRecompiler::RecompileResult>> compiledStages;
+    std::vector<Graphics::CompiledShader> stages;
+    std::uint32_t pushCursorBytes = 0;
+    std::vector<const ShaderRecompiler::RecompileResult*> programResults;
+    std::vector<StageCapture> stageCaptures;
+    std::vector<std::shared_ptr<DispatchVariant>> matched;
+    std::vector<std::vector<ShaderRecompiler::MemoryRegion>> matchedRegions;
+    std::vector<std::shared_ptr<DispatchVariant>> fresh;
+    std::vector<bool> recompiled;
+    DrawDataCandidates dataCandidates;
+    std::vector<bool> reused;
+    std::vector<StageDataHit> stageData;
+    bool drawHit = false;
+    bool verifyHit = false;
+    bool stageEntries = false;
+    bool patchedStages = false;
+    const Submission* submission = nullptr;
+    std::uint32_t submissionQueue = 0;
+    std::uint64_t submissionSerial = 0;
+    std::size_t packetOffset = 0;
+    std::uint32_t packetOpcode = 0;
+    std::uint32_t reg207 = 0;
+    std::uint64_t epoch = 0;
+    QueueState* queue = nullptr;
+    std::span<const std::uint32_t> packet;
+    std::string* rejected = nullptr;
+    std::string ownRejected;
+    GpuLock* gpuLock = nullptr;
+    GpuLock ownLock{GuestMemory::GpuMutex(), std::defer_lock};
+};
+
+namespace {
+
+void noteFinisherHandOff(bool handed) {
+    static std::atomic<std::uint64_t> handedOff{0}, kept{0};
+    static std::atomic<std::int64_t> reportedAt{0};
+    (handed ? handedOff : kept).fetch_add(1, std::memory_order_relaxed);
+    static const bool stats = std::getenv("APS5_PIPELINE_STATS") != nullptr || std::getenv("APS5_PROFILE_DRAW") != nullptr;
+    if (!stats) return;
+    const auto tenSeconds = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now().time_since_epoch()).count() / 10;
+    if (reportedAt.exchange(tenSeconds, std::memory_order_relaxed) == tenSeconds) return;
+    std::fprintf(stderr, "[finisher] draws (10 s): %llu handed to the draw finisher, %llu finished on the worker%s\n", static_cast<unsigned long long>(handedOff.exchange(0)), static_cast<unsigned long long>(kept.exchange(0)), DrawFinisher::DrainText().c_str());
+}
+
+}
+
+namespace {
+
+enum class HeldReason : std::size_t { Known, NoDecode, Size, NullVertex, NullFragment, NullOther, Bda, Buffer, Storage, Count };
+std::array<std::atomic<std::uint64_t>, static_cast<std::size_t>(HeldReason::Count)> heldReasons{};
+
+void noteHeld(HeldReason reason) {
+    heldReasons[static_cast<std::size_t>(reason)].fetch_add(1, std::memory_order_relaxed);
+    static const bool profile = std::getenv("APS5_PIPELINE_STATS") != nullptr || std::getenv("APS5_PROFILE_DRAW") != nullptr;
+    if (!profile) return;
+    static std::atomic<std::uint64_t> total{0};
+    const auto count = ++total;
+    if (count % 262144u != 0u && (count > 262144u || (count & (count - 1u)) != 0u)) return;
+    constexpr std::array<const char*, static_cast<std::size_t>(HeldReason::Count)> names{"targets only", "no decode", "stage count", "no variant (vertex)", "no variant (fragment)", "no variant (other)", "BDA stores", "written buffer", "storage image"};
+    std::string text;
+    for (std::size_t i = 0; i < names.size(); ++i) {
+        char item[64];
+        std::snprintf(item, sizeof(item), "%s %s %llu", i == 0 ? "" : ",", names[i], static_cast<unsigned long long>(count >= 262144u ? heldReasons[i].exchange(0) : heldReasons[i].load()));
+        text += item;
+    }
+    std::fprintf(stderr, "[finisher-held] held draws' writes:%s\n", text.c_str());
+}
+
+std::optional<HeldReason> writesBeyondTargets(const ShaderRecompiler::RecompileResult& compiled) {
+    using Role = ShaderRecompiler::DescriptorRole;
+    for (const auto& binding : compiled.bindings) {
+        if (binding.role == Role::BdaPagetable && Graphics::ShaderStoresThroughAddresses(compiled.spirv)) return HeldReason::Bda;
+        if (binding.role == Role::GuestBuffers) {
+            for (std::uint32_t element = 0; element < binding.count; ++element) {
+                if (element >= binding.bufferWritten.size() || binding.bufferWritten[element]) return HeldReason::Buffer;
+            }
+        }
+        if (binding.role == Role::GuestImages && binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) {
+            for (std::uint32_t element = 0; element < binding.count; ++element) {
+                if (element >= binding.imageWritten.size() || binding.imageWritten[element]) return HeldReason::Storage;
+            }
+        }
+    }
+    return std::nullopt;
+}
+
+DrawFinisher::Pending heldWrites(const DrawTail& tail) {
+    DrawFinisher::Pending held;
+    held.key = tail.drawKey;
+    held.shape = tail.shapeKey;
+    if (tail.decode == nullptr) {
+        noteHeld(HeldReason::NoDecode);
+        return held;
+    }
+    if (tail.matched.size() != tail.programs.size()) {
+        noteHeld(HeldReason::Size);
+        return held;
+    }
+    const auto& roles = tail.decode->roles;
+    for (std::size_t i = 0; i < tail.matched.size(); ++i) {
+        if (i < roles.size() && roles[i] == ShaderRecompiler::ProgramRole::GeometryBack) continue;
+        const auto& variant = tail.matched[i];
+        if (variant == nullptr || variant->compiled == nullptr) {
+            const bool candidates = !tail.drawHit && i < tail.dataCandidates.size() && !tail.dataCandidates[i].empty();
+            const auto& program = tail.programs[i];
+            if (!candidates && program.snapshot != nullptr && !ProgramWritesMemory(*program.snapshot, program.codeOffset) && (program.tailContinuation == nullptr || !ProgramWritesMemory(*program.tailContinuation, 0))) continue;
+            if (!candidates) {
+                const auto stage = tail.programs[i].binary.stage;
+                noteHeld(stage == ShaderRecompiler::ShaderStage::Fragment ? HeldReason::NullFragment : stage == ShaderRecompiler::ShaderStage::Vertex ? HeldReason::NullVertex : HeldReason::NullOther);
+                return held;
+            }
+            for (const auto& [candidate, regions] : tail.dataCandidates[i]) {
+                if (candidate == nullptr || candidate->compiled == nullptr) {
+                    noteHeld(HeldReason::NullOther);
+                    return held;
+                }
+                if (const auto reason = writesBeyondTargets(*candidate->compiled)) {
+                    noteHeld(*reason);
+                    return held;
+                }
+            }
+            continue;
+        }
+        if (const auto reason = writesBeyondTargets(*variant->compiled)) {
+            noteHeld(*reason);
+            return held;
+        }
+        if (i < tail.stageData.size() && tail.stageData[i].compiled != nullptr) {
+            if (const auto reason = writesBeyondTargets(*tail.stageData[i].compiled)) {
+                noteHeld(*reason);
+                return held;
+            }
+        }
+    }
+    noteHeld(HeldReason::Known);
+    held.writes = DrawWriteRanges(tail.decode->state, {}, ExactDrawWrites());
+    held.unknown = false;
+    return held;
+}
+
+bool EmptyIndirectSkip() {
+    static const bool enabled = std::getenv("APS5_NO_EMPTY_INDIRECT_SKIP") == nullptr;
+    return enabled;
+}
+
+struct CpuIndirectProgramsStorage {};
+
+std::unordered_map<std::uint64_t, bool>& CpuIndirectPrograms() {
+    return HostThreadLocal<std::unordered_map<std::uint64_t, bool>, CpuIndirectProgramsStorage>();
+}
+
+}
+
 DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission, std::string& rejected) {
     PerformanceTimer timing("Driver.Draw");
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
-    std::array<double, DrawDriverPhaseCount> phaseMs{};
-    std::uint64_t captures = 0;
-    auto phaseLap = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
-    DrawPhaseTiming phaseTiming{profile, phaseMs, phaseLap};
+    auto tailHolder = std::make_shared<DrawTail>();
+    auto& tail = *tailHolder;
+    auto& phaseMs = tail.phaseMs;
+    auto& captures = tail.captures;
+    auto& phaseLap = tail.phaseLap;
+    phaseLap = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
+    tail.profile = profile;
+    tail.phaseTiming.profile = profile;
+    auto& phaseTiming = tail.phaseTiming;
     if (profile && packetStartedAt() != std::chrono::steady_clock::time_point{}) phaseMs[DrawRowPrologue] = std::chrono::duration<double, std::milli>(phaseLap - packetStartedAt()).count();
 
     const auto drawn = [&] {
@@ -35,8 +230,10 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         pending.tailAt = phaseLap;
         return DrawVerdict::Drawn;
     };
-    auto drawParameters = Pm4::ResolveDraw(packet, queue);
-    const bool pipelined = DrawPipeline::Active();
+    auto& drawParameters = tail.drawParameters;
+    drawParameters = Pm4::ResolveDraw(packet, queue);
+    tail.pipelined = DrawPipeline::Active();
+    const bool pipelined = tail.pipelined;
     static const bool pipelinedIndirect = std::getenv("APS5_NO_PIPELINED_INDIRECT") == nullptr;
     if (pipelined && drawParameters.indirect) {
         const auto& indirect = *drawParameters.indirect;
@@ -44,10 +241,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         const bool overlaps = pipeline.Busy() && (pipeline.Overlaps(indirect.arguments, static_cast<std::size_t>(std::max<std::uint64_t>(indirect.RangeBytes(), indirect.recordBytes))) || (indirect.countIndirect && pipeline.Overlaps(indirect.countAddress, 4)));
         if (!pipelinedIndirect || overlaps) pipeline.Drain(DrawPipeline::DrainReason::Indirect);
     }
-    const auto drainBeforeInline = [&] {
-        if (pipelined && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Indirect);
-    };
-    bool traceIndirect = false;
+    auto& traceIndirect = tail.traceIndirect;
     if (const auto verdict = precheckDraw(queue, submission, packet, drawParameters, rejected, traceIndirect)) return *verdict;
     phaseTiming.Phase(DrawRowPrecheck);
     using Stage = ShaderRecompiler::ShaderStage;
@@ -59,7 +253,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
 
     static const bool lockedPrepare = std::getenv("APS5_LOCKED_DRAW_PREPARE") != nullptr;
     std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock);
-    std::shared_ptr<VulkanDevice> localDevice;
+    auto& localDevice = tail.localDevice;
     if (lockedPrepare) {
 
         GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw);
@@ -81,13 +275,14 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     phaseTiming.Phase(DrawRowVectors);
 
     const bool useDrawEntries = drawEntries() && !ShaderRecompiler::DebugProbeActive() && dumpTarget == 0 && dumpSlot1 == 0;
-    const bool registerKey = useDrawEntries && registerKeyEnabled();
-    std::uint64_t drawKey = 0;
+    tail.registerKey = useDrawEntries && registerKeyEnabled();
+    const bool registerKey = tail.registerKey;
+    auto& drawKey = tail.drawKey;
     std::array<std::uint64_t, 5> drawKeyComponents{};
-    std::shared_ptr<DrawEntry> entry;
-    std::shared_ptr<const DrawDecode> decode;
-    std::shared_ptr<DrawEntry> dataEntry;
-    std::uint64_t shapeKey = 0;
+    auto& entry = tail.entry;
+    auto& decode = tail.decode;
+    auto& dataEntry = tail.dataEntry;
+    auto& shapeKey = tail.shapeKey;
     std::uint64_t drawIdentity = 0, drawAlias = 0;
     std::vector<DrawUserPatch> userPatches;
     if (registerKey) {
@@ -95,6 +290,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         std::shared_ptr<const DrawRelocation> relocation;
         bool aliasFirst = false;
         if (drawRelocationEnabled()) {
+            if (DrawFinisher::Busy() && drawRelocationPending.key != 0) DrawFinisher::Get().Drain();
             drawIdentity = drawIdentityKey(queue, *submission.shaders);
             std::lock_guard cacheLock(drawCacheMutex);
             learnPendingRelocation();
@@ -107,6 +303,17 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
             }
         }
         drawKey = drawRegisterKey(queue, *submission.shaders, localDevice->Serial(), &shapeKey, traceDrawCache() ? &drawKeyComponents : nullptr, aliasFirst ? relocation.get() : nullptr, aliasFirst ? &drawAlias : nullptr);
+        if (DrawFinisher::Busy()) {
+            bool shapeMatters = false;
+            if (DrawFinisher::Overlap() && drawDataHits()) {
+                std::lock_guard peekLock(drawCacheMutex);
+                const auto shape = drawShapes.find(shapeKey);
+                shapeMatters = !drawCache.contains(drawKey) && (shape == drawShapes.end() || !drawCache.contains(shape->second));
+            }
+            const bool drain = !DrawFinisher::Overlap() || DrawFinisher::Get().PendingKey(drawKey, shapeMatters ? shapeKey : 0);
+            if (drain) DrawFinisher::Get().Drain();
+            if (DrawFinisher::Overlap()) DrawFinisher::NoteDrain(DrawFinisher::DrainSite::Lookup, drain);
+        }
         std::unique_lock cacheLock(drawCacheMutex);
         ++drawEntryCounters.lookups;
         ++drawEntryCounters.registerKeyLookups;
@@ -186,16 +393,34 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     }
     resolveDrawDecode(queue, submission, localDevice->graphicsContext(), decode, registerKey, drawKey, profile);
     const auto& graphics = decode->state;
-    auto pixel = decode->pixel;
-    const bool writesPrograms = graphics.stages.mesh || drawParameters.indirect || !userPatches.empty();
-    std::vector<DrawProgram> writablePrograms;
-    if (writesPrograms) writablePrograms = decode->programs;
-    for (const auto& patch : userPatches) writablePrograms[patch.program].userData[patch.index] = patch.value;
-    const std::vector<DrawProgram>& programs = writesPrograms ? writablePrograms : decode->programs;
+    auto& pixel = tail.pixel;
+    pixel = decode->pixel;
+    auto& programs = tail.programs;
+    programs = decode->programs;
+    for (const auto& patch : userPatches) programs[patch.program].userData[patch.index] = patch.value;
     if (const auto skipped = SkippedShader(); skipped != 0 && std::any_of(programs.begin(), programs.end(), [skipped](const DrawProgram& program) { return program.binary.codeAddress == skipped; })) return drawn();
+    const bool emptyIndirectSkip = EmptyIndirectSkip();
+    auto& cpuIndirectPrograms = CpuIndirectPrograms();
+    const auto recordDraws = [&](const Pm4::DrawArguments& arguments) { return arguments.count != 0 && arguments.instances != 0 && (!drawParameters.indexed || arguments.firstVertexOrIndex < drawParameters.indexCount); };
+    if (emptyIndirectSkip && drawParameters.indirect && !programs.empty()) {
+        const auto known = cpuIndirectPrograms.find(programs.front().binary.codeAddress);
+        if (known != cpuIndirectPrograms.end() && known->second) {
+            const auto& indirect = *drawParameters.indirect;
+            recordQueuedLabelsBeforeRead(submission.queue);
+            const auto readStart = std::chrono::steady_clock::now();
+            const auto count = std::min(indirect.countIndirect ? Pm4::ReadDrawCount(indirect) : indirect.count, indirect.count);
+            bool any = false;
+            for (std::uint32_t record = 0; record < count && !any; ++record) any = recordDraws(Pm4::ReadDrawArguments(indirect, record));
+            if (!any) {
+                Graphics::CountEmptyIndirectDraw(std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - readStart).count());
+                timing.Mark("empty_indirect_skipped");
+                return drawn();
+            }
+        }
+    }
     const auto setMeshIndexBuffer = [&](const Pm4::DrawParameters& parameters) {
         if (!graphics.stages.mesh) return;
-        auto& words = writablePrograms.front().userData;
+        auto& words = programs.front().userData;
         require(programs.front().firstUserSgpr == 0 && words.size() >= ShaderRecompiler::MeshIndexBufferUserWord + 4, "mesh program lacks the hidden user words");
         const auto descriptor = Graphics::MeshIndexBufferDescriptor(parameters);
         std::copy(descriptor.begin(), descriptor.end(), words.begin() + ShaderRecompiler::MeshIndexBufferUserWord);
@@ -231,10 +456,15 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         indirect.startInstanceSgpr = sgprOf(indirect.startInstanceLocation);
         indirect.drawIndexSgpr = sgprOf(indirect.drawIndexLocation);
     }
-    std::vector<ShaderRecompiler::MemoryRegion> memory;
+    if (DrawFinisher::Busy()) {
+        const bool drain = !DrawFinisher::Overlap();
+        if (drain) DrawFinisher::Get().Drain();
+        else DrawFinisher::NoteDrain(DrawFinisher::DrainSite::Reads, false);
+    }
+    auto& memory = tail.memory;
+    auto& linked = tail.linked;
     memory.reserve(2 * programs.size() + 32);
     AppendProgramMemory(memory, programs);
-    std::vector<ShaderRecompiler::LinkedProgram> linked;
     linked.reserve(programs.size());
     for (std::size_t i = 0; i < programs.size(); ++i) {
         const auto& program = programs[i];
@@ -243,47 +473,59 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     timing.Mark("prepare");
     phaseTiming.Phase(DrawRowProgramPrepare);
 
-    std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>> vertexInfos(programs.size());
-    std::vector<std::vector<Graphics::DecodeRead>> decodeReads(programs.size());
+    auto& vertexInfos = tail.vertexInfos;
+    vertexInfos.resize(programs.size());
+    auto& decodeReads = tail.decodeReads;
+    decodeReads.resize(programs.size());
     const auto decodeVertexInfo = [&](std::size_t i) {
         const auto& program = programs[i];
         if (program.binary.stage == Stage::Fragment || roles[i] == Role::GeometryBack) return;
-        decodeReads[i].clear();
-        vertexInfos[i] = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, &decodeReads[i], false, 8u - program.firstUserSgpr);
-        if (vertexInfos[i]) vertexInfos[i]->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, readRegister(queue.context, 0x207), program.tailContinuation.get()) | graphics.layerExports;
+        vertexInfos[i] = DecodeOrderedVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, decodeReads[i], 8u - program.firstUserSgpr);
         if (vertexInfos[i]) {
+            vertexInfos[i]->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, readRegister(queue.context, 0x207), program.tailContinuation.get()) | graphics.layerExports;
             vertexInfos[i]->depthRemapMode = graphics.depthRemap.mode;
+            Graphics::ApplyPointSize(graphics, *vertexInfos[i]);
         }
     };
     if (!registerKey) {
         for (std::size_t i = 0; i < programs.size(); ++i) decodeVertexInfo(i);
         phaseTiming.Phase(DrawRowDecode);
     }
-    auto shaderMemory = std::make_unique<ShaderMemory>(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter(), &QueryPendingSpan);
-    std::vector<ShaderRecompiler::RecompileResult> results;
-    std::vector<std::shared_ptr<const ShaderRecompiler::RecompileResult>> compiledStages;
+    auto& shaderMemory = tail.shaderMemory;
+    shaderMemory = std::make_unique<ShaderMemory>(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter(), &QueryPendingSpan);
+    auto& results = tail.results;
+    auto& compiledStages = tail.compiledStages;
     compiledStages.reserve(programs.size());
-    std::vector<Graphics::CompiledShader> stages;
+    auto& stages = tail.stages;
     results.reserve(programs.size() + 2u);
     stages.reserve(programs.size());
-    std::uint32_t pushCursorBytes = 0;
+    auto& pushCursorBytes = tail.pushCursorBytes;
 
-    std::vector<const ShaderRecompiler::RecompileResult*> programResults(programs.size(), nullptr);
+    auto& programResults = tail.programResults;
+    programResults.assign(programs.size(), nullptr);
 
-    std::vector<StageCapture> stageCaptures(programs.size());
-    std::vector<std::shared_ptr<DispatchVariant>> matched(programs.size());
-    std::vector<std::vector<ShaderRecompiler::MemoryRegion>> matchedRegions(programs.size());
+    auto& stageCaptures = tail.stageCaptures;
+    stageCaptures.resize(programs.size());
+    auto& matched = tail.matched;
+    matched.resize(programs.size());
+    auto& matchedRegions = tail.matchedRegions;
+    matchedRegions.resize(programs.size());
 
-    std::vector<std::shared_ptr<DispatchVariant>> fresh(programs.size());
+    auto& fresh = tail.fresh;
+    fresh.resize(programs.size());
 
-    std::vector<bool> recompiled(programs.size(), false);
-    std::vector<std::vector<std::pair<std::shared_ptr<DispatchVariant>, std::vector<ShaderRecompiler::MemoryRegion>>>> dataCandidates(programs.size());
-    std::vector<bool> reused(programs.size(), false);
-    std::vector<StageDataHit> stageData(programs.size());
-    std::uint64_t dataStagesReused = 0, dataStagesCompiled = 0, dataVerified = 0;
-    bool drawHit = false;
-    bool verifyHit = false;
-    const bool stageEntries = useDrawEntries && !(graphics.stages.tessellation && !drawParameters.indirect && drawParameters.instanceCount > 1u);
+    auto& recompiled = tail.recompiled;
+    recompiled.assign(programs.size(), false);
+    auto& dataCandidates = tail.dataCandidates;
+    dataCandidates.resize(programs.size());
+    auto& reused = tail.reused;
+    reused.assign(programs.size(), false);
+    auto& stageData = tail.stageData;
+    stageData.resize(programs.size());
+    auto& drawHit = tail.drawHit;
+    auto& verifyHit = tail.verifyHit;
+    tail.stageEntries = useDrawEntries && !(graphics.stages.tessellation && !drawParameters.indirect && drawParameters.instanceCount > 1u);
+    const bool stageEntries = tail.stageEntries;
     lookupDraw(submission, localDevice, graphics, pixel, programs, roles, vertexInfos, stageEntries, registerKey, profile, drawKey, entry, matched, matchedRegions, drawHit, verifyHit, phaseTiming, phaseMs, dataEntry, dataCandidates, stageData);
     for (std::size_t i = 0; i < matched.size(); ++i) {
         if (matched[i] == nullptr || matched[i]->shaderDataPatches.empty()) continue;
@@ -304,8 +546,8 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         for (const auto& patch : userPatches) current->programs[patch.program].userData[patch.index] = patch.value;
         decode = std::move(current);
     }
-    const bool patchedStages = std::any_of(stageData.begin(), stageData.end(), [](const StageDataHit& hit) { return hit.compiled != nullptr; });
-    std::uint64_t stageDataVerified = 0;
+    tail.patchedStages = std::any_of(stageData.begin(), stageData.end(), [](const StageDataHit& hit) { return hit.compiled != nullptr; });
+    tail.relocatedDecode = relocatedDecode;
 
     if (registerKey) {
 
@@ -330,6 +572,137 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         phaseTiming.Phase(DrawRowDecode);
     }
 
+    tail.submission = &submission;
+    tail.submissionQueue = submission.queue;
+    tail.packetOpcode = (packet[0] >> 8u) & 0xffu;
+    tail.reg207 = readRegister(queue.context, 0x207);
+    tail.dumpTarget = dumpTarget;
+    tail.dumpSlot1 = dumpSlot1;
+    const bool handOff = pipelined && !drawParameters.indirect && !profile && !lockedPrepare && dumpTarget == 0 && dumpSlot1 == 0 && !tracePrograms && !verifyDrawDataHits() && deferredLabels().labels.empty() && DrawFinisher::Enabled();
+    if (handOff) {
+        tail.epoch = DrawPipeline::EpochToken().load(std::memory_order_relaxed);
+        tail.submissionSerial = submission.serial;
+        tail.packetOffset = static_cast<std::size_t>(packet.data() - submission.commands.data());
+        noteFinisherHandOff(true);
+        DrawFinisher::Pending held;
+        if (DrawFinisher::Overlap()) held = heldWrites(tail);
+        DrawFinisher::Get().Enqueue([this, tailHolder] {
+            auto& job = *tailHolder;
+            DrawPipeline::FollowEpoch(job.epoch);
+            GuestMemory::SetCurrentPacket(job.packetOpcode, job.submissionQueue);
+            PerformanceTimer finishTiming("Driver.DrawFinish");
+            try {
+                if (finishDraw(job, finishTiming) == DrawVerdict::Rejected) throw std::runtime_error(job.ownRejected);
+            } catch (const std::exception& error) {
+                CaptureTrace::Log("draw-error submission=%llu offset=%zu finisher=1 reason=%.256s", static_cast<unsigned long long>(job.submissionSerial), job.packetOffset, error.what());
+                throw std::runtime_error("draw at submission " + std::to_string(job.submissionSerial) + " offset " + std::to_string(job.packetOffset) + " (finished on the draw finisher): " + error.what());
+            }
+        }, std::move(held));
+        timing.Mark("draw_handed_off");
+        return drawn();
+    }
+    noteFinisherHandOff(false);
+    if (DrawFinisher::Busy()) {
+        DrawFinisher::Get().Drain();
+        if (DrawFinisher::Overlap()) DrawFinisher::NoteDrain(DrawFinisher::DrainSite::Kept, true);
+    }
+    tail.queue = &queue;
+    tail.packet = packet;
+    tail.rejected = &rejected;
+    tail.gpuLock = &gpuLock;
+    return finishDraw(tail, timing);
+}
+
+DrawVerdict Driver::finishDraw(DrawTail& tail, PerformanceTimer& timing) {
+    using Stage = ShaderRecompiler::ShaderStage;
+    using Role = ShaderRecompiler::ProgramRole;
+    const bool profile = tail.profile;
+    auto& phaseMs = tail.phaseMs;
+    auto& phaseLap = tail.phaseLap;
+    auto& captures = tail.captures;
+    auto& phaseTiming = tail.phaseTiming;
+    auto& drawParameters = tail.drawParameters;
+    const bool pipelined = tail.pipelined;
+    static const bool pipelinedIndirect = std::getenv("APS5_NO_PIPELINED_INDIRECT") == nullptr;
+    const auto drainBeforeInline = [&] {
+        if (pipelined && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Indirect);
+    };
+    auto& traceIndirect = tail.traceIndirect;
+    const auto dumpTarget = tail.dumpTarget;
+    const auto dumpSlot1 = tail.dumpSlot1;
+    auto& gpuLock = tail.gpuLock != nullptr ? *tail.gpuLock : tail.ownLock;
+    auto& localDevice = tail.localDevice;
+    const bool registerKey = tail.registerKey;
+    auto& drawKey = tail.drawKey;
+    auto& entry = tail.entry;
+    auto& decode = tail.decode;
+    auto& dataEntry = tail.dataEntry;
+    auto& shapeKey = tail.shapeKey;
+    const auto& graphics = decode->state;
+    auto& pixel = tail.pixel;
+    auto& programs = tail.programs;
+    const std::vector<Role>& roles = decode->roles;
+    auto& memory = tail.memory;
+    auto& linked = tail.linked;
+    auto& vertexInfos = tail.vertexInfos;
+    auto& decodeReads = tail.decodeReads;
+    auto& shaderMemory = tail.shaderMemory;
+    auto& results = tail.results;
+    auto& compiledStages = tail.compiledStages;
+    auto& stages = tail.stages;
+    auto& pushCursorBytes = tail.pushCursorBytes;
+    auto& programResults = tail.programResults;
+    auto& stageCaptures = tail.stageCaptures;
+    auto& matched = tail.matched;
+    auto& matchedRegions = tail.matchedRegions;
+    auto& fresh = tail.fresh;
+    auto& recompiled = tail.recompiled;
+    auto& dataCandidates = tail.dataCandidates;
+    auto& reused = tail.reused;
+    auto& stageData = tail.stageData;
+    std::uint64_t dataStagesReused = 0, dataStagesCompiled = 0, dataVerified = 0;
+    auto& drawHit = tail.drawHit;
+    auto& verifyHit = tail.verifyHit;
+    const bool stageEntries = tail.stageEntries;
+    const bool patchedStages = tail.patchedStages;
+    std::uint64_t stageDataVerified = 0;
+    const Submission& submission = *tail.submission;
+    std::string& rejected = tail.rejected != nullptr ? *tail.rejected : tail.ownRejected;
+    static const QueueState noQueue{};
+    const QueueState& queue = tail.queue != nullptr ? *tail.queue : noQueue;
+    const auto drawn = [&] {
+        phaseTiming.Phase(DrawRowVectors);
+        if (!profile) return DrawVerdict::Drawn;
+        auto& pending = pendingDrawPhases();
+        pending.phases = true;
+        pending.captures = captures;
+        pending.ms = phaseMs;
+        pending.tailAt = phaseLap;
+        return DrawVerdict::Drawn;
+    };
+    const auto setMeshIndexBuffer = [&](const Pm4::DrawParameters& parameters) {
+        if (!graphics.stages.mesh) return;
+        auto& words = programs.front().userData;
+        require(programs.front().firstUserSgpr == 0 && words.size() >= ShaderRecompiler::MeshIndexBufferUserWord + 4, "mesh program lacks the hidden user words");
+        const auto descriptor = Graphics::MeshIndexBufferDescriptor(parameters);
+        std::copy(descriptor.begin(), descriptor.end(), words.begin() + ShaderRecompiler::MeshIndexBufferUserWord);
+    };
+    const auto locate = [&](std::uint32_t location) -> std::optional<std::pair<std::size_t, std::size_t>> {
+        if (location == 0x280u) return std::nullopt;
+        for (std::size_t i = 0; i < programs.size(); ++i) {
+            if (roles[i] == Role::Fragment || roles[i] == Role::GeometryBack || location < programs[i].userDataBase) continue;
+            const auto word = location - programs[i].userDataBase + (8u - programs[i].firstUserSgpr);
+            if (word < programs[i].userData.size()) return std::make_pair(i, static_cast<std::size_t>(word));
+        }
+        return std::nullopt;
+    };
+    const auto decodeVertexInfo = [&](std::size_t i) {
+        const auto& program = programs[i];
+        if (program.binary.stage == Stage::Fragment || roles[i] == Role::GeometryBack) return;
+        decodeReads[i].clear();
+        vertexInfos[i] = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, &decodeReads[i], false, 8u - program.firstUserSgpr);
+        if (vertexInfos[i]) vertexInfos[i]->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, tail.reg207, program.tailContinuation.get()) | graphics.layerExports;
+    };
     const auto fold = [&](const ShaderRecompiler::RecompileResult& main, Pm4::DrawParameters& parameters) { FoldDrawOffsets(main, programs.front(), parameters); };
 
     const auto freshRuns = [&](std::size_t i, std::vector<std::pair<std::uint64_t, std::uint64_t>>& runs, std::vector<std::uint32_t>& words) {
@@ -414,6 +787,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         const auto& result = *programResults[i];
         if (i == 0 && drawParameters.indirect) {
             indirectCpu = ClassifyIndirectDraw(result, graphics, programs.front(), localDevice, drawParameters, traceIndirect);
+            if (EmptyIndirectSkip()) CpuIndirectPrograms()[programs.front().binary.codeAddress] = indirectCpu.has_value();
         } else if (i == 0) {
             fold(result, drawParameters);
         }
@@ -433,15 +807,36 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         drawEntryCounters.dataStagesCompiled += dataStagesCompiled;
         drawEntryCounters.dataVerified += dataVerified;
     }
-    cacheDrawStages(stageEntries, drawHit, drawParameters, indirectCpu, programs, stageCaptures, vertexInfos, decodeReads, verifyHit, matched, fresh, drawKey, registerKey, decode, phaseTiming, reused, entry, shapeKey);
+    const bool shapeRefresh = Graphics::DrawPipelineEnabled();
+    const auto refreshSeenTwice = [&] {
+        std::uint64_t data = shapeKey;
+        bool stored = false;
+        const auto mix = [&data](std::uint64_t value) {
+            data ^= value + 0x9e3779b97f4a7c15ull + (data << 6u) + (data >> 2u);
+        };
+        for (std::size_t i = 0; i < programs.size(); ++i) {
+            const auto& capture = stageCaptures[i];
+            if (reused[i] || capture.compiled == nullptr || !CacheableResult(*capture.compiled)) continue;
+            stored = true;
+            const auto& words = programs[i].userData;
+            for (std::size_t w = 0; w < words.size(); ++w) {
+                if (w >= 64u || (capture.userDataMask >> w & 1u) != 0) mix((static_cast<std::uint64_t>(i) << 40u) ^ (static_cast<std::uint64_t>(w) << 32u) ^ words[w]);
+            }
+        }
+        if (!stored) return false;
+        static std::array<std::atomic<std::uint64_t>, 4096> lastData{};
+        return lastData[shapeKey % lastData.size()].exchange(data, std::memory_order_relaxed) == data;
+    };
+    const bool shapeNeedsEntry = shapeRefresh && registerKey && shapeKey != 0 && drawDataHits() && !drawParameters.indirect && dataEntry != nullptr && dataStagesCompiled != 0 && refreshSeenTwice();
+    cacheDrawStages(stageEntries, drawHit, drawParameters, indirectCpu, programs, stageCaptures, vertexInfos, decodeReads, verifyHit, matched, stageData, fresh, drawKey, registerKey, decode, phaseTiming, reused, entry, shapeKey, shapeNeedsEntry);
     timing.Mark("shader_compile_and_link");
 
     for (const auto& reads : decodeReads) {
         for (const auto& read : reads) memory.push_back({read.address, read.Bytes()});
     }
 
-    if (pipelined && !deferredLabels().labels.empty()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Labels);
-    if (recordQueuedLabelsAfterCapture(submission.queue, memory)) return draw(queue, packet, submission, rejected);
+    if (tail.queue != nullptr && pipelined && !deferredLabels().labels.empty()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Labels);
+    if (tail.queue != nullptr && recordQueuedLabelsAfterCapture(submission.queue, memory)) return draw(*tail.queue, tail.packet, submission, rejected);
 
     bool rectListBuilt = false;
 
@@ -480,14 +875,14 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
             require(programResults[0]->pushConstants.size() == pushBytes && programResults[0]->parameterExports == exports, "pairing the mesh program with its pixel shader changed the push constants or the parameter exports the pixel shader was built for");
         }
         require(programResults[0]->meshBarycentrics == meshBarycentrics, "the mesh program was compiled for a different barycentric layout than its pixel shader");
-    } else if (!programs.empty() && programResults.back() != nullptr && programResults.back()->barycentricEmulation.active) {
-        require(!graphics.rectList && graphics.stages.path == Graphics::ShaderPath::Vertex && programs.size() == 2 && programResults[0] != nullptr && stages.size() == 2, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric needs a vertex shader before it");
+    } else if (!graphics.rectList && !programs.empty() && programResults.back() != nullptr && programResults.back()->barycentricEmulation.active) {
+        require(graphics.stages.path == Graphics::ShaderPath::Vertex && programs.size() == 2 && programResults[0] != nullptr && stages.size() == 2, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric needs a vertex shader before it");
         const auto geometryLimits = localDevice->GeometryLimits();
         if (geometryLimits || !localDevice->Target().tessellation) {
             results.push_back(ShaderRecompiler::BuildBarycentricGeometryShader(*programResults[0], *programResults[1], localDevice->Target(), geometryLimits));
             stages.insert(stages.begin() + 1, Graphics::CompiledShader{Stage::Geometry, &results.back(), 0});
         } else {
-            require(graphics.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST && !graphics.primitiveRestart, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric or geometry shaders needs a triangle list without primitive restart");
+            require((graphics.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST || graphics.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP) && !graphics.primitiveRestart, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric or geometry shaders needs a triangle list or strip without primitive restart");
             auto patches = ShaderRecompiler::BuildBarycentricTessellationShaders(*programResults[0], *programResults[1], localDevice->Target(), localDevice->MaxFragmentInputComponents(), graphics.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT ? 2u : 0u);
             const auto first = results.size();
             results.push_back(std::move(patches.control));
@@ -504,6 +899,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     snapshot();
     timing.Mark("post_compile_prepare");
     const auto lockForDraw = [&] {
+        require(tail.queue != nullptr, "the draw finisher reached a draw it cannot commit through the pipeline");
         if (gpuLock.owns_lock()) return;
         phaseTiming.Phase(DrawRowVectors);
         GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw);
@@ -553,7 +949,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
             std::set<std::size_t> patched;
             const auto patch = [&](const std::vector<std::pair<std::size_t, std::size_t>>& words, std::uint32_t value) {
                 for (const auto& [program, word] : words) {
-                    writablePrograms[program].userData[word] = value;
+                    programs[program].userData[word] = value;
                     patched.insert(program);
                 }
             };
@@ -598,7 +994,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
                 phaseTiming.Phase(DrawRowVectors);
                 auto writes = drawWriteRanges(graphics, stages);
                 const auto commitQueue = submission.queue;
-                const auto commitPacket = (packet[0] >> 8u) & 0xffu;
+                const auto commitPacket = tail.packetOpcode;
                 const auto epoch = DrawPipeline::EpochToken().load(std::memory_order_relaxed);
                 auto commit = [this, localDevice = std::move(localDevice), decode = std::move(decode), direct, stages = std::move(stages), snapshots = std::move(snapshots), commitQueue, commitPacket, epoch, programs = std::move(programs), results = std::move(results), memory = std::move(memory), linked = std::move(linked), stageCaptures = std::move(stageCaptures), matched = std::move(matched), matchedRegions = std::move(matchedRegions), fresh = std::move(fresh), decodeReads = std::move(decodeReads), shaderMemory = std::move(shaderMemory), entry = std::move(entry), dataEntry = std::move(dataEntry), compiledStages = std::move(compiledStages), stageData = std::move(stageData)] {
                     DrawPipeline::FollowEpoch(epoch);
@@ -639,20 +1035,20 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
             return DrawVerdict::Rejected;
         }
     }
-    if (pipelined && !drawParameters.indirect && deferredLabels().labels.empty()) {
+    if (pipelined && !drawParameters.indirect && (tail.queue == nullptr || deferredLabels().labels.empty())) {
         phaseTiming.Phase(DrawRowVectors);
         auto writes = drawWriteRanges(graphics, stages);
         const auto commitQueue = submission.queue;
-        const auto commitPacket = (packet[0] >> 8u) & 0xffu;
-        const auto epoch = DrawPipeline::EpochToken().load(std::memory_order_relaxed);
+        const auto commitPacket = tail.packetOpcode;
+        const auto epoch = DrawFinisher::OnFinisherThread() ? tail.epoch : DrawPipeline::EpochToken().load(std::memory_order_relaxed);
         auto reads = QueuedDrawReadRanges(drawParameters, stages);
-        auto commit = [this, localDevice = std::move(localDevice), decode = std::move(decode), drawParameters, stages = std::move(stages), snapshots = std::move(snapshots), recipe = std::move(recipe), recipeStages = std::move(recipeStages), drawKey, commitQueue, commitPacket, epoch, programs = std::move(programs), results = std::move(results), memory = std::move(memory), linked = std::move(linked), stageCaptures = std::move(stageCaptures), matched = std::move(matched), matchedRegions = std::move(matchedRegions), fresh = std::move(fresh), decodeReads = std::move(decodeReads), shaderMemory = std::move(shaderMemory), entry = std::move(entry), dataEntry = std::move(dataEntry), compiledStages = std::move(compiledStages), stageData = std::move(stageData)] {
+        auto commit = [this, localDevice = std::move(localDevice), decode = std::move(decode), drawParameters, stages = std::move(stages), snapshots = std::move(snapshots), recipe = std::move(recipe), recipeStages = std::move(recipeStages), drawKey, commitQueue, commitPacket, epoch, programs = std::move(programs), results = std::move(results), memory = std::move(memory), linked = std::move(linked), stageCaptures = std::move(stageCaptures), matched = std::move(matched), matchedRegions = std::move(matchedRegions), fresh = std::move(fresh), decodeReads = std::move(decodeReads), shaderMemory = std::move(shaderMemory), entry = std::move(entry), dataEntry = std::move(dataEntry), compiledStages = std::move(compiledStages), stageData = std::move(stageData)]() mutable {
             DrawPipeline::FollowEpoch(epoch);
             GuestMemory::SetCurrentPacket(commitPacket, commitQueue);
             commitDraw(localDevice, commitQueue, decode->state, drawParameters, stages, snapshots, recipe, recipeStages, drawKey);
         };
         auto held = std::make_shared<decltype(commit)>(std::move(commit));
-        DrawPipeline::Queue0().Enqueue([held] { (*held)(); }, std::move(writes), 0, {}, std::move(reads));
+        DrawPipeline::Queue0().Enqueue([held] { (*held)(); }, std::move(writes), 0, {}, std::move(reads), true);
         phaseTiming.Phase(DrawRowLockWait);
         timing.Mark("draw_enqueued");
         return drawn();

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp

@@ -123,10 +123,24 @@ bool Driver::admitDrawKey(std::uint64_t key, std::uint64_t shape) {
     return drawShapeSightings->SeenBefore(shape);
 }
 
-void Driver::insertDrawEntry(std::uint64_t key, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::shared_ptr<const DrawDecode> decode, std::uint64_t shape) {
+void Driver::insertDrawEntry(std::uint64_t key, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::shared_ptr<const DrawDecode> decode, std::uint64_t shape, bool replaceShape) {
     std::lock_guard cacheLock(drawCacheMutex);
     auto& counters = drawEntryCounters;
-    ++counters.inserts;
+    if (replaceShape && shape != 0) {
+        ++counters.shapeUpdates;
+        if (const auto shapeOf = drawShapes.find(shape); shapeOf != drawShapes.end() && shapeOf->second != key) {
+            if (const auto previous = drawCache.find(shapeOf->second); previous != drawCache.end()) {
+                for (const auto& variants : previous->second->stages) {
+                    for (const auto& variant : variants) accountDrawVariant(*variant, false);
+                }
+                drawOrder.erase(previous->second->order);
+                drawCache.erase(previous);
+            }
+            drawShapes.erase(shapeOf);
+        }
+    } else {
+        ++counters.inserts;
+    }
     const auto found = drawCache.find(key);
     auto replacement = std::make_shared<DrawEntry>();
     replacement->stages.resize(fresh.size());

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCacheStatistics.cpp

@@ -62,7 +62,7 @@ std::string Driver::profileDrawCacheLines(const DrawEntryCounters& counters) {
         std::snprintf(item, sizeof(item), "%s%llu", rank == 0 ? "" : " / ", count(counters.variantHitsByRank[rank]));
         ranks += item;
     }
-    append("[draw-cache] %llu lookups (10 s): %llu no entry, %llu validated in %.1f us each (key, lookup and every stage): %llu hits (every stage equal; stage hits by variant rank 1..k %s), misses by reason: front stage differing %llu, fragment differing %llu, other stage differing %llu, layout (push offset) %llu, gate %llu, stage count %llu; %llu stage validations (%llu equal, %.2f variants compared each); %llu inserts (%llu variants inserted, %llu evicted beyond k, %llu already present, %llu unstable), %zu entries (%llu variants, ~%.1f MiB), %llu evictions in total, %llu LRU moves; verify: %llu hits captured again, %llu stages differed\n", count(counters.lookups), count(counters.absent), count(validated), validated != 0 ? counters.validateUs / static_cast<double>(validated) : 0.0, count(counters.hits), ranks.c_str(), miss(DrawMiss::FrontDiffering), miss(DrawMiss::FragmentDiffering), miss(DrawMiss::OtherDiffering), miss(DrawMiss::Layout), miss(DrawMiss::Gate), miss(DrawMiss::Stages), count(counters.stageValidations), count(counters.stageEqual), counters.stageValidations != 0 ? static_cast<double>(counters.variantsCompared) / static_cast<double>(counters.stageValidations) : 0.0, count(counters.inserts), count(counters.variantsInserted), count(counters.variantsEvicted), count(counters.present), count(counters.unstable), drawCache.size(), count(drawCacheVariants), static_cast<double>(drawCacheVariantBytes) / (1024.0 * 1024.0), count(drawCacheEvictions), count(counters.touches), count(counters.verifyHits), count(counters.verifyMismatches));
+    append("[draw-cache] %llu lookups (10 s): %llu no entry, %llu validated in %.1f us each (key, lookup and every stage): %llu hits (every stage equal; stage hits by variant rank 1..k %s), misses by reason: front stage differing %llu, fragment differing %llu, other stage differing %llu, layout (push offset) %llu, gate %llu, stage count %llu; %llu stage validations (%llu equal, %.2f variants compared each); %llu inserts (%llu shape updates, %llu variants inserted, %llu evicted beyond k, %llu already present, %llu unstable), %zu entries (%llu variants, ~%.1f MiB), %llu evictions in total, %llu LRU moves; verify: %llu hits captured again, %llu stages differed\n", count(counters.lookups), count(counters.absent), count(validated), validated != 0 ? counters.validateUs / static_cast<double>(validated) : 0.0, count(counters.hits), ranks.c_str(), miss(DrawMiss::FrontDiffering), miss(DrawMiss::FragmentDiffering), miss(DrawMiss::OtherDiffering), miss(DrawMiss::Layout), miss(DrawMiss::Gate), miss(DrawMiss::Stages), count(counters.stageValidations), count(counters.stageEqual), counters.stageValidations != 0 ? static_cast<double>(counters.variantsCompared) / static_cast<double>(counters.stageValidations) : 0.0, count(counters.inserts), count(counters.shapeUpdates), count(counters.variantsInserted), count(counters.variantsEvicted), count(counters.present), count(counters.unstable), drawCache.size(), count(drawCacheVariants), static_cast<double>(drawCacheVariantBytes) / (1024.0 * 1024.0), count(drawCacheEvictions), count(counters.touches), count(counters.verifyHits), count(counters.verifyMismatches));
     append("[draw-cache] register key (10 s): %llu lookups, %llu hits, key build %.1f us each; decode skipped %llu, partial (state/pixel/programs from the entry) %llu; facade log != table %llu; verify: %llu decodes compared, %llu differed\n", count(counters.registerKeyLookups), count(counters.registerKeyHits), counters.registerKeyLookups != 0 ? counters.keyUs / static_cast<double>(counters.registerKeyLookups) : 0.0, count(counters.decodeSkipped), count(counters.decodePartial), count(counters.facadeMismatches), count(counters.verifyDecodes), count(counters.verifyDecodeMismatches));
     append("[draw-cache] stage data hits (10 s): %llu stages (%llu words refreshed), verified %llu; stage inserts with data positions %llu (%.1f positions each)\n", count(counters.stageDataHits), count(counters.stageDataWords), count(counters.stageDataVerified), count(counters.stageDataInserts), counters.stageDataInserts != 0 ? static_cast<double>(counters.stageDataPositions) / static_cast<double>(counters.stageDataInserts) : 0.0);
     return text;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp

@@ -152,8 +152,11 @@ void Driver::recordStageDataPositions(DispatchVariant& variant, const ShaderReco
     variant.dataSlots.resize(kept);
 }
 
-void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawParameters& drawParameters, const std::optional<Graphics::IndirectDrawPath>& indirectCpu, const std::vector<DrawProgram>& programs, const std::vector<StageCapture>& stageCaptures, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, const std::vector<std::vector<Graphics::DecodeRead>>& decodeReads, bool verifyHit, const std::vector<std::shared_ptr<DispatchVariant>>& matched, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::uint64_t drawKey, bool registerKey, const std::shared_ptr<const DrawDecode>& decode, DrawPhaseTiming& phaseTiming, const std::vector<bool>& reused, const std::shared_ptr<DrawEntry>& entry, std::uint64_t shapeKey) {
-    if (useDrawEntries && !drawHit && !(drawParameters.indirect && indirectCpu) && (verifyHit || admitDrawKey(drawKey, registerKey && drawDataHits() ? shapeKey : 0))) {
+void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawParameters& drawParameters, const std::optional<Graphics::IndirectDrawPath>& indirectCpu, const std::vector<DrawProgram>& programs, const std::vector<StageCapture>& stageCaptures, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, const std::vector<std::vector<Graphics::DecodeRead>>& decodeReads, bool verifyHit, const std::vector<std::shared_ptr<DispatchVariant>>& matched, const std::vector<StageDataHit>& stageData, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::uint64_t drawKey, bool registerKey, const std::shared_ptr<const DrawDecode>& decode, DrawPhaseTiming& phaseTiming, const std::vector<bool>& reused, const std::shared_ptr<DrawEntry>& entry, std::uint64_t shapeKey, bool shapeNeedsEntry) {
+    const bool cacheable = useDrawEntries && !drawHit && !(drawParameters.indirect && indirectCpu);
+    const bool admitted = cacheable && (verifyHit || admitDrawKey(drawKey, registerKey && drawDataHits() ? shapeKey : 0));
+    const bool replaceShape = cacheable && !admitted && shapeNeedsEntry;
+    if (admitted || replaceShape) {
         phaseTiming.Phase(DrawRowVectors);
         std::uint64_t unstable = 0, mismatches = 0, differingSameRuns = 0, differingWords = 0, differingRunsChanged = 0;
         for (std::size_t i = 0; i < programs.size(); ++i) {
@@ -189,17 +192,17 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP
                     ++differingRunsChanged;
                 }
             }
-            const auto sameContent = [](const DispatchVariant& a, const DispatchVariant& b) {
-                const auto flatten = [](const DispatchVariant& source) {
+            const auto sameContent = [](const DispatchVariant& a, const std::vector<std::uint32_t>& words, const DispatchVariant& b) {
+                const auto flatten = [](const DispatchVariant& source, const std::vector<std::uint32_t>& values) {
                     std::vector<std::pair<std::uint64_t, std::uint32_t>> flat;
                     std::size_t word = 0;
                     for (const auto& [begin, end] : source.runs) {
-                        for (auto address = begin; address < end && word < source.words.size(); address += sizeof(std::uint32_t)) flat.emplace_back(address, source.words[word++]);
+                        for (auto address = begin; address < end && word < values.size(); address += sizeof(std::uint32_t)) flat.emplace_back(address, values[word++]);
                     }
                     std::sort(flat.begin(), flat.end());
                     return flat;
                 };
-                return (a.runs == b.runs && a.words == b.words) || flatten(a) == flatten(b);
+                return (a.runs == b.runs && words == b.words) || flatten(a, words) == flatten(b, b.words);
             };
             const auto sameForwardedWords = [](const ShaderRecompiler::RecompileResult& a, const ShaderRecompiler::RecompileResult& b, std::uint32_t hidden) {
                 if (a.pushConstants.size() != b.pushConstants.size() || a.shaderDataUserWords != b.shaderDataUserWords) return false;
@@ -208,10 +211,14 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP
                 }
                 return true;
             };
-            if (verifyHit && matched[i] != nullptr && (!sameContent(*matched[i], *variant) || !sameForwardedWords(*matched[i]->compiled, *variant->compiled, programs[i].hiddenUserWords))) {
-                ++mismatches;
-                static std::atomic<std::uint64_t> reports{0};
-                if (reports.fetch_add(1) < 20) std::fprintf(stderr, "[draw-cache] verify: stage %zu (program 0x%llx) of a hit captured differently: %zu runs / %zu words matched, %zu / %zu fresh\n", i, static_cast<unsigned long long>(programs[i].binary.codeAddress), matched[i]->runs.size(), matched[i]->words.size(), variant->runs.size(), variant->words.size());
+            if (verifyHit && matched[i] != nullptr) {
+                const auto& words = stageData[i].compiled != nullptr ? stageData[i].words : matched[i]->words;
+                const auto& compiled = stageData[i].compiled != nullptr ? *stageData[i].compiled : *matched[i]->compiled;
+                if (!sameContent(*matched[i], words, *variant) || !sameForwardedWords(compiled, *variant->compiled, programs[i].hiddenUserWords)) {
+                    ++mismatches;
+                    static std::atomic<std::uint64_t> reports{0};
+                    if (reports.fetch_add(1) < 20) std::fprintf(stderr, "[draw-cache] verify: stage %zu (program 0x%llx) of a hit captured differently: %zu runs / %zu words matched, %zu / %zu fresh\n", i, static_cast<unsigned long long>(programs[i].binary.codeAddress), matched[i]->runs.size(), words.size(), variant->runs.size(), variant->words.size());
+                }
             }
             if (insertCompare()) {
                 const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DrawCache);
@@ -222,7 +229,7 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP
             }
             fresh[i] = std::move(variant);
         }
-        insertDrawEntry(drawKey, fresh, registerKey ? decode : nullptr, registerKey && drawDataHits() ? shapeKey : 0);
+        insertDrawEntry(drawKey, fresh, registerKey ? decode : nullptr, registerKey && drawDataHits() ? shapeKey : 0, replaceShape);
         std::uint64_t dataInserts = 0, dataPositions = 0;
         for (std::size_t i = 0; i < programs.size(); ++i) {
             if (fresh[i] == nullptr || reused[i] || fresh[i]->dataPositions.empty()) continue;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp

@@ -1,6 +1,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include <algorithm>
 #include <spirv/unified1/spirv.hpp>
@@ -30,6 +31,25 @@ void ReadUserWords(const QueueState& queue, DrawProgram& program, bool staticAbi
     if ((program.binary.stage == ShaderRecompiler::ShaderStage::Vertex || program.binary.stage == ShaderRecompiler::ShaderStage::Fragment) && program.snapshot->header.empty() && program.snapshot->userDataTailRegister) program.tailContinuation = ReadUserDataTailContinuation(program.binary.code, program.firstUserSgpr, program.userData);
 }
 
+ShaderRecompiler::ShaderVertexStageInfo DecodeOrderedVertexStageInfo(std::span<const std::byte> header, std::uint64_t headerAddress, std::span<const std::uint32_t> userData, std::vector<Graphics::DecodeRead>& reads, std::uint32_t hiddenUserWords) {
+    const auto decode = [&] {
+        reads.clear();
+        return Graphics::DecodeVertexStageInfo(header, headerAddress, userData, &reads, false, hiddenUserWords);
+    };
+    if (!DrawPipeline::Active()) return decode();
+    auto& queue = DrawPipeline::Queue0();
+    const auto watch = queue.WatchWrites();
+    auto info = decode();
+    if (watch.items == 0 && !queue.Busy() && queue.Retired() == watch.retired) return info;
+    std::vector<DrawPipeline::Range> ranges;
+    ranges.reserve(reads.size());
+    for (const auto& read : reads) ranges.emplace_back(read.address, read.address + read.size);
+    if (!queue.WrittenSince(watch, ranges)) return info;
+    queue.NoteDecodeDrain();
+    queue.Drain(DrawPipeline::DrainReason::Capture);
+    return decode();
+}
+
 void InitializeMerged(const QueueState& queue, DrawProgram& program, std::uint32_t pointerBase, bool pointerRequired, bool staticAbi) {
     program.merged = true;
     program.mergedPointer = pointerBase;
@@ -165,6 +185,7 @@ std::shared_ptr<DrawDecode> Driver::decodeDraw(const QueueState& queue, const Su
     product->pixel.targetExportPacking = Graphics::ExportPackings(product->state);
     product->pixel.dualSourceBlend = product->state.dualSourceBlend;
     product->pixel.reversedBlend = product->state.reversedBlend;
+    product->pixel.targetMerge = Graphics::ExportTargetMerge(product->state);
     product->pixel.alphaToCoverage = product->state.alphaToCoverage;
     product->pixel.quadPixelMask = Graphics::QuadPixelMask(product->state);
     if (product->state.sampleLocationGrid) {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp

@@ -2,6 +2,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include <algorithm>
+#include <bit>
 
 namespace AgcDriver::DriverDetail {
 
@@ -51,6 +52,10 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptr<Vulk
                 mix(vertex.fetchBufferReg);
                 mix(vertex.fetchEmbedded);
                 mix(vertex.paClVsOutCntl);
+                mix(vertex.pointList);
+                mix(std::bit_cast<std::uint32_t>(vertex.pointSize));
+                mix(std::bit_cast<std::uint32_t>(vertex.pointSizeMin));
+                mix(std::bit_cast<std::uint32_t>(vertex.pointSizeMax));
                 for (std::uint32_t r = 0; r < vertex.resourcesNum; ++r) {
                     for (const auto field : vertex.resources[r].fields) mix(field);
                     const auto& destination = vertex.resourcesDst[r];
@@ -74,6 +79,12 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptr<Vulk
             for (const auto value : pixel.targetExportPacking) mix(static_cast<std::uint64_t>(value));
             mix(pixel.dualSourceBlend);
             mix(pixel.reversedBlend);
+            mix(pixel.targetMerge.target);
+            mix(pixel.targetMerge.mask);
+            mix(pixel.targetMerge.samples);
+            for (const auto word : pixel.targetMerge.descriptor) mix(word);
+            mix(pixel.targetMerge.depthStencilWrites);
+            mix(pixel.targetMerge.depthStencilTests);
             mix(pixel.alphaToCoverage);
             mix(static_cast<std::uint64_t>(pixel.rasterPrimitive));
             mix(pixel.ditheredAlphaToCoverage);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp

@@ -4,7 +4,11 @@
 #include "prx/libSceAgcDriver/Execution/include/HandoffSpin.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp"
 #include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp"
+#include "prx/libc/include/HostThreadLocal.hpp"
 #include <algorithm>
+#include <array>
+#include <limits>
+#include <string>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
@@ -29,6 +33,212 @@ std::atomic<std::uint64_t> heldReadHits{0};
 
 }
 
+namespace {
+thread_local bool finisherThread = false;
+thread_local bool finisherProducer = false;
+}
+
+DrawFinisher& DrawFinisher::Get() {
+    static DrawFinisher* finisher = new DrawFinisher();
+    return *finisher;
+}
+
+bool DrawFinisher::Enabled() {
+    return Graphics::DrawPipelineEnabled();
+}
+
+bool DrawPipeline::MetadataPassesPipelined() {
+    return Graphics::DrawPipelineEnabled();
+}
+
+namespace {
+std::atomic<std::uint64_t> pipelinedMetadataPasses{0};
+}
+
+std::uint64_t DrawPipeline::PipelinedMetadataPasses() {
+    return pipelinedMetadataPasses.load(std::memory_order_relaxed);
+}
+
+void DrawPipeline::CountPipelinedMetadataPass() {
+    pipelinedMetadataPasses.fetch_add(1, std::memory_order_relaxed);
+}
+
+bool DrawFinisher::Overlap() {
+    return Graphics::DrawPipelineEnabled();
+}
+
+bool DrawFinisher::PendingOverlaps(std::uint64_t begin, std::uint64_t end) {
+    if (!Busy() || !finisherProducer) return false;
+    std::lock_guard lock(mutex);
+    for (const auto& held : pending) {
+        if (held.unknown) return true;
+        for (const auto& [first, last] : held.writes) {
+            if (begin < last && first < end) return true;
+        }
+    }
+    return false;
+}
+
+DrawFinisher::LabelLookup DrawFinisher::PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t& value) {
+    if (!Busy() || !finisherProducer) return LabelLookup::NotHeld;
+    const auto end = address + bytes;
+    std::lock_guard lock(mutex);
+    for (auto held = pending.rbegin(); held != pending.rend(); ++held) {
+        if (held->unknown) return LabelLookup::Unknown;
+        const bool overlaps = std::any_of(held->writes.begin(), held->writes.end(), [&](const Range& range) { return address < range.second && range.first < end; });
+        if (!overlaps) continue;
+        if (held->labelBytes.empty() || address < held->labelAddress || end > held->labelAddress + held->labelBytes.size()) return LabelLookup::Unknown;
+        value = 0;
+        std::memcpy(&value, held->labelBytes.data() + (address - held->labelAddress), bytes);
+        return LabelLookup::Value;
+    }
+    return LabelLookup::NotHeld;
+}
+
+bool DrawFinisher::PendingOverlapsAny(std::span<const Range> ranges) {
+    if (!Busy() || !finisherProducer) return false;
+    std::lock_guard lock(mutex);
+    for (const auto& held : pending) {
+        if (held.unknown) return true;
+        for (const auto& [first, last] : held.writes) {
+            for (const auto& [begin, end] : ranges) {
+                if (begin < last && first < end) return true;
+            }
+        }
+    }
+    return false;
+}
+
+bool DrawFinisher::PendingKey(std::uint64_t key, std::uint64_t shape) {
+    if (!Busy() || !finisherProducer) return false;
+    std::lock_guard lock(mutex);
+    return std::any_of(pending.begin(), pending.end(), [&](const Pending& held) { return held.key == key || (shape != 0 && held.shape == shape); });
+}
+
+namespace {
+std::array<std::atomic<std::uint64_t>, static_cast<std::size_t>(DrawFinisher::DrainSite::Count)> finisherDrains{}, finisherSkips{};
+}
+
+void DrawFinisher::NoteDrain(DrainSite site, bool drained) {
+    (drained ? finisherDrains : finisherSkips)[static_cast<std::size_t>(site)].fetch_add(1, std::memory_order_relaxed);
+}
+
+namespace {
+std::array<std::atomic<std::uint64_t>, 257> drainedOpcodes{}, drainedBeside{};
+}
+
+void DrawFinisher::NoteDrainedOpcode(std::uint32_t opcode, bool besideCommits) {
+    drainedOpcodes[std::min<std::uint32_t>(opcode, 256u)].fetch_add(1, std::memory_order_relaxed);
+    if (besideCommits) drainedBeside[std::min<std::uint32_t>(opcode, 256u)].fetch_add(1, std::memory_order_relaxed);
+}
+
+std::string DrawFinisher::DrainText() {
+    constexpr std::array<const char*, static_cast<std::size_t>(DrainSite::Count)> names{"lookup", "reads", "packet", "kept draw"};
+    std::string text = "; worker drains of the finisher (drained/skipped):";
+    for (std::size_t i = 0; i < names.size(); ++i) {
+        char item[64];
+        std::snprintf(item, sizeof(item), " %s %llu/%llu", names[i], static_cast<unsigned long long>(finisherDrains[i].exchange(0)), static_cast<unsigned long long>(finisherSkips[i].exchange(0)));
+        text += item;
+    }
+    text += "; drained by opcode (of them beside commits):";
+    for (int shown = 0; shown < 6; ++shown) {
+        std::size_t top = 0;
+        for (std::size_t i = 1; i < drainedOpcodes.size(); ++i) {
+            if (drainedOpcodes[i].load() > drainedOpcodes[top].load()) top = i;
+        }
+        const auto count = drainedOpcodes[top].exchange(0);
+        if (count == 0) break;
+        char item[48];
+        std::snprintf(item, sizeof(item), " 0x%02zx x%llu (%llu)", top, static_cast<unsigned long long>(count), static_cast<unsigned long long>(drainedBeside[top].exchange(0)));
+        text += item;
+    }
+    return text;
+}
+
+bool DrawFinisher::OnFinisherThread() {
+    return finisherThread;
+}
+
+void DrawFinisher::Enqueue(Job job) {
+    Enqueue(std::move(job), Pending{});
+}
+
+void DrawFinisher::Enqueue(Job job, Pending held) {
+    finisherProducer = true;
+    constexpr std::size_t depth = 16;
+    struct ReclaimStorage {};
+    auto& reclaim = HostThreadLocal<std::vector<Job>, ReclaimStorage>();
+    reclaim.clear();
+    std::unique_lock lock(mutex);
+    if (!ran.empty()) std::swap(ran, reclaim);
+    if (failure != nullptr) std::rethrow_exception(std::exchange(failure, nullptr));
+    if (jobs.size() >= depth && GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw finisher full while the enqueuing thread holds the GPU lock");
+    idle.wait(lock, [&] { return jobs.size() < depth; });
+    jobs.push_back(std::move(job));
+    pending.push_back(std::move(held));
+    outstanding.fetch_add(1, std::memory_order_release);
+    if (!running) {
+        running = true;
+        thread = std::thread([this] {
+            finisherThread = true;
+            run();
+        });
+    }
+    if (sleeping) wake.notify_one();
+}
+
+void DrawFinisher::run() {
+    PerformanceTrace::BindThread("draw finisher");
+    for (;;) {
+        Job* job = nullptr;
+        bool failed = false;
+        SpinUntil(HandoffSpinWindow(), [&] { return outstanding.load(std::memory_order_acquire) != 0; });
+        {
+            std::unique_lock lock(mutex);
+            sleeping = true;
+            wake.wait(lock, [&] { return !jobs.empty(); });
+            sleeping = false;
+            job = &jobs.front();
+            failed = failure != nullptr;
+        }
+        std::exception_ptr error;
+        try {
+            if (!failed) (*job)();
+        } catch (...) {
+            error = std::current_exception();
+        }
+        std::lock_guard lock(mutex);
+        if (error != nullptr && failure == nullptr) failure = error;
+        ran.push_back(std::move(jobs.front()));
+        jobs.pop_front();
+        pending.pop_front();
+        outstanding.fetch_sub(1, std::memory_order_release);
+        idle.notify_all();
+    }
+}
+
+void DrawFinisher::Settle() {
+    if (!Busy() || OnFinisherThread()) return;
+    if (GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw finisher settle under the GPU lock");
+    std::unique_lock lock(mutex);
+    idle.wait(lock, [&] { return jobs.empty(); });
+}
+
+void DrawFinisher::Drain() {
+    if (OnFinisherThread()) return;
+    if (Busy() && GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw finisher drain under the GPU lock");
+    SpinUntil(HandoffSpinWindow() * 2, [&] { return outstanding.load(std::memory_order_acquire) == 0; });
+    std::unique_lock lock(mutex);
+    if (!jobs.empty()) idle.wait(lock, [&] { return jobs.empty(); });
+    if (failure != nullptr) std::rethrow_exception(std::exchange(failure, nullptr));
+}
+
+void DrawFinisher::ReleaseRan() {
+    std::vector<Job> done;
+    std::lock_guard lock(mutex);
+    std::swap(done, ran);
+}
+
 DrawPipeline& DrawPipeline::Queue0() {
     static DrawPipeline* pipeline = new DrawPipeline();
     return *pipeline;
@@ -78,9 +288,16 @@ void DrawPipeline::rethrowFailure() {
     std::rethrow_exception(error);
 }
 
-void DrawPipeline::Enqueue(Commit commit, std::vector<Range> writes, std::uint64_t labelAddress, std::vector<std::byte> labelBytes, std::vector<Range> reads) {
+void DrawPipeline::Enqueue(Commit commit, std::vector<Range> writes, std::uint64_t labelAddress, std::vector<std::byte> labelBytes, std::vector<Range> reads, bool draw) {
+    if (DrawFinisher::Busy() && !DrawFinisher::OnFinisherThread()) DrawFinisher::Get().Drain();
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
+    struct ReclaimStorage {};
+    auto& reclaim = HostThreadLocal<std::vector<Item>, ReclaimStorage>();
+    reclaim.clear();
+    const auto lockStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
     std::unique_lock lock(mutex);
+    if (!retired.empty()) std::swap(retired, reclaim);
+    if (profile) lockWaitNs += elapsedNs(lockStart);
     rethrowFailure();
     if (closing) {
         lock.unlock();
@@ -90,23 +307,33 @@ void DrawPipeline::Enqueue(Commit commit, std::vector<Range> writes, std::uint64
     if (items.size() >= Depth()) {
         if (GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw pipeline full while the enqueuing thread holds the GPU lock");
         const auto start = std::chrono::steady_clock::now();
+        resumeAt = Depth() - 1;
         ++idleWaiters;
-        idle.wait(lock, [&] { return items.size() < Depth(); });
+        idle.wait(lock, [&] { return items.size() <= resumeAt; });
         --idleWaiters;
+        resumeAt = 0;
         fullWaitNs += elapsedNs(start);
         rethrowFailure();
     }
     depthSum += items.size();
     ++enqueued;
     heldReadRanges.fetch_add(reads.size(), std::memory_order_relaxed);
-    items.push_back({std::move(commit), std::move(writes), labelAddress, std::move(labelBytes), std::move(reads)});
+    countWrites(writes, true);
+    items.push_back({std::move(commit), std::move(writes), labelAddress, std::move(labelBytes), std::move(reads), draw});
     outstanding.fetch_add(1, std::memory_order_release);
     if (!thread.joinable()) {
-        thread = std::thread([this] { run(); });
+        thread = std::thread([this] {
+            GuestMemory::MarkProofThread();
+            run();
+        });
         runningCommitters.fetch_add(1, std::memory_order_acq_rel);
     }
-    if (committerWaiting) wake.notify_one();
-    if (!profile) return;
+    if (committerWaiting) {
+        ++wakes;
+        wake.notify_one();
+    }
+    static const bool stats = std::getenv("APS5_PIPELINE_STATS") != nullptr;
+    if (!profile && !stats) return;
     const auto now = std::chrono::steady_clock::now();
     if (now - lastReport >= std::chrono::seconds(10)) report(now);
 }
@@ -132,20 +359,24 @@ void DrawPipeline::report(std::chrono::steady_clock::time_point now) {
     const auto perCommit = commits == 0 ? 0.0 : static_cast<double>(commitNs) / 1e3 / static_cast<double>(commits);
     const auto perDraw = enqueued == 0 ? 0.0 : static_cast<double>(fullWaitNs) / 1e3 / static_cast<double>(enqueued);
     const auto queued = enqueued == 0 ? 0.0 : static_cast<double>(depthSum) / static_cast<double>(enqueued);
-    char reads[64];
-    std::snprintf(reads, sizeof(reads), " %llu read ranges, %llu stores held", static_cast<unsigned long long>(heldReadRanges.load(std::memory_order_relaxed)), static_cast<unsigned long long>(heldReadHits.exchange(0, std::memory_order_relaxed)));
+    const auto decodeDrainCount = decodeDrains.load(std::memory_order_relaxed);
+    char reads[96];
+    std::snprintf(reads, sizeof(reads), " %llu read ranges, %llu stores held, %llu decode drains", static_cast<unsigned long long>(heldReadRanges.load(std::memory_order_relaxed)), static_cast<unsigned long long>(heldReadHits.exchange(0, std::memory_order_relaxed)), static_cast<unsigned long long>(decodeDrainCount - reportedDecodeDrains));
+    reportedDecodeDrains = decodeDrainCount;
     drainText += reads;
-    std::fprintf(stderr, "[draw] pipelined (10 s, depth %zu): %llu commits at %.1f us each on the committer, %llu failed; %llu enqueued behind %.2f on average, waiting %.1f us per draw for room;%s; drains:%s\n", Depth(), static_cast<unsigned long long>(commits), perCommit, static_cast<unsigned long long>(commitErrors), static_cast<unsigned long long>(enqueued), queued, perDraw, reads, drainText.c_str());
-    commits = commitNs = commitErrors = enqueued = fullWaitNs = depthSum = 0;
+    std::fprintf(stderr, "[draw] pipelined (10 s, depth %zu): %llu commits at %.1f us each on the committer, %llu failed; %llu enqueued behind %.2f on average, waiting %.1f us per draw for room, %.2f us for the lock; %llu committer wakes;%s; drains:%s\n", Depth(), static_cast<unsigned long long>(commits), perCommit, static_cast<unsigned long long>(commitErrors), static_cast<unsigned long long>(enqueued), queued, perDraw, enqueued == 0 ? 0.0 : static_cast<double>(lockWaitNs) / 1e3 / static_cast<double>(enqueued), static_cast<unsigned long long>(wakes), reads, drainText.c_str());
+    commits = commitNs = commitErrors = enqueued = fullWaitNs = depthSum = wakes = lockWaitNs = 0;
     drains = {};
     drainWaitNs = {};
 }
 
+
 void DrawPipeline::run() {
     PerformanceTrace::BindThread("draw committer");
     StartCommitterSampler();
     for (;;) {
         Commit* commit = nullptr;
+        bool isDraw = false;
         SpinUntil(HandoffSpinWindow(), [&] { return outstanding.load(std::memory_order_acquire) != 0; });
         {
             std::unique_lock lock(mutex);
@@ -154,6 +385,7 @@ void DrawPipeline::run() {
             committerWaiting = false;
             if (items.empty()) break;
             commit = stopped ? nullptr : &items.front().commit;
+            isDraw = items.front().draw;
         }
         const auto start = std::chrono::steady_clock::now();
         std::exception_ptr fatal;
@@ -161,6 +393,7 @@ void DrawPipeline::run() {
         bool shutdown = false;
         try {
             if (commit != nullptr) (*commit)();
+            if (!isDraw) GuestMemory::BreakProofs();
         } catch (const std::exception&) {
             failed = true;
             fatal = std::current_exception();
@@ -176,13 +409,16 @@ void DrawPipeline::run() {
             if (!shutdown) Driver::Get().ReportFailure(fatal);
         }
         std::lock_guard lock(mutex);
+        countWrites(items.front().writes, false);
+        retired.push_back(std::move(items.front()));
         items.pop_front();
+        retiredItems.fetch_add(1, std::memory_order_release);
         ++commits;
         commitNs += spent;
         if (failed) ++commitErrors;
         if (fatal != nullptr && failure == nullptr) failure = fatal;
         outstanding.fetch_sub(1, std::memory_order_release);
-        if (idleWaiters != 0) idle.notify_all();
+        if (idleWaiters != 0 && (items.empty() || items.size() <= resumeAt)) idle.notify_all();
     }
     runningCommitters.fetch_sub(1, std::memory_order_acq_rel);
 }
@@ -200,10 +436,18 @@ void DrawPipeline::Stop() {
     else committer.join();
 }
 
+void DrawPipeline::ReleaseRetired() {
+    std::vector<Item> done;
+    std::lock_guard lock(mutex);
+    std::swap(done, retired);
+}
+
 void DrawPipeline::Drain(DrainReason reason, std::uint32_t opcode) {
     drainRequests[std::min<std::uint32_t>(opcode, 0x100)].fetch_add(1, std::memory_order_relaxed);
+    const bool finisherBusy = DrawFinisher::Busy() && !DrawFinisher::OnFinisherThread();
+    if ((finisherBusy || Busy()) && GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw pipeline drain under the GPU lock (reason " + std::to_string(static_cast<unsigned>(reason)) + ", opcode " + std::to_string(opcode) + ")");
+    if (finisherBusy) DrawFinisher::Get().Drain();
     if (!Busy()) return;
-    if (GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw pipeline drain under the GPU lock (reason " + std::to_string(static_cast<unsigned>(reason)) + ", opcode " + std::to_string(opcode) + ")");
     const auto start = std::chrono::steady_clock::now();
     SpinUntil(HandoffSpinWindow() * 2, [&] { return !Busy(); });
     std::unique_lock lock(mutex);
@@ -244,6 +488,12 @@ void DrawPipeline::ReleaseRunningReads() {
 
 std::optional<std::uint64_t> DrawPipeline::PendingLabel(std::uint64_t address, std::size_t bytes) {
     if (bytes == 0 || bytes > 8) return std::nullopt;
+    if (DrawFinisher::Overlap()) {
+        std::uint64_t held = 0;
+        const auto lookup = DrawFinisher::Get().PendingLabel(address, bytes, held);
+        if (lookup == DrawFinisher::LabelLookup::Value) return held;
+        if (lookup == DrawFinisher::LabelLookup::Unknown) return std::nullopt;
+    }
     const auto end = address + bytes;
     std::lock_guard lock(mutex);
     for (auto item = items.rbegin(); item != items.rend(); ++item) {
@@ -257,15 +507,64 @@ std::optional<std::uint64_t> DrawPipeline::PendingLabel(std::uint64_t address, s
     return std::nullopt;
 }
 
-bool DrawPipeline::Overlaps(std::uint64_t address, std::size_t bytes) {
-    const auto end = address + bytes;
+void DrawPipeline::countWrites(const std::vector<Range>& writes, bool added) {
+    for (const auto& range : writes) {
+        const auto found = std::find_if(queuedWrites.begin(), queuedWrites.end(), [&](const auto& entry) { return entry.first == range; });
+        if (added) {
+            if (found != queuedWrites.end()) ++found->second;
+            else queuedWrites.emplace_back(range, 1u);
+        } else if (found != queuedWrites.end() && --found->second == 0) {
+            *found = queuedWrites.back();
+            queuedWrites.pop_back();
+        }
+    }
+}
+
+DrawPipeline::Watch DrawPipeline::WatchWrites() {
+    if (!Busy()) return {Retired(), 0, {}};
     std::lock_guard lock(mutex);
-    for (const auto& item : items) {
-        for (const auto& [begin, limit] : item.writes) {
-            if (address < limit && begin < end) return true;
+    Watch watch{retiredItems.load(std::memory_order_relaxed), items.size(), {}};
+    for (const auto& item : items) watch.writes.insert(watch.writes.end(), item.writes.begin(), item.writes.end());
+    return watch;
+}
+
+bool DrawPipeline::WrittenSince(const Watch& watch, const std::vector<Range>& reads) {
+    const auto overlaps = [&reads](const std::vector<Range>& writes) {
+        return std::any_of(reads.begin(), reads.end(), [&writes](const Range& read) {
+            return std::any_of(writes.begin(), writes.end(), [&read](const Range& write) { return read.first < write.second && write.first < read.second; });
+        });
+    };
+    if (DrawFinisher::Overlap() && DrawFinisher::Get().PendingOverlapsAny(reads)) return true;
+    std::lock_guard lock(mutex);
+    if (std::any_of(items.begin(), items.end(), [&](const Item& item) { return overlaps(item.writes); })) return true;
+    const auto retiredNow = retiredItems.load(std::memory_order_relaxed);
+    if (retiredNow == watch.retired) return false;
+    return retiredNow - watch.retired > watch.items || overlaps(watch.writes);
+}
+
+bool DrawPipeline::Busy() const {
+    return outstanding.load(std::memory_order_acquire) != 0 || (finisherProducer && DrawFinisher::Busy() && DrawFinisher::Overlap());
+}
+
+bool DrawPipeline::OverlapsAny(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges) {
+    if (DrawFinisher::Overlap() && DrawFinisher::Get().PendingOverlapsAny(ranges)) return true;
+    std::lock_guard lock(mutex);
+    for (const auto& [range, count] : queuedWrites) {
+        for (const auto& [address, end] : ranges) {
+            if (address < range.second && range.first < end) return true;
         }
     }
     return false;
 }
 
+bool DrawPipeline::Overlaps(std::uint64_t address, std::size_t bytes) {
+    const auto end = address + bytes;
+    if (DrawFinisher::Overlap() && DrawFinisher::Get().PendingOverlaps(address, end)) return true;
+    std::lock_guard lock(mutex);
+    for (const auto& [range, count] : queuedWrites) {
+        if (address < range.second && range.first < end) return true;
+    }
+    return false;
+}
+
 }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp

@@ -3,6 +3,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
 #include <cstdlib>
 
 namespace AgcDriver::DriverDetail {
@@ -16,6 +17,25 @@ std::optional<DrawVerdict> Driver::precheckDraw(const QueueState& queue, const S
     if (!drawParameters.indirect && !drawParameters.indexed && (drawParameters.indexCount == 0 || drawParameters.instanceCount == 0)) return DrawVerdict::Nothing;
     if (const auto pass = Graphics::DecodeColorMetadataPass(queue)) {
         require(!drawParameters.indirect, "indirect CB metadata passes are unsupported");
+        if (DrawPipeline::Active() && DrawPipeline::MetadataPassesPipelined() && deferredLabels().labels.empty()) {
+            auto writes = Graphics::PipelinedColorMetadataPassWrites(*pass);
+            const auto localDevice = device.Load();
+            if (!writes.empty() && localDevice != nullptr) {
+                const auto queueId = submission.queue;
+                const auto opcode = (packet[0] >> 8u) & 0xffu;
+                DrawPipeline::Queue0().Enqueue([this, localDevice, queueId, opcode, pass = *pass] {
+                    GuestMemory::SetCurrentPacket(opcode, queueId);
+                    GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw);
+                    std::lock_guard gpuLock(GuestMemory::GpuMutex());
+                    auto current = device.Load();
+                    const auto& target = current != nullptr ? current : localDevice;
+                    recordLabelsForPacket(target.get(), queueId);
+                    target->ColorMetadataPass(pass);
+                }, std::move(writes));
+                DrawPipeline::CountPipelinedMetadataPass();
+                return DrawVerdict::Drawn;
+            }
+        }
         if (DrawPipeline::Active()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, (packet[0] >> 8u) & 0xffu);
         GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw);
         std::lock_guard gpuLock(GuestMemory::GpuMutex());

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp

@@ -213,7 +213,7 @@ void Driver::traceDrawKeyLookup(const std::array<std::uint64_t, 5>& components,
 }
 
 bool Driver::sameVertexInfo(const ShaderRecompiler::ShaderVertexStageInfo& a, const ShaderRecompiler::ShaderVertexStageInfo& b) {
-    if (a.resourcesNum != b.resourcesNum || a.fetchAttribReg != b.fetchAttribReg || a.fetchBufferReg != b.fetchBufferReg || a.fetchEmbedded != b.fetchEmbedded || a.paClVsOutCntl != b.paClVsOutCntl) return false;
+    if (a.resourcesNum != b.resourcesNum || a.fetchAttribReg != b.fetchAttribReg || a.fetchBufferReg != b.fetchBufferReg || a.fetchEmbedded != b.fetchEmbedded || a.paClVsOutCntl != b.paClVsOutCntl || a.pointList != b.pointList || a.pointSize != b.pointSize || a.pointSizeMin != b.pointSizeMin || a.pointSizeMax != b.pointSizeMax) return false;
     if (a.depthRemapMode != b.depthRemapMode) return false;
     for (std::uint32_t i = 0; i < a.resourcesNum && i < a.resources.size(); ++i) {
         if (a.resources[i].fields != b.resources[i].fields) return false;
@@ -228,7 +228,7 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) {
     const auto& s = a.state;
     const auto& t = b.state;
     const auto sameColor = [](const Graphics::ColorTarget& x, const Graphics::ColorTarget& y) {
-        return x.address == y.address && x.extent.width == y.extent.width && x.extent.height == y.extent.height && x.format == y.format && x.bytes == y.bytes && x.componentMapping == y.componentMapping && x.tileMode == y.tileMode && x.elementBytes == y.elementBytes && x.dccAddress == y.dccAddress && x.dccAlphaOnMsb == y.dccAlphaOnMsb && x.slot == y.slot && x.exportIndex == y.exportIndex;
+        return x.address == y.address && x.extent.width == y.extent.width && x.extent.height == y.extent.height && x.format == y.format && x.bytes == y.bytes && x.componentMapping == y.componentMapping && x.tileMode == y.tileMode && x.elementBytes == y.elementBytes && x.dccAddress == y.dccAddress && x.dccAlphaOnMsb == y.dccAlphaOnMsb && x.slot == y.slot && x.exportIndex == y.exportIndex && x.partialWriteMask == y.partialWriteMask;
     };
     const auto sameBlend = [](const VkPipelineColorBlendAttachmentState& x, const VkPipelineColorBlendAttachmentState& y) {
         return x.blendEnable == y.blendEnable && x.srcColorBlendFactor == y.srcColorBlendFactor && x.dstColorBlendFactor == y.dstColorBlendFactor && x.colorBlendOp == y.colorBlendOp && x.srcAlphaBlendFactor == y.srcAlphaBlendFactor && x.dstAlphaBlendFactor == y.dstAlphaBlendFactor && x.alphaBlendOp == y.alphaBlendOp && x.colorWriteMask == y.colorWriteMask;
@@ -262,8 +262,8 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) {
         const auto& y = b.programs[i];
         if (x.binary.stage != y.binary.stage || x.binary.codeAddress != y.binary.codeAddress || x.userDataBase != y.userDataBase || x.firstUserSgpr != y.firstUserSgpr || x.userData != y.userData || x.snapshot != y.snapshot || x.codeOffset != y.codeOffset || x.tailContinuation != y.tailContinuation) return false;
     }
-    if (p.targetExportPacking != q.targetExportPacking || p.dualSourceBlend != q.dualSourceBlend || p.reversedBlend != q.reversedBlend || p.alphaToCoverage != q.alphaToCoverage || p.rasterPrimitive != q.rasterPrimitive || p.ditheredAlphaToCoverage != q.ditheredAlphaToCoverage || p.quadPixelMask != q.quadPixelMask || p.quadSampleMask != q.quadSampleMask || p.iteratesSamples != q.iteratesSamples || p.positionLocation != q.positionLocation || p.sampleLocations != q.sampleLocations || p.sampleOrder != q.sampleOrder) return false;
-    if (s.samples != t.samples || s.mergedCoverage != t.mergedCoverage || s.quadSampleMask != t.quadSampleMask || s.pixelEarlyZ != t.pixelEarlyZ) return false;
+    if (p.targetExportPacking != q.targetExportPacking || p.dualSourceBlend != q.dualSourceBlend || p.reversedBlend != q.reversedBlend || p.targetMerge != q.targetMerge || p.alphaToCoverage != q.alphaToCoverage || p.rasterPrimitive != q.rasterPrimitive || p.ditheredAlphaToCoverage != q.ditheredAlphaToCoverage || p.quadPixelMask != q.quadPixelMask || p.quadSampleMask != q.quadSampleMask || p.iteratesSamples != q.iteratesSamples || p.positionLocation != q.positionLocation || p.sampleLocations != q.sampleLocations || p.sampleOrder != q.sampleOrder) return false;
+    if (s.samples != t.samples || s.mergedCoverage != t.mergedCoverage || s.truncatedCoverage != t.truncatedCoverage || s.quadSampleMask != t.quadSampleMask || s.pixelEarlyZ != t.pixelEarlyZ || s.pixelKill != t.pixelKill) return false;
     return true;
 }
 

core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp

@@ -1,3 +1,4 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ThreadPriority.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Screenshot.hpp"
@@ -160,3 +161,8 @@ extern "C" void AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span<const Sh
 } catch (const ProcessShutdown&) {
     LibcAwaitExit_nid_postfix();
 }
+
+extern "C" void AgcDriverPrepareCommandWrite_nid_no_patch(std::uint64_t address, std::size_t bytes) {
+    if (AgcDriver::Graphics::Recorder::SnapshotWriteOverlaps(address, bytes))
+        AgcDriver::GuestMemory::FlushGpuWrites(address, bytes);
+}

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp

@@ -14,6 +14,7 @@
 #include "prx/libc/include/GuestAllocations.hpp"
 #include "ShaderDiskCache.hpp"
 #include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
 #include <atomic>
 #include <cstdint>
 #include <cstdio>
@@ -87,6 +88,9 @@ void MapDriverMemory() {
     auto* slot = reinterpret_cast<std::uint64_t*>(FsTableAddress);
     slot[0] = (FsRoutineTableAddress & 0xffffffffffffull) | 16ull << 48u;
     slot[1] = FsTableDword3 << 32u | FsRoutineRecords;
+#ifndef _WIN32
+    GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(mapped, DriverMemoryBytes);
+#endif
     GuestAllocations::Mutation mutation;
     mutation.Add(mapped, DriverMemoryBytes, true, true, true);
 }
@@ -161,7 +165,13 @@ void Driver::stop() {
     for (auto& [queue, worker] : workers) {
         if (worker.thread.joinable()) worker.thread.join();
     }
+    try {
+        DrawFinisher::Get().Drain();
+    } catch (...) {
+    }
     DrawPipeline::Queue0().Stop();
+    DrawPipeline::Queue0().ReleaseRetired();
+    DrawFinisher::Get().ReleaseRan();
     StopWorkerSampler();
     CloseRegistrationPreparation();
     Graphics::StopPipelineLibraryWorker();
@@ -224,13 +234,17 @@ Driver::Driver() {
 
 void Driver::WaitIdle() {
     require(!onWorkerThread(), "worker cannot wait for itself");
-    std::unique_lock lock(mutex);
-    const auto target = accepted;
-    ++idleWaiters;
-    changed.wait(lock, [&] { return failure != nullptr || stopping || completed >= target; });
-    --idleWaiters;
-    rethrowFailure();
-    checkStopping();
+    {
+        std::unique_lock lock(mutex);
+        const auto target = accepted;
+        ++idleWaiters;
+        changed.wait(lock, [&] { return failure != nullptr || stopping || completed >= target; });
+        --idleWaiters;
+        rethrowFailure();
+        checkStopping();
+    }
+    std::lock_guard gpuLock(GuestMemory::GpuMutex());
+    if (const auto localDevice = device.Load()) localDevice->WaitIdle();
 }
 
 void Driver::CheckFailure() {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferCopy.cpp

@@ -2,6 +2,8 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferCopy.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 #include <algorithm>
 #include <cstdlib>
 #include <cstring>
@@ -16,14 +18,14 @@ bool Driver::matchesCopyKernel(std::span<const std::uint32_t> code, const std::v
     static const bool enabled = std::getenv("APS5_NO_COPY_HLE") == nullptr;
     if (!enabled || userData.size() != 12 || compute.numThreads[0] != 64 || compute.numThreads[1] != 1 || compute.numThreads[2] != 1 || !compute.groupIdEnable[0]) return false;
     static constexpr std::size_t copyKernelWords = 38;
-    static constexpr std::uint64_t copyKernelHash = 0x6ec00fe8aa95f99aull;
+    static constexpr std::array<std::uint64_t, 2> copyKernelHashes{0x6ec00fe8aa95f99aull, 0xc771c42e4d069d62ull};
     if (code.size() < copyKernelWords || code[0] != 0xbfa00002u || code[copyKernelWords - 1] != 0xbf810000u) return false;
     std::uint64_t hash = 0xcbf29ce484222325ull;
     for (const auto word : code.first(copyKernelWords)) {
         hash ^= word;
         hash *= 0x100000001b3ull;
     }
-    if (hash != copyKernelHash) return false;
+    if (std::find(copyKernelHashes.begin(), copyKernelHashes.end(), hash) == copyKernelHashes.end()) return false;
     const auto stride = [&](std::size_t word) { return (userData[word] >> 16u) & 0x3fffu; };
     const auto swizzled = [&](std::size_t word) { return ((userData[word] >> 31u) & 1u) != 0; };
     return userData[3] == 0x00014004u && userData[7] == 0x00014004u && userData[11] == 0x0004dfacu && stride(1) == 4 && stride(5) == 4 && stride(9) == 16 && !swizzled(1) && !swizzled(5) && !swizzled(9) && userData[10] >= 1;
@@ -52,13 +54,35 @@ std::optional<StreamCopy> StreamCopyRange(const std::vector<std::uint32_t>& user
     return copy;
 }
 
+bool MatchesIndexedCopyKernel(std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) {
+    static const bool enabled = std::getenv("APS5_NO_COPY_HLE") == nullptr;
+    if (!enabled || userData.size() != 12 || compute.numThreads[0] != 64 || compute.numThreads[1] != 1 || compute.numThreads[2] != 1 || !compute.groupIdEnable[0]) return false;
+    static constexpr std::array<std::uint32_t, 19> kernel{0xbfa00001u, 0xd7460000u, 0x04010c0cu, 0xf4201a84u, 0xfa000000u, 0xbf8cc07fu, 0x7da8006au, 0xbf88000au, 0xf4241a84u, 0xfa000004u, 0xbf8cc07fu, 0x4a08006bu, 0x4a00006au, 0xe0382000u, 0x80000000u, 0xbf8c3f70u, 0xe0782000u, 0x80010004u, 0xbf810000u};
+    if (code.size() < kernel.size() || !std::equal(kernel.begin(), kernel.end(), code.begin())) return false;
+    const auto stride = [&](std::size_t word) { return (userData[word] >> 16u) & 0x3fffu; };
+    const auto swizzled = [&](std::size_t word) { return ((userData[word] >> 31u) & 1u) != 0; };
+    const bool addTid = ((userData[3] >> 23u) & 1u) != 0;
+    const auto outOfBounds = (userData[3] >> 28u) & 3u;
+    const auto type = userData[3] >> 30u;
+    return stride(1) == 16 && stride(5) == 16 && stride(9) == 16 && !swizzled(1) && !swizzled(5) && !swizzled(9) && userData[3] == userData[7] && type == 0 && !addTid && outOfBounds == 0 && userData[10] >= 1;
+}
+
+std::optional<StreamCopy> IndexedCopyRange(const std::vector<std::uint32_t>& userData, std::uint32_t groups, std::uint32_t count, std::uint32_t sourceOffset, std::uint32_t destinationOffset) {
+    const auto base = [&](std::size_t word) { return userData[word] | (static_cast<std::uint64_t>(userData[word + 1] & 0xffffu) << 32u); };
+    const auto elements = std::min<std::uint64_t>(std::uint64_t{groups} * 64u, count);
+    if (elements == 0 || std::uint64_t{sourceOffset} + elements > userData[2] || std::uint64_t{destinationOffset} + elements > userData[6]) return std::nullopt;
+    const StreamCopy copy{base(0) + std::uint64_t{sourceOffset} * 16u, base(4) + std::uint64_t{destinationOffset} * 16u, static_cast<std::size_t>(elements) * 16u};
+    if (copy.bytes > (16u << 20u) || copy.source % 16 != 0 || copy.destination % 16 != 0) return std::nullopt;
+    return copy;
+}
+
 void Driver::countCopy(int path, std::size_t bytes, std::chrono::steady_clock::time_point started, std::chrono::steady_clock::time_point locked, const VulkanDevice::CopyOutcome* outcome) {
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
     if (path < 0 || path >= CopyPaths) return;
     copyPathCounts[path].fetch_add(1, std::memory_order_relaxed);
     if (!profile) return;
     using Outcome = VulkanDevice::CopyOutcome;
-    static std::mutex countsMutex;
+    static AgcDriver::Mutex countsMutex;
     static std::uint64_t counts[CopyPaths] = {};
     static std::uint64_t refused[Outcome::Refusals] = {};
     static std::uint64_t cpuBytes = 0, gpuBytes = 0, aliasedBytes = 0, syncs = 0, settledBySignal = 0, sourceWaits = 0;
@@ -119,7 +143,9 @@ bool Driver::knownValueVerify() {
 }
 
 bool Driver::copyBuffer(QueueState& queue, std::uint32_t queueId, std::span<const std::uint32_t> packet, std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, const std::shared_ptr<VulkanDevice>& localDevice, std::uint64_t programAddress) {
-    const bool stream = !matchesCopyKernel(code, userData, compute);
+    const bool hashed = matchesCopyKernel(code, userData, compute);
+    const bool indexed = !hashed && MatchesIndexedCopyKernel(code, userData, compute);
+    const bool stream = !hashed && !indexed;
     if (stream && !MatchesStreamCopyKernel(code, userData, compute)) return false;
     const auto started = std::chrono::steady_clock::now();
     static const std::size_t cpuMax = [] {
@@ -132,8 +158,8 @@ bool Driver::copyBuffer(QueueState& queue, std::uint32_t queueId, std::span<cons
         return text ? static_cast<std::size_t>(std::strtoull(text, nullptr, 10)) : std::numeric_limits<std::size_t>::max();
     }();
     const auto base = [&](std::size_t word) { return userData[word] | (static_cast<std::uint64_t>(userData[word + 1] & 0xffffu) << 32u); };
-    const auto source = base(0);
-    const auto destination = base(4);
+    auto source = base(0);
+    auto destination = base(4);
     const auto record = stream ? std::uint64_t{0} : base(8);
     std::array<std::uint32_t, 3> groups{packet[1], packet[2], packet[3]};
     if ((packet[4] & 0x20u) != 0) {
@@ -168,6 +194,28 @@ bool Driver::copyBuffer(QueueState& queue, std::uint32_t queueId, std::span<cons
             return false;
         }
         bytes = range->bytes;
+    } else if (indexed) {
+        Graphics::FinishLeaseWork();
+        const bool recordAccessible = GuestMemory::Accessible(reinterpret_cast<const void*>(record), 12);
+
+        bool published = false;
+        if (!recordAccessible || Graphics::Recorder::SnapshotWriteOverlaps(record, 12) || Graphics::StorageTexture::FlushPending(record, 12, nullptr, "indexed copy record", Graphics::PublishScope::Whole, &published) || published || labelPending(record, 12)) {
+            if (traceCopy()) traceCopyPending(recordAccessible ? "copy record" : "inaccessible copy record", record, 12, source, destination, 0, localDevice);
+            countCopy(CopyRecordPending, 0, started, locked);
+            return false;
+        }
+        std::uint32_t count = 0, sourceOffset = 0, destinationOffset = 0;
+        std::memcpy(&count, reinterpret_cast<const void*>(record), 4);
+        std::memcpy(&sourceOffset, reinterpret_cast<const void*>(record + 4), 4);
+        std::memcpy(&destinationOffset, reinterpret_cast<const void*>(record + 8), 4);
+        const auto range = IndexedCopyRange(userData, groups[0], count, sourceOffset, destinationOffset);
+        if (!range || (record < range->destination + range->bytes && range->destination < record + 12)) {
+            countCopy(CopyShape, range ? range->bytes : 0, started, locked);
+            return false;
+        }
+        source = range->source;
+        destination = range->destination;
+        bytes = range->bytes;
     } else {
         const bool recordAccessible = GuestMemory::Accessible(reinterpret_cast<const void*>(record), 8);
 

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferFill.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 #include "ThreadOwned.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DwordPatternFill.hpp"
@@ -186,7 +187,7 @@ void Driver::fillClearCount(const Graphics::StorageTexture::FillCoverage& covera
     if (!profile) return;
     using FillCover = Graphics::StorageTexture::FillCover;
     static constexpr std::array<const char*, 8> names{"none", "exact", "inside", "around", "straddle", "several", "keys", "layer"};
-    static std::mutex countsMutex;
+    static AgcDriver::Mutex countsMutex;
     static std::array<std::uint64_t, 8> counts{}, coverBytes{};
     static std::map<std::uint32_t, std::uint64_t> coveredFormats;
     static std::map<std::string, std::uint64_t> refusals;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp

@@ -181,10 +181,11 @@ std::vector<std::pair<std::uint64_t, std::uint64_t>> QueuedDrawReadRanges(const
     }
     if (stages.empty() || stages.front().program == nullptr) return ranges;
     for (const auto& attribute : stages.front().program->vertexAttributes) {
-        if (Graphics::NullVertexDescriptor(attribute) || Graphics::VertexFetchOutOfRange(attribute)) continue;
+        if (Graphics::NullVertexDescriptor(attribute)) continue;
         const auto address = attribute.resource.fields[0] | (static_cast<std::uint64_t>(attribute.resource.fields[1] & 0xffffu) << 32u);
         std::uint64_t bytes = 0;
         try {
+            if (Graphics::VertexFetchOutOfRange(attribute)) continue;
             bytes = Graphics::VertexBufferExtent(attribute);
         } catch (const std::runtime_error&) {
             continue;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp

@@ -34,6 +34,24 @@ bool pipelineDma() {
     return enabled;
 }
 
+bool keepsDrawFinisher(std::uint32_t header, std::uint32_t opcode) {
+    if (header == FlipPacketHeader || header == RenderingWaitPacketHeader) return false;
+    if (Pm4::DrawOpcode(opcode)) return true;
+    switch (opcode) {
+        case 0x13: case 0x26: case 0x2a: case 0x2f: case 0x69: case 0x76: case 0x79: return true;
+        case 0x10: return ((header >> 2u) & 0x3fu) == 0;
+        default: return false;
+    }
+}
+
+bool waitTakesPendingLabel(std::span<const std::uint32_t> packet) {
+    if (packet.size() < 5) return false;
+    const auto awaitedBytes = Pm4::WaitAwaitedBytes(packet);
+    const std::uint64_t awaited = (packet[2] & (awaitedBytes == 8 ? ~7u : ~3u)) | (static_cast<std::uint64_t>(packet[3]) << 32u);
+    const auto value = DrawPipeline::Queue0().PendingLabel(awaited, awaitedBytes);
+    return value.has_value() && Pm4::WaitComparesValue(packet, *value);
+}
+
 bool keepsDrawPipeline(std::span<const std::uint32_t> packet, std::uint32_t header, std::uint32_t opcode) {
     if (header == FlipPacketHeader || header == RenderingWaitPacketHeader) return false;
     if (Pm4::DrawOpcode(opcode)) return true;
@@ -108,13 +126,17 @@ void Driver::deliverFlipEopInterrupt(std::uint32_t queue) {
 
 template <typename TWork>
 void Driver::timed(double WorkerProfile::*bucket, TWork&& work) {
+    static const bool report = std::getenv("APS5_PROFILE_DRAW") != nullptr;
+    if (!report) {
+        work();
+        return;
+    }
     static thread_local WorkerProfile profile;
     const auto begin = std::chrono::steady_clock::now();
     work();
     const auto end = std::chrono::steady_clock::now();
     profile.*bucket += std::chrono::duration<double, std::milli>(end - begin).count();
-    static const bool report = std::getenv("APS5_PROFILE_DRAW") != nullptr;
-    if (report && end - profile.reported > std::chrono::seconds(10)) {
+    if (end - profile.reported > std::chrono::seconds(10)) {
         profile.reported = end;
         AgcDriver::ProfilePrint_nid_no_patch( "[gpu] worker at %.0f s: dispatch %.1f s, draw %.1f s, wait %.1f s\n", std::chrono::duration<double>(end - profile.start).count(), profile.dispatchMs / 1000, profile.drawMs / 1000, profile.waitMs / 1000);
     }
@@ -226,6 +248,12 @@ void Driver::execute(const Submission& submission) {
     struct ClearAhead {
         ~ClearAhead() { clearResolvedAhead(); }
     } clearAhead;
+    struct SettleFinisher {
+        bool queue0;
+        ~SettleFinisher() {
+            if (queue0) DrawFinisher::Get().Settle();
+        }
+    } settleFinisher{submission.queue == 0};
     for (std::size_t cursor = 0; cursor < submission.commands.size();) {
         if (packetEpoch()) bumpEpoch(&EpochBumps::packets);
         CheckFailure();
@@ -235,10 +263,20 @@ void Driver::execute(const Submission& submission) {
         const auto packet = std::span(submission.commands).subspan(cursor, count);
         const auto opcode = (header >> 8u) & 0xffu;
         auto nextCursor = cursor + count;
+        const bool predicated = Pm4::Predicated(header) && queue.predication.operation != 0;
+        if (submission.queue == 0 && DrawFinisher::Busy() && !keepsDrawFinisher(header, opcode)) {
+            const bool queuesLabel = pipelined && !predicated && header != FlipPacketHeader && header != RenderingWaitPacketHeader && !submission.flips.contains(cursor) && labelPacketQueues(packet, opcode);
+            const bool heldLabelWait = pipelined && !predicated && orderedWaits && (opcode == 0x3c || opcode == 0x93) && header != RenderingWaitPacketHeader && waitTakesPendingLabel(packet);
+            const bool beside = DrawFinisher::Overlap() && opcode != 0x15 && (queuesLabel || heldLabelWait || keepsDrawPipeline(packet, header, opcode));
+            if (!beside) DrawFinisher::Get().Drain();
+            if (DrawFinisher::Overlap()) {
+                DrawFinisher::NoteDrain(DrawFinisher::DrainSite::Packet, !beside);
+                if (!beside) DrawFinisher::NoteDrainedOpcode(header == FlipPacketHeader ? 0xffu : opcode, keepsDrawPipeline(packet, header, opcode));
+            }
+        }
         if (APS5_ENABLE_TIMING_LOG && submission.queue == 0 && pendingFrameTiming == nullptr) includeTimingSubmission(submission, false);
         PerformanceContext timingContext(submission.queue == 0 ? pendingFrameTiming.get() : FrameTiming::Async());
         PerformanceTimer timing("Driver.Packet");
-        const bool predicated = Pm4::Predicated(header) && queue.predication.operation != 0;
         if (pipelined && !predicated && header != FlipPacketHeader && header != RenderingWaitPacketHeader && !submission.flips.contains(cursor) && (enqueueLabelPacket(packet, opcode, submission.queue) || enqueueDmaPacket(packet, opcode, submission.queue, queue))) {
             traceLabel(packet, submission.queue);
             recent.Record(cursor);
@@ -265,8 +303,10 @@ void Driver::execute(const Submission& submission) {
                 continue;
             }
         }
+        timing.Mark("pipeline_prepare");
         if (pipelined && DrawPipeline::Queue0().Busy() && !keepsDrawPipeline(packet, header, opcode)) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, header == FlipPacketHeader ? 0xffu : opcode);
         if (pipelined && predicated && DrawPipeline::Queue0().Busy() && DrawPipeline::Queue0().Overlaps(queue.predication.address, queue.predication.operation == 3 ? 8 : 4)) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, opcode);
+        timing.Mark("pipeline_drain");
         std::shared_lock deviceUse(deviceReplacement, std::defer_lock);
         if (opcode != 0x3c && opcode != 0x93 && header != RenderingWaitPacketHeader && header != FlipPacketHeader) deviceUse.lock();
 
@@ -289,7 +329,7 @@ void Driver::execute(const Submission& submission) {
         const auto flushStart = profilePackets ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
         flushBetweenPackets(submission.queue, header, opcode == 0x49 || opcode == 0x37);
         timing.Mark("flush_between_packets");
-        PacketTimer packetTimer{profilePackets, header == FlipPacketHeader ? 0xffffu : opcode, submission.queue, packetProfile, std::chrono::steady_clock::now()};
+        PacketTimer packetTimer{profilePackets, header == FlipPacketHeader ? 0xffffu : opcode, submission.queue, packetProfile, profilePackets ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}};
 
         if (profilePackets) {
             packetStartedAt() = packetTimer.start;
@@ -404,7 +444,7 @@ void Driver::execute(const Submission& submission) {
             finishDispatchPacket(opcode == 0x16);
         } else if (opcode == 0x3c || opcode == 0x93) {
             static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr;
-            const auto waitStart = std::chrono::steady_clock::now();
+            const auto waitStart = traceGpu ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
             timed(&WorkerProfile::waitMs, [&] { waitMemory(packet, submission.queue, recent, submission.received, submission.heldAtSubmit.contains(cursor)); });
             gateOpened() = true;
             if (traceGpu && std::chrono::steady_clock::now() - waitStart > std::chrono::milliseconds(200)) {
@@ -519,6 +559,7 @@ void Driver::execute(const Submission& submission) {
         cursor = nextCursor;
     }
 
+    if (submission.queue == 0) DrawFinisher::Get().Drain();
     PerformanceContext endContext(submission.queue == 0 ? frameTiming() : FrameTiming::Async());
     PerformanceTimer endTiming("Driver.EndSubmission");
     static const bool submitAtEnd = std::getenv("APS5_SUBMIT_AT_END") != nullptr;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp

@@ -52,6 +52,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
                 countLabelOutcome(reason);
                 ++immediateLabels;
             } else {
+                if (localDevice != nullptr) Graphics::StorageTexture::StoreReadbackBeforeLabel("readback before interrupt");
                 ++noOpLabels;
             }
             if (reason == 0 || reason == 5) {
@@ -380,9 +381,15 @@ void Driver::commitDmaStore(std::uint32_t queue, std::uint64_t address, std::spa
     Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration());
 }
 
-bool Driver::enqueueLabelPacket(std::span<const std::uint32_t> packet, std::uint32_t opcode, std::uint32_t queue) {
+bool Driver::labelPacketQueues(std::span<const std::uint32_t> packet, std::uint32_t opcode) {
     static const bool ordered = std::getenv("APS5_PIPELINE_DRAIN_LABELS") == nullptr;
     if (!ordered || !DeferLabels() || (opcode != 0x49 && opcode != 0x37) || !deferredLabels().labels.empty()) return false;
+    if (Pm4::DecodeLabelWrite(packet).has_value()) return true;
+    return opcode == 0x49 && ((packet[2] >> 29u) == 0 || (packet[3] | (static_cast<std::uint64_t>(packet[4]) << 32u)) == 0);
+}
+
+bool Driver::enqueueLabelPacket(std::span<const std::uint32_t> packet, std::uint32_t opcode, std::uint32_t queue) {
+    if (!labelPacketQueues(packet, opcode)) return false;
     const auto interruptSelect = opcode == 0x49 ? (packet[2] >> 24u) & 7u : 0u;
     const bool endOfPipeInterrupt = interruptSelect != 0 && interruptSelect != 3;
     const auto label = Pm4::DecodeLabelWrite(packet);
@@ -406,10 +413,23 @@ bool Driver::enqueueLabelPacket(std::span<const std::uint32_t> packet, std::uint
     }
     if (endOfPipeInterrupt) bumpEpoch(&EpochBumps::drains);
     auto stored = bytes;
-    DrawPipeline::Queue0().Enqueue([this, queue, opcode, address, bytes = std::move(bytes), endOfPipeInterrupt] {
-        GuestMemory::SetCurrentPacket(opcode, queue);
-        commitLabel(queue, address, bytes, endOfPipeInterrupt);
-    }, std::move(writes), address, std::move(stored));
+    auto labelBytes = bytes;
+    auto enqueue =[this, queue, opcode, address, bytes = std::move(bytes), endOfPipeInterrupt, writes, stored = std::move(stored)]() mutable {
+        DrawPipeline::Queue0().Enqueue([this, queue, opcode, address, bytes = std::move(bytes), endOfPipeInterrupt] {
+            GuestMemory::SetCurrentPacket(opcode, queue);
+            commitLabel(queue, address, bytes, endOfPipeInterrupt);
+        }, std::move(writes), address, std::move(stored));
+    };
+    if (queue == 0 && DrawFinisher::Overlap() && DrawFinisher::Busy()) {
+        DrawFinisher::Pending held;
+        held.unknown = false;
+        held.writes = std::move(writes);
+        held.labelAddress = address;
+        held.labelBytes = std::move(labelBytes);
+        DrawFinisher::Get().Enqueue(std::move(enqueue), std::move(held));
+        return true;
+    }
+    enqueue();
     return true;
 }
 

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp

@@ -282,6 +282,7 @@ void Driver::executeRewindTail(const Submission& stalled) {
             checkStopping();
             noteAwaitingTitle(stalled.queue, reinterpret_cast<std::uint64_t>(stalled.rewindTail - 1));
             stall.Check("REWIND at", reinterpret_cast<std::uint64_t>(stalled.rewindTail - 1));
+            if (stalled.queue == 0 && completionsPending()) reapCompletionLabels();
             PollSleep();
         }
     }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderDiagnostics.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 #include "ControlFlow/RequestSerializer.hpp"
 #include <cstdlib>
 #include <functional>
@@ -7,7 +8,7 @@
 namespace AgcDriver::DriverDetail {
 
 void Driver::reportSkip(const char* kind, const std::string& what) {
-    static std::mutex reportedMutex;
+    static AgcDriver::Mutex reportedMutex;
     static std::set<std::size_t> reported;
     static std::map<std::string, int> requestDumps;
 
@@ -50,7 +51,7 @@ void Driver::reportSkip(const char* kind, const std::string& what) {
 }
 
 std::string Driver::dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request, std::string_view suffix) {
-    static std::mutex dumpMutex;
+    static AgcDriver::Mutex dumpMutex;
     static std::set<std::string> dumped;
     char hex[32];
     std::snprintf(hex, sizeof(hex), "%llx", static_cast<unsigned long long>(address));

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp

@@ -1,9 +1,11 @@
 #include "prx/libc/include/general/LogMacros.hpp"
 #include "ControlFlow/GraphBuilder.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
+#include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "CacheKey.hpp"
@@ -220,6 +222,24 @@ bool CommittedAbi(const ShaderSnapshot& snapshot, std::span<const std::uint64_t>
     return std::ranges::any_of(snapshot.prepared->registeredAbis, [&](const auto& abi) { return std::ranges::equal(abi, key); });
 }
 
+std::optional<std::vector<std::weak_ptr<const ShaderSnapshot>>> CommittedGraphicsAbi(const ShaderSnapshot& owner, std::span<const std::uint64_t> key, const std::shared_ptr<const ShaderRegistry>& registry) {
+    std::unique_lock lock(owner.prepared->mutex);
+    AwaitRegisteredPreparation(*owner.prepared, lock);
+    const auto& abis = owner.prepared->graphicsAbis;
+    const auto found = std::ranges::find_if(abis, [&](const auto& abi) { return std::ranges::equal(abi.key, key) && abi.registry.lock() == registry; });
+    if (found == abis.end()) return std::nullopt;
+    return found->stages;
+}
+
+bool CommittedLink(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint32_t primitiveType) {
+    if (primitiveType == 7 || primitiveType == 17) return false;
+    std::unique_lock lock(front.prepared->mutex);
+    AwaitRegisteredPreparation(*front.prepared, lock);
+    const auto& fragments = front.prepared->fragments;
+    if (front.prepared->rectangleRequested || std::ranges::any_of(fragments, [](const auto& entry) { return entry.expired(); })) return false;
+    return fragment == nullptr || std::ranges::any_of(fragments, [&](const auto& entry) { return entry.lock() == fragment; });
+}
+
 }
 
 bool ProgramUsesWorkgroup(const ShaderSnapshot& snapshot, std::size_t codeOffset) {
@@ -236,6 +256,216 @@ bool ProgramUsesWorkgroup(const ShaderSnapshot& snapshot, std::size_t codeOffset
     return used;
 }
 
+namespace {
+
+bool readOnlyMemoryOp(ShaderRecompiler::RdnaOpcode op) {
+    using Op = ShaderRecompiler::RdnaOpcode;
+    switch (op) {
+        case Op::SLoadDword:
+        case Op::SLoadDwordx2:
+        case Op::SLoadDwordx4:
+        case Op::SLoadDwordx8:
+        case Op::SLoadDwordx16:
+        case Op::BufferLoadFormatX:
+        case Op::BufferLoadFormatXy:
+        case Op::BufferLoadFormatXyz:
+        case Op::BufferLoadFormatXyzw:
+        case Op::BufferLoadDword:
+        case Op::BufferLoadDwordx2:
+        case Op::BufferLoadDwordx4:
+        case Op::ImageSample:
+        case Op::ImageSampleLz:
+        case Op::ImageSampleBCl:
+        case Op::ImageSampleBClO:
+        case Op::ImageSampleBO:
+        case Op::ImageSampleC:
+        case Op::ImageSampleCB:
+        case Op::ImageSampleCBCl:
+        case Op::ImageSampleCBClO:
+        case Op::ImageSampleCBO:
+        case Op::ImageSampleCCl:
+        case Op::ImageSampleCClO:
+        case Op::ImageSampleCD:
+        case Op::ImageSampleCDCl:
+        case Op::ImageSampleCDClO:
+        case Op::ImageSampleCDO:
+        case Op::ImageSampleCL:
+        case Op::ImageSampleCLzO:
+        case Op::ImageSampleCLO:
+        case Op::ImageSampleCO:
+        case Op::ImageSampleCl:
+        case Op::ImageSampleClO:
+        case Op::ImageSampleD:
+        case Op::ImageSampleDCl:
+        case Op::ImageSampleDO:
+        case Op::ImageSampleLzO:
+        case Op::ImageSampleO:
+        case Op::ImageSampleCd:
+        case Op::ImageSampleCdCl:
+        case Op::ImageSampleCCd:
+        case Op::ImageSampleCCdCl:
+        case Op::ImageSampleCdO:
+        case Op::ImageSampleCdClO:
+        case Op::ImageSampleCCdO:
+        case Op::ImageSampleCCdClO:
+        case Op::ImageSampleDG16:
+        case Op::ImageSampleDClG16:
+        case Op::ImageSampleCDG16:
+        case Op::ImageSampleCDClG16:
+        case Op::ImageSampleDOG16:
+        case Op::ImageSampleDClOG16:
+        case Op::ImageSampleCDOG16:
+        case Op::ImageSampleCDClOG16:
+        case Op::ImageSampleCdG16:
+        case Op::ImageSampleCdClG16:
+        case Op::ImageSampleCCdG16:
+        case Op::ImageSampleCCdClG16:
+        case Op::ImageSampleCdOG16:
+        case Op::ImageSampleCdClOG16:
+        case Op::ImageSampleCCdOG16:
+        case Op::ImageSampleCCdClOG16:
+        case Op::ImageLoad:
+        case Op::ImageGetResinfo:
+        case Op::SBufferLoadDword:
+        case Op::SBufferLoadDwordx2:
+        case Op::SBufferLoadDwordx4:
+        case Op::SBufferLoadDwordx8:
+        case Op::SBufferLoadDwordx16:
+        case Op::SScratchLoadDword:
+        case Op::SScratchLoadDwordx2:
+        case Op::SScratchLoadDwordx4:
+        case Op::BufferLoadUbyte:
+        case Op::BufferLoadSbyte:
+        case Op::BufferLoadUshort:
+        case Op::BufferLoadSshort:
+        case Op::BufferLoadDwordx3:
+        case Op::BufferLoadUbyteD16:
+        case Op::BufferLoadUbyteD16Hi:
+        case Op::BufferLoadSbyteD16:
+        case Op::BufferLoadSbyteD16Hi:
+        case Op::BufferLoadShortD16:
+        case Op::BufferLoadShortD16Hi:
+        case Op::TbufferLoadFormatX:
+        case Op::TbufferLoadFormatXy:
+        case Op::TbufferLoadFormatXyz:
+        case Op::TbufferLoadFormatXyzw:
+        case Op::TbufferLoadFormatD16X:
+        case Op::TbufferLoadFormatD16Xy:
+        case Op::TbufferLoadFormatD16Xyz:
+        case Op::TbufferLoadFormatD16Xyzw:
+        case Op::BufferLoadFormatD16X:
+        case Op::BufferLoadFormatD16Xy:
+        case Op::BufferLoadFormatD16Xyz:
+        case Op::BufferLoadFormatD16Xyzw:
+        case Op::BufferLoadFormatD16HiX:
+        case Op::FlatLoadUbyte:
+        case Op::FlatLoadSbyte:
+        case Op::FlatLoadUshort:
+        case Op::FlatLoadSshort:
+        case Op::FlatLoadDword:
+        case Op::FlatLoadDwordx2:
+        case Op::FlatLoadDwordx3:
+        case Op::FlatLoadDwordx4:
+        case Op::GlobalLoadDwordAddtid:
+        case Op::FlatLoadUbyteD16:
+        case Op::FlatLoadUbyteD16Hi:
+        case Op::FlatLoadSbyteD16:
+        case Op::FlatLoadSbyteD16Hi:
+        case Op::FlatLoadShortD16:
+        case Op::FlatLoadShortD16Hi:
+        case Op::ImageGetLod:
+        case Op::ImageLoadMip:
+        case Op::ImageLoadPck:
+        case Op::ImageLoadPckSgn:
+        case Op::ImageLoadMipPck:
+        case Op::ImageLoadMipPckSgn:
+        case Op::ImageLoadBy2:
+        case Op::ImageLoadBy4:
+        case Op::ImageLoadMipBy2:
+        case Op::ImageLoadMipBy4:
+        case Op::ImageLoadPck2:
+        case Op::ImageLoadPck4:
+        case Op::ImageLoadMipPck2:
+        case Op::ImageLoadMipPck4:
+        case Op::ImageMsaaLoad:
+        case Op::ImageGather4Lz:
+        case Op::ImageGather4C:
+        case Op::ImageGather4CLz:
+        case Op::ImageGather4LzO:
+        case Op::ImageGather4CO:
+        case Op::ImageGather4CLzO:
+        case Op::ImageGather4h:
+        case Op::ImageGather4hPck:
+        case Op::ImageGather4:
+        case Op::ImageGather4B:
+        case Op::ImageGather4BCl:
+        case Op::ImageGather4BClO:
+        case Op::ImageGather4BO:
+        case Op::ImageGather4Cl:
+        case Op::ImageGather4ClO:
+        case Op::ImageGather4CB:
+        case Op::ImageGather4CBCl:
+        case Op::ImageGather4CBClO:
+        case Op::ImageGather4CBO:
+        case Op::ImageGather4CCl:
+        case Op::ImageGather4CClO:
+        case Op::ImageGather4CL:
+        case Op::ImageGather4CLO:
+        case Op::ImageGather4L:
+        case Op::ImageGather4LO:
+        case Op::ImageGather4O:
+        case Op::ImageBvhIntersectRay:
+        case Op::SGl1Inv:
+        case Op::SDcacheInv:
+        case Op::SDcacheWb:
+        case Op::SMemtime:
+        case Op::SMemrealtime:
+        case Op::SDcacheDiscard:
+        case Op::SDcacheDiscardX2:
+        case Op::BufferGl0Inv:
+        case Op::BufferGl1Inv:
+        return true;
+    default:
+        return false;
+    }
+}
+
+bool decodedWritesMemory(std::span<const std::uint32_t> code) {
+    using Family = ShaderRecompiler::RdnaInstructionFamily;
+    using Op = ShaderRecompiler::RdnaOpcode;
+    try {
+        const auto decoded = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code);
+        return std::any_of(decoded.instructions.begin(), decoded.instructions.end(), [](const auto& instruction) {
+            switch (instruction.family) {
+            case Family::MUBUF: case Family::MTBUF: case Family::FLAT: case Family::MIMG: case Family::SMEM:
+                return !readOnlyMemoryOp(instruction.op);
+            case Family::DS:
+                return instruction.gds || instruction.op == Op::DsGwsInit || instruction.op == Op::DsGwsBarrier || instruction.op == Op::DsGwsSemaV || instruction.op == Op::DsGwsSemaP || instruction.op == Op::DsGwsSemaBr || instruction.op == Op::DsGwsSemaReleaseAll || instruction.op == Op::DsOrderedCount || instruction.op == Op::DsAppend || instruction.op == Op::DsConsume;
+            case Family::Unknown:
+                return true;
+            default:
+                return instruction.op == Op::SSetpcB64 || instruction.op == Op::SSwappcB64 || instruction.op == Op::SRfeB64;
+            }
+        });
+    } catch (const std::exception&) {
+        return true;
+    }
+}
+
+}
+
+bool ProgramWritesMemory(const ShaderSnapshot& snapshot, std::size_t codeOffset) {
+    constexpr std::uint8_t Unknown = 0, Reads = 1, Writes = 2;
+    if (codeOffset == 0) {
+        const auto cached = snapshot.memoryWrites->load(std::memory_order_acquire);
+        if (cached != Unknown) return cached == Writes;
+    }
+    if (codeOffset > snapshot.code.size()) return true;
+    const bool writes = decodedWritesMemory(std::span(snapshot.code).subspan(codeOffset));
+    if (codeOffset == 0) snapshot.memoryWrites->store(writes ? Writes : Reads, std::memory_order_release);
+    return writes;
+}
+
 std::uint32_t PositionExportLayout(const ShaderSnapshot* snapshot, std::size_t codeOffset, std::span<const std::uint32_t> code, std::uint32_t paClVsOutCntl, const ShaderSnapshot* continuation) {
     constexpr std::uint8_t Unknown = 0, Unused = 1, Used = 2;
     const auto bits = paClVsOutCntl & Graphics::PositionExportLayoutBits;
@@ -351,6 +581,10 @@ const PreparedShaderState& ShaderPreparationTransaction::Read(const ShaderSnapsh
     return changes.emplace(snapshot.prepared.get(), State::Change{snapshot.prepared, *snapshot.prepared, false, snapshot.prepared->entries.size(), snapshot.prepared->registeredAbis.size()}).first->second.prepared;
 }
 
+bool ShaderPreparationTransaction::Tracks(const ShaderSnapshot& snapshot) const {
+    return root->changes.contains(snapshot.prepared.get());
+}
+
 void ShaderPreparationTransaction::Hold() {
     require(!committed, "shader preparation transaction is already committed");
     root->Hold();
@@ -591,6 +825,65 @@ void PublishRegisteredShader(std::shared_ptr<ShaderRegistry>& registry, const st
 
 namespace {
 
+bool LazyShaderPreparation() {
+    static const bool lazy = std::getenv("APS5_LAZY_SHADER_PREPARE") != nullptr;
+    return lazy;
+}
+
+using PreparedAtUse = std::optional<std::vector<const ShaderRecompiler::SourceHandle*>>;
+
+PreparedAtUse PrepareDeferredAbis(const ShaderSnapshot& snapshot, std::unique_lock<std::mutex>& lock);
+
+struct DeferredAbiUseCounters {
+    std::atomic<std::uint64_t> firstUses{0};
+    std::atomic<std::uint64_t> found{0};
+    std::atomic<std::uint64_t> own{0};
+    std::atomic<std::uint64_t> entries{0};
+    std::atomic<std::uint64_t> matchedEntries{0};
+    std::atomic<std::int64_t> lastReport{0};
+};
+
+DeferredAbiUseCounters& DeferredAbiCounters() {
+    static DeferredAbiUseCounters counters;
+    return counters;
+}
+
+void ReportDeferredAbiUses() {
+    static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
+    if (!profile) return;
+    auto& counters = DeferredAbiCounters();
+    const auto now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
+    auto previous = counters.lastReport.load(std::memory_order_relaxed);
+    if ((previous != 0 && now - previous < 60000) || !counters.lastReport.compare_exchange_strong(previous, now, std::memory_order_relaxed)) return;
+    const auto entries = counters.entries.load(std::memory_order_relaxed);
+    const auto matched = counters.matchedEntries.load(std::memory_order_relaxed);
+    AgcDriver::ProfilePrint_nid_no_patch("[shaders] deferred ABIs: %llu prepared at first use, %llu found their artifact there, %llu prepared their own; %llu of the %llu entries prepared from them never matched a use\n", static_cast<unsigned long long>(counters.firstUses.load(std::memory_order_relaxed)), static_cast<unsigned long long>(counters.found.load(std::memory_order_relaxed)), static_cast<unsigned long long>(counters.own.load(std::memory_order_relaxed)), static_cast<unsigned long long>(entries - std::min(entries, matched)), static_cast<unsigned long long>(entries));
+}
+
+void NoteMatchedEntry(PreparedShaders::Entry& entry, const PreparedAtUse& preparedAtUse) {
+    auto& counters = DeferredAbiCounters();
+    bool changed = false;
+    if (entry.fromDeferredAbi && !entry.matched) {
+        entry.matched = true;
+        counters.matchedEntries.fetch_add(1, std::memory_order_relaxed);
+        changed = true;
+    }
+    if (preparedAtUse) {
+        counters.firstUses.fetch_add(1, std::memory_order_relaxed);
+        if (std::ranges::find(*preparedAtUse, entry.handle.get()) != preparedAtUse->end()) counters.found.fetch_add(1, std::memory_order_relaxed);
+        changed = true;
+    }
+    if (changed) ReportDeferredAbiUses();
+}
+
+void NoteOwnPreparation(const PreparedAtUse& preparedAtUse) {
+    if (!preparedAtUse) return;
+    auto& counters = DeferredAbiCounters();
+    counters.firstUses.fetch_add(1, std::memory_order_relaxed);
+    counters.own.fetch_add(1, std::memory_order_relaxed);
+    ReportDeferredAbiUses();
+}
+
 void ReportPreparedAtUse(const ShaderSnapshot& snapshot, const ShaderRecompiler::RecompileRequest& request) {
     if (request.context.convertedSamples) {
         APS5_LOG_OUT_DEBUG("Shader 0x%llx samples a converted image; preparing its converted-sample variant at use", static_cast<unsigned long long>(request.shader.codeAddress));
@@ -626,9 +919,15 @@ std::shared_ptr<const ShaderRecompiler::SourceHandle> SourceHandleFor(const Shad
     ShaderRecompiler::BuildPreparedShaderKey(request, key);
     std::unique_lock lock(snapshot.prepared->mutex);
     AwaitRegisteredPreparation(*snapshot.prepared, lock);
-    for (const auto& entry : snapshot.prepared->entries) {
-        if (entry.codeOffset == codeOffset && ShaderRecompiler::MatchesPreparedShader(request, *entry.handle, key)) return entry.handle;
+    PreparedAtUse preparedAtUse;
+    if (!snapshot.prepared->deferredAbis.empty() || snapshot.prepared->deferredPreparing) preparedAtUse = PrepareDeferredAbis(snapshot, lock);
+    for (auto& entry : snapshot.prepared->entries) {
+        if (entry.codeOffset == codeOffset && ShaderRecompiler::MatchesPreparedShader(request, *entry.handle, key)) {
+            NoteMatchedEntry(entry, preparedAtUse);
+            return entry.handle;
+        }
     }
+    NoteOwnPreparation(preparedAtUse);
     ReportPreparedAtUse(snapshot, request);
     auto handle = PrepareShaderWithDiagnostics(request);
     snapshot.prepared->entries.push_back({codeOffset, handle});
@@ -779,12 +1078,18 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s
     struct PreparedKeyStorage {};
     auto& key = HostThreadLocal<std::vector<std::uint64_t>, PreparedKeyStorage>();
     ShaderRecompiler::BuildPreparedShaderKey(source, key);
-    for (const auto& entry : snapshot.prepared->entries) {
+    PreparedAtUse preparedAtUse;
+    if (!snapshot.prepared->deferredAbis.empty() || snapshot.prepared->deferredPreparing) preparedAtUse = PrepareDeferredAbis(snapshot, lock);
+    for (auto& entry : snapshot.prepared->entries) {
         if (entry.codeOffset != codeOffset) continue;
         invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*entry.handle);
         if (!linked.code.empty() && !std::ranges::equal(invocationRequest.shader.code, linked.code)) continue;
-        if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, entry.handle, key)) return std::move(*invocation);
+        if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, entry.handle, key)) {
+            NoteMatchedEntry(entry, preparedAtUse);
+            return std::move(*invocation);
+        }
     }
+    NoteOwnPreparation(preparedAtUse);
     ReportPreparedAtUse(snapshot, source);
     auto handle = PrepareShaderWithDiagnostics(source);
     invocationRequest = source;
@@ -1071,6 +1376,60 @@ std::vector<PreparedShaders::Entry> PrepareRegistered(const ShaderSnapshot& snap
     return plan != nullptr ? PrepareRegistered(*plan) : std::vector<PreparedShaders::Entry>{};
 }
 
+PreparedAtUse PrepareDeferredAbis(const ShaderSnapshot& snapshot, std::unique_lock<std::mutex>& lock) {
+    auto& prepared = *snapshot.prepared;
+    if (prepared.deferredPreparing) {
+        prepared.settled.wait(lock, [&] { return !prepared.deferredPreparing; });
+        return std::nullopt;
+    }
+    if (prepared.deferredAbis.empty()) return std::nullopt;
+    std::vector<PreparedShaders::DeferredAbi> deferred;
+    deferred.swap(prepared.deferredAbis);
+    prepared.deferredPreparing = true;
+    struct Preparing {
+        PreparedShaders& prepared;
+        std::unique_lock<std::mutex>& lock;
+        ~Preparing() {
+            if (!lock.owns_lock()) lock.lock();
+            prepared.deferredPreparing = false;
+            prepared.settled.notify_all();
+        }
+    } preparing{prepared, lock};
+    lock.unlock();
+    std::vector<PreparedShaders::Entry> entries;
+    for (const auto& abi : deferred) {
+        const auto device = abi.device.lock();
+        if (device == nullptr || snapshot.registeredState == nullptr) continue;
+        QueueState state{};
+        state.shader = snapshot.registeredState->shader;
+        state.context = snapshot.registeredState->context;
+        state.userConfig = snapshot.registeredState->userConfig;
+        for (const auto& [offset, value] : abi.context) state.context[offset] = value;
+        for (const auto& [offset, value] : abi.primitive) state.userConfig[offset] = value;
+        try {
+            const auto plan = PlanRegistered(snapshot, *device, state, false);
+            if (plan == nullptr) continue;
+            plan->prebuildDevice = device;
+            auto made = PrepareRegistered(*plan);
+            std::ranges::move(made, std::back_inserter(entries));
+        } catch (const std::exception& error) {
+            APS5_LOG_ERR("Shader 0x%llx could not be prepared at its first use for its resolved ABI (%s); preparing it for the use instead", static_cast<unsigned long long>(snapshot.codeAddress), error.what());
+            continue;
+        }
+    }
+    lock.lock();
+    std::vector<const ShaderRecompiler::SourceHandle*> added;
+    for (auto& entry : entries) {
+        const auto duplicate = std::ranges::any_of(prepared.entries, [&](const auto& existing) { return existing.codeOffset == entry.codeOffset && existing.handle->artifact == entry.handle->artifact; });
+        if (duplicate) continue;
+        entry.fromDeferredAbi = true;
+        added.push_back(entry.handle.get());
+        prepared.entries.push_back(std::move(entry));
+    }
+    DeferredAbiCounters().entries.fetch_add(added.size(), std::memory_order_relaxed);
+    return added;
+}
+
 void DeferPipelineStage(PreparedShaderState& prepared, const ShaderSnapshot& snapshot, const char* reason) {
     if (!prepared.deferred) APS5_LOG_ERR("Shader 0x%llx could not be prepared when the title created its pipeline (%s); preparing it when it is used", static_cast<unsigned long long>(snapshot.codeAddress), reason);
     prepared.deferred = true;
@@ -1146,9 +1505,8 @@ public:
             Closing() = true;
             pools = {Existing(), ExistingPrebuild()};
         }
-        for (auto* pool : pools) {
-            if (pool != nullptr) pool->Close();
-        }
+        if (pools[0] != nullptr) pools[0]->Close(false);
+        if (pools[1] != nullptr) pools[1]->Close(true);
     }
 
 private:
@@ -1204,11 +1562,17 @@ public:
         idle.wait(lock, [&] { return unfinished.empty() || *unfinished.begin() > ticket; });
     }
 
-    void Close() {
+    void Close(bool dropQueued) {
         std::vector<Worker> joined;
+        std::deque<Task> dropped;
         {
             std::unique_lock lock(mutex);
             closed = true;
+            if (dropQueued) {
+                dropped.swap(tasks);
+                for (const auto& task : dropped) unfinished.erase(task.sequence);
+                idle.notify_all();
+            }
             idle.wait(lock, [&] { return tasks.empty() && running == 0; });
             joined.swap(workers);
         }
@@ -1435,7 +1799,10 @@ std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decod
             prepared.push_back({program.snapshot, {program.codeOffset, nullptr}, program.binary.stage, offset, nullptr});
             continue;
         }
-        if (vertex) vertex->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, decoded.state.paClVsOutCntl, program.tailContinuation.get()) | decoded.state.layerExports;
+        if (vertex) {
+            vertex->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, decoded.state.paClVsOutCntl, program.tailContinuation.get()) | decoded.state.layerExports;
+            Graphics::ApplyPointSize(decoded.state, *vertex);
+        }
         ShaderRecompiler::RecompileRequest request{program.binary, {wave, program.firstUserSgpr, program.userData, {}, fragment ? std::optional(decoded.pixel) : std::nullopt, vertex, memory, RegisteredFloatMode(*program.snapshot)}, target, {0, 0, offset, room}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, decoded.state.stages.mesh, decoded.state.stages.tessellation, {}}};
         std::vector<std::uint64_t> key;
         ShaderRecompiler::BuildPreparedShaderKey(request, key);
@@ -1531,14 +1898,23 @@ void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std
     auto& key = HostThreadLocal<std::vector<std::uint64_t>, GraphicsAbiKeyStorage>();
     BuildRegisteredAbiKey(state, *localDevice, key);
     const auto primitiveType = state.userConfig.find(0x242u);
-    for (const auto& abi : transaction.Read(*owner).graphicsAbis) {
-        if (abi.key != key || abi.registry.lock() != registry) continue;
-        for (const auto& stage : abi.stages) {
+    const auto resolveStages = [&](const std::vector<std::weak_ptr<const ShaderSnapshot>>& stages) {
+        for (const auto& stage : stages) {
             const auto snapshot = stage.lock();
             require(snapshot != nullptr, "prepared graphics ABI lost a registered stage");
             if (snapshot->type != 1) ResolvePreparedGraphics(*snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target(), true);
         }
         transaction.Commit();
+    };
+    if (!transaction.Tracks(*owner)) {
+        if (const auto committed = CommittedGraphicsAbi(*owner, key, registry)) {
+            resolveStages(*committed);
+            return;
+        }
+    }
+    for (const auto& abi : transaction.Read(*owner).graphicsAbis) {
+        if (abi.key != key || abi.registry.lock() != registry) continue;
+        resolveStages(abi.stages);
         return;
     }
     DrawDecode decoded{};
@@ -1574,6 +1950,55 @@ void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std
     transaction.Commit();
 }
 
+std::size_t Driver::PreparedEntries(const Shader* shader) {
+    std::shared_ptr<const ShaderSnapshot> snapshot;
+    {
+        std::lock_guard lock(mutex);
+        if (shaders == nullptr) return 0;
+        snapshot = RegisteredHeader(*shaders, shader);
+    }
+    if (snapshot == nullptr) return 0;
+    std::unique_lock lock(snapshot->prepared->mutex);
+    AwaitRegisteredPreparation(*snapshot->prepared, lock);
+    return snapshot->prepared->entries.size();
+}
+
+Driver::DeferredAbiUseCounts Driver::DeferredAbiUses() {
+    auto& counters = DeferredAbiCounters();
+    const auto entries = counters.entries.load(std::memory_order_relaxed);
+    const auto matched = counters.matchedEntries.load(std::memory_order_relaxed);
+    return {counters.firstUses.load(std::memory_order_relaxed), counters.found.load(std::memory_order_relaxed), counters.own.load(std::memory_order_relaxed), entries, entries - std::min(entries, matched)};
+}
+
+std::vector<std::array<std::uint64_t, 2>> Driver::PreparedEntryHashes(const Shader* shader) {
+    std::shared_ptr<const ShaderSnapshot> snapshot;
+    {
+        std::lock_guard lock(mutex);
+        if (shaders == nullptr) return {};
+        snapshot = RegisteredHeader(*shaders, shader);
+    }
+    if (snapshot == nullptr) return {};
+    std::unique_lock lock(snapshot->prepared->mutex);
+    AwaitRegisteredPreparation(*snapshot->prepared, lock);
+    std::vector<std::array<std::uint64_t, 2>> hashes;
+    for (const auto& entry : snapshot->prepared->entries) {
+        if (entry.handle == nullptr || entry.handle->artifact == nullptr) continue;
+        const auto hash = entry.handle->artifact->artifact.spirv.ContentHash();
+        std::uint64_t key = 0xcbf29ce484222325ull;
+        for (const auto word : entry.handle->staticKey) key = (key ^ word) * 0x100000001b3ull;
+        hashes.push_back({hash.low, hash.high ^ key});
+    }
+    std::ranges::sort(hashes);
+    return hashes;
+}
+
+std::array<std::uint64_t, 4> Driver::PrebuiltLibraries() {
+    if (auto* pool = RegistrationPreparePool::ExistingPrebuild()) pool->AwaitSubmitted();
+    const auto localDevice = device.Load();
+    if (localDevice == nullptr) return {};
+    return Graphics::PipelineLibraryCountersOf(localDevice->Device()).built;
+}
+
 void Driver::ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive) {
     BindTitleThread();
     PerformanceContext timingContext(FrameTiming::Preparation());
@@ -1611,6 +2036,31 @@ void Driver::ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegist
     struct RegisteredAbiKeyStorage {};
     auto& key = HostThreadLocal<std::vector<std::uint64_t>, RegisteredAbiKeyStorage>();
     BuildRegisteredAbiKey(state, *localDevice, key);
+    if (LazyShaderPreparation()) {
+        const auto& resolved = transaction.Read(*snapshot).registeredAbis;
+        if (std::ranges::find(resolved, key) != resolved.end()) {
+            transaction.Commit();
+            return;
+        }
+        std::optional<std::string> invalid;
+        try {
+            static_cast<void>(PlanRegistered(*snapshot, *localDevice, state, false));
+        } catch (const std::exception& error) {
+            invalid = error.what();
+        }
+        auto& prepared = transaction.Edit(*snapshot);
+        if (invalid) DeferPipelineStage(prepared, *snapshot, invalid->c_str());
+        prepared.registeredAbis.push_back(key);
+        prepared.deferred = true;
+        transaction.Commit();
+        if (invalid) return;
+        PreparedShaders::DeferredAbi deferred{key, {}, {}, localDevice};
+        for (const auto reg : context) deferred.context.emplace_back(reg.offset, reg.value);
+        for (const auto reg : primitive) deferred.primitive.emplace_back(reg.offset, reg.value);
+        std::lock_guard lock(snapshot->prepared->mutex);
+        snapshot->prepared->deferredAbis.push_back(std::move(deferred));
+        return;
+    }
     std::optional<std::string> deferral;
     const auto prepare = [&] {
         try {
@@ -1624,6 +2074,10 @@ void Driver::ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegist
         }
     };
     const bool committedAbi = CommittedAbi(*snapshot, key);
+    if (committedAbi && !transaction.Tracks(*snapshot)) {
+        transaction.Commit();
+        return;
+    }
     auto entries = committedAbi ? std::vector<PreparedShaders::Entry>{} : prepare();
     const auto& registeredAbis = transaction.Read(*snapshot).registeredAbis;
     if (std::ranges::find(registeredAbis, key) != registeredAbis.end()) {
@@ -1709,6 +2163,10 @@ void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<
             return;
         }
     }
+    if (!transaction.Tracks(front) && CommittedLink(front, fragment, primitiveType)) {
+        transaction.Commit();
+        return;
+    }
     std::vector<std::shared_ptr<const ShaderSnapshot>> fragments;
     {
         const auto& current = transaction.Read(front);
@@ -1846,13 +2304,24 @@ void Driver::RegisterShader(const Shader* shader) {
         localDevice = device;
     }
     snapshot.registeredState = std::make_shared<const RegisteredShaderState>(DecodeRegisteredState(snapshot));
+    {
+        std::lock_guard lock(mutex);
+        if (shaders != nullptr) {
+            if (const auto found = shaders->find(snapshot.codeAddress); found != shaders->end()) {
+                for (const auto& current : found->second) {
+                    if (current->registeredState == nullptr || current->type != snapshot.type || current->code != snapshot.code || *current->registeredState != *snapshot.registeredState) continue;
+                    snapshot.registeredState = current->registeredState;
+                    break;
+                }
+            }
+        }
+    }
     QueueState registered{};
     registered.shader = snapshot.registeredState->shader;
     registered.context = snapshot.registeredState->context;
     registered.userConfig = snapshot.registeredState->userConfig;
     std::shared_ptr<RegisteredPreparation> preparation;
-    static const bool lazy = std::getenv("APS5_LAZY_SHADER_PREPARE") != nullptr;
-    if (lazy) {
+    if (LazyShaderPreparation()) {
         static_cast<void>(PlanRegistered(snapshot, *localDevice, registered, true, &snapshot.prepared->deferred));
         snapshot.prepared->deferred = true;
     } else if (AsyncRegistrationPrepare()) {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/DeferredLabels.cpp

@@ -2,14 +2,20 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libc/include/HostThreadSlot.hpp"
 #include "ThreadOwned.hpp"
 #include <cstdlib>
 
 namespace AgcDriver::DriverDetail {
 
 DeferredLabels& deferredLabels() {
-    static thread_local DeferredLabels* deferred = nullptr;
-    return ShaderRecompiler::ThreadOwned(deferred);
+    struct Slot {};
+    auto* deferred = HostThreadSlot<DeferredLabels*, Slot>::Get();
+    if (deferred == nullptr) {
+        ShaderRecompiler::ThreadOwned(deferred);
+        HostThreadSlot<DeferredLabels*, Slot>::Set(deferred);
+    }
+    return *deferred;
 }
 
 bool DeferLabels() {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/LabelHistory.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 #include <cstdlib>
 #include <cstring>
 #include <string_view>
@@ -60,7 +61,7 @@ void Driver::traceLabel(std::span<const std::uint32_t> packet, std::uint32_t que
     }
     if (kind == nullptr) return;
     if (std::string_view(kind) != "WAIT_REG_MEM") {
-        static std::mutex historyMutex;
+        static AgcDriver::Mutex historyMutex;
         std::lock_guard lock(historyMutex);
         auto& entry = writeHistory()[writeCursor()++ % writeHistory().size()];
         entry = {target, length, queue, opcode};

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -1,12 +1,14 @@
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/HandoffSpin.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp"
 #include "prx/libSceAgcDriver/Execution/include/WriteWatchCoverage.hpp"
 #include "ThreadOwned.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
 #include "prx/libc/include/GuestArena.hpp"
 #include "prx/libc/include/GuestWriteWatch.hpp"
+#include "prx/libc/include/HostThreadSlot.hpp"
 #include <mutex>
 #include <array>
 #include <chrono>
@@ -20,6 +22,7 @@
 #include <cstring>
 #include <limits>
 #include <memory>
+#include <new>
 #include <stdexcept>
 #include <string>
 #include <tuple>
@@ -152,13 +155,13 @@ struct MemoryCounters {
 };
 
 struct MemoryProfile {
-    std::mutex threadsMutex;
+    AgcDriver::Mutex threadsMutex;
     std::vector<const MemoryCounters*> threads;
     MemoryCounters retired;
     MemoryCounters shared;
     std::atomic<std::int64_t> lastReport{0};
     // Sampled callers of Read and Write (module offsets), to name what touches guest memory most.
-    std::mutex callersMutex;
+    AgcDriver::Mutex callersMutex;
     std::array<std::pair<unsigned long long, std::uint64_t>, 16> callers{};
     std::array<std::pair<unsigned long long, std::uint64_t>, 16> writeCallers{};
 };
@@ -222,19 +225,19 @@ void CountCaller(std::array<std::pair<unsigned long long, std::uint64_t>, 16>& c
     }
 }
 
-// Attribution tags (see the header): the packet a queue worker executes and the read site the
-// driver code named. Plain thread-locals; a store per packet costs nothing measurable.
-thread_local PacketTag currentPacket{NoPacket, 0xffffffffu};
-thread_local ReadSite currentReadSite = ReadSite::Unknown;
+struct CurrentPacketSlot {};
+struct CurrentReadSiteSlot {};
+constexpr std::uint64_t NoPacketBits = NoPacket | (std::uint64_t{0xffffffffu} << 32u);
 // Sampled reads by site (every 256th Read/ReadCommitted, like the callers), for the [guestmem] line.
 std::atomic<std::uint64_t> readSiteSamples[static_cast<std::size_t>(ReadSite::Count)] = {};
 
 void SetCurrentPacket(std::uint32_t opcode, std::uint32_t queue) {
-    currentPacket = {opcode, queue};
+    HostThreadSlot<std::uint64_t, CurrentPacketSlot>::Set((opcode | (std::uint64_t{queue} << 32u)) ^ NoPacketBits);
 }
 
 PacketTag CurrentPacket() {
-    return currentPacket;
+    const auto bits = HostThreadSlot<std::uint64_t, CurrentPacketSlot>::Get() ^ NoPacketBits;
+    return {static_cast<std::uint32_t>(bits), static_cast<std::uint32_t>(bits >> 32u)};
 }
 
 const char* ReadSiteName(ReadSite site) {
@@ -244,11 +247,13 @@ const char* ReadSiteName(ReadSite site) {
 }
 
 ReadSite SetReadSite(ReadSite site) {
-    return std::exchange(currentReadSite, site);
+    const auto previous = HostThreadSlot<ReadSite, CurrentReadSiteSlot>::Get();
+    HostThreadSlot<ReadSite, CurrentReadSiteSlot>::Set(site);
+    return previous;
 }
 
 ReadSite CurrentReadSite() {
-    return currentReadSite;
+    return HostThreadSlot<ReadSite, CurrentReadSiteSlot>::Get();
 }
 
 std::size_t CaptureCallerOffsets(std::span<unsigned long long> frames, unsigned skip) {
@@ -289,7 +294,7 @@ std::size_t CaptureCallerOffsets(std::span<unsigned long long> frames, unsigned
 void CountReadCaller(const void* returnAddress) {
     thread_local std::uint32_t sampled = 0;
     // Sampled at the same rate as the callers, in step with them (CountCaller advances the counter).
-    if ((sampled + 1) % 256 == 0) readSiteSamples[static_cast<std::size_t>(currentReadSite)].fetch_add(256, std::memory_order_relaxed);
+    if ((sampled + 1) % 256 == 0) readSiteSamples[static_cast<std::size_t>(CurrentReadSite())].fetch_add(256, std::memory_order_relaxed);
     CountCaller(Profile().callers, sampled, 256, returnAddress);
 }
 
@@ -368,18 +373,38 @@ struct PageSpan {
     std::size_t size = 0;
     // Per page: access flags, zero while unknown.
     std::uint8_t* pages = nullptr;
+    static constexpr std::size_t ChunkEntries = 65536;
+    struct alignas(8) Chunk { std::array<std::uint8_t, ChunkEntries> entries{}; };
+    std::atomic<Chunk*>* chunks = nullptr;
+
+    std::uint8_t* chunkData(std::size_t index, bool create) const {
+        auto* chunk = chunks[index].load(std::memory_order_acquire);
+        if (chunk == nullptr && create) {
+            auto fresh = std::unique_ptr<Chunk>(new (std::nothrow) Chunk{});
+            if (fresh != nullptr && chunks[index].compare_exchange_strong(chunk, fresh.get(), std::memory_order_acq_rel, std::memory_order_acquire)) chunk = fresh.release();
+        }
+        return chunk == nullptr ? nullptr : chunk->entries.data();
+    }
+
+    std::uint8_t* entry(std::size_t index, bool create) const {
+        if (chunks == nullptr) return pages + index;
+        auto* data = chunkData(index / ChunkEntries, create);
+        return data == nullptr ? nullptr : data + index % ChunkEntries;
+    }
 
-    // Reserves and commits the table; untouched table pages cost nothing until a guest page is recorded.
     bool allocate() {
         if (size == 0) return false;
         const auto count = (size / PageBytes + 8) & ~static_cast<std::size_t>(7);
 #ifdef _WIN32
-        pages = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, count, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        const auto chunkCount = (count + ChunkEntries - 1) / ChunkEntries;
+        chunks = new (std::nothrow) std::atomic<Chunk*>[chunkCount]{};
+        if (chunks == nullptr) size = 0;
+        return chunks != nullptr;
 #else
         pages = static_cast<std::uint8_t*>(std::calloc(count, 1));
-#endif
         if (pages == nullptr) size = 0;
         return pages != nullptr;
+#endif
     }
 
     bool covers(std::uintptr_t address) const {
@@ -387,11 +412,23 @@ struct PageSpan {
     }
 
     std::uint8_t load(std::uintptr_t address) const {
-        return std::atomic_ref<std::uint8_t>(const_cast<std::uint8_t&>(pages[(address - base) / PageBytes])).load(std::memory_order_relaxed);
+        auto* value = entry((address - base) / PageBytes, false);
+        return value == nullptr ? 0 : std::atomic_ref<std::uint8_t>(*value).load(std::memory_order_relaxed);
     }
 
     void store(std::uintptr_t address, std::uint8_t value) {
-        std::atomic_ref<std::uint8_t>(pages[(address - base) / PageBytes]).store(value, std::memory_order_relaxed);
+        if (auto* at = entry((address - base) / PageBytes, value != 0 && value != PageInaccessible)) std::atomic_ref<std::uint8_t>(*at).store(value, std::memory_order_relaxed);
+    }
+
+    void store(std::uintptr_t first, std::uintptr_t last, std::uint8_t value) {
+        auto index = (first - base) / PageBytes;
+        const auto stop = (last - 1 - base) / PageBytes + 1;
+        while (index < stop) {
+            const auto next = std::min(stop, (index / ChunkEntries + 1) * ChunkEntries);
+            if (auto* at = entry(index, value != 0 && value != PageInaccessible)) {
+                for (; index < next; ++index, ++at) std::atomic_ref<std::uint8_t>(*at).store(value, std::memory_order_relaxed);
+            } else index = next;
+        }
     }
 
     // End of the run of pages equal to `value` that starts at `address`, at most `limit`.
@@ -400,11 +437,17 @@ struct PageSpan {
         const auto stop = std::min<std::uintptr_t>((std::min(limit, base + size) - base + PageBytes - 1) / PageBytes, size / PageBytes);
         const std::uint64_t wide = 0x0101010101010101ull * value;
         while (index < stop) {
-            if (index % 8 == 0 && index + 8 <= stop && std::atomic_ref<std::uint64_t>(*reinterpret_cast<std::uint64_t*>(const_cast<std::uint8_t*>(pages + index))).load(std::memory_order_relaxed) == wide) {
+            auto* at = entry(index, false);
+            if (at == nullptr) {
+                if (value != 0) break;
+                index = std::min(stop, (index / ChunkEntries + 1) * ChunkEntries);
+                continue;
+            }
+            if (index % 8 == 0 && index + 8 <= stop && std::atomic_ref<std::uint64_t>(*reinterpret_cast<std::uint64_t*>(at)).load(std::memory_order_relaxed) == wide) {
                 index += 8;
                 continue;
             }
-            if (std::atomic_ref<std::uint8_t>(const_cast<std::uint8_t&>(pages[index])).load(std::memory_order_relaxed) != value) break;
+            if (std::atomic_ref<std::uint8_t>(*at).load(std::memory_order_relaxed) != value) break;
             ++index;
         }
         return std::min(limit, base + index * PageBytes);
@@ -414,7 +457,7 @@ struct PageSpan {
         if (size == 0 || bytes == 0 || address >= base + size || address + bytes <= base) return;
         const auto first = (std::max(address, base) - base) / PageBytes;
         const auto last = (std::min(address + bytes, base + size) - 1 - base) / PageBytes;
-        for (auto page = first; page <= last; ++page) std::atomic_ref<std::uint8_t>(pages[page]).store(0, std::memory_order_relaxed);
+        store(base + first * PageBytes, base + (last + 1) * PageBytes, 0);
     }
 };
 
@@ -445,7 +488,7 @@ struct PageStates {
             const std::uint8_t value = readable ? static_cast<std::uint8_t>(PageReadable | (writable ? PageWritable : 0u)) : PageInaccessible;
             const auto first = std::max(base, span->base);
             const auto last = std::min(regionEnd, span->base + span->size);
-            for (auto at = first; at < last; at += PageBytes) span->store(at, value);
+            span->store(first, last, value);
             if (GuestAllocations::GuestAllocationsGeneration_nid_postfix() != generation || accessSerial.load(std::memory_order_acquire) != access) span->forget(first, static_cast<std::size_t>(last - first));
         }
     }
@@ -848,7 +891,7 @@ void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_
 #endif
 
 struct WriteTracker {
-    std::mutex mutex;
+    AgcDriver::Mutex mutex;
     bool initialized = false;
     bool watched = false;
     static constexpr std::size_t CoarseBlocks = 64;
@@ -874,6 +917,7 @@ struct WriteTracker {
     // bumps, or an image refreshed after a stamp would keep failing UnchangedSince until the next
     // walk of its range and re-upload at every use of the epoch.
     std::atomic<std::uint32_t> generation{1};
+    std::atomic<std::uint32_t> stable{1};
     std::vector<void*> pages;
     // Collect memo: page-rounded ranges walked during the current epoch (see BumpCollectEpoch). A
     // collect fully inside one skips GetWriteWatch, whose cost is proportional to the range's pages,
@@ -971,8 +1015,10 @@ struct WriteTracker {
     struct DriverPieces {
         std::uint32_t whole = 0;
         std::uint32_t dropped = 0;
-        std::uint32_t next = 0;
         std::array<DriverPiece, 4> pieces{};
+        std::uint32_t possibleWhole = 0;
+        std::uint32_t possibleDropped = 0;
+        std::array<DriverPiece, 4> possiblePieces{};
 
         bool covers(std::uint32_t begin, std::uint32_t end, std::uint64_t generation) const {
             if (generation == 0 || begin >= end || dropped > generation) return false;
@@ -1025,7 +1071,7 @@ struct WriteTracker {
 #endif
     }
 
-    void stampWatchedPart(std::uint64_t address, std::uint64_t end, std::uint32_t stampGeneration) {
+    void stampWatchedPart(std::uint64_t address, std::uint64_t end, std::uint32_t stampGeneration, bool complete) {
 #ifdef _WIN32
         address = std::max<std::uint64_t>(address, base);
         end = std::min<std::uint64_t>(end, base + size);
@@ -1039,24 +1085,45 @@ struct WriteTracker {
             const auto stop = std::min<std::uint64_t>(end, blockBegin(block) + WriteBlockBytes);
             if (!blockPartWatched(first, stop)) continue;
             stamp(block, stampGeneration, StampKind::Driver);
-            noteDriverStore(block, address, end, stampGeneration);
+            noteDriverStore(block, address, end, stampGeneration, complete);
         }
     }
 
-    void noteDriverStore(std::uint64_t block, std::uint64_t address, std::uint64_t end, std::uint32_t stampGeneration) {
+    void noteDriverStore(std::uint64_t block, std::uint64_t address, std::uint64_t end, std::uint32_t stampGeneration, bool complete) {
         const auto begin = blockBegin(block);
         const auto from = static_cast<std::uint32_t>(std::max(address, begin) - begin);
         const auto to = static_cast<std::uint32_t>(std::min<std::uint64_t>(end, begin + WriteBlockBytes) - begin);
         if (from == 0 && to == WriteBlockBytes) {
             auto& entry = driverPieces[block];
-            entry = DriverPieces{};
-            entry.whole = stampGeneration;
+            if (complete) {
+                entry = DriverPieces{};
+                entry.whole = stampGeneration;
+            } else {
+                entry.possibleWhole = stampGeneration;
+                entry.possibleDropped = 0;
+                entry.possiblePieces = {};
+            }
             return;
         }
         auto& entry = driverPieces[block];
-        auto& slot = entry.pieces[entry.next++ % entry.pieces.size()];
-        entry.dropped = std::max(entry.dropped, slot.generation);
-        slot = {stampGeneration, from, to};
+        auto& pieces = complete ? entry.pieces : entry.possiblePieces;
+        auto& dropped = complete ? entry.dropped : entry.possibleDropped;
+        DriverPiece* contained = nullptr;
+        for (auto& piece : pieces) {
+            if (piece.generation == 0 || piece.begin < from || piece.end > to) continue;
+            if (contained == nullptr) contained = &piece;
+            else piece = {};
+        }
+        if (contained != nullptr) {
+            *contained = {stampGeneration, from, to};
+            return;
+        }
+        auto* oldest = &pieces.front();
+        for (auto& piece : pieces) {
+            if (piece.generation < oldest->generation) oldest = &piece;
+        }
+        dropped = std::max(dropped, oldest->generation);
+        *oldest = {stampGeneration, from, to};
     }
 
     Leaf& leafFor(std::uint64_t block) {
@@ -1167,10 +1234,13 @@ struct WriteTracker {
                 continue;
             }
             const auto& entry = found->second;
-            if (entry.whole > generation || entry.dropped > generation) return true;
+            if (entry.whole > generation || entry.dropped > generation || entry.possibleWhole > generation || entry.possibleDropped > generation) return true;
             for (const auto& piece : entry.pieces) {
                 if (piece.generation > generation && piece.begin < to && from < piece.end) return true;
             }
+            for (const auto& piece : entry.possiblePieces) {
+                if (piece.generation > generation && piece.begin < to && from < piece.end) return true;
+            }
         }
         return false;
     }
@@ -1199,16 +1269,51 @@ WriteTracker& Tracker() {
 
 // Takes the tracker mutex; under APS5_PROFILE_DRAW an acquisition that found it held is counted
 // ('tracker waits N / M' in [guestmem]), which says whether walking outside the lock would pay.
-std::unique_lock<std::mutex> lockTracker(WriteTracker& tracker) {
-    static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
-    if (!profile) return std::unique_lock(tracker.mutex);
-    std::unique_lock lock(tracker.mutex, std::try_to_lock);
-    if (!lock.owns_lock()) {
-        trackerWaits.fetch_add(1, std::memory_order_relaxed);
+class TrackerLock {
+public:
+    explicit TrackerLock(WriteTracker& tracker) : tracker(tracker), lock(tracker.mutex, std::defer_lock) {
+        static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
+        if (!profile) {
+            lock.lock();
+            return;
+        }
+        if (!lock.try_lock()) {
+            trackerWaits.fetch_add(1, std::memory_order_relaxed);
+            lock.lock();
+        }
+        trackerAcquisitions.fetch_add(1, std::memory_order_relaxed);
+    }
+    ~TrackerLock() {
+        if (lock.owns_lock()) publish();
+    }
+    TrackerLock(const TrackerLock&) = delete;
+    TrackerLock& operator=(const TrackerLock&) = delete;
+
+    void Unlock() {
+        publish();
+        lock.unlock();
+    }
+
+    void Suspend() {
+        lock.unlock();
+    }
+
+    void Resume() {
         lock.lock();
     }
-    trackerAcquisitions.fetch_add(1, std::memory_order_relaxed);
-    return lock;
+
+private:
+    void publish() {
+        const auto current = tracker.generation.load(std::memory_order_relaxed);
+        if (tracker.stable.load(std::memory_order_relaxed) != current) tracker.stable.store(current, std::memory_order_release);
+    }
+
+    WriteTracker& tracker;
+    std::unique_lock<AgcDriver::Mutex> lock;
+};
+
+TrackerLock lockTracker(WriteTracker& tracker) {
+    return TrackerLock(tracker);
 }
 
 #ifdef _WIN32
@@ -1234,11 +1339,11 @@ void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_
 // thread that never bumped is given a new epoch for every collect: its walks are never reused (the
 // presenter, game threads inside the flush hook), as nothing orders the CPU's writes for it.
 std::atomic<std::uint64_t> nextCollectEpoch{1};
-thread_local std::uint64_t threadCollectEpoch = 0;
+struct CollectEpochSlot {};
 std::atomic<std::uint64_t> lastBumpedEpoch{0};
 
 std::uint64_t currentCollectEpoch() {
-    if (threadCollectEpoch != 0) return threadCollectEpoch;
+    if (const auto epoch = HostThreadSlot<std::uint64_t, CollectEpochSlot>::Get(); epoch != 0) return epoch;
     return nextCollectEpoch.fetch_add(1, std::memory_order_relaxed) + 1;
 }
 
@@ -1255,7 +1360,15 @@ struct ThreadCollectMemo {
     std::array<WriteTracker::Memo, 64> entries{};
     std::size_t next = 0;
 };
-thread_local ThreadCollectMemo threadCollectMemo;
+ThreadCollectMemo& threadCollectMemo() {
+    struct Slot {};
+    auto* memo = HostThreadSlot<ThreadCollectMemo*, Slot>::Get();
+    if (memo == nullptr) {
+        ShaderRecompiler::ThreadOwned(memo);
+        HostThreadSlot<ThreadCollectMemo*, Slot>::Set(memo);
+    }
+    return *memo;
+}
 
 struct EpochCollectMap {
     std::uint64_t epoch = 0;
@@ -1373,16 +1486,16 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi
             const auto& map = epochCollectMap(epoch);
             if (const auto found = map.ranges.find(first); found != map.ranges.end() && found->second.unwatched == unwatched && found->second.mapping == mapping && stop <= found->second.end) {
                 collectMemoHits.fetch_add(1, std::memory_order_relaxed);
-                return tracker.generation.load(std::memory_order_relaxed);
+                return tracker.stable.load(std::memory_order_acquire);
             }
         }
-        for (const auto& entry : threadCollectMemo.entries) {
+        for (const auto& entry : threadCollectMemo().entries) {
             if (entry.epoch == epoch && entry.unwatched == unwatched && entry.mapping == mapping && entry.begin <= first && stop <= entry.end) {
                 // The current generation, not the memoized one: blocks stamped since (MarkWritten, an
                 // overlapping collect) were written before this caller reads, and an older value would
                 // make every later UnchangedSince fail until the epoch ends.
                 collectMemoHits.fetch_add(1, std::memory_order_relaxed);
-                return tracker.generation.load(std::memory_order_relaxed);
+                return tracker.stable.load(std::memory_order_acquire);
             }
         }
     }
@@ -1400,7 +1513,7 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi
     }
     const TimedAccess timed(CounterCollect, bytes);
 #ifndef _WIN32
-    if (useMemo && threadCollectEpoch != 0 && bytes <= WriteBlockBytes) {
+    if (useMemo && HostThreadSlot<std::uint64_t, CollectEpochSlot>::Get() != 0 && bytes <= WriteBlockBytes) {
         const auto begin = first & ~(WriteBlockBytes - 1);
         const auto end = (stop + WriteBlockBytes - 1) & ~(WriteBlockBytes - 1);
         if (end > begin && tracker.covers(begin, end - begin)) {
@@ -1415,7 +1528,8 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi
         const auto serial = unwatchSerial.load(std::memory_order_relaxed);
         if (sharedCollectMemo()) tracker.memo[tracker.nextMemo++ % tracker.memo.size()] = {first, stop, epoch, serial, mapping};
         else {
-            threadCollectMemo.entries[threadCollectMemo.next++ % threadCollectMemo.entries.size()] = {first, stop, epoch, serial, mapping};
+            auto& memo = threadCollectMemo();
+            memo.entries[memo.next++ % memo.entries.size()] = {first, stop, epoch, serial, mapping};
             if (epochCollectMapEnabled()) {
                 auto& entry = epochCollectMap(epoch).ranges[first];
                 if (entry.unwatched != serial || entry.mapping != mapping || entry.end < stop) entry = {first, stop, epoch, serial, mapping};
@@ -1436,14 +1550,15 @@ std::uint64_t CollectWritesUncached(std::uint64_t address, std::size_t bytes) {
 }
 
 void BumpCollectEpoch() {
-    threadCollectEpoch = nextCollectEpoch.fetch_add(1, std::memory_order_relaxed) + 1;
+    const auto epoch = nextCollectEpoch.fetch_add(1, std::memory_order_relaxed) + 1;
+    HostThreadSlot<std::uint64_t, CollectEpochSlot>::Set(epoch);
     auto last = lastBumpedEpoch.load(std::memory_order_relaxed);
-    while (last < threadCollectEpoch && !lastBumpedEpoch.compare_exchange_weak(last, threadCollectEpoch, std::memory_order_release, std::memory_order_relaxed)) {}
+    while (last < epoch && !lastBumpedEpoch.compare_exchange_weak(last, epoch, std::memory_order_release, std::memory_order_relaxed)) {}
     collectEpochBumps.fetch_add(1, std::memory_order_relaxed);
 }
 
 std::uint64_t ThreadCollectEpoch() {
-    return collectMemoEnabled() ? threadCollectEpoch : 0;
+    return collectMemoEnabled() ? HostThreadSlot<std::uint64_t, CollectEpochSlot>::Get() : 0;
 }
 
 std::uint64_t UnwatchSerial() {
@@ -1452,15 +1567,32 @@ std::uint64_t UnwatchSerial() {
 
 void AdoptLatestCollectEpoch() {
     const auto latest = lastBumpedEpoch.load(std::memory_order_acquire);
-    if (latest > threadCollectEpoch) threadCollectEpoch = latest;
+    if (latest > HostThreadSlot<std::uint64_t, CollectEpochSlot>::Get()) HostThreadSlot<std::uint64_t, CollectEpochSlot>::Set(latest);
 }
 
 std::uint64_t CollectEpochBumps() {
     return collectEpochBumps.load(std::memory_order_relaxed);
 }
 
+namespace {
+std::atomic<std::uint64_t> proofSerial{1};
+thread_local bool proofThread = false;
+}
+
+std::uint64_t ProofSerial() {
+    return proofSerial.load(std::memory_order_acquire);
+}
+
+void BreakProofs() {
+    proofSerial.fetch_add(1, std::memory_order_acq_rel);
+}
+
+void MarkProofThread() {
+    proofThread = true;
+}
+
 std::uint64_t CollectEpoch() {
-    return collectMemoEnabled() ? threadCollectEpoch : 0;
+    return collectMemoEnabled() ? HostThreadSlot<std::uint64_t, CollectEpochSlot>::Get() : 0;
 }
 
 namespace {
@@ -1552,7 +1684,7 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function
     auto lock = lockTracker(tracker);
     tracker.initialize();
     if (!tracker.watched) {
-        lock.unlock();
+        lock.Unlock();
         return import();
     }
     constexpr std::uint64_t page = 4096;
@@ -1666,17 +1798,18 @@ bool UnchangedCommittedSince(std::span<const std::pair<std::uint64_t, std::uint6
 
 std::uint64_t storeOwn(std::uint64_t address, std::size_t bytes, const std::function<std::pair<std::uint64_t, std::uint64_t>()>& store) {
     if (bytes == 0) return 0;
+    BreakProofs();
     const auto stampStored = [](WriteTracker& tracker, std::pair<std::uint64_t, std::uint64_t> stored) -> std::uint64_t {
         if (stored.second <= stored.first) return 0;
         ++tracker.generation;
         if (!tracker.watched || !tracker.covers(stored.first, static_cast<std::size_t>(stored.second - stored.first))) {
             tracker.noteUnstampedStore(stored.first, stored.second);
-            if (tracker.watched) tracker.stampWatchedPart(stored.first, stored.second, tracker.generation);
+            if (tracker.watched) tracker.stampWatchedPart(stored.first, stored.second, tracker.generation, true);
             return 0;
         }
         for (auto block = tracker.blockOf(stored.first); block <= tracker.blockOf(stored.second - 1); ++block) {
             tracker.stamp(block, tracker.generation, StampKind::Driver);
-            tracker.noteDriverStore(block, stored.first, stored.second, tracker.generation);
+            tracker.noteDriverStore(block, stored.first, stored.second, tracker.generation, true);
         }
         return tracker.generation;
     };
@@ -1709,7 +1842,8 @@ std::uint64_t StoreOwnBytes(std::uint64_t address, std::span<const std::byte> so
     });
 }
 
-std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) {
+static std::uint64_t markWritten(std::uint64_t address, std::size_t bytes, bool complete) {
+    BreakProofs();
     auto& tracker = Tracker();
     auto lock = lockTracker(tracker);
     tracker.initialize();
@@ -1717,7 +1851,7 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) {
     ++tracker.generation;
     if (!tracker.watched || !tracker.covers(address, bytes)) {
         tracker.noteUnstampedStore(address, address + bytes);
-        if (tracker.watched) tracker.stampWatchedPart(address, address + bytes, tracker.generation);
+        if (tracker.watched) tracker.stampWatchedPart(address, address + bytes, tracker.generation, complete);
         return 0;
     }
     {
@@ -1725,14 +1859,14 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) {
         const auto blockStop = (address + bytes + WriteBlockBytes - 1) & ~(std::uint64_t{WriteBlockBytes} - 1);
         StampRuns runs{tracker, StampKind::Fresh};
 #ifdef _WIN32
-        const bool complete = collectWindowsWrites(runs, blockFirst, blockStop, false);
+        const bool collected = collectWindowsWrites(runs, blockFirst, blockStop, false);
 #else
-        const bool complete = GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(static_cast<std::uintptr_t>(blockFirst), static_cast<std::size_t>(blockStop - blockFirst), &stampWrittenRun, &runs);
+        const bool collected = GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(static_cast<std::uintptr_t>(blockFirst), static_cast<std::size_t>(blockStop - blockFirst), &stampWrittenRun, &runs);
 #endif
-        if (!complete) {
-            lock.unlock();
+        if (!collected) {
+            lock.Suspend();
             beforeUnwatch(blockFirst, static_cast<std::size_t>(blockStop - blockFirst));
-            lock.lock();
+            lock.Resume();
             unwatchLocked(tracker, blockFirst, static_cast<std::size_t>(blockStop - blockFirst));
             tracker.noteUnstampedStore(address, address + bytes);
             return 0;
@@ -1743,11 +1877,19 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) {
     const auto last = tracker.blockOf(address + bytes - 1);
     for (auto block = first; block <= last; ++block) {
         tracker.stamp(block, tracker.generation, StampKind::Driver);
-        tracker.noteDriverStore(block, address, address + bytes, tracker.generation);
+        tracker.noteDriverStore(block, address, address + bytes, tracker.generation, complete);
     }
     return tracker.generation;
 }
 
+std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) {
+    return markWritten(address, bytes, true);
+}
+
+std::uint64_t MarkPossiblyWritten(std::uint64_t address, std::size_t bytes) {
+    return markWritten(address, bytes, false);
+}
+
 bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generation) {
     auto& tracker = Tracker();
     const auto lock = lockTracker(tracker);
@@ -2053,24 +2195,29 @@ std::string WaitedBehindReport(const HolderMatrixValues& waited) {
 
 }
 
-// The owner token is a thread-local's address: unique per live thread and free to read.
 namespace {
-const void* ThreadToken() {
-    thread_local char token;
-    return &token;
-}
-
 std::atomic<std::uint64_t> gpuHoldSerial{0};
 }
 
+void GpuMutexType::enter() {
+    if (!HeldByThisThread()) LockSpinning(mutex, HandoffSpinWindow());
+}
+
+bool GpuMutexType::tryEnter() {
+    return HeldByThisThread() || mutex.try_lock();
+}
+
 void GpuMutexType::acquired() {
-    owner.store(ThreadToken(), std::memory_order_relaxed);
-    if (++depth == 1) gpuHoldSerial.fetch_add(1, std::memory_order_relaxed);
+    owner.store(CurrentThreadToken(), std::memory_order_relaxed);
+    if (++depth == 1) {
+        gpuHoldSerial.fetch_add(1, std::memory_order_relaxed);
+        if (!proofThread) BreakProofs();
+    }
 }
 
 bool GpuMutexType::try_lock() {
     if (!GpuLockProfiled()) {
-        if (!mutex.try_lock()) return false;
+        if (!tryEnter()) return false;
         acquired();
         return true;
     }
@@ -2079,7 +2226,7 @@ bool GpuMutexType::try_lock() {
     auto& stats = LockStats();
     const auto tagged = std::exchange(stats.nextSite, GpuLockSite::Other);
     ++stats.tries;
-    if (!mutex.try_lock()) {
+    if (!tryEnter()) {
         ++stats.triesFailed;
         return false;
     }
@@ -2106,8 +2253,8 @@ void GpuMutexType::unlock() {
         owner.store(nullptr, std::memory_order_relaxed);
         // Still the holder here: the hold ends when the mutex is given up, not before.
         if (GpuHoldProfiled()) EndHold();
+        mutex.unlock();
     }
-    mutex.unlock();
     // After the release, so what the hook does (destroying a batch's kept objects) is neither part
     // of the hold nor waited for by the next holder.
     if (outermost) {
@@ -2116,7 +2263,7 @@ void GpuMutexType::unlock() {
 }
 
 bool GpuMutexType::HeldByThisThread() const {
-    return owner.load(std::memory_order_relaxed) == ThreadToken();
+    return owner.load(std::memory_order_relaxed) == CurrentThreadToken();
 }
 
 std::uint32_t GpuMutexType::DepthOnThisThread() const {
@@ -2135,10 +2282,10 @@ void GpuMutexType::lock() {
     if (!GpuLockProfiled()) {
         auto* trace = PerformanceTrace::Recording() ? PerformanceTrace::Thread() : nullptr;
         if (trace == nullptr) {
-            LockSpinning(mutex, HandoffSpinWindow());
+            enter();
         } else {
             const auto site = static_cast<std::size_t>(std::exchange(LockStats().nextSite, GpuLockSite::Other));
-            if (!mutex.try_lock()) {
+            if (!tryEnter()) {
                 const auto start = std::chrono::steady_clock::now();
                 LockSpinning(mutex, HandoffSpinWindow());
                 trace->Add("GpuMutex.wait", GpuLockSiteNames[site], start, std::chrono::steady_clock::now());
@@ -2153,7 +2300,7 @@ void GpuMutexType::lock() {
     ++stats.siteAcquisitions[site];
     // An uncontended (or recursive) acquisition costs no clock reads for the wait; only a wait is
     // timed. The hold timer (outermost acquisitions only) starts once the mutex is held.
-    if (!mutex.try_lock()) {
+    if (!tryEnter()) {
         const bool holders = GpuHoldProfiled();
         HolderMatrixValues before{};
         if (holders) Holders().Snapshot(before);
@@ -2420,6 +2567,7 @@ void WriteChangedCommitted(std::uint64_t address, std::span<const std::byte> cur
 
 void WriteChanged(std::uint64_t address, std::span<const std::byte> current, std::span<const std::byte> original) {
     require(current.size() == original.size(), "write-back snapshot sizes differ");
+    BreakProofs();
     if (current.empty()) return;
     // A store: the hook sync it makes orders this write after a pending GPU write, whatever a read
     // site above it was doing (the [hooksync] line counts stores apart).

core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp

@@ -450,7 +450,7 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
         }
         case 0x59:
             size(2);
-            require((packet[1] & 0x7fffffffu) == 0, "unsupported REWIND payload bits");
+            require((packet[1] & ~(RewindValid | RewindOffloadEnable)) == 0, "unsupported REWIND payload bits");
             break;
         case 0x3c: case 0x93: {
             size(opcode == 0x3c ? 7 : 9);
@@ -846,7 +846,7 @@ DrawParameters resolveIndirectDraw(std::span<const std::uint32_t> packet, const
     indirect.drawIndexLocation = multi ? location(packet[4] & 0xffffu) : 0x280u;
     indirect.drawIndexEnabled = multi && (packet[4] >> 31u) != 0;
     indirect.indxOffset = 0;
-    require(indirect.count <= (std::numeric_limits<std::uint64_t>::max() - indirect.arguments - indirect.recordBytes) / indirect.stride, "indirect draw record range overflow");
+    require(indirect.RangeBytes() <= std::numeric_limits<std::uint64_t>::max() - indirect.arguments, "indirect draw record range overflow");
     // The counts are unknown here (the CP takes them from the record; NUM_INSTANCES is ignored).
     DrawParameters draw{0, 0, 0, 0, packet.back() & 0x20u, indexed, 0, 0};
     if (!indexed) {

core/libs/prx/libSceAgcDriver/Execution/src/Screenshot.cpp

@@ -16,8 +16,12 @@ namespace {
 std::atomic<bool> requested{false};
 
 std::filesystem::path directory() {
+#ifdef _WIN32
+    const wchar_t* configured = _wgetenv(L"ANYPS5_SCREENSHOTS");
+#else
     const char* configured = std::getenv("ANYPS5_SCREENSHOTS");
-    if (configured != nullptr && configured[0] != '\0') return configured;
+#endif
+    if (configured != nullptr && configured[0] != 0) return configured;
     const std::unique_ptr<char, decltype(&SDL_free)> basePath(SDL_GetBasePath(), SDL_free);
     if (!basePath) throw std::runtime_error(std::string("Screenshot: cannot locate the executable: ") + SDL_GetError());
     return std::filesystem::path(reinterpret_cast<const char8_t*>(basePath.get())) / "anyps5-screenshots";

core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp

@@ -18,6 +18,8 @@
 #include <memory>
 #include <stdexcept>
 #include <vector>
+#include <mutex>
+#include <unordered_map>
 
 namespace AgcDriver {
 namespace {
@@ -130,6 +132,23 @@ std::vector<std::unique_ptr<TPage>>& pagePool() {
 
 constexpr std::size_t PagePoolLimit = 64;
 
+bool sharedPagePools() {
+    static const bool enabled = std::getenv("APS5_DRAW_PIPELINE") != nullptr;
+    return enabled;
+}
+
+template<typename TPage>
+struct SharedPagePool {
+    std::mutex mutex;
+    std::vector<std::unique_ptr<TPage>> pages;
+};
+
+template<typename TPage>
+SharedPagePool<TPage>& sharedPagePool() {
+    static auto* pool = new SharedPagePool<TPage>();
+    return *pool;
+}
+
 }
 
 void ShaderMemory::SetWaitedMsProvider(WaitedMsProvider provider) {
@@ -144,6 +163,15 @@ ShaderMemory::ShaderMemory(std::span<const ShaderRecompiler::MemoryRegion> regio
 }
 
 ShaderMemory::~ShaderMemory() {
+    if (sharedPagePools() && !pages.empty()) {
+        auto& shared = sharedPagePool<Page>();
+        std::lock_guard lock(shared.mutex);
+        for (const auto& [base, page] : pages) {
+            if (shared.pages.size() < 1024) shared.pages.emplace_back(page);
+            else delete page;
+        }
+        return;
+    }
     auto& pool = pagePool<Page>();
     for (const auto& [base, page] : pages) {
         if (pool.size() < PagePoolLimit) pool.emplace_back(page);
@@ -191,6 +219,14 @@ std::size_t ShaderMemory::pageAtOrAfter(std::uint64_t base) const {
 
 ShaderMemory::Page& ShaderMemory::addPage(std::size_t position, std::uint64_t base) {
     auto& pool = pagePool<Page>();
+    if (pool.empty() && sharedPagePools()) {
+        auto& shared = sharedPagePool<Page>();
+        std::lock_guard lock(shared.mutex);
+        for (std::size_t i = 0; i < 32 && !shared.pages.empty(); ++i) {
+            pool.push_back(std::move(shared.pages.back()));
+            shared.pages.pop_back();
+        }
+    }
     std::unique_ptr<Page> owned;
     if (pool.empty()) {
         owned = std::make_unique<Page>();

core/libs/prx/libSceAgcDriver/Execution/src/SubmissionCapture.cpp

@@ -101,8 +101,12 @@ SubmissionCapture::SubmissionCapture(const std::filesystem::path& directory) : d
 
 SubmissionCapture* SubmissionCapture::Active() {
     static SubmissionCapture* const capture = [] {
+#ifdef _WIN32
+        const wchar_t* directory = _wgetenv(L"APS5_DUMP_SUBMITS");
+#else
         const char* directory = std::getenv("APS5_DUMP_SUBMITS");
-        return directory != nullptr && *directory != '\0' ? new SubmissionCapture(directory) : nullptr;
+#endif
+        return directory != nullptr && *directory != 0 ? new SubmissionCapture(directory) : nullptr;
     }();
     return capture;
 }

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -2,6 +2,7 @@
 #include "prx/libSceAgcDriver/Execution/include/FrameDumpIo.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp"
 #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 #include "BdaAbi.hpp"
 #if ANYPS5_ENABLE_SPIRV_TOOLS
 #include "SpirvBackend/SpirvOptimizer.hpp"
@@ -28,6 +29,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Screenshot.hpp"
 #include "prx/libkernel/AppMetadata/include/AppMetadata.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/PlaceholderImages.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
 #include "prx/libSceAgcDriver/Execution/include/DisplayFormat.hpp"
@@ -80,6 +82,44 @@
 #include <utility>
 
 namespace AgcDriver {
+
+DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> families, std::uint32_t family, bool settles, bool sparseBinding) {
+    DeviceQueuePlan plan;
+    const auto count = static_cast<std::uint32_t>(families.size());
+    if (settles) {
+        if (family < count && families[family].queueCount >= 2) {
+            plan.mainCount = 2;
+            plan.settle = true;
+            plan.settleFamily = family;
+            plan.settleIndex = 1;
+        } else {
+            const auto copies = [&](std::uint32_t index) { return index != family && families[index].queueCount != 0 && (families[index].queueFlags & (VK_QUEUE_TRANSFER_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) != 0; };
+            for (std::uint32_t i = 0; i < count && !plan.settle; ++i) {
+                if (copies(i) && (families[i].queueFlags & (VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) == 0) plan.settle = true, plan.settleFamily = i;
+            }
+            for (std::uint32_t i = 0; i < count && !plan.settle; ++i) {
+                if (copies(i)) plan.settle = true, plan.settleFamily = i;
+            }
+        }
+    }
+    if (!sparseBinding) return plan;
+    const auto taken = [&](std::uint32_t index) { return plan.settle && index == plan.settleFamily ? plan.settleIndex + 1 : 0u; };
+    const auto binds = [&](std::uint32_t index) { return index != family && families[index].queueCount > taken(index) && (families[index].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) != 0; };
+    for (std::uint32_t i = 0; i < count && plan.sparseFamily == VK_QUEUE_FAMILY_IGNORED; ++i) {
+        if (binds(i) && families[i].queueFlags == VK_QUEUE_SPARSE_BINDING_BIT) plan.sparseFamily = i;
+    }
+    for (std::uint32_t i = 0; i < count && plan.sparseFamily == VK_QUEUE_FAMILY_IGNORED; ++i) {
+        if (binds(i)) plan.sparseFamily = i;
+    }
+    if (plan.sparseFamily != VK_QUEUE_FAMILY_IGNORED) {
+        plan.sparseIndex = taken(plan.sparseFamily);
+        plan.sparseBinding = true;
+    } else {
+        plan.sparseBinding = family < count && (families[family].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) != 0;
+    }
+    return plan;
+}
+
 namespace {
 
 void check(VkResult result, const char* operation) {
@@ -129,6 +169,12 @@ void CreateComputePipeline(VkDevice device, PFN_vkCreateComputePipelines create,
     PerformanceCounters::CountPipeline();
 }
 
+#ifdef _WIN32
+const wchar_t* ComputePipelineDumpDirectory() { return _wgetenv(L"APS5_DUMP_COMPUTE_PIPELINES"); }
+#else
+const char* ComputePipelineDumpDirectory() { return std::getenv("APS5_DUMP_COMPUTE_PIPELINES"); }
+#endif
+
 bool CaptureComputePipeline(std::size_t words) {
     const char* minimum = std::getenv("APS5_DUMP_COMPUTE_MIN_WORDS");
     if (minimum == nullptr) return true;
@@ -156,20 +202,22 @@ public:
             std::lock_guard lock(mutex);
             stopping = true;
         }
-        changed.notify_one();
+        changed.notify_all();
         if (worker.joinable()) worker.join();
     }
 
     void Enqueue(int index, std::uint32_t width, std::uint32_t height, std::uint32_t scale, bool screenshot, std::vector<std::byte> pixels) {
-        std::lock_guard lock(mutex);
+        std::unique_lock lock(mutex);
         require(!stopping, "frame dump writer has stopped");
-        require(pixels.size() <= maxBytes - queuedBytes, "frame dump queue exceeded 64 MiB");
         if (!screenshot && !frameDumpOutput.Enabled_nid_no_patch()) return;
+        require(pixels.size() <= maxBytes, "a frame dump exceeds the 64 MiB queue");
+        changed.wait(lock, [&] { return stopping || pixels.size() <= maxBytes - queuedBytes; });
+        require(!stopping, "frame dump writer has stopped");
         if (!worker.joinable()) worker = std::thread([this] { run(); });
         const auto bytes = pixels.size();
         pending.push_back({index, width, height, scale, screenshot, std::move(pixels)});
         queuedBytes += bytes;
-        changed.notify_one();
+        changed.notify_all();
     }
 
     bool Idle() {
@@ -202,6 +250,7 @@ private:
                     else WriteFrameBmp(frame.index, frame.width, frame.height, frame.pixels, frame.scale);
                     std::lock_guard lock(mutex);
                     queuedBytes -= frame.pixels.size();
+                    changed.notify_all();
                 }
             }
         } catch (const std::exception& error) {
@@ -299,9 +348,16 @@ struct VulkanDevice::State {
     VkDeviceSize uploadSize = 0;
     VkPhysicalDeviceProperties properties{};
     std::string driverInfo;
+    VkDriverId driverId = VK_DRIVER_ID_MAX_ENUM;
+    std::uint32_t pixelFunctionStorageDwords = 4096u;
     bool individuallyFreeDescriptorSets = false;
     PFN_vkGetCalibratedTimestampsKHR calibratedTimestamps = nullptr;
     PFN_vkCmdSetCheckpointNV checkpointMarker = nullptr;
+    PFN_vkCmdWriteBufferMarkerAMD bufferMarker = nullptr;
+    VkBuffer markerBuffer = VK_NULL_HANDLE;
+    VkDeviceMemory markerMemory = VK_NULL_HANDLE;
+    void* markerMapping = nullptr;
+    BufferMarkerLog markerLog;
     bool deviceLostHook = false;
     VkTimeDomainKHR hostTimeDomain = VK_TIME_DOMAIN_DEVICE_KHR;
     VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES};
@@ -312,7 +368,9 @@ struct VulkanDevice::State {
     bool meshShader = false;
     bool fragmentShaderBarycentric = false;
     bool shaderTerminateInvocation = false;
+    bool fusedExactArithmetic = false;
     bool geometryShader = false;
+    bool largePoints = false;
     bool sampleRateShading = false;
     bool storageImageMultisample = false;
     bool storageImageWriteWithoutFormat = false;
@@ -330,6 +388,7 @@ struct VulkanDevice::State {
     bool earlyFragmentSampleMaskTestBeforeSampleCounting = false;
     bool robustBufferAccess2 = false;
     bool subgroupControlBarriers = true;
+    bool workgroupScopeFences = true;
     // VK_EXT_descriptor_indexing with non-uniform image array indexing (bindless image tables in
     // graphics stages, and compute workgroups wider than a wave).
     bool descriptorIndexing = false;
@@ -342,6 +401,7 @@ struct VulkanDevice::State {
     bool imageInt64Atomics = false;
     bool bufferInt64Atomics = false;
     bool viewportIndexLayer = false;
+    bool imageFormatList = false;
     bool primitiveListRestart = false;
     bool depthClipControl = false;
     bool graphicsPipelineLibrary = false;
@@ -367,8 +427,11 @@ struct VulkanDevice::State {
     bool drawInputsInPlace = true;
     bool dmaBufImport = false;
     bool sparseBinding = false;
+    bool sparseResidencyBuffer = false;
+    bool sparseResidencyAliased = false;
     std::uint32_t queueFamily = 0;
     std::uint32_t sparseFamily = VK_QUEUE_FAMILY_IGNORED;
+    std::uint32_t sparseIndex = 0;
     VkQueue sparseQueue = VK_NULL_HANDLE;
     std::mutex sparseQueueMutex;
     VkQueue settleQueue = VK_NULL_HANDLE;
@@ -382,6 +445,8 @@ struct VulkanDevice::State {
     bool samplerFilterMinmax = false;
     bool nonSeamlessCubeMap = false;
     VkSampleCountFlags mergedCoverageSampleCounts = 0;
+    VkSampleCountFlags truncatedCoverageSampleCounts = 0;
+    bool mixedAttachmentSamples = false;
     bool fragmentShaderPixelInterlock = false;
     bool conservativeRasterization = false;
     bool provokingVertexLast = false;
@@ -414,7 +479,7 @@ struct VulkanDevice::State {
     // Compute pipeline objects by variant (and push-constant use), under their own mutex: the
     // dispatch's find-or-insert and the verify switch's lookups touch the map, recipes hold weak
     // references to its objects.
-    std::mutex computePipelinesMutex;
+    AgcDriver::Mutex computePipelinesMutex;
     std::map<ComputePipelineKey, std::shared_ptr<ComputePipelineObjects>> computePipelines;
     std::map<ComputePipelineKey, std::shared_future<std::shared_ptr<ComputePipelineObjects>>> computePipelinesBuilding;
     // The journal replay building this device's pipelines on its own threads; drained first at teardown.
@@ -495,7 +560,7 @@ struct VulkanDevice::State {
         std::uint64_t throughSerial;
     };
     std::deque<RetiredRange> retiredRanges;
-    std::mutex presentMutex;
+    AgcDriver::Mutex presentMutex;
     // The presentation present() submitted and QueuePresent has not handed to the swapchain.
     bool queuePending = false;
     std::uint32_t queueIndex = 0;
@@ -570,6 +635,33 @@ struct VulkanDevice::State {
         return workgroupMemory;
     }
 
+    void CreateMarkerBuffer() {
+        bufferMarker = DeviceFunction<PFN_vkCmdWriteBufferMarkerAMD>("vkCmdWriteBufferMarkerAMD");
+        VkBufferCreateInfo buffer{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
+        buffer.size = 2 * sizeof(std::uint32_t);
+        buffer.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
+        buffer.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
+        check(DeviceFunction<PFN_vkCreateBuffer>("vkCreateBuffer")(device, &buffer, nullptr, &markerBuffer), "vkCreateBuffer buffer markers");
+        VkMemoryRequirements requirements{};
+        DeviceFunction<PFN_vkGetBufferMemoryRequirements>("vkGetBufferMemoryRequirements")(device, markerBuffer, &requirements);
+        std::uint32_t memoryType = memoryProperties.memoryTypeCount;
+        const auto flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
+        for (std::uint32_t i = 0; i < memoryProperties.memoryTypeCount; ++i) {
+            if ((requirements.memoryTypeBits & (1u << i)) != 0 && (memoryProperties.memoryTypes[i].propertyFlags & flags) == flags) {
+                memoryType = i;
+                break;
+            }
+        }
+        require(memoryType < memoryProperties.memoryTypeCount, "coherent host memory for buffer markers is unavailable");
+        VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
+        allocation.allocationSize = requirements.size;
+        allocation.memoryTypeIndex = memoryType;
+        check(DeviceFunction<PFN_vkAllocateMemory>("vkAllocateMemory")(device, &allocation, nullptr, &markerMemory), "vkAllocateMemory buffer markers");
+        check(DeviceFunction<PFN_vkBindBufferMemory>("vkBindBufferMemory")(device, markerBuffer, markerMemory, 0), "vkBindBufferMemory buffer markers");
+        check(DeviceFunction<PFN_vkMapMemory>("vkMapMemory")(device, markerMemory, 0, buffer.size, 0, &markerMapping), "vkMapMemory buffer markers");
+        std::memset(markerMapping, 0, buffer.size);
+    }
+
     void Upload(std::span<const std::byte> pixels) {
         if (uploadSize < pixels.size()) {
             if (uploadMapping) DeviceFunction<PFN_vkUnmapMemory>("vkUnmapMemory")(device, uploadMemory);
@@ -830,6 +922,9 @@ struct VulkanDevice::State {
             if (uploadMapping) reinterpret_cast<PFN_vkUnmapMemory>(deviceProc(device, "vkUnmapMemory"))(device, uploadMemory);
             if (uploadBuffer) reinterpret_cast<PFN_vkDestroyBuffer>(deviceProc(device, "vkDestroyBuffer"))(device, uploadBuffer, nullptr);
             if (uploadMemory) reinterpret_cast<PFN_vkFreeMemory>(deviceProc(device, "vkFreeMemory"))(device, uploadMemory, nullptr);
+            if (markerMapping) reinterpret_cast<PFN_vkUnmapMemory>(deviceProc(device, "vkUnmapMemory"))(device, markerMemory);
+            if (markerBuffer) reinterpret_cast<PFN_vkDestroyBuffer>(deviceProc(device, "vkDestroyBuffer"))(device, markerBuffer, nullptr);
+            if (markerMemory) reinterpret_cast<PFN_vkFreeMemory>(deviceProc(device, "vkFreeMemory"))(device, markerMemory, nullptr);
             if (swapchain) reinterpret_cast<PFN_vkDestroySwapchainKHR>(deviceProc(device, "vkDestroySwapchainKHR"))(device, swapchain, nullptr);
             const auto destroyPool = reinterpret_cast<PFN_vkDestroyCommandPool>(deviceProc(device, "vkDestroyCommandPool"));
             const auto destroyDevice = reinterpret_cast<PFN_vkDestroyDevice>(deviceProc(device, "vkDestroyDevice"));
@@ -1176,8 +1271,22 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &driverProperties);
         driverId = driver.driverID;
         state->driverInfo = std::string(driver.driverName) + " " + driver.driverInfo;
+        state->driverId = driver.driverID;
         state->individuallyFreeDescriptorSets = driver.driverID == VK_DRIVER_ID_MESA_LLVMPIPE;
+        state->fusedExactArithmetic = driver.driverID == VK_DRIVER_ID_MOLTENVK;
         state->subgroupControlBarriers = SubgroupControlBarrierUsable(driver.driverID);
+        state->workgroupScopeFences = WorkgroupScopeFenceUsable(driver.driverID);
+    }
+    if (driverId == VK_DRIVER_ID_NVIDIA_PROPRIETARY && hasExtension(VK_NV_SHADER_SM_BUILTINS_EXTENSION_NAME)) {
+        VkPhysicalDeviceShaderSMBuiltinsPropertiesNV smBuiltins{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_SM_BUILTINS_PROPERTIES_NV};
+        VkPhysicalDeviceProperties2 smProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &smBuiltins};
+        state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &smProperties);
+        const auto subgroup = state->subgroup.subgroupSize;
+        if (smBuiltins.shaderSMCount != 0u && smBuiltins.shaderWarpsPerSM != 0u && subgroup != 0u) {
+            const auto invocations = static_cast<std::uint64_t>(smBuiltins.shaderSMCount) * smBuiltins.shaderWarpsPerSM * subgroup;
+            const auto budgetDwords = (4ull << 30) / (invocations * 4ull);
+            if (budgetDwords < state->pixelFunctionStorageDwords) state->pixelFunctionStorageDwords = static_cast<std::uint32_t>(budgetDwords);
+        }
     }
     auto byteFeatures = QueryBdaByteFeatures(selected, state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2"), availableExtensions);
     auto bdaFeatures = QueryBdaFeatures(selected, state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2"), availableExtensions);
@@ -1313,6 +1422,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         state->spirvExtensions.push_back("SPV_EXT_shader_viewport_index_layer");
         state->viewportIndexLayer = true;
     }
+    if (hasExtension(VK_KHR_IMAGE_FORMAT_LIST_EXTENSION_NAME)) {
+        deviceExtensions.push_back(VK_KHR_IMAGE_FORMAT_LIST_EXTENSION_NAME);
+        state->imageFormatList = true;
+    }
     if (state->demoteToHelperInvocation) {
         deviceExtensions.push_back(VK_EXT_SHADER_DEMOTE_TO_HELPER_INVOCATION_EXTENSION_NAME);
         state->capabilities.push_back(spv::CapabilityDemoteToHelperInvocation);
@@ -1409,7 +1522,8 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     // Guest memory is host memory: importing it lets address-based shaders use it in place instead of
     // copying every registered allocation per draw.
     if (hasExtension(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME) && std::getenv("APS5_NO_HOST_IMPORT") == nullptr) {
-        VkPhysicalDeviceDriverProperties driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES};
+        VkPhysicalDeviceMaintenance3Properties maintenance3{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_3_PROPERTIES};
+        VkPhysicalDeviceDriverProperties driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES, &maintenance3};
         VkPhysicalDeviceExternalMemoryHostPropertiesEXT hostProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT, &driverProperties};
         VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &hostProperties};
         state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &properties);
@@ -1421,6 +1535,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
 #endif
         state->drawInputsInPlace = std::getenv("APS5_NO_DRAW_INPUTS_IN_PLACE") == nullptr && (std::getenv("APS5_DRAW_INPUTS_IN_PLACE") != nullptr || DrawInputsInPlaceUsable(driverProperties.driverID, windows));
         if (!compileOnly) Graphics::SetHostImportWindow(8ull << 20u);
+        if (!compileOnly) Graphics::SetHostImportMaxBytes(maintenance3.maxMemoryAllocationSize);
 #ifdef _WIN32
         // Imported pages are locked; locking is bounded by the process working-set minimum.
         if (const char* value = std::getenv("APS5_WORKING_SET_MIB"); value != nullptr && !compileOnly) {
@@ -1507,12 +1622,17 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
             check(combinations(selected, &count, supported.data()), "vkGetPhysicalDeviceSupportedFramebufferMixedSamplesCombinationsNV");
             for (const auto& combination : supported) {
                 if (combination.coverageReductionMode == VK_COVERAGE_REDUCTION_MODE_MERGE_NV && combination.colorSamples == VK_SAMPLE_COUNT_1_BIT && combination.depthStencilSamples == 0) state->mergedCoverageSampleCounts |= combination.rasterizationSamples;
+                if (combination.coverageReductionMode == VK_COVERAGE_REDUCTION_MODE_TRUNCATE_NV && combination.colorSamples == VK_SAMPLE_COUNT_1_BIT && combination.depthStencilSamples == combination.rasterizationSamples) state->truncatedCoverageSampleCounts |= combination.rasterizationSamples;
             }
         }
-        if (state->mergedCoverageSampleCounts != 0) {
+        if (state->mergedCoverageSampleCounts != 0 || state->truncatedCoverageSampleCounts != 0) {
             deviceExtensions.push_back(VK_NV_FRAMEBUFFER_MIXED_SAMPLES_EXTENSION_NAME);
             deviceExtensions.push_back(VK_NV_COVERAGE_REDUCTION_MODE_EXTENSION_NAME);
         }
+    } else if (hasExtension(VK_AMD_MIXED_ATTACHMENT_SAMPLES_EXTENSION_NAME)) {
+        state->mixedAttachmentSamples = true;
+        state->truncatedCoverageSampleCounts = state->properties.limits.framebufferDepthSampleCounts & state->properties.limits.framebufferColorSampleCounts & ~static_cast<VkSampleCountFlags>(VK_SAMPLE_COUNT_1_BIT);
+        deviceExtensions.push_back(VK_AMD_MIXED_ATTACHMENT_SAMPLES_EXTENSION_NAME);
     }
     coverageReductionFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COVERAGE_REDUCTION_MODE_FEATURES_NV};
     coverageReductionFeatures.coverageReductionMode = VK_TRUE;
@@ -1562,35 +1682,18 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     queueInfo.queueCount = 1;
     queueInfo.pQueuePriorities = &priority;
     const float priorities[2]{1.0f, 1.0f};
-    bool settleQueue = false;
-    std::uint32_t settleFamily = family;
-    std::uint32_t settleIndex = 0;
-    std::vector<VkQueueFamilyProperties> queues;
 #ifdef _WIN32
     const bool settles = true;
 #else
     const bool settles = state->dmaBufImport;
 #endif
-    if (settles) {
+    std::vector<VkQueueFamilyProperties> queues;
+    {
         std::uint32_t families = 0;
         auto getFamilies = state->InstanceFunction<PFN_vkGetPhysicalDeviceQueueFamilyProperties>("vkGetPhysicalDeviceQueueFamilyProperties");
         getFamilies(selected, &families, nullptr);
         queues.resize(families);
         getFamilies(selected, &families, queues.data());
-        if (family < families && queues[family].queueCount >= 2) {
-            queueInfo.queueCount = 2;
-            queueInfo.pQueuePriorities = priorities;
-            settleQueue = true;
-            settleIndex = 1;
-        } else {
-            const auto copies = [&](std::uint32_t index) { return index != family && queues[index].queueCount != 0 && (queues[index].queueFlags & (VK_QUEUE_TRANSFER_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) != 0; };
-            for (std::uint32_t i = 0; i < families && !settleQueue; ++i) {
-                if (copies(i) && (queues[i].queueFlags & (VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT)) == 0) settleFamily = i, settleQueue = true;
-            }
-            for (std::uint32_t i = 0; i < families && !settleQueue; ++i) {
-                if (copies(i)) settleFamily = i, settleQueue = true;
-            }
-        }
     }
     VkDeviceCreateInfo deviceInfo{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO};
     deviceInfo.queueCreateInfoCount = 1;
@@ -1625,50 +1728,38 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     // Guest shaders routinely read past descriptor ranges; robust access turns that into zeros
     // instead of a GPU fault that loses the device.
     enabled.robustBufferAccess = available.robustBufferAccess;
+    const auto plan = PlanDeviceQueues(queues, family, settles, available.sparseBinding == VK_TRUE);
+    const bool settleQueue = plan.settle;
+    const auto settleFamily = plan.settleFamily;
+    const auto settleIndex = plan.settleIndex;
     VkDeviceQueueCreateInfo queueInfos[3]{queueInfo, {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}, {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}};
-    if (available.sparseBinding && state->dmaBufImport) {
-        std::uint32_t families = 0;
-        auto getFamilies = state->InstanceFunction<PFN_vkGetPhysicalDeviceQueueFamilyProperties>("vkGetPhysicalDeviceQueueFamilyProperties");
-        getFamilies(selected, &families, nullptr);
-        std::vector<VkQueueFamilyProperties> queues(families);
-        getFamilies(selected, &families, queues.data());
-        const auto binds = [&](std::uint32_t index) { return index != family && queues[index].queueCount != 0 && (queues[index].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) != 0; };
-        std::uint32_t sparseFamily = families;
-        for (std::uint32_t i = 0; i < families && sparseFamily == families; ++i) {
-            if (binds(i) && queues[i].queueFlags == VK_QUEUE_SPARSE_BINDING_BIT) sparseFamily = i;
+    queueInfos[0].queueCount = plan.mainCount;
+    queueInfos[0].pQueuePriorities = plan.mainCount == 2 ? priorities : &priority;
+    deviceInfo.queueCreateInfoCount = 1;
+    const auto addQueue = [&](std::uint32_t queueFamily, std::uint32_t index) {
+        for (std::uint32_t info = 1; info < deviceInfo.queueCreateInfoCount; ++info) {
+            if (queueInfos[info].queueFamilyIndex != queueFamily) continue;
+            queueInfos[info].queueCount = std::max(queueInfos[info].queueCount, index + 1);
+            queueInfos[info].pQueuePriorities = queueInfos[info].queueCount == 2 ? priorities : &priority;
+            return;
         }
-        for (std::uint32_t i = 0; i < families && sparseFamily == families; ++i) {
-            if (binds(i)) sparseFamily = i;
-        }
-        if (sparseFamily != families) {
-            queueInfos[1].queueFamilyIndex = sparseFamily;
-            queueInfos[1].queueCount = 1;
-            queueInfos[1].pQueuePriorities = &priority;
-            deviceInfo.queueCreateInfoCount = 2;
-            state->sparseFamily = sparseFamily;
-            state->sparseBinding = true;
-        } else {
-            state->sparseBinding = family < families && (queues[family].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) != 0;
-        }
-        enabled.sparseBinding = state->sparseBinding ? VK_TRUE : VK_FALSE;
-    }
+        auto& added = queueInfos[deviceInfo.queueCreateInfoCount++];
+        added = {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};
+        added.queueFamilyIndex = queueFamily;
+        added.queueCount = index + 1;
+        added.pQueuePriorities = added.queueCount == 2 ? priorities : &priority;
+    };
+    if (plan.settle && plan.settleFamily != family) addQueue(plan.settleFamily, plan.settleIndex);
+    if (plan.sparseFamily != VK_QUEUE_FAMILY_IGNORED) addQueue(plan.sparseFamily, plan.sparseIndex);
     deviceInfo.pQueueCreateInfos = queueInfos;
-    if (settleQueue && settleFamily != family) {
-        if (deviceInfo.queueCreateInfoCount == 2 && queueInfos[1].queueFamilyIndex == settleFamily) {
-            settleQueue = queues[settleFamily].queueCount >= 2;
-            if (settleQueue) {
-                queueInfos[1].queueCount = 2;
-                queueInfos[1].pQueuePriorities = priorities;
-                settleIndex = 1;
-            }
-        } else {
-            auto& settleInfo = queueInfos[deviceInfo.queueCreateInfoCount++];
-            settleInfo = {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};
-            settleInfo.queueFamilyIndex = settleFamily;
-            settleInfo.queueCount = 1;
-            settleInfo.pQueuePriorities = &priority;
-        }
-    }
+    state->sparseFamily = plan.sparseFamily;
+    state->sparseIndex = plan.sparseIndex;
+    state->sparseBinding = plan.sparseBinding;
+    enabled.sparseBinding = state->sparseBinding ? VK_TRUE : VK_FALSE;
+    enabled.sparseResidencyBuffer = state->sparseBinding ? available.sparseResidencyBuffer : VK_FALSE;
+    enabled.sparseResidencyAliased = state->sparseBinding ? available.sparseResidencyAliased : VK_FALSE;
+    state->sparseResidencyBuffer = enabled.sparseResidencyBuffer == VK_TRUE;
+    state->sparseResidencyAliased = enabled.sparseResidencyAliased == VK_TRUE;
     // Indirect draw records with a non-zero first instance, and several records per call.
     enabled.drawIndirectFirstInstance = available.drawIndirectFirstInstance;
     enabled.multiDrawIndirect = available.multiDrawIndirect;
@@ -1703,6 +1794,8 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     enabled.sampleRateShading = available.sampleRateShading;
     enabled.geometryShader = available.geometryShader;
     state->geometryShader = enabled.geometryShader == VK_TRUE;
+    enabled.largePoints = available.largePoints;
+    state->largePoints = enabled.largePoints == VK_TRUE;
     state->sampleRateShading = enabled.sampleRateShading == VK_TRUE;
     if (enabled.sampleRateShading) state->capabilities.push_back(spv::CapabilityInterpolationFunction);
     if (enabled.geometryShader) state->capabilities.push_back(spv::CapabilityGeometry);
@@ -1803,7 +1896,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         nonSeamlessCubeFeatures.pNext = const_cast<void*>(deviceInfo.pNext);
         deviceInfo.pNext = &nonSeamlessCubeFeatures;
     }
-    if (state->mergedCoverageSampleCounts != 0) {
+    if (!state->mixedAttachmentSamples && (state->mergedCoverageSampleCounts != 0 || state->truncatedCoverageSampleCounts != 0)) {
         coverageReductionFeatures.pNext = const_cast<void*>(deviceInfo.pNext);
         deviceInfo.pNext = &coverageReductionFeatures;
     }
@@ -1932,9 +2025,11 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         faultFeatures.pNext = byteFeatures.pNext;
         byteFeatures.pNext = &faultFeatures;
     }
-    const bool checkpoints = std::getenv("APS5_GPU_CHECKPOINTS") != nullptr && hasExtension(VK_NV_DEVICE_DIAGNOSTIC_CHECKPOINTS_EXTENSION_NAME);
-    if (checkpoints) {
-        deviceExtensions.push_back(VK_NV_DEVICE_DIAGNOSTIC_CHECKPOINTS_EXTENSION_NAME);
+    const bool checkpointsRequested = std::getenv("APS5_GPU_CHECKPOINTS") != nullptr;
+    const bool checkpoints = checkpointsRequested && hasExtension(VK_NV_DEVICE_DIAGNOSTIC_CHECKPOINTS_EXTENSION_NAME);
+    const bool bufferMarkers = checkpointsRequested && !checkpoints && hasExtension(VK_AMD_BUFFER_MARKER_EXTENSION_NAME);
+    if (checkpoints || bufferMarkers) {
+        deviceExtensions.push_back(checkpoints ? VK_NV_DEVICE_DIAGNOSTIC_CHECKPOINTS_EXTENSION_NAME : VK_AMD_BUFFER_MARKER_EXTENSION_NAME);
         deviceInfo.enabledExtensionCount = static_cast<std::uint32_t>(deviceExtensions.size());
         deviceInfo.ppEnabledExtensionNames = deviceExtensions.data();
     }
@@ -1986,12 +2081,18 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         state->settleFamily = settleFamily;
         state->DeviceFunction<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(state->device, settleFamily, settleIndex, &state->settleQueue);
     }
-    if (state->sparseFamily != VK_QUEUE_FAMILY_IGNORED) state->DeviceFunction<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(state->device, state->sparseFamily, 0, &state->sparseQueue);
+    if (state->sparseFamily != VK_QUEUE_FAMILY_IGNORED) state->DeviceFunction<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(state->device, state->sparseFamily, state->sparseIndex, &state->sparseQueue);
     if (checkpoints) state->checkpointMarker = state->DeviceFunction<PFN_vkCmdSetCheckpointNV>("vkCmdSetCheckpointNV");
-    if (deviceFault || checkpoints) {
+    if (bufferMarkers) state->CreateMarkerBuffer();
+    if (deviceFault || checkpoints || bufferMarkers) {
         const auto faultInfo = deviceFault ? state->DeviceFunction<PFN_vkGetDeviceFaultInfoEXT>("vkGetDeviceFaultInfoEXT") : nullptr;
         const auto checkpointData = checkpoints ? state->DeviceFunction<PFN_vkGetQueueCheckpointDataNV>("vkGetQueueCheckpointDataNV") : nullptr;
-        Graphics::DeviceLostHook() = [device = state->device, queue = state->queue, faultInfo, checkpointData] {
+        Graphics::DeviceLostHook() = [device = state->device, queue = state->queue, faultInfo, checkpointData, owner = state.get()] {
+            if (owner->markerMapping != nullptr) {
+                std::uint32_t words[2];
+                std::memcpy(words, owner->markerMapping, sizeof(words));
+                for (const auto& line : owner->markerLog.Report(words[0], words[1])) std::fprintf(stderr, "%s\n", line.c_str());
+            }
             for (const auto& line : DeviceFaultReport(device, faultInfo, queue, checkpointData)) std::fprintf(stderr, "%s\n", line.c_str());
         };
         Graphics::DeviceLostOwner() = state.get();
@@ -2206,6 +2307,7 @@ void VulkanDevice::WaitIdle() {
         state->recorder->Sync();
     }
     check(state->DeviceFunction<PFN_vkDeviceWaitIdle>("vkDeviceWaitIdle")(state->device), "vkDeviceWaitIdle");
+    Graphics::CheckPixelFunctionLds(graphicsContext());
     APS5_LOG_CHARS_OUT_DEBUG("VulkanDevice::WaitIdle complete");
 }
 
@@ -2293,6 +2395,7 @@ bool VulkanDevice::StoresPendingOver(std::uint64_t address, std::size_t bytes) c
 int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst) {
     static const bool storeAtLabel = Graphics::StorageTexture::StoreAtLabelRequested(std::getenv("APS5_STORE_AT_LABEL"));
     if (storeAtLabel) Graphics::StorageTexture::FlushAllPending("label");
+    Graphics::StorageTexture::StoreReadbackBeforeLabel("readback before label");
     if (!state->recorder || bytes.empty() || bytes.size() > 65536 || address % 4 != 0 || bytes.size() % 4 != 0) return 4;
     auto& recorder = *state->recorder;
     // Batches that already finished are retired before the checks: their completions ran, so they
@@ -2318,10 +2421,37 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::by
     // stores of a run against each other and against the work before and after, the queued DCC
     // key stores included).
     const auto recordStore = [&] { recorder.RecordStore(import->buffer, Graphics::HostImportOffset(*import, address), bytes, address); };
-    // Only a write-back that can land over these bytes (a listed copied writer of the range, as
-    // FillBuffer tests) puts the label behind the completions; any other pending completion
-    // stores elsewhere. Debug aid: APS5_LABEL_GATE_ALL=1 gates on any completion, as before (a
-    // completion label there registered its own completion, so every later label followed).
+    // On the console every store a queue made before a label is in memory once the label is
+    // visible. A listed copied writer (a dispatch or draw whose written buffer the device holds as a
+    // CPU copy) is written back by the CPU when its batch is reaped, after the GPU already ran any
+    // store recorded here, so a title polling the label could rewrite the written region and have
+    // the late write-back land over its bytes (a title that rebuilds its command buffer there sees
+    // a stale packet). While one is listed, a label of 8 bytes or less (every RELEASE_MEM, the fence
+    // WRITE_DATAs) is therefore not recorded on the GPU at all: a completion action of the open (or
+    // newest in-flight) batch stores it when that batch is reaped. Completions run in push order and
+    // batches finish front-first, so it lands after every earlier write-back, whatever range that
+    // one wrote. The GPU store plus completion re-store below would not do: its GPU store is
+    // visible before the write-back, and its re-store runs only over a write-back that overlapped
+    // the label. While such a completion store is pending (PendingCompletionStores: no end-of-pipe
+    // delivery and no write-back counts there), later small labels follow it even after every
+    // writer was written back, or a GPU store would become visible before the earlier completion
+    // store lands (#1458), or land under it at the reap when both are at one address. Larger stores
+    // (DUMP_CONST_RAM, COPY_DATA and DMA_DATA payloads that later GPU work reads through the import)
+    // stay on the GPU and follow only a pending completion store over their own range. Readers of a
+    // range stored this way sync through its batch first (the label is a pending write of its batch
+    // and counted in PendingCompletionLabels). Debug aid: APS5_DRAIN_COMPLETION_LABELS=1 drains
+    // (reason 2) instead and stores the label on the CPU.
+    const bool listed = !state->copiedWriters->empty() || !Graphics::DrawCopiedWriters()->empty();
+    if (drain) {
+        if (listed) return 2;
+    } else if (const bool storesPending = Graphics::Recorder::PendingCompletionStores() != 0; bytes.size() <= 8 ? (listed || storesPending) : (storesPending && recorder.CompletionLabelIn(address, bytes.size()))) {
+        recorder.AfterCompletions(address, bytes, stamp, queue, false);
+        return 5;
+    }
+    // A larger store over a listed copied writer's range (as FillBuffer tests it) goes behind the
+    // completions too, below; any other one stores on the GPU. Debug aid: APS5_LABEL_GATE_ALL=1
+    // gates on any completion, as before (a completion label there registered its own completion,
+    // so every later label followed).
     static const bool gateAll = std::getenv("APS5_LABEL_GATE_ALL") != nullptr;
     if (gateAll ? recorder.HasCompletions() : state->CopiedWriterOverlaps(address, bytes.size())) {
         // A copied buffer's write-back is still to run: the store must land after it. It is stored
@@ -3573,6 +3703,7 @@ double VulkanDevice::RetirePresents(std::size_t keepInFlight) {
     const auto status = state->DeviceFunction<PFN_vkGetFenceStatus>("vkGetFenceStatus");
     auto inFlight = static_cast<std::size_t>(std::count_if(slots.begin(), slots.end(), [](const auto& slot) { return slot.inFlight; }));
     double waitedMs = 0;
+    bool retired = false;
     // Oldest first from the cursor; the fences of one queue signal in submission order, so the
     // first unsignaled one ends the scan, and the ones beyond the bound are the oldest.
     for (std::size_t i = 0; i < slots.size() && inFlight != 0; ++i) {
@@ -3591,9 +3722,11 @@ double VulkanDevice::RetirePresents(std::size_t keepInFlight) {
         }
         const bool dump = slot.dumpRecorded;
         state->RetireSlot(slot);
+        retired = true;
         if (dump) timing.Mark("dump");
         --inFlight;
     }
+    if (retired) Graphics::CheckPixelFunctionLds(graphicsContext());
     return waitedMs;
 }
 
@@ -3676,6 +3809,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::Target() const {
     require(state->shaderProfile != nullptr, "shader device profile is unavailable");
     auto target = state->shaderProfile->Target();
     target.float32FlushToZero = state->float32FlushToZero;
+    target.fusedExactArithmetic = state->fusedExactArithmetic;
     target.float32DenormPreserve = state->float32DenormPreserve;
     return target;
 }
@@ -3699,6 +3833,9 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const {
     target.registrationModules = state->graphicsPipelineLibrary;
     target.robustBufferAccess2 = state->robustBufferAccess2;
     target.waveLdsMemoryBarriers = !state->subgroupControlBarriers;
+    target.largePixelFunctionLdsBuffer = state->driverId == VK_DRIVER_ID_NVIDIA_PROPRIETARY;
+    target.pixelFunctionStorageDwords = state->pixelFunctionStorageDwords;
+    target.deviceScopeLdsMemoryBarriers = !state->workgroupScopeFences;
     target.srgbDecodeFormats = state->srgbDecodeFormats & ~state->srgbCopyFormats;
     target.vertexSubgroups = (state->subgroup.supportedStages & VK_SHADER_STAGE_VERTEX_BIT) != 0;
     if (state->meshShader) {
@@ -3786,6 +3923,8 @@ Graphics::Context VulkanDevice::buildContext() const {
     context.individuallyFreeDescriptorSets = state->individuallyFreeDescriptorSets;
     context.dmaBufImport = state->dmaBufImport;
     context.sparseBinding = state->sparseBinding;
+    context.sparseResidencyBuffer = state->sparseResidencyBuffer;
+    context.sparseResidencyAliased = state->sparseResidencyAliased;
     context.queueFamily = state->queueFamily;
     context.sparseFamily = state->sparseFamily;
     context.sparseQueue = state->sparseQueue;
@@ -3806,12 +3945,15 @@ Graphics::Context VulkanDevice::buildContext() const {
     context.samplerFilterMinmax = state->samplerFilterMinmax;
     context.nonSeamlessCubeMap = state->nonSeamlessCubeMap;
     context.mergedCoverageSampleCounts = state->mergedCoverageSampleCounts;
+    context.truncatedCoverageSampleCounts = state->truncatedCoverageSampleCounts;
+    context.mixedAttachmentSamples = state->mixedAttachmentSamples;
     context.conservativeRasterization = state->conservativeRasterization;
     context.meshWave32 = state->meshWave32;
     context.provokingVertexLast = state->provokingVertexLast;
     context.graphicsPipelineLibrary = state->graphicsPipelineLibrary;
     context.pipelineLinkTimeOptimization = state->pipelineLinkTimeOptimization;
     context.drawInputsInPlace = state->drawInputsInPlace;
+    context.discreteGpu = state->properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
     context.provokingVertexModePerPipeline = state->provokingVertexModePerPipeline;
     context.drawIndirectCount = state->drawIndirectCount;
     context.occlusionQueryPrecise = state->occlusionQueryPrecise;
@@ -3830,6 +3972,7 @@ Graphics::Context VulkanDevice::buildContext() const {
     context.imageTableUpdateAfterBind = ImageTableUpdateAfterBind(context);
     context.samplerTableUpdateAfterBind = SamplerTableUpdateAfterBind(context);
     context.geometryShader = state->geometryShader;
+    context.largePoints = state->largePoints;
     context.sampleRateShading = state->sampleRateShading;
     context.fillModeNonSolid = state->fillModeNonSolid;
     context.maintenance5 = state->maintenance5;
@@ -3842,6 +3985,7 @@ Graphics::Context VulkanDevice::buildContext() const {
     context.bufferInt64Atomics = state->bufferInt64Atomics;
     context.placeholderImages = state->placeholderImages.get();
     context.viewportIndexLayer = state->viewportIndexLayer;
+    context.imageFormatList = state->imageFormatList;
     context.demoteToHelperInvocation = state->demoteToHelperInvocation;
     context.earlyFragmentSampleMaskTestBeforeSampleCounting = state->earlyFragmentSampleMaskTestBeforeSampleCounting;
     context.primitiveListRestart = state->primitiveListRestart;
@@ -4090,7 +4234,7 @@ void WatchMemory(std::uint64_t programAddress) {
         return parsed;
     }();
     if (ranges.empty()) return;
-    static std::mutex mutex;
+    static AgcDriver::Mutex mutex;
     static std::deque<std::pair<std::uint64_t, std::vector<std::uint8_t>>> history;
     static bool written = false;
     std::lock_guard lock(mutex);
@@ -4403,6 +4547,7 @@ struct RecordedDispatch {
     // A template hit's revalidate time, charged to the [rescache] refresh accounting.
     double revalidateMs;
     DispatchTimer* timer;
+    std::optional<std::pair<std::uint64_t, std::size_t>> htileClear;
     // Whether the data refresh recorded anything.
     bool refreshed = false;
     bool resourceKeyMatched = false;
@@ -4413,6 +4558,24 @@ struct RecordedDispatch {
     std::array<std::uint32_t, 3> threadLimit{};
 };
 
+struct DispatchCompletion {
+    const Graphics::Context& context;
+    const std::array<Graphics::CompiledShader, 1>& shaders;
+    VkShaderStageFlags pushStages;
+    const std::array<std::byte, Graphics::PipelinePushConstantBytes>& pushBytes;
+    const std::shared_ptr<ComputePipelineObjects>& objects;
+    const std::shared_ptr<Graphics::ShaderResources>& resources;
+    const ResourceCache::Key& contentKey;
+    bool cacheable;
+    bool fromCache;
+    double revalidateMs;
+    const std::shared_ptr<Graphics::ShaderResources::DrawBindings>& overlay;
+    std::uint64_t lookupsBefore;
+    DispatchTimer& timer;
+    PerformanceTimer& timing;
+    char* groupsText;
+};
+
 VkDevice VulkanDevice::Device() const {
     return state->device;
 }
@@ -4663,8 +4826,8 @@ std::shared_ptr<RecipeHit> VulkanDevice::PrepareRecipe(const std::shared_ptr<con
     return hit;
 }
 
-void VulkanDevice::Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t programAddress, std::shared_ptr<PreparedDispatch> prepared, std::shared_ptr<const Recipe>* recipe) {
-    static_cast<void>(dispatch(shader, x, y, z, 0, snapshots, programAddress, std::move(prepared), recipe));
+void VulkanDevice::Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t programAddress, std::shared_ptr<PreparedDispatch> prepared, std::shared_ptr<const Recipe>* recipe, std::optional<std::pair<std::uint64_t, std::size_t>> htileClear) {
+    static_cast<void>(dispatch(shader, x, y, z, 0, snapshots, programAddress, std::move(prepared), recipe, {}, htileClear));
 }
 
 VulkanDevice::IndirectOutcome VulkanDevice::DispatchIndirect(const ShaderRecompiler::RecompileResult& shader, std::uint64_t arguments, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t programAddress, std::shared_ptr<PreparedDispatch> prepared, std::shared_ptr<const Recipe>* recipe, const std::array<std::uint32_t, 3>& threadGroup) {
@@ -4886,6 +5049,8 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
     }
     const auto gpuTiming = recorder.BeginGpuTiming(record.programAddress != 0 ? record.programAddress : record.shader->program->variantId);
     if (state->checkpointMarker != nullptr) state->checkpointMarker(commands, reinterpret_cast<const void*>(static_cast<std::uintptr_t>(record.programAddress)));
+    const auto marker = state->bufferMarker != nullptr ? state->markerLog.Next(record.programAddress) : 0u;
+    if (marker != 0) state->bufferMarker(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, state->markerBuffer, 0, marker);
     if (argumentImport != nullptr) {
         context.Resolved(&Graphics::DeviceFunctions::cmdDispatchIndirect, "vkCmdDispatchIndirect")(commands, indirectBuffer, indirectOffset);
     } else {
@@ -4918,6 +5083,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
         }
     }
     recorder.EndGpuTiming(gpuTiming);
+    if (marker != 0) state->bufferMarker(commands, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, state->markerBuffer, sizeof(std::uint32_t), marker);
     const auto trailingTiming = recorder.BeginGpuTiming(CommandClass::DispatchTrailing);
     constexpr VkAccessFlags dispatchedAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT;
     Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT, dispatchedAccess);
@@ -4925,21 +5091,31 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
     recorder.EndGpuTiming(trailingTiming);
     recorder.MarkCovered(dispatchedAccess);
     recordStep(PhaseRecordBind);
+    if (record.htileClear.has_value()) Graphics::ArmHtileDepthClear(record.htileClear->first, record.htileClear->second);
     resources.MarkGpuWrites(recorder);
+    if (record.htileClear.has_value()) {
+        Graphics::QueueHtileDepthClear(record.htileClear->first, record.htileClear->second);
+        recorder.NotePendingWrite(record.htileClear->first, record.htileClear->second);
+    }
     recordStep(PhaseRecordMarks);
     // Only copied written buffers (and BDA fault checks) need work once the GPU is done; without them
     // the batch can signal its labels from the GPU.
-    if (resources.NeedsCompletion() && resources.Reusable()) {
+    const bool completionNeeded = resources.NeedsCompletion() || record.htileClear.has_value();
+    if (completionNeeded && resources.Reusable()) {
         auto kept = record.resources;
-        recorder.OnComplete([kept] { kept->CheckFaults(); });
-    } else if (resources.NeedsCompletion()) {
+        recorder.OnComplete([kept, htileClear = record.htileClear] {
+            kept->CheckFaults();
+            if (htileClear.has_value()) Graphics::CompleteHtileDepthClear(htileClear->first, htileClear->second);
+        });
+    } else if (completionNeeded) {
         // Listed for DispatchIndirect until the write-back ran (a non-reusable object, so once).
         auto writers = state->copiedWriters;
         auto kept = record.resources;
-        recorder.OnComplete([kept, writers] {
+        recorder.OnComplete([kept, writers, htileClear = record.htileClear] {
             // Delisted before the write-back: one that fails must not keep indirect dispatches on the CPU.
             writers->erase(std::remove(writers->begin(), writers->end(), kept), writers->end());
             kept->WriteBackBuffers();
+            if (htileClear.has_value()) Graphics::CompleteHtileDepthClear(htileClear->first, htileClear->second);
         });
         // Listed after the registration: a throw there leaves nothing that would pin the CPU path forever.
         if (resources.HasCopiedWrites()) writers->push_back(kept);
@@ -4973,7 +5149,7 @@ std::shared_ptr<ComputePipelineObjects> VulkanDevice::createComputePipeline(cons
     const Graphics::PipelineSpecialization specialization(shader);
     const auto subgroupSize = computeSubgroupSize(shader);
     const auto stageFlags = computeStageFlags(shader);
-    if (const char* directory = std::getenv("APS5_DUMP_COMPUTE_PIPELINES"); directory != nullptr && *directory != '\0' && ComputePipelineCapture::OptionalEnabled() && CaptureComputePipeline(shader.spirv.size())) {
+    if (const auto* directory = ComputePipelineDumpDirectory(); directory != nullptr && *directory != 0 && !state->probingFloatControls && ComputePipelineCapture::OptionalEnabled() && CaptureComputePipeline(shader.spirv.size())) {
         ComputePipelineCapture capture;
         capture.spirv = shader.spirv.Copy();
         ComputeDescriptorSetLayout resourcesLayout;
@@ -5070,7 +5246,7 @@ std::shared_ptr<ComputePipelineObjects> VulkanDevice::computePipeline(const Grap
     return finish(std::move(objects), nullptr);
 }
 
-VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t programAddress, std::shared_ptr<PreparedDispatch> prepared, std::shared_ptr<const Recipe>* recipeOut, const std::array<std::uint32_t, 3>& threadGroup) {
+VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t programAddress, std::shared_ptr<PreparedDispatch> prepared, std::shared_ptr<const Recipe>* recipeOut, const std::array<std::uint32_t, 3>& threadGroup, std::optional<std::pair<std::uint64_t, std::size_t>> htileClear) {
     startJournalReplay();
     for (const auto& binding : shader.bindings) {
         if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue;
@@ -5318,11 +5494,32 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
         timing.Mark("pipeline_create");
         timer.phase(PhasePipeline);
     }
+    DispatchCompletion completion{context, shaders, pushStages, pushBytes, objects, resources, contentKey, cacheable, fromCache, revalidateMs, overlay, lookupsBefore, timer, timing, groupsText};
+    return recordCompletedDispatch(shader, x, y, z, arguments, programAddress, recipeOut, threadGroup, htileClear, completion);
+}
+
+VulkanDevice::IndirectOutcome VulkanDevice::recordCompletedDispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::uint64_t programAddress, std::shared_ptr<const Recipe>* recipeOut, const std::array<std::uint32_t, 3>& threadGroup, std::optional<std::pair<std::uint64_t, std::size_t>> htileClear, DispatchCompletion& completion) {
+    static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
+    const auto& context = completion.context;
+    const auto& shaders = completion.shaders;
+    const auto pushStages = completion.pushStages;
+    const auto& pushBytes = completion.pushBytes;
+    const auto& objects = completion.objects;
+    const auto& resources = completion.resources;
+    const auto& contentKey = completion.contentKey;
+    const bool cacheable = completion.cacheable;
+    const bool fromCache = completion.fromCache;
+    const double revalidateMs = completion.revalidateMs;
+    const auto& overlay = completion.overlay;
+    const auto lookupsBefore = completion.lookupsBefore;
+    auto& timer = completion.timer;
+    auto& timing = completion.timing;
+    auto* groupsText = completion.groupsText;
     // Recorded into the device's open batch: the CPU moves on to the next command while the GPU
     // works; the batch is waited for where the guest expects results (labels, flips, CPU reads).
     auto& recorder = *state->recorder;
     IndirectOutcome outcome{0, 0};
-    RecordedDispatch record{&context, &shaders[0], resources, objects, pushStages, &pushBytes, x, y, z, arguments, nullptr, programAddress, fromCache && TemplateDataRefresh() ? RecordedDispatch::DataRefresh::Words : RecordedDispatch::DataRefresh::None, 0, revalidateMs, &timer};
+    RecordedDispatch record{&context, &shaders[0], resources, objects, pushStages, &pushBytes, x, y, z, arguments, nullptr, programAddress, fromCache && TemplateDataRefresh() ? RecordedDispatch::DataRefresh::Words : RecordedDispatch::DataRefresh::None, 0, revalidateMs, &timer, htileClear};
     record.resourceKeyMatched = fromCache;
     record.bindings = overlay;
     record.threadGroup = threadGroup;

core/libs/prx/libSceAgcDriver/Execution/src/WorkerSampler.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
@@ -43,7 +44,7 @@ struct Sampler {
     std::uint64_t stackHigh = 0;
     std::vector<DwarfFrames> modules;
     std::uint64_t pendingModule = 0;
-    std::mutex mutex;
+    AgcDriver::Mutex mutex;
     std::map<std::uint64_t, std::pair<std::uint64_t, std::uint64_t>> counts;
     CollapsedStacks stacks;
     std::uint64_t samples = 0;

core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp

@@ -3,6 +3,7 @@
 
 #include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
 #include <chrono>
+#include "prx/libSceAgcDriver/Execution/include/Mutex.hpp"
 #include <cstdint>
 #include <deque>
 #include <list>
@@ -65,13 +66,14 @@ public:
     // The size a buffer for `bytes` is created with: its size class, or `bytes` itself when large.
     static std::size_t Capacity(std::size_t bytes, VkMemoryPropertyFlags properties);
     static VkBufferUsageFlags Usage(VkBufferUsageFlags usage, VkMemoryPropertyFlags properties);
-    VkDeviceMemory AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional<std::uint32_t> hostType = std::nullopt, std::uint32_t* allocatedType = nullptr);
+    VkDeviceMemory AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional<std::uint32_t> hostType = std::nullopt, std::uint32_t* allocatedType = nullptr, bool newImage = false);
     static VkDeviceSize ImageBudget(VkDeviceSize deviceBudget);
-    VkDeviceMemory TakeImageMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional<std::uint32_t> hostType = std::nullopt, std::uint32_t* allocatedType = nullptr);
+    VkDeviceMemory TakeImageMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional<std::uint32_t> hostType = std::nullopt, std::uint32_t* allocatedType = nullptr, bool newImage = false);
     void PutImageMemory(VkDeviceMemory memory, VkDeviceSize bytes, std::uint32_t memoryType) noexcept;
     std::size_t RetainedImages();
     std::optional<BufferAllocation> Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
     void Put(const BufferAllocation& allocation) noexcept;
+    VkDeviceSize ReclaimIdleDevice(VkDeviceSize bytes, std::chrono::steady_clock::time_point now);
     static bool SlabEligible(std::size_t capacity, VkDeviceSize alignment, VkDeviceSize size, VkDeviceSize atom);
     static VkDeviceSize SlabBlockBytes(std::size_t capacity);
     SlabSlot TakeSlot(const Context& context, std::uint32_t memoryType, std::size_t capacity, bool addressable);
@@ -142,7 +144,7 @@ private:
     // The retained-slot bound of each tier (APS5_BUFFER_POOL_SLOTS, default `defaultSlots`), read once.
     static std::size_t MaxSlots();
     std::size_t maxSlots;
-    VkDeviceSize (*cacheReclaim)(VkDeviceSize bytes);
+    VkDeviceSize (*cacheReclaim)(VkDeviceSize bytes, bool newImage);
     VkDevice device;
     PFN_vkUnmapMemory unmap;
     PFN_vkDestroyBuffer destroyBuffer;
@@ -152,7 +154,7 @@ private:
     VkPhysicalDevice physical = VK_NULL_HANDLE;
     VkPhysicalDeviceMemoryProperties memoryProperties{};
     PFN_vkGetPhysicalDeviceMemoryProperties2 memoryProperties2 = nullptr;
-    std::mutex mutex;
+    AgcDriver::Mutex mutex;
     // Not `small`/`large`: <rpcndr.h> (via <windows.h>) defines `small` as a macro.
     Tier smallTier;
     Tier largeTier;
@@ -167,7 +169,11 @@ private:
     std::uint64_t imageEvictions = 0;
     std::list<Slot*> imageOrder;
     std::uint64_t clock = 0;
-    std::mutex slabMutex;
+    std::optional<std::chrono::steady_clock::time_point> idleRead;
+    std::uint64_t idleClock = 0;
+    std::uint64_t reclaimBefore = 0;
+    std::uint64_t scannedIdleClock = 0;
+    AgcDriver::Mutex slabMutex;
     std::unordered_map<std::uint64_t, Slab> slabs;
     std::unordered_map<VkDeviceMemory, std::unique_ptr<SlabBlock>> slabBlocks;
     static constexpr VkDeviceSize budget = 512ull * 1024 * 1024;
@@ -181,7 +187,7 @@ private:
 };
 
 std::shared_ptr<BufferPool> GetBufferPool(const Context& context);
-VkDeviceMemory AllocateImageMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation, std::uint32_t* allocatedType = nullptr);
+VkDeviceMemory AllocateImageMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation, std::uint32_t* allocatedType = nullptr, bool newImage = false);
 void ReleaseImageMemory(const Context& context, VkDeviceMemory memory, VkDeviceSize bytes, std::uint32_t memoryType) noexcept;
 
 }

core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp

@@ -180,8 +180,11 @@ struct Context {
     // Nonzero when VK_EXT_external_memory_host is enabled: the required host pointer alignment.
     VkDeviceSize hostImportAlignment = 0;
     bool drawInputsInPlace = true;
+    bool discreteGpu = false;
     bool dmaBufImport = false;
     bool sparseBinding = false;
+    bool sparseResidencyBuffer = false;
+    bool sparseResidencyAliased = false;
     std::uint32_t queueFamily = 0;
     std::uint32_t sparseFamily = VK_QUEUE_FAMILY_IGNORED;
     VkQueue sparseQueue = VK_NULL_HANDLE;
@@ -191,7 +194,7 @@ struct Context {
     std::uint32_t settleFamily = 0;
     std::mutex* settleQueueMutex = nullptr;
     PFN_vkGetPhysicalDeviceMemoryProperties2 memoryProperties2 = nullptr;
-    VkDeviceSize (*cacheReclaim)(VkDeviceSize bytes) = nullptr;
+    VkDeviceSize (*cacheReclaim)(VkDeviceSize bytes, bool newImage) = nullptr;
     RenderCache* renderCache = nullptr;
     DrawQueue* drawQueue = nullptr;
     GraphicsPipelineCache* graphicsPipelines = nullptr;
@@ -213,6 +216,8 @@ struct Context {
     bool samplerFilterMinmax = false;
     bool nonSeamlessCubeMap = false;
     VkSampleCountFlags mergedCoverageSampleCounts = 0;
+    VkSampleCountFlags truncatedCoverageSampleCounts = 0;
+    bool mixedAttachmentSamples = false;
     bool conservativeRasterization = false;
     bool meshWave32 = false;
     VkBuffer emptyBuffer = VK_NULL_HANDLE;
@@ -237,11 +242,13 @@ struct Context {
     bool imageTableUpdateAfterBind = false;
     bool samplerTableUpdateAfterBind = false;
     bool geometryShader = false;
+    bool largePoints = false;
     bool sampleRateShading = false;
     bool fillModeNonSolid = false;
     bool maintenance5 = false;
     bool storageImageMultisample = false;
     bool storageImageWriteWithoutFormat = false;
+    bool imageFormatList = false;
     bool shaderClipDistance = false;
     bool shaderCullDistance = false;
     bool nullDescriptors = false;

core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp

@@ -81,6 +81,16 @@ struct DccKeyProof {
 // APS5_NO_KEY_FAST_PATH=1 scans on every call and stores nothing.
 DccKeys ProvedClearKeys(const GuestTextureResource& resource, std::uint64_t guestBytes, DccKeyProof& proof);
 bool KeyFastPath();
+bool SingleKeysReadable(const GuestTextureResource& resource);
+bool ReadSingleElement(const GuestTextureResource& resource, std::span<std::byte> element);
+struct DccSingleProof {
+    std::uint64_t address = 0;
+    std::uint64_t bytes = 0;
+    std::uint64_t generation = 0;
+    std::uint32_t format = 0;
+    bool uniform = false;
+};
+bool ProvedSingleUniform(const GuestTextureResource& resource, std::uint64_t guestBytes, std::uint64_t collected, DccSingleProof& proof);
 // Cumulative outcomes of ProvedClearKeys: calls answered by a proof, scans made, and scans not kept.
 struct DccKeyProofCounts {
     std::uint64_t proved;

core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp

@@ -29,6 +29,7 @@ struct DepthPlaneStoreStatistics {
     std::uint64_t bytes = 0;
 };
 DepthPlaneStoreStatistics DepthPlaneStoreCounts();
+void NoteDepthSurfaceWrite(const Context& context, const DepthTarget& target);
 bool SampleLocationsCompatibleDepth(const Context& context, VkFormat format, std::uint32_t samples);
 std::uint64_t DepthSurfacesSerial();
 std::uint64_t DepthSurfacesReleased();
@@ -54,7 +55,10 @@ bool DepthSurfaceOverlaps(VkDevice device, std::uint64_t address, std::uint64_t
 bool DepthSurfaceAt(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& resource);
 std::uint64_t HtileDepthClearAddress(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const std::array<std::uint32_t, 3>& numThreads);
 std::size_t HtileDepthClearBytes_nid_no_patch(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, std::span<const std::uint32_t> packet, std::uint64_t indirectArguments);
-void NoteHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes, std::size_t nativeBytes = 0u);
+bool NoteHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes, std::size_t nativeBytes = 0u);
+void ArmHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes);
+void QueueHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes);
+void CompleteHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes);
 struct MemoryValueFill {
     std::uint64_t address;
     std::size_t bytes;

core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp

@@ -18,6 +18,17 @@ struct HostImport;
 // built for its draw-cache entry (design_cpu_final M8; null otherwise, and always under
 // APS5_NO_DRAW_RECIPE=1).
 void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span<const CompiledShader> shaders, std::span<const GuestMemorySnapshot> snapshots = {}, std::shared_ptr<const DrawRecipe>* recipe = nullptr);
+
+struct PixelFunctionLdsLayouts {
+    VkDescriptorSetLayout empty = VK_NULL_HANDLE;
+    VkDescriptorSetLayout buffer = VK_NULL_HANDLE;
+};
+
+PixelFunctionLdsLayouts PixelFunctionLdsSetLayouts(const Context& context);
+void PreparePixelFunctionLds(const Context& context, VkCommandBuffer commands, std::uint32_t subgroupSize);
+void BindPixelFunctionLds(const Context& context, VkCommandBuffer commands, VkPipelineLayout layout);
+void CheckPixelFunctionLds(const Context& context);
+void ReleasePixelFunctionLds(VkDevice device);
 std::optional<std::string> KnownValidationFailure(const Context& context, std::span<const CompiledShader> shaders, const State& state);
 std::uint64_t DrawRenderPassKey(const Context& context, const State& state, std::span<const VkImageView> targetViews);
 
@@ -94,6 +105,8 @@ bool DrawRecipes();
 enum class IndirectDrawPath : std::uint8_t { Gpu, NotFolded, FetchUnknown, NonVertexPath, IndxOffset, DrawIndex, VertexRange, FeatureGap, PendingImage, PendingLabelOrCopy, NotImported, Disabled, IndexRange, Count };
 void CountIndirectDraw(IndirectDrawPath path, double readMs, bool rewritten = false);
 std::array<std::uint64_t, static_cast<std::size_t>(IndirectDrawPath::Count)> IndirectDrawCounts();
+void CountEmptyIndirectDraw(double readMs);
+std::uint64_t EmptyIndirectDrawCount();
 
 struct DrawPassCounts {
     std::uint64_t draws = 0;
@@ -128,6 +141,9 @@ std::shared_ptr<std::vector<std::shared_ptr<ShaderResources>>> DrawCopiedWriters
 
 void RunColorMetadataPass(const Context& context, const ColorMetadataPass& pass);
 void RunDepthCopyPass(const Context& context, const DepthCopyPass& pass);
+std::vector<std::pair<std::uint64_t, std::uint64_t>> PipelinedColorMetadataPassWrites(const ColorMetadataPass& pass);
+bool DrawPipelineEnabled();
+bool CommitTrims();
 
 }
 

core/libs/prx/libSceAgcDriver/Graphics/include/FirstSampleCopy.hpp

@@ -0,0 +1,40 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_FIRSTSAMPLECOPY_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_FIRSTSAMPLECOPY_HPP
+
+#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
+
+namespace AgcDriver::Graphics {
+
+class Recorder;
+
+class FirstSampleCopy {
+public:
+    FirstSampleCopy(const Context& context, VkFormat format, VkExtent2D extent);
+    ~FirstSampleCopy();
+    FirstSampleCopy(const FirstSampleCopy&) = delete;
+    FirstSampleCopy& operator=(const FirstSampleCopy&) = delete;
+
+    void Record(Recorder& recorder, VkImageView source, VkImageLayout layout);
+    VkImage Image() const { return image; }
+
+private:
+    void release() noexcept;
+
+    Context context;
+    VkExtent2D extent;
+    VkImage image = VK_NULL_HANDLE;
+    VkDeviceMemory memory = VK_NULL_HANDLE;
+    VkImageView view = VK_NULL_HANDLE;
+    VkSampler sampler = VK_NULL_HANDLE;
+    VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE;
+    VkPipelineLayout pipelineLayout = VK_NULL_HANDLE;
+    VkRenderPass renderPass = VK_NULL_HANDLE;
+    VkFramebuffer framebuffer = VK_NULL_HANDLE;
+    VkShaderModule vertex = VK_NULL_HANDLE;
+    VkShaderModule fragment = VK_NULL_HANDLE;
+    VkPipeline pipeline = VK_NULL_HANDLE;
+};
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp

@@ -113,6 +113,7 @@ void SetHostImportDeferredSettles(bool enabled);
 // The larger of 6 GiB and half the physical memory, at most the largest host heap less the larger of 2 GiB and half
 // that heap: the other half stays for the buffers, mirrors and textures that share it.
 std::uint64_t HostImportBudget(const VkPhysicalDeviceMemoryProperties& memory, std::uint64_t physicalBytes);
+void SetHostImportMaxBytes(std::uint64_t bytes);
 std::uint64_t HostImportLiveBytes();
 std::uint64_t HostImportBudgetRemaining(const Context& context);
 struct HostImportSettleCounts {
@@ -248,6 +249,11 @@ struct MirrorStats {
 };
 MirrorStats MirrorCounters();
 std::uint64_t SnapshotMismatches();
+struct HeapMirrorHold {
+    std::uint64_t mirrors = 0;
+    std::uint64_t bytes = 0;
+};
+HeapMirrorHold HeapMirrorsWithin(std::uint64_t begin, std::uint64_t end);
 void ClearSnapshotCopies(VkDevice device);
 std::vector<std::uint64_t> AddressTableKey(std::span<const std::pair<std::uint64_t, std::uint64_t>> writes, std::span<const ShaderRecompiler::BdaAbi::Range> own);
 void ClearImageMirrors(VkDevice device);
@@ -285,6 +291,7 @@ public:
     // calls RecordCopyBacks (a dispatch), since a staged region's results reach guest memory by
     // that copy alone. Call before Upload.
     void AllowDeviceStaging() { stagingAllowed = true; }
+    void DisallowAddressStaging() { addressStaging = false; }
     void AllowAdjustedRegions() { adjustedRegions = true; }
     // Records the copy-in of every staged region anew for another use of this upload (a resource
     // cache hit, from ShaderResources::Revalidate, under GuestMemory::GpuMutex, once the imports
@@ -421,6 +428,7 @@ private:
         // The device refused the shadow (memory or allocations exhausted): the region takes the
         // path it would take without staging, in both upload stages.
         bool unstaged = false;
+        bool overlay = false;
     };
 
     // How [begin, end) lies against the space's base regions.
@@ -462,10 +470,12 @@ private:
     // APS5_GPU_COPY_MAX_KIB allows, and APS5_CPU_COPIES unset.
     bool gpuCopyEligible(const Region& region) const;
     // Whether a region inside a host import is staged in device memory instead of bound in place
-    // (see GuestBufferMemory.cpp): staging allowed, not address-based, a written element inside,
-    // and an atomic element or a size within the written-shadow window. Independent of the import,
-    // so UploadPrepare and UploadFinish decide alike.
+    // (see GuestBufferMemory.cpp): staging allowed, not address-based unless an overlay region, a
+    // written element inside, and an atomic element or a size within the written-shadow window.
+    // Independent of the import, so UploadPrepare and UploadFinish decide alike.
     bool stagingEligible(const Region& region, bool addressable) const;
+    bool overlayCandidate(const Region& inside, bool readOnly) const;
+    void resolveOverlays();
     bool bindableInPlace(std::uint64_t offset, bool addressable) const;
     // Records the import-to-buffer copies of the given gpuCopy regions into the open batch, with
     // the barriers that order them after earlier recorded writes and before the shaders reading them.
@@ -473,6 +483,7 @@ private:
     void takeHeapReferences();
     Context context;
     bool stagingAllowed = false;
+    bool addressStaging = true;
     bool adjustedRegions = false;
     GuestAllocations::Lease lease;
     // The cached address space this build maps through (its lease pins the ranges); `regions` then
@@ -491,11 +502,14 @@ private:
     bool regionsOverlap = false;
     std::vector<std::pair<std::uint64_t, std::uint64_t>> writes;
     std::vector<std::pair<std::uint64_t, std::uint64_t>> readsOverBase;
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> servedInsideBase;
     std::vector<std::pair<std::uint64_t, std::vector<std::byte>>> heapReferences;
+    std::vector<std::pair<std::uint64_t, std::shared_ptr<Buffer>>> heapReadbacks;
     // UploadPrepare ran (regions are frozen); `uploaded` once UploadFinish ran.
     bool prepared = false;
     bool uploaded = false;
     bool committed = false;
+    mutable std::uint64_t directWritesGeneration = 0;
 };
 
 }

core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp

@@ -90,12 +90,14 @@ struct GuestTextureResource {
     std::uint32_t minLod = 0;
     std::uint32_t allocatedMipCount = 0;
     std::uint32_t samples = 1;
+    std::uint64_t fmaskCmaskAddress = 0;
 };
 
 float EffectiveMinLod(const GuestTextureResource& resource);
 
 bool IsNullTextureResource(std::span<const std::uint32_t> words);
 GuestTextureResource DecodeTextureResource(std::span<const std::uint32_t> words);
+std::array<std::uint32_t, 8> EncodeTextureResource(const GuestTextureResource& resource);
 bool DescriptorSingleLevel(std::span<const std::uint32_t> words);
 bool FirstLayerDescriptor(std::span<const std::uint32_t> words, std::array<std::uint32_t, 8>& narrowed);
 bool MatchesGuestDimension(ShaderRecompiler::DescriptorImageShape shape, TextureDimension dimension);