Merge upstream/main into fix/relinker-plt-before-code

Resolve the docs/dev/debt/relinker.md conflict with #5994: keep its
sce_module syscall-scan entry, which now also stops at the end of the
last unwind function when the PLT comes before the code (it calls the
same FindSceCodeEnd), and its /libexec/ld-elf.so.1 start in the code
analysis entry, with this PR's end for a PLT before the code.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

.github/pull_request_template.md

@@ -13,6 +13,7 @@
 - [ ] New third-party code is a submodule built from source
 - [ ] Generic behaviour, not specific to one title
 - [ ] Depends on: <!-- #PR, or none -->
+- [ ] Fixes: <!-- issue it closes as Fixes #N, or none -->
 - [ ] Unblocks: <!-- title ID and Game bring-up discussion, e.g. PPSA01325 #3232, or none -->
 - [ ] GPUs tested: <!-- for AGC, shader recompiler or VideoOut changes: vendor, model, driver and OS of each GPU the GPU tests ran on; otherwise n/a -->
 - [ ] AI-assisted: <!-- yes / no -->

.github/workflows/area-labels.yml

@@ -26,7 +26,7 @@ jobs:
           state=$RUNNER_TEMP/area-state.json
           [ -s "$state" ] || echo '{}' > "$state"
           config=$(gh api "repos/$GITHUB_REPOSITORY/contents/.github/labeler.yml" --jq .sha)
-          gh pr list -R "$GITHUB_REPOSITORY" --base "${{ github.event.repository.default_branch }}" -L 200 --json number,headRefOid \
+          gh pr list -R "$GITHUB_REPOSITORY" --base "${{ github.event.repository.default_branch }}" -L 1000 --json number,headRefOid \
             --jq "map({key: (.number | tostring), value: (.headRefOid + \"-$config\")}) | from_entries" > "$RUNNER_TEMP/area-open.json"
           jq -r --slurpfile old "$state" 'to_entries[] | select($old[0][.key] != .value) | .key' "$RUNNER_TEMP/area-open.json" | head -n 40 > "$RUNNER_TEMP/area-todo"
           jq -c --slurpfile open "$RUNNER_TEMP/area-open.json" --rawfile todo "$RUNNER_TEMP/area-todo" \

.github/workflows/build.yml

@@ -9,7 +9,7 @@ on:
   workflow_dispatch:
     inputs:
       pr:
-        description: Pull request numbers to build together, separated by spaces or commas (up to 8)
+        description: Pull request numbers to build together, separated by spaces or commas (up to 40)
         required: true
         type: string
       linux:
@@ -22,13 +22,18 @@ on:
         required: false
         default: true
         type: boolean
+      select:
+        description: Build and run only the tests whose build inputs the pull requests change (push builds of main always run every test)
+        required: false
+        default: false
+        type: boolean
 
 permissions:
   contents: read
 
 concurrency:
   group: build-${{ github.event_name }}-${{ github.event.pull_request.number || inputs.pr || github.ref }}
-  cancel-in-progress: ${{ github.event_name != 'push' }}
+  cancel-in-progress: true
 
 env:
   CONFIGURE: >-
@@ -65,23 +70,77 @@ jobs:
             const { owner, repo } = context.repo;
             const workflow = '.github/workflows/build.yml';
             const libc = 'core/libs/prx/libc/';
+            const jobOfLine = (text) => {
+              let inJobs = false;
+              let job = '';
+              return text.split('\n').map((line) => {
+                if (/^\S/.test(line)) {
+                  inJobs = /^jobs:\s*$/.test(line);
+                  job = '';
+                } else if (inJobs) {
+                  job = /^  ([\w-]+):\s*$/.exec(line)?.[1] ?? job;
+                }
+                return job;
+              });
+            };
+            const patchedJobs = (patch, text) => {
+              const jobs = jobOfLine(text);
+              const touched = new Set();
+              let line = 1;
+              for (const row of patch.split('\n')) {
+                const hunk = /^@@ -\d+(?:,\d+)? \+(\d+)/.exec(row);
+                if (hunk) {
+                  line = Number(hunk[1]);
+                } else if (row.startsWith('+')) {
+                  touched.add(jobs[line - 1] ?? '');
+                  line += 1;
+                } else if (row.startsWith('-')) {
+                  touched.add(jobs[line - 2] ?? '').add(jobs[line - 1] ?? '');
+                } else if (row.startsWith(' ')) {
+                  line += 1;
+                }
+              }
+              return touched;
+            };
+            const workflowJobs = new Set();
+            const readWorkflow = async (files, ref) => {
+              const file = files.find((file) => file.filename === workflow || file.previous_filename === workflow);
+              if (!file) {
+                return;
+              }
+              if (file.filename !== workflow || !file.patch) {
+                workflowJobs.add('*');
+                return;
+              }
+              try {
+                const { data } = await github.rest.repos.getContent({ owner, repo, path: workflow, ref });
+                const text = Buffer.from(data.content, 'base64').toString('utf8');
+                patchedJobs(file.patch, text).forEach((job) => workflowJobs.add(job));
+              } catch (error) {
+                core.warning(`Could not read ${workflow} at ${ref}, so every test job runs: ${error.message}`);
+                workflowJobs.add('*');
+              }
+            };
             const select = (paths, all) => {
               const touches = (...prefixes) => all ||
                 paths.some((path) => prefixes.some((prefix) => prefix.endsWith('/') ? path.startsWith(prefix) : path === prefix));
+              const edits = (job) => ['*', '', 'eligibility', job].some((name) => workflowJobs.has(name));
               return {
-                relinker: touches(
+                relinker: edits('relinker') || touches(
                   'core/relinker/', libc, 'core/libs/prx/libkernel/AppMetadata/', 'core/libs/tests/ParamJsonParser.cpp',
                   'core/shader/recompiler/tests/CacheFileReadTests.cpp', 'core/shader/recompiler/ShaderCacheDirectory.hpp',
-                  'tools/', 'cmake/', 'CMakeLists.txt', workflow
+                  'tools/', 'cmake/', 'CMakeLists.txt'
                 ),
-                registerContext: touches('core/libs/tests/register_context/', 'core/libs/tests/RegisterContext.cpp', libc, workflow),
-                writeWatch: touches('core/libs/tests/write_watch/', 'core/libs/tests/WriteWatch.cpp', libc, workflow),
+                registerContext: edits('register-context') ||
+                  touches('core/libs/tests/register_context/', 'core/libs/tests/RegisterContext.cpp', libc),
+                writeWatch: edits('write-watch') || touches('core/libs/tests/write_watch/', 'core/libs/tests/WriteWatch.cpp', libc),
               };
             };
             const relinkerHosts = (full, linux) => JSON.stringify([
               ...(full && linux ? [] : [{ os: 'ubuntu-latest', compiler: 'g++' }]),
               ...(full ? [] : [{ os: 'windows-latest', compiler: 'g++' }]),
               { os: 'macos-latest', compiler: 'clang++' },
+              { os: 'macos-latest', compiler: 'clang++', architectures: 'x86_64' },
             ]);
             if (context.eventName === 'push') {
               if (context.ref !== 'refs/heads/main') {
@@ -97,6 +156,7 @@ jobs:
                   const files = data.files ?? [];
                   all = files.length >= 300;
                   paths = files.flatMap((file) => [file.filename, file.previous_filename].filter(Boolean));
+                  await readWorkflow(files, context.sha);
                 } catch (error) {
                   core.warning(`Could not list the files of this push, so every test job runs: ${error.message}`);
                 }
@@ -127,8 +187,8 @@ jobs:
                   throw new Error('Only collaborators with write access can start or rerun builds');
                 }
               }
-              if (!/^[1-9][0-9]*([ ,]+[1-9][0-9]*){0,7}$/.test(process.env.PR_NUMBER.trim())) {
-                throw new Error('Enter up to 8 positive pull request numbers separated by spaces or commas');
+              if (!/^[1-9][0-9]*([ ,]+[1-9][0-9]*){0,39}$/.test(process.env.PR_NUMBER.trim())) {
+                throw new Error('Enter up to 40 positive pull request numbers separated by spaces or commas');
               }
             }
             const numbers = manual
@@ -140,7 +200,6 @@ jobs:
             const pulls = [];
             const paths = [];
             let truncated = false;
-            let relinker = manual;
             for (const pull_number of numbers) {
               let { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number });
               for (let attempt = 0; pr.mergeable === null && attempt < 10; ++attempt) {
@@ -160,19 +219,35 @@ jobs:
               if (pr.mergeable === false) {
                 throw new Error(`Pull request #${pull_number} does not merge cleanly into main; rebase it first`);
               }
-              if (!manual) {
-                const { data: commits } = await github.rest.repos.listCommits({
-                  owner, repo, sha: 'main', author: pr.user.login, per_page: 1
-                });
-                relinker = commits.length > 0;
-              }
               const files = await github.paginate(github.rest.pulls.listFiles, { owner, repo, pull_number, per_page: 100 });
               truncated ||= files.length >= 3000;
               paths.push(...files.flatMap((file) => [file.filename, file.previous_filename].filter(Boolean)));
+              await readWorkflow(files, pr.head.sha);
               pulls.push({ number: pull_number, head: pr.head.sha });
             }
             const run = select(paths, truncated);
-            relinker = relinker && run.relinker;
+            let relinker = run.relinker;
+            if (relinker && !manual) {
+              const author = context.payload.pull_request.user.login;
+              const { data: commits } = await github.rest.repos.listCommits({
+                owner, repo, sha: 'main', author, per_page: 1
+              });
+              relinker = commits.length > 0;
+              if (!relinker) {
+                try {
+                  const { data: mergedPRs } = await github.rest.search.issuesAndPullRequests({
+                    q: `repo:${owner}/${repo} is:pr is:merged base:main author:${author}`,
+                    per_page: 1
+                  });
+                  if (mergedPRs.incomplete_results) {
+                    throw new Error('the search results are incomplete');
+                  }
+                  relinker = mergedPRs.total_count > 0;
+                } catch (error) {
+                  core.warning(`Skipping the relinker: could not search the merged pull requests of ${author}: ${error.message}`);
+                }
+              }
+            }
             const registerContext = run.registerContext;
             const writeWatch = run.writeWatch;
             const { data: main } = await github.rest.repos.getBranch({ owner, repo, branch: 'main' });
@@ -188,7 +263,7 @@ jobs:
             merged.forEach((line) => core.info(line));
             await core.summary.addRaw(
               `${merged.join(', then ')}: full builds ${full ? 'enabled' : 'disabled'}, ` +
-              `relinker ${relinker ? (full ? (linux ? 'enabled on macOS (the full builds run the Linux and Windows relinker tests)' : 'enabled on Linux and macOS (the full Windows build runs the Windows relinker tests)') : 'enabled') : 'skipped: no relinker file changed or the author has no commit in main'}, ` +
+              `relinker ${relinker ? (full ? (linux ? 'enabled on macOS (the full builds run the Linux and Windows relinker tests)' : 'enabled on Linux and macOS (the full Windows build runs the Windows relinker tests)') : 'enabled') : 'skipped: no relinker file changed, or no commit or merged pull request in main was found for the author'}, ` +
               `register context tests ${registerContext ? 'enabled' : 'skipped'}, write watch tests ${writeWatch ? 'enabled' : 'skipped'}`
             ).write();
 
@@ -240,9 +315,40 @@ jobs:
           7z x "$RUNNER_TEMP/winlibs.zip" -oC:/winlibs > /dev/null
       - if: runner.os == 'Windows'
         run: echo 'C:\winlibs\mingw64\bin' >> "$GITHUB_PATH"
-      - run: cmake -S . -B build-relinker -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DANYPS5_RELINKER_ONLY=ON -DBUILD_TESTING=ON
+      - if: ${{ matrix.architectures == 'x86_64' }}
+        run: sudo softwareupdate --install-rosetta --agree-to-license
+      - run: cmake -S . -B build-relinker -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DANYPS5_RELINKER_ONLY=ON -DBUILD_TESTING=ON -DCMAKE_OSX_ARCHITECTURES=${{ matrix.architectures }}
       - run: cmake --build build-relinker --parallel 3
-      - run: ctest --test-dir build-relinker --output-on-failure --timeout 120
+      - run: ctest --test-dir build-relinker --output-on-failure --timeout 120 --parallel 4
+
+  relinker-asan:
+    needs: eligibility
+    if: ${{ needs.eligibility.outputs.relinker == 'true' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ needs.eligibility.outputs.sha }}
+          persist-credentials: false
+      - name: Merge the pull request into main
+        if: ${{ needs.eligibility.outputs.head != '' }}
+        env:
+          PR_NUMBER: ${{ needs.eligibility.outputs.pr }}
+          PR_HEAD: ${{ needs.eligibility.outputs.head }}
+        run: |
+          read -ra numbers <<< "$PR_NUMBER"
+          read -ra heads <<< "$PR_HEAD"
+          refspecs=()
+          for number in "${numbers[@]}"; do refspecs+=("+refs/pull/$number/head:refs/remotes/pull/$number"); done
+          git fetch --no-tags --unshallow origin "${refspecs[@]}"
+          for i in "${!numbers[@]}"; do
+            if [ "$(git rev-parse "refs/remotes/pull/${numbers[i]}")" != "${heads[i]}" ]; then echo "::error::Pull request #${numbers[i]} has newer commits; start a new build"; exit 1; fi
+            if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit "refs/remotes/pull/${numbers[i]}"; then echo "::error::Pull request #${numbers[i]} does not merge cleanly into main and the pull requests before it; rebase it first"; exit 1; fi
+          done
+      - run: cmake -S . -B build-relinker-asan -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=g++ -DANYPS5_RELINKER_ONLY=ON -DANYPS5_ENABLE_ASAN=ON -DBUILD_TESTING=ON
+      - run: cmake --build build-relinker-asan --parallel 4
+      - run: ctest --test-dir build-relinker-asan --output-on-failure --timeout 120 --label-regex '^tls_function_coverage$' --no-tests=error
 
   register-context:
     needs: eligibility
@@ -256,6 +362,25 @@ jobs:
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@v7
+        with:
+          ref: ${{ needs.eligibility.outputs.sha }}
+          persist-credentials: false
+      - name: Merge the pull request into main
+        if: ${{ needs.eligibility.outputs.head != '' }}
+        shell: bash
+        env:
+          PR_NUMBER: ${{ needs.eligibility.outputs.pr }}
+          PR_HEAD: ${{ needs.eligibility.outputs.head }}
+        run: |
+          read -ra numbers <<< "$PR_NUMBER"
+          read -ra heads <<< "$PR_HEAD"
+          refspecs=()
+          for number in "${numbers[@]}"; do refspecs+=("+refs/pull/$number/head:refs/remotes/pull/$number"); done
+          git fetch --no-tags --unshallow origin "${refspecs[@]}"
+          for i in "${!numbers[@]}"; do
+            if [ "$(git rev-parse "refs/remotes/pull/${numbers[i]}")" != "${heads[i]}" ]; then echo "::error::Pull request #${numbers[i]} has newer commits; start a new build"; exit 1; fi
+            if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit "refs/remotes/pull/${numbers[i]}"; then echo "::error::Pull request #${numbers[i]} does not merge cleanly into main and the pull requests before it; rebase it first"; exit 1; fi
+          done
       - run: cmake -S core/libs/tests/register_context -B build/register-context -DCMAKE_BUILD_TYPE=${{ matrix.configuration }}
       - run: cmake --build build/register-context --parallel
       - run: ctest --test-dir build/register-context --output-on-failure --repeat until-fail:3 --no-tests=error
@@ -271,6 +396,25 @@ jobs:
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@v7
+        with:
+          ref: ${{ needs.eligibility.outputs.sha }}
+          persist-credentials: false
+      - name: Merge the pull request into main
+        if: ${{ needs.eligibility.outputs.head != '' }}
+        shell: bash
+        env:
+          PR_NUMBER: ${{ needs.eligibility.outputs.pr }}
+          PR_HEAD: ${{ needs.eligibility.outputs.head }}
+        run: |
+          read -ra numbers <<< "$PR_NUMBER"
+          read -ra heads <<< "$PR_HEAD"
+          refspecs=()
+          for number in "${numbers[@]}"; do refspecs+=("+refs/pull/$number/head:refs/remotes/pull/$number"); done
+          git fetch --no-tags --unshallow origin "${refspecs[@]}"
+          for i in "${!numbers[@]}"; do
+            if [ "$(git rev-parse "refs/remotes/pull/${numbers[i]}")" != "${heads[i]}" ]; then echo "::error::Pull request #${numbers[i]} has newer commits; start a new build"; exit 1; fi
+            if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit "refs/remotes/pull/${numbers[i]}"; then echo "::error::Pull request #${numbers[i]} does not merge cleanly into main and the pull requests before it; rebase it first"; exit 1; fi
+          done
       - run: cmake -S core/libs/tests/write_watch -B build/write-watch -DCMAKE_BUILD_TYPE=Release
       - run: cmake --build build/write-watch --parallel
       - run: ctest --test-dir build/write-watch --output-on-failure --repeat until-fail:3 --no-tests=error
@@ -286,8 +430,6 @@ jobs:
       CCACHE_COMPILERCHECK: content
       MESA_SHADER_CACHE_DIR: /var/cache/anyps5/mesa
       MESA_SHADER_CACHE_MAX_SIZE: 4G
-      CMAKE_BUILD_PARALLEL_LEVEL: 1
-      CTEST_PARALLEL_LEVEL: 1
     steps:
       - uses: actions/checkout@v7
         with:
@@ -321,16 +463,87 @@ jobs:
           ccache --show-config
           dpkg-query -W mesa-vulkan-drivers
           grep -m1 '^model name' /proc/cpuinfo
-      - run: cmake -S . -B build $CONFIGURE -DANYPS5_ENABLE_SPIRV_TOOLS=ON
+      - name: Install the wild linker
+        shell: bash
+        env:
+          WILD_VERSION: 0.10.0
+          WILD_SHA256: 641265506a7c06cfb03181b8916ab663ec8407855db6d4db7f8450667d105283
+        run: |
+          tools=$RUNNER_TOOL_CACHE/wild
+          mkdir -p "$tools" 2>/dev/null && test -w "$tools" || { tools=$RUNNER_TEMP/wild; mkdir -p "$tools"; }
+          dir=$tools/$WILD_VERSION
+          tmp=
+          probe=$(mktemp -d "$RUNNER_TEMP/wild-probe.XXXXXX")
+          trap 'rm -rf "$tmp" "$probe"' EXIT
+          if [ ! -x "$dir/bin/ld" ]; then
+            tmp=$(mktemp -d "$tools/.wild.XXXXXX")
+            name=wild-linker-$WILD_VERSION-x86_64-unknown-linux-gnu
+            curl -fsSL --retry 3 -o "$tmp/$name.tar.gz" "https://github.com/davidlattimore/wild/releases/download/$WILD_VERSION/$name.tar.gz"
+            echo "$WILD_SHA256  $tmp/$name.tar.gz" | sha256sum -c -
+            tar -xzf "$tmp/$name.tar.gz" -C "$tmp"
+            mkdir -p "$tmp/out/bin"
+            mv "$tmp/$name/wild" "$tmp/out/bin/wild"
+            ln -s wild "$tmp/out/bin/ld"
+            mv -T "$tmp/out" "$dir" 2>/dev/null || test -x "$dir/bin/ld"
+          fi
+          echo 'int main(void) { return 0; }' > "$probe/main.c"
+          gcc "-B$dir/bin" "$probe/main.c" -o "$probe/main"
+          readelf -p .comment "$probe/main" | grep -F "Linker: Wild $WILD_VERSION"
+          echo "LINKER_FLAGS=-B$dir/bin" >> "$GITHUB_ENV"
+      - name: Size parallelism to what the shared host has free
+        run: |
+          cpus=$(nproc)
+          busy=$(awk '{ printf "%d", $1 + 0.5 }' /proc/loadavg)
+          mem_mb=$(awk '/^MemTotal:/ { print int($2 / 1024) }' /proc/meminfo)
+          slots=$(( (mem_mb - 2048) / 1024 ))
+          slots=$(( slots < 1 ? 1 : slots ))
+          tests=$(( cpus - busy ))
+          tests=$(( tests < 1 ? 1 : tests ))
+          slot_dir=$CCACHE_DIR/compile-slots
+          mkdir -p -m 1777 "$slot_dir"
+          test -w "$slot_dir"
+          echo "cpus=$cpus load=$busy mem_total_mb=$mem_mb -> build jobs $cpus, cold compiles $slots per host, tests $tests"
+          echo "CMAKE_BUILD_PARALLEL_LEVEL=$cpus" >> "$GITHUB_ENV"
+          echo "CTEST_PARALLEL_LEVEL=$tests" >> "$GITHUB_ENV"
+          echo "CCACHE_PREFIX=$GITHUB_WORKSPACE/tools/compile_slot.sh" >> "$GITHUB_ENV"
+          echo "ANYPS5_COMPILE_SLOT_DIR=$slot_dir" >> "$GITHUB_ENV"
+          echo "ANYPS5_COMPILE_SLOTS=$slots" >> "$GITHUB_ENV"
+      - run: cmake -S . -B build $CONFIGURE -DANYPS5_ENABLE_SPIRV_TOOLS=ON "-DCMAKE_EXE_LINKER_FLAGS=$LINKER_FLAGS" "-DCMAKE_SHARED_LINKER_FLAGS=$LINKER_FLAGS" "-DCMAKE_MODULE_LINKER_FLAGS=$LINKER_FLAGS"
       - name: Check that every test executable is registered with ctest
         run: python3 tools/check_test_registration.py build
-      - run: cmake --build build
-      - run: cmake --build build --target libs
-      - run: ctest --test-dir build --output-on-failure --timeout 120 --output-log ctest.log --output-junit ctest.xml
+      - if: ${{ inputs.select }}
+        uses: actions/cache/restore@v6
+        with:
+          path: test-graph.json
+          key: test-graph-${{ runner.os }}-${{ needs.eligibility.outputs.sha }}
+          restore-keys: test-graph-${{ runner.os }}-
+      - name: Select the tests the change reaches
+        if: ${{ inputs.select }}
+        env:
+          BASE: ${{ needs.eligibility.outputs.sha }}
+        run: |
+          git diff --name-only "$BASE" HEAD > changed.txt
+          python3 tools/select_tests.py select build test-graph.json changed.txt
+      - id: build
+        run: |
+          if [ -f build/select.ninja ]; then ninja -C build -f select.ninja -j "$CMAKE_BUILD_PARALLEL_LEVEL" -l "$(nproc)" selected; else cmake --build build -- -l "$(nproc)"; fi
+      - id: libs
+        run: cmake --build build --target libs -- -l "$(nproc)"
+      - run: python3 tools/select_tests.py ctest build --test-load "$(nproc)" --output-on-failure --timeout 120 --output-log ctest.log --output-junit ctest.xml
         env:
           ANYPS5_REQUIRE_VULKAN: 1
           GALLIUM_OVERRIDE_CPU_CAPS: avx
       - run: python3 tools/check_skips.py build/ctest.xml
+      - name: Write the test graph of main
+        if: ${{ !cancelled() && github.event_name == 'push' && steps.build.outcome == 'success' && steps.libs.outcome == 'success' }}
+        continue-on-error: true
+        run: python3 tools/select_tests.py dump build test-graph.json
+      - if: ${{ !cancelled() && github.event_name == 'push' && hashFiles('test-graph.json') != '' }}
+        continue-on-error: true
+        uses: actions/cache/save@v6
+        with:
+          path: test-graph.json
+          key: test-graph-${{ runner.os }}-${{ github.sha }}
       - name: Report compiler cache statistics
         if: ${{ !cancelled() }}
         run: ccache --show-stats
@@ -350,7 +563,6 @@ jobs:
     env:
       WINLIBS_RELEASE: 15.2.0posix-14.0.0-ucrt-r7
       WINLIBS_ZIP: winlibs-x86_64-posix-seh-gcc-15.2.0-mingw-w64ucrt-14.0.0-r7.zip
-      SSL_CERT_FILE: C:\Program Files\Git\mingw64\etc\ssl\certs\ca-bundle.crt
     defaults:
       run:
         shell: bash
@@ -359,7 +571,6 @@ jobs:
         with:
           ref: ${{ needs.eligibility.outputs.sha }}
           persist-credentials: false
-          submodules: recursive
       - name: Merge the pull request into main
         if: ${{ needs.eligibility.outputs.head != '' }}
         shell: bash
@@ -376,7 +587,35 @@ jobs:
             if [ "$(git rev-parse "refs/remotes/pull/${numbers[i]}")" != "${heads[i]}" ]; then echo "::error::Pull request #${numbers[i]} has newer commits; start a new build"; exit 1; fi
             if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit "refs/remotes/pull/${numbers[i]}"; then echo "::error::Pull request #${numbers[i]} does not merge cleanly into main and the pull requests before it; rebase it first"; exit 1; fi
           done
-          git submodule update --init --recursive
+      - id: submodules
+        name: Key the submodules on their commits
+        run: |
+          echo "key=$(git ls-files -s | awk '$1 == "160000" { print $2, $4 }' | cat - .gitmodules | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
+          echo "ffmpeg=$(git rev-parse HEAD:3rdparty/ffmpeg-core)" >> "$GITHUB_OUTPUT"
+      - id: submodule-cache
+        uses: actions/cache/restore@v6
+        with:
+          path: ${{ runner.temp }}/submodule-gitdirs.tar
+          key: submodule-gitdirs-${{ runner.os }}-${{ steps.submodules.outputs.key }}
+      - name: Check out the submodules
+        run: |
+          if [ -f "$RUNNER_TEMP/submodule-gitdirs.tar" ]; then tar --force-local -xf "$RUNNER_TEMP/submodule-gitdirs.tar"; fi
+          git submodule update --init --recursive --depth 1 --jobs 8
+      - if: steps.submodule-cache.outputs.cache-hit != 'true'
+        run: tar --force-local -cf "$RUNNER_TEMP/submodule-gitdirs.tar" .git/modules
+      - if: steps.submodule-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v6
+        continue-on-error: true
+        with:
+          path: ${{ runner.temp }}/submodule-gitdirs.tar
+          key: submodule-gitdirs-${{ runner.os }}-${{ steps.submodules.outputs.key }}
+      - name: Point the WinLibs CMake at the CA bundle of Git for Windows
+        run: |
+          for prefix in ucrt64 mingw64; do
+            bundle="C:/Program Files/Git/$prefix/etc/ssl/certs/ca-bundle.crt"
+            if [ -f "$bundle" ]; then echo "SSL_CERT_FILE=$bundle" >> "$GITHUB_ENV"; exit 0; fi
+          done
+          echo "::error::No CA bundle found in Git for Windows"; exit 1
       - id: toolchain
         uses: actions/cache@v6
         with:
@@ -389,21 +628,74 @@ jobs:
       - run: echo 'C:\winlibs\mingw64\bin' >> "$GITHUB_PATH"
       - uses: hendrikmuhs/[email protected]
         with:
-          key: windows
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+          key: ${{ github.event_name == 'workflow_dispatch' && 'dispatch-windows' || 'windows' }}
+          restore-keys: ${{ github.event_name == 'workflow_dispatch' && 'windows' || '' }}
+          save: ${{ github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }}
           evict-old-files: job
+      - id: ffmpeg-cache
+        uses: actions/cache/restore@v6
+        with:
+          path: build/externals
+          key: ffmpeg-${{ runner.os }}-${{ steps.submodules.outputs.ffmpeg }}
+      - id: seed-cache
+        uses: actions/cache/restore@v6
+        with:
+          path: configure-seed.cmake
+          key: configure-seed-${{ env.WINLIBS_ZIP }}-${{ steps.submodules.outputs.key }}-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'core/libs/CMakeLists.txt', '.github/workflows/build.yml') }}
       - shell: cmd
-        run: cmake -S . -B build %CONFIGURE% "-DANYPS5_AGC_EXECUTION_TESTS=F16ToF32Unpack;F64FmaSubnormals;RegistrationPrepareShutdown"
+        run: if exist configure-seed.cmake (cmake -C configure-seed.cmake -S . -B build %CONFIGURE% "-DANYPS5_AGC_EXECUTION_TESTS=F16ToF32Unpack;F64FmaSubnormals;RegistrationPrepareShutdown") else (cmake -S . -B build %CONFIGURE% "-DANYPS5_AGC_EXECUTION_TESTS=F16ToF32Unpack;F64FmaSubnormals;RegistrationPrepareShutdown")
+      - if: steps.seed-cache.outputs.cache-hit != 'true'
+        shell: cmd
+        run: python tools/configure_seed.py build/CMakeCache.txt configure-seed.cmake
+      - if: steps.seed-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v6
+        continue-on-error: true
+        with:
+          path: configure-seed.cmake
+          key: ${{ steps.seed-cache.outputs.cache-primary-key }}
+      - if: steps.ffmpeg-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v6
+        continue-on-error: true
+        with:
+          path: build/externals
+          key: ${{ steps.ffmpeg-cache.outputs.cache-primary-key }}
       - name: Check that every test executable is registered with ctest
         shell: cmd
         run: python tools/check_test_registration.py build
-      - shell: cmd
-        run: cmake --build build
-      - shell: cmd
+      - if: ${{ inputs.select }}
+        uses: actions/cache/restore@v6
+        with:
+          path: test-graph.json
+          key: test-graph-${{ runner.os }}-${{ needs.eligibility.outputs.sha }}
+          restore-keys: test-graph-${{ runner.os }}-
+      - name: Select the tests the change reaches
+        if: ${{ inputs.select }}
+        env:
+          BASE: ${{ needs.eligibility.outputs.sha }}
+        shell: cmd
+        run: |
+          git diff --name-only %BASE% HEAD > changed.txt
+          python tools/select_tests.py select build test-graph.json changed.txt
+      - id: build
+        shell: cmd
+        run: if exist build\select.ninja (ninja -C build -f select.ninja selected) else (cmake --build build)
+      - id: libs
+        shell: cmd
         run: cmake --build build --target libs
       - shell: cmd
-        run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit ctest.xml
+        run: python tools/select_tests.py ctest build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit ctest.xml
       - run: python tools/check_skips.py build/ctest.xml --job windows
+      - name: Write the test graph of main
+        if: ${{ !cancelled() && github.event_name == 'push' && steps.build.outcome == 'success' && steps.libs.outcome == 'success' }}
+        continue-on-error: true
+        shell: cmd
+        run: python tools/select_tests.py dump build test-graph.json
+      - if: ${{ !cancelled() && github.event_name == 'push' && hashFiles('test-graph.json') != '' }}
+        continue-on-error: true
+        uses: actions/cache/save@v6
+        with:
+          path: test-graph.json
+          key: test-graph-${{ runner.os }}-${{ github.sha }}
       - name: Report tests slower than 30 s
         if: ${{ !cancelled() }}
         shell: bash
@@ -411,3 +703,4 @@ jobs:
           [ -f ctest.log ] || exit 0
           awk '/Test +#[0-9]+: / && $NF == "sec" { for (i = 1; i <= NF; i++) if ($i ~ /^#[0-9]+:$/) name = $(i + 1); t = $(NF - 1) + 0; if (t > max[name]) max[name] = t }
             END { for (name in max) if (max[name] > 30) printf "::warning title=Slow test::%s took %.1f s on %s; tests must finish in 30 s (CONTRIBUTING.md)\n", name, max[name], ENVIRON["RUNNER_OS"] }' ctest.log
+

.github/workflows/fix-label.yml

@@ -0,0 +1,44 @@
+name: Fix label
+
+on:
+  schedule:
+    - cron: '45 * * * *'
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: fix-label
+  cancel-in-progress: true
+
+jobs:
+  label:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: read
+      pull-requests: write
+    env:
+      GH_TOKEN: ${{ github.token }}
+      LABEL: fix error
+    steps:
+      - run: |
+          gh api graphql --paginate -f owner="${GITHUB_REPOSITORY%/*}" -f name="${GITHUB_REPOSITORY#*/}" -f query='
+            query($owner: String!, $name: String!, $endCursor: String) {
+              repository(owner: $owner, name: $name) {
+                pullRequests(states: OPEN, first: 100, after: $endCursor) {
+                  pageInfo { hasNextPage endCursor }
+                  nodes { number closingIssuesReferences { totalCount } labels(first: 100) { nodes { name } } }
+                }
+              }
+            }' --jq '.data.repository.pullRequests.nodes[]
+              | (.closingIssuesReferences.totalCount > 0) as $fix
+              | ([.labels.nodes[].name] | index(env.LABEL) != null) as $has
+              | if $fix and ($has | not) then "add \(.number)" elif $has and ($fix | not) then "remove \(.number)" else empty end' > "$RUNNER_TEMP/fix-todo"
+          label=$(jq -rn 'env.LABEL | @uri')
+          while read -r action number; do
+            if [ "$action" = add ]; then
+              gh api "repos/$GITHUB_REPOSITORY/issues/$number/labels" -f "labels[]=$LABEL" --silent
+            else
+              gh api -X DELETE "repos/$GITHUB_REPOSITORY/issues/$number/labels/$label" --silent
+            fi
+          done < "$RUNNER_TEMP/fix-todo"

.github/workflows/labels.yml

@@ -35,4 +35,5 @@ jobs:
           label prx       c5def5 'Other PRX libraries'
           label build     bfd4f2 'CMake, submodules, 3rdparty'
           label docs      0075ca 'Documentation'
+          label 'fix error' d73a4a 'Closes an issue (Fixes #N)'
           label stale     ededed 'Conflicting with main and no activity for 7 days'

.github/workflows/pages-progress.yml

@@ -0,0 +1,40 @@
+name: Pages progress
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - core/libs/prx/**
+      - core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp
+
+permissions:
+  contents: read
+  actions: write
+  pages: read
+
+concurrency:
+  group: pages-progress
+  cancel-in-progress: true
+
+jobs:
+  progress:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: Deploy Pages if the progress changed
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+        run: |
+          python3 tools/progress.py "$RUNNER_TEMP/progress"
+          site=$(gh api "repos/$GH_REPO/pages" --jq .html_url)
+          if curl -fsSL "${site%/}/progress.json?$GITHUB_SHA" -o "$RUNNER_TEMP/deployed.json" &&
+            cmp -s "$RUNNER_TEMP/deployed.json" "$RUNNER_TEMP/progress/progress.json"; then
+            echo "Progress unchanged, Pages not deployed" >> "$GITHUB_STEP_SUMMARY"
+          else
+            gh workflow run pages.yml --ref main
+            echo "Progress changed, Pages deployment dispatched" >> "$GITHUB_STEP_SUMMARY"
+          fi

.github/workflows/pages.yml

@@ -3,6 +3,17 @@ name: Pages
 on:
   push:
     branches: [main]
+    paths:
+      - .github/workflows/pages.yml
+      - CMakeLists.txt
+      - core/relinker/**
+      - demo/web/**
+      - tools/build_web.py
+      - tools/nid_names.py
+      - tools/progress.py
+      - tools/progress_history.py
+      - tools/rdna_isa.txt
+      - tools/test_web.py
   workflow_dispatch:
   release:
     types: [published]

.github/workflows/pr-intake.yml

@@ -27,6 +27,27 @@ jobs:
             if (pr.state !== 'open' || pr.base.ref !== 'main') {
               return;
             }
+            if (context.payload.action === 'reopened') {
+              let maintainer = false;
+              try {
+                const { data: access } = await github.rest.repos.getCollaboratorPermissionLevel({
+                  owner, repo, username: context.payload.sender.login
+                });
+                maintainer = ['admin', 'maintain', 'write'].includes(access.role_name ?? access.permission);
+              } catch (error) {
+                core.warning(`Could not check the permission of ${context.payload.sender.login}, so the open PR limit applies: ${error.message}`);
+              }
+              if (maintainer) {
+                try {
+                  await github.rest.issues.removeLabel({ owner, repo, issue_number: pull_number, name: 'open PR limit' });
+                } catch (error) {
+                  if (error.status !== 404) {
+                    core.warning(`Could not remove the open PR limit label: ${error.message}`);
+                  }
+                }
+                return;
+              }
+            }
             const { data: mergedPRs } = await github.rest.search.issuesAndPullRequests({
               q: `repo:${owner}/${repo} is:pr is:merged base:main author:${pr.user.login}`,
               per_page: 1
@@ -42,7 +63,10 @@ jobs:
             const openPRs = await github.paginate(github.rest.pulls.list, {
               owner, repo, state: 'open', base: 'main', per_page: 100
             });
-            const open = new Set(openPRs.filter((p) => p.user.login === pr.user.login).map((p) => p.number));
+            const reopened = context.payload.action === 'reopened';
+            const open = new Set(openPRs
+              .filter((p) => p.user.login === pr.user.login && (reopened || p.number < pull_number))
+              .map((p) => p.number));
             open.add(pull_number);
             if (open.size <= allowedOpen) {
               return;

.github/workflows/release.yml

@@ -42,6 +42,15 @@ jobs:
       - uses: actions/checkout@v7
         with:
           submodules: recursive
+      - name: Point the WinLibs CMake at the CA bundle of Git for Windows
+        if: matrix.platform == 'windows'
+        shell: bash
+        run: |
+          for prefix in ucrt64 mingw64; do
+            bundle="C:/Program Files/Git/$prefix/etc/ssl/certs/ca-bundle.crt"
+            if [ -f "$bundle" ]; then echo "SSL_CERT_FILE=$bundle" >> "$GITHUB_ENV"; exit 0; fi
+          done
+          echo "::error::No CA bundle found in Git for Windows"; exit 1
       - id: toolchain
         if: matrix.platform == 'windows'
         uses: actions/cache@v6
@@ -49,8 +58,6 @@ jobs:
           path: C:\winlibs
           key: ${{ env.WINLIBS_ZIP }}
       - if: matrix.platform == 'windows' && steps.toolchain.outputs.cache-hit != 'true'
-        env:
-          SSL_CERT_FILE: 'C:\Program Files\Git\mingw64\etc\ssl\certs\ca-bundle.crt'
         run: |
           curl -sSfL -o "$RUNNER_TEMP/winlibs.zip" "https://github.com/brechtsanders/winlibs_mingw/releases/download/$WINLIBS_RELEASE/$WINLIBS_ZIP"
           7z x "$RUNNER_TEMP/winlibs.zip" -oC:/winlibs
@@ -69,8 +76,6 @@ jobs:
       - name: Configure Windows
         if: matrix.platform == 'windows'
         shell: cmd
-        env:
-          SSL_CERT_FILE: 'C:\Program Files\Git\mingw64\etc\ssl\certs\ca-bundle.crt'
         run: cmake -S . -B build %CONFIGURE%
       - name: Build project and tests (Linux)
         if: matrix.platform == 'linux'

.github/workflows/stale-conflicts.yml

@@ -14,6 +14,14 @@ jobs:
       issues: write
       pull-requests: write
     steps:
+      - env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh pr list -R "$GITHUB_REPOSITORY" --label stale -L 1000 --json number,labels \
+            --jq '.[] | select(all(.labels[]; .name != "needs rebase")) | .number' |
+            while read -r number; do
+              gh api -X DELETE "repos/$GITHUB_REPOSITORY/issues/$number/labels/stale" --silent
+            done
       - uses: actions/stale@v11
         with:
           only-pr-labels: needs rebase

.github/workflows/web-tests.yml

@@ -22,6 +22,12 @@ jobs:
       - uses: actions/checkout@v7
         with:
           persist-credentials: false
+      - name: Test archive root selection
+        shell: bash
+        run: |
+          zip_url=$(python3 -c 'from pathlib import Path; import re; print(re.search(r"from \"(https://cdn.jsdelivr.net/[^\"]+)\"", Path("demo/web/archive.mjs").read_text(encoding="utf-8")).group(1))')
+          curl -fsSL "$zip_url" -o "$RUNNER_TEMP/zip.mjs"
+          ANYPS5_TEST_ZIP_URL="$zip_url" ANYPS5_TEST_ZIP_PATH="$(python3 -c 'from pathlib import Path; import sys; print(Path(sys.argv[1]).resolve().as_uri())' "$RUNNER_TEMP/zip.mjs")" node demo/web/tests/archive-root.test.mjs demo/web
       - name: Check browser module syntax
         run: |
           node --check demo/web/app.mjs

.gitmodules

@@ -40,3 +40,9 @@
 [submodule "3rdparty/mbedtls"]
 	path = 3rdparty/mbedtls
 	url = https://github.com/Mbed-TLS/mbedtls.git
+[submodule "3rdparty/zstd"]
+	path = 3rdparty/zstd
+	url = https://github.com/facebook/zstd.git
+[submodule "3rdparty/ZArchive"]
+	path = 3rdparty/ZArchive
+	url = https://github.com/Exzap/ZArchive.git

AGENTS.md

@@ -0,0 +1,14 @@
+# AGENTS.md
+
+AnyPS5 relinks PS5 executables into native Linux or Windows programs that run against reimplemented system libraries; it is not an emulator. The rules are [CONTRIBUTING.md](CONTRIBUTING.md) and [CONVENTIONS](docs/dev/CONVENTIONS.md); these are the ones agents break.
+
+- No comments in code, except `#endif` and namespace ends.
+- An export you cannot implement calls `NotImplemented_nid_no_patch(__func__)`; never return a made-up success.
+- Implement the function for every input. Never special-case a value, ID or path seen in one title's trace; if the general behaviour is unknown, throw and record it in `docs/dev/debt/<library>.md`.
+- New third-party code is a submodule at `3rdparty/<name>`: add its `.gitmodules` entry (`git submodule add <url> 3rdparty/<name>`) and build it from source with CMake. Never `find_package` or copy sources in.
+- Known gaps go in `docs/dev/debt/<library>.md`; notes, reports and logs go in the pull request. Add no other files.
+- Commit subjects and PR titles are Conventional Commits.
+- PR body: copy [the template](.github/pull_request_template.md), keep every line, tick every box, and fill `What`, `Tested`, `Depends on` (#N or none), `Unblocks` (title ID and discussion, or none), `GPUs tested` (AGC, shader recompiler, VideoOut; otherwise n/a) and `AI-assisted` (yes or no).
+- Before pushing run `python3 tools/check_conventions.py --base origin/main`; before opening run `python3 tools/pr_overlap.py --branch <branch>`.
+- Build and test: [BUILD](docs/dev/BUILD.md); a relinker-only change builds with `-DANYPS5_RELINKER_ONLY=ON`.
+- A speed claim shows `tools/trace_digest.py` summaries of `main` and of the change from the same machine and scene.

CMakeLists.txt

@@ -6,10 +6,21 @@ endif()
 project(Relinker CXX)
 
 set(ANYPS5_TEST_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests")
+set(ANYPS5_WINDOWS_TEST_EXIT "${CMAKE_CURRENT_SOURCE_DIR}/cmake/WindowsTestExit.cpp")
 
 function(add_test_executable target)
     add_executable(${target} ${ARGN})
     set_target_properties(${target} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$<1:${ANYPS5_TEST_OUTPUT_DIRECTORY}>")
+    if(MINGW)
+        target_sources(${target} PRIVATE "${ANYPS5_WINDOWS_TEST_EXIT}")
+        target_link_options(${target} PRIVATE LINKER:--wrap=main)
+    endif()
+endfunction()
+
+function(add_asan_test_options test options)
+    if(ANYPS5_ENABLE_ASAN)
+        set_property(TEST ${test} APPEND PROPERTY ENVIRONMENT_MODIFICATION "ASAN_OPTIONS=string_append::${options}")
+    endif()
 endfunction()
 
 option(BUILD_TESTING "Build and register relinker tests" OFF)
@@ -86,6 +97,17 @@ if(BUILD_TESTING)
     add_test(NAME request_serializer COMMAND request_serializer_tests)
     set_tests_properties(request_serializer PROPERTIES TIMEOUT 20)
     find_package(Python3 QUIET COMPONENTS Interpreter)
+    add_test_executable(shader_ir_use_rewrite_tests
+        core/shader/recompiler/IntermediateRepresentation/tests/UseRewrite.cpp
+        core/shader/recompiler/IntermediateRepresentation/tests/UseRewriteAllocation.cpp
+        core/shader/recompiler/IntermediateRepresentation/src/IrValue.cpp
+        core/shader/recompiler/IntermediateRepresentation/src/IrBlock.cpp
+        core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp
+        core/shader/recompiler/IntermediateRepresentation/src/GuestRegister.cpp)
+    target_include_directories(shader_ir_use_rewrite_tests PRIVATE
+        core/shader/recompiler/IntermediateRepresentation/include)
+    add_test(NAME shader_ir_use_rewrite COMMAND shader_ir_use_rewrite_tests)
+    set_tests_properties(shader_ir_use_rewrite PROPERTIES TIMEOUT 20)
     if(Python3_Interpreter_FOUND)
         add_test(NAME shader_cache_native_paths COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/tests/test_shader_cache_native_paths.py
@@ -113,8 +135,16 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_import_audit.py)
         add_test(NAME unwrap_self COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_unwrap_self.py)
-        add_test(NAME conventions_function_bodies COMMAND ${Python3_EXECUTABLE}
-            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_comments.py)
+        foreach(shard RANGE 0 11)
+            if(shard EQUAL 0)
+                set(conventions_test_name conventions_function_bodies)
+            else()
+                set(conventions_test_name conventions_function_bodies_shard_${shard})
+            endif()
+            add_test(NAME ${conventions_test_name} COMMAND ${Python3_EXECUTABLE}
+                ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_comments.py --shard ${shard}/12)
+            set_tests_properties(${conventions_test_name} PROPERTIES TIMEOUT 30)
+        endforeach()
         add_test(NAME conventions_wrapped_declarations COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_wrapped_declarations.py)
         add_test(NAME conventions_ci_gate COMMAND ${Python3_EXECUTABLE}
@@ -123,10 +153,16 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_doc_links.py)
         add_test(NAME test_registration COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_test_registration.py)
+        add_test(NAME build_eligibility COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_build_eligibility.py)
         add_test(NAME check_skips COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_skips.py)
+        add_test(NAME select_tests COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_select_tests.py)
         add_test(NAME check_main_sync COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_main_sync.py)
+        add_test(NAME configure_seed COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_configure_seed.py)
         add_test(NAME stubgen COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_stubgen.py)
         add_test(NAME debt_split COMMAND ${Python3_EXECUTABLE}
@@ -159,6 +195,12 @@ if(BUILD_TESTING)
         add_test(NAME conventions_bom_shebang COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_bom_shebang.py)
         set_tests_properties(conventions_bom_shebang PROPERTIES TIMEOUT 20)
+        add_test(NAME env_reference COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_env_reference.py)
+        set_tests_properties(env_reference PROPERTIES TIMEOUT 20)
+        add_test(NAME conventions_env_reference COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_env_reference.py)
+        set_tests_properties(conventions_env_reference PROPERTIES TIMEOUT 60)
         add_test(NAME conventions_body_bom COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_body_bom.py)
         add_test(NAME conventions_blob_reads COMMAND ${Python3_EXECUTABLE}
@@ -257,6 +299,8 @@ set(LIBUSB_ENABLE_UDEV OFF CACHE BOOL "" FORCE)
 add_subdirectory(3rdparty/libusb-cmake EXCLUDE_FROM_ALL)
 set_target_properties(usb-1.0 PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
+include(cmake/ZArchive.cmake)
+
 add_subdirectory(core/shader/recompiler)
 add_subdirectory(core/Decoder/Jpeg)
 add_subdirectory(core/Decoder/Png)

CONTRIBUTING.md

@@ -37,7 +37,7 @@ For pull requests, full Linux and Windows builds and tests run only when a colla
 gh workflow run build.yml --repo boykopovar/AnyPS5 --ref main --field pr=1234
 ```
 
-This manual run also builds and tests the relinker on Linux, Windows and macOS, regardless of the author's previous contributions or the changed files. The relinker matrix also runs automatically when a pull request targeting `main` is opened, updated or reopened, if its author already has a commit in `main` and it changes a file the relinker-only build uses; full builds never run automatically for pull requests. Every job that runs automatically for pull requests runs only when the pull request changes its files: a `paths` filter on the `pull_request` trigger, or `needs: eligibility` with an `if` on the files output it sets in `build.yml`. The Conventions check fails a workflow change that adds a job without one. All build jobs build the selected pull request merged into the current `main` (GitHub's merge commit), so a pull request that no longer merges cleanly must be rebased first. New commits require another manual run for full builds.
+This manual run also builds and tests the relinker on Linux, Windows and macOS when the pull request changes a file the relinker-only build uses, regardless of the author's previous contributions. The relinker matrix also runs automatically when a pull request targeting `main` is opened, updated or reopened, if its author already has a commit in `main` and it changes a file the relinker-only build uses; full builds never run automatically for pull requests. Every job that runs automatically for pull requests runs only when the pull request changes its files: a `paths` filter on the `pull_request` trigger, or `needs: eligibility` with an `if` on the files output it sets in `build.yml`. The Conventions check fails a workflow change that adds a job without one. All build jobs build the selected pull request merged into the current `main` (GitHub's merge commit), so a pull request that no longer merges cleanly must be rebased first. New commits require another manual run for full builds.
 
 With `--field full=false`, the manual run skips the full builds and runs only the relinker matrix on Linux, Windows and macOS, if the pull request changes a file the relinker-only build uses, such as the Python tools and their tests in `tools/`.
 
@@ -57,7 +57,9 @@ The [pull request template](.github/pull_request_template.md) is the checklist f
 - Keep the branch up to date with `main` and resolve conflicts yourself; rebasing and force-pushing is fine.
 - A pull request that conflicts with `main` and has no activity for 7 days is labeled `stale` with a reminder. It is not closed.
 - If a pull request needs another one first, say so on the checklist line (`Depends on: #N`; `none` otherwise).
+- If a pull request fixes a reported issue, write `Fixes #N` on its checklist line (`none` otherwise). GitHub closes the issue on merge, and the `fix error` label is added within the hour.
 - If a pull request fixes a stop or a wrong result seen on a title, name it on the checklist line with the title ID and its [Game bring-up](https://github.com/boykopovar/AnyPS5/discussions/categories/game-bring-up) discussion (`Unblocks: PPSA01325 #3232`; `none` otherwise). Reviewers review these first.
+- For GPU changes, list each tested GPU's vendor, model, driver and OS on the checklist line. CI runs GPU tests only on lavapipe, so reviewers may ask for a run on another vendor before merging.
 - Fixes a title needs go to `main` as pull requests. A Game bring-up discussion lists the open pull requests and local changes the title still needs; update the list when one is merged, closed or replaced.
 - The PR overlap comment lists open pull requests that change the same lines or exports; agree with their authors which one goes first and declare it with `Depends on`.
 - Run the tests before opening the pull request and describe what was tested (OS, title or homebrew).

README.md

@@ -2,6 +2,8 @@
 
 Tool for automatic executables porting to Linux and Windows.
 
+The relinker-only build runs on macOS, including Apple silicon.
+
 [**Web converter**](https://boykopovar.github.io/AnyPS5/) - files stay on your device.
 
 ![anyps5_collage1](https://github.com/user-attachments/assets/ed2f0b2d-33d3-46d7-ab88-767ca872fc99)

cmake/WindowsTestExit.cpp

@@ -0,0 +1,31 @@
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+#include <process.h>
+#include <windows.h>
+
+extern "C" int __real_main(int argc, char** argv);
+
+namespace {
+
+template<typename TFunction>
+TFunction LibcFunction(const char* name) {
+    const HMODULE libc = GetModuleHandleW(L"libc.prx");
+    if (libc == nullptr) return nullptr;
+    return reinterpret_cast<TFunction>(reinterpret_cast<void*>(GetProcAddress(libc, name)));
+}
+
+}
+
+extern "C" int __wrap_main(int argc, char** argv) {
+    const int code = __real_main(argc, argv);
+    const auto terminate = LibcFunction<void (*)(int)>("LibcTerminate_nid_no_patch");
+    if (terminate == nullptr) return code;
+    _cexit();
+    std::cout.flush();
+    std::clog.flush();
+    std::cerr.flush();
+    std::fflush(nullptr);
+    terminate(code);
+    return code;
+}

cmake/ZArchive.cmake

@@ -0,0 +1,25 @@
+enable_language(C)
+
+set(ZSTD_BUILD_PROGRAMS OFF CACHE BOOL "" FORCE)
+set(ZSTD_BUILD_TESTS OFF CACHE BOOL "" FORCE)
+set(ZSTD_BUILD_CONTRIB OFF CACHE BOOL "" FORCE)
+set(ZSTD_BUILD_SHARED OFF CACHE BOOL "" FORCE)
+set(ZSTD_BUILD_STATIC ON CACHE BOOL "" FORCE)
+set(ZSTD_LEGACY_SUPPORT OFF CACHE BOOL "" FORCE)
+set(ZSTD_MULTITHREAD_SUPPORT OFF CACHE BOOL "" FORCE)
+add_subdirectory(${CMAKE_SOURCE_DIR}/3rdparty/zstd/build/cmake ${CMAKE_BINARY_DIR}/3rdparty/zstd EXCLUDE_FROM_ALL)
+set_target_properties(libzstd_static PROPERTIES POSITION_INDEPENDENT_CODE ON)
+
+set(ZARCHIVE_SOURCE_DIR ${CMAKE_SOURCE_DIR}/3rdparty/ZArchive)
+add_library(zarchive STATIC EXCLUDE_FROM_ALL
+    ${ZARCHIVE_SOURCE_DIR}/src/zarchivereader.cpp
+    ${ZARCHIVE_SOURCE_DIR}/src/zarchivewriter.cpp
+    ${ZARCHIVE_SOURCE_DIR}/src/sha_256.c
+)
+target_include_directories(zarchive SYSTEM PUBLIC ${ZARCHIVE_SOURCE_DIR}/include)
+target_compile_features(zarchive PUBLIC cxx_std_20)
+target_link_libraries(zarchive PRIVATE libzstd_static)
+set_target_properties(zarchive PROPERTIES POSITION_INDEPENDENT_CODE ON)
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    target_link_options(zarchive INTERFACE LINKER:--exclude-libs,libzarchive.a:libzstd.a)
+endif()

core/libs/CMakeLists.txt

@@ -1,5 +1,7 @@
 if(NOT WIN32)
-    string(REPLACE "-static " "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ")
+    foreach(runtimeLinkFlag IN ITEMS -static-libgcc -static-libstdc++ -static)
+        string(REPLACE "${runtimeLinkFlag}" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
+    endforeach()
     string(STRIP "${CMAKE_EXE_LINKER_FLAGS}" CMAKE_EXE_LINKER_FLAGS)
 endif()
 
@@ -221,6 +223,8 @@ if(BUILD_TESTING)
     if(Python3_Interpreter_FOUND)
         add_test(NAME nid_gnu_hash COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_gnu_hash.py $<TARGET_FILE:nid_patcher>)
+        add_test(NAME nid_elf_duplicates COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_elf_duplicates.py $<TARGET_FILE:nid_patcher>)
         add_test(NAME nid_patcher_macho COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_macho_nids.py $<TARGET_FILE:nid_patcher>)
         add_test(NAME nid_patcher_native_paths COMMAND ${Python3_EXECUTABLE}
@@ -238,6 +242,17 @@ if(BUILD_TESTING)
         target_link_options(patched_libraries_tests PRIVATE -static -static-libgcc -static-libstdc++)
     endif()
     add_test(NAME patched_libraries COMMAND patched_libraries_tests ${PATCHED_DIR})
+    if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        add_library(host_libstdcxx_exception_module SHARED tests/HostLibstdcxxExceptionModule.cpp)
+        target_link_libraries(host_libstdcxx_exception_module PRIVATE ${PATCHED_DIR}/libc.prx)
+        add_dependencies(host_libstdcxx_exception_module libs)
+        add_test_executable(host_libstdcxx_exception_tests tests/HostLibstdcxxException.cpp)
+        target_link_libraries(host_libstdcxx_exception_tests PRIVATE host_libstdcxx_exception_module ${PATCHED_DIR}/libc.prx)
+        add_dependencies(host_libstdcxx_exception_tests libs)
+        set_target_properties(host_libstdcxx_exception_module host_libstdcxx_exception_tests PROPERTIES BUILD_RPATH ${PATCHED_DIR})
+        add_test(NAME host_libstdcxx_exception COMMAND host_libstdcxx_exception_tests)
+        set_tests_properties(host_libstdcxx_exception PROPERTIES TIMEOUT 20)
+    endif()
     if(Python3_Interpreter_FOUND)
         add_test(NAME nid_elf_layout COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_elf_layout.py $<TARGET_FILE:nid_patcher>)
@@ -257,6 +272,7 @@ if(BUILD_TESTING)
     endif()
     if(APPLE)
         add_subdirectory(tests/write_watch)
+        add_subdirectory(tests/guest_arena)
     endif()
     if(WIN32)
         add_test_executable(windows_write_watch_tests tests/WindowsWriteWatch.cpp)
@@ -277,6 +293,35 @@ if(BUILD_TESTING)
             PASS_REGULAR_EXPRESSION "C\\+\\+ exception St9bad_alloc raised on thread [0-9]+ cannot be delivered: 0x[0-9a-f]+ undeliverable_host_exception_tests\\.exe\\+0x[0-9a-f]+ has no unwind data")
     endif()
 
+    add_test_executable(crash_text_tests tests/CrashText.cpp)
+    target_include_directories(crash_text_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    if(MINGW)
+        target_compile_options(crash_text_tests PRIVATE -fasynchronous-unwind-tables)
+    endif()
+    add_test(NAME crash_text COMMAND crash_text_tests)
+
+    if(WIN32)
+        add_test_executable(windows_crash_text_tests tests/WindowsCrashText.cpp)
+        add_dependencies(windows_crash_text_tests libc)
+        if(MINGW)
+            target_compile_options(windows_crash_text_tests PRIVATE -fasynchronous-unwind-tables)
+        endif()
+        add_test(NAME windows_crash_text COMMAND windows_crash_text_tests $<TARGET_FILE:libc>)
+        set_tests_properties(windows_crash_text PROPERTIES TIMEOUT 20
+            PASS_REGULAR_EXPRESSION "faulting[^~]*FATAL: unhandled exception 0xc0000005[^~]*  \\[r12\\] \"allocation failed in the heap\"[\r\n]+  \\[r13\\] u\"LowLevelFatalError \\[File:Unknown\\] \\[Line: 42\\] Out of memory\"[\r\n]+  \\[r14\\] \"PAGE END\"[.][.][.]"
+            FAIL_REGULAR_EXPRESSION "\\[rbx\\] u?\"|did not fault")
+    endif()
+
+    add_test_executable(guest_voice_port_attr_tests tests/GuestVoicePortAttr.cpp)
+    target_include_directories(guest_voice_port_attr_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_voice_port_attr_tests PRIVATE libSceVoice libc)
+    configure_windows_unwind(guest_voice_port_attr_tests)
+    add_test(NAME guest_voice_port_attr COMMAND guest_voice_port_attr_tests)
+    if(WIN32)
+        set_tests_properties(guest_voice_port_attr PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceVoice>")
+    endif()
+
     add_test_executable(guest_offline_imports_tests tests/GuestOfflineImports.cpp)
     target_include_directories(guest_offline_imports_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_offline_imports_tests PRIVATE libSceHttp libSceNpEntitlementAccess libSceRudp libc)
@@ -351,6 +396,16 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceRudp>")
     endif()
 
+    add_test_executable(guest_rudp_context_lifecycle_tests tests/GuestRudpContextLifecycle.cpp)
+    target_include_directories(guest_rudp_context_lifecycle_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_rudp_context_lifecycle_tests PRIVATE libSceRudp libc)
+    configure_windows_unwind(guest_rudp_context_lifecycle_tests)
+    add_test(NAME guest_rudp_context_lifecycle COMMAND guest_rudp_context_lifecycle_tests)
+    if(WIN32)
+        set_tests_properties(guest_rudp_context_lifecycle PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceRudp>")
+    endif()
+
     add_test_executable(guest_memory_pool_stats_tests tests/GuestMemoryPoolStats.cpp)
     target_include_directories(guest_memory_pool_stats_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_memory_pool_stats_tests PRIVATE libSceSsl libSceHttp2 libSceNet libkernel libc)
@@ -458,16 +513,6 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceSaveData.native>")
     endif()
 
-    add_test_executable(guest_ppsa01530_exports_tests tests/GuestPPSA01530Exports.cpp)
-    target_include_directories(guest_ppsa01530_exports_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
-    target_link_libraries(guest_ppsa01530_exports_tests PRIVATE libkernel libSceJson2 libSceNet libScePad libc)
-    configure_windows_unwind(guest_ppsa01530_exports_tests)
-    add_test(NAME guest_ppsa01530_exports COMMAND guest_ppsa01530_exports_tests)
-    if(WIN32)
-        set_tests_properties(guest_ppsa01530_exports PROPERTIES ENVIRONMENT_MODIFICATION
-            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
-    endif()
-
     add_test_executable(guest_posix_random_tests tests/GuestPosixRandom.cpp)
     target_include_directories(guest_posix_random_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(guest_posix_random_tests PRIVATE libScePosixForWebKit)

core/libs/SceTypes.hpp

@@ -160,18 +160,23 @@ struct VirtualQueryInfo {
     std::uint64_t offset;
     std::int32_t protection;
     std::int32_t memory_type;
-    std::uint32_t is_flexible : 1;
-    std::uint32_t is_direct : 1;
-    std::uint32_t is_stack : 1;
-    std::uint32_t is_pooled : 1;
-    std::uint32_t is_committed : 1;
-    std::uint32_t is_gpu_prt : 1;
-    std::uint32_t amm_usage : 1;
-    std::uint32_t reserved : 1;
+    std::uint8_t is_flexible : 1;
+    std::uint8_t is_direct : 1;
+    std::uint8_t is_stack : 1;
+    std::uint8_t is_pooled : 1;
+    std::uint8_t is_committed : 1;
+    std::uint8_t is_gpu_prt : 1;
+    std::uint8_t amm_usage : 1;
+    std::uint8_t reserved : 1;
     char name[32];
     std::uint8_t gpu_mask_id;
     std::uint8_t reserved2;
 };
+static_assert(sizeof(VirtualQueryInfo) == 72);
+static_assert(alignof(VirtualQueryInfo) == 8);
+static_assert(offsetof(VirtualQueryInfo, name) == 33);
+static_assert(offsetof(VirtualQueryInfo, gpu_mask_id) == 65);
+static_assert(offsetof(VirtualQueryInfo, reserved2) == 66);
 
 struct KernelSemaPrivate;
 struct KernelEventFlagPrivate;
@@ -675,10 +680,13 @@ struct AvPlayerVideoEx {
     std::uint8_t luma_bit_depth;
     std::uint8_t chroma_bit_depth;
     bool video_full_range_flag;
-    std::uint8_t reserved1[37];
+    std::uint8_t reserved1[5];
+    double frame_rate;
+    std::uint8_t reserved2[24];
 };
 static_assert(sizeof(AvPlayerVideoEx) == 80);
 static_assert(offsetof(AvPlayerVideoEx, crop_left_offset) == 20 && offsetof(AvPlayerVideoEx, pitch) == 36);
+static_assert(offsetof(AvPlayerVideoEx, frame_rate) == 48);
 
 struct AvPlayerTimedTextEx {
     std::uint8_t language_code[4];
@@ -913,6 +921,14 @@ struct VoicePortInfo {
     std::uint16_t reserved;
 };
 
+struct VoiceResourceInfo {
+    std::uint16_t max_voice_input_ports;
+    std::uint16_t max_voice_output_ports;
+    std::uint16_t max_device_input_ports;
+    std::uint16_t max_device_output_ports;
+    std::uint16_t max_ports;
+};
+
 struct VoiceStartParam {
     void* container;
     std::uint32_t mem_size;
@@ -1364,10 +1380,6 @@ struct NpUniversalDataSystemMemoryStat {
     std::size_t current_inuse_size;
 };
 
-struct NpUniversalDataSystemEvent {};
-struct NpUniversalDataSystemEventPropertyObject {};
-struct NpUniversalDataSystemEventPropertyArray {};
-
 struct NpUniversalDataSystemStorageStat {
     std::size_t in_events;
     std::size_t out_events;
@@ -1960,6 +1972,42 @@ struct VideoOutVblankStatus {
     std::uint8_t pad1[6] = {};
 };
 
+struct VideoOutBufferAttribute {
+    std::uint32_t pixel_format;
+    std::uint32_t tiling_mode;
+    std::uint32_t aspect_ratio;
+    std::uint32_t width;
+    std::uint32_t height;
+    std::uint32_t pitch_in_pixel;
+    std::uint32_t option;
+    std::uint32_t reserved0;
+    std::uint64_t reserved1;
+};
+static_assert(sizeof(VideoOutBufferAttribute) == 40);
+
+struct VideoOutFlipStatusPs4 {
+    std::uint64_t count = 0;
+    std::uint64_t processTime = 0;
+    std::uint64_t tsc = 0;
+    std::int64_t flipArg = 0;
+    std::uint64_t submitTsc = 0;
+    std::uint64_t reserved0 = 0;
+    std::int32_t gcQueueNum = 0;
+    std::int32_t flipPendingNum = 0;
+    std::int32_t currentBuffer = 0;
+    std::uint32_t reserved1 = 0;
+};
+static_assert(sizeof(VideoOutFlipStatusPs4) == 64);
+
+struct VideoOutVblankStatusPs4 {
+    std::uint64_t count = 0;
+    std::uint64_t processTime = 0;
+    std::uint64_t tsc = 0;
+    std::uint8_t flags = 0;
+    std::uint8_t pad[7] = {};
+};
+static_assert(sizeof(VideoOutVblankStatusPs4) == 32);
+
 struct VideoOutOutputStatus {
     std::uint32_t resolution = 0;
     std::uint32_t dynamicRange = 0;
@@ -1988,6 +2036,11 @@ struct VideoOutColorSettings {
     std::uint32_t reserved[3] = {};
 };
 
+struct VideoOutColorSpaceConversionSettings {
+    std::uint32_t internalData[8] = {};
+};
+static_assert(sizeof(VideoOutColorSpaceConversionSettings) == 32);
+
 
 using atexit_func_t = void (*)();
 

core/libs/nid/src/ElfNidPatcher.cpp

@@ -4,6 +4,7 @@
 #include <nid/NidCompute.hpp>
 #include <algorithm>
 #include <unordered_map>
+#include <utility>
 #include <cstdint>
 #include <cstring>
 #include <stdexcept>
@@ -408,6 +409,7 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
     };
 
     std::vector<SymbolNameUse> uses;
+    std::unordered_map<std::string, std::pair<std::uint64_t, std::uint16_t>> patchedExports;
     for (std::size_t i = 1u; i < symCount; ++i) {
         const std::size_t symOffset = dynSymOffset + i * sizeof(Elf64_Sym);
         const auto sym = Read<Elf64_Sym>(elf, symOffset);
@@ -424,6 +426,9 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
             const auto it = nidMap.find(symName);
             if (it != nidMap.end())
                 newValue = it->second;
+            const auto [existing, inserted] = patchedExports.try_emplace(newValue, sym.st_value, sym.st_shndx);
+            if (!inserted && existing->second != std::pair{sym.st_value, sym.st_shndx})
+                throw std::runtime_error("duplicate patched export name: " + newValue);
         } else if (binding != kStbLocal && sym.st_shndx == kShnUndef) {
             const bool isUnversioned = versymOffset == 0u ||
                 (Read<std::uint16_t>(elf, versymOffset + i * sizeof(std::uint16_t)) <= 1u);

core/libs/nid/tests/test_elf_duplicates.py

@@ -0,0 +1,69 @@
+import base64
+import hashlib
+import sys
+import unittest
+from pathlib import Path
+
+from test_symbol_versions import build_image, patch, read_dynsym, IMPORTS, NID_NAME, SHN_UNDEF, STT_FUNC
+
+
+def nid(name):
+    digest = hashlib.sha1(name.encode() + bytes.fromhex("518d64a635ded8c1e6b039b1c3e55230")).digest()
+    return base64.b64encode(digest[:8][::-1]).decode().rstrip("=").replace("/", "-")
+
+
+def image(names, addresses=None):
+    if addresses is None:
+        addresses = [0x1200 + index * 0x20 for index in range(len(names))]
+    entries = [(None, 1, SHN_UNDEF, 0, 0)]
+    entries += [(name, 1, SHN_UNDEF, 0, 0) for name in IMPORTS]
+    entries += [(name, 1, 9, STT_FUNC, address) for name, address in zip(names, addresses)]
+    return bytes(build_image(entries, False))
+
+
+class ElfDuplicateTests(unittest.TestCase):
+    def test_exports_patched_to_one_name_are_rejected(self):
+        cases = (
+            ["sceAlphaComputeLength", "sceAlphaComputeLength_nid_disambig1", "sceBetaCopyString"],
+            ["sceAlphaComputeLength_nid_postfix", "sceAlphaComputeLength_nid_disambig12", "sceBetaCopyString"],
+            ["sceBetaCopyString", "sceAlphaComputeLength_nid_no_patch", "sceAlphaComputeLength_nid_no_patch_nid_no_patch_cut"],
+        )
+        for names in cases:
+            with self.subTest(names=names):
+                before = image(names)
+                result, after = patch(PATCHER, before, "duplicates.so")
+                self.assertEqual(result.returncode, 2, (result.returncode, result.stderr))
+                self.assertIn(b"duplicate patched export name", result.stderr)
+                self.assertEqual(after, before)
+
+    def test_aliases_of_one_address_are_patched(self):
+        names = ["sceAlphaComputeLength", "sceAlphaComputeLength_nid_postfix", "sceAlphaComputeLength_nid_no_patch_cut", "sceBetaCopyString"]
+        result, after = patch(PATCHER, image(names, [0x1200, 0x1200, 0x1200, 0x1220]), "aliases.so")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        patched = [(entry[0], entry[2]) for entry in read_dynsym(after) if entry[1] == 9]
+        self.assertEqual(sorted(name for name, address in patched if address == 0x1200),
+                         sorted(["sceAlphaComputeLength", "sceAlphaComputeLength", nid("sceAlphaComputeLength")]))
+        self.assertEqual([name for name, address in patched if address == 0x1220], [nid("sceBetaCopyString")])
+        self.assertTrue(NID_NAME.match(nid("sceAlphaComputeLength")))
+
+    def test_aliases_of_different_addresses_are_rejected(self):
+        names = ["sceAlphaComputeLength", "sceAlphaComputeLength_nid_postfix", "sceAlphaComputeLength_nid_no_patch_cut"]
+        before = image(names, [0x1200, 0x1200, 0x1220])
+        result, after = patch(PATCHER, before, "aliases.so")
+        self.assertEqual(result.returncode, 2, (result.returncode, result.stderr))
+        self.assertIn(b"duplicate patched export name: sceAlphaComputeLength", result.stderr)
+        self.assertEqual(after, before)
+
+    def test_distinct_exports_are_still_patched(self):
+        names = ["sceAlphaComputeLength_nid_disambig1", "sceBetaCopyString_nid_postfix", "sceGammaKeep_nid_no_patch_cut"]
+        result, after = patch(PATCHER, image(names), "distinct.so")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        patched = [entry[0] for entry in read_dynsym(after) if entry[1] == 9]
+        self.assertEqual(len(patched), 3)
+        self.assertEqual(len(set(patched)), 3)
+        self.assertIn("sceGammaKeep", patched)
+
+
+if __name__ == "__main__":
+    PATCHER = Path(sys.argv.pop(1)).resolve()
+    unittest.main()

core/libs/prx/libSceAgc/Command/include/RegisterDefaults.hpp

@@ -6,6 +6,7 @@
 namespace Agc::Command {
 
 void* GetRegisterDefaults(std::uint32_t version, bool internal, const char* function);
+int GetDefaultCxStateFlat(void* destination, std::uint32_t size);
 
 }
 

core/libs/prx/libSceAgc/Command/src/Packet.cpp

@@ -107,7 +107,10 @@ std::uint32_t* WriteRegisterRange(CommandBuffer* buffer, std::uint32_t opcode, s
 }
 
 std::uint32_t* WriteRegisters(CommandBuffer* buffer, std::uint32_t opcode, const volatile ShaderRegister* registers, std::uint32_t count, bool snapshotAll, const char* function) {
-    Require(count != 0, function, "empty register list");
+    if (count == 0) {
+        Require(buffer != nullptr, function, "null command buffer");
+        return buffer->cursor_up;
+    }
     CheckAddress(reinterpret_cast<std::uintptr_t>(registers), 4, function);
     std::vector<ShaderRegister> snapshot;
     if (snapshotAll) {

core/libs/prx/libSceAgc/Command/src/RegisterDefaults.cpp

@@ -4,6 +4,7 @@
 #include <algorithm>
 #include <array>
 #include <cstddef>
+#include <cstring>
 #include <mutex>
 #include <vector>
 
@@ -44,6 +45,33 @@ struct Storage {
 std::mutex defaultsMutex;
 std::array<std::array<Storage, 14>, 2> storage;
 
+struct AgcContextState {
+    std::uint32_t data[512];
+};
+
+static_assert(sizeof(AgcContextState) == 2048);
+
+std::mutex defaultStateMutex;
+AgcContextState defaultState{};
+bool defaultStateInitialized = false;
+
+void InitDefaultCxState() {
+    std::memset(&defaultState, 0, sizeof(defaultState));
+    const auto* source = agcPublicRegDefaultsByVersion[8];
+    for (std::size_t bank = 0; bank < 4; ++bank) {
+        const auto& b = source->banks[bank];
+        if (b.registers == nullptr) continue;
+        for (std::uint32_t r = 0; r < b.registerCount; ++r) {
+            const auto offset = b.registers[r].offset;
+            const auto value = b.registers[r].value;
+            if (offset < 512) {
+                defaultState.data[offset] = value;
+            }
+        }
+    }
+    defaultStateInitialized = true;
+}
+
 }
 
 void* GetRegisterDefaults(std::uint32_t version, bool internal, const char* function) {
@@ -76,4 +104,15 @@ void* GetRegisterDefaults(std::uint32_t version, bool internal, const char* func
     return &destination.defaults;
 }
 
+int GetDefaultCxStateFlat(void* destination, std::uint32_t size) {
+    if (destination == nullptr) return static_cast<int>(0x80890001);
+    std::lock_guard lock(defaultStateMutex);
+    if (!defaultStateInitialized) {
+        InitDefaultCxState();
+    }
+    const auto copyBytes = std::min(size, static_cast<std::uint32_t>(sizeof(AgcContextState)));
+    std::memcpy(destination, defaultState.data, copyBytes);
+    return 0;
+}
+
 }

core/libs/prx/libSceAgc/Misc/include/RegisterDefaults.hpp

@@ -1,4 +1,9 @@
 #ifndef CORE_LIBS_PRX_LIBSCEAGC_MISC_INCLUDE_REGISTERDEFAULTS_HPP
 #define CORE_LIBS_PRX_LIBSCEAGC_MISC_INCLUDE_REGISTERDEFAULTS_HPP
+#include "SceTypes.hpp"
+#include <cstdint>
 
+extern "C" {
+int APS5_VABI sceAgcGetDefaultCxStateFlat(void* out_state, std::uint32_t size);
+}
 #endif

core/libs/prx/libSceAgc/Misc/src/RegisterDefaults.cpp

@@ -26,4 +26,8 @@ void* APS5_VABI sceAgcGetRegisterDefaults2Internal(std::uint32_t version) {
     return Agc::Command::GetRegisterDefaults(version, true, __func__);
 }
 
+int APS5_VABI sceAgcGetDefaultCxStateFlat(void* out_state, std::uint32_t size) {
+    return Agc::Command::GetDefaultCxStateFlat(out_state, size);
+}
+
 }

core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp

@@ -127,13 +127,11 @@ int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Sh
     return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValue, true);
 }
 
-APS5_EXPORT("dbOlWdppb4o", sceAgcUnknownCreateInterpolantMapping);
-int APS5_VABI sceAgcUnknownCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
+int APS5_VABI sceAgcCreateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
     return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValueSplitF16, true);
 }
 
-APS5_EXPORT("vieBRwlh1Lw", sceAgcUnknown_vieBRwlh1Lw);
-int APS5_VABI sceAgcUnknown_vieBRwlh1Lw(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
+int APS5_VABI sceAgcUpdateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
     return CreateInterpolantMapping(__func__, regs, gs, ps, UpdateInterpolantValue, false);
 }
 

core/libs/prx/libSceAgc/Unimplemented.cpp

@@ -30,11 +30,6 @@ int APS5_VABI sceAgcDcbSetMarkerSpan() {
  return 0;
 }
 
-int APS5_VABI sceAgcGetDefaultCxStateFlat() {
- NotImplemented_nid_no_patch(__func__);
- return 0;
-}
-
 int APS5_VABI sceAgcSetSemaphoreMemory() {
  NotImplemented_nid_no_patch(__func__);
  return 0;

core/libs/prx/libSceAgc/tests/Commands.cpp

@@ -21,6 +21,7 @@ extern "C" int APS5_VABI sceAgcInit(std::uint32_t version);
 extern "C" void* APS5_VABI sceAgcGetRegisterDefaults();
 extern "C" void* APS5_VABI sceAgcGetRegisterDefaultsInternal();
 extern "C" void* APS5_VABI sceAgcGetRegisterDefaults2Internal(std::uint32_t version);
+extern "C" int APS5_VABI sceAgcGetDefaultCxStateFlat(void* out_state, std::uint32_t size);
 extern "C" int APS5_VABI sceAgcInit_0090(std::uint32_t* state, std::uint32_t version);
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexAuto(CommandBuffer* buf, std::uint32_t indexCount, std::uint64_t modifier);
 extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexIndirect(CommandBuffer* buf, std::uint32_t dataOffsetInBytes, std::uint64_t modifier);
@@ -637,6 +638,21 @@ void testDefaults() {
     auto* internalDefaults = sceAgcGetRegisterDefaultsInternal();
     check(internalDefaults != nullptr && internalDefaults == Agc::Command::GetRegisterDefaults(0, true, __func__) && internalDefaults == sceAgcGetRegisterDefaults2Internal(0), "internal register defaults are not the baseline internal table");
     check(internalDefaults != sceAgcGetRegisterDefaults(), "internal register defaults returned the public table");
+    constexpr int kInvalidArgument = static_cast<int>(0x80890001u);
+    std::array<std::uint32_t, 512> flatCxBuffer{};
+    check(sceAgcGetDefaultCxStateFlat(nullptr, sizeof(flatCxBuffer)) == kInvalidArgument, "null destination was accepted");
+    check(sceAgcGetDefaultCxStateFlat(flatCxBuffer.data(), 0) == 0, "zero size copy failed");
+    check(sceAgcGetDefaultCxStateFlat(flatCxBuffer.data(), sizeof(flatCxBuffer)) == 0, "flat CX state default retrieval failed");
+    check(flatCxBuffer[0x001] == 0x11000100u, "flat CX state register 0x001 mismatch");
+    check(flatCxBuffer[0x007] == 0x0000ffffu, "flat CX state register 0x007 mismatch");
+    check(flatCxBuffer[0x12d] == 0x3f800000u, "flat CX state register 0x12d mismatch");
+    check(flatCxBuffer[0x1e0] == 0x20010001u, "flat CX state register 0x1e0 mismatch");
+    std::array<std::uint32_t, 513> overBuffer{};
+    overBuffer[512] = 0x12345678u;
+    check(sceAgcGetDefaultCxStateFlat(overBuffer.data(), sizeof(overBuffer)) == 0, "oversized copy failed");
+    check(overBuffer[512] == 0x12345678u, "oversized copy overflowed buffer");
+    check(Agc::Command::GetDefaultCxStateFlat(nullptr, sizeof(flatCxBuffer)) == kInvalidArgument, "null command destination was accepted");
+    check(Agc::Command::GetDefaultCxStateFlat(flatCxBuffer.data(), sizeof(flatCxBuffer)) == 0, "command flat CX state retrieval failed");
     expectFailure([] { Agc::Command::GetRegisterDefaults(14, false, __func__); });
     expectFailure([] { Agc::Command::GetRegisterDefaults(0xffffffffu, true, __func__); });
 }

core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp

@@ -12,8 +12,8 @@
 #include <span>
 
 extern "C" int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Shader* gs, const Shader* ps);
-extern "C" int APS5_VABI sceAgcUnknownCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps);
-extern "C" int APS5_VABI sceAgcUnknown_vieBRwlh1Lw(ShaderRegister* regs, const Shader* gs, const Shader* ps);
+extern "C" int APS5_VABI sceAgcCreateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps);
+extern "C" int APS5_VABI sceAgcUpdateInterpolantMapping_10000(ShaderRegister* regs, const Shader* gs, const Shader* ps);
 
 namespace {
 
@@ -61,14 +61,14 @@ void checkIdentity(const Registers& regs, std::uint32_t first, const char* messa
 
 void testIdentity() {
     auto regs = filled();
-    check(sceAgcUnknownCreateInterpolantMapping(regs.data(), nullptr, nullptr) == 0, "mapping without shaders failed");
+    check(sceAgcCreateInterpolantMapping_10000(regs.data(), nullptr, nullptr) == 0, "mapping without shaders failed");
     checkIdentity(regs, 0, "mapping without a pixel shader is not the identity");
     Shader ps{};
     regs = filled();
-    check(sceAgcUnknownCreateInterpolantMapping(regs.data(), nullptr, &ps) == 0, "mapping without inputs failed");
+    check(sceAgcCreateInterpolantMapping_10000(regs.data(), nullptr, &ps) == 0, "mapping without inputs failed");
     checkIdentity(regs, 0, "mapping without inputs is not the identity");
     regs = filled();
-    check(sceAgcUnknown_vieBRwlh1Lw(regs.data(), nullptr, &ps) == 0, "update without inputs failed");
+    check(sceAgcUpdateInterpolantMapping_10000(regs.data(), nullptr, &ps) == 0, "update without inputs failed");
     check(preparedCount == 0u, "update without inputs prepared registers");
     const auto untouched = filled();
     check(std::memcmp(regs.data(), untouched.data(), sizeof(regs)) == 0, "update without inputs wrote registers");
@@ -94,7 +94,7 @@ void testMapping() {
     ps.num_input_semantics = static_cast<std::uint32_t>(inputs.size());
 
     auto split = filled();
-    check(sceAgcUnknownCreateInterpolantMapping(split.data(), &gs, &ps) == 0, "split f16 mapping failed");
+    check(sceAgcCreateInterpolantMapping_10000(split.data(), &gs, &ps) == 0, "split f16 mapping failed");
     check(std::memcmp(split.data(), preparedMapping.data(), sizeof(split)) == 0, "prepared split mapping differs from published registers");
     for (std::uint32_t i = 0; i < inputs.size(); ++i) {
         check(split[i].offset == ShaderRegs::SPI_PS_INPUT_CNTL_0 + i, "interpolant register offset changed");
@@ -109,7 +109,7 @@ void testMapping() {
     auto update = filled();
     const auto untouched = filled();
     const std::size_t written = inputs.size() * sizeof(ShaderRegister);
-    check(sceAgcUnknown_vieBRwlh1Lw(update.data(), &gs, &ps) == 0, "update mapping failed");
+    check(sceAgcUpdateInterpolantMapping_10000(update.data(), &gs, &ps) == 0, "update mapping failed");
     check(preparedCount == inputs.size(), "update prepared registers past its inputs");
     check(std::memcmp(update.data(), preparedMapping.data(), written) == 0, "prepared update differs from published registers");
     check(std::memcmp(update.data(), split.data(), written) == 0, "update mapping differs from the split f16 mapping");
@@ -142,7 +142,7 @@ void testUpdateMapping() {
 
     auto update = filled();
     const auto untouched = filled();
-    check(sceAgcUnknown_vieBRwlh1Lw(update.data(), &gs, &ps) == 0, "update mapping failed");
+    check(sceAgcUpdateInterpolantMapping_10000(update.data(), &gs, &ps) == 0, "update mapping failed");
     check(update[0].value == 0x00000625u, "custom interpolant update changed");
     check(update[1].value == 0x01380107u, "low f16 half update changed");
     check(update[2].value == 0x02480220u, "unmatched high f16 half update changed");
@@ -154,14 +154,14 @@ void testUpdateMapping() {
     check(std::memcmp(update.data() + inputs.size(), untouched.data(), sizeof(update) - written) == 0, "update wrote past its inputs");
 
     auto create = filled();
-    check(sceAgcUnknownCreateInterpolantMapping(create.data(), &gs, &ps) == 0, "split f16 mapping failed");
+    check(sceAgcCreateInterpolantMapping_10000(create.data(), &gs, &ps) == 0, "split f16 mapping failed");
     check(std::memcmp(create.data(), update.data(), written) == 0, "update differs from the split f16 mapping");
     checkIdentity(create, static_cast<std::uint32_t>(inputs.size()), "unused interpolants are not the identity");
 
     for (const std::uint32_t index : {1u, 3u}) {
         inputs[index].default_value = 0;
         update = filled();
-        expectFailure([&] { sceAgcUnknown_vieBRwlh1Lw(update.data(), &gs, &ps); });
+        expectFailure([&] { sceAgcUpdateInterpolantMapping_10000(update.data(), &gs, &ps); });
         check(std::memcmp(update.data(), untouched.data(), sizeof(update)) == 0, "rejected update wrote registers");
         inputs[index].default_value = inputs[index].default_value_hi;
     }
@@ -177,15 +177,15 @@ void testRejections() {
     std::memset(regs.data(), 0xcc, sizeof(regs));
     const auto saved = regs;
     expectFailure([&] { sceAgcCreateInterpolantMapping_0100(regs.data(), &gs, &ps); });
-    expectFailure([&] { sceAgcUnknownCreateInterpolantMapping(regs.data(), &gs, &ps); });
-    expectFailure([&] { sceAgcUnknown_vieBRwlh1Lw(regs.data(), &gs, &ps); });
+    expectFailure([&] { sceAgcCreateInterpolantMapping_10000(regs.data(), &gs, &ps); });
+    expectFailure([&] { sceAgcUpdateInterpolantMapping_10000(regs.data(), &gs, &ps); });
     check(std::memcmp(regs.data(), saved.data(), sizeof(regs)) == 0, "rejected mapping wrote registers");
     ps.num_input_semantics = 1;
-    expectFailure([&] { sceAgcUnknownCreateInterpolantMapping(regs.data(), nullptr, &ps); });
-    expectFailure([&] { sceAgcUnknownCreateInterpolantMapping(nullptr, &gs, &ps); });
-    expectFailure([&] { sceAgcUnknown_vieBRwlh1Lw(regs.data(), nullptr, &ps); });
-    expectFailure([&] { sceAgcUnknown_vieBRwlh1Lw(nullptr, &gs, &ps); });
-    expectFailure([&] { sceAgcUnknown_vieBRwlh1Lw(regs.data(), &gs, nullptr); });
+    expectFailure([&] { sceAgcCreateInterpolantMapping_10000(regs.data(), nullptr, &ps); });
+    expectFailure([&] { sceAgcCreateInterpolantMapping_10000(nullptr, &gs, &ps); });
+    expectFailure([&] { sceAgcUpdateInterpolantMapping_10000(regs.data(), nullptr, &ps); });
+    expectFailure([&] { sceAgcUpdateInterpolantMapping_10000(nullptr, &gs, &ps); });
+    expectFailure([&] { sceAgcUpdateInterpolantMapping_10000(regs.data(), &gs, nullptr); });
 }
 
 }

core/libs/prx/libSceAgc/tests/RegisterCommands.cpp

@@ -193,6 +193,10 @@ void testDirectList() {
         packet = writers[i](&split.buffer, paired.data(), 4);
         check(std::equal(expectedSplit.begin(), expectedSplit.end(), packet) && split.buffer.cursor_up == packet + expectedSplit.size(), "register pairs were not written as two ranges");
         check(sizes[i](0) == 0, "an empty register list has a size");
+        Storage empty;
+        empty.words.fill(sentinel);
+        check(writers[i](&empty.buffer, nullptr, 0) == empty.words.data() && empty.buffer.cursor_up == empty.words.data() && empty.words.front() == sentinel, "an empty register list wrote a packet");
+        expectFailure([&] { writers[i](nullptr, nullptr, 0); });
         check(sizes[i](0x15555555u) == 0xfffffffcu, "largest register list size mismatch");
         for (const auto count : {0x15555556u, 0x40000000u, 0xffffffffu}) {
             expectFailure([&] { sizes[i](count); });

core/libs/prx/libSceAgc/tests/ShaderHelpers.cpp

@@ -2,6 +2,7 @@
 #include "prx/libSceAgc/Shader/include/PrimState.hpp"
 #include "prx/libSceAgc/Shader/include/InterpolantMapping.hpp"
 #include "prx/libSceAgc/Shader/include/ShaderConstants.hpp"
+#include "prx/libc/include/Shutdown.hpp"
 #include "prx/libc/include/general/VabiMacros.hpp"
 #include <algorithm>
 #include <array>
@@ -194,9 +195,11 @@ int main() {
         Require(preparations == 0 && mappings == 0 && links == 0);
         OptionalGeometryOutputRegister(special, vertex, hull);
         HandBuiltUserVgprRegister();
+        LibcRunShutdown_nid_postfix();
         std::cout << "shader helper preparation tests passed\n";
     } catch (const std::exception& error) {
         std::cerr << error.what() << '\n';
+        try { LibcRunShutdown_nid_postfix(); } catch (...) {}
         return 1;
     }
 }

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -83,6 +83,19 @@ add_custom_command(
         VERBATIM
 )
 
+set(agcTextureRepackShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/TextureRepack.comp)
+set(agcTextureRepackShaderSpv ${agcTextureDetileGeneratedDir}/TextureRepack.spv)
+set(agcTextureRepackShaderHeader ${agcTextureDetileGeneratedDir}/TextureRepack_spv.h)
+add_custom_command(
+        OUTPUT "${agcTextureRepackShaderHeader}"
+        COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}"
+        COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -o "${agcTextureRepackShaderSpv}" "${agcTextureRepackShaderSource}"
+        COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTextureRepackShaderSpv} -DOUTPUT=${agcTextureRepackShaderHeader}
+        -DSYMBOL=TEXTURE_REPACK_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
+        DEPENDS "${agcTextureRepackShaderSource}" glslang-standalone
+        VERBATIM
+)
+
 set(agcSampleCounterShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/SampleCounter.comp)
 set(agcSampleCounterShaderSpv ${agcTextureDetileGeneratedDir}/SampleCounter.spv)
 set(agcSampleCounterShaderHeader ${agcTextureDetileGeneratedDir}/SampleCounter_spv.h)
@@ -125,6 +138,23 @@ foreach(agcPresentationStage IN ITEMS Vert Frag)
     )
     list(APPEND agcPresentationShaderHeaders ${agcPresentationShaderHeader})
 endforeach()
+foreach(agcOverlayStage IN ITEMS Vert Frag)
+    string(TOLOWER "${agcOverlayStage}" agcOverlayExtension)
+    string(TOUPPER "${agcOverlayStage}" agcOverlaySymbol)
+    set(agcOverlayShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/PerformanceOverlay.${agcOverlayExtension})
+    set(agcOverlayShaderSpv ${agcTextureDetileGeneratedDir}/PerformanceOverlay${agcOverlayStage}.spv)
+    set(agcOverlayShaderHeader ${agcTextureDetileGeneratedDir}/PerformanceOverlay${agcOverlayStage}_spv.h)
+    add_custom_command(
+            OUTPUT "${agcOverlayShaderHeader}"
+            COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}"
+            COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -o "${agcOverlayShaderSpv}" "${agcOverlayShaderSource}"
+            COMMAND ${CMAKE_COMMAND} -DINPUT=${agcOverlayShaderSpv} -DOUTPUT=${agcOverlayShaderHeader}
+            -DSYMBOL=PERFORMANCE_OVERLAY_${agcOverlaySymbol}_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
+            DEPENDS "${agcOverlayShaderSource}" glslang-standalone
+            VERBATIM
+    )
+    list(APPEND agcPresentationShaderHeaders ${agcOverlayShaderHeader})
+endforeach()
 
 set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Unimplemented.cpp
@@ -154,6 +184,7 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawPipeline.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawPrecheck.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawRegisterKey.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawRelocate.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawTiming.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/IndirectDraw.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/EntryPoints.cpp
@@ -173,6 +204,7 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Presentation.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Queues/QueueWorker.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Queues/Submission.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Queues/SubmissionRing.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Queues/WorkerAffinity.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Shaders/ShaderDiagnostics.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Shaders/ShaderRegistry.cpp
@@ -183,6 +215,7 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Synchronization/WaitStatistics.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/WorkerSampler.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/PerformanceTrace.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/PerformanceTraceThread.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/TraceFile.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/ThreadPriority.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/GuestMemory.cpp
@@ -196,6 +229,7 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/AspectFit.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/PresentationScaler.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/PresentationPass.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/PerformanceOverlay.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Draw.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DepthSurface.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/MultisampleTarget.cpp
@@ -204,6 +238,7 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Pipeline.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineLibrary.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineCache.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineJournal.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Resources.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PlaceholderImages.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Recorder.cpp
@@ -230,6 +265,7 @@ set(agcDriverSources
         ${agcCmaskClearShaderHeader}
         ${agcMeshArgumentsShaderHeader}
         ${agcTextureDetileImageShaderHeader}
+        ${agcTextureRepackShaderHeader}
         ${agcSampleCounterShaderHeader}
         ${agcSampleDumpsShaderHeader}
         ${agcPresentationShaderHeaders}
@@ -316,6 +352,17 @@ set_target_properties(libSceAgcDriver PROPERTIES CXX_EXTENSIONS OFF)
 configure_windows_unwind(libSceAgcDriver)
 
 if(BUILD_TESTING)
+    add_test_executable(agc_frame_dump_io_tests tests/FrameDumpIo.cpp)
+    target_include_directories(agc_frame_dump_io_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(agc_frame_dump_io_tests PRIVATE libc)
+    configure_windows_unwind(agc_frame_dump_io_tests)
+    add_test(NAME agc_frame_dump_io COMMAND agc_frame_dump_io_tests)
+    set_tests_properties(agc_frame_dump_io PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(agc_frame_dump_io PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+
     add_test_executable(agc_write_watch_coverage_tests tests/WriteWatchCoverage.cpp)
     target_include_directories(agc_write_watch_coverage_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(agc_write_watch_coverage_tests PRIVATE libc)
@@ -337,6 +384,18 @@ if(BUILD_TESTING)
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 
+    add_test_executable(agc_async_log_tests tests/AsyncLog.cpp)
+    target_include_directories(agc_async_log_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(agc_async_log_tests PRIVATE libc)
+    configure_windows_unwind(agc_async_log_tests)
+    add_test(NAME agc_async_log COMMAND agc_async_log_tests)
+    add_test(NAME agc_async_log_fatal COMMAND agc_async_log_tests fatal)
+    set_tests_properties(agc_async_log agc_async_log_fatal PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(agc_async_log agc_async_log_fatal PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+
     add_test_executable(agc_stack_walk_tests tests/StackWalk.cpp)
     target_include_directories(agc_stack_walk_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(agc_stack_walk_tests PRIVATE libc)
@@ -358,6 +417,17 @@ if(BUILD_TESTING)
         set_tests_properties(agc_performance_trace PROPERTIES ENVIRONMENT_MODIFICATION
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+
+    add_test_executable(agc_handoff_spin_tests tests/HandoffSpin.cpp)
+    target_include_directories(agc_handoff_spin_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(agc_handoff_spin_tests PRIVATE libc)
+    configure_windows_unwind(agc_handoff_spin_tests)
+    add_test(NAME agc_handoff_spin COMMAND agc_handoff_spin_tests)
+    set_tests_properties(agc_handoff_spin PROPERTIES TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_handoff_spin PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
 endif()
 
 if(WIN32)
@@ -457,6 +527,7 @@ add_test_executable(agc_shader_memory_tests EXCLUDE_FROM_ALL
         tests/ShaderMemory.cpp
         Execution/src/ShaderMemory.cpp
         Execution/src/GuestMemory.cpp
+        Execution/src/PerformanceTraceThread.cpp
 )
 target_include_directories(agc_shader_memory_tests PRIVATE ${LIBS_INCLUDE_DIR})
 target_link_libraries(agc_shader_memory_tests PRIVATE shader_recompiler_agc_driver libc)
@@ -472,6 +543,34 @@ target_link_libraries(agc_vertex_input_tests PRIVATE shader_recompiler_agc_drive
 set_target_properties(agc_vertex_input_tests PROPERTIES CXX_EXTENSIONS OFF)
 configure_windows_unwind(agc_vertex_input_tests)
 
+add_test_executable(agc_guarded_memory_loads_tests EXCLUDE_FROM_ALL tests/GuardedMemoryLoads.cpp)
+target_link_libraries(agc_guarded_memory_loads_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_guarded_memory_loads_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_guarded_memory_loads_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_guarded_memory_loads_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_guarded_memory_loads COMMAND agc_guarded_memory_loads_tests)
+    set_tests_properties(agc_guarded_memory_loads PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_guarded_memory_loads PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
+add_test_executable(agc_lds_lock_budget_tests EXCLUDE_FROM_ALL tests/LdsLockBudget.cpp)
+target_link_libraries(agc_lds_lock_budget_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_lds_lock_budget_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_lds_lock_budget_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_lds_lock_budget_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_lds_lock_budget COMMAND agc_lds_lock_budget_tests)
+    set_tests_properties(agc_lds_lock_budget PROPERTIES TIMEOUT 30 ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_lds_lock_budget PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_shared_write_tests EXCLUDE_FROM_ALL tests/SharedWrites.cpp)
 target_include_directories(agc_shared_write_tests PRIVATE ${LIBS_INCLUDE_DIR})
 target_link_libraries(agc_shared_write_tests PRIVATE libSceAgcDriver libkernel libc)
@@ -481,6 +580,7 @@ configure_windows_unwind(agc_shared_write_tests)
 add_test_executable(agc_write_tracking_tests EXCLUDE_FROM_ALL
         tests/WriteTracking.cpp
         Execution/src/GuestMemory.cpp
+        Execution/src/PerformanceTraceThread.cpp
 )
 target_include_directories(agc_write_tracking_tests PRIVATE ${LIBS_INCLUDE_DIR})
 target_link_libraries(agc_write_tracking_tests PRIVATE shader_recompiler_agc_driver libc)
@@ -542,6 +642,15 @@ if(BUILD_TESTING)
     add_test(NAME agc_pipeline_link_optimization COMMAND agc_pipeline_link_optimization_tests)
 endif()
 
+add_test_executable(agc_robust_buffer_access_tests EXCLUDE_FROM_ALL tests/RobustBufferAccess.cpp)
+target_include_directories(agc_robust_buffer_access_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
+set_target_properties(agc_robust_buffer_access_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_robust_buffer_access_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_robust_buffer_access_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_robust_buffer_access COMMAND agc_robust_buffer_access_tests)
+endif()
+
 add_test_executable(agc_device_fault_tests EXCLUDE_FROM_ALL tests/DeviceFault.cpp)
 target_include_directories(agc_device_fault_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
 set_target_properties(agc_device_fault_tests PROPERTIES CXX_EXTENSIONS OFF)
@@ -750,6 +859,24 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_pipeline_queue_failure_tests EXCLUDE_FROM_ALL tests/PipelineQueueFailure.cpp
+    Graphics/src/PipelineLibrary.cpp tests/AllocationFailure.cpp)
+target_include_directories(agc_pipeline_queue_failure_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
+target_link_libraries(agc_pipeline_queue_failure_tests PRIVATE libSceAgcDriver libc)
+set_target_properties(agc_pipeline_queue_failure_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_pipeline_queue_failure_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_pipeline_queue_failure_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    foreach(mode wait clear stop)
+        add_test(NAME agc_pipeline_queue_failure_${mode} COMMAND agc_pipeline_queue_failure_tests ${mode})
+        set_tests_properties(agc_pipeline_queue_failure_${mode} PROPERTIES TIMEOUT 15)
+        if(WIN32)
+            set_tests_properties(agc_pipeline_queue_failure_${mode} PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        endif()
+    endforeach()
+endif()
+
 add_test_executable(agc_pass_hazards_tests EXCLUDE_FROM_ALL tests/PassHazards.cpp Graphics/src/PassHazards.cpp)
 target_include_directories(agc_pass_hazards_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
 target_link_libraries(agc_pass_hazards_tests PRIVATE libc)
@@ -771,6 +898,7 @@ if(BUILD_TESTING)
 endif()
 
 add_test_executable(agc_buffer_pool_tests EXCLUDE_FROM_ALL tests/BufferPool.cpp)
+target_sources(agc_buffer_pool_tests PRIVATE Graphics/src/BufferPool.cpp tests/AllocationFailure.cpp)
 target_include_directories(agc_buffer_pool_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
 target_link_libraries(agc_buffer_pool_tests PRIVATE libSceAgcDriver libc)
 set_target_properties(agc_buffer_pool_tests PROPERTIES CXX_EXTENSIONS OFF)
@@ -778,8 +906,14 @@ configure_windows_unwind(agc_buffer_pool_tests)
 if(BUILD_TESTING)
     set_target_properties(agc_buffer_pool_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
     add_test(NAME agc_buffer_pool COMMAND agc_buffer_pool_tests)
+    add_test(NAME agc_buffer_pool_small_device_shared COMMAND agc_buffer_pool_tests --small-device-retention)
+    set_tests_properties(agc_buffer_pool_small_device_shared PROPERTIES ENVIRONMENT
+        "APS5_STAGING_POOL_MIB=256;APS5_BUFFER_POOL_SHARED=1")
+    add_test(NAME agc_buffer_pool_small_device_no_staging COMMAND agc_buffer_pool_tests --small-device-retention)
+    set_tests_properties(agc_buffer_pool_small_device_no_staging PROPERTIES ENVIRONMENT
+        "APS5_STAGING_POOL_MIB=0")
     if(WIN32)
-        set_tests_properties(agc_buffer_pool PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_buffer_pool agc_buffer_pool_small_device_shared agc_buffer_pool_small_device_no_staging PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 endif()
@@ -1003,6 +1137,7 @@ add_custom_command(
 
 add_test_executable(agc_driver_recorder_tests EXCLUDE_FROM_ALL
         tests/Recorder.cpp
+        tests/HostImportLifetime.cpp
         tests/ResidentPresent.cpp
         ${agcTriangleHeaders}
         ${agcSampleLodShaderHeader}
@@ -1029,6 +1164,11 @@ set_target_properties(agc_driver_recorder_tests PROPERTIES CXX_EXTENSIONS OFF)
 configure_windows_unwind(agc_driver_recorder_tests)
 if(BUILD_TESTING)
     set_target_properties(agc_driver_recorder_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_recorder_device_selection COMMAND agc_driver_recorder_tests --device-selection-only)
+    if(WIN32)
+        set_tests_properties(agc_recorder_device_selection PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_pipeline_cache COMMAND agc_driver_recorder_tests --pipeline-cache-only)
     add_test(NAME agc_pipeline_cache_file COMMAND agc_driver_recorder_tests --pipeline-cache-file-only)
     add_test(NAME agc_pipeline_library COMMAND agc_driver_recorder_tests --pipeline-library-only)
@@ -1051,7 +1191,14 @@ if(BUILD_TESTING)
     if(WIN32)
         add_test(NAME agc_import_watch_other_family COMMAND agc_driver_recorder_tests --import-watch-other-family-only)
         set_tests_properties(agc_import_watch_other_family PROPERTIES SKIP_RETURN_CODE 77)
-        set_tests_properties(agc_import_watch agc_import_watch_other_family PROPERTIES ENVIRONMENT_MODIFICATION
+        add_test(NAME agc_import_settle_deferred COMMAND agc_driver_recorder_tests --deferred-settle-only)
+        add_test(NAME agc_import_settle_deferred_other_family COMMAND agc_driver_recorder_tests --deferred-settle-other-family-only)
+        add_test(NAME agc_import_settle_first_use COMMAND agc_driver_recorder_tests --settle-first-use-only)
+        add_test(NAME agc_import_settle_earlier_import COMMAND agc_driver_recorder_tests --settle-earlier-import-only)
+        add_test(NAME agc_import_settle_retire COMMAND agc_driver_recorder_tests --settle-retire-only)
+        add_test(NAME agc_import_settle_storage_image COMMAND agc_driver_recorder_tests --settle-storage-image-only)
+        set_tests_properties(agc_import_settle_deferred agc_import_settle_deferred_other_family agc_import_settle_first_use agc_import_settle_earlier_import agc_import_settle_retire agc_import_settle_storage_image PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 60)
+        set_tests_properties(agc_import_watch agc_import_watch_other_family agc_import_settle_deferred agc_import_settle_deferred_other_family agc_import_settle_first_use agc_import_settle_earlier_import agc_import_settle_retire agc_import_settle_storage_image PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_unchanged_cpu_stamps COMMAND agc_driver_recorder_tests --unchanged-cpu-stamps-only)
@@ -1074,6 +1221,12 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_storage_upload_lifetime COMMAND agc_driver_recorder_tests --storage-upload-lifetime-only)
+    add_test(NAME agc_sampled_storage_base_level_usage COMMAND agc_driver_recorder_tests --sampled-storage-base-level-usage-only)
+    set_tests_properties(agc_sampled_storage_base_level_usage PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_sampled_storage_base_level_usage PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_storage_upload_failure COMMAND agc_driver_recorder_tests --storage-upload-failure-only)
     set_tests_properties(agc_storage_upload_lifetime agc_storage_upload_failure PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
     if(WIN32)
@@ -1166,10 +1319,28 @@ if(BUILD_TESTING)
     set_tests_properties(agc_shadowed_key_stores PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
     add_test(NAME agc_sample_dumps COMMAND agc_driver_recorder_tests --sample-dumps-only)
     set_tests_properties(agc_sample_dumps PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    add_test(NAME agc_draw_descriptors COMMAND agc_driver_recorder_tests --draw-descriptors-only)
+    set_tests_properties(agc_draw_descriptors PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 60)
     if(WIN32)
-        set_tests_properties(agc_pending_key_stores agc_shadowed_key_stores agc_sample_dumps PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_pending_key_stores agc_shadowed_key_stores agc_sample_dumps agc_draw_descriptors PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_driver_host_import_lifetime COMMAND agc_driver_recorder_tests --host-import-lifetime-only)
+    add_test(NAME agc_driver_host_import_mutation_race COMMAND agc_driver_recorder_tests --host-import-mutation-race-only)
+    set_tests_properties(agc_driver_host_import_lifetime agc_driver_host_import_mutation_race PROPERTIES
+        ENVIRONMENT "APS5_PIN_WAIT_MS=2000" SKIP_RETURN_CODE 77 TIMEOUT 30)
+    add_test(NAME agc_driver_unit_shadow COMMAND agc_driver_recorder_tests --unit-shadow-only)
+    add_test(NAME agc_driver_unit_shadow_budget COMMAND agc_driver_recorder_tests --unit-shadow-only)
+    set_tests_properties(agc_driver_unit_shadow PROPERTIES
+        ENVIRONMENT "APS5_PIN_WAIT_MS=2000" SKIP_RETURN_CODE 77 TIMEOUT 30)
+    set_tests_properties(agc_driver_unit_shadow_budget PROPERTIES
+        ENVIRONMENT "APS5_PIN_WAIT_MS=2000;APS5_UNIT_SHADOW_MIB=8" SKIP_RETURN_CODE 77 TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_driver_host_import_lifetime agc_driver_host_import_mutation_race agc_driver_unit_shadow agc_driver_unit_shadow_budget PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+    set_property(TEST agc_driver_host_import_lifetime APPEND PROPERTY ENVIRONMENT_MODIFICATION
+        "APS5_RELEASE_UNDER_LOCK=unset:" "APS5_RELEASE_ON_UNLOCK=unset:")
 endif()
 
 add_test_executable(agc_recorder_kept_bytes_tests EXCLUDE_FROM_ALL tests/RecorderKeptBytes.cpp)
@@ -1186,6 +1357,23 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_command_batch_failure_tests EXCLUDE_FROM_ALL
+        tests/CommandBatch.cpp tests/CommandBatchFailure.cpp Graphics/src/Resources.cpp tests/AllocationFailure.cpp)
+target_include_directories(agc_command_batch_failure_tests PRIVATE
+        ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
+target_link_libraries(agc_command_batch_failure_tests PRIVATE libSceAgcDriver libc)
+set_target_properties(agc_command_batch_failure_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_command_batch_failure_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_command_batch_failure_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_command_batch_failure COMMAND agc_command_batch_failure_tests)
+    set_tests_properties(agc_command_batch_failure PROPERTIES TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_command_batch_failure PROPERTIES
+                ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_host_import_retire_tests EXCLUDE_FROM_ALL tests/HostImportRetireHold.cpp)
 target_include_directories(agc_host_import_retire_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
 target_link_libraries(agc_host_import_retire_tests PRIVATE libSceAgcDriver libc)
@@ -1243,6 +1431,48 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_performance_overlay_statistics_tests EXCLUDE_FROM_ALL
+        tests/PerformanceOverlayStatistics.cpp
+)
+target_include_directories(agc_performance_overlay_statistics_tests PRIVATE
+        ${LIBS_INCLUDE_DIR}
+        ${CMAKE_SOURCE_DIR}/core/shader/recompiler
+        ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}
+)
+target_link_libraries(agc_performance_overlay_statistics_tests PRIVATE
+        libSceAgcDriver
+        libc
+)
+set_target_properties(agc_performance_overlay_statistics_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_performance_overlay_statistics_tests)
+
+add_test_executable(agc_performance_overlay_tests EXCLUDE_FROM_ALL
+        tests/PerformanceOverlay.cpp
+)
+target_include_directories(agc_performance_overlay_tests PRIVATE
+        ${LIBS_INCLUDE_DIR}
+        ${CMAKE_SOURCE_DIR}/core/shader/recompiler
+        ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}
+)
+target_link_libraries(agc_performance_overlay_tests PRIVATE
+        libSceAgcDriver
+        SDL2-static
+        libc
+)
+set_target_properties(agc_performance_overlay_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_performance_overlay_tests)
+
+if(BUILD_TESTING)
+    set_target_properties(agc_performance_overlay_statistics_tests agc_performance_overlay_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_performance_overlay_statistics COMMAND agc_performance_overlay_statistics_tests)
+    add_test(NAME agc_performance_overlay COMMAND agc_performance_overlay_tests)
+    set_tests_properties(agc_performance_overlay PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_performance_overlay_statistics agc_performance_overlay PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_driver_pm4_tests EXCLUDE_FROM_ALL
         tests/Pm4.cpp
 )
@@ -1275,6 +1505,22 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL
         tests/DepthSurfaceReuse.cpp
 )
 
+if(BUILD_TESTING)
+    add_test_executable(agc_descriptor_cache_failure_tests
+        tests/DescriptorCacheFailure.cpp
+        Graphics/src/ShaderResources.cpp)
+    target_include_directories(agc_descriptor_cache_failure_tests PRIVATE
+        $<TARGET_PROPERTY:libSceAgcDriver,INCLUDE_DIRECTORIES>)
+    target_link_libraries(agc_descriptor_cache_failure_tests PRIVATE libSceAgcDriver shader_recompiler_agc_driver libc libkernel)
+    configure_windows_unwind(agc_descriptor_cache_failure_tests)
+    add_test(NAME agc_descriptor_cache_failure COMMAND agc_descriptor_cache_failure_tests)
+    set_tests_properties(agc_descriptor_cache_failure PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(agc_descriptor_cache_failure PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 target_include_directories(agc_driver_graphics_tests PRIVATE
         ${LIBS_INCLUDE_DIR}
         $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>
@@ -1375,14 +1621,33 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources)
     configure_windows_unwind(${agcExecutionTest})
     list(APPEND agcExecutionTests ${agcExecutionTest})
 endforeach()
+if(BUILD_TESTING)
+    add_test(NAME agc_driver_address_passes_targets_split COMMAND agc_driver_address_passes_tests --targets)
+    set_tests_properties(agc_driver_address_passes_targets_split PROPERTIES ENVIRONMENT "APS5_NO_ADDRESS_READ_PASSES=1" SKIP_RETURN_CODE 77 TIMEOUT 60)
+    if(WIN32)
+        set_tests_properties(agc_driver_address_passes_targets_split PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
 foreach(agcExecutionTestName IN LISTS ANYPS5_AGC_EXECUTION_TESTS)
     if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/execution/${agcExecutionTestName}.cpp)
         message(FATAL_ERROR "ANYPS5_AGC_EXECUTION_TESTS names ${agcExecutionTestName}, which has no source in tests/execution")
     endif()
 endforeach()
+if(ANYPS5_ENABLE_SPIRV_TOOLS)
+    target_compile_definitions(agc_driver_stencil_null_feedback_tests PRIVATE ANYPS5_ENABLE_SPIRV_TOOLS=1)
+    target_link_libraries(agc_driver_stencil_null_feedback_tests PRIVATE SPIRV-Tools-static)
+endif()
+
+if(NOT ANYPS5_AGC_EXECUTION_TESTS OR "PooledBufferRange" IN_LIST ANYPS5_AGC_EXECUTION_TESTS)
+    target_sources(agc_driver_pooled_buffer_range_tests PRIVATE
+            Graphics/src/BufferPool.cpp Graphics/src/Resources.cpp tests/AllocationFailure.cpp tests/CommandBatchFailure.cpp)
+endif()
 
 target_link_libraries(agc_driver_gpu_map_on_fiber_tests PRIVATE libSceFiber)
 
+target_link_libraries(agc_driver_queue_zero_late_interrupt_tests PRIVATE libkernel)
+
 target_link_libraries(agc_driver_preparation_scope_settle_tests PRIVATE libSceAgc)
 
 target_include_directories(agc_driver_alpha_to_coverage_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
@@ -1390,19 +1655,46 @@ target_include_directories(agc_driver_alpha_to_coverage_tests PRIVATE $<TARGET_P
 target_include_directories(agc_driver_sample_multisample_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
 if(BUILD_TESTING)
+    add_test(NAME agc_driver_store_pending_identity COMMAND agc_driver_store_over_untracked_tests --pending-identity)
+    set_tests_properties(agc_driver_store_pending_identity PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_store_pending_identity PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
+if(BUILD_TESTING)
+    add_test(NAME agc_driver_native_msaa_vertex_input COMMAND agc_driver_sample_multisample_target_tests --native-vertex-input)
+    set_tests_properties(agc_driver_native_msaa_vertex_input PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" TIMEOUT 30 SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_native_msaa_vertex_input PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_undithered_alpha_to_coverage_state COMMAND agc_driver_graphics_tests --undithered-alpha-to-coverage-state)
+    add_test(NAME agc_driver_msaa_dcc_state COMMAND agc_driver_graphics_tests --multisample-dcc-state)
     set_tests_properties(agc_driver_undithered_alpha_to_coverage_state PROPERTIES
             ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_driver_alpha_to_coverage_state COMMAND agc_driver_graphics_tests --alpha-to-coverage-state)
     set_tests_properties(agc_driver_alpha_to_coverage_state PROPERTIES
             ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_driver_dithered_alpha_to_coverage COMMAND agc_driver_alpha_to_coverage_tests --dithered8)
+    set(multisampledDccClearTests)
+    foreach(samples IN ITEMS 2 4 8)
+        set(testName "agc_driver_msaa_dcc_clear_${samples}")
+        add_test(NAME ${testName} COMMAND agc_driver_alpha_to_coverage_tests --dcc-fast-clear ${samples})
+        list(APPEND multisampledDccClearTests ${testName})
+    endforeach()
+    set_tests_properties(agc_driver_msaa_dcc_state ${multisampledDccClearTests} PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    set_tests_properties(${multisampledDccClearTests} PROPERTIES SKIP_RETURN_CODE 77)
     set_tests_properties(agc_driver_dithered_alpha_to_coverage PROPERTIES
             ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     if(WIN32)
-        set_tests_properties(agc_driver_alpha_to_coverage_state agc_driver_undithered_alpha_to_coverage_state PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_driver_alpha_to_coverage_state agc_driver_undithered_alpha_to_coverage_state agc_driver_msaa_dcc_state PROPERTIES ENVIRONMENT_MODIFICATION
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
-        set_tests_properties(agc_driver_dithered_alpha_to_coverage PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_driver_dithered_alpha_to_coverage ${multisampledDccClearTests} PROPERTIES ENVIRONMENT_MODIFICATION
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 endif()
@@ -1414,10 +1706,16 @@ target_include_directories(agc_driver_reversed_order_blend_tests PRIVATE $<TARGE
 
 target_include_directories(agc_driver_swap_alt_blend_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_packed_float_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
+target_link_libraries(agc_driver_job_manager_compute_tests PRIVATE libSceJobManager libSceAgc libkernel)
+
 target_include_directories(agc_driver_packed_unorm_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
 target_include_directories(agc_driver_color1555_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_color565_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
 foreach(stage IN ITEMS vert frag)
     set(agcProvokingShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/ProvokingVertex.${stage})
     set(agcProvokingShaderSpv ${agcTestShaderGeneratedRoot}/ProvokingVertex_${stage}.spv)
@@ -1438,8 +1736,12 @@ target_include_directories(agc_driver_provoking_vertex_tests PRIVATE ${agcTestSh
 
 target_include_directories(agc_driver_prepared_exports_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_pipeline_journal_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
 target_include_directories(agc_driver_buffer_descriptor_runtime_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_include_directories(agc_driver_shutdown_threads_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
+
 target_include_directories(agc_driver_prepared_publication_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
 target_include_directories(agc_driver_prepared_graphics_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
@@ -1447,6 +1749,8 @@ target_link_libraries(agc_driver_interpolation_modes_tests PRIVATE shader_recomp
 
 target_include_directories(agc_driver_prepared_unregistered_graphics_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
+target_link_libraries(agc_driver_guest_memory_lifetime_tests PRIVATE libkernel)
+
 target_include_directories(agc_driver_prepared_shaders_tests PRIVATE
         $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>
 )
@@ -1498,8 +1802,10 @@ if(BUILD_TESTING)
     add_test(NAME agc_shader_layout_pair_off COMMAND agc_shader_memory_tests --layout-pair)
     set_tests_properties(agc_shader_layout_pair PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     set_tests_properties(agc_shader_layout_pair_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_LAYOUT_PAIR=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_shader_entry_lane_proof_budget COMMAND agc_shader_memory_tests --entry-lane-proof-budget)
+    set_tests_properties(agc_shader_entry_lane_proof_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     if(WIN32)
-        set_tests_properties(agc_shader_bindless agc_shader_srt_read_patching agc_shader_type_zero_image_materialization agc_shader_frontend_pair agc_shader_frontend_pair_off agc_shader_layout_pair agc_shader_layout_pair_off PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_shader_bindless agc_shader_srt_read_patching agc_shader_type_zero_image_materialization agc_shader_frontend_pair agc_shader_frontend_pair_off agc_shader_layout_pair agc_shader_layout_pair_off agc_shader_entry_lane_proof_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     foreach(agcTest IN ITEMS agc_vertex_input_tests agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests agc_formatted_buffer_byte_stores_tests ${agcExecutionTests} agc_fma_mix_once_tests)
         if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests" OR agcTest STREQUAL "agc_driver_pipeline_optimized_readback_tests")
@@ -1519,23 +1825,60 @@ if(BUILD_TESTING)
     if(WIN32)
         set_tests_properties(agc_driver_partially_resident_pending_stores PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_shader_disk_cache_large_file COMMAND agc_shader_disk_cache_tests --large-file)
+    set_tests_properties(agc_shader_disk_cache_large_file PROPERTIES SKIP_RETURN_CODE 77 RUN_SERIAL TRUE WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 240)
+    if(WIN32)
+        set_tests_properties(agc_shader_disk_cache_large_file PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     if(WIN32)
         add_test(NAME agc_driver_untracked_snapshot_hint COMMAND agc_driver_partially_resident_snapshot_tests --untracked-hint)
         set_tests_properties(agc_driver_untracked_snapshot_hint PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30
             ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    set_tests_properties(agc_driver_registration_trace PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TRACE_FRAMES=0:1;APS5_TRACE_EVENTS=64" TIMEOUT 30)
+    set_property(TEST agc_driver_registration_trace APPEND PROPERTY ENVIRONMENT_MODIFICATION
+            "APS5_TRACE_SECONDS=unset:" "APS5_TRACE_SUMMARY_EVERY=unset:")
+    add_test(NAME agc_driver_registration_trace_disabled COMMAND ${CMAKE_COMMAND} -E env
+            --unset=APS5_TRACE_FRAMES --unset=APS5_TRACE_SECONDS --unset=APS5_TRACE_SUMMARY_EVERY
+            $<TARGET_FILE:agc_driver_registration_trace_tests> disabled)
+    set_tests_properties(agc_driver_registration_trace_disabled PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_driver_registration_trace_disabled PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_sparse_buffer_holes COMMAND agc_driver_graphics_tests --sparse-buffer-holes)
     set_tests_properties(agc_driver_native_multisample_depth PROPERTIES TIMEOUT 30)
     add_test(NAME agc_driver_native_depth_retirement COMMAND agc_driver_native_multisample_depth_tests --retirement)
     set_tests_properties(agc_driver_native_depth_retirement PROPERTIES TIMEOUT 30 ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    add_test(NAME agc_driver_native_depth_clears COMMAND agc_driver_native_multisample_depth_tests --clears)
+    add_test(NAME agc_driver_native_depth_clear_refusals COMMAND agc_driver_native_multisample_depth_tests --clear-refusals)
+    set_tests_properties(agc_driver_native_depth_clears agc_driver_native_depth_clear_refusals PROPERTIES TIMEOUT 30 ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     if(WIN32)
         set_tests_properties(agc_driver_native_depth_retirement PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_driver_native_depth_clears agc_driver_native_depth_clear_refusals PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     set_tests_properties(agc_sparse_buffer_holes PROPERTIES TIMEOUT 30)
+    add_test(NAME agc_pipeline_journal COMMAND agc_driver_graphics_tests --pipeline-journal)
+    add_test(NAME agc_pipeline_library_clear_waits_for_build COMMAND agc_driver_graphics_tests --pipeline-library-clear-waits-for-build)
+    add_test(NAME agc_pipeline_store_lazy_buffer_pool COMMAND agc_driver_graphics_tests --pipeline-store-lazy-buffer-pool)
+    set_tests_properties(agc_pipeline_journal PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    set_tests_properties(agc_pipeline_library_clear_waits_for_build PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    set_tests_properties(agc_pipeline_store_lazy_buffer_pool PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_pipeline_journal agc_pipeline_library_clear_waits_for_build agc_pipeline_store_lazy_buffer_pool PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     set_tests_properties(agc_driver_sparse_buffer_holes PROPERTIES TIMEOUT 30)
     if(WIN32)
         set_tests_properties(agc_sparse_buffer_holes PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_driver_lds_lock_budget COMMAND agc_driver_lds_mixed_width_atomics_tests --lock-budget)
+    set_tests_properties(agc_driver_lds_lock_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_driver_lds_lock_budget PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+
     add_test(NAME agc_driver_emulated_color_compare_srgb_decode COMMAND agc_driver_emulated_color_compare_tests)
     set_tests_properties(agc_driver_emulated_color_compare_srgb_decode PROPERTIES
             ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_SRGB_SHADER_DECODE=1"
@@ -1545,6 +1888,14 @@ if(BUILD_TESTING)
         set_tests_properties(agc_driver_emulated_color_compare_srgb_decode PROPERTIES ENVIRONMENT_MODIFICATION
                 "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_driver_image_srgb_sample_copy_forced COMMAND agc_driver_image_srgb_sample_copy_tests)
+    set_tests_properties(agc_driver_image_srgb_sample_copy_forced PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_SRGB_SHADER_DECODE=copy"
+            SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_image_srgb_sample_copy_forced PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_shared_write_partial_collect COMMAND agc_shared_write_tests --partial-collect)
     set_tests_properties(agc_shared_write_partial_collect PROPERTIES SKIP_RETURN_CODE 77 ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
     if(WIN32)
@@ -1622,6 +1973,48 @@ if(BUILD_TESTING)
         set_tests_properties(agc_driver_uint8x4_color_target_no_resident PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    set(agcGuestMemoryLifetimeTests agc_driver_guest_memory_lifetime)
+    foreach(mode IN ITEMS copy direct locked heap heap_uncached)
+        set(testName agc_driver_guest_memory_lifetime_${mode})
+        if(mode STREQUAL "heap_uncached")
+            add_test(NAME ${testName} COMMAND agc_driver_guest_memory_lifetime_tests heap)
+        else()
+            add_test(NAME ${testName} COMMAND agc_driver_guest_memory_lifetime_tests ${mode})
+        endif()
+        list(APPEND agcGuestMemoryLifetimeTests ${testName})
+    endforeach()
+    set_tests_properties(${agcGuestMemoryLifetimeTests} PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIN_WAIT_MS=1000;APS5_HEAP_CACHE_MIB=4"
+            SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    set_tests_properties(agc_driver_guest_memory_lifetime_heap_uncached PROPERTIES
+            ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIN_WAIT_MS=1000;APS5_HEAP_CACHE_MIB=0")
+    if(WIN32)
+        set_tests_properties(${agcGuestMemoryLifetimeTests} PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+    if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND Python3_Interpreter_FOUND)
+        add_library(agc_guest_memory_lifetime_fixture SHARED tests/GuestMemoryLifetimeFixture.cpp)
+        target_include_directories(agc_guest_memory_lifetime_fixture PRIVATE
+                ${LIBS_INCLUDE_DIR}
+                ${CMAKE_SOURCE_DIR}/core/shader/recompiler
+                ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
+                ${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include)
+        target_link_libraries(agc_guest_memory_lifetime_fixture PRIVATE libSceAgcDriver SDL2-static libc)
+        add_dependencies(agc_guest_memory_lifetime_fixture relinker libkernel)
+        set_target_properties(agc_guest_memory_lifetime_fixture PROPERTIES
+                OUTPUT_NAME GuestMemoryFixture SUFFIX ".prx" CXX_EXTENSIONS OFF
+                BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH "$ORIGIN")
+        add_test(NAME agc_driver_guest_memory_lifetime_converted COMMAND ${Python3_EXECUTABLE}
+                ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_guest_memory_lifetime.py
+                ${CMAKE_SOURCE_DIR}/core/relinker/relinker/tests
+                $<TARGET_FILE:relinker>
+                $<TARGET_FILE:agc_guest_memory_lifetime_fixture>
+                $<TARGET_FILE:libkernel>
+                $<TARGET_FILE:libc>
+                $<TARGET_FILE:libSceAgcDriver>)
+        set_tests_properties(agc_driver_guest_memory_lifetime_converted PROPERTIES
+                ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIN_WAIT_MS=1000" SKIP_RETURN_CODE 77 TIMEOUT 30)
+    endif()
     set_tests_properties(agc_driver_flat_store PROPERTIES RUN_SERIAL TRUE)
     set_target_properties(agc_driver_flat_apertures_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
     foreach(flatAperturesPart RANGE 13)
@@ -1633,6 +2026,12 @@ if(BUILD_TESTING)
         endif()
     endforeach()
     set_tests_properties(agc_driver_storage_cache_budget PROPERTIES TIMEOUT 30)
+    add_test(NAME agc_driver_storage_cache_retained_batch COMMAND agc_driver_storage_cache_budget_tests --retained-batch)
+    set_tests_properties(agc_driver_storage_cache_retained_batch PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_storage_cache_retained_batch PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_sampled_cache_budget_memory COMMAND agc_driver_sampled_cache_budget_tests memory)
     set_tests_properties(agc_driver_sampled_cache_budget_memory PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77)
     add_test(NAME agc_driver_resource_cache_storage_admission COMMAND agc_driver_sampled_cache_budget_tests resource-admission)
@@ -1653,13 +2052,31 @@ if(BUILD_TESTING)
     endif()
     add_test(NAME agc_driver_pm4_failure COMMAND agc_driver_pm4_tests failure)
     set_tests_properties(agc_driver_pm4_failure PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 30)
+    set(pm4RingCases "ring|APS5_PM4_RING=4" "ring-empty|APS5_PM4_RING=16" "ring-disabled|APS5_PM4_RING=0" "ring-invalid|APS5_PM4_RING=4x" "ring-report|APS5_PM4_RING=2" "stall-watchdog|APS5_STALL_WATCHDOG_SECS=1"
+            "stall-watchdog-invalid|APS5_STALL_WATCHDOG_SECS=1x" "stall-watchdog-rewind|APS5_STALL_WATCHDOG_SECS=1" "stall-watchdog-order|APS5_STALL_WATCHDOG_SECS=1")
+    if(WIN32)
+        list(APPEND pm4RingCases "ring-report-heap|APS5_PM4_RING=2")
+    endif()
+    foreach(pm4RingCase IN LISTS pm4RingCases)
+        string(REPLACE "|" ";" pm4RingCase "${pm4RingCase}")
+        list(GET pm4RingCase 0 pm4RingMode)
+        list(GET pm4RingCase 1 pm4RingEnvironment)
+        string(REPLACE "-" "_" pm4RingTest "agc_driver_pm4_${pm4RingMode}")
+        add_test(NAME ${pm4RingTest} COMMAND agc_driver_pm4_tests ${pm4RingMode})
+        set_tests_properties(${pm4RingTest} PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;${pm4RingEnvironment}" SKIP_RETURN_CODE 77 TIMEOUT 30)
+        if(WIN32)
+            set_tests_properties(${pm4RingTest} PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        endif()
+    endforeach()
     add_test(NAME agc_descriptor_pool_reset_failure COMMAND agc_driver_graphics_tests --descriptor-pool-reset-failure)
+    add_test(NAME agc_descriptor_cache COMMAND agc_driver_graphics_tests --descriptor-cache)
     add_test(NAME agc_driver_address_space_teardown COMMAND agc_driver_graphics_tests --address-space-teardown)
     set_tests_properties(agc_driver_address_space_teardown PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_driver_pm4_failure_inline COMMAND agc_driver_pm4_tests failure)
     set_tests_properties(agc_driver_pm4_failure_inline PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=0" SKIP_RETURN_CODE 77)
     if(WIN32)
-        set_tests_properties(agc_driver_pm4_failure agc_driver_pm4_failure_inline agc_descriptor_pool_reset_failure agc_driver_address_space_teardown PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_driver_pm4_failure agc_driver_pm4_failure_inline agc_descriptor_pool_reset_failure agc_descriptor_cache agc_driver_address_space_teardown PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     if(NOT WIN32)
@@ -1695,6 +2112,12 @@ if(BUILD_TESTING)
         set_tests_properties(agc_driver_stencil_replay_no_gpl PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_driver_pipelined_dispatch_item COMMAND agc_driver_pipelined_dispatch_tests)
+    set_tests_properties(agc_driver_pipelined_dispatch_item PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINE_DISPATCH=1" SKIP_RETURN_CODE 77)
+    if(WIN32)
+        set_tests_properties(agc_driver_pipelined_dispatch_item PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
 
     foreach(agcTest IN ITEMS agc_spirv_narrow_constant_store_tests agc_spirv13_image_tests agc_spirv_optimizer_tests agc_driver_constant_store_tests)
         if(NOT TARGET ${agcTest})
@@ -1726,7 +2149,7 @@ if(BUILD_TESTING)
             endif()
         endforeach()
     endforeach()
-    foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable registered_workgroup registered_workgroup_sync)
+    foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable registered_workgroup registered_workgroup_sync lazy lazy_sync)
         set(preparedTest "agc_driver_prepared_shaders_${preparedMode}")
         if(preparedMode STREQUAL "indirect")
             add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --indirect)
@@ -1734,6 +2157,8 @@ if(BUILD_TESTING)
             add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --deferred)
         elseif(preparedMode STREQUAL "deferred_undecodable")
             add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --deferred-undecodable)
+        elseif(preparedMode STREQUAL "lazy" OR preparedMode STREQUAL "lazy_sync")
+            add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --lazy)
         elseif(preparedMode STREQUAL "registered_workgroup" OR preparedMode STREQUAL "registered_workgroup_sync")
             add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --registered-workgroup)
         elseif(preparedMode STREQUAL "debug_failure")
@@ -1747,6 +2172,11 @@ if(BUILD_TESTING)
             set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_NO_DISPATCH_CACHE=1")
         elseif(preparedMode STREQUAL "registered_workgroup_sync")
             set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_SYNC_SHADER_PREPARE=1")
+        elseif(preparedMode STREQUAL "lazy" OR preparedMode STREQUAL "lazy_sync")
+            set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_LAZY_SHADER_PREPARE=1" "APS5_SUSPEND_DRAIN=1")
+            if(preparedMode STREQUAL "lazy_sync")
+                set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_SYNC_SHADER_PREPARE=1")
+            endif()
         elseif(preparedMode STREQUAL "debug" OR preparedMode STREQUAL "debug_failure")
             set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_PROFILE_DRAW=1" "APS5_NO_UNLOCKED_DEVICE=1")
         endif()
@@ -1754,6 +2184,17 @@ if(BUILD_TESTING)
             set_tests_properties(${preparedTest} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
         endif()
     endforeach()
+    add_test(NAME agc_driver_shutdown_threads_joined COMMAND agc_driver_shutdown_threads_tests --joined)
+    add_test(NAME agc_driver_shutdown_threads_compare_helpers COMMAND agc_driver_shutdown_threads_tests --compare-helpers)
+    add_test(NAME agc_driver_shutdown_threads_draw_committer COMMAND agc_driver_shutdown_threads_tests --draw-committer)
+    add_test(NAME agc_driver_shutdown_threads_shader_cache_writer COMMAND agc_driver_shutdown_threads_tests --shader-cache-writer)
+    add_test(NAME agc_driver_shutdown_threads_late_shader_cache COMMAND agc_driver_shutdown_threads_tests --late-shader-cache)
+    add_test(NAME agc_driver_shutdown_threads_async_log_held COMMAND agc_driver_shutdown_threads_tests --async-log-held)
+    add_test(NAME agc_driver_shutdown_threads_exit_after_start COMMAND agc_driver_shutdown_threads_tests --exit-after-start)
+    set_tests_properties(agc_driver_shutdown_threads_joined agc_driver_shutdown_threads_compare_helpers agc_driver_shutdown_threads_draw_committer agc_driver_shutdown_threads_shader_cache_writer agc_driver_shutdown_threads_late_shader_cache agc_driver_shutdown_threads_async_log_held agc_driver_shutdown_threads_exit_after_start PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_driver_shutdown_threads_joined agc_driver_shutdown_threads_compare_helpers agc_driver_shutdown_threads_draw_committer agc_driver_shutdown_threads_shader_cache_writer agc_driver_shutdown_threads_late_shader_cache agc_driver_shutdown_threads_async_log_held agc_driver_shutdown_threads_exit_after_start PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_compute_user_data_calls_registered COMMAND agc_driver_compute_user_data_calls_tests --registered)
     set_tests_properties(agc_driver_compute_user_data_calls_registered PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     if(WIN32)
@@ -1868,6 +2309,25 @@ endif()
 set_target_properties(agc_shader_replay PROPERTIES CXX_EXTENSIONS OFF)
 configure_windows_unwind(agc_shader_replay)
 
+if(BUILD_TESTING)
+    add_test_executable(agc_shader_replay_requests_tests tests/ShaderReplayRequests.cpp)
+    target_link_libraries(agc_shader_replay_requests_tests PRIVATE shader_recompiler_agc_driver libc)
+    set_target_properties(agc_shader_replay_requests_tests PROPERTIES CXX_EXTENSIONS OFF)
+    configure_windows_unwind(agc_shader_replay_requests_tests)
+    add_dependencies(agc_shader_replay_requests_tests agc_shader_replay)
+    set(agcShaderReplayCapture "${CMAKE_CURRENT_BINARY_DIR}/shader-replay-capture")
+    add_test(NAME agc_shader_replay_requests COMMAND agc_shader_replay_requests_tests "${agcShaderReplayCapture}")
+    set_tests_properties(agc_shader_replay_requests PROPERTIES FIXTURES_SETUP agc_shader_replay_capture TIMEOUT 30)
+    add_test(NAME agc_shader_replay_tool COMMAND ${CMAKE_COMMAND} -DTOOL=$<TARGET_FILE:agc_shader_replay> "-DCAPTURE=${agcShaderReplayCapture}"
+            -P ${CMAKE_CURRENT_SOURCE_DIR}/tests/ShaderReplayTool.cmake)
+    set_tests_properties(agc_shader_replay_tool PROPERTIES FIXTURES_REQUIRED agc_shader_replay_capture TIMEOUT 30)
+    set_tests_properties(agc_shader_replay_requests agc_shader_replay_tool PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1")
+    if(WIN32)
+        set_tests_properties(agc_shader_replay_requests agc_shader_replay_tool PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_executable(agc_pm4_replay EXCLUDE_FROM_ALL
         tools/Pm4Replay.cpp
         tools/Pm4Analysis.cpp
@@ -2341,3 +2801,22 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 endif()
+
+add_test_executable(agc_constant_select_tests EXCLUDE_FROM_ALL tests/ConstantSelect.cpp)
+target_link_libraries(agc_constant_select_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_constant_select_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_constant_select_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_constant_select_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_constant_select COMMAND agc_constant_select_tests)
+    if(WIN32)
+        set_tests_properties(agc_constant_select PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
+if(BUILD_TESTING)
+    get_directory_property(agcRegisteredTests TESTS)
+    list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal|agc_driver_pipeline_journal)$")
+    set_property(TEST ${agcRegisteredTests} APPEND PROPERTY ENVIRONMENT "APS5_NO_PIPELINE_JOURNAL=1")
+endif()

core/libs/prx/libSceAgcDriver/Execution/include/AsyncLog.hpp

@@ -0,0 +1,418 @@
+#pragma once
+
+#include "prx/libc/include/Shutdown.hpp"
+#include "prx/libc/include/general/LogMacros.hpp"
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <deque>
+#include <initializer_list>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+namespace AgcDriver {
+
+struct LogLiteral {
+    const char* text;
+};
+
+namespace AsyncLogDetail {
+
+template<typename T>
+inline constexpr bool Deferrable = std::is_arithmetic_v<T> || std::is_enum_v<T> || std::is_same_v<T, LogLiteral>;
+
+inline const char* Unwrap(LogLiteral value) {
+    return value.text;
+}
+
+template<typename T>
+T Unwrap(T value) {
+    return value;
+}
+
+class Formatter {
+public:
+    virtual ~Formatter() = default;
+    virtual void Format(std::string& out) const = 0;
+};
+
+template<typename... TArgs>
+class Bound final : public Formatter {
+public:
+    Bound(const char* format, TArgs... args) : format(format), values(args...) {}
+
+    void Format(std::string& out) const override {
+        std::apply([&](const auto&... value) { Append(out, value...); }, values);
+    }
+
+private:
+    void Append(std::string& out, const auto&... value) const {
+        char small[512];
+        const int size = std::snprintf(small, sizeof(small), format, Unwrap(value)...);
+        if (size < 0) return;
+        if (static_cast<std::size_t>(size) < sizeof(small)) {
+            out.append(small, static_cast<std::size_t>(size));
+            return;
+        }
+        std::string large(static_cast<std::size_t>(size) + 1, '\0');
+        std::snprintf(large.data(), large.size(), format, Unwrap(value)...);
+        out.append(large.data(), static_cast<std::size_t>(size));
+    }
+
+    const char* format;
+    std::tuple<TArgs...> values;
+};
+
+struct Entry {
+    std::FILE* stream = nullptr;
+    long long micros = -1;
+    std::string text;
+    std::unique_ptr<Formatter> deferred;
+    std::size_t cost = 0;
+};
+
+}
+
+class AsyncLog {
+public:
+    using Clock = std::chrono::steady_clock;
+
+    static constexpr std::size_t Capacity = 16u * 1024u * 1024u;
+
+    static AsyncLog& Get() {
+        static AsyncLog* const instance = new AsyncLog();
+        return *instance;
+    }
+
+    int Write(std::FILE* stream, const char* text, std::size_t length) {
+        const auto start = Clock::now();
+        if (!asynchronous) return WriteNow(stream, text, length);
+        AsyncLogDetail::Entry entry;
+        entry.stream = stream;
+        entry.text.assign(text, length);
+        entry.cost = sizeof(entry) + length;
+        Enqueue(std::move(entry));
+        callerNanoseconds.fetch_add(Elapsed(start), std::memory_order_relaxed);
+        return 0;
+    }
+
+    template<typename... TArgs>
+    void Defer(std::FILE* stream, const char* format, TArgs... args) {
+        static_assert((AsyncLogDetail::Deferrable<TArgs> && ...), "deferred log arguments are numbers, enums or LogLiteral");
+        const auto start = Clock::now();
+        const auto micros = LogTimestampMicros();
+        auto bound = std::make_unique<AsyncLogDetail::Bound<TArgs...>>(format, args...);
+        if (!asynchronous) {
+            std::string text;
+            if (micros >= 0) AppendTimestamp(text, micros);
+            bound->Format(text);
+            WriteNow(stream, text.data(), text.size());
+            return;
+        }
+        AsyncLogDetail::Entry entry;
+        entry.stream = stream;
+        entry.micros = micros;
+        entry.deferred = std::move(bound);
+        entry.cost = sizeof(entry) + sizeof(AsyncLogDetail::Bound<TArgs...>);
+        Enqueue(std::move(entry));
+        callerNanoseconds.fetch_add(Elapsed(start), std::memory_order_relaxed);
+    }
+
+    bool Admit(const std::string& line) {
+        const auto now = Clock::now();
+        std::string summary;
+        {
+            std::lock_guard lock(coalesceMutex);
+            auto found = windows.find(line);
+            if (found == windows.end()) {
+                if (windows.size() >= CoalesceKeys) return true;
+                windows.emplace(line, Window{now, 1, 0});
+                return true;
+            }
+            auto& window = found->second;
+            if (now - window.first < CoalesceWindow) {
+                if (++window.seen <= PrintedRepeats) return true;
+                ++window.suppressed;
+                coalescedTotal.fetch_add(1, std::memory_order_relaxed);
+                return false;
+            }
+            summary = Summary(found->first, window.suppressed);
+            window = Window{now, 1, 0};
+        }
+        if (!summary.empty()) WriteNow(stderr, summary.data(), summary.size());
+        return true;
+    }
+
+    void Flush() {
+        if (!asynchronous) return;
+        std::unique_lock lock(mutex);
+        const auto target = queued.load(std::memory_order_relaxed);
+        idle.wait(lock, [&] { return finished.load(std::memory_order_relaxed) >= target; });
+    }
+
+    bool FlushFor(std::chrono::milliseconds limit) {
+        if (!asynchronous) return true;
+        const auto target = queued.load(std::memory_order_acquire);
+        const auto deadline = Clock::now() + limit;
+        while (finished.load(std::memory_order_acquire) < target) {
+            if (Clock::now() >= deadline) return false;
+            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }
+        return true;
+    }
+
+    static void FlushAtEnd() {
+        auto& log = Get();
+        if (log.abandoned.load(std::memory_order_acquire)) return;
+        if (!log.FlushFor(EndFlushLimit)) log.abandoned.store(true, std::memory_order_release);
+    }
+
+    std::unique_lock<std::mutex> Hold() {
+        return std::unique_lock(mutex);
+    }
+
+private:
+    struct Window {
+        Clock::time_point first;
+        std::uint64_t seen;
+        std::uint64_t suppressed;
+    };
+
+    static constexpr std::size_t CoalesceKeys = 128;
+    static constexpr std::uint64_t PrintedRepeats = 3;
+    static constexpr std::chrono::seconds CoalesceWindow{10};
+    static constexpr std::chrono::seconds StatsWindow{10};
+    static constexpr std::chrono::milliseconds Tick{500};
+    static constexpr std::chrono::milliseconds EndFlushLimit{2000};
+
+    AsyncLog() {
+        asynchronous = std::getenv("APS5_LOG_SYNC") == nullptr;
+        const char* stats = std::getenv("APS5_LOG_STATS");
+        statsEnabled = stats == nullptr || std::string(stats) != "0";
+        lastStats = Clock::now();
+        if (!asynchronous) return;
+        LogSink().store(&AsyncLog::Sink, std::memory_order_release);
+        std::atexit(&AsyncLog::FlushAtEnd);
+        InitializeWaits();
+        std::thread([this] { Run(); }).detach();
+        LibcSetEndFlush_nid_no_patch(&AsyncLog::FlushAtEnd);
+    }
+
+    void InitializeWaits() {
+        std::unique_lock lock(mutex);
+        for (auto* condition : {&wake, &space, &idle}) static_cast<void>(condition->wait_for(lock, std::chrono::nanoseconds(0)));
+    }
+
+    static int Sink(std::FILE* stream, const char* text, std::size_t length) {
+        return Get().Write(stream, text, length);
+    }
+
+    static std::uint64_t Elapsed(Clock::time_point start) {
+        return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(Clock::now() - start).count());
+    }
+
+    static void AppendTimestamp(std::string& out, long long micros) {
+        char prefix[64];
+        const int size = FormatLogTimestamp(prefix, sizeof(prefix), micros);
+        if (size > 0) out.append(prefix, static_cast<std::size_t>(size));
+    }
+
+    static std::string Summary(const std::string& line, std::uint64_t suppressed) {
+        if (suppressed == 0) return {};
+        std::string text = line;
+        while (!text.empty() && (text.back() == '\n' || text.back() == '\r')) text.pop_back();
+        text += " (repeated " + std::to_string(suppressed) + " more times in the last 10 s)\n";
+        return text;
+    }
+
+    static int WriteNow(std::FILE* stream, const char* text, std::size_t length) {
+        std::fwrite(text, 1, length, stream);
+        return std::fflush(stream);
+    }
+
+    void Enqueue(AsyncLogDetail::Entry&& entry) {
+        bool notify = false;
+        {
+            std::unique_lock lock(mutex);
+            space.wait(lock, [&] { return bytes == 0 || bytes + entry.cost <= Capacity; });
+            bytes += entry.cost;
+            peak = std::max(peak, bytes);
+            pending.push_back(std::move(entry));
+            queued.fetch_add(1, std::memory_order_release);
+            notify = sleeping;
+            sleeping = false;
+        }
+        if (notify) wake.notify_one();
+    }
+
+    void Run() noexcept {
+        std::deque<AsyncLogDetail::Entry> batch;
+        std::string buffer;
+        for (;;) {
+            {
+                std::unique_lock lock(mutex);
+                sleeping = true;
+                wake.wait_for(lock, Tick, [&] { return !pending.empty(); });
+                sleeping = false;
+                batch.swap(pending);
+                bytes = 0;
+            }
+            space.notify_all();
+            if (!batch.empty()) {
+                const auto start = Clock::now();
+                const auto count = batch.size();
+                try {
+                    Drain(batch, buffer);
+                } catch (...) {
+                    buffer.clear();
+                }
+                batch.clear();
+                loggerNanoseconds.fetch_add(Elapsed(start), std::memory_order_relaxed);
+                {
+                    std::lock_guard lock(mutex);
+                    finished.fetch_add(count, std::memory_order_release);
+                }
+                idle.notify_all();
+            }
+            try {
+                Housekeeping();
+            } catch (...) {
+            }
+        }
+    }
+
+    void Drain(std::deque<AsyncLogDetail::Entry>& batch, std::string& buffer) {
+        std::FILE* current = nullptr;
+        buffer.clear();
+        std::uint64_t outLines = 0;
+        std::uint64_t errLines = 0;
+        std::uint64_t written = 0;
+        const auto emit = [&] {
+            if (current != nullptr && !buffer.empty()) {
+                written += buffer.size();
+                WriteNow(current, buffer.data(), buffer.size());
+            }
+            buffer.clear();
+        };
+        for (auto& entry : batch) {
+            if (entry.stream != current) {
+                emit();
+                current = entry.stream;
+            }
+            if (entry.deferred) {
+                if (entry.micros >= 0) AppendTimestamp(buffer, entry.micros);
+                entry.deferred->Format(buffer);
+            } else {
+                buffer += entry.text;
+            }
+            (entry.stream == stderr ? errLines : outLines) += 1;
+            if (buffer.size() >= 65536) emit();
+        }
+        emit();
+        stdoutLines.fetch_add(outLines, std::memory_order_relaxed);
+        stderrLines.fetch_add(errLines, std::memory_order_relaxed);
+        loggedBytes.fetch_add(written, std::memory_order_relaxed);
+    }
+
+    void Housekeeping() {
+        const auto now = Clock::now();
+        std::vector<std::string> summaries;
+        {
+            std::lock_guard lock(coalesceMutex);
+            for (auto it = windows.begin(); it != windows.end();) {
+                if (now - it->second.first >= CoalesceWindow) {
+                    const auto summary = Summary(it->first, it->second.suppressed);
+                    if (!summary.empty()) summaries.push_back(summary);
+                    it = windows.erase(it);
+                } else {
+                    ++it;
+                }
+            }
+        }
+        for (const auto& summary : summaries) WriteNow(stderr, summary.data(), summary.size());
+        if (now - lastStats < StatsWindow) return;
+        const double seconds = std::chrono::duration<double>(now - lastStats).count();
+        lastStats = now;
+        const auto outLines = stdoutLines.exchange(0, std::memory_order_relaxed);
+        const auto errLines = stderrLines.exchange(0, std::memory_order_relaxed);
+        const auto written = loggedBytes.exchange(0, std::memory_order_relaxed);
+        const auto caller = callerNanoseconds.exchange(0, std::memory_order_relaxed);
+        const auto logger = loggerNanoseconds.exchange(0, std::memory_order_relaxed);
+        const auto coalesced = coalescedTotal.exchange(0, std::memory_order_relaxed);
+        std::size_t high = 0;
+        {
+            std::lock_guard lock(mutex);
+            high = peak;
+            peak = bytes;
+        }
+        if (!statsEnabled || outLines + errLines + coalesced == 0) return;
+        char line[512];
+        const int size = std::snprintf(line, sizeof(line), "[log] %.1f s: %llu lines (%llu stdout, %llu stderr), %.1f KiB written, %llu repeats coalesced; callers spent %.2f ms queueing and formatting (%.3f%% of one thread), logger thread %.2f ms, queue peak %.1f KiB\n", seconds, static_cast<unsigned long long>(outLines + errLines), static_cast<unsigned long long>(outLines), static_cast<unsigned long long>(errLines), static_cast<double>(written) / 1024.0, static_cast<unsigned long long>(coalesced), static_cast<double>(caller) / 1e6, static_cast<double>(caller) / 1e9 / seconds * 100.0, static_cast<double>(logger) / 1e6, static_cast<double>(high) / 1024.0);
+        if (size > 0) WriteNow(stderr, line, std::min(static_cast<std::size_t>(size), sizeof(line) - 1));
+    }
+
+    bool asynchronous = true;
+    bool statsEnabled = true;
+    std::mutex mutex;
+    std::condition_variable wake;
+    std::condition_variable space;
+    std::condition_variable idle;
+    std::deque<AsyncLogDetail::Entry> pending;
+    std::size_t bytes = 0;
+    std::size_t peak = 0;
+    bool sleeping = false;
+    std::atomic<std::uint64_t> queued{0};
+    std::atomic<std::uint64_t> finished{0};
+    std::atomic<bool> abandoned{false};
+    std::mutex coalesceMutex;
+    std::map<std::string, Window> windows;
+    Clock::time_point lastStats;
+    std::atomic<std::uint64_t> stdoutLines{0};
+    std::atomic<std::uint64_t> stderrLines{0};
+    std::atomic<std::uint64_t> loggedBytes{0};
+    std::atomic<std::uint64_t> callerNanoseconds{0};
+    std::atomic<std::uint64_t> loggerNanoseconds{0};
+    std::atomic<std::uint64_t> coalescedTotal{0};
+};
+
+inline void InstallAsyncLog_nid_no_patch() {
+    static_cast<void>(AsyncLog::Get());
+}
+
+inline void FlushAsyncLog_nid_no_patch() {
+    AsyncLog::Get().Flush();
+}
+
+inline void FlushAsyncLogAtEnd_nid_no_patch() {
+    AsyncLog::FlushAtEnd();
+}
+
+std::unique_lock<std::mutex> HoldAsyncLog();
+
+template<typename... TArgs>
+void LogCoalescedStderr_nid_no_patch(const char* format, TArgs... args) {
+    char small[1024];
+    const int size = std::snprintf(small, sizeof(small), format, args...);
+    if (size < 0) return;
+    std::string line = static_cast<std::size_t>(size) < sizeof(small) ? std::string(small, static_cast<std::size_t>(size)) : std::string(static_cast<std::size_t>(size), '\0');
+    if (static_cast<std::size_t>(size) >= sizeof(small)) std::snprintf(line.data(), line.size() + 1, format, args...);
+    if (!AsyncLog::Get().Admit(line)) return;
+    std::fwrite(line.data(), 1, line.size(), stderr);
+    std::fflush(stderr);
+}
+
+}
+
+#define AGC_LOG_DEFERRED(stream, fmt, ...) \
+::AgcDriver::AsyncLog::Get().Defer(stream, "[%s:%d %s] " fmt "\n", ::AgcDriver::LogLiteral{_APS5_FILE_}, __LINE__, ::AgcDriver::LogLiteral{TrimNidPostfix(__func__)}, __VA_ARGS__)

core/libs/prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp

@@ -22,6 +22,7 @@ struct DisplayBuffer {
     std::uint32_t pitchInPixel = 0;
     std::uint64_t dccAddress = 0;
     std::uint64_t dccClearColor = 0;
+    std::uint32_t alignment = 65536;
 };
 
 std::size_t DisplayBufferSize(const DisplayBuffer& buffer);

core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp

@@ -1,6 +1,7 @@
 #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_HPP
 #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_HPP
 
+#include <cstdint>
 #include <span>
 #include "SceTypes.hpp"
 #include "SceShaders.hpp"
@@ -17,6 +18,7 @@ extern "C" void AgcDriverLockVulkanLoader_nid_postfix();
 extern "C" void AgcDriverUnlockVulkanLoader_nid_postfix();
 extern "C" void AgcDriverShutdown_nid_postfix();
 extern "C" void AgcDriverSuspendPoint_nid_postfix();
+extern "C" std::uint64_t AgcDriverDispatchesAdoptedAhead_nid_postfix();
 extern "C" void AgcDriverRegisterShader_nid_postfix(const Shader* shader);
 extern "C" void AgcDriverResolveShaderAbi_nid_postfix(const Shader* shader, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);
 extern "C" void AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span<const Shader* const> stages, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp

@@ -29,6 +29,7 @@ struct DispatchVariant {
     std::vector<std::uint32_t> words;
     std::uint64_t forgetSerial = 0;
     std::shared_ptr<const ShaderRecompiler::RecompileResult> compiled;
+    std::array<std::uint32_t, 4> meshIndexDescriptor{};
 
     std::shared_ptr<const ShaderSnapshot> shader;
 
@@ -40,6 +41,9 @@ struct DispatchVariant {
     std::uint32_t flatBinding = NoFlatBinding;
 
     std::uint32_t pushOffset = 0;
+    std::vector<std::int8_t> runBases;
+    std::uint64_t relocatableWords = 0;
+    std::vector<std::pair<std::uint32_t, std::uint32_t>> shaderDataPatches;
     std::shared_ptr<const ShaderRecompiler::ShaderVertexStageInfo> vertexInfo;
 
     std::shared_ptr<ShaderMemory> memory;

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp

@@ -1,6 +1,7 @@
 #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAWCACHE_HPP
 #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAWCACHE_HPP
 
+#include "UserDataUse.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp"
 #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
 #include <array>
@@ -30,6 +31,7 @@ struct DrawProgram {
     bool merged = false;
     bool mergedPointerRequired = false;
     std::uint32_t mergedPointer = 0;
+    std::uint32_t hiddenUserWords = 0;
 };
 
 struct DrawDecode {
@@ -118,6 +120,36 @@ struct StageCapture {
     std::vector<ShaderRecompiler::MemoryRegion> regions;
     std::uint64_t forgetSerial = 0;
     std::uint32_t pushOffset = 0;
+    ShaderRecompiler::SrtUserDataUse userDataUse;
+};
+
+struct RelocatableUserWord {
+    std::uint32_t reg;
+    std::vector<std::pair<std::int64_t, std::uint32_t>> reads;
+};
+
+struct DrawRelocation {
+    std::vector<RelocatableUserWord> words;
+    std::vector<std::uint32_t> banned;
+};
+
+struct DrawUserPatch {
+    std::size_t program;
+    std::uint32_t index, value;
+};
+
+struct DrawRelocationPending {
+    std::uint64_t key = 0, alias = 0, identity = 0;
+};
+
+struct DrawRelocationTrial {
+    static constexpr std::uint32_t Window = 1024, MinimumHitShare = 16, IdleDraws = 16384, ProbationAdmissions = 2;
+    std::uint32_t lookups = 0, hits = 0, idle = 0, admissions = 0;
+    bool productive = false;
+};
+
+struct DrawRelocationCounters {
+    std::uint64_t lookups = 0, donors = 0, refused = 0, hits = 0, evicted = 0;
 };
 
 bool DecodeProgramsCurrent(const DrawDecode& decode, const ShaderRegistry& registry);

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp

@@ -26,12 +26,16 @@ public:
 
     static DrawPipeline& Queue0();
     static std::size_t Depth();
+    static std::size_t QueuedItems();
     static bool& Active();
     static std::atomic<std::uint64_t>& EpochToken();
     static void FollowEpoch(std::uint64_t token);
+    static unsigned Committers();
 
     void Enqueue(Commit commit, std::vector<Range> writes, std::uint64_t labelAddress = 0, std::vector<std::byte> labelBytes = {});
     void Drain(DrainReason reason, std::uint32_t opcode = 0x100);
+    void Stop();
+    static std::uint64_t DrainRequestsByOpcode(std::uint32_t opcode);
     bool Busy() const { return outstanding.load(std::memory_order_acquire) != 0; }
     std::size_t Outstanding() const { return outstanding.load(std::memory_order_acquire); }
     bool Overlaps(std::uint64_t address, std::size_t bytes);
@@ -58,6 +62,7 @@ private:
     std::uint32_t idleWaiters = 0;
     bool committerWaiting = false;
     bool stopped = false;
+    bool closing = false;
     std::thread thread;
     std::uint64_t commits = 0;
     std::uint64_t commitNs = 0;
@@ -67,6 +72,7 @@ private:
     std::array<std::uint64_t, static_cast<std::size_t>(DrainReason::Count)> drains{};
     std::array<std::uint64_t, static_cast<std::size_t>(DrainReason::Count)> drainWaitNs{};
     std::array<std::uint64_t, 257> drainOpcodes{};
+    std::array<std::atomic<std::uint64_t>, 257> drainRequests{};
     std::uint64_t depthSum = 0;
     std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now();
 };

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -49,6 +49,7 @@ public:
     ~Driver();
     void Shutdown();
     void WaitIdle();
+    std::uint64_t DispatchesAdoptedAhead() const;
     void CheckFailure();
     void ReportFailure(std::exception_ptr error);
     void Submit(const Packet* packet, std::uint32_t queue);
@@ -74,6 +75,7 @@ public:
     };
     DispatchCacheFootprint DispatchCacheSize();
     static std::uint64_t DispatchCacheBytes();
+    std::shared_ptr<const ShaderRegistry> RegisteredShaders();
 
 private:
     friend class SampledReadScope;
@@ -101,6 +103,7 @@ private:
     static bool& completionDeferred();
     void deferCompletion(const Submission& submission);
     void submitOpenWork();
+    void wakeQueueZeroForCompletions();
     void completeSubmission(std::uint64_t serial, std::uint64_t received, std::span<const std::uint64_t> labelWrites);
     static bool waitFree(const Submission& submission);
     bool queue0Before(std::uint64_t received) const;
@@ -110,6 +113,7 @@ private:
     static bool stampValidate();
     static bool dataHits();
     static bool verifyDataHits();
+    static bool pipelineDispatch();
     static std::size_t dispatchVariants();
     static bool insertCompare();
     static std::size_t dispatchCacheEntries();
@@ -141,8 +145,16 @@ private:
     void maybeReportDrawCache(bool profile);
     void reportDrawCache(DrawEntryCounters& counters, bool profile);
     std::string profileDrawCacheLines(const DrawEntryCounters& counters);
-    static std::uint64_t drawRegisterKey(const QueueState& queue, const ShaderRegistry& registry, std::uint64_t deviceSerial, std::uint64_t* shape = nullptr, std::array<std::uint64_t, 5>* components = nullptr);
+    static std::uint64_t drawRegisterKey(const QueueState& queue, const ShaderRegistry& registry, std::uint64_t deviceSerial, std::uint64_t* shape = nullptr, std::array<std::uint64_t, 5>* components = nullptr, const DrawRelocation* relocation = nullptr, std::uint64_t* alias = nullptr);
     void traceDrawKeyLookup(const std::array<std::uint64_t, 5>& components, bool cacheMiss);
+    static std::uint64_t drawIdentityKey(const QueueState& queue, const ShaderRegistry& registry);
+    static bool drawRelocationEnabled();
+    static std::uint64_t relocatedContentKey(const RelocatableUserWord& word, std::uint32_t value, bool& readable);
+    static void tagVariantRuns(DispatchVariant& variant, std::span<const ShaderRecompiler::MemoryRegion> regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords);
+    static void applyShaderDataPatches(DispatchVariant& variant);
+    void learnPendingRelocation();
+    void relocateDrawEntry(const QueueState& queue, std::uint64_t alias, std::shared_ptr<DrawEntry>& entry, std::shared_ptr<const DrawDecode>& decode, std::vector<DrawUserPatch>& patches);
+    void reportDrawRelocation();
     static bool sameVertexInfo(const ShaderRecompiler::ShaderVertexStageInfo& a, const ShaderRecompiler::ShaderVertexStageInfo& b);
     static bool sameDecode(const DrawDecode& a, const DrawDecode& b);
     std::shared_ptr<DrawDecode> decodeDraw(const QueueState& queue, const Submission& submission);
@@ -184,6 +196,7 @@ private:
     static bool traceCopy();
     void traceCopyPending(const char* what, std::uint64_t address, std::size_t bytes, std::uint64_t source, std::uint64_t destination, std::size_t copyBytes, const std::shared_ptr<VulkanDevice>& localDevice);
     static void countIndirect(int path, double readMs);
+    static int indirectArgumentsPath(std::span<const std::uint32_t> packet, std::uint64_t arguments);
     void dispatchIndirect(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission);
     static bool& sampledRead();
     static bool writeEvidenceEnabled();
@@ -346,6 +359,11 @@ private:
     std::unordered_map<std::uint64_t, std::uint64_t> drawShapes;
     std::unordered_set<std::uint64_t> traceShapes, traceKeysSeen, traceKeysEvicted;
     std::list<std::uint64_t> drawOrder;
+    std::unordered_map<std::uint64_t, std::shared_ptr<const DrawRelocation>> drawRelocations;
+    std::unordered_map<std::uint64_t, std::shared_ptr<DrawEntry>> drawAliases;
+    DrawRelocationPending drawRelocationPending;
+    std::unordered_map<std::uint64_t, DrawRelocationTrial> drawRelocationTrials;
+    DrawRelocationCounters drawRelocationCounters;
     std::mutex drawCacheMutex;
     std::uint64_t drawCacheHits = 0, drawCacheEvictions = 0, drawCacheVariants = 0, drawCacheVariantBytes = 0;
     std::optional<DrawKeySightings> drawKeySightings;
@@ -399,6 +417,7 @@ private:
 
     std::atomic<std::uint64_t> evidenceReads{0};
     std::atomic<std::uint64_t> evidenceValidations{0};
+    std::atomic<std::uint64_t> dispatchesAdoptedAhead{0};
 
     std::mutex writtenBuffersMutex;
     std::deque<WrittenBuffer> writtenBuffers;

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp

@@ -11,6 +11,7 @@
 namespace AgcDriver::DriverDetail {
 
 std::vector<std::pair<std::uint64_t, std::uint64_t>> DrawWriteRanges(const Graphics::State& graphics, std::span<const Graphics::CompiledShader> stages, bool exact);
+std::vector<std::pair<std::uint64_t, std::uint64_t>> DispatchWriteRanges(const ShaderRecompiler::RecompileResult& compiled, bool& bounded);
 bool ExactDrawWrites();
 
 }

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/SubmissionRing.hpp

@@ -0,0 +1,40 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_QUEUES_SUBMISSIONRING_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_QUEUES_SUBMISSIONRING_HPP
+
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <span>
+#include <string_view>
+
+namespace AgcDriver::DriverDetail {
+
+inline constexpr std::uint32_t SubmissionQueueEnd = 0x58;
+
+std::size_t SubmissionRingCapacity();
+
+std::chrono::seconds StallWatchdogPeriod();
+
+bool ReadSubmissionRingSettings();
+
+void PrepareSubmissionRing(std::uint32_t queue);
+
+void RecordSubmission(std::uint32_t queue, std::uint64_t serial, std::span<const std::uint32_t> commands, const void* guest, bool rewindTail);
+
+void DumpSubmissionRings(std::string_view reason);
+
+class StallWatch {
+public:
+    explicit StallWatch(std::uint32_t queue) : queue(queue) {}
+    void Progressed() { since = {}; }
+    void Check(const char* waiter, std::uint64_t value);
+
+private:
+    std::uint32_t queue;
+    std::chrono::steady_clock::time_point since{};
+    bool dumped = false;
+};
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp

@@ -13,10 +13,12 @@
 #include <exception>
 #include <functional>
 #include <initializer_list>
+#include <iterator>
 #include <map>
 #include <memory>
 #include <mutex>
 #include <string>
+#include <utility>
 #include <vector>
 
 struct Shader;
@@ -32,7 +34,94 @@ class VulkanDevice;
 namespace AgcDriver::DriverDetail {
 
 struct ShaderSnapshot;
-using ShaderRegistry = std::map<std::uint64_t, std::vector<std::shared_ptr<const ShaderSnapshot>>>;
+
+class ShaderRegistry {
+    using Chunk = std::map<std::uint64_t, std::vector<std::shared_ptr<const ShaderSnapshot>>>;
+    using Chunks = std::vector<std::shared_ptr<Chunk>>;
+
+public:
+    using key_type = std::uint64_t;
+    using mapped_type = Chunk::mapped_type;
+    using value_type = Chunk::value_type;
+
+    class const_iterator {
+    public:
+        using iterator_category = std::bidirectional_iterator_tag;
+        using value_type = ShaderRegistry::value_type;
+        using difference_type = std::ptrdiff_t;
+        using pointer = const value_type*;
+        using reference = const value_type&;
+
+        const_iterator() = default;
+        reference operator*() const { return *entry; }
+        pointer operator->() const { return &*entry; }
+        const_iterator& operator++() {
+            if (++entry == (*chunk)->end()) {
+                ++chunk;
+                entry = chunk == last ? Chunk::const_iterator{} : (*chunk)->cbegin();
+            }
+            return *this;
+        }
+        const_iterator operator++(int) {
+            auto previous = *this;
+            ++*this;
+            return previous;
+        }
+        const_iterator& operator--() {
+            if (chunk == last || entry == (*chunk)->cbegin()) {
+                --chunk;
+                entry = std::prev((*chunk)->cend());
+            } else {
+                --entry;
+            }
+            return *this;
+        }
+        const_iterator operator--(int) {
+            auto previous = *this;
+            --*this;
+            return previous;
+        }
+        bool operator==(const const_iterator& other) const { return chunk == other.chunk && entry == other.entry; }
+
+    private:
+        friend class ShaderRegistry;
+        const_iterator(Chunks::const_iterator chunk, Chunks::const_iterator last, Chunk::const_iterator entry) : chunk(chunk), last(last), entry(entry) {}
+        Chunks::const_iterator chunk{};
+        Chunks::const_iterator last{};
+        Chunk::const_iterator entry{};
+    };
+    using iterator = const_iterator;
+
+    ShaderRegistry() = default;
+    ShaderRegistry(std::initializer_list<value_type> entries) {
+        for (const auto& [address, headers] : entries) emplace(address, headers);
+    }
+
+    const_iterator begin() const { return chunks.empty() ? end() : const_iterator(chunks.cbegin(), chunks.cend(), chunks.front()->cbegin()); }
+    const_iterator end() const { return const_iterator(chunks.cend(), chunks.cend(), {}); }
+    const_iterator find(std::uint64_t address) const;
+    const_iterator upper_bound(std::uint64_t address) const;
+    bool contains(std::uint64_t address) const { return find(address) != end(); }
+    const mapped_type& at(std::uint64_t address) const;
+    std::size_t size() const { return count; }
+    bool empty() const { return count == 0; }
+    mapped_type& operator[](std::uint64_t address);
+    std::pair<const_iterator, bool> emplace(std::uint64_t address, mapped_type headers);
+    std::size_t erase(std::uint64_t address);
+    void clear() {
+        chunks.clear();
+        count = 0;
+    }
+
+private:
+    static constexpr std::size_t ChunkEntries = 64;
+
+    std::size_t chunkFor(std::uint64_t address) const;
+    Chunk& writable(std::size_t index);
+
+    Chunks chunks;
+    std::size_t count = 0;
+};
 
 struct PreparedShaderState {
     struct Entry {
@@ -87,6 +176,7 @@ struct ShaderSnapshot {
     std::shared_ptr<const RegisteredShaderState> registeredState;
     std::shared_ptr<std::atomic<std::uint8_t>> workgroupUse = std::make_shared<std::atomic<std::uint8_t>>(0u);
     std::shared_ptr<std::atomic<std::uint8_t>> positionVectorUse = std::make_shared<std::atomic<std::uint8_t>>(0u);
+    std::shared_ptr<std::atomic<bool>> rawGraphicsNoted = std::make_shared<std::atomic<bool>>(false);
     std::optional<std::uint32_t> userDataTailRegister;
 };
 
@@ -94,8 +184,9 @@ struct ShaderSnapshot {
 std::shared_ptr<const ShaderSnapshot> ReadRawShader(std::uint64_t address);
 std::shared_ptr<const ShaderSnapshot> ReadUserDataTailContinuation(std::span<const std::uint32_t> code, std::uint32_t userDataBase, std::span<const std::uint32_t> userData);
 std::shared_ptr<const ShaderSnapshot> ProgramSnapshot(const ShaderRegistry& shaders, std::uint64_t address, std::initializer_list<std::uint8_t> types);
-void NoteRawGraphicsDraw();
+void NoteRawGraphicsDraw(const ShaderSnapshot& snapshot);
 bool RawGraphicsDrawn();
+bool RawGraphicsDrawnIn(std::uint64_t begin, std::uint64_t end);
 bool ProgramUsesWorkgroup(const ShaderSnapshot& snapshot, std::size_t codeOffset);
 // The PA_CL_VS_OUT_CNTL position export bits a vertex program is compiled with: none for a program
 // without pos1..pos3 exports, so its artifact does not depend on them.
@@ -115,7 +206,9 @@ void SubmitRegistrationPreparation(std::function<void()> task, const PreparedSha
 void PrioritizeRegistrationPreparation(PreparedShaders& prepared);
 void AwaitRegistrationPreparation();
 void SubmitLibraryPrebuild(std::function<void()> task);
+void AwaitLibraryPrebuilds();
 void CloseRegistrationPreparation();
+unsigned RegistrationPreparationThreads();
 
 void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target);
 

core/libs/prx/libSceAgcDriver/Execution/include/FrameDumpIo.hpp

@@ -0,0 +1,60 @@
+#pragma once
+#include <atomic>
+#include <cerrno>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <span>
+
+namespace AgcDriver {
+
+struct FrameDumpIoFailure {
+    const char* operation = nullptr;
+    int error = 0;
+    explicit operator bool() const noexcept { return operation != nullptr; }
+};
+
+inline FrameDumpIoFailure WriteBmpStream_nid_no_patch(std::FILE* file, std::uint32_t width, std::uint32_t height, std::span<const std::byte> pixels) noexcept {
+    const std::uint32_t imageBytes = width * height * 4;
+    const std::uint32_t header[13] = {0, 0, 54, 40, width, static_cast<std::uint32_t>(-static_cast<std::int32_t>(height)), 1u | (32u << 16u), 0, imageBytes, 2835, 2835, 0, 0};
+    const std::uint16_t magic = 0x4d42;
+    const std::uint32_t fileBytes = 54 + imageBytes;
+    const auto write = [&](const void* data, std::size_t size, const char* operation) {
+        errno = 0;
+        if (std::fwrite(data, 1, size, file) != size) return FrameDumpIoFailure{operation, errno};
+        return FrameDumpIoFailure{};
+    };
+    if (const auto failure = write(&magic, 2, "header")) return failure;
+    if (const auto failure = write(&fileBytes, 4, "header")) return failure;
+    if (const auto failure = write(header + 1, 48, "header")) return failure;
+    return write(pixels.data(), pixels.size(), "pixels");
+}
+
+inline FrameDumpIoFailure WriteBmpAndClose_nid_no_patch(std::FILE* file, std::uint32_t width, std::uint32_t height, std::span<const std::byte> pixels) noexcept {
+    auto failure = WriteBmpStream_nid_no_patch(file, width, height, pixels);
+    errno = 0;
+    if (std::fclose(file) != 0 && !failure) failure = {"close", errno};
+    return failure;
+}
+
+class FrameDumpOutput {
+public:
+    bool Enabled_nid_no_patch() const noexcept { return enabled.load(std::memory_order_relaxed); }
+
+    bool Write_nid_no_patch(const char* path, std::uint32_t width, std::uint32_t height, std::span<const std::byte> pixels) noexcept {
+        if (!Enabled_nid_no_patch()) return false;
+        errno = 0;
+        auto* file = std::fopen(path, "wb");
+        const auto failure = file != nullptr ? WriteBmpAndClose_nid_no_patch(file, width, height, pixels) : FrameDumpIoFailure{"open", errno};
+        if (!failure) return true;
+        if (enabled.exchange(false, std::memory_order_relaxed)) {
+            std::fprintf(stderr, "[gpu] optional BMP dumps disabled: %s failed (errno=%d)\n", failure.operation, failure.error);
+        }
+        return false;
+    }
+
+private:
+    std::atomic<bool> enabled{true};
+};
+
+}

core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp

@@ -26,6 +26,7 @@ struct Commitment {
     bool whole = false;
 };
 Commitment DescribeCommitted(std::uint64_t address, std::size_t bytes, bool writable = false);
+void CopyGuestBytes(void* destination, const void* source, std::size_t bytes);
 void Read(std::uint64_t address, std::span<std::byte> destination, std::size_t alignment = 1);
 // Variants for resources in heaps the guest commits on demand (partially resident textures, GPU
 // scratch heaps): uncommitted pages read as zeros, are not compared, and are never stored.
@@ -60,6 +61,7 @@ void WriteChanged(std::uint64_t address, std::span<const std::byte> current, std
 // UnchangedSince is always false, so callers fall back to comparing bytes.
 bool WriteWatched();
 void Unwatch(std::uint64_t address, std::size_t bytes);
+void Rewatch(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges);
 bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched = true);
 void SetUnwatchHook(void (*hook)(std::uint64_t address, std::size_t bytes));
 bool Watched(std::uint64_t address, std::size_t bytes);
@@ -81,6 +83,7 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes);
 std::uint64_t StoreOwnBytes(std::uint64_t address, std::size_t bytes, const std::function<void()>& store);
 std::uint64_t StoreOwnBytes(std::uint64_t address, std::span<const std::byte> source);
 bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generation);
+bool StoredAllSince_nid_no_patch(std::uint64_t address, std::size_t bytes, std::uint64_t generation);
 // Collect epoch: within one epoch a range already collected is not walked again, CollectWrites
 // returns the current generation instead. A guest write landing between two collects of the same
 // epoch is seen by the next epoch, which is the ordering real hardware gives a CPU write made while a

core/libs/prx/libSceAgcDriver/Execution/include/HandoffSpin.hpp

@@ -0,0 +1,61 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_HANDOFFSPIN_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_HANDOFFSPIN_HPP
+
+#include "prx/libc/include/CpuTopology.hpp"
+#include <algorithm>
+#include <bitset>
+#include <chrono>
+#include <cstdint>
+#include <cstdlib>
+#include <thread>
+#if defined(__x86_64__) || defined(_M_X64)
+#include <immintrin.h>
+#endif
+
+namespace AgcDriver {
+
+inline void SpinPause() {
+#if defined(__x86_64__) || defined(_M_X64)
+    _mm_pause();
+#endif
+}
+
+inline unsigned UsableProcessors() {
+    const auto count = static_cast<unsigned>(std::bitset<64>(CpuTopology::Get().process).count());
+    return count != 0 ? count : std::thread::hardware_concurrency();
+}
+
+inline std::chrono::nanoseconds HandoffSpinWindowFor(const char* configured, unsigned usableProcessors) {
+    constexpr std::chrono::microseconds automaticWindow{150};
+    constexpr unsigned automaticProcessorsAbove = 8;
+    constexpr long long longestWindowUs = 10000;
+    if (configured != nullptr) return std::chrono::microseconds(std::clamp<long long>(std::strtoll(configured, nullptr, 10), 0, longestWindowUs));
+    return usableProcessors > automaticProcessorsAbove ? std::chrono::nanoseconds(automaticWindow) : std::chrono::nanoseconds::zero();
+}
+
+inline std::chrono::nanoseconds HandoffSpinWindow() {
+    static const std::chrono::nanoseconds window = HandoffSpinWindowFor(std::getenv("APS5_HANDOFF_SPIN_US"), UsableProcessors());
+    return window;
+}
+
+template<typename TReady>
+bool SpinUntil(std::chrono::nanoseconds window, TReady&& ready) {
+    if (ready()) return true;
+    if (window.count() <= 0) return false;
+    const auto until = std::chrono::steady_clock::now() + window;
+    do {
+        SpinPause();
+        if (ready()) return true;
+    } while (std::chrono::steady_clock::now() < until);
+    return false;
+}
+
+template<typename TMutex>
+void LockSpinning(TMutex& mutex, std::chrono::nanoseconds window) {
+    if (window.count() > 0 && SpinUntil(window, [&] { return mutex.try_lock(); })) return;
+    mutex.lock();
+}
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Execution/include/PerformanceOverlay.hpp

@@ -0,0 +1,223 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_PERFORMANCEOVERLAY_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_PERFORMANCEOVERLAY_HPP
+
+#define VK_NO_PROTOTYPES
+#include <vulkan/vulkan.h>
+#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
+#include <array>
+#include <atomic>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <deque>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+namespace AgcDriver {
+
+namespace Graphics {
+class Buffer;
+}
+
+enum class OverlayCorner : std::uint8_t { TopLeft, TopRight, BottomLeft, BottomRight };
+
+struct OverlaySettings {
+    bool visible = false;
+    OverlayCorner corner = OverlayCorner::TopLeft;
+};
+
+OverlaySettings ParseOverlaySettings(const char* enabled, const char* position);
+
+class PerformanceCounters {
+public:
+    static void AddGpuNanoseconds(std::uint64_t nanoseconds);
+    static void AddDriverNanoseconds(std::uint64_t nanoseconds);
+    static void CountPipeline();
+    static std::uint64_t GpuNanoseconds();
+    static std::uint64_t DriverNanoseconds();
+    static std::uint64_t Pipelines();
+    static bool GpuEnabled();
+    static bool DriverEnabled();
+};
+
+class PacketCounters {
+public:
+    struct Totals {
+        std::uint64_t draws = 0;
+        std::uint64_t dispatches = 0;
+    };
+
+    static PacketCounters& ThisThread();
+    static Totals Sum();
+
+    void CountDraw() { draws.store(draws.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); }
+    void CountDispatch() { dispatches.store(dispatches.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); }
+
+private:
+    alignas(64) std::atomic<std::uint64_t> draws{0};
+    std::atomic<std::uint64_t> dispatches{0};
+};
+
+struct OverlayCounters {
+    std::uint64_t gpuNanoseconds = 0;
+    std::uint64_t driverNanoseconds = 0;
+    std::uint64_t draws = 0;
+    std::uint64_t dispatches = 0;
+    std::uint64_t pipelines = 0;
+};
+
+class OverlayStatistics {
+public:
+    using Clock = std::chrono::steady_clock;
+    static constexpr std::size_t FrameHistory = 120;
+    static constexpr Clock::duration Window = std::chrono::seconds(1);
+
+    struct Range {
+        double minimum = 0;
+        double average = 0;
+        double maximum = 0;
+    };
+
+    void AddPresent(Clock::time_point at, std::uint64_t gpuNanoseconds, std::uint64_t driverNanoseconds);
+    void AddPresent(Clock::time_point at, const OverlayCounters& counters);
+    void Reset();
+    double Fps() const;
+    double LastFrameMs() const;
+    std::vector<double> FrameTimes() const;
+    Range FrameTimeRange() const;
+    double GpuMsPerFrame() const;
+    double DriverMsPerFrame() const;
+    double DrawsPerFrame() const;
+    double DispatchesPerFrame() const;
+    std::uint64_t PipelinesInWindow() const;
+    OverlayCounters Latest() const;
+
+private:
+    struct Sample {
+        Clock::time_point at;
+        OverlayCounters counters;
+    };
+
+    double perFrame(std::uint64_t OverlayCounters::*counter) const;
+
+    std::deque<Sample> window;
+    std::optional<Sample> previous;
+    std::array<double, FrameHistory> frameTimes{};
+    std::size_t frameCount = 0;
+    std::size_t frameNext = 0;
+};
+
+struct OverlayRect {
+    std::int32_t x = 0;
+    std::int32_t y = 0;
+    std::int32_t width = 0;
+    std::int32_t height = 0;
+    std::uint32_t color = 0;
+};
+
+enum class OverlayColor : std::uint32_t { Background = 0, Text = 1, Graph = 2 };
+inline constexpr std::size_t OverlayColorCount = 3;
+
+struct OverlayContent {
+    std::vector<std::string> above;
+    std::vector<double> frameTimes;
+    std::vector<std::string> below;
+};
+
+struct OverlayMemory {
+    std::uint64_t used = 0;
+    std::uint64_t budget = 0;
+};
+
+struct OverlayDetails {
+    std::optional<OverlayMemory> vram;
+    std::optional<std::uint64_t> ram;
+    VkExtent2D display{};
+    VkExtent2D window{};
+};
+
+struct OverlayLayout {
+    OverlayRect bounds;
+    std::vector<OverlayRect> rects;
+    std::int32_t scale = 1;
+};
+
+struct OverlayMetrics {
+    static constexpr std::int32_t GlyphWidth = 5;
+    static constexpr std::int32_t GlyphHeight = 7;
+    static constexpr std::int32_t Advance = 6;
+    static constexpr std::int32_t LinePitch = 9;
+    static constexpr std::int32_t Columns = 27;
+    static constexpr std::int32_t Padding = 3;
+    static constexpr std::int32_t Margin = 4;
+    static constexpr std::int32_t GraphHeight = 20;
+    static constexpr std::int32_t GraphGap = 2;
+    static constexpr std::int32_t InnerWidth = Columns * Advance - 1;
+};
+
+const std::array<std::uint8_t, OverlayMetrics::GlyphHeight>& OverlayGlyph(char character);
+std::string FormatOverlayValue(double value);
+std::string FormatOverlayRate(double value);
+std::string FormatOverlayCount(double value);
+std::string FormatOverlayGigabytes(std::uint64_t bytes);
+OverlayContent ComposeOverlay(const OverlayStatistics& statistics, bool gpu, bool driver);
+OverlayContent ComposeOverlay(const OverlayStatistics& statistics, bool gpu, bool driver, const OverlayDetails& details);
+bool OverlaySampleDue(std::optional<OverlayStatistics::Clock::time_point> sampledAt, OverlayStatistics::Clock::time_point now);
+std::optional<OverlayMemory> ReadVramBudget(const Graphics::Context& context);
+std::optional<std::uint64_t> ReadProcessPrivateBytes();
+std::int32_t OverlayScale(VkExtent2D extent, const OverlayContent& content);
+OverlayLayout LayOutOverlay(const OverlayContent& content, VkExtent2D extent, OverlayCorner corner);
+std::array<std::array<float, 4>, OverlayColorCount> OverlayPalette(VkFormat format, VkColorSpaceKHR colorSpace);
+
+class PerformanceOverlay {
+public:
+    PerformanceOverlay(const Graphics::Context& context, std::size_t slots);
+    ~PerformanceOverlay();
+    PerformanceOverlay(const PerformanceOverlay&) = delete;
+    PerformanceOverlay& operator=(const PerformanceOverlay&) = delete;
+
+    static bool Visible();
+    static void Toggle();
+    static OverlayCorner Corner();
+
+    void NotePresent(OverlayStatistics::Clock::time_point at, VkExtent2D display);
+    const OverlayStatistics& Statistics() const { return statistics; }
+    const OverlayDetails& Details() const { return details; }
+    OverlayRect Record(VkCommandBuffer commands, std::size_t slot, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout);
+    OverlayRect Record(VkCommandBuffer commands, std::size_t slot, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout, const OverlayContent& content, OverlayCorner corner);
+    void Release(std::size_t slot) noexcept;
+
+private:
+    struct Slot {
+        std::unique_ptr<Graphics::Buffer> rects;
+        std::size_t capacity = 0;
+        std::vector<std::pair<VkImageView, VkFramebuffer>> transient;
+    };
+
+    VkRenderPass renderPass(VkFormat format, VkImageLayout initialLayout, VkImageLayout finalLayout);
+    VkPipeline pipeline(VkFormat format, VkRenderPass compatible);
+    void releaseTransient(Slot& slot) noexcept;
+    void release() noexcept;
+    void restartIfToggled();
+
+    Graphics::Context context;
+    VkPipelineLayout pipelineLayout = VK_NULL_HANDLE;
+    VkShaderModule vertex = VK_NULL_HANDLE;
+    VkShaderModule fragment = VK_NULL_HANDLE;
+    std::map<std::tuple<VkFormat, VkImageLayout, VkImageLayout>, VkRenderPass> renderPasses;
+    std::map<VkFormat, VkPipeline> pipelines;
+    std::vector<Slot> slots;
+    OverlayStatistics statistics;
+    OverlayDetails details;
+    std::optional<OverlayStatistics::Clock::time_point> sampledAt;
+    std::uint64_t toggles = 0;
+};
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp

@@ -21,6 +21,13 @@
 #include <utility>
 #include <vector>
 
+namespace AgcDriver {
+class TraceThread;
+}
+
+extern "C" bool AgcDriverTraceRecording_nid_postfix();
+extern "C" AgcDriver::TraceThread* AgcDriverTraceThread_nid_postfix();
+
 namespace AgcDriver {
 
 class VulkanDevice;
@@ -532,23 +539,26 @@ public:
     }
 
     static bool Recording() {
-        return recording.load(std::memory_order_relaxed);
+        return AgcDriverTraceRecording_nid_postfix();
     }
 
     static TraceThread* Thread() {
-        return local;
+        return AgcDriverTraceThread_nid_postfix();
     }
 
     static void StopOutput();
 
 private:
+    friend bool ::AgcDriverTraceRecording_nid_postfix();
+    friend TraceThread* ::AgcDriverTraceThread_nid_postfix();
+
     static void OpenInterval(std::chrono::steady_clock::time_point now, std::uint64_t present);
     static void FlushInterval(std::chrono::steady_clock::time_point now, std::uint32_t width, std::uint32_t height, const VulkanDevice* device);
 
-    inline static std::atomic<bool> recording{false};
+    static std::atomic<bool> recording;
     inline static std::atomic<bool> buffering{false};
     inline static std::atomic<std::uint64_t> presentCount{0};
-    inline static thread_local TraceThread* local = nullptr;
+    static thread_local TraceThread* local;
     inline static std::mutex gpuMutex;
     inline static std::vector<TraceGpuRange> gpuRanges;
     inline static std::uint64_t gpuLost = 0;

core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp

@@ -78,6 +78,7 @@ std::uint32_t ReadDrawCount(const DrawParameters::IndirectDraw& indirect);
 inline bool IndirectDrawOpcode(std::uint32_t opcode) { return opcode == 0x24 || opcode == 0x25 || opcode == 0x2c || opcode == 0x38; }
 inline bool DrawOpcode(std::uint32_t opcode) { return opcode == 0x27 || opcode == 0x2d || opcode == 0x35 || IndirectDrawOpcode(opcode); }
 
+std::string_view KnownName(std::uint32_t header);
 std::string Name(std::uint32_t header);
 // A PM4 type-2 packet is a one-dword filler (command-buffer padding); type 3 and type 0 carry a
 // dword count in bits 29:16. Type 1 is undefined.

core/libs/prx/libSceAgcDriver/Execution/include/Presentation.hpp

@@ -29,6 +29,7 @@ extern "C" void AgcDriverAttachWindow_nid_postfix(const AgcDriver::PresentationW
 extern "C" void AgcDriverPresentClear_nid_postfix(const AgcDriver::PresentationWindow& window, bool opaque, void (*gpuReady)(void*), void* context);
 extern "C" void AgcDriverPresentBuffer_nid_postfix(const AgcDriver::PresentationWindow& window, const AgcDriver::DisplayBuffer& buffer, void (*gpuReady)(void*), void* context);
 extern "C" void AgcDriverReleaseWindow_nid_postfix(void* window);
+extern "C" void AgcDriverTogglePerformanceOverlay_nid_postfix();
 extern "C" void AgcDriverReportFailure_nid_postfix(std::exception_ptr error);
 
 #endif

core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp

@@ -47,7 +47,7 @@ struct Recipe {
     std::uint64_t dataWordsHash = 0;
     // The template's PresyncSurfaces() as of the attach and the host-import identity they were
     // computed under (a surface is CPU-read only while its memory has no import).
-    std::vector<std::pair<std::uint64_t, std::uint64_t>> presyncSurfaces;
+    std::vector<Graphics::PresyncSurface> presyncSurfaces;
     Graphics::HostImportsProof presyncProof;
     // Both false for every recipe (only reusable objects get one); kept for the asserts.
     bool needsCompletion = false;

core/libs/prx/libSceAgcDriver/Execution/include/RobustBufferAccess.hpp

@@ -0,0 +1,14 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_ROBUSTBUFFERACCESS_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_ROBUSTBUFFERACCESS_HPP
+
+#include <vulkan/vulkan.h>
+
+namespace AgcDriver {
+
+inline bool RobustBufferAccess2Usable(VkDriverId driver) {
+    return driver != VK_DRIVER_ID_AMD_PROPRIETARY;
+}
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp

@@ -30,6 +30,8 @@ struct ComputePipelineObjects;
 struct RecipeHit;
 // The record tail shared by an ordinary dispatch and a recipe hit (VulkanDevice::recordDispatch).
 struct RecordedDispatch;
+// The module, layout and pipeline a compute dispatch binds (VulkanDevice::createComputePipeline).
+struct ComputePipelineObjects;
 
 class VulkanDevice {
 public:
@@ -37,6 +39,14 @@ public:
     struct CompileOnly {};
     explicit VulkanDevice(CompileOnly);
     ComputePipelineReplayResult ReplayComputePipeline(const ComputePipelineCapture& capture, std::span<const std::byte> initialCache = {});
+    // Builds the pipeline of a journaled compute result ahead of its first dispatch (the journal
+    // replay's hook, on its threads): false when the device already holds a pipeline for the module.
+    bool ReplayComputePipeline(const ShaderRecompiler::RecompileResult& shader, bool pushConstants);
+    // Starts the journal replay if the device has not (a headless device starts it at its first
+    // draw or dispatch) and joins it (tests; the teardown joins it anyway).
+    void WaitForPipelineJournalReplay();
+    // Compute pipelines this device built, by dispatches and by the journal replay (tests).
+    std::uint64_t ComputePipelinesCreated() const;
     ~VulkanDevice();
     VulkanDevice(const VulkanDevice&) = delete;
     VulkanDevice& operator=(const VulkanDevice&) = delete;
@@ -45,9 +55,11 @@ public:
     bool CalibrateClock(double& gpuNs, std::chrono::steady_clock::time_point& cpu) const;
     ShaderRecompiler::SpirvTarget Target() const;
     std::optional<ShaderRecompiler::GeometryStageLimits> GeometryLimits() const;
+    std::uint32_t MaxFragmentInputComponents() const;
     const std::uint32_t* WorkGroupCountLimit() const;
     std::uint32_t VendorId() const;
     ShaderRecompiler::SpirvTarget ComputeTarget(std::uint32_t waveSize) const;
+    bool ComputeFullSubgroups() const;
     // Distinguishes this device from every earlier one in the process (a memo keyed by the device
     // cannot rely on the pointer, which a replacement may reuse).
     std::uint64_t Serial() const { return serial; }
@@ -58,6 +70,7 @@ public:
     // three-step form below so the GPU wait happens without the mutex.
     void WaitIdle();
     void PrepareForReplacement();
+    void startJournalReplay();
     bool ImportGuestMemory(const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice);
     // Sends recorded work to the GPU without waiting for it. With `reapFirst` it first retires batches
     // that already finished, so the in-flight list stays short (APS5_NO_OPPORTUNISTIC_REAP=1 skips
@@ -173,6 +186,10 @@ public:
         std::uint64_t created, underGpuMutex;
     };
     static ComputePipelineCreation ComputePipelineCounts();
+    struct DispatchMoveCount {
+        std::uint64_t checked, exact, moved, refused, filtered, rebuilt, kept, served;
+    };
+    static DispatchMoveCount DispatchMoveCounts();
     void ResolveMemory(std::uint64_t address, std::size_t bytes, bool writable);
     void* Window() const;
     void Resize(std::uint32_t width, std::uint32_t height);
@@ -232,6 +249,10 @@ public:
         double gpuBusyMs, gpuGapMs, lastSubmitAfterFlipMs, blitSubmitAfterFlipMs;
     };
     static PresentStatistics PresentCounts();
+    struct PresyncStatistics {
+        std::uint64_t checked, presyncs, keyChecks, keyWaits;
+    };
+    static PresyncStatistics PresyncCounts();
     // Stage A of a dispatch's resource build, run WITHOUT GuestMemory::GpuMutex before Dispatch or
     // DispatchIndirect: the binding plan, data buffers, imports already serving the guest buffers,
     // read-only copies and the descriptor set (see ShaderResources). The dispatch completes it
@@ -316,12 +337,16 @@ private:
     // The device's Graphics::Context: a copy of the one built at setup (its instance functions
     // resolved then, its function table filled then), or with APS5_NO_CONTEXT_CACHE=1 built anew.
     Graphics::Context graphicsContext() const;
+    VkPipelineCreateFlags computePipelineFlags() const;
+    std::uint32_t computeSubgroupSize(const ShaderRecompiler::RecompileResult& shader) const;
+    VkPipelineShaderStageCreateFlags computeStageFlags(const ShaderRecompiler::RecompileResult& shader) const;
+    std::shared_ptr<ComputePipelineObjects> createComputePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span<const std::uint32_t> layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress);
     ShaderRecompiler::SpirvTarget buildTarget() const;
     Graphics::Context buildContext() const;
     // Body of Dispatch and DispatchIndirect: `arguments` 0 dispatches x, y, z groups.
     IndirectOutcome dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t programAddress, std::shared_ptr<PreparedDispatch> prepared, std::shared_ptr<const Recipe>* recipe);
     // The stage-A pre-sync over `surfaces` (see PrepareDispatch): the serial waited for, 0 none.
-    std::uint64_t presync(std::span<const std::pair<std::uint64_t, std::uint64_t>> surfaces);
+    std::uint64_t presync(std::span<const Graphics::PresyncSurface> surfaces);
     std::shared_ptr<ComputePipelineObjects> computePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span<const std::uint32_t> layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress);
     // A DISPATCH_INDIRECT's argument path (see DispatchIndirect): GPU-side from the host import, or
     // the counts read on the CPU (the record then continues as a direct dispatch).

core/libs/prx/libSceAgcDriver/Execution/src/ComputePipelineCapture.cpp

@@ -94,7 +94,7 @@ void ComputePipelineCapture::Validate() const {
     RequireCapture(mainEntry, "compute main entry point missing");
     RequireCapture(deviceName.size() < VK_MAX_PHYSICAL_DEVICE_NAME_SIZE && deviceName.find('\0') == std::string::npos, "invalid device name");
     RequireCapture((flags & ~(VK_PIPELINE_CREATE_DISPATCH_BASE_BIT | VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR)) == 0, "unsupported pipeline flags");
-    RequireCapture(stageFlags == 0, "unsupported shader stage flags");
+    RequireCapture(stageFlags == 0 || (stageFlags == VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT && requiredSubgroupSize != 0), "unsupported shader stage flags");
     RequireCapture(requiredSubgroupSize == 0 || (requiredSubgroupSize <= 128 && (requiredSubgroupSize & (requiredSubgroupSize - 1u)) == 0), "invalid required subgroup size");
     RequireCapture(sets.size() <= 8, "too many descriptor sets");
     std::size_t totalBindings = 0;

core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp

@@ -130,7 +130,8 @@ std::size_t DisplayBufferSize(const DisplayBuffer& buffer) {
         throw std::runtime_error(message);
     }
     require(!halfFloat || buffer.dccAddress == 0, "VideoOut: DCC metadata on a 16-bit float display buffer is not implemented");
-    require(buffer.address != 0 && (buffer.address & 65535u) == 0, "VideoOut: display buffer requires 64 KiB alignment");
+    require(buffer.alignment == 65536u || (buffer.alignment == 4096u && buffer.tilingMode == 1), "VideoOut: 4 KiB display buffer alignment is only supported for linear buffers");
+    require(buffer.address != 0 && (buffer.address & (buffer.alignment - 1u)) == 0, buffer.alignment == 65536u ? "VideoOut: display buffer requires 64 KiB alignment" : "VideoOut: display buffer requires 4 KiB alignment");
     require(buffer.tilingMode <= 1, "VideoOut: unsupported display tiling mode");
     require(buffer.tilingMode != 0 || buffer.pitchInPixel == 0, "VideoOut: tiled display pitch is unsupported");
     require(buffer.dccAddress == 0 || buffer.tilingMode == 0, "VideoOut: a linear display buffer cannot carry DCC metadata");
@@ -249,7 +250,7 @@ std::vector<std::byte> ReadDisplayBuffer(const DisplayBuffer& buffer) {
         std::fprintf(stderr, "[gpu] scanout display buffer 0x%llx %ux%u format 0x%016llx at %.1f s\n", static_cast<unsigned long long>(buffer.address), buffer.width, buffer.height, static_cast<unsigned long long>(buffer.pixelFormat), std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count());
     }
     GuestMemory::FlushGpuWrites(buffer.address, size);
-    GuestMemory::CheckRange(reinterpret_cast<const void*>(buffer.address), size, 65536);
+    GuestMemory::CheckRange(reinterpret_cast<const void*>(buffer.address), size, buffer.alignment);
     const auto* source = reinterpret_cast<const std::byte*>(buffer.address);
     return DecodeDisplayBuffer(buffer, {source, size});
 }
@@ -267,7 +268,7 @@ std::vector<std::byte> DisplayBufferTexels(const DisplayBuffer& buffer, std::spa
 std::vector<std::byte> ReadDisplayBufferTexels(const DisplayBuffer& buffer) {
     const auto size = DisplayBufferSize(buffer);
     GuestMemory::FlushGpuWrites(buffer.address, size);
-    GuestMemory::CheckRange(reinterpret_cast<const void*>(buffer.address), size, 65536);
+    GuestMemory::CheckRange(reinterpret_cast<const void*>(buffer.address), size, buffer.alignment);
     return DisplayBufferTexels(buffer, {reinterpret_cast<const std::byte*>(buffer.address), size});
 }
 

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp

@@ -2,9 +2,11 @@
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferCopy.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
@@ -15,6 +17,47 @@
 
 namespace AgcDriver::DriverDetail {
 
+namespace {
+
+bool DispatchItemsEnabled() {
+    static const bool enabled = [] {
+        const char* text = std::getenv("APS5_PIPELINE_DISPATCH");
+        return text != nullptr && text[0] != '\0' && text[0] != '0';
+    }();
+    return enabled;
+}
+
+enum class ItemFallback : std::size_t { Miss, Reads, NoRecipe, Writes, Verify, Count };
+constexpr std::array<const char*, static_cast<std::size_t>(ItemFallback::Count)> ItemFallbackNames{"miss", "reads", "no recipe", "writes", "verify"};
+std::array<std::atomic<std::uint64_t>, static_cast<std::size_t>(ItemFallback::Count)> itemFallbacks{};
+std::atomic<std::uint64_t> itemCommits{0};
+std::atomic<std::uint64_t> itemRebuilds{0};
+
+void NoteItemFallback(ItemFallback reason) {
+    itemFallbacks[static_cast<std::size_t>(reason)].fetch_add(1, std::memory_order_relaxed);
+}
+
+void ReportDispatchItems(bool profile) {
+    static auto lastReport = std::chrono::steady_clock::now();
+    if (!profile) return;
+    const auto now = std::chrono::steady_clock::now();
+    if (now - lastReport < std::chrono::seconds(10)) return;
+    lastReport = now;
+    std::string drained;
+    for (std::size_t i = 0; i < ItemFallbackNames.size(); ++i) {
+        char item[48];
+        std::snprintf(item, sizeof(item), " %s %llu", ItemFallbackNames[i], static_cast<unsigned long long>(itemFallbacks[i].load(std::memory_order_relaxed)));
+        drained += item;
+    }
+    std::fprintf(stderr, "[dispatch-item] %llu commit items (%llu recipes rebuilt on the committer); drained instead, cumulative:%s\n", static_cast<unsigned long long>(itemCommits.load(std::memory_order_relaxed)), static_cast<unsigned long long>(itemRebuilds.load(std::memory_order_relaxed)), drained.c_str());
+}
+
+}
+
+bool Driver::pipelineDispatch() {
+    return DispatchItemsEnabled();
+}
+
 void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission, std::uint64_t indirectArguments) {
     PerformanceTimer timing("Driver.Dispatch");
     const auto address = (static_cast<std::uint64_t>(readRegister(queue.shader, 0x20c)) << 8u) | (static_cast<std::uint64_t>(readRegister(queue.shader, 0x20d) & 0xffu) << 40u);
@@ -72,6 +115,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
     if (indirectArguments == 0 && (packet[4] & 0x2u) != 0) {
         partialGroups = Pm4::PartialGroupThreads({packet[1], packet[2], packet[3]}, {readRegister(queue.shader, 0x207), readRegister(queue.shader, 0x208), readRegister(queue.shader, 0x209)});
     }
+    const bool pipelineCandidate = !resolving && !fromAhead && DispatchItemsEnabled() && DrawPipeline::Active() && indirectArguments == 0 && deferredLabels().labels.empty() && !matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute) && !MatchesStreamCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute);
     const bool partialLastGroup = partialGroups != std::array<std::uint32_t, 3>{};
     if (resolving) {
         if (!partialLastGroup && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return;
@@ -84,10 +128,11 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
             pendingDispatchPhases().outcome = DispatchOutcome::FillHle;
             return;
         }
-        if (DrawPipeline::Active()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, 0x15);
+        if (DrawPipeline::Active() && !pipelineCandidate) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, 0x15);
         if (const auto htile = Graphics::HtileDepthClearAddress(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads); htile != 0) {
             const auto threads = (packet[4] & 0x20u) != 0 ? static_cast<std::uint64_t>(packet[1]) : static_cast<std::uint64_t>(packet[1]) * compute.numThreads[0];
-            Graphics::NoteHtileDepthClear(htile, static_cast<std::size_t>(threads * 16u));
+            const auto nativeBytes = Graphics::HtileDepthClearBytes_nid_no_patch(std::span(snapshot.code).subspan(codeOffset), userData, compute, packet, indirectArguments);
+            Graphics::NoteHtileDepthClear(htile, static_cast<std::size_t>(threads * 16u), nativeBytes);
         }
         if (!partialLastGroup && indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) {
             pendingDispatchPhases().outcome = DispatchOutcome::CopyHle;
@@ -222,6 +267,27 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
     }
     phaseTiming.Phase(PhaseKey);
     timing.Mark("decode_key_device_setup");
+    bool readsClear = false;
+    if (pipelineCandidate) {
+        std::vector<ShaderRecompiler::MemoryRegion> reads;
+        bool entryFound = false;
+        {
+            std::lock_guard cacheLock(dispatchCacheMutex);
+            if (const auto found = dispatchCache.find(key); found != dispatchCache.end()) {
+                entryFound = true;
+                for (const auto& variant : found->second->variants) appendEntryRegions(*variant, reads);
+            }
+        }
+        if (!entryFound || noDispatchCache || stampValidate()) {
+            NoteItemFallback(ItemFallback::Miss);
+        } else if (verifyDataHits()) {
+            NoteItemFallback(ItemFallback::Verify);
+        } else {
+            readsClear = std::none_of(reads.begin(), reads.end(), [](const auto& region) { return DrawPipeline::Queue0().Overlaps(region.guestAddress, region.bytes.size()); });
+            if (!readsClear) NoteItemFallback(ItemFallback::Reads);
+        }
+        if (!readsClear && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, 0x15);
+    }
     if (fromAhead) {
         compiledResult = std::move(ahead.compiledResult);
         keepVariant = std::move(ahead.keepVariant);
@@ -234,6 +300,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
         cached = ahead.cached;
         validated = ahead.validated;
         ++groupStats.adopted;
+        ++dispatchesAdoptedAhead;
         const auto age = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - ahead.resolvedAt).count();
         groupStats.ageMs += age;
         groupStats.ageMaxMs = std::max(groupStats.ageMaxMs, age);
@@ -242,6 +309,19 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
         lookupDispatch(address, submission, key, noDispatchCache, traceCache, profile, memory, phaseTiming, phaseMs, compiledResult, keepVariant, captured, liveWords, dataHit, cached, validated, missedEntry, missedDiffering);
     }
     timing.Mark("lookup_cache");
+    std::vector<DrawPipeline::Range> itemWrites;
+    bool pipelined = false;
+    if (readsClear) {
+        if (cached && keepVariant != nullptr && keepVariant->recipe.load(std::memory_order_acquire) != nullptr) {
+            bool bounded = false;
+            itemWrites = DispatchWriteRanges(*compiledResult, bounded);
+            pipelined = bounded;
+            if (!bounded) NoteItemFallback(ItemFallback::Writes);
+        } else {
+            NoteItemFallback(ItemFallback::NoRecipe);
+        }
+        if (!pipelined && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, 0x15);
+    }
     if (cached) {
         require(keepVariant != nullptr && keepVariant->shader == registeredShader, "dispatch cache belongs to another registered shader");
         captureMs += phaseTiming.Elapsed();
@@ -359,6 +439,64 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
     };
     phaseTiming.Phase(PhaseSnapshots);
 
+    if (pipelined) {
+        const auto recipe = keepVariant->recipe.load(std::memory_order_acquire);
+        const auto epoch = DrawPipeline::EpochToken().load(std::memory_order_relaxed);
+        const auto queueId = submission.queue;
+        itemCommits.fetch_add(1, std::memory_order_relaxed);
+        DrawPipeline::Queue0().Enqueue([this, localDevice = std::move(localDevice), result = compiledResult, snapshots = std::move(snapshots), groups, address, queueId, recipe, keepVariant, dataHit, epoch] {
+            DrawPipeline::FollowEpoch(epoch);
+            GuestMemory::SetCurrentPacket(0x15, queueId);
+            const auto& compiledCommit = *result;
+            const bool noteWrites = writeEvidenceEnabled() || traceCapSync();
+            try {
+                std::shared_ptr<RecipeHit> recipeHit = localDevice->PrepareRecipe(recipe, false);
+                std::shared_ptr<const Recipe> builtRecipe;
+                bool writersNoted = false;
+                for (;;) {
+                    std::shared_ptr<PreparedDispatch> prepared;
+                    if (recipeHit == nullptr || VulkanDevice::VerifyRecipes()) prepared = localDevice->PrepareDispatch(compiledCommit, snapshots);
+                    GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Dispatch);
+                    std::lock_guard gpuLock(GuestMemory::GpuMutex());
+                    recordLabelsForPacket(localDevice.get(), queueId);
+                    if (noteWrites && writerKeyedEvidence() && !writersNoted) {
+                        noteWrittenBuffers(address, queueId, compiledCommit);
+                        writersNoted = true;
+                    }
+                    if (recipeHit != nullptr) {
+                        VulkanDevice::IndirectOutcome outcome{0, 0};
+                        const auto recipeResult = localDevice->DispatchRecipe(compiledCommit, groups[0], groups[1], groups[2], 0, address, recipeHit, outcome, VulkanDevice::VerifyRecipes() ? prepared : nullptr, dataHit);
+                        if (recipeResult == RecipeOutcome::Rebuild) {
+                            recipeHit = nullptr;
+                            VulkanDevice::NoteRecipe(VulkanDevice::RecipeEvent::Restart, false);
+                            itemRebuilds.fetch_add(1, std::memory_order_relaxed);
+                            continue;
+                        }
+                    } else {
+                        localDevice->Dispatch(compiledCommit, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared), &builtRecipe);
+                    }
+                    break;
+                }
+                if (noteWrites && !writerKeyedEvidence()) noteWrittenBuffers(address, queueId, compiledCommit);
+                if (builtRecipe != nullptr) {
+                    if (dataHit) {
+                        auto own = std::make_shared<Recipe>(*builtRecipe);
+                        own->dataWordsHash = Graphics::ShaderResources::DataWordsHash({ShaderRecompiler::ShaderStage::Compute, keepVariant->compiled.get(), 0});
+                        builtRecipe = std::move(own);
+                    }
+                    keepVariant->recipe.store(std::move(builtRecipe), std::memory_order_release);
+                    VulkanDevice::NoteRecipe(VulkanDevice::RecipeEvent::Attach, false);
+                }
+            } catch (const std::exception& error) {
+                char where[64];
+                std::snprintf(where, sizeof(where), "compute shader 0x%llx: ", static_cast<unsigned long long>(address));
+                throw std::runtime_error(where + std::string(error.what()));
+            }
+        }, std::move(itemWrites));
+        ReportDispatchItems(profile);
+        return;
+    }
+
     std::shared_ptr<RecipeHit> recipeHit;
     if (cached && keepVariant != nullptr && !stampValidate()) {
         recipeHit = localDevice->PrepareRecipe(keepVariant->recipe.load(std::memory_order_acquire), indirectArguments != 0);
@@ -370,6 +508,19 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
     bool writersNoted = false;
     const bool noteWrites = writeEvidenceEnabled() || traceCapSync();
 
+    std::optional<Graphics::DepthPlaneClear> depthPlaneClear;
+    if (indirectArguments == 0 && !partialLastGroup) {
+        if (const auto clear = Graphics::MatchStorageImageClear(std::span(snapshot.code).subspan(codeOffset), userData, compute, groups)) {
+            const auto clearAddress = Graphics::StorageImageClearAddress(*clear);
+            if (clearAddress != 0 && Graphics::DepthSurfaceAt(clearAddress)) {
+                GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Dispatch);
+                std::lock_guard clearLock(GuestMemory::GpuMutex());
+                recordLabelsForPacket(localDevice.get(), submission.queue);
+                depthPlaneClear.emplace(*clear);
+            }
+        }
+    }
+
     for (;;) {
 
         std::shared_ptr<PreparedDispatch> prepared;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp

@@ -35,6 +35,10 @@ bool& Driver::gateOpened() {
     return opened;
 }
 
+std::uint64_t Driver::DispatchesAdoptedAhead() const {
+    return dispatchesAdoptedAhead.load(std::memory_order_relaxed);
+}
+
 void Driver::clearResolvedAhead() {
     auto& resolved = resolvedAhead();
     if (resolved.empty()) return;
@@ -146,7 +150,7 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l
                 const auto argumentsEnd = arguments + 3u * sizeof(std::uint32_t);
                 const bool written = std::any_of(writes.begin(), writes.end(), [&](const auto& range) { return range.first < argumentsEnd && arguments < range.second; })
                     || std::any_of(labels.begin(), labels.end(), [&](const RunLabel& label) { return label.address < argumentsEnd && arguments < label.address + label.size; });
-                if (unknownWrites || written) {
+                if ((unknownWrites || written) && indirectArgumentsPath(packet, arguments) != IndirectGpu) {
                     ++stats.writtenBefore;
                     break;
                 }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp

@@ -23,14 +23,18 @@ void Driver::countIndirect(int path, double readMs) {
     std::fprintf(stderr, "[indirect] gpu-side %llu, cpu-side %llu (thread dimensions %llu, fill kernel %llu, pending image results %llu, pending label or copied write %llu, not imported %llu, misaligned %llu, disabled %llu, device memory LDS %llu); CPU argument reads took %.1f s\n", static_cast<unsigned long long>(counts[IndirectGpu]), static_cast<unsigned long long>(cpu), static_cast<unsigned long long>(counts[IndirectThreadDimensions]), static_cast<unsigned long long>(counts[IndirectFillKernel]), static_cast<unsigned long long>(counts[IndirectPendingImage]), static_cast<unsigned long long>(counts[IndirectCopiedWrite]), static_cast<unsigned long long>(counts[IndirectNotImported]), static_cast<unsigned long long>(counts[IndirectMisaligned]), static_cast<unsigned long long>(counts[IndirectDisabled]), static_cast<unsigned long long>(counts[IndirectWorkgroupMemory]), readWaitedMs / 1000);
 }
 
-void Driver::dispatchIndirect(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission) {
+int Driver::indirectArgumentsPath(std::span<const std::uint32_t> packet, std::uint64_t arguments) {
     static const bool gpuIndirect = std::getenv("APS5_NO_GPU_INDIRECT") == nullptr;
+    if (!gpuIndirect) return IndirectDisabled;
+    if ((packet.back() & 0x20u) != 0) return IndirectThreadDimensions;
+    if (arguments % 4 != 0) return IndirectMisaligned;
+    return IndirectGpu;
+}
+
+void Driver::dispatchIndirect(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission) {
     const auto arguments = Pm4::DispatchArgumentAddress(packet, queue);
     const auto initiator = packet.back();
-    int path = IndirectGpu;
-    if (!gpuIndirect) path = IndirectDisabled;
-    else if ((initiator & 0x20u) != 0) path = IndirectThreadDimensions;
-    else if (arguments % 4 != 0) path = IndirectMisaligned;
+    const int path = indirectArgumentsPath(packet, arguments);
     if (path != IndirectGpu) {
         if (!resolvingAhead()) {
             const auto& resolved = resolvedAhead();

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp

@@ -77,10 +77,26 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     std::shared_ptr<const DrawDecode> decode;
     std::shared_ptr<DrawEntry> dataEntry;
     std::uint64_t shapeKey = 0;
+    std::uint64_t drawIdentity = 0, drawAlias = 0;
+    std::vector<DrawUserPatch> userPatches;
     if (registerKey) {
         const auto keyStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
-        drawKey = drawRegisterKey(queue, *submission.shaders, localDevice->Serial(), &shapeKey, traceDrawCache() ? &drawKeyComponents : nullptr);
-        std::lock_guard cacheLock(drawCacheMutex);
+        std::shared_ptr<const DrawRelocation> relocation;
+        bool aliasFirst = false;
+        if (drawRelocationEnabled()) {
+            drawIdentity = drawIdentityKey(queue, *submission.shaders);
+            std::lock_guard cacheLock(drawCacheMutex);
+            learnPendingRelocation();
+            const auto known = drawRelocations.find(drawIdentity);
+            if (known != drawRelocations.end()) {
+                auto& trial = drawRelocationTrials[drawIdentity];
+                if (trial.idle != 0) --trial.idle;
+                else relocation = known->second;
+                aliasFirst = relocation != nullptr && trial.productive;
+            }
+        }
+        drawKey = drawRegisterKey(queue, *submission.shaders, localDevice->Serial(), &shapeKey, traceDrawCache() ? &drawKeyComponents : nullptr, aliasFirst ? relocation.get() : nullptr, aliasFirst ? &drawAlias : nullptr);
+        std::unique_lock cacheLock(drawCacheMutex);
         ++drawEntryCounters.lookups;
         ++drawEntryCounters.registerKeyLookups;
         if (profile) drawEntryCounters.keyUs += std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - keyStart).count();
@@ -90,6 +106,22 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
             decode = entry->decode;
         } else {
             ++drawEntryCounters.absent;
+            if (relocation != nullptr && !aliasFirst) {
+                cacheLock.unlock();
+                drawRegisterKey(queue, *submission.shaders, localDevice->Serial(), nullptr, nullptr, relocation.get(), &drawAlias);
+                cacheLock.lock();
+            }
+            if (drawIdentity != 0) drawRelocationPending = {drawKey, drawAlias, drawIdentity};
+            if (drawAlias != 0) relocateDrawEntry(queue, drawAlias, entry, decode, userPatches);
+            if (relocation != nullptr) {
+                auto& trial = drawRelocationTrials[drawIdentity];
+                ++trial.lookups;
+                if (trial.lookups >= DrawRelocationTrial::Window) {
+                    trial.productive = trial.hits * DrawRelocationTrial::MinimumHitShare >= trial.lookups;
+                    if (!trial.productive) trial.idle = DrawRelocationTrial::IdleDraws;
+                    trial.lookups = trial.hits = 0;
+                }
+            }
             if (traceDrawCache()) {
                 auto& counters = drawEntryCounters;
                 if (traceKeysEvicted.contains(drawKey)) ++counters.absentEvicted;
@@ -97,7 +129,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
                 else if (traceShapes.contains(shapeKey)) ++counters.absentUserWords;
                 else ++counters.absentNewRegisters;
             }
-            if (drawDataHits() && !drawParameters.indirect) {
+            if (decode == nullptr && drawDataHits() && !drawParameters.indirect) {
                 if (const auto shape = drawShapes.find(shapeKey); shape != drawShapes.end()) {
                     const auto kept = drawCache.find(shape->second);
                     if (kept != drawCache.end() && kept->second->decode != nullptr && kept->second->shape == shapeKey) dataEntry = kept->second;
@@ -113,6 +145,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         maybeReportDrawCache(profile);
     }
     if (decode != nullptr && !DecodeProgramsCurrent(*decode, *submission.shaders)) {
+        userPatches.clear();
         std::lock_guard cacheLock(drawCacheMutex);
         eraseDrawEntry(drawKey, entry);
         ++drawEntryCounters.absent;
@@ -144,6 +177,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     const auto& graphics = decode->state;
     auto pixel = decode->pixel;
     std::vector<DrawProgram> programs = decode->programs;
+    for (const auto& patch : userPatches) programs[patch.program].userData[patch.index] = patch.value;
     if (const auto skipped = SkippedShader(); skipped != 0 && std::any_of(programs.begin(), programs.end(), [skipped](const DrawProgram& program) { return program.binary.codeAddress == skipped; })) return drawn();
     const auto setMeshIndexBuffer = [&](const Pm4::DrawParameters& parameters) {
         if (!graphics.stages.mesh) return;
@@ -238,6 +272,25 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     bool verifyHit = false;
     const bool stageEntries = useDrawEntries && !(graphics.stages.tessellation && !drawParameters.indirect && drawParameters.instanceCount > 1u);
     lookupDraw(submission, localDevice, graphics, pixel, programs, roles, vertexInfos, stageEntries, registerKey, profile, drawKey, entry, matched, matchedRegions, drawHit, verifyHit, phaseTiming, phaseMs, dataEntry, dataCandidates, stageData);
+    for (std::size_t i = 0; i < matched.size(); ++i) {
+        if (matched[i] == nullptr || matched[i]->shaderDataPatches.empty()) continue;
+        applyShaderDataPatches(*matched[i]);
+        if (stageData[i].compiled != nullptr) stageData[i].compiled = patchDataWords(*matched[i], stageData[i].words);
+    }
+    const auto relocatedDecode = decode;
+    if (!userPatches.empty()) {
+        std::lock_guard cacheLock(drawCacheMutex);
+        if (drawHit || verifyHit) {
+            ++drawRelocationCounters.hits;
+            ++drawRelocationTrials[drawIdentity].hits;
+        }
+        if (profile && drawRelocationCounters.lookups >= 100000) reportDrawRelocation();
+    }
+    if (!userPatches.empty() && !drawHit) {
+        auto current = std::make_shared<DrawDecode>(*decode);
+        for (const auto& patch : userPatches) current->programs[patch.program].userData[patch.index] = patch.value;
+        decode = std::move(current);
+    }
     const bool patchedStages = std::any_of(stageData.begin(), stageData.end(), [](const StageDataHit& hit) { return hit.compiled != nullptr; });
     std::uint64_t stageDataVerified = 0;
 
@@ -383,15 +436,16 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         phaseTiming.Phase(DrawRowVectors);
         require(programs.size() == 2 && programResults[0] != nullptr && programResults[1] != nullptr, "rect-list requires vertex and fragment programs");
         auto rectangle = DrawRectangle(*programs[0].snapshot, programs[1].snapshot, programResults[0]->variantId, programResults[1]->variantId, localDevice->Target());
+        auto& control = graphics.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT ? rectangle.lastVertexControl : rectangle.control;
         if (rectListBuilt) {
-            results[rectIndex] = std::move(rectangle.control);
+            results[rectIndex] = std::move(control);
             results[rectIndex + 1] = std::move(rectangle.evaluation);
             phaseTiming.Phase(DrawRowRectList);
             return;
         }
         require(stages.size() == 2, "rect-list requires vertex and fragment programs");
         rectIndex = results.size();
-        results.push_back(std::move(rectangle.control));
+        results.push_back(std::move(control));
         results.push_back(std::move(rectangle.evaluation));
         stages.insert(stages.begin() + 1, {{Stage::TessellationControl, &results[rectIndex], 0}, {Stage::TessellationEvaluation, &results[rectIndex + 1], 0}});
         rectListBuilt = true;
@@ -400,8 +454,18 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     if (graphics.rectList) buildRectList();
     if (!programs.empty() && programResults.back() != nullptr && programResults.back()->barycentricEmulation.active) {
         require(!graphics.rectList && graphics.stages.path == Graphics::ShaderPath::Vertex && programs.size() == 2 && programResults[0] != nullptr && stages.size() == 2, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric needs a vertex shader before it");
-        results.push_back(ShaderRecompiler::BuildBarycentricGeometryShader(*programResults[0], *programResults[1], localDevice->Target(), localDevice->GeometryLimits()));
-        stages.insert(stages.begin() + 1, Graphics::CompiledShader{Stage::Geometry, &results.back(), 0});
+        const auto geometryLimits = localDevice->GeometryLimits();
+        if (geometryLimits || !localDevice->Target().tessellation) {
+            results.push_back(ShaderRecompiler::BuildBarycentricGeometryShader(*programResults[0], *programResults[1], localDevice->Target(), geometryLimits));
+            stages.insert(stages.begin() + 1, Graphics::CompiledShader{Stage::Geometry, &results.back(), 0});
+        } else {
+            require(graphics.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST && !graphics.primitiveRestart, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric or geometry shaders needs a triangle list without primitive restart");
+            auto patches = ShaderRecompiler::BuildBarycentricTessellationShaders(*programResults[0], *programResults[1], localDevice->Target(), localDevice->MaxFragmentInputComponents());
+            const auto first = results.size();
+            results.push_back(std::move(patches.control));
+            results.push_back(std::move(patches.evaluation));
+            stages.insert(stages.begin() + 1, {{Stage::TessellationControl, &results[first], 0}, {Stage::TessellationEvaluation, &results[first + 1], 0}});
+        }
     }
     std::vector<Graphics::GuestMemorySnapshot> snapshots;
     const auto snapshot = [&] {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp

@@ -102,7 +102,22 @@ bool Driver::admitDrawKey(std::uint64_t key, std::uint64_t shape) {
     std::lock_guard cacheLock(drawCacheMutex);
     if (drawCache.contains(key)) return true;
     if (!drawKeySightings) drawKeySightings.emplace(drawCacheEntries() * DrawKeySightings::Ways);
-    const bool seen = drawKeySightings->SeenBefore(key);
+    bool seen = drawKeySightings->SeenBefore(key);
+    const auto& pending = drawRelocationPending;
+    if (pending.key == key && pending.identity != 0) {
+        if (pending.alias != 0) {
+            const auto trial = drawRelocationTrials.find(pending.identity);
+            if (drawKeySightings->SeenBefore(pending.alias ^ 0x9e3779b97f4a7c15ull) && trial != drawRelocationTrials.end()) {
+                auto& state = trial->second;
+                if (state.productive || state.hits != 0 || state.admissions < DrawRelocationTrial::ProbationAdmissions) {
+                    if (!state.productive && state.hits == 0) ++state.admissions;
+                    seen = true;
+                }
+            }
+        } else if (!drawRelocations.contains(pending.identity)) {
+            if (drawKeySightings->SeenBefore(pending.identity ^ 0xc2b2ae3d27d4eb4full)) seen = true;
+        }
+    }
     if (seen || shape == 0 || drawShapes.contains(shape)) return seen;
     if (!drawShapeSightings) drawShapeSightings.emplace(drawCacheEntries() * DrawKeySightings::Ways);
     return drawShapeSightings->SeenBefore(shape);
@@ -125,7 +140,7 @@ void Driver::insertDrawEntry(std::uint64_t key, std::vector<std::shared_ptr<Disp
     for (std::size_t i = 0; i < fresh.size(); ++i) {
         if (fresh[i] == nullptr) continue;
         auto& variants = replacement->stages[i];
-        const auto present = std::find_if(variants.begin(), variants.end(), [&](const std::shared_ptr<DispatchVariant>& kept) { return kept->pushOffset == fresh[i]->pushOffset && kept->runs == fresh[i]->runs && kept->words == fresh[i]->words; });
+        const auto present = std::find_if(variants.begin(), variants.end(), [&](const std::shared_ptr<DispatchVariant>& kept) { return kept->pushOffset == fresh[i]->pushOffset && kept->meshIndexDescriptor == fresh[i]->meshIndexDescriptor && kept->runs == fresh[i]->runs && kept->words == fresh[i]->words; });
         if (present != variants.end()) {
             ++counters.present;
             fresh[i] = *present;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp

@@ -5,6 +5,7 @@
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "Optimization/ResourceProgram.hpp"
+#include <algorithm>
 #include <cstdlib>
 #include <cstring>
 #include <sstream>
@@ -55,6 +56,7 @@ std::shared_ptr<const ShaderRecompiler::RecompileResult> Driver::materializeDraw
     }();
 
     stageCapture.regions = shaderMemory.TakeRecentRegions();
+    stageCapture.userDataUse = capture->userDataUse;
     recompiled[i] = true;
     memory = shaderMemory.Regions();
 
@@ -148,6 +150,9 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP
             if (stageCapture.compiled == nullptr || !CacheableResult(*stageCapture.compiled)) continue;
             auto variant = std::make_shared<DispatchVariant>();
             variant->compiled = stageCapture.compiled;
+            if (programs[i].binary.stage == ShaderRecompiler::ShaderStage::Mesh) {
+                std::copy_n(programs[i].userData.begin() + ShaderRecompiler::MeshIndexBufferUserWord, variant->meshIndexDescriptor.size(), variant->meshIndexDescriptor.begin());
+            }
             variant->shader = programs[i].snapshot;
             variant->forgetSerial = stageCapture.forgetSerial;
             variant->pushOffset = stageCapture.pushOffset;
@@ -156,13 +161,7 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP
             std::vector<ShaderRecompiler::MemoryRegion> regions(stageCapture.regions.begin(), stageCapture.regions.end());
             for (const auto& read : decodeReads[i]) regions.push_back({read.address, std::as_bytes(std::span(read.bytes))});
             std::stable_sort(regions.begin(), regions.end(), [](const ShaderRecompiler::MemoryRegion& a, const ShaderRecompiler::MemoryRegion& b) { return a.guestAddress < b.guestAddress; });
-            for (const auto& region : regions) {
-                variant->runs.emplace_back(region.guestAddress, region.guestAddress + region.bytes.size());
-                const auto count = region.bytes.size() / sizeof(std::uint32_t);
-                const auto offset = variant->words.size();
-                variant->words.resize(offset + count);
-                std::memcpy(variant->words.data() + offset, region.bytes.data(), count * sizeof(std::uint32_t));
-            }
+            tagVariantRuns(*variant, regions, stageCapture.userDataUse, decodeReads[i].empty() && (!vertexInfos[i] || vertexInfos[i]->resourcesNum == 0), programs[i].hiddenUserWords);
             if (drawStageDataHits() && stageCapture.capture != nullptr && !stageCapture.capture->readTrace.leaves.empty()) recordStageDataPositions(*variant, *stageCapture.capture, decodeReads[i]);
             if (traceDrawCache() && entry != nullptr && matched[i] == nullptr && i < entry->stages.size()) {
                 const auto& variants = entry->stages[i];
@@ -174,7 +173,26 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP
                     ++differingRunsChanged;
                 }
             }
-            if (verifyHit && matched[i] != nullptr && (matched[i]->runs != variant->runs || matched[i]->words != variant->words)) {
+            const auto sameContent = [](const DispatchVariant& a, const DispatchVariant& b) {
+                const auto flatten = [](const DispatchVariant& source) {
+                    std::vector<std::pair<std::uint64_t, std::uint32_t>> flat;
+                    std::size_t word = 0;
+                    for (const auto& [begin, end] : source.runs) {
+                        for (auto address = begin; address < end && word < source.words.size(); address += sizeof(std::uint32_t)) flat.emplace_back(address, source.words[word++]);
+                    }
+                    std::sort(flat.begin(), flat.end());
+                    return flat;
+                };
+                return (a.runs == b.runs && a.words == b.words) || flatten(a) == flatten(b);
+            };
+            const auto sameForwardedWords = [](const ShaderRecompiler::RecompileResult& a, const ShaderRecompiler::RecompileResult& b, std::uint32_t hidden) {
+                if (a.pushConstants.size() != b.pushConstants.size() || a.shaderDataUserWords != b.shaderDataUserWords) return false;
+                for (std::size_t slot = 0; slot < a.shaderDataUserWords.size() && (slot + 1) * sizeof(std::uint32_t) <= a.pushConstants.size(); ++slot) {
+                    if (a.shaderDataUserWords[slot] >= hidden && std::memcmp(a.pushConstants.data() + slot * sizeof(std::uint32_t), b.pushConstants.data() + slot * sizeof(std::uint32_t), sizeof(std::uint32_t)) != 0) return false;
+                }
+                return true;
+            };
+            if (verifyHit && matched[i] != nullptr && (!sameContent(*matched[i], *variant) || !sameForwardedWords(*matched[i]->compiled, *variant->compiled, programs[i].hiddenUserWords))) {
                 ++mismatches;
                 static std::atomic<std::uint64_t> reports{0};
                 if (reports.fetch_add(1) < 20) std::fprintf(stderr, "[draw-cache] verify: stage %zu (program 0x%llx) of a hit captured differently: %zu runs / %zu words matched, %zu / %zu fresh\n", i, static_cast<unsigned long long>(programs[i].binary.codeAddress), matched[i]->runs.size(), matched[i]->words.size(), variant->runs.size(), variant->words.size());

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp

@@ -35,6 +35,7 @@ void InitializeMerged(const QueueState& queue, DrawProgram& program, std::uint32
     program.mergedPointerRequired = pointerRequired;
     program.firstUserSgpr = 0;
     program.userData.insert(program.userData.begin(), 8, 0);
+    program.hiddenUserWords = 8;
     Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, pointerBase);
     Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, pointerBase + 1);
     if (staticAbi) return;
@@ -67,7 +68,7 @@ void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const
         const auto& snapshot = *program;
         require((address - snapshot.codeAddress) % sizeof(std::uint32_t) == 0, "graphics entry point is not dword aligned");
         require(snapshot.header.empty() || snapshot.type == type, "graphics program refers to an incompatible shader binary type");
-        if (!nullPixel && snapshot.header.empty()) NoteRawGraphicsDraw();
+        if (!nullPixel && snapshot.header.empty()) NoteRawGraphicsDraw(snapshot);
         const auto codeOffset = static_cast<std::size_t>((address - snapshot.codeAddress) / sizeof(std::uint32_t));
         DrawProgram result{
             {stage, address, std::span(snapshot.code).subspan(codeOffset), snapshot.headerAddress, snapshot.header},

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp

@@ -1,6 +1,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include <algorithm>
 
 namespace AgcDriver::DriverDetail {
 
@@ -21,7 +22,7 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptr<Vulk
             mix(graphics.stages.mesh.has_value());
             if (graphics.stages.mesh) {
                 const auto& mesh = *graphics.stages.mesh;
-                for (const auto value : {mesh.inputPrimitive, mesh.primitivesPerGroup, mesh.verticesPerGroup, mesh.maxVertices, mesh.maxPrimitives, mesh.threadsPerGroup, mesh.ldsSizeDwords, mesh.provokingVertex, mesh.esgsItemSize}) mix(value);
+                for (const auto value : {mesh.inputPrimitive, mesh.primitivesPerGroup, mesh.verticesPerGroup, mesh.maxVertices, mesh.maxPrimitives, mesh.threadsPerGroup, mesh.ldsSizeDwords, mesh.provokingVertex, mesh.esgsItemSize, mesh.outputPrimitiveSize}) mix(value);
             }
             mix(graphics.stages.tessellation.has_value());
             if (graphics.stages.tessellation) {
@@ -107,6 +108,11 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptr<Vulk
                     for (std::size_t rank = 0; rank < variants.size(); ++rank) {
                         const auto& variant = variants[rank];
                         if (variant->pushOffset != cursor) continue;
+                        if (programs[i].binary.stage == ShaderRecompiler::ShaderStage::Mesh && !std::equal(variant->meshIndexDescriptor.begin(), variant->meshIndexDescriptor.end(), programs[i].userData.begin() + ShaderRecompiler::MeshIndexBufferUserWord)) {
+                            if (!anyLayout) outcome = EntryOutcome::Differing;
+                            anyLayout = true;
+                            continue;
+                        }
                         auto& regions = matchedRegions[i];
                         regions.clear();
                         appendEntryRegions(*variant, regions);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp

@@ -1,6 +1,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/HandoffSpin.hpp"
 #include <algorithm>
 #include <cstdio>
 #include <cstdlib>
@@ -14,6 +15,8 @@ namespace {
 
 constexpr std::array<const char*, static_cast<std::size_t>(DrawPipeline::DrainReason::Count)> DrainNames{"packet", "flush", "labels", "capture", "indirect", "submission"};
 
+std::atomic<unsigned> runningCommitters{0};
+
 std::uint64_t elapsedNs(std::chrono::steady_clock::time_point since) {
     return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - since).count());
 }
@@ -36,6 +39,12 @@ std::size_t DrawPipeline::Depth() {
     return depth;
 }
 
+std::size_t DrawPipeline::QueuedItems() {
+    auto& pipeline = Queue0();
+    std::lock_guard lock(pipeline.mutex);
+    return pipeline.items.size();
+}
+
 bool& DrawPipeline::Active() {
     static thread_local bool active = false;
     return active;
@@ -53,6 +62,10 @@ void DrawPipeline::FollowEpoch(std::uint64_t token) {
     GuestMemory::BumpCollectEpoch();
 }
 
+unsigned DrawPipeline::Committers() {
+    return runningCommitters.load(std::memory_order_acquire);
+}
+
 void DrawPipeline::rethrowFailure() {
     if (failure == nullptr) return;
     auto error = std::exchange(failure, nullptr);
@@ -63,6 +76,11 @@ void DrawPipeline::Enqueue(Commit commit, std::vector<Range> writes, std::uint64
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
     std::unique_lock lock(mutex);
     rethrowFailure();
+    if (closing) {
+        lock.unlock();
+        commit();
+        return;
+    }
     if (items.size() >= Depth()) {
         if (GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw pipeline full while the enqueuing thread holds the GPU lock");
         const auto start = std::chrono::steady_clock::now();
@@ -76,7 +94,10 @@ void DrawPipeline::Enqueue(Commit commit, std::vector<Range> writes, std::uint64
     ++enqueued;
     items.push_back({std::move(commit), std::move(writes), labelAddress, std::move(labelBytes)});
     outstanding.fetch_add(1, std::memory_order_release);
-    if (!thread.joinable()) thread = std::thread([this] { run(); });
+    if (!thread.joinable()) {
+        thread = std::thread([this] { run(); });
+        runningCommitters.fetch_add(1, std::memory_order_acq_rel);
+    }
     if (committerWaiting) wake.notify_one();
     if (!profile) return;
     const auto now = std::chrono::steady_clock::now();
@@ -113,11 +134,13 @@ void DrawPipeline::report(std::chrono::steady_clock::time_point now) {
 void DrawPipeline::run() {
     for (;;) {
         Commit* commit = nullptr;
+        SpinUntil(HandoffSpinWindow(), [&] { return outstanding.load(std::memory_order_acquire) != 0; });
         {
             std::unique_lock lock(mutex);
             committerWaiting = true;
-            wake.wait(lock, [&] { return !items.empty(); });
+            wake.wait(lock, [&] { return !items.empty() || closing; });
             committerWaiting = false;
+            if (items.empty()) break;
             commit = stopped ? nullptr : &items.front().commit;
         }
         const auto start = std::chrono::steady_clock::now();
@@ -149,12 +172,28 @@ void DrawPipeline::run() {
         outstanding.fetch_sub(1, std::memory_order_release);
         if (idleWaiters != 0) idle.notify_all();
     }
+    runningCommitters.fetch_sub(1, std::memory_order_acq_rel);
+}
+
+void DrawPipeline::Stop() {
+    std::thread committer;
+    {
+        std::lock_guard lock(mutex);
+        closing = true;
+        committer = std::move(thread);
+    }
+    wake.notify_all();
+    if (!committer.joinable()) return;
+    if (committer.get_id() == std::this_thread::get_id()) committer.detach();
+    else committer.join();
 }
 
 void DrawPipeline::Drain(DrainReason reason, std::uint32_t opcode) {
+    drainRequests[std::min<std::uint32_t>(opcode, 0x100)].fetch_add(1, std::memory_order_relaxed);
     if (!Busy()) return;
     if (GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw pipeline drain under the GPU lock (reason " + std::to_string(static_cast<unsigned>(reason)) + ", opcode " + std::to_string(opcode) + ")");
     const auto start = std::chrono::steady_clock::now();
+    SpinUntil(HandoffSpinWindow() * 2, [&] { return !Busy(); });
     std::unique_lock lock(mutex);
     ++idleWaiters;
     idle.wait(lock, [&] { return items.empty(); });
@@ -166,6 +205,10 @@ void DrawPipeline::Drain(DrainReason reason, std::uint32_t opcode) {
     rethrowFailure();
 }
 
+std::uint64_t DrawPipeline::DrainRequestsByOpcode(std::uint32_t opcode) {
+    return Queue0().drainRequests[std::min<std::uint32_t>(opcode, 0x100)].load(std::memory_order_relaxed);
+}
+
 std::optional<std::uint64_t> DrawPipeline::PendingLabel(std::uint64_t address, std::size_t bytes) {
     if (bytes == 0 || bytes > 8) return std::nullopt;
     const auto end = address + bytes;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp

@@ -47,7 +47,7 @@ std::optional<DrawVerdict> Driver::precheckDraw(const QueueState& queue, const S
             const auto primitive = queue.userConfig.find(0x242);
             const std::uint32_t allOnes = drawParameters.indexSize == 2 ? 0xffffu : 0xffffffffu;
             const auto type = primitive == queue.userConfig.end() ? 0u : primitive->second & 0x3fu;
-            const bool strip = type == 3 || type == 5 || type == 6;
+            const bool strip = type == 3 || type == 5 || type == 6 || type == 21;
             const bool list = type == 1 || type == 2 || type == 4;
             const auto restartDevice = device.Load();
             if (!strip && !(list && restartDevice != nullptr && restartDevice->PrimitiveListRestart())) {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawRegisterKey.hpp"
+#include "prx/libc/include/HostThreadLocal.hpp"
 #include <algorithm>
 #include <array>
 #include <cstdlib>
@@ -32,6 +33,20 @@ const auto& registerMasks() {
     return masks;
 }
 
+template <typename TVisit>
+void forEachKeyedUserWord(const QueueState& queue, bool allUserWords, TVisit&& visit) {
+    const auto words = [&](std::uint32_t base, std::uint32_t count) {
+        for (auto it = queue.shader.lower_bound(base); it != queue.shader.end() && it->first < base + count; ++it) visit(it->first, it->second);
+    };
+    for (const auto base : {0x00cu, 0x08cu, 0x10cu}) {
+        const auto resources = queue.shader.find(base - 1);
+        const auto count = resources == queue.shader.end() ? 0u : ((resources->second >> 1u) & 0x1fu) | (((resources->second >> 27u) & 1u) << 5u);
+        words(base, allUserWords ? 32u : std::min(count, 32u));
+    }
+    words(0x082u, 2);
+    words(0x102u, 2);
+}
+
 }
 
 std::uint64_t RegisterStateKey(const QueueState& queue, bool allUserWords, bool incremental, std::uint64_t* shapeKey, RegisterStateKeyParts* parts) {
@@ -85,13 +100,59 @@ std::uint64_t DrawShaderSetKey(const QueueState& queue, const ShaderRegistry& re
     return key;
 }
 
-std::uint64_t Driver::drawRegisterKey(const QueueState& queue, const ShaderRegistry& registry, std::uint64_t deviceSerial, std::uint64_t* shape, std::array<std::uint64_t, 5>* components) {
+std::uint64_t Driver::drawIdentityKey(const QueueState& queue, const ShaderRegistry& registry) {
+    std::uint64_t key = 0x84222325cbf29ce4ull;
+    const auto mix = [&](std::uint64_t value) {
+        key ^= value;
+        key *= 0x100000001b3ull;
+    };
+    for (const auto base : {0x008u, 0x088u, 0x0c8u, 0x108u, 0x148u}) {
+        const auto low = queue.shader.find(base);
+        const auto high = queue.shader.find(base + 1);
+        if (low == queue.shader.end() || high == queue.shader.end()) {
+            mix(0);
+            continue;
+        }
+        mix(low->second);
+        mix(high->second);
+        const auto resources = queue.shader.find(base + 3);
+        mix(resources != queue.shader.end() ? resources->second : 0);
+        const auto address = (static_cast<std::uint64_t>(low->second) << 8u) | (static_cast<std::uint64_t>(high->second & 0xffu) << 40u);
+        auto it = registry.upper_bound(address);
+        if (it == registry.begin()) {
+            mix(1);
+            continue;
+        }
+        --it;
+        for (const auto& snapshot : it->second) mix(reinterpret_cast<std::uintptr_t>(snapshot.get()));
+        mix(address - it->first);
+    }
+    return key == 0 ? 1 : key;
+}
+
+std::uint64_t Driver::drawRegisterKey(const QueueState& queue, const ShaderRegistry& registry, std::uint64_t deviceSerial, std::uint64_t* shape, std::array<std::uint64_t, 5>* components, const DrawRelocation* relocation, std::uint64_t* alias) {
+    const bool aliased = alias != nullptr && relocation != nullptr && !relocation->words.empty();
+    bool readable = true;
+    std::uint64_t other = 0x6c62272e07bb0142ull;
+    const auto mixOther = [&](std::uint64_t value) {
+        other ^= value;
+        other *= 0x100000001b3ull;
+    };
     static const bool allUserWords = std::getenv("APS5_DRAW_KEY_ALL_USER_WORDS") != nullptr;
     static const bool verify = std::getenv("APS5_VERIFY_REGISTER_KEYS") != nullptr;
     std::uint64_t shapeKey = 0;
     RegisterStateKeyParts parts;
     auto key = RegisterStateKey(queue, allUserWords, true, &shapeKey, components != nullptr ? &parts : nullptr);
     if (verify && key != RegisterStateKey(queue, allUserWords, false)) throw std::runtime_error("incremental register fingerprint disagrees with full state");
+    if (aliased) {
+        mixOther(shapeKey);
+        forEachKeyedUserWord(queue, allUserWords, [&](std::uint32_t offset, std::uint32_t value) {
+            mixOther(offset);
+            const auto word = std::find_if(relocation->words.begin(), relocation->words.end(), [&](const RelocatableUserWord& candidate) { return candidate.reg == offset; });
+            mixOther(word != relocation->words.end() ? relocatedContentKey(*word, value, readable) : value);
+        });
+        mixOther(deviceSerial);
+    }
     const auto mix = [&](std::uint64_t value) {
         key ^= value;
         key *= 0x100000001b3ull;
@@ -122,15 +183,21 @@ std::uint64_t Driver::drawRegisterKey(const QueueState& queue, const ShaderRegis
         --it;
         for (const auto& snapshot : it->second) mix(reinterpret_cast<std::uintptr_t>(snapshot.get()));
         mix(address - it->first);
+        if (aliased) {
+            for (const auto& snapshot : it->second) mixOther(reinterpret_cast<std::uintptr_t>(snapshot.get()));
+            mixOther(address - it->first);
+        }
     }
     if (shape != nullptr) *shape = shapeKey;
+    if (aliased && readable) *alias = other == 0 ? 1 : other;
     return key;
 }
 
 void Driver::traceDrawKeyLookup(const std::array<std::uint64_t, 5>& components, bool cacheMiss) {
     if (!traceDrawCache()) return;
     struct Previous { std::array<std::uint64_t, 4> parts{}; };
-    thread_local std::unordered_map<std::uint64_t, Previous> previous;
+    struct PreviousStorage {};
+    auto& previous = HostThreadLocal<std::unordered_map<std::uint64_t, Previous>, PreviousStorage>();
     const auto shaderSet = components[4];
     if (previous.size() >= 2048 && previous.find(shaderSet) == previous.end()) previous.clear();
     const auto [it, inserted] = previous.try_emplace(shaderSet);
@@ -168,7 +235,7 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) {
     const auto sameMesh = [](const std::optional<ShaderRecompiler::MeshConfiguration>& x, const std::optional<ShaderRecompiler::MeshConfiguration>& y) {
         if (x.has_value() != y.has_value()) return false;
         if (!x) return true;
-        return x->inputPrimitive == y->inputPrimitive && x->primitivesPerGroup == y->primitivesPerGroup && x->verticesPerGroup == y->verticesPerGroup && x->maxVertices == y->maxVertices && x->maxPrimitives == y->maxPrimitives && x->threadsPerGroup == y->threadsPerGroup && x->ldsSizeDwords == y->ldsSizeDwords && x->provokingVertex == y->provokingVertex && x->esgsItemSize == y->esgsItemSize;
+        return x->inputPrimitive == y->inputPrimitive && x->primitivesPerGroup == y->primitivesPerGroup && x->verticesPerGroup == y->verticesPerGroup && x->maxVertices == y->maxVertices && x->maxPrimitives == y->maxPrimitives && x->threadsPerGroup == y->threadsPerGroup && x->ldsSizeDwords == y->ldsSizeDwords && x->provokingVertex == y->provokingVertex && x->esgsItemSize == y->esgsItemSize && x->outputPrimitiveSize == y->outputPrimitiveSize;
     };
     const auto sameTess = [](const std::optional<ShaderRecompiler::TessellationConfiguration>& x, const std::optional<ShaderRecompiler::TessellationConfiguration>& y) {
         if (x.has_value() != y.has_value()) return false;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp

@@ -0,0 +1,269 @@
+#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+
+namespace AgcDriver::DriverDetail {
+
+namespace {
+
+constexpr std::size_t MaxAliases = 16384;
+constexpr std::size_t MaxContentWords = 64;
+
+std::uint64_t AlignedWord(std::uint32_t word) {
+    return word & ~std::uint32_t {3};
+}
+
+}
+
+bool Driver::drawRelocationEnabled() {
+    static const bool enabled = std::getenv("APS5_NO_DRAW_RELOCATION") == nullptr;
+    return enabled;
+}
+
+std::uint64_t Driver::relocatedContentKey(const RelocatableUserWord& word, std::uint32_t value, bool& readable) {
+    std::uint64_t hash = 0x9e3779b97f4a7c15ull;
+    const auto mix = [&](std::uint64_t mixed) {
+        hash ^= mixed;
+        hash *= 0x100000001b3ull;
+    };
+    mix(value & 3u);
+    if (value == 0) {
+        readable = false;
+        return hash;
+    }
+    std::array<std::uint32_t, MaxContentWords> content;
+    for (const auto& [offset, words] : word.reads) {
+        mix(static_cast<std::uint64_t>(offset));
+        mix(words);
+        for (std::size_t done = 0; done < words; done += MaxContentWords) {
+            const auto count = std::min<std::size_t>(words - done, MaxContentWords);
+            const auto address = AlignedWord(value) + static_cast<std::uint64_t>(offset) + done * sizeof(std::uint32_t);
+            if (GuestMemory::CopyMapped(address, std::as_writable_bytes(std::span(content).first(count))) != GuestMemory::Compare::Equal) {
+                readable = false;
+                return hash;
+            }
+            for (std::size_t i = 0; i < count; ++i) mix(content[i]);
+        }
+    }
+    return hash;
+}
+
+void Driver::tagVariantRuns(DispatchVariant& variant, std::span<const ShaderRecompiler::MemoryRegion> regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords) {
+    std::uint64_t relocatable = eligible && drawRelocationEnabled() ? use.asBase & ~use.asValue : 0;
+    if (hiddenWords != 0) relocatable &= hiddenWords >= 64 ? 0 : ~((std::uint64_t {1} << hiddenWords) - 1);
+    std::vector<std::pair<std::uint64_t, std::int32_t>> reads;
+    const auto covered = [&](std::uint64_t address) {
+        return std::any_of(regions.begin(), regions.end(), [&](const ShaderRecompiler::MemoryRegion& region) { return address >= region.guestAddress && address - region.guestAddress + sizeof(std::uint32_t) <= region.bytes.size(); });
+    };
+    if (relocatable != 0) {
+        reads = use.reads;
+        std::sort(reads.begin(), reads.end());
+        reads.erase(std::unique(reads.begin(), reads.end()), reads.end());
+        const auto drop = [&](std::int32_t origin) {
+            if (origin >= 0) relocatable &= ~(std::uint64_t {1} << origin);
+        };
+        for (std::size_t i = 0; i < reads.size(); ++i) {
+            if (i + 1 < reads.size() && reads[i].first == reads[i + 1].first) {
+                drop(reads[i].second);
+                drop(reads[i + 1].second);
+            }
+            if (reads[i].second >= 0 && ((reads[i].first & 3u) != 0 || !covered(reads[i].first))) drop(reads[i].second);
+        }
+    }
+    const auto origin = [&](std::uint64_t address) -> std::int8_t {
+        const auto found = std::lower_bound(reads.begin(), reads.end(), std::make_pair(address, std::numeric_limits<std::int32_t>::min()));
+        if (found == reads.end() || found->first != address || found->second < 0) return -1;
+        return (relocatable >> found->second & 1u) != 0 ? static_cast<std::int8_t>(found->second) : std::int8_t {-1};
+    };
+    variant.relocatableWords = relocatable;
+    for (const auto& region : regions) {
+        const auto count = region.bytes.size() / sizeof(std::uint32_t);
+        const auto offset = variant.words.size();
+        variant.words.resize(offset + count);
+        std::memcpy(variant.words.data() + offset, region.bytes.data(), count * sizeof(std::uint32_t));
+        if (relocatable == 0) {
+            variant.runs.emplace_back(region.guestAddress, region.guestAddress + region.bytes.size());
+            continue;
+        }
+        std::size_t start = 0;
+        auto current = count != 0 ? origin(region.guestAddress) : std::int8_t {-1};
+        for (std::size_t word = 1; word <= count; ++word) {
+            const auto next = word < count ? origin(region.guestAddress + word * sizeof(std::uint32_t)) : std::int8_t {-2};
+            if (next == current) continue;
+            variant.runs.emplace_back(region.guestAddress + start * sizeof(std::uint32_t), region.guestAddress + word * sizeof(std::uint32_t));
+            variant.runBases.push_back(current);
+            start = word;
+            current = next;
+        }
+    }
+}
+
+namespace {
+
+template <typename TMap>
+std::size_t dropEveryOther(TMap& map) {
+    std::size_t dropped = 0;
+    bool drop = false;
+    for (auto it = map.begin(); it != map.end();) {
+        drop = !drop;
+        if (!drop) {
+            ++it;
+            continue;
+        }
+        it = map.erase(it);
+        ++dropped;
+    }
+    return dropped;
+}
+
+}
+
+void Driver::learnPendingRelocation() {
+    const auto pending = drawRelocationPending;
+    drawRelocationPending = {};
+    if (pending.key == 0) return;
+    const auto inserted = drawCache.find(pending.key);
+    if (inserted == drawCache.end() || inserted->second->decode == nullptr) return;
+    const auto& entry = *inserted->second;
+    const auto& programs = entry.decode->programs;
+    if (entry.stages.size() != programs.size()) return;
+    const auto known = drawRelocations.find(pending.identity);
+    auto shape = std::make_shared<DrawRelocation>();
+    if (known != drawRelocations.end()) *shape = *known->second;
+    bool changed = known == drawRelocations.end();
+    for (std::size_t i = 0; i < programs.size(); ++i) {
+        if (entry.stages[i].empty()) continue;
+        const auto& variant = *entry.stages[i].front();
+        const auto& program = programs[i];
+        for (std::uint32_t index = program.hiddenUserWords; index < program.userData.size() && index < 64; ++index) {
+            const auto reg = program.userDataBase + index - program.hiddenUserWords;
+            const auto held = std::find_if(shape->words.begin(), shape->words.end(), [&](const RelocatableUserWord& word) { return word.reg == reg; });
+            if ((variant.relocatableWords >> index & 1u) == 0) {
+                if (held == shape->words.end()) continue;
+                shape->words.erase(held);
+                shape->banned.push_back(reg);
+                changed = true;
+                continue;
+            }
+            if (held != shape->words.end() || std::find(shape->banned.begin(), shape->banned.end(), reg) != shape->banned.end()) continue;
+            RelocatableUserWord word{reg, {}};
+            const auto base = AlignedWord(program.userData[index]);
+            for (std::size_t run = 0; run < variant.runs.size() && run < variant.runBases.size(); ++run) {
+                if (variant.runBases[run] != static_cast<std::int8_t>(index)) continue;
+                word.reads.emplace_back(static_cast<std::int64_t>(variant.runs[run].first - base), static_cast<std::uint32_t>((variant.runs[run].second - variant.runs[run].first) / sizeof(std::uint32_t)));
+            }
+            shape->words.push_back(std::move(word));
+            changed = true;
+        }
+    }
+    if (changed) {
+        std::sort(shape->words.begin(), shape->words.end(), [](const RelocatableUserWord& a, const RelocatableUserWord& b) { return a.reg < b.reg; });
+        drawRelocations[pending.identity] = std::move(shape);
+    }
+    if (pending.alias == 0) return;
+    if (drawAliases.size() >= MaxAliases) drawRelocationCounters.evicted += dropEveryOther(drawAliases);
+    drawAliases[pending.alias] = inserted->second;
+}
+
+void Driver::relocateDrawEntry(const QueueState& queue, std::uint64_t alias, std::shared_ptr<DrawEntry>& entry, std::shared_ptr<const DrawDecode>& decode, std::vector<DrawUserPatch>& patches) {
+    auto& counters = drawRelocationCounters;
+    ++counters.lookups;
+    const auto found = drawAliases.find(alias);
+    if (found == drawAliases.end() || found->second->decode == nullptr) return;
+    const auto donor = found->second;
+    const auto& programs = donor->decode->programs;
+    if (donor->stages.size() != programs.size()) return;
+    struct Move {
+        std::size_t program;
+        std::uint32_t index, from, to;
+    };
+    std::vector<Move> moves;
+    for (std::size_t i = 0; i < programs.size(); ++i) {
+        const auto& program = programs[i];
+        for (std::uint32_t index = program.hiddenUserWords; index < program.userData.size(); ++index) {
+            const auto current = readUserData(queue.shader, program.userDataBase + index - program.hiddenUserWords);
+            if (current != program.userData[index]) moves.push_back({i, index, program.userData[index], current});
+        }
+    }
+    if (moves.empty()) return;
+    ++counters.donors;
+    auto relocated = std::make_shared<DrawEntry>();
+    relocated->decode = donor->decode;
+    relocated->stages.resize(donor->stages.size());
+    for (std::size_t i = 0; i < donor->stages.size(); ++i) {
+        const bool moved = std::any_of(moves.begin(), moves.end(), [&](const Move& move) { return move.program == i; });
+        if (!moved) {
+            relocated->stages[i] = donor->stages[i];
+            continue;
+        }
+        for (const auto& variant : donor->stages[i]) {
+            if (variant == nullptr || variant->compiled == nullptr) continue;
+            const bool allowed = std::all_of(moves.begin(), moves.end(), [&](const Move& move) { return move.program != i || (move.index < 64 && (variant->relocatableWords >> move.index & 1u) != 0 && move.from != 0 && move.to != 0 && ((move.from ^ move.to) & 3u) == 0); });
+            if (!allowed) {
+                ++counters.refused;
+                continue;
+            }
+            auto copy = std::make_shared<DispatchVariant>();
+            copy->runs = variant->runs;
+            copy->runBases = variant->runBases;
+            copy->relocatableWords = variant->relocatableWords;
+            copy->words = variant->words;
+            copy->forgetSerial = variant->forgetSerial;
+            copy->compiled = variant->compiled;
+            copy->meshIndexDescriptor = variant->meshIndexDescriptor;
+            copy->shader = variant->shader;
+            copy->dataPositions = variant->dataPositions;
+            copy->dataSlots = variant->dataSlots;
+            copy->flatBinding = variant->flatBinding;
+            copy->pushOffset = variant->pushOffset;
+            copy->vertexInfo = variant->vertexInfo;
+            for (const auto& move : moves) {
+                if (move.program != i) continue;
+                const auto delta = AlignedWord(move.to) - AlignedWord(move.from);
+                for (std::size_t run = 0; run < copy->runs.size() && run < copy->runBases.size(); ++run) {
+                    if (copy->runBases[run] != static_cast<std::int8_t>(move.index)) continue;
+                    copy->runs[run].first += delta;
+                    copy->runs[run].second += delta;
+                }
+                const auto& forwarded = variant->compiled->shaderDataUserWords;
+                for (std::uint32_t slot = 0; slot < forwarded.size(); ++slot) {
+                    if (forwarded[slot] == move.index) copy->shaderDataPatches.emplace_back(slot, move.to);
+                }
+            }
+            relocated->stages[i].push_back(std::move(copy));
+        }
+    }
+    patches.clear();
+    for (const auto& move : moves) patches.push_back({move.program, move.index, move.to});
+    entry = std::move(relocated);
+    decode = donor->decode;
+}
+
+void Driver::applyShaderDataPatches(DispatchVariant& variant) {
+    if (variant.shaderDataPatches.empty()) return;
+    auto rebased = std::make_shared<ShaderRecompiler::RecompileResult>(*variant.compiled);
+    for (const auto& [slot, value] : variant.shaderDataPatches) {
+        const auto at = static_cast<std::size_t>(slot) * sizeof(std::uint32_t);
+        if (at + sizeof(std::uint32_t) <= rebased->pushConstants.size()) std::memcpy(rebased->pushConstants.data() + at, &value, sizeof(value));
+        for (auto& binding : rebased->bindings) {
+            if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData && slot < binding.guestDescriptor.size()) binding.guestDescriptor[slot] = value;
+        }
+    }
+    variant.compiled = std::move(rebased);
+    variant.shaderDataPatches.clear();
+}
+
+void Driver::reportDrawRelocation() {
+    auto& counters = drawRelocationCounters;
+    AgcDriver::ProfilePrint_nid_no_patch("[draw-relocate] %llu alias lookups: %llu donors with moved user words, %llu refusals, %llu hits through a relocated entry; %zu shader sets learned, %zu aliases held (%llu dropped at the limit)\n", static_cast<unsigned long long>(counters.lookups), static_cast<unsigned long long>(counters.donors), static_cast<unsigned long long>(counters.refused), static_cast<unsigned long long>(counters.hits), drawRelocations.size(), drawAliases.size(), static_cast<unsigned long long>(counters.evicted));
+    counters = {};
+}
+
+}

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp

@@ -31,7 +31,9 @@ std::optional<Graphics::IndirectDrawPath> Driver::ClassifyIndirectDraw(const Sha
     constexpr std::uint32_t NoLocation = 0x280u;
     const bool meshPath = graphics.stages.mesh.has_value() && !graphics.stages.tessellation && !graphics.rectList;
     const bool meshGpuSide = meshPath && drawParameters.indexed && indirect.count == 1 && !indirect.countIndirect && indirect.baseVertexLocation == NoLocation && indirect.startInstanceLocation == NoLocation && (!indirect.drawIndexEnabled || indirect.drawIndexLocation == NoLocation);
-    if ((graphics.stages.path != Graphics::ShaderPath::Vertex || graphics.rectList) && !meshGpuSide) fail(Path::NonVertexPath);
+    static const bool rectListOnGpu = std::getenv("APS5_NO_RECT_INDIRECT") == nullptr;
+    const bool rectListGpuSide = rectListOnGpu && graphics.rectList && !graphics.stages.mesh && !graphics.stages.tessellation;
+    if ((graphics.stages.path != Graphics::ShaderPath::Vertex || (graphics.rectList && !rectListGpuSide)) && !meshGpuSide) fail(Path::NonVertexPath);
     if (indirect.drawIndexEnabled && indirect.drawIndexSgpr >= 0) fail(Path::DrawIndex);
     if (indirect.countIndirect && !support.count) fail(Path::FeatureGap);
     if (indirect.countIndirect && indirect.count > 1 && !support.multi) fail(Path::FeatureGap);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp

@@ -99,6 +99,10 @@ extern "C" std::uint32_t AgcDriverReserveEopFlip_nid_postfix(const AgcDriver::Fl
     return AgcDriver::DriverDetail::Driver::Get().ReserveEopFlip(info);
 }
 
+extern "C" std::uint64_t AgcDriverDispatchesAdoptedAhead_nid_postfix() {
+    return AgcDriver::DriverDetail::Driver::Get().DispatchesAdoptedAhead();
+}
+
 extern "C" void AgcDriverRegisterVideoOutput_nid_postfix(std::uint32_t handle, const std::shared_ptr<AgcDriver::IVideoOutput>& output) {
     AgcDriver::RegisterVideoOutput(handle, output);
 }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp

@@ -1,12 +1,18 @@
+#include "prx/libSceAgcDriver/Execution/include/AsyncLog.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
+#include "ShaderDiskCache.hpp"
 #include <atomic>
 #include <cstdint>
 #include <cstdio>
@@ -38,6 +44,7 @@ void PrintFailure(std::exception_ptr error) {
     } catch (...) {
         std::fprintf(stderr, "[gpu] driver failure with a non-standard exception\n");
     }
+    FlushAsyncLog_nid_no_patch();
     std::fflush(stderr);
 }
 
@@ -66,8 +73,7 @@ bool StackHoldsImport() {
     const auto low = bounds.low;
     const auto high = bounds.high;
 #endif
-    const volatile unsigned char marker = 0;
-    const auto frame = reinterpret_cast<std::uintptr_t>(&marker);
+    const auto frame = reinterpret_cast<std::uintptr_t>(__builtin_frame_address(0));
     return frame >= low && frame < high && frame - low >= InlineImportStackBytes;
 }
 
@@ -112,18 +118,25 @@ void Driver::stop() {
     for (auto& [queue, worker] : workers) {
         if (worker.thread.joinable()) worker.thread.join();
     }
+    DrawPipeline::Queue0().Stop();
     StopWorkerSampler();
     CloseRegistrationPreparation();
+    Graphics::StopPipelineLibraryWorker();
     std::lock_guard gpuLock(GuestMemory::GpuMutex());
     device.Reset();
     replacedDevices.clear();
+    Graphics::FlushPipelineJournal();
     Graphics::ShutdownGuestBufferWorkers();
+    Graphics::StopCompareHelpers();
+    ShaderRecompiler::ShaderDiskCache::Shutdown();
     PerformanceTrace::StopOutput();
+    FlushAsyncLogAtEnd_nid_no_patch();
     StopProfileOutput_nid_no_patch();
     stopped = true;
 }
 
 Driver::Driver() {
+    InstallAsyncLog_nid_no_patch();
     static_cast<void>(GuestMemory::TrackerGeneration());
     ShaderMemory::SetWaitedMsProvider(&Graphics::Recorder::ThreadWaitedMs);
     try {
@@ -216,3 +229,11 @@ bool& Driver::onWorkerThread() {
 }
 
 }
+
+namespace AgcDriver {
+
+std::unique_lock<std::mutex> HoldAsyncLog() {
+    return AsyncLog::Get().Hold();
+}
+
+}

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp

@@ -50,6 +50,31 @@ void Driver::noteWrittenBuffers(std::uint64_t program, std::uint32_t queue, cons
     while (writtenBuffers.size() > WrittenBufferRing) writtenBuffers.pop_front();
 }
 
+std::vector<std::pair<std::uint64_t, std::uint64_t>> DispatchWriteRanges(const ShaderRecompiler::RecompileResult& compiled, bool& bounded) {
+    bounded = compiled.bdaAbiVersion == 0;
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> ranges;
+    constexpr std::uint64_t UnknownImageBytes = 64ull * 1024 * 1024;
+    for (const auto& binding : compiled.bindings) {
+        if (binding.role == ShaderRecompiler::DescriptorRole::Gds || binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable || binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer) bounded = false;
+        if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages || binding.kind != ShaderRecompiler::DescriptorKind::StorageImage) continue;
+        for (std::uint32_t element = 0; element < binding.count; ++element) {
+            const bool written = element >= binding.imageWritten.size() || binding.imageWritten[element];
+            if (!written) continue;
+            if (binding.guestDescriptor.size() < (static_cast<std::size_t>(element) + 1) * 8) {
+                bounded = false;
+                continue;
+            }
+            const auto resource = Graphics::DecodeTextureResource(std::span<const std::uint32_t>(binding.guestDescriptor).subspan(static_cast<std::size_t>(element) * 8, 8));
+            if (resource.baseAddress == 0) continue;
+            const auto surfaceBytes = Graphics::DescribeSurface(resource).guestBytes;
+            ranges.emplace_back(resource.baseAddress, resource.baseAddress + (surfaceBytes != 0 ? surfaceBytes : UnknownImageBytes));
+            if (resource.dccAddress != 0) ranges.emplace_back(resource.dccAddress, resource.dccAddress + (surfaceBytes != 0 ? surfaceBytes / 256 + 1 : UnknownImageBytes));
+        }
+    }
+    visitWrittenBuffers(compiled, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool) { ranges.emplace_back(begin, end); });
+    return ranges;
+}
+
 void Driver::noteForeignWriter(std::uint64_t begin, std::uint64_t end, std::uint32_t queue) {
     if (!foreignWriters() || !(writeEvidenceEnabled() || traceCapSync()) || end <= begin) return;
     std::lock_guard lock(writtenBuffersMutex);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp

@@ -7,6 +7,7 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
 #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/PerformanceOverlay.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
 #include "prx/libSceAgcDriver/Eq/include/Event.hpp"
@@ -54,6 +55,7 @@ void Driver::timed(double WorkerProfile::*bucket, TWork&& work) {
         profile.reported = end;
         AgcDriver::ProfilePrint_nid_no_patch( "[gpu] worker at %.0f s: dispatch %.1f s, draw %.1f s, wait %.1f s\n", std::chrono::duration<double>(end - profile.start).count(), profile.dispatchMs / 1000, profile.drawMs / 1000, profile.waitMs / 1000);
     }
+    if (report && bucket != &WorkerProfile::waitMs) PerformanceCounters::AddDriverNanoseconds(static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(end - begin).count()));
 }
 
 void Driver::execute(const Submission& submission) {
@@ -149,6 +151,7 @@ void Driver::execute(const Submission& submission) {
     thread_local PacketProfile* packetProfileSlot = nullptr;
     auto& packetProfile = ShaderRecompiler::ThreadOwned(packetProfileSlot);
     ++packetProfile.submissions;
+    auto& packetCounters = PacketCounters::ThisThread();
 
     static const bool epochEach = std::getenv("APS5_SUBMISSION_EPOCH_EACH") != nullptr;
     if (epochEach || submission.enqueuedAt == std::chrono::steady_clock::time_point{} || submission.enqueuedAt > lastEpochBump()) {
@@ -310,12 +313,15 @@ void Driver::execute(const Submission& submission) {
             flipHold.Release();
             submission.flips.at(cursor)->GpuReady(frame);
         } else if (opcode == 0x15 || opcode == 0x16) {
+            packetCounters.CountDispatch();
             landPendingWaits(submission.queue);
             currentPacketOffset() = cursor;
             if (gateOpened()) {
                 gateOpened() = false;
-                if (pipelined && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, opcode);
-                timed(&WorkerProfile::dispatchMs, [&] { resolveGroupAhead(submission, queue, cursor); });
+                if (!pipelineDispatch() || submission.queue != 0) {
+                    if (pipelined && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, opcode);
+                    timed(&WorkerProfile::dispatchMs, [&] { resolveGroupAhead(submission, queue, cursor); });
+                }
             }
             if (opcode == 0x15) timed(&WorkerProfile::dispatchMs, [&] { dispatch(queue, packet, submission); });
             else timed(&WorkerProfile::dispatchMs, [&] { dispatchIndirect(queue, packet, submission); });
@@ -341,6 +347,7 @@ void Driver::execute(const Submission& submission) {
             }
         } else if (drawPacket) {
             landPendingWaits(submission.queue);
+            packetCounters.CountDraw();
             bool drawn = false;
             timed(&WorkerProfile::drawMs, [&] {
                 static const bool traceDraws = std::getenv("APS5_TRACE_DRAWS") != nullptr;
@@ -409,7 +416,10 @@ void Driver::execute(const Submission& submission) {
             return;
         }
         if ((std::atomic_ref<std::uint32_t>(*const_cast<std::uint32_t*>(submission.rewindTail - 1)).load(std::memory_order_acquire) & 0x80000000u) != 0) {
-            DrawPipeline::Queue0().Enqueue([this] { submitOpenWork(); }, {});
+            DrawPipeline::Queue0().Enqueue([this] {
+                submitOpenWork();
+                wakeQueueZeroForCompletions();
+            }, {});
             executeRewindTail(submission);
             return;
         }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Queues/SubmissionRing.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
@@ -84,6 +85,12 @@ void Driver::submitOpenWork() {
     if (localDevice != nullptr) localDevice->SubmitRecorded(true);
 }
 
+void Driver::wakeQueueZeroForCompletions() {
+    if (!completionsPending()) return;
+    std::lock_guard lock(mutex);
+    changed.notify_all();
+}
+
 void Driver::completeSubmission(std::uint64_t serial, std::uint64_t received, std::span<const std::uint64_t> labelWrites) {
     struct Complete {
         Driver& driver;
@@ -97,7 +104,7 @@ void Driver::completeSubmission(std::uint64_t serial, std::uint64_t received, st
                 driver.forgetUnfinishedWrites(driver.workers.at(0), labelWrites);
                 const auto found = std::find(driver.queue0Uncommitted.begin(), driver.queue0Uncommitted.end(), received);
                 if (found != driver.queue0Uncommitted.end()) driver.queue0Uncommitted.erase(found);
-                notify = driver.idleWaiters != 0 || driver.orderHolders.load(std::memory_order_acquire) != 0;
+                notify = driver.idleWaiters != 0 || driver.orderHolders.load(std::memory_order_acquire) != 0 || completionsPending();
             }
             if (notify) driver.changed.notify_all();
         }
@@ -210,12 +217,12 @@ void Driver::run(std::uint32_t id) noexcept {
                         lock.lock();
                         if (ready()) break;
                         if (id == 0 && allQueuesIdle() && !completionsPending()
-                            && (!storeIdleResults || idleEpoch.load(std::memory_order_acquire) != idleEpochAtCheck)) continue;
+                            && ((!storeIdleResults && Graphics::StorageTexture::AnyPending()) || idleEpoch.load(std::memory_order_acquire) != idleEpochAtCheck)) continue;
                         if (id == 0) queue0Dormant.store(true, std::memory_order_relaxed);
                         const auto idleEpochAtWait = idleEpoch.load(std::memory_order_acquire);
-                        changed.wait(lock, [&] { return ready() || (id == 0 && idleEpoch.load(std::memory_order_acquire) != idleEpochAtWait); });
+                        changed.wait(lock, [&] { return ready() || (id == 0 && (idleEpoch.load(std::memory_order_acquire) != idleEpochAtWait || completionsPending())); });
                         if (id == 0) queue0Dormant.store(false, std::memory_order_relaxed);
-                        if (id == 0 && !ready() && idleEpoch.load(std::memory_order_acquire) != idleEpochAtWait) continue;
+                        if (id == 0 && !ready() && (idleEpoch.load(std::memory_order_acquire) != idleEpochAtWait || completionsPending())) continue;
                         break;
                     }
                     if (poll(ready)) break;
@@ -233,10 +240,18 @@ void Driver::run(std::uint32_t id) noexcept {
                 if (APS5_ENABLE_TIMING_LOG) submission.dequeuedAt = std::chrono::steady_clock::now();
                 if (id == 0) queue0Executing = submission.suspend ? 0 : submission.received;
                 if (submission.waitFree) {
-                    orderHolders.fetch_add(1, std::memory_order_acq_rel);
-                    const auto released = [&] { return failure || stopping || orderReleased(id, submission.received); };
-                    while (!released()) poll(released);
-                    orderHolders.fetch_sub(1, std::memory_order_acq_rel);
+                    {
+                        struct OrderHold {
+                            std::atomic<std::uint32_t>& holders;
+                            explicit OrderHold(std::atomic<std::uint32_t>& holders) : holders(holders) { holders.fetch_add(1, std::memory_order_acq_rel); }
+                            ~OrderHold() { holders.fetch_sub(1, std::memory_order_acq_rel); }
+                            OrderHold(const OrderHold&) = delete;
+                            OrderHold& operator=(const OrderHold&) = delete;
+                        } hold{orderHolders};
+                        const auto released = [&] { return failure || stopping || orderReleased(id, submission.received); };
+                        StallWatch stall(id);
+                        while (!released() && !poll(released)) stall.Check("order behind queue 0 for event", submission.received);
+                    }
                     rethrowFailure();
                 }
                 if (APS5_ENABLE_TIMING_LOG) submission.orderedAt = std::chrono::steady_clock::now();

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp

@@ -1,6 +1,7 @@
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Queues/SubmissionRing.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
@@ -259,6 +260,8 @@ void Driver::reserveOutputs(Submission& submission) {
 
 void Driver::executeRewindTail(const Submission& stalled) {
     PerformanceTimer timing("Driver.Rewind");
+    static const bool ring = ReadSubmissionRingSettings();
+    if (ring) PrepareSubmissionRing(stalled.queue);
     std::atomic_ref<std::uint32_t> control(*const_cast<std::uint32_t*>(stalled.rewindTail - 1));
     if ((control.load(std::memory_order_acquire) & Pm4::RewindValid) == 0) {
         noteWaitBlocked(stalled.queue, reinterpret_cast<std::uint64_t>(stalled.rewindTail - 1), true);
@@ -267,10 +270,12 @@ void Driver::executeRewindTail(const Submission& stalled) {
             std::uint32_t queue;
             ~BlockedWait() { driver.noteWaitBlocked(queue, 0, false); }
         } blocked{*this, stalled.queue};
+        StallWatch stall(stalled.queue);
         while ((control.load(std::memory_order_acquire) & Pm4::RewindValid) == 0) {
             CheckFailure();
             checkStopping();
             noteAwaitingTitle(stalled.queue, reinterpret_cast<std::uint64_t>(stalled.rewindTail - 1));
+            stall.Check("REWIND at", reinterpret_cast<std::uint64_t>(stalled.rewindTail - 1));
             PollSleep();
         }
     }
@@ -303,6 +308,7 @@ void Driver::executeRewindTail(const Submission& stalled) {
         tail.serial = stalled.serial;
         tail.received = ++eventSerial;
         if (capture != nullptr) capture->Take(captured, tail.serial, captureTime(), tail.heldAtSubmit);
+        if (ring) RecordSubmission(tail.queue, tail.serial, tail.commands, stalled.rewindTail, true);
     }
     if (capture != nullptr) writeCaptured(*this, *capture, captured);
     if (APS5_ENABLE_TIMING_LOG) tail.enqueuedAt = tail.dequeuedAt = tail.orderedAt = std::chrono::steady_clock::now();
@@ -313,7 +319,9 @@ void Driver::executeRewindTail(const Submission& stalled) {
 void Driver::Submit(const Packet* packet, std::uint32_t queue) {
     const auto receivedAt = APS5_ENABLE_TIMING_LOG ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
     CheckFailure();
-    require(queue == 0 || (queue >= 0x20 && queue < 0x58), "unsupported compute queue");
+    require(queue == 0 || (queue >= 0x20 && queue < SubmissionQueueEnd), "unsupported compute queue");
+    static const bool ring = ReadSubmissionRingSettings();
+    if (ring) PrepareSubmissionRing(queue);
     GuestMemory::CheckRange(packet, sizeof(Packet), alignof(Packet));
     const auto descriptor = *packet;
     require(descriptor.flags == 0, "nonzero submission flags are not implemented");
@@ -355,6 +363,7 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) {
         const auto now = std::chrono::steady_clock::now();
         submission.enqueuedAt = now;
         if (capture != nullptr) capture->Take(captured, submission.serial, captureTime(), submission.heldAtSubmit);
+        if (ring) RecordSubmission(queue, submission.serial, submission.commands, descriptor.addr, false);
         if (profile) {
             ++costs.submissions;
             costs.validateNs += static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(validated - copied).count());

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/SubmissionRing.cpp

@@ -0,0 +1,281 @@
+#include "prx/libSceAgcDriver/Execution/include/Driver/Queues/SubmissionRing.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
+#include "prx/libc/include/FailureReport.hpp"
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <cerrno>
+#include <charconv>
+#include <cstdarg>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <thread>
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
+
+namespace AgcDriver::DriverDetail {
+
+namespace {
+
+constexpr std::size_t KeptWords = 512;
+constexpr std::size_t MaximumCapacity = 4096;
+constexpr std::size_t MaximumStallSeconds = 86400;
+constexpr std::size_t PayloadWordsShown = 8;
+
+struct Entry {
+    std::uint64_t tenthsOfMs = 0;
+    std::uint64_t serial = 0;
+    std::uint64_t guest = 0;
+    std::size_t dwords = 0;
+    std::size_t kept = 0;
+    bool rewindTail = false;
+    std::array<std::uint32_t, KeptWords> words{};
+};
+
+struct Ring {
+    std::mutex mutex;
+    std::once_flag allocated;
+    std::atomic<std::uint64_t> recorded{0};
+    std::unique_ptr<Entry[]> entries;
+};
+
+void reportFailure() noexcept;
+
+struct Rings {
+    std::array<Ring, SubmissionQueueEnd> queues;
+};
+
+Rings& rings() {
+    static auto* value = new Rings;
+    return *value;
+}
+
+struct Registration {
+    Registration() { LibcRegisterFailureReport_nid_no_patch(reportFailure); }
+    ~Registration() { LibcUnregisterFailureReport_nid_no_patch(reportFailure); }
+    Registration(const Registration&) = delete;
+    Registration& operator=(const Registration&) = delete;
+};
+
+std::size_t environmentCount(const char* name, std::size_t fallback, std::size_t maximum, const char* reason) {
+    const char* text = std::getenv(name);
+    if (text == nullptr) return fallback;
+    const auto* end = text + std::strlen(text);
+    std::size_t value = 0;
+    const auto [last, error] = std::from_chars(text, end, value);
+    require(error == std::errc{} && last == end && value <= maximum, reason);
+    return value;
+}
+
+bool writeLine(const char* text, std::size_t length) {
+#ifdef _WIN32
+    const HANDLE handle = GetStdHandle(STD_ERROR_HANDLE);
+    if (handle == nullptr || handle == INVALID_HANDLE_VALUE) return false;
+    DWORD written = 0;
+    return WriteFile(handle, text, static_cast<DWORD>(length), &written, nullptr) != 0 && written == length;
+#else
+    while (length != 0) {
+        const auto written = ::write(2, text, length);
+        if (written < 0 && errno == EINTR) continue;
+        if (written <= 0) return false;
+        text += written;
+        length -= static_cast<std::size_t>(written);
+    }
+    return true;
+#endif
+}
+
+class Line {
+public:
+    void Add(const char* format, ...) {
+        if (length >= sizeof(text) - 2) return;
+        va_list arguments;
+        va_start(arguments, format);
+        const int added = std::vsnprintf(text + length, sizeof(text) - 1 - length, format, arguments);
+        va_end(arguments);
+        if (added > 0) length = std::min(length + static_cast<std::size_t>(added), sizeof(text) - 2);
+    }
+
+    bool Write() {
+        text[length] = '\n';
+        const bool written = writeLine(text, length + 1);
+        length = 0;
+        return written;
+    }
+
+private:
+    char text[512];
+    std::size_t length = 0;
+};
+
+void addName(Line& line, std::uint32_t header) {
+    const auto name = Pm4::KnownName(header);
+    if (name.empty()) line.Add("UNKNOWN_0x%02x", static_cast<unsigned>((header >> 8u) & 0xffu));
+    else line.Add("%.*s", static_cast<int>(name.size()), name.data());
+}
+
+bool writePackets(const Entry& entry, Line& line) {
+    bool written = true;
+    for (std::size_t cursor = 0; cursor < entry.kept;) {
+        const auto header = entry.words[cursor];
+        line.Add("[pm4 ring]   +0x%04zx ", cursor);
+        if (Pm4::FillerPacket(header)) {
+            std::size_t run = 1;
+            while (cursor + run < entry.kept && Pm4::FillerPacket(entry.words[cursor + run])) ++run;
+            addName(line, header);
+            line.Add(" x%zu", run);
+            written = line.Write() && written;
+            cursor += run;
+            continue;
+        }
+        if ((header >> 30u) != 3u) {
+            line.Add("not a type-3 header:");
+            for (std::size_t i = 0; i <= PayloadWordsShown && cursor + i < entry.kept; ++i) line.Add(" %08x", entry.words[cursor + i]);
+            written = line.Write() && written;
+            break;
+        }
+        const auto words = Pm4::PacketWords(header);
+        const auto shown = std::min(words, PayloadWordsShown + 1);
+        addName(line, header);
+        for (std::size_t i = 1; i < shown && cursor + i < entry.kept; ++i) line.Add(" %08x", entry.words[cursor + i]);
+        if (cursor + shown > entry.kept) line.Add(" (cut)");
+        else if (words > shown) line.Add(" (+%zu dwords)", words - shown);
+        written = line.Write() && written;
+        cursor += words;
+    }
+    return written;
+}
+
+bool dump(std::string_view reason, bool failing) {
+    const auto capacity = SubmissionRingCapacity();
+    const auto label = static_cast<int>(std::min<std::size_t>(reason.size(), 200));
+    Line line;
+    if (capacity == 0) {
+        line.Add("[pm4 ring] %.*s: disabled (APS5_PM4_RING=0)", label, reason.data());
+        return line.Write();
+    }
+    auto& state = rings();
+    std::size_t used = 0;
+    for (const auto& ring : state.queues) used += ring.recorded.load(std::memory_order_acquire) != 0 ? 1 : 0;
+    if (used == 0) {
+        line.Add("[pm4 ring] %.*s: no submissions recorded", label, reason.data());
+        return line.Write();
+    }
+    line.Add("[pm4 ring] %.*s: %zu queue(s), up to %zu submissions each, oldest first", label, reason.data(), used, capacity);
+    bool written = line.Write();
+    for (std::size_t queue = 0; queue < state.queues.size(); ++queue) {
+        auto& ring = state.queues[queue];
+        if (ring.recorded.load(std::memory_order_acquire) == 0) continue;
+        std::unique_lock lock(ring.mutex, std::defer_lock);
+        if (failing) {
+            for (int attempt = 0; attempt < 100 && !lock.try_lock(); ++attempt) std::this_thread::yield();
+        } else {
+            lock.lock();
+        }
+        const auto recorded = ring.recorded.load(std::memory_order_acquire);
+        if (!lock.owns_lock()) {
+            line.Add("[pm4 ring] queue 0x%zx: %llu submissions recorded, ring busy", queue, static_cast<unsigned long long>(recorded));
+            written = line.Write() && written;
+            continue;
+        }
+        const auto first = recorded > capacity ? recorded - capacity : 0;
+        line.Add("[pm4 ring] queue 0x%zx: %llu submissions recorded, last %llu kept", queue, static_cast<unsigned long long>(recorded), static_cast<unsigned long long>(recorded - first));
+        written = line.Write() && written;
+        for (auto ordinal = first; ordinal < recorded; ++ordinal) {
+            const auto& entry = ring.entries[ordinal % capacity];
+            line.Add("[pm4 ring] queue 0x%zx #%llu serial %llu t=%llu.%llu ms: %zu dwords from 0x%llx%s, first %zu kept", queue, static_cast<unsigned long long>(ordinal + 1), static_cast<unsigned long long>(entry.serial),
+                     static_cast<unsigned long long>(entry.tenthsOfMs / 10), static_cast<unsigned long long>(entry.tenthsOfMs % 10),
+                     entry.dwords, static_cast<unsigned long long>(entry.guest), entry.rewindTail ? " (rewind tail)" : "", entry.kept);
+            written = line.Write() && written;
+            written = writePackets(entry, line) && written;
+        }
+    }
+    return written;
+}
+
+void reportFailure() noexcept {
+    try {
+        dump("failure report", true);
+    } catch (...) {
+        static constexpr char text[] = "[pm4 ring] failure report: dump failed\n";
+        writeLine(text, sizeof(text) - 1);
+    }
+}
+
+}
+
+std::size_t SubmissionRingCapacity() {
+    static const std::size_t value = environmentCount("APS5_PM4_RING", 16, MaximumCapacity, "APS5_PM4_RING must be a number of submissions from 0 to 4096");
+    return value;
+}
+
+std::chrono::seconds StallWatchdogPeriod() {
+    static const std::chrono::seconds value(environmentCount("APS5_STALL_WATCHDOG_SECS", 0, MaximumStallSeconds, "APS5_STALL_WATCHDOG_SECS must be a number of seconds from 0 to 86400"));
+    return value;
+}
+
+bool ReadSubmissionRingSettings() {
+    StallWatchdogPeriod();
+    return SubmissionRingCapacity() != 0;
+}
+
+void PrepareSubmissionRing(std::uint32_t queue) {
+    const auto capacity = SubmissionRingCapacity();
+    require(capacity != 0, "the PM4 ring is disabled");
+    require(queue < SubmissionQueueEnd, "PM4 ring queue out of range");
+    static const Registration registration;
+    auto& ring = rings().queues[queue];
+    std::call_once(ring.allocated, [&] { ring.entries = std::make_unique<Entry[]>(capacity); });
+}
+
+void RecordSubmission(std::uint32_t queue, std::uint64_t serial, std::span<const std::uint32_t> commands, const void* guest, bool rewindTail) {
+    const auto capacity = SubmissionRingCapacity();
+    require(capacity != 0, "the PM4 ring is disabled");
+    require(queue < SubmissionQueueEnd, "PM4 ring queue out of range");
+    auto& ring = rings().queues[queue];
+    require(ring.entries != nullptr, "PM4 ring recorded before it was prepared");
+    const auto tenthsOfMs = static_cast<std::uint64_t>(std::max(TraceMs(), 0.0) * 10.0 + 0.5);
+    std::lock_guard lock(ring.mutex);
+    const auto recorded = ring.recorded.load(std::memory_order_relaxed);
+    auto& entry = ring.entries[recorded % capacity];
+    entry.tenthsOfMs = tenthsOfMs;
+    entry.serial = serial;
+    entry.guest = reinterpret_cast<std::uintptr_t>(guest);
+    entry.dwords = commands.size();
+    entry.kept = std::min(commands.size(), KeptWords);
+    entry.rewindTail = rewindTail;
+    std::copy_n(commands.begin(), entry.kept, entry.words.begin());
+    ring.recorded.store(recorded + 1, std::memory_order_release);
+}
+
+void DumpSubmissionRings(std::string_view reason) {
+    require(dump(reason, false), "cannot write the PM4 ring dump to stderr");
+}
+
+void StallWatch::Check(const char* waiter, std::uint64_t value) {
+    if (dumped) return;
+    const auto period = StallWatchdogPeriod();
+    if (period.count() == 0) return;
+    const auto now = std::chrono::steady_clock::now();
+    if (since == std::chrono::steady_clock::time_point{}) {
+        since = now;
+        return;
+    }
+    if (now - since < period) return;
+    dumped = true;
+    char reason[192];
+    std::snprintf(reason, sizeof(reason), "queue 0x%x %s 0x%llx has not progressed for %lld s (APS5_STALL_WATCHDOG_SECS)", queue, waiter, static_cast<unsigned long long>(value), static_cast<long long>(period.count()));
+    DumpSubmissionRings(reason);
+}
+
+}

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp

@@ -11,6 +11,7 @@
 #include "CompiledVariant.hpp"
 #include "Optimization/ResourceProgram.hpp"
 #include <algorithm>
+#include <array>
 #include <atomic>
 #include <chrono>
 #include <set>
@@ -24,7 +25,10 @@
 #include <deque>
 #include <exception>
 #include <functional>
+#include <iterator>
 #include <list>
+#include <map>
+#include <mutex>
 #include <atomic>
 #include <pthread.h>
 #ifdef __GLIBC__
@@ -143,15 +147,35 @@ std::shared_ptr<const ShaderSnapshot> ReadUserDataTailContinuation(std::span<con
 
 std::shared_ptr<const ShaderSnapshot> ProgramSnapshot(const ShaderRegistry& shaders, std::uint64_t address, std::initializer_list<std::uint8_t> types) {
     if (auto registered = RegisteredProgram(shaders, address, types)) return registered;
-    require(shaders.empty(), "graphics program does not belong to a compatible registered shader");
     return ReadRawShader(address);
 }
 
 namespace {
 std::atomic<bool> rawGraphicsDrawn{false};
+std::mutex rawGraphicsMutex;
+std::map<std::uint64_t, std::uint64_t> rawGraphicsRanges;
 }
 
-void NoteRawGraphicsDraw() {
+void NoteRawGraphicsDraw(const ShaderSnapshot& snapshot) {
+    if (snapshot.rawGraphicsNoted->load(std::memory_order_acquire)) return;
+    auto begin = snapshot.codeAddress;
+    auto end = snapshot.codeAddress + snapshot.code.size() * sizeof(std::uint32_t);
+    {
+        std::lock_guard lock(rawGraphicsMutex);
+        auto next = rawGraphicsRanges.upper_bound(begin);
+        if (next != rawGraphicsRanges.begin() && std::prev(next)->second >= begin) {
+            const auto previous = std::prev(next);
+            begin = previous->first;
+            end = std::max(end, previous->second);
+            rawGraphicsRanges.erase(previous);
+        }
+        while (next != rawGraphicsRanges.end() && next->first <= end) {
+            end = std::max(end, next->second);
+            next = rawGraphicsRanges.erase(next);
+        }
+        rawGraphicsRanges.emplace(begin, end);
+    }
+    snapshot.rawGraphicsNoted->store(true, std::memory_order_release);
     rawGraphicsDrawn.store(true, std::memory_order_relaxed);
 }
 
@@ -159,6 +183,13 @@ bool RawGraphicsDrawn() {
     return rawGraphicsDrawn.load(std::memory_order_relaxed);
 }
 
+bool RawGraphicsDrawnIn(std::uint64_t begin, std::uint64_t end) {
+    if (begin >= end) return false;
+    std::lock_guard lock(rawGraphicsMutex);
+    const auto next = rawGraphicsRanges.lower_bound(end);
+    return next != rawGraphicsRanges.begin() && std::prev(next)->second > begin;
+}
+
 namespace {
 
 void AwaitRegisteredPreparation(PreparedShaders& prepared, std::unique_lock<std::mutex>& lock) {
@@ -384,6 +415,72 @@ bool SameHeader(const ShaderSnapshot& snapshot, const Shader* shader) {
     return std::memcmp(&used, &registered, offsetof(Shader, num_sh_registers) + sizeof(Shader::num_sh_registers)) == 0 || RelocatedRegisters(snapshot, used, registered) || FusedHullHeader(snapshot, used, registered);
 }
 
+std::size_t ShaderRegistry::chunkFor(std::uint64_t address) const {
+    const auto above = std::upper_bound(chunks.begin(), chunks.end(), address, [](std::uint64_t key, const std::shared_ptr<Chunk>& chunk) { return key < chunk->begin()->first; });
+    return above == chunks.begin() ? 0u : static_cast<std::size_t>(above - chunks.begin()) - 1u;
+}
+
+ShaderRegistry::Chunk& ShaderRegistry::writable(std::size_t index) {
+    auto& chunk = chunks[index];
+    if (chunk.use_count() != 1) chunk = std::make_shared<Chunk>(*chunk);
+    return *chunk;
+}
+
+ShaderRegistry::const_iterator ShaderRegistry::find(std::uint64_t address) const {
+    if (chunks.empty()) return end();
+    const auto index = chunkFor(address);
+    const auto found = chunks[index]->find(address);
+    if (found == chunks[index]->end()) return end();
+    return const_iterator(chunks.cbegin() + static_cast<std::ptrdiff_t>(index), chunks.cend(), found);
+}
+
+ShaderRegistry::const_iterator ShaderRegistry::upper_bound(std::uint64_t address) const {
+    if (chunks.empty()) return end();
+    const auto index = chunkFor(address);
+    const auto found = chunks[index]->upper_bound(address);
+    if (found != chunks[index]->end()) return const_iterator(chunks.cbegin() + static_cast<std::ptrdiff_t>(index), chunks.cend(), found);
+    if (index + 1u == chunks.size()) return end();
+    return const_iterator(chunks.cbegin() + static_cast<std::ptrdiff_t>(index + 1u), chunks.cend(), chunks[index + 1u]->cbegin());
+}
+
+const ShaderRegistry::mapped_type& ShaderRegistry::at(std::uint64_t address) const {
+    const auto found = find(address);
+    if (found == end()) throw std::out_of_range("shader registry has no entry for the code address");
+    return found->second;
+}
+
+ShaderRegistry::mapped_type& ShaderRegistry::operator[](std::uint64_t address) {
+    if (chunks.empty()) chunks.push_back(std::make_shared<Chunk>());
+    const auto index = chunks.front()->empty() ? 0u : chunkFor(address);
+    auto& chunk = writable(index);
+    const auto [entry, inserted] = chunk.try_emplace(address);
+    auto& headers = entry->second;
+    if (!inserted) return headers;
+    ++count;
+    if (chunk.size() > ChunkEntries) {
+        auto upper = std::make_shared<Chunk>();
+        for (auto moved = std::next(chunk.begin(), static_cast<std::ptrdiff_t>(chunk.size() / 2u)); moved != chunk.end();) upper->insert(upper->end(), chunk.extract(moved++));
+        chunks.insert(chunks.begin() + static_cast<std::ptrdiff_t>(index + 1u), std::move(upper));
+    }
+    return headers;
+}
+
+std::pair<ShaderRegistry::const_iterator, bool> ShaderRegistry::emplace(std::uint64_t address, mapped_type headers) {
+    if (const auto found = find(address); found != end()) return {found, false};
+    (*this)[address] = std::move(headers);
+    return {find(address), true};
+}
+
+std::size_t ShaderRegistry::erase(std::uint64_t address) {
+    if (!contains(address)) return 0;
+    const auto index = chunkFor(address);
+    auto& chunk = writable(index);
+    chunk.erase(address);
+    --count;
+    if (chunk.empty()) chunks.erase(chunks.begin() + static_cast<std::ptrdiff_t>(index));
+    return 1;
+}
+
 std::shared_ptr<const ShaderSnapshot> RegisteredHeader(const ShaderRegistry& registry, const Shader* shader) {
     const auto found = registry.find(reinterpret_cast<std::uintptr_t>(const_cast<const void*>(GuestShaderFields(shader).code)));
     if (found == registry.end()) return {};
@@ -903,7 +1000,8 @@ class RegistrationPreparePool {
 public:
     static RegistrationPreparePool& Instance() {
         static auto* pool = [] {
-            auto* created = new RegistrationPreparePool(std::max(2u, std::thread::hardware_concurrency() / 2u));
+            std::lock_guard lifecycle(Lifecycle());
+            auto* created = new RegistrationPreparePool(Closing() ? 0u : std::max(2u, std::thread::hardware_concurrency() / 2u));
             PreparationSlot().store(created);
             return created;
         }();
@@ -917,7 +1015,8 @@ public:
     static RegistrationPreparePool& Prebuild() {
         static auto* pool = [] {
             const char* text = std::getenv("APS5_PREBUILD_THREADS");
-            auto* created = new RegistrationPreparePool(text != nullptr ? std::max(1u, static_cast<unsigned>(std::strtoul(text, nullptr, 10))) : 1u);
+            std::lock_guard lifecycle(Lifecycle());
+            auto* created = new RegistrationPreparePool(Closing() ? 0u : text != nullptr ? std::max(1u, static_cast<unsigned>(std::strtoul(text, nullptr, 10))) : 1u);
             PrebuildSlot().store(created);
             return created;
         }();
@@ -928,7 +1027,33 @@ public:
         return PrebuildSlot().load();
     }
 
+    static unsigned LiveWorkers() {
+        return liveWorkers.load(std::memory_order_acquire);
+    }
+
+    static void CloseAll() {
+        std::array<RegistrationPreparePool*, 2> pools{};
+        {
+            std::lock_guard lifecycle(Lifecycle());
+            Closing() = true;
+            pools = {Existing(), ExistingPrebuild()};
+        }
+        for (auto* pool : pools) {
+            if (pool != nullptr) pool->Close();
+        }
+    }
+
 private:
+    static std::mutex& Lifecycle() {
+        static std::mutex lifecycle;
+        return lifecycle;
+    }
+
+    static bool& Closing() {
+        static bool closing = false;
+        return closing;
+    }
+
     static std::atomic<RegistrationPreparePool*>& PreparationSlot() {
         static std::atomic<RegistrationPreparePool*> slot{nullptr};
         return slot;
@@ -972,9 +1097,17 @@ public:
     }
 
     void Close() {
-        std::unique_lock lock(mutex);
-        closed = true;
-        idle.wait(lock, [&] { return tasks.empty() && running == 0; });
+        std::vector<pthread_t> joined;
+        {
+            std::unique_lock lock(mutex);
+            closed = true;
+            idle.wait(lock, [&] { return tasks.empty() && running == 0; });
+            joined.swap(workers);
+        }
+        ready.notify_all();
+        for (const auto worker : joined) {
+            if (!pthread_equal(worker, pthread_self())) pthread_join(worker, nullptr);
+        }
     }
 
 private:
@@ -985,8 +1118,9 @@ private:
         for (unsigned index = 0; index < count; ++index) {
             pthread_t thread;
             if (pthread_create(&thread, &attributes, &RegistrationPreparePool::Main, this) == 0) {
-                pthread_detach(thread);
+                workers.push_back(thread);
                 ++threads;
+                liveWorkers.fetch_add(1, std::memory_order_acq_rel);
             }
         }
         pthread_attr_destroy(&attributes);
@@ -999,7 +1133,11 @@ private:
             Task task;
             {
                 std::unique_lock lock(pool.mutex);
-                pool.ready.wait(lock, [&] { return !pool.tasks.empty(); });
+                pool.ready.wait(lock, [&] { return !pool.tasks.empty() || pool.closed; });
+                if (pool.tasks.empty()) {
+                    liveWorkers.fetch_sub(1, std::memory_order_acq_rel);
+                    return nullptr;
+                }
                 task = std::move(pool.tasks.front());
                 pool.tasks.pop_front();
                 ++pool.running;
@@ -1032,7 +1170,9 @@ private:
     std::condition_variable ready;
     std::condition_variable idle;
     std::deque<Task> tasks;
+    std::vector<pthread_t> workers;
     unsigned threads = 0;
+    inline static std::atomic<unsigned> liveWorkers{0};
     unsigned running = 0;
     std::uint64_t submitted = 0;
     std::set<std::uint64_t> unfinished;
@@ -1089,9 +1229,16 @@ void SubmitLibraryPrebuild(std::function<void()> task) {
     RegistrationPreparePool::Prebuild().Submit(std::move(task));
 }
 
+void AwaitLibraryPrebuilds() {
+    if (auto* pool = RegistrationPreparePool::ExistingPrebuild()) pool->AwaitSubmitted();
+}
+
 void CloseRegistrationPreparation() {
-    RegistrationPreparePool::Instance().Close();
-    if (auto* prebuild = RegistrationPreparePool::ExistingPrebuild()) prebuild->Close();
+    RegistrationPreparePool::CloseAll();
+}
+
+unsigned RegistrationPreparationThreads() {
+    return RegistrationPreparePool::LiveWorkers();
 }
 
 std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target) {
@@ -1164,7 +1311,16 @@ std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decod
     return prepared;
 }
 
+namespace {
+
+void BindTitleThread() {
+    if (PerformanceTrace::Configured()) PerformanceTrace::BindThread("title", TraceThread::DefaultCapacity / 8);
+}
+
+}
+
 void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive) {
+    BindTitleThread();
     PerformanceContext timingContext(FrameTiming::Preparation());
     PerformanceTimer timing("Shader.ResolveGraphicsStagesAbi");
     ShaderPreparationTransaction transaction;
@@ -1241,6 +1397,7 @@ void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std
 }
 
 void Driver::ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive) {
+    BindTitleThread();
     PerformanceContext timingContext(FrameTiming::Preparation());
     PerformanceTimer timing("Shader.ResolveAbi");
     ShaderPreparationTransaction transaction;
@@ -1312,6 +1469,7 @@ std::uint64_t NullPixelProgramAddress() {
 }
 
 void Driver::ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType) {
+    BindTitleThread();
     PerformanceContext timingContext(FrameTiming::Preparation());
     PerformanceTimer timing("Shader.ResolveGraphicsAbi");
     CheckFailure();
@@ -1432,12 +1590,17 @@ ShaderSnapshot PrepareNullPixelProgram(const VulkanDevice& device) {
     return null;
 }
 
+std::shared_ptr<const ShaderRegistry> Driver::RegisteredShaders() {
+    std::lock_guard lock(mutex);
+    return shaders;
+}
+
 void Driver::RegisterShader(const Shader* shader) {
+    BindTitleThread();
     PerformanceContext timingContext(FrameTiming::Preparation());
     PerformanceTimer timing("Shader.Register");
     ShaderPreparationTransaction transaction;
     CheckFailure();
-    require(!RawGraphicsDrawn(), "a shader was registered after graphics programs were drawn without registration");
     GuestMemory::CheckRange(shader, sizeof(Shader), 1);
     Shader fields;
     std::memcpy(&fields, static_cast<const void*>(shader), sizeof(Shader));
@@ -1447,6 +1610,7 @@ void Driver::RegisterShader(const Shader* shader) {
     GuestMemory::CheckRange(shader, fields.header_size, 1);
     const auto* code = const_cast<const void*>(fields.code);
     GuestMemory::CheckRange(code, fields.shader_size, 256);
+    require(!RawGraphicsDrawnIn(reinterpret_cast<std::uintptr_t>(code), reinterpret_cast<std::uintptr_t>(code) + fields.shader_size), "a shader was registered over a graphics program that was drawn without registration");
     ShaderSnapshot snapshot{reinterpret_cast<std::uintptr_t>(code), reinterpret_cast<std::uintptr_t>(shader), fields.type, {}, {}};
     snapshot.code.resize(fields.shader_size / sizeof(std::uint32_t));
     std::memcpy(snapshot.code.data(), code, fields.shader_size);
@@ -1484,7 +1648,11 @@ void Driver::RegisterShader(const Shader* shader) {
     registered.context = snapshot.registeredState->context;
     registered.userConfig = snapshot.registeredState->userConfig;
     std::shared_ptr<RegisteredPreparation> preparation;
-    if (AsyncRegistrationPrepare()) {
+    static const bool lazy = std::getenv("APS5_LAZY_SHADER_PREPARE") != nullptr;
+    if (lazy) {
+        static_cast<void>(PlanRegistered(snapshot, *localDevice, registered, true, &snapshot.prepared->deferred));
+        snapshot.prepared->deferred = true;
+    } else if (AsyncRegistrationPrepare()) {
         preparation = PlanRegistered(snapshot, *localDevice, registered, true, &snapshot.prepared->deferred);
         if (preparation != nullptr) preparation->prebuildDevice = localDevice;
         if (preparation != nullptr && (snapshot.type == 0 || snapshot.type == 1)) BuildRegisteredAbiKey(registered, *localDevice, preparation->abiKey);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp

@@ -1,6 +1,7 @@
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/WaitMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Queues/SubmissionRing.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
@@ -330,6 +331,7 @@ void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t que
     const auto spinStart = std::chrono::steady_clock::now();
     bool spinning = pauseSpin;
     std::uint32_t polls = 0;
+    StallWatch stall(queue);
     while (!Pm4::WaitSatisfiedUnchecked(packet)) {
         if (std::uint32_t writer = queue; !requireMemory && storedSince(packet, awaited, awaitedBytes, received, &writer)) {
             if (writer != queue) notePendingWait(packet, queue, received);
@@ -358,7 +360,9 @@ void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t que
         if (const auto done = packetsDone.load(); done != lastDone || (!requireMemory && packetsInFlight.load() != 0)) {
             lastDone = done;
             start = std::chrono::steady_clock::now();
+            stall.Progressed();
         }
+        stall.Check(((packet[0] >> 8u) & 0xffu) == 0x93u ? "WAIT_REG_MEM_64 at" : "WAIT_REG_MEM at", awaited);
 
         if (!warned && std::chrono::steady_clock::now() - start > std::chrono::milliseconds(waitTimeoutMs())) {
             warned = true;

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -1,5 +1,6 @@
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/HandoffSpin.hpp"
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp"
 #include "prx/libSceAgcDriver/Execution/include/WriteWatchCoverage.hpp"
 #include "ThreadOwned.hpp"
@@ -657,8 +658,7 @@ bool onOwnLiveStack(std::uintptr_t address, std::size_t bytes) {
         }
     };
     thread_local const Bounds bounds;
-    const volatile unsigned char marker = 0;
-    const auto frame = reinterpret_cast<std::uintptr_t>(&marker);
+    const auto frame = reinterpret_cast<std::uintptr_t>(__builtin_frame_address(0));
     return bounds.high != 0 && frame >= bounds.low && frame < bounds.high && address >= frame && address < bounds.high && bytes <= bounds.high - address;
 }
 
@@ -883,6 +883,20 @@ struct WriteTracker {
         std::uint32_t dropped = 0;
         std::uint32_t next = 0;
         std::array<DriverPiece, 4> pieces{};
+
+        bool covers(std::uint32_t begin, std::uint32_t end, std::uint64_t generation) const {
+            if (generation == 0 || begin >= end || dropped > generation) return false;
+            if (whole > generation) return true;
+            while (begin < end) {
+                auto covered = begin;
+                for (const auto& piece : pieces) {
+                    if (piece.generation > generation && piece.begin <= begin && piece.end > covered) covered = piece.end;
+                }
+                if (covered == begin) return false;
+                begin = covered;
+            }
+            return true;
+        }
     };
     std::unordered_map<std::uint64_t, DriverPieces> driverPieces;
 
@@ -904,15 +918,40 @@ struct WriteTracker {
         }
     }
 
+    bool blockPartWatched(std::uint64_t first, std::uint64_t stop) const {
+#ifdef _WIN32
+        return covers(first, 1) || covers(stop - 1, 1);
+#else
+        return GuestWriteWatch::GuestWriteWatchOverlaps_nid_postfix(static_cast<std::uintptr_t>(first), static_cast<std::size_t>(stop - first));
+#endif
+    }
+
+    void stampWatchedPart(std::uint64_t address, std::uint64_t end, std::uint32_t stampGeneration) {
+#ifdef _WIN32
+        address = std::max<std::uint64_t>(address, base);
+        end = std::min<std::uint64_t>(end, base + size);
+        if (address >= end) return;
+#else
+        end = std::min<std::uint64_t>(end, LeafCount * LeafBlocks * WriteBlockBytes);
+        if (address >= end || !GuestWriteWatch::GuestWriteWatchOverlaps_nid_postfix(static_cast<std::uintptr_t>(address), static_cast<std::size_t>(end - address))) return;
+#endif
+        for (auto block = blockOf(address); block <= blockOf(end - 1); ++block) {
+            const auto first = std::max(address, blockBegin(block));
+            const auto stop = std::min<std::uint64_t>(end, blockBegin(block) + WriteBlockBytes);
+            if (!blockPartWatched(first, stop)) continue;
+            stamp(block, stampGeneration, StampKind::Driver);
+            noteDriverStore(block, address, end, stampGeneration);
+        }
+    }
+
     void noteDriverStore(std::uint64_t block, std::uint64_t address, std::uint64_t end, std::uint32_t stampGeneration) {
         const auto begin = blockBegin(block);
         const auto from = static_cast<std::uint32_t>(std::max(address, begin) - begin);
         const auto to = static_cast<std::uint32_t>(std::min<std::uint64_t>(end, begin + WriteBlockBytes) - begin);
         if (from == 0 && to == WriteBlockBytes) {
-            if (const auto found = driverPieces.find(block); found != driverPieces.end()) {
-                found->second = DriverPieces{};
-                found->second.whole = stampGeneration;
-            }
+            auto& entry = driverPieces[block];
+            entry = DriverPieces{};
+            entry.whole = stampGeneration;
             return;
         }
         auto& entry = driverPieces[block];
@@ -1309,6 +1348,31 @@ void Unwatch(std::uint64_t address, std::size_t bytes) {
     unwatchLocked(tracker, address, bytes);
 }
 
+void Rewatch(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges) {
+#ifdef _WIN32
+    auto& tracker = Tracker();
+    const auto lock = lockTracker(tracker);
+    tracker.initialize();
+    if (!tracker.watched) return;
+    bool restored = false;
+    for (const auto& [begin, end] : ranges) {
+        if (end <= begin || begin < tracker.base || end - tracker.base > tracker.size || (begin - tracker.base) % WriteBlockBytes != 0 || (end - tracker.base) % WriteBlockBytes != 0) continue;
+        const auto bytes = static_cast<std::size_t>(end - begin);
+        if (tracker.coverage.Covers(begin, bytes) || !walkWrites(tracker, begin, end, StampKind::Cpu) || !tracker.coverage.Restore(begin, bytes)) continue;
+        ++tracker.generation;
+        tracker.noteCpuStore(begin, end, tracker.generation);
+        for (auto block = tracker.blockOf(begin); block <= tracker.blockOf(end - 1); ++block) {
+            tracker.driverPieces.erase(block);
+            tracker.stamp(block, tracker.generation, StampKind::Cpu);
+        }
+        restored = true;
+    }
+    if (restored) unwatchSerial.fetch_add(1, std::memory_order_release);
+#else
+    static_cast<void>(ranges);
+#endif
+}
+
 bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched) {
     if (bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return import();
     if (!staysWatched) {
@@ -1434,6 +1498,7 @@ std::uint64_t storeOwn(std::uint64_t address, std::size_t bytes, const std::func
         ++tracker.generation;
         if (!tracker.watched || !tracker.covers(stored.first, static_cast<std::size_t>(stored.second - stored.first))) {
             tracker.noteUnstampedStore(stored.first, stored.second);
+            if (tracker.watched) tracker.stampWatchedPart(stored.first, stored.second, tracker.generation);
             return 0;
         }
         for (auto block = tracker.blockOf(stored.first); block <= tracker.blockOf(stored.second - 1); ++block) {
@@ -1479,6 +1544,7 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) {
     ++tracker.generation;
     if (!tracker.watched || !tracker.covers(address, bytes)) {
         tracker.noteUnstampedStore(address, address + bytes);
+        if (tracker.watched) tracker.stampWatchedPart(address, address + bytes, tracker.generation);
         return 0;
     }
     {
@@ -1543,6 +1609,33 @@ bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generati
     return false;
 }
 
+bool StoredAllSince_nid_no_patch(std::uint64_t address, std::size_t bytes, std::uint64_t generation) {
+    if (generation == 0 || bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return false;
+    auto& tracker = Tracker();
+    const auto lock = lockTracker(tracker);
+    tracker.initialize();
+    if (!tracker.watched || !tracker.covers(address, bytes)) return false;
+    const auto end = address + bytes;
+    if (!tracker.unstampedStores.empty()) {
+        const auto first = address / WriteBlockBytes;
+        const auto last = (end - 1u) / WriteBlockBytes;
+        if (last - first >= WriteTracker::UnstampedSlots) {
+            if (std::ranges::any_of(tracker.unstampedStores, [&](auto stamp) { return stamp > generation; })) return false;
+        } else {
+            for (auto block = first; block <= last; ++block) if (tracker.unstampedStores[block % WriteTracker::UnstampedSlots] > generation) return false;
+        }
+    }
+    for (auto block = tracker.blockOf(address); block <= tracker.blockOf(end - 1u); ++block) {
+        const auto found = tracker.driverPieces.find(block);
+        if (found == tracker.driverPieces.end()) return false;
+        const auto start = tracker.blockBegin(block);
+        const auto from = static_cast<std::uint32_t>(std::max(address, start) - start);
+        const auto to = static_cast<std::uint32_t>(std::min<std::uint64_t>(end, start + WriteBlockBytes) - start);
+        if (!found->second.covers(from, to, generation)) return false;
+    }
+    return true;
+}
+
 std::uint64_t TrackerGeneration() {
     auto& tracker = Tracker();
     // Under the mutex: a collect bumps the generation before its walk, and a walk still running
@@ -1891,12 +1984,12 @@ void GpuMutexType::lock() {
     if (!GpuLockProfiled()) {
         auto* trace = PerformanceTrace::Recording() ? PerformanceTrace::Thread() : nullptr;
         if (trace == nullptr) {
-            mutex.lock();
+            LockSpinning(mutex, HandoffSpinWindow());
         } else {
             const auto site = static_cast<std::size_t>(std::exchange(LockStats().nextSite, GpuLockSite::Other));
             if (!mutex.try_lock()) {
                 const auto start = std::chrono::steady_clock::now();
-                mutex.lock();
+                LockSpinning(mutex, HandoffSpinWindow());
                 trace->Add("GpuMutex.wait", GpuLockSiteNames[site], start, std::chrono::steady_clock::now());
             }
         }
@@ -1914,7 +2007,7 @@ void GpuMutexType::lock() {
         HolderMatrixValues before{};
         if (holders) Holders().Snapshot(before);
         const auto start = std::chrono::steady_clock::now();
-        mutex.lock();
+        LockSpinning(mutex, HandoffSpinWindow());
         acquired();
         const auto now = std::chrono::steady_clock::now();
         if (auto* trace = PerformanceTrace::Recording() ? PerformanceTrace::Thread() : nullptr) trace->Add("GpuMutex.wait", GpuLockSiteNames[site], start, now);
@@ -2060,9 +2153,26 @@ void Read(std::uint64_t address, std::span<std::byte> destination, std::size_t a
         if (trace) std::fprintf(stderr, "[unreadable]   read from +0x%llx\n", ModuleOffset(__builtin_return_address(0)));
         throw;
     }
-    std::memcpy(destination.data(), source, destination.size());
+    CopyGuestBytes(destination.data(), source, destination.size());
 }
 
+#ifdef ANYPS5_ENABLE_ASAN
+[[gnu::no_sanitize("address")]] void CopyGuestBytes(void* destination, const void* source, std::size_t bytes) {
+    auto* to = static_cast<unsigned char*>(destination);
+    const auto* from = static_cast<const unsigned char*>(source);
+    std::size_t index = 0;
+    for (; index + sizeof(std::uint64_t) <= bytes; index += sizeof(std::uint64_t)) {
+        const std::uint64_t word = *reinterpret_cast<const volatile std::uint64_t*>(from + index);
+        std::memcpy(to + index, &word, sizeof(word));
+    }
+    for (; index < bytes; ++index) to[index] = static_cast<const volatile unsigned char*>(from)[index];
+}
+#else
+void CopyGuestBytes(void* destination, const void* source, std::size_t bytes) {
+    std::memcpy(destination, source, bytes);
+}
+#endif
+
 namespace {
 
 void copyCommitted(std::uint64_t address, std::span<std::byte> destination) {

core/libs/prx/libSceAgcDriver/Execution/src/PerformanceOverlay.cpp

@@ -0,0 +1,791 @@
+#include "prx/libSceAgcDriver/Execution/include/PerformanceOverlay.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Presentation.hpp"
+#include "prx/libSceAgcDriver/Execution/include/PqToSdr.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
+#include "prx/libSceAgcDriver/Graphics/shaders/PerformanceOverlayFrag_spv.h"
+#include "prx/libSceAgcDriver/Graphics/shaders/PerformanceOverlayVert_spv.h"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
+#include "prx/libc/include/HostThreadLocal.hpp"
+#include <algorithm>
+#include <atomic>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <initializer_list>
+#include <mutex>
+#include <stdexcept>
+#include <string_view>
+#ifdef _WIN32
+#define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#include <psapi.h>
+#endif
+
+namespace AgcDriver {
+namespace {
+
+static_assert(sizeof(OverlayRect) == 20, "overlay rectangles are read as one ivec4 and one uint per instance");
+
+struct OverlayParameters {
+    std::array<std::array<float, 4>, OverlayColorCount> palette;
+    std::array<float, 2> extent;
+};
+
+std::atomic<std::uint64_t> toggleCount{0};
+std::atomic<std::uint64_t> gpuTotal{0};
+std::atomic<std::uint64_t> driverTotal{0};
+std::atomic<std::uint64_t> pipelineTotal{0};
+
+struct PacketCounterRegistry {
+    std::mutex mutex;
+    std::deque<PacketCounters> counters;
+    std::vector<PacketCounters*> unused;
+};
+
+PacketCounterRegistry& packetCounterRegistry() {
+    static auto* registry = new PacketCounterRegistry;
+    return *registry;
+}
+
+struct PacketCounterLease {
+    PacketCounters* counters = nullptr;
+    ~PacketCounterLease() {
+        if (counters == nullptr) return;
+        auto& registry = packetCounterRegistry();
+        std::lock_guard lock(registry.mutex);
+        registry.unused.push_back(counters);
+    }
+};
+
+constexpr double GigabyteBytes = 1024.0 * 1024.0 * 1024.0;
+
+const OverlaySettings& settings() {
+    static const OverlaySettings parsed = ParseOverlaySettings(std::getenv("ANYPS5_OVERLAY"), std::getenv("ANYPS5_OVERLAY_POS"));
+    return parsed;
+}
+
+std::int32_t panelWidth(std::int32_t scale) {
+    return (OverlayMetrics::InnerWidth + 2 * OverlayMetrics::Padding) * scale;
+}
+
+std::int32_t panelHeight(const OverlayContent& content, std::int32_t scale) {
+    const auto lines = static_cast<std::int32_t>(content.above.size() + content.below.size());
+    const auto glyphGap = OverlayMetrics::LinePitch - OverlayMetrics::GlyphHeight;
+    return (2 * OverlayMetrics::Padding + lines * OverlayMetrics::LinePitch + OverlayMetrics::GraphHeight + OverlayMetrics::GraphGap - glyphGap) * scale;
+}
+
+bool fits(VkExtent2D extent, const OverlayContent& content, std::int32_t scale) {
+    const auto margins = 2 * OverlayMetrics::Margin * scale;
+    return panelWidth(scale) + margins <= static_cast<std::int32_t>(extent.width) && panelHeight(content, scale) + margins <= static_cast<std::int32_t>(extent.height);
+}
+
+float srgbDecode(float value) {
+    return value <= 0.04045f ? value / 12.92f : std::pow((value + 0.055f) / 1.055f, 2.4f);
+}
+
+float pqEncode(float nits) {
+    constexpr double m1 = 2610.0 / 16384.0;
+    constexpr double m2 = 2523.0 / 4096.0 * 128.0;
+    constexpr double c1 = 3424.0 / 4096.0;
+    constexpr double c2 = 2413.0 / 4096.0 * 32.0;
+    constexpr double c3 = 2392.0 / 4096.0 * 32.0;
+    const double power = std::pow(std::clamp(static_cast<double>(nits) / 10000.0, 0.0, 1.0), m1);
+    return static_cast<float>(std::pow((c1 + c2 * power) / (1.0 + c3 * power), m2));
+}
+
+bool srgbFormat(VkFormat format) {
+    return format == VK_FORMAT_B8G8R8A8_SRGB || format == VK_FORMAT_R8G8B8A8_SRGB || format == VK_FORMAT_A8B8G8R8_SRGB_PACK32;
+}
+
+}
+
+OverlaySettings ParseOverlaySettings(const char* enabled, const char* position) {
+    OverlaySettings parsed;
+    const std::string_view visible = enabled != nullptr ? enabled : "";
+    if (visible == "1") parsed.visible = true;
+    else if (!visible.empty() && visible != "0") throw std::runtime_error("ANYPS5_OVERLAY must be 1 or 0, not '" + std::string(visible) + "'");
+    const std::string_view corner = position != nullptr ? position : "";
+    if (corner.empty() || corner == "tl") parsed.corner = OverlayCorner::TopLeft;
+    else if (corner == "tr") parsed.corner = OverlayCorner::TopRight;
+    else if (corner == "bl") parsed.corner = OverlayCorner::BottomLeft;
+    else if (corner == "br") parsed.corner = OverlayCorner::BottomRight;
+    else throw std::runtime_error("ANYPS5_OVERLAY_POS must be tl, tr, bl or br, not '" + std::string(corner) + "'");
+    return parsed;
+}
+
+void PerformanceCounters::AddGpuNanoseconds(std::uint64_t nanoseconds) {
+    gpuTotal.fetch_add(nanoseconds, std::memory_order_relaxed);
+}
+
+void PerformanceCounters::AddDriverNanoseconds(std::uint64_t nanoseconds) {
+    driverTotal.fetch_add(nanoseconds, std::memory_order_relaxed);
+}
+
+void PerformanceCounters::CountPipeline() {
+    pipelineTotal.fetch_add(1, std::memory_order_relaxed);
+}
+
+std::uint64_t PerformanceCounters::GpuNanoseconds() {
+    return gpuTotal.load(std::memory_order_relaxed);
+}
+
+std::uint64_t PerformanceCounters::DriverNanoseconds() {
+    return driverTotal.load(std::memory_order_relaxed);
+}
+
+std::uint64_t PerformanceCounters::Pipelines() {
+    return pipelineTotal.load(std::memory_order_relaxed);
+}
+
+PacketCounters& PacketCounters::ThisThread() {
+    auto& lease = HostThreadLocal<PacketCounterLease, PacketCounterLease>();
+    if (lease.counters == nullptr) {
+        auto& registry = packetCounterRegistry();
+        std::lock_guard lock(registry.mutex);
+        if (registry.unused.empty()) {
+            lease.counters = &registry.counters.emplace_back();
+        } else {
+            lease.counters = registry.unused.back();
+            registry.unused.pop_back();
+        }
+    }
+    return *lease.counters;
+}
+
+PacketCounters::Totals PacketCounters::Sum() {
+    auto& registry = packetCounterRegistry();
+    std::lock_guard lock(registry.mutex);
+    Totals totals;
+    for (const auto& counters : registry.counters) {
+        totals.draws += counters.draws.load(std::memory_order_relaxed);
+        totals.dispatches += counters.dispatches.load(std::memory_order_relaxed);
+    }
+    return totals;
+}
+
+bool PerformanceCounters::GpuEnabled() {
+    return Graphics::Recorder::GpuTimingEnabled();
+}
+
+bool PerformanceCounters::DriverEnabled() {
+    static const bool enabled = std::getenv("APS5_PROFILE_DRAW") != nullptr;
+    return enabled;
+}
+
+void OverlayStatistics::AddPresent(Clock::time_point at, std::uint64_t gpuNanoseconds, std::uint64_t driverNanoseconds) {
+    AddPresent(at, OverlayCounters{gpuNanoseconds, driverNanoseconds});
+}
+
+void OverlayStatistics::AddPresent(Clock::time_point at, const OverlayCounters& counters) {
+    if (!window.empty()) {
+        Graphics::Require(at >= window.back().at, "presents reached the performance overlay out of order");
+        frameTimes[frameNext] = std::chrono::duration<double, std::milli>(at - window.back().at).count();
+        frameNext = (frameNext + 1) % FrameHistory;
+        frameCount = std::min(frameCount + 1, FrameHistory);
+        previous = window.back();
+    }
+    window.push_back({at, counters});
+    while (at - window.front().at > Window) window.pop_front();
+}
+
+void OverlayStatistics::Reset() {
+    window.clear();
+    previous.reset();
+    frameTimes = {};
+    frameCount = 0;
+    frameNext = 0;
+}
+
+double OverlayStatistics::Fps() const {
+    if (window.size() < 2) {
+        const auto frameMs = LastFrameMs();
+        return frameMs > 0 ? 1000.0 / frameMs : 0;
+    }
+    const auto seconds = std::chrono::duration<double>(window.back().at - window.front().at).count();
+    return seconds > 0 ? static_cast<double>(window.size() - 1) / seconds : 0;
+}
+
+double OverlayStatistics::LastFrameMs() const {
+    return frameCount == 0 ? 0 : frameTimes[(frameNext + FrameHistory - 1) % FrameHistory];
+}
+
+std::vector<double> OverlayStatistics::FrameTimes() const {
+    std::vector<double> ordered;
+    ordered.reserve(frameCount);
+    for (std::size_t i = 0; i < frameCount; ++i) ordered.push_back(frameTimes[(frameNext + FrameHistory - frameCount + i) % FrameHistory]);
+    return ordered;
+}
+
+OverlayStatistics::Range OverlayStatistics::FrameTimeRange() const {
+    const auto times = FrameTimes();
+    if (times.empty()) return {};
+    Range range{times.front(), 0, times.front()};
+    double sum = 0;
+    for (const auto time : times) {
+        range.minimum = std::min(range.minimum, time);
+        range.maximum = std::max(range.maximum, time);
+        sum += time;
+    }
+    range.average = sum / static_cast<double>(times.size());
+    return range;
+}
+
+double OverlayStatistics::perFrame(std::uint64_t OverlayCounters::*counter) const {
+    const bool windowed = window.size() >= 2;
+    if (!windowed && !previous) return 0;
+    const auto first = windowed ? window.front().counters.*counter : previous->counters.*counter;
+    const auto last = window.back().counters.*counter;
+    const auto frames = windowed ? window.size() - 1 : std::size_t{1};
+    return last >= first ? static_cast<double>(last - first) / static_cast<double>(frames) : 0;
+}
+
+double OverlayStatistics::GpuMsPerFrame() const {
+    return perFrame(&OverlayCounters::gpuNanoseconds) / 1e6;
+}
+
+double OverlayStatistics::DriverMsPerFrame() const {
+    return perFrame(&OverlayCounters::driverNanoseconds) / 1e6;
+}
+
+double OverlayStatistics::DrawsPerFrame() const {
+    return perFrame(&OverlayCounters::draws);
+}
+
+double OverlayStatistics::DispatchesPerFrame() const {
+    return perFrame(&OverlayCounters::dispatches);
+}
+
+std::uint64_t OverlayStatistics::PipelinesInWindow() const {
+    if (window.empty()) return 0;
+    const auto first = window.size() < 2 && previous ? previous->counters.pipelines : window.front().counters.pipelines;
+    const auto last = window.back().counters.pipelines;
+    return last >= first ? last - first : 0;
+}
+
+OverlayCounters OverlayStatistics::Latest() const {
+    return window.empty() ? OverlayCounters{} : window.back().counters;
+}
+
+const std::array<std::uint8_t, OverlayMetrics::GlyphHeight>& OverlayGlyph(char character) {
+    using Glyph = std::array<std::uint8_t, OverlayMetrics::GlyphHeight>;
+    static constexpr Glyph space{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+    static constexpr std::array<Glyph, 10> digits{{
+        {0x0e, 0x11, 0x13, 0x15, 0x19, 0x11, 0x0e},
+        {0x04, 0x0c, 0x04, 0x04, 0x04, 0x04, 0x0e},
+        {0x0e, 0x11, 0x01, 0x02, 0x04, 0x08, 0x1f},
+        {0x1f, 0x02, 0x04, 0x02, 0x01, 0x11, 0x0e},
+        {0x02, 0x06, 0x0a, 0x12, 0x1f, 0x02, 0x02},
+        {0x1f, 0x10, 0x1e, 0x01, 0x01, 0x11, 0x0e},
+        {0x06, 0x08, 0x10, 0x1e, 0x11, 0x11, 0x0e},
+        {0x1f, 0x01, 0x02, 0x04, 0x08, 0x08, 0x08},
+        {0x0e, 0x11, 0x11, 0x0e, 0x11, 0x11, 0x0e},
+        {0x0e, 0x11, 0x11, 0x0f, 0x01, 0x02, 0x0c},
+    }};
+    static constexpr Glyph period{0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x0c};
+    static constexpr Glyph colon{0x00, 0x0c, 0x0c, 0x00, 0x0c, 0x0c, 0x00};
+    static constexpr Glyph slash{0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x00};
+    static constexpr Glyph a{0x0e, 0x11, 0x11, 0x11, 0x1f, 0x11, 0x11};
+    static constexpr Glyph b{0x1e, 0x11, 0x11, 0x1e, 0x11, 0x11, 0x1e};
+    static constexpr Glyph c{0x0e, 0x11, 0x10, 0x10, 0x10, 0x11, 0x0e};
+    static constexpr Glyph d{0x1c, 0x12, 0x11, 0x11, 0x11, 0x12, 0x1c};
+    static constexpr Glyph e{0x1f, 0x10, 0x10, 0x1e, 0x10, 0x10, 0x1f};
+    static constexpr Glyph f{0x1f, 0x10, 0x10, 0x1e, 0x10, 0x10, 0x10};
+    static constexpr Glyph g{0x0e, 0x11, 0x10, 0x17, 0x11, 0x11, 0x0f};
+    static constexpr Glyph h{0x11, 0x11, 0x11, 0x1f, 0x11, 0x11, 0x11};
+    static constexpr Glyph i{0x0e, 0x04, 0x04, 0x04, 0x04, 0x04, 0x0e};
+    static constexpr Glyph k{0x11, 0x12, 0x14, 0x18, 0x14, 0x12, 0x11};
+    static constexpr Glyph l{0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x1f};
+    static constexpr Glyph m{0x11, 0x1b, 0x15, 0x15, 0x11, 0x11, 0x11};
+    static constexpr Glyph n{0x11, 0x11, 0x19, 0x15, 0x13, 0x11, 0x11};
+    static constexpr Glyph o{0x0e, 0x11, 0x11, 0x11, 0x11, 0x11, 0x0e};
+    static constexpr Glyph p{0x1e, 0x11, 0x11, 0x1e, 0x10, 0x10, 0x10};
+    static constexpr Glyph r{0x1e, 0x11, 0x11, 0x1e, 0x14, 0x12, 0x11};
+    static constexpr Glyph s{0x0f, 0x10, 0x10, 0x0e, 0x01, 0x01, 0x1e};
+    static constexpr Glyph t{0x1f, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04};
+    static constexpr Glyph u{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x0e};
+    static constexpr Glyph v{0x11, 0x11, 0x11, 0x11, 0x11, 0x0a, 0x04};
+    static constexpr Glyph w{0x11, 0x11, 0x11, 0x15, 0x15, 0x15, 0x0a};
+    static constexpr Glyph x{0x11, 0x11, 0x0a, 0x04, 0x0a, 0x11, 0x11};
+    if (character >= '0' && character <= '9') return digits[static_cast<std::size_t>(character - '0')];
+    switch (character) {
+        case ' ': return space;
+        case '.': return period;
+        case ':': return colon;
+        case '/': return slash;
+        case 'A': return a;
+        case 'B': return b;
+        case 'C': return c;
+        case 'D': return d;
+        case 'E': return e;
+        case 'F': return f;
+        case 'G': return g;
+        case 'H': return h;
+        case 'I': return i;
+        case 'K': return k;
+        case 'L': return l;
+        case 'M': return m;
+        case 'N': return n;
+        case 'O': return o;
+        case 'P': return p;
+        case 'R': return r;
+        case 'S': return s;
+        case 'T': return t;
+        case 'U': return u;
+        case 'V': return v;
+        case 'W': return w;
+        case 'X': return x;
+        default: throw std::runtime_error(std::string("the performance overlay font has no glyph for '") + character + "'");
+    }
+}
+
+std::string FormatOverlayValue(double value) {
+    const double clamped = std::isfinite(value) ? std::clamp(value, 0.0, 9999.0) : 0.0;
+    char text[16];
+    std::snprintf(text, sizeof(text), clamped < 99.95 ? "%.1f" : "%.0f", clamped);
+    return text;
+}
+
+std::string FormatOverlayRate(double value) {
+    if (!std::isfinite(value) || value <= 0 || value >= 0.995) return FormatOverlayValue(value);
+    char text[16];
+    std::snprintf(text, sizeof(text), "%.2f", value);
+    return text;
+}
+
+std::string FormatOverlayCount(double value) {
+    const long long rounded = std::isfinite(value) && value > 0 ? std::llround(std::min(value, 1e12)) : 0;
+    char text[24];
+    if (rounded < 10000) std::snprintf(text, sizeof(text), "%lld", rounded);
+    else if (rounded < 1000000) std::snprintf(text, sizeof(text), "%lldK", rounded / 1000);
+    else std::snprintf(text, sizeof(text), "%lldM", std::min<long long>(rounded / 1000000, 999));
+    return text;
+}
+
+std::string FormatOverlayGigabytes(std::uint64_t bytes) {
+    return FormatOverlayValue(static_cast<double>(bytes) / GigabyteBytes);
+}
+
+OverlayContent ComposeOverlay(const OverlayStatistics& statistics, bool gpu, bool driver) {
+    return ComposeOverlay(statistics, gpu, driver, OverlayDetails{});
+}
+
+OverlayContent ComposeOverlay(const OverlayStatistics& statistics, bool gpu, bool driver, const OverlayDetails& details) {
+    OverlayContent content;
+    if (gpu || driver) content.above.push_back("PROFILED");
+    content.above.push_back("FPS: " + FormatOverlayRate(statistics.Fps()));
+    content.above.push_back("FRAME: " + FormatOverlayValue(statistics.LastFrameMs()) + " MS");
+    content.frameTimes = statistics.FrameTimes();
+    const auto range = statistics.FrameTimeRange();
+    content.below.push_back("MIN/AVG/MAX: " + FormatOverlayValue(range.minimum) + "/" + FormatOverlayValue(range.average) + "/" + FormatOverlayValue(range.maximum));
+    if (gpu) content.below.push_back("GPU: " + FormatOverlayValue(statistics.GpuMsPerFrame()) + " MS");
+    if (driver) content.below.push_back("DRIVER: " + FormatOverlayValue(statistics.DriverMsPerFrame()) + " MS");
+    const auto latest = statistics.Latest();
+    if (latest.draws != 0 || latest.dispatches != 0) content.below.push_back("DRAWS: " + FormatOverlayCount(statistics.DrawsPerFrame()) + "  DISPATCH: " + FormatOverlayCount(statistics.DispatchesPerFrame()));
+    if (latest.pipelines != 0) content.below.push_back("SHADERS: " + FormatOverlayCount(static_cast<double>(statistics.PipelinesInWindow())) + "  TOTAL: " + FormatOverlayCount(static_cast<double>(latest.pipelines)));
+    if (details.vram) content.below.push_back("VRAM: " + FormatOverlayGigabytes(details.vram->used) + "/" + FormatOverlayGigabytes(details.vram->budget) + " GB");
+    if (details.ram) content.below.push_back("RAM: " + FormatOverlayGigabytes(*details.ram) + " GB");
+    if (details.display.width != 0 && details.display.height != 0 && details.window.width != 0 && details.window.height != 0) {
+        const auto size = [](VkExtent2D extent) { return std::to_string(extent.width) + "X" + std::to_string(extent.height); };
+        for (const std::string_view separator : {" / ", "/"}) {
+            auto line = "RES: " + size(details.display) + std::string(separator) + size(details.window);
+            if (line.size() > static_cast<std::size_t>(OverlayMetrics::Columns)) continue;
+            content.below.push_back(std::move(line));
+            break;
+        }
+    }
+    return content;
+}
+
+bool OverlaySampleDue(std::optional<OverlayStatistics::Clock::time_point> sampledAt, OverlayStatistics::Clock::time_point now) {
+    return !sampledAt || now - *sampledAt >= std::chrono::seconds(1);
+}
+
+std::optional<OverlayMemory> ReadVramBudget(const Graphics::Context& context) {
+    if (context.memoryProperties2 == nullptr || context.physical == VK_NULL_HANDLE) return std::nullopt;
+    const auto heap = Graphics::LargestDeviceLocalHeap(context.memory);
+    if (!heap) return std::nullopt;
+    VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
+    VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, &reported};
+    context.memoryProperties2(context.physical, &properties);
+    if (reported.heapBudget[*heap] == 0) return std::nullopt;
+    return OverlayMemory{reported.heapUsage[*heap], reported.heapBudget[*heap]};
+}
+
+std::optional<std::uint64_t> ReadProcessPrivateBytes() {
+#ifdef _WIN32
+    PROCESS_MEMORY_COUNTERS_EX counters{};
+    counters.cb = sizeof(counters);
+    if (K32GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PROCESS_MEMORY_COUNTERS*>(&counters), sizeof(counters)) == 0) return std::nullopt;
+    return static_cast<std::uint64_t>(counters.PrivateUsage);
+#else
+    std::FILE* status = std::fopen("/proc/self/status", "r");
+    if (status == nullptr) return std::nullopt;
+    std::optional<std::uint64_t> anonymousKb;
+    std::uint64_t swapKb = 0;
+    char line[256];
+    while (std::fgets(line, sizeof(line), status) != nullptr) {
+        unsigned long long value = 0;
+        if (std::sscanf(line, "RssAnon: %llu kB", &value) == 1) anonymousKb = value;
+        else if (std::sscanf(line, "VmSwap: %llu kB", &value) == 1) swapKb = value;
+    }
+    std::fclose(status);
+    if (!anonymousKb) return std::nullopt;
+    return (*anonymousKb + swapKb) * 1024u;
+#endif
+}
+
+std::int32_t OverlayScale(VkExtent2D extent, const OverlayContent& content) {
+    auto scale = std::max<std::int32_t>(1, static_cast<std::int32_t>((extent.height + 240) / 480));
+    while (scale > 1 && !fits(extent, content, scale)) --scale;
+    return scale;
+}
+
+OverlayLayout LayOutOverlay(const OverlayContent& content, VkExtent2D extent, OverlayCorner corner) {
+    Graphics::Require(extent.width != 0 && extent.height != 0, "the performance overlay needs a destination");
+    Graphics::Require(content.frameTimes.size() <= OverlayStatistics::FrameHistory, "the performance overlay graph holds at most 120 frames");
+    OverlayLayout layout;
+    const auto scale = OverlayScale(extent, content);
+    layout.scale = scale;
+    const auto width = panelWidth(scale);
+    const auto height = panelHeight(content, scale);
+    const auto margin = OverlayMetrics::Margin * scale;
+    const bool right = corner == OverlayCorner::TopRight || corner == OverlayCorner::BottomRight;
+    const bool bottom = corner == OverlayCorner::BottomLeft || corner == OverlayCorner::BottomRight;
+    const auto left = std::max(0, right ? static_cast<std::int32_t>(extent.width) - margin - width : margin);
+    const auto top = std::max(0, bottom ? static_cast<std::int32_t>(extent.height) - margin - height : margin);
+    layout.bounds = {left, top, width, height, static_cast<std::uint32_t>(OverlayColor::Background)};
+    layout.rects.push_back(layout.bounds);
+    const auto text = [&](const std::string& line, std::int32_t row) {
+        Graphics::Require(line.size() <= static_cast<std::size_t>(OverlayMetrics::Columns), "a performance overlay line is wider than the panel");
+        for (std::size_t column = 0; column < line.size(); ++column) {
+            const auto& glyph = OverlayGlyph(line[column]);
+            const auto glyphLeft = left + (OverlayMetrics::Padding + static_cast<std::int32_t>(column) * OverlayMetrics::Advance) * scale;
+            for (std::int32_t y = 0; y < OverlayMetrics::GlyphHeight; ++y) {
+                const auto bits = glyph[static_cast<std::size_t>(y)];
+                std::int32_t x = 0;
+                while (x < OverlayMetrics::GlyphWidth) {
+                    if ((bits & (0x10u >> x)) == 0) {
+                        ++x;
+                        continue;
+                    }
+                    auto end = x;
+                    while (end < OverlayMetrics::GlyphWidth && (bits & (0x10u >> end)) != 0) ++end;
+                    layout.rects.push_back({glyphLeft + x * scale, top + (row + y) * scale, (end - x) * scale, scale, static_cast<std::uint32_t>(OverlayColor::Text)});
+                    x = end;
+                }
+            }
+        }
+    };
+    auto row = OverlayMetrics::Padding;
+    for (const auto& line : content.above) {
+        text(line, row);
+        row += OverlayMetrics::LinePitch;
+    }
+    const auto graphBottom = top + (row + OverlayMetrics::GraphHeight) * scale;
+    if (!content.frameTimes.empty()) {
+        double maximum = 0;
+        double sum = 0;
+        for (const auto time : content.frameTimes) {
+            maximum = std::max(maximum, time);
+            sum += time;
+        }
+        const double fullScale = std::max(maximum, 2.0 * sum / static_cast<double>(content.frameTimes.size()));
+        const auto first = static_cast<std::int32_t>(OverlayStatistics::FrameHistory - content.frameTimes.size());
+        for (std::size_t i = 0; i < content.frameTimes.size() && fullScale > 0; ++i) {
+            const auto pixels = std::clamp(static_cast<std::int32_t>(std::lround(content.frameTimes[i] / fullScale * OverlayMetrics::GraphHeight * scale)), 1, OverlayMetrics::GraphHeight * scale);
+            layout.rects.push_back({left + (OverlayMetrics::Padding + first + static_cast<std::int32_t>(i)) * scale, graphBottom - pixels, scale, pixels, static_cast<std::uint32_t>(OverlayColor::Graph)});
+        }
+    }
+    row += OverlayMetrics::GraphHeight + OverlayMetrics::GraphGap;
+    for (const auto& line : content.below) {
+        text(line, row);
+        row += OverlayMetrics::LinePitch;
+    }
+    return layout;
+}
+
+std::array<std::array<float, 4>, OverlayColorCount> OverlayPalette(VkFormat format, VkColorSpaceKHR colorSpace) {
+    std::array<std::array<float, 4>, OverlayColorCount> palette{{{0.0f, 0.0f, 0.0f, 0.6f}, {1.0f, 1.0f, 1.0f, 1.0f}, {0.35f, 0.8f, 0.35f, 1.0f}}};
+    if (colorSpace == VK_COLOR_SPACE_HDR10_ST2084_EXT) {
+        for (auto& color : palette) {
+            const float red = srgbDecode(color[0]);
+            const float green = srgbDecode(color[1]);
+            const float blue = srgbDecode(color[2]);
+            const std::array<float, 3> bt2020{
+                0.6274f * red + 0.3293f * green + 0.0433f * blue,
+                0.0691f * red + 0.9195f * green + 0.0114f * blue,
+                0.0164f * red + 0.0880f * green + 0.8956f * blue};
+            for (std::size_t channel = 0; channel < 3; ++channel) color[channel] = pqEncode(bt2020[channel] * PqReferenceWhiteNits);
+        }
+    } else if (srgbFormat(format)) {
+        for (auto& color : palette) {
+            for (std::size_t channel = 0; channel < 3; ++channel) color[channel] = srgbDecode(color[channel]);
+        }
+    }
+    return palette;
+}
+
+PerformanceOverlay::PerformanceOverlay(const Graphics::Context& context, std::size_t slotCount) : context(context), slots(slotCount) {
+    Graphics::Require(slotCount != 0, "the performance overlay needs at least one slot");
+    try {
+        const VkPushConstantRange push{VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(OverlayParameters)};
+        VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
+        layoutInfo.pushConstantRangeCount = 1;
+        layoutInfo.pPushConstantRanges = &push;
+        Graphics::Check(context.Function<PFN_vkCreatePipelineLayout>("vkCreatePipelineLayout")(context.device, &layoutInfo, nullptr, &pipelineLayout), "vkCreatePipelineLayout performance overlay");
+        const auto createModule = context.Function<PFN_vkCreateShaderModule>("vkCreateShaderModule");
+        VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
+        moduleInfo.codeSize = sizeof(PERFORMANCE_OVERLAY_VERT_SPV);
+        moduleInfo.pCode = PERFORMANCE_OVERLAY_VERT_SPV;
+        Graphics::Check(createModule(context.device, &moduleInfo, nullptr, &vertex), "vkCreateShaderModule performance overlay vertex");
+        moduleInfo.codeSize = sizeof(PERFORMANCE_OVERLAY_FRAG_SPV);
+        moduleInfo.pCode = PERFORMANCE_OVERLAY_FRAG_SPV;
+        Graphics::Check(createModule(context.device, &moduleInfo, nullptr, &fragment), "vkCreateShaderModule performance overlay fragment");
+    } catch (...) {
+        release();
+        throw;
+    }
+}
+
+PerformanceOverlay::~PerformanceOverlay() {
+    release();
+}
+
+void PerformanceOverlay::releaseTransient(Slot& slot) noexcept {
+    for (const auto& [view, framebuffer] : slot.transient) {
+        if (framebuffer) context.Function<PFN_vkDestroyFramebuffer>("vkDestroyFramebuffer")(context.device, framebuffer, nullptr);
+        if (view) context.Function<PFN_vkDestroyImageView>("vkDestroyImageView")(context.device, view, nullptr);
+    }
+    slot.transient.clear();
+}
+
+void PerformanceOverlay::Release(std::size_t index) noexcept {
+    if (index < slots.size()) releaseTransient(slots[index]);
+}
+
+void PerformanceOverlay::release() noexcept {
+    for (auto& slot : slots) {
+        releaseTransient(slot);
+        slot.rects.reset();
+        slot.capacity = 0;
+    }
+    for (const auto& [format, entry] : pipelines) context.Function<PFN_vkDestroyPipeline>("vkDestroyPipeline")(context.device, entry, nullptr);
+    pipelines.clear();
+    for (const auto& [key, entry] : renderPasses) context.Function<PFN_vkDestroyRenderPass>("vkDestroyRenderPass")(context.device, entry, nullptr);
+    renderPasses.clear();
+    if (fragment) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, fragment, nullptr);
+    if (vertex) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, vertex, nullptr);
+    if (pipelineLayout) context.Function<PFN_vkDestroyPipelineLayout>("vkDestroyPipelineLayout")(context.device, pipelineLayout, nullptr);
+    fragment = vertex = VK_NULL_HANDLE;
+    pipelineLayout = VK_NULL_HANDLE;
+}
+
+bool PerformanceOverlay::Visible() {
+    return settings().visible != ((toggleCount.load(std::memory_order_relaxed) & 1u) != 0);
+}
+
+void PerformanceOverlay::Toggle() {
+    toggleCount.fetch_add(1, std::memory_order_relaxed);
+}
+
+OverlayCorner PerformanceOverlay::Corner() {
+    return settings().corner;
+}
+
+void PerformanceOverlay::NotePresent(OverlayStatistics::Clock::time_point at, VkExtent2D display) {
+    restartIfToggled();
+    if (OverlaySampleDue(sampledAt, at)) {
+        details.vram = ReadVramBudget(context);
+        details.ram = ReadProcessPrivateBytes();
+        sampledAt = at;
+    }
+    details.display = display;
+    const auto packets = PacketCounters::Sum();
+    statistics.AddPresent(at, OverlayCounters{PerformanceCounters::GpuEnabled() ? PerformanceCounters::GpuNanoseconds() : 0, PerformanceCounters::DriverEnabled() ? PerformanceCounters::DriverNanoseconds() : 0, packets.draws, packets.dispatches, PerformanceCounters::Pipelines()});
+}
+
+void PerformanceOverlay::restartIfToggled() {
+    const auto current = toggleCount.load(std::memory_order_relaxed);
+    if (current == toggles) return;
+    statistics.Reset();
+    details = {};
+    sampledAt.reset();
+    toggles = current;
+}
+
+VkRenderPass PerformanceOverlay::renderPass(VkFormat format, VkImageLayout initialLayout, VkImageLayout finalLayout) {
+    const auto key = std::tuple{format, initialLayout, finalLayout};
+    if (const auto found = renderPasses.find(key); found != renderPasses.end()) return found->second;
+    VkFormatProperties properties{};
+    context.formatProperties(context.physical, format, &properties);
+    Graphics::Require((properties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT) != 0, "the performance overlay cannot blend into this format");
+    VkAttachmentDescription attachment{};
+    attachment.format = format;
+    attachment.samples = VK_SAMPLE_COUNT_1_BIT;
+    attachment.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
+    attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
+    attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
+    attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
+    attachment.initialLayout = initialLayout;
+    attachment.finalLayout = finalLayout;
+    const VkAttachmentReference reference{0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL};
+    VkSubpassDescription subpass{};
+    subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS;
+    subpass.colorAttachmentCount = 1;
+    subpass.pColorAttachments = &reference;
+    const bool transfer = finalLayout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
+    const VkSubpassDependency dependencies[2] = {
+        {VK_SUBPASS_EXTERNAL, 0, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, 0},
+        {0, VK_SUBPASS_EXTERNAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, transfer ? VK_PIPELINE_STAGE_TRANSFER_BIT : VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, transfer ? VK_ACCESS_TRANSFER_READ_BIT : VkAccessFlags{0}, 0},
+    };
+    VkRenderPassCreateInfo info{VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO};
+    info.attachmentCount = 1;
+    info.pAttachments = &attachment;
+    info.subpassCount = 1;
+    info.pSubpasses = &subpass;
+    info.dependencyCount = 2;
+    info.pDependencies = dependencies;
+    VkRenderPass created = VK_NULL_HANDLE;
+    Graphics::Check(context.Function<PFN_vkCreateRenderPass>("vkCreateRenderPass")(context.device, &info, nullptr, &created), "vkCreateRenderPass performance overlay");
+    return renderPasses.emplace(key, created).first->second;
+}
+
+VkPipeline PerformanceOverlay::pipeline(VkFormat format, VkRenderPass compatible) {
+    if (const auto found = pipelines.find(format); found != pipelines.end()) return found->second;
+    VkPipelineShaderStageCreateInfo stages[2]{{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}, {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}};
+    stages[0].stage = VK_SHADER_STAGE_VERTEX_BIT;
+    stages[0].module = vertex;
+    stages[0].pName = "main";
+    stages[1].stage = VK_SHADER_STAGE_FRAGMENT_BIT;
+    stages[1].module = fragment;
+    stages[1].pName = "main";
+    const VkVertexInputBindingDescription binding{0, sizeof(OverlayRect), VK_VERTEX_INPUT_RATE_INSTANCE};
+    const VkVertexInputAttributeDescription attributes[2] = {{0, 0, VK_FORMAT_R32G32B32A32_SINT, 0}, {1, 0, VK_FORMAT_R32_UINT, 16}};
+    VkPipelineVertexInputStateCreateInfo vertexInput{VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO};
+    vertexInput.vertexBindingDescriptionCount = 1;
+    vertexInput.pVertexBindingDescriptions = &binding;
+    vertexInput.vertexAttributeDescriptionCount = 2;
+    vertexInput.pVertexAttributeDescriptions = attributes;
+    VkPipelineInputAssemblyStateCreateInfo assembly{VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO};
+    assembly.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP;
+    VkPipelineViewportStateCreateInfo viewport{VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO};
+    viewport.viewportCount = 1;
+    viewport.scissorCount = 1;
+    VkPipelineRasterizationStateCreateInfo rasterization{VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO};
+    rasterization.polygonMode = VK_POLYGON_MODE_FILL;
+    rasterization.cullMode = VK_CULL_MODE_NONE;
+    rasterization.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    rasterization.lineWidth = 1.0f;
+    VkPipelineMultisampleStateCreateInfo multisample{VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO};
+    multisample.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT;
+    VkPipelineColorBlendAttachmentState blendAttachment{};
+    blendAttachment.blendEnable = VK_TRUE;
+    blendAttachment.srcColorBlendFactor = VK_BLEND_FACTOR_SRC_ALPHA;
+    blendAttachment.dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA;
+    blendAttachment.colorBlendOp = VK_BLEND_OP_ADD;
+    blendAttachment.srcAlphaBlendFactor = VK_BLEND_FACTOR_ZERO;
+    blendAttachment.dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE;
+    blendAttachment.alphaBlendOp = VK_BLEND_OP_ADD;
+    blendAttachment.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT;
+    VkPipelineColorBlendStateCreateInfo blend{VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO};
+    blend.attachmentCount = 1;
+    blend.pAttachments = &blendAttachment;
+    const VkDynamicState dynamicStates[2] = {VK_DYNAMIC_STATE_VIEWPORT, VK_DYNAMIC_STATE_SCISSOR};
+    VkPipelineDynamicStateCreateInfo dynamic{VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO};
+    dynamic.dynamicStateCount = 2;
+    dynamic.pDynamicStates = dynamicStates;
+    VkGraphicsPipelineCreateInfo info{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO};
+    info.stageCount = 2;
+    info.pStages = stages;
+    info.pVertexInputState = &vertexInput;
+    info.pInputAssemblyState = &assembly;
+    info.pViewportState = &viewport;
+    info.pRasterizationState = &rasterization;
+    info.pMultisampleState = &multisample;
+    info.pColorBlendState = &blend;
+    info.pDynamicState = &dynamic;
+    info.layout = pipelineLayout;
+    info.renderPass = compatible;
+    VkPipeline created = VK_NULL_HANDLE;
+    Graphics::Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, VK_NULL_HANDLE, 1, &info, nullptr, &created), "vkCreateGraphicsPipelines performance overlay");
+    return pipelines.emplace(format, created).first->second;
+}
+
+OverlayRect PerformanceOverlay::Record(VkCommandBuffer commands, std::size_t slot, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout) {
+    auto shown = details;
+    shown.window = extent;
+    return Record(commands, slot, destination, format, colorSpace, extent, initialLayout, finalLayout, ComposeOverlay(statistics, PerformanceCounters::GpuEnabled(), PerformanceCounters::DriverEnabled(), shown), Corner());
+}
+
+OverlayRect PerformanceOverlay::Record(VkCommandBuffer commands, std::size_t index, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout, const OverlayContent& content, OverlayCorner corner) {
+    Graphics::Require(index < slots.size(), "performance overlay slot is out of range");
+    Graphics::Require(destination != VK_NULL_HANDLE && extent.width != 0 && extent.height != 0, "the performance overlay destination is unavailable");
+    auto& slot = slots[index];
+    releaseTransient(slot);
+    const auto layout = LayOutOverlay(content, extent, corner);
+    if (slot.capacity < layout.rects.size()) {
+        const auto capacity = std::max<std::size_t>(layout.rects.size(), 1024);
+        slot.rects = std::make_unique<Graphics::Buffer>(context, capacity * sizeof(OverlayRect), VK_BUFFER_USAGE_VERTEX_BUFFER_BIT);
+        slot.capacity = capacity;
+    }
+    std::memcpy(slot.rects->Bytes().data(), layout.rects.data(), layout.rects.size() * sizeof(OverlayRect));
+    const auto pass = renderPass(format, initialLayout, finalLayout);
+    const auto drawing = pipeline(format, pass);
+    VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
+    viewInfo.image = destination;
+    viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D;
+    viewInfo.format = format;
+    viewInfo.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY};
+    viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
+    VkImageViewUsageCreateInfo usageInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO};
+    usageInfo.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
+    for (const auto layout : {initialLayout, finalLayout}) {
+        if (layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) usageInfo.usage |= VK_IMAGE_USAGE_TRANSFER_DST_BIT;
+        if (layout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) usageInfo.usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
+    }
+    viewInfo.pNext = &usageInfo;
+    VkImageView view = VK_NULL_HANDLE;
+    Graphics::Check(context.Function<PFN_vkCreateImageView>("vkCreateImageView")(context.device, &viewInfo, nullptr, &view), "vkCreateImageView performance overlay");
+    VkFramebufferCreateInfo framebufferInfo{VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO};
+    framebufferInfo.renderPass = pass;
+    framebufferInfo.attachmentCount = 1;
+    framebufferInfo.pAttachments = &view;
+    framebufferInfo.width = extent.width;
+    framebufferInfo.height = extent.height;
+    framebufferInfo.layers = 1;
+    VkFramebuffer framebuffer = VK_NULL_HANDLE;
+    const auto created = context.Function<PFN_vkCreateFramebuffer>("vkCreateFramebuffer")(context.device, &framebufferInfo, nullptr, &framebuffer);
+    slot.transient.emplace_back(view, framebuffer);
+    Graphics::Check(created, "vkCreateFramebuffer performance overlay");
+    const auto right = std::min<std::int32_t>(layout.bounds.x + layout.bounds.width, static_cast<std::int32_t>(extent.width));
+    const auto bottom = std::min<std::int32_t>(layout.bounds.y + layout.bounds.height, static_cast<std::int32_t>(extent.height));
+    const VkRect2D area{{layout.bounds.x, layout.bounds.y}, {static_cast<std::uint32_t>(std::max(1, right - layout.bounds.x)), static_cast<std::uint32_t>(std::max(1, bottom - layout.bounds.y))}};
+    VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO};
+    begin.renderPass = pass;
+    begin.framebuffer = framebuffer;
+    begin.renderArea = area;
+    context.Function<PFN_vkCmdBeginRenderPass>("vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE);
+    context.Function<PFN_vkCmdBindPipeline>("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, drawing);
+    const VkViewport viewport{0.0f, 0.0f, static_cast<float>(extent.width), static_cast<float>(extent.height), 0.0f, 1.0f};
+    context.Function<PFN_vkCmdSetViewport>("vkCmdSetViewport")(commands, 0, 1, &viewport);
+    context.Function<PFN_vkCmdSetScissor>("vkCmdSetScissor")(commands, 0, 1, &area);
+    const OverlayParameters parameters{OverlayPalette(format, colorSpace), {static_cast<float>(extent.width), static_cast<float>(extent.height)}};
+    context.Function<PFN_vkCmdPushConstants>("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(parameters), &parameters);
+    const VkBuffer buffer = slot.rects->Handle();
+    const VkDeviceSize offset = 0;
+    context.Function<PFN_vkCmdBindVertexBuffers>("vkCmdBindVertexBuffers")(commands, 0, 1, &buffer, &offset);
+    context.Function<PFN_vkCmdDraw>("vkCmdDraw")(commands, 4, static_cast<std::uint32_t>(layout.rects.size()), 0, 0);
+    context.Function<PFN_vkCmdEndRenderPass>("vkCmdEndRenderPass")(commands);
+    return layout.bounds;
+}
+
+}
+
+extern "C" void AgcDriverTogglePerformanceOverlay_nid_postfix() {
+    AgcDriver::PerformanceOverlay::Toggle();
+}

core/libs/prx/libSceAgcDriver/Execution/src/PerformanceTraceThread.cpp

@@ -0,0 +1,16 @@
+#include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp"
+
+namespace AgcDriver {
+
+std::atomic<bool> PerformanceTrace::recording{false};
+thread_local TraceThread* PerformanceTrace::local = nullptr;
+
+}
+
+extern "C" bool AgcDriverTraceRecording_nid_postfix() {
+    return AgcDriver::PerformanceTrace::recording.load(std::memory_order_relaxed);
+}
+
+extern "C" AgcDriver::TraceThread* AgcDriverTraceThread_nid_postfix() {
+    return AgcDriver::PerformanceTrace::local;
+}

core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp

@@ -175,7 +175,7 @@ std::vector<std::byte> copyDataSource(std::span<const std::uint32_t> packet, con
 
 }
 
-std::string Name(std::uint32_t header) {
+std::string_view KnownName(std::uint32_t header) {
     if (FillerPacket(header)) return "PM4_FILLER";
     const auto opcode = (header >> 8u) & 0xffu;
     if (opcode == 0x10 && (header & 0xfcu) != 0) {
@@ -194,9 +194,14 @@ std::string Name(std::uint32_t header) {
             default: return "UNKNOWN_CUSTOM";
         }
     }
-    for (const auto& entry : Opcodes) if (entry.value == opcode) return std::string(entry.name);
+    for (const auto& entry : Opcodes) if (entry.value == opcode) return entry.name;
+    return {};
+}
+
+std::string Name(std::uint32_t header) {
+    if (const auto known = KnownName(header); !known.empty()) return std::string(known);
     char text[24]{};
-    std::snprintf(text, sizeof(text), "UNKNOWN_0x%02x", opcode);
+    std::snprintf(text, sizeof(text), "UNKNOWN_0x%02x", (header >> 8u) & 0xffu);
     return text;
 }
 
@@ -373,9 +378,11 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
             require((packet[1] & 0x7fffffffu) == 0 && (packet[2] & 0x7fffffffu) == 0, "CONTEXT_CONTROL register loading and shadowing are not implemented");
             break;
         case 0x46: {
-            require((packet[1] & ~0x73fu) == 0, "unsupported EVENT_WRITE flags or reserved bits");
             const auto eventType = packet[1] & 0x3fu;
-            const auto eventIndex = (packet[1] >> 8u) & 7u;
+            const auto eventIndex = (packet[1] >> 8u) & 0xfu;
+            const auto streamOutStatistics = (eventType >= 0x01u && eventType <= 0x03u) || eventType == 0x20u;
+            require(!streamOutStatistics || (eventIndex != 3 && (eventIndex < 8 || eventIndex > 11)), "stream-out statistics sampling to memory is not implemented");
+            require((packet[1] & ~0x73fu) == 0, "unsupported EVENT_WRITE flags or reserved bits");
             switch (eventType) {
                 case 0x07: case 0x0f: case 0x10:
                     size(2);
@@ -396,6 +403,11 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
                     size(2);
                     require(eventIndex == 0, "invalid VGT_FLUSH event index");
                     break;
+                case 0x01: case 0x02: case 0x03: case 0x20:
+                    graphics();
+                    size(2);
+                    require(eventIndex == 0, "invalid stream-out statistics event index");
+                    break;
                 case 0x39:
                     graphics();
                     size(4);

core/libs/prx/libSceAgcDriver/Execution/src/ShaderDeviceProfile.cpp

@@ -31,6 +31,7 @@ ShaderDeviceProfile::ShaderDeviceProfile(const ShaderRecompiler::SpirvTarget& ta
     const auto* mesh = findFeatures<VkPhysicalDeviceMeshShaderFeaturesEXT>(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MESH_SHADER_FEATURES_EXT);
     const auto* clock = findFeatures<VkPhysicalDeviceShaderClockFeaturesKHR>(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CLOCK_FEATURES_KHR);
     const auto* drawParameters = findFeatures<VkPhysicalDeviceShaderDrawParametersFeatures>(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_DRAW_PARAMETERS_FEATURES);
+    const auto* terminate = findFeatures<VkPhysicalDeviceShaderTerminateInvocationFeaturesKHR>(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_TERMINATE_INVOCATION_FEATURES_KHR);
     const auto* maintenance8 = findFeatures<VkPhysicalDeviceMaintenance8FeaturesKHR>(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_8_FEATURES_KHR);
     Require(target.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version, "shader device profile has an incompatible BDA ABI");
     Require(bda != nullptr && bda->bufferDeviceAddress == VK_TRUE && core.shaderInt64 == VK_TRUE, "shader runtime requires enabled bufferDeviceAddress and shaderInt64");
@@ -46,6 +47,7 @@ ShaderDeviceProfile::ShaderDeviceProfile(const ShaderRecompiler::SpirvTarget& ta
         Require(!hasCapability(capability) || enabled, std::string("shader device profile advertises a disabled feature: ") + name);
     };
     checkFeature(spv::CapabilityFloat64, core.shaderFloat64 == VK_TRUE, "shaderFloat64");
+    checkFeature(spv::CapabilityGeometry, core.geometryShader == VK_TRUE, "geometryShader");
     checkFeature(spv::CapabilityTessellation, core.tessellationShader == VK_TRUE, "tessellationShader");
     checkFeature(spv::CapabilityImageGatherExtended, core.shaderImageGatherExtended == VK_TRUE, "shaderImageGatherExtended");
     checkFeature(spv::CapabilityClipDistance, core.shaderClipDistance == VK_TRUE, "shaderClipDistance");
@@ -76,6 +78,7 @@ ShaderDeviceProfile::ShaderDeviceProfile(const ShaderRecompiler::SpirvTarget& ta
     extensions.erase(std::unique(extensions.begin(), extensions.end()), extensions.end());
     for (const auto& extension : extensions) extensionViews.emplace_back(extension);
     Require(std::ranges::find(extensionViews, "SPV_KHR_physical_storage_buffer") != extensionViews.end() && std::ranges::find(extensionViews, "SPV_KHR_8bit_storage") != extensionViews.end(), "shader device profile is missing runtime extensions");
+    Require(std::ranges::find(extensionViews, "SPV_KHR_terminate_invocation") == extensionViews.end() || (terminate != nullptr && terminate->shaderTerminateInvocation == VK_TRUE), "shader device profile advertises a disabled feature: shaderTerminateInvocation");
     this->target.supportedCapabilities = capabilities;
     this->target.supportedExtensions = extensionViews;
 }

core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp

@@ -219,7 +219,7 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val
     const auto index = static_cast<std::size_t>((address % PageBytes) / sizeof(*value));
     if (!page.valid.Test(index) && page.lazy) {
         const auto first = index / BlockWords * BlockWords;
-        std::memcpy(&page.words[first], reinterpret_cast<const void*>((address & ~static_cast<std::uint64_t>(PageBytes - 1)) + first * sizeof(std::uint32_t)), BlockWords * sizeof(std::uint32_t));
+        GuestMemory::CopyGuestBytes(&page.words[first], reinterpret_cast<const void*>((address & ~static_cast<std::uint64_t>(PageBytes - 1)) + first * sizeof(std::uint32_t)), BlockWords * sizeof(std::uint32_t));
         page.valid.SetBlock(index);
     }
     if (!page.valid.Test(index)) {

core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp

@@ -54,7 +54,8 @@ struct SlabSlot {
 class BufferPool {
 public:
     explicit BufferPool(const Context& context);
-    static VkDeviceSize DeviceBudget(const VkPhysicalDeviceMemoryProperties& memory);
+    static VkDeviceSize DeviceBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported = nullptr);
+    static VkDeviceSize ReclaimTarget(VkDeviceSize bytes);
     ~BufferPool();
     BufferPool(const BufferPool&) = delete;
     BufferPool& operator=(const BufferPool&) = delete;
@@ -116,6 +117,7 @@ private:
     // when shared; the device tier for device-local memory while it has a budget).
     Tier& tierFor(std::size_t capacity, VkMemoryPropertyFlags properties);
     static bool DeviceTierEnabled();
+    static VkDeviceSize deviceBudget(const Context& context);
     static bool DeviceTiered(VkMemoryPropertyFlags properties);
     static std::size_t NextCapacity(std::size_t capacity);
     VkDeviceSize destroy(const BufferAllocation& allocation) noexcept;
@@ -127,6 +129,7 @@ private:
     // The retained-slot bound of each tier (APS5_BUFFER_POOL_SLOTS, default `defaultSlots`), read once.
     static std::size_t MaxSlots();
     std::size_t maxSlots;
+    VkDeviceSize (*cacheReclaim)(VkDeviceSize bytes);
     VkDevice device;
     PFN_vkUnmapMemory unmap;
     PFN_vkDestroyBuffer destroyBuffer;

core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp

@@ -171,6 +171,7 @@ struct Context {
     std::uint32_t settleFamily = 0;
     std::mutex* settleQueueMutex = nullptr;
     PFN_vkGetPhysicalDeviceMemoryProperties2 memoryProperties2 = nullptr;
+    VkDeviceSize (*cacheReclaim)(VkDeviceSize bytes) = nullptr;
     RenderCache* renderCache = nullptr;
     DrawQueue* drawQueue = nullptr;
     GraphicsPipelineCache* graphicsPipelines = nullptr;
@@ -210,8 +211,10 @@ struct Context {
     bool variableDescriptorCount = false;
     std::uint32_t imageTableCapacity = 0;
     std::uint32_t samplerTableCapacity = 0;
+    bool individuallyFreeDescriptorSets = false;
     bool geometryShader = false;
     bool sampleRateShading = false;
+    bool storageImageMultisample = false;
     bool shaderClipDistance = false;
     bool shaderCullDistance = false;
     bool nullDescriptors = false;
@@ -226,12 +229,14 @@ struct Context {
     bool graphicsPipelineLibrary = false;
     bool pipelineLinkTimeOptimization = true;
     std::uint32_t srgbDecodeFormats = 0;
+    std::uint32_t srgbCopyFormats = 0;
     bool provokingVertexLast = false;
     bool provokingVertexModePerPipeline = false;
     bool singlePassStorage = false;
     VkSampleCountFlags sampleLocationSampleCounts = 0;
     VkSampleCountFlags sampleLocationGridSampleCounts = 0;
     PipelineCacheSynchronization* pipelineCacheSynchronization = nullptr;
+    bool shaderTerminateInvocation = false;
 
     PipelineCacheSynchronization::PipelineUseLock LockPipelineCacheForUse() const {
         return pipelineCacheSynchronization != nullptr ? pipelineCacheSynchronization->LockForPipelineUse() : PipelineCacheSynchronization::PipelineUseLock{};
@@ -317,6 +322,7 @@ inline void EndRenderPass(const Context& context, VkCommandBuffer commands) {
 }
 
 std::uint32_t ImageTableCapacity(const Context& context);
+std::uint32_t StorageTableCapacity(const Context& context);
 std::uint32_t SamplerTableCapacity(const Context& context);
 
 }

core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp

@@ -99,6 +99,16 @@ DccKeys ProvedCurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceByt
 void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span<std::byte> bytes);
 void NoteKeysFillOnGpu(std::uint64_t begin, std::size_t count, DccKeys keys);
 std::optional<DccKeys> WaitForKeyWriters(const GuestTextureResource& resource, std::uint64_t guestBytes);
+bool DccKeyWritersPending(std::uint64_t metaAddress, std::size_t count);
+bool DccKeyWriterNeedsSync(std::uint64_t metaAddress, std::size_t count);
+struct DccKeyWaitCounts {
+    std::uint64_t calls;
+    std::uint64_t noPending;
+    std::uint64_t known;
+    std::uint64_t fillsUncompressed;
+    std::uint64_t synced;
+};
+DccKeyWaitCounts KeyWaitCounts();
 
 }
 

core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp

@@ -26,9 +26,11 @@ void RequireNoNativeDepthFeedback_nid_no_patch(const Context& context, const Dep
 void ClearDepthSurfaces(VkDevice device);
 void RetireDepthSurfaces(VkDevice device, std::uint64_t address, std::uint64_t bytes);
 bool DepthSurfaceAt(std::uint64_t address);
+bool DepthSurfaceOverlaps(VkDevice device, std::uint64_t address, std::uint64_t bytes);
 bool DepthSurfaceAt(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& resource);
 std::uint64_t HtileDepthClearAddress(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const std::array<std::uint32_t, 3>& numThreads);
-void NoteHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes);
+std::size_t HtileDepthClearBytes_nid_no_patch(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, std::span<const std::uint32_t> packet, std::uint64_t indirectArguments);
+void NoteHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes, std::size_t nativeBytes = 0u);
 struct MemoryValueFill {
     std::uint64_t address;
     std::size_t bytes;
@@ -47,6 +49,49 @@ float HostDepthBiasConstant(const Context& context, VkFormat format, float const
 std::shared_ptr<StorageTexture> DepthSurfaceStorage(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& resource, std::uint32_t mip);
 bool DepthSurfaceStoreBack(const Context& context, const StorageTexture& storage);
 
+struct StorageImageClear {
+    std::array<std::uint32_t, 8> image{};
+    std::array<std::uint32_t, 4> color{};
+    std::uint32_t width = 0;
+    std::uint32_t height = 0;
+};
+std::optional<StorageImageClear> MatchStorageImageClear(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, const std::array<std::uint32_t, 3>& groups);
+std::uint64_t StorageImageClearAddress(const StorageImageClear& clear);
+struct DepthPlaneOverwrite {
+    std::array<std::uint32_t, 8> image{};
+    GuestTextureResource view{};
+    std::uint32_t bits = 0;
+    std::uint32_t width = 0;
+    std::uint32_t height = 0;
+    std::uint32_t raw = 0;
+    std::uint64_t colorAddress = 0;
+    std::uint32_t colorBits = 0;
+    float depth = 0.0f;
+};
+class DepthPlaneClear {
+public:
+    explicit DepthPlaneClear(const StorageImageClear& clear);
+    ~DepthPlaneClear();
+    DepthPlaneClear(const DepthPlaneClear&) = delete;
+    DepthPlaneClear& operator=(const DepthPlaneClear&) = delete;
+    bool Active() const { return installed; }
+
+private:
+    DepthPlaneOverwrite value;
+    DepthPlaneOverwrite* previous = nullptr;
+    bool installed = false;
+};
+struct DepthClearStatistics {
+    std::uint64_t matched = 0;
+    std::uint64_t taken = 0;
+    std::uint64_t copiesSkipped = 0;
+    std::uint64_t clears = 0;
+    std::uint64_t bytesSkipped = 0;
+    std::array<std::uint64_t, 8> declined{};
+};
+DepthClearStatistics DepthClearCounts();
+void SetDepthPlaneClearForTest(bool enabled);
+
 }
 
 #endif

core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp

@@ -31,7 +31,7 @@ struct DrawInputCopy {
 DrawInputCopy CopyDrawInput(const Context& context, Recorder* recorder, std::uint64_t address, std::size_t bytes, std::size_t alignment, Recorder::SnapshotUse use);
 DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes);
 void KeepDrawInput(Recorder* recorder, std::uint64_t address, const DrawInputCopy& copy, Recorder::SnapshotUse use, std::uint32_t derived);
-const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment);
+const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment, bool* behindGpuWrite = nullptr);
 struct InPlaceIndices {
     std::optional<std::uint32_t> highest;
     bool reused = false;
@@ -93,6 +93,16 @@ bool DrawRecipes();
 // `rewritten` counts a GPU-side draw whose records were copied and patched with a constant.
 enum class IndirectDrawPath : std::uint8_t { Gpu, NotFolded, FetchUnknown, NonVertexPath, IndxOffset, DrawIndex, VertexRange, FeatureGap, PendingImage, PendingLabelOrCopy, NotImported, Disabled, Count };
 void CountIndirectDraw(IndirectDrawPath path, double readMs, bool rewritten = false);
+std::array<std::uint64_t, static_cast<std::size_t>(IndirectDrawPath::Count)> IndirectDrawCounts();
+
+struct DrawPassCounts {
+    std::uint64_t draws = 0;
+    std::uint64_t passesBegun = 0;
+    std::uint64_t passesContinued = 0;
+    std::uint64_t keyStoreFlushes = 0;
+    std::uint64_t storeFlushes = 0;
+};
+std::array<DrawPassCounts, 2> DrawPassTotals();
 const char* IndirectDrawPathName(IndirectDrawPath path);
 
 // Draw packets that drew nothing, for the [draws] line: Nothing (an empty count, or no color writes

core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp

@@ -40,8 +40,13 @@ struct HostImport {
     bool joined = false;
     std::shared_ptr<void> chunk;
     mutable std::uint64_t lastUse = 0;
+    std::uint64_t lead = 0;
 };
 
+inline VkDeviceSize HostImportOffset(const HostImport& import, std::uint64_t address) {
+    return address - import.base + import.lead;
+}
+
 enum class ImportWatch : std::uint8_t { Watch, Unwatch };
 
 struct ImportProbe {
@@ -63,14 +68,27 @@ ImportWatch PrepareImportWatch(const Context& context);
 void SetImportWatch(const Context& context, ImportWatch watch);
 
 // The host import of the registered allocation containing [address, address + bytes), made on demand
-// (alignment and budget permitting), or null. Bytes at `address` are at `address - import->base` in
-// the import's buffer.
+// (alignment and budget permitting), or null. Bytes at `address` are at `HostImportOffset(*import,
+// address)` in the import's buffer.
 const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes);
+const HostImport* ExistingHostImportFor(const Context& context, std::uint64_t address, std::size_t bytes);
 void SetHostImportWindow(std::uint64_t bytes);
 // Tests: the window of one large registered range (64 MiB) and the import budget (0: the configured one).
 void SetHostImportRangeWindow(std::uint64_t bytes);
 void SetHostImportBudget(std::uint64_t bytes);
+void SetHostImportDeferredSettles(bool enabled);
 std::uint64_t HostImportLiveBytes();
+struct HostImportSettleCounts {
+    std::uint64_t submitted = 0;
+    std::uint64_t waited = 0;
+    std::uint64_t rewatched = 0;
+    std::uint64_t parked = 0;
+    std::uint64_t unmapWaits = 0;
+    std::uint64_t keptCompared = 0;
+};
+HostImportSettleCounts HostImportSettles();
+// Tests: how many imports settleImport has used on the settle queue.
+std::uint64_t HostImportsSettled();
 bool ImportMappedRanges(const Context& context, const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice);
 // Whether an existing import covers [address, address + bytes), without reconciling the imports
 // with the registry or making one (HostImportFor may take a registry lease): a hint for choices
@@ -374,7 +392,7 @@ private:
     void completeSpaceWrites() const;
     // The region containing `address`: this build's own regions (sorted after UploadPrepare) or
     // the space's base regions, or null.
-    const Region* owner(std::uint64_t address) const;
+    const Region* owner(std::uint64_t address, std::uint64_t end) const;
     // Builds this build's regions from a lease as the per-build path always did: `base` gets the
     // ranges served in place (imports, mirrors), `copied` the ones copied per build.
     struct CopiedRange {
@@ -429,6 +447,7 @@ private:
     // Whether `regions` is in ascending address order (true right after AcquireRegistered, whose
     // regions follow the registry's order), so AddSnapshot can search instead of scanning.
     bool regionsSorted = false;
+    bool regionsOverlap = false;
     std::vector<std::pair<std::uint64_t, std::uint64_t>> writes;
     std::vector<std::pair<std::uint64_t, std::uint64_t>> readsOverBase;
     std::vector<std::pair<std::uint64_t, std::vector<std::byte>>> heapReferences;

core/libs/prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp

@@ -8,6 +8,7 @@
 #include <cstdint>
 #include <memory>
 #include <optional>
+#include <span>
 #include <string>
 
 namespace AgcDriver::Graphics {
@@ -49,11 +50,15 @@ private:
 };
 
 std::uint64_t MultisampledColorBytes(const ColorTarget& target);
+std::size_t MultisampledDccBytes(const ColorTarget& target);
+VkClearColorValue MultisampledDccClear(const ColorTarget& target, std::span<const std::byte> dcc, std::span<const std::byte> cmask);
 bool MultisampledFillClearColor(const ColorTarget& target, std::uint32_t fill, VkClearColorValue& clear);
 bool TakeMultisampledClear(MultisampledCmask& cmask, MultisampledColorFill& colorFill, const ColorTarget& use, VkClearColorValue& clear);
 VkImageView MultisampleTargetView(const Context& context, const ColorTarget& target);
 std::shared_ptr<Texture> MultisampleTargetTexture(const Context& context, const GuestTextureResource& resource, VkComponentMapping components);
 void MaterializeMultisampleTargetClear(const Context& context, const ColorTarget& target);
+VkImageView MultisampleTargetStorageView(const Context& context, const GuestTextureResource& resource);
+bool MultisampleTargetStorable(const Context& context, const ColorTarget& target);
 void ResolveMultisampleTarget(const Context& context, const ColorTarget& source, StorageTexture& destination, VkImageView destinationView);
 bool MultisampledColorFillAbsorbs(std::uint64_t address, std::size_t bytes, std::uint32_t pattern);
 void NoteColorMetadataFill(std::uint64_t address, std::size_t bytes, std::uint32_t pattern);

core/libs/prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp

@@ -16,6 +16,8 @@ struct CompareSpan {
 void CompareSpans(std::span<const CompareSpan> spans, std::span<std::uint8_t> equal);
 void CompareSpansFrom(std::span<const CompareSpan> spans, std::span<std::uint8_t> equal, std::size_t parallelBytes);
 unsigned CompareHelpers();
+void StopCompareHelpers();
+unsigned CompareHelpersRunning();
 
 }
 

core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp

@@ -94,14 +94,29 @@ private:
     std::vector<CachedFramebuffer> framebuffers;
 };
 
-// The pipeline for this configuration, shared across draws and frames: keyed by the stages' variant
-// ids (the rect-list tessellation pair by the vertex and fragment ids that generate it), the vertex
-// input layout, the descriptor set layout key, push constant stages and every pipeline state field.
-// A configuration whose stage has no variant id gets a private pipeline. Entries are LRU-bounded and
-// only evicted once no recorded draw holds them. Debug aid: APS5_NO_PIPELINE_CACHE=1 builds one per
-// draw as before.
+// The pipeline for this configuration, shared across draws and frames: keyed by the stages'
+// identities (the rect-list tessellation pair and the geometry stage by the vertex and fragment
+// results that generate them), the vertex input layout, the descriptor set layout key, push constant
+// stages and every pipeline state field. A configuration whose stage has no module gets a private
+// pipeline. A miss builds the pipeline outside the store's mutex (a concurrent build of the same
+// key keeps the first inserted) and records its recipe in the pipeline journal. Entries are
+// LRU-bounded (APS5_PIPELINE_STORE_ENTRIES, 1024) and only evicted once no recorded draw holds
+// them. Debug aid: APS5_NO_PIPELINE_CACHE=1 builds one per draw as before.
 std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
-std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+// `created` reports whether this call built the pipeline (false: served from the store or private).
+std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, bool* created = nullptr);
+// Cumulative pipeline store outcomes since start-up (the [pipecache] line reports 10 s windows).
+struct PipelineStoreCounters {
+    std::uint64_t hits = 0;
+    std::uint64_t misses = 0;
+    std::uint64_t uncached = 0;
+    std::uint64_t evicted = 0;
+    std::uint64_t cached = 0;
+    std::uint64_t bound = 0;
+};
+PipelineStoreCounters PipelineStoreTotals();
+// The pipeline store's key of a configuration (empty when the configuration gets a private pipeline).
+void BuildPipelineStoreKey(std::vector<std::byte>& key, const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout);
 struct DescribedSetLayouts {
     PipelineSetLayouts layouts;
     std::vector<std::uint32_t> mainKey;
@@ -119,6 +134,7 @@ bool HasStencil(VkFormat format);
 void ValidateViewport(const Context& context, const VkViewport& viewport);
 void ValidateDepthBounds(const Context& context, const State& state);
 void ValidateProvokingVertex(const Context& context, const State& state, std::span<const CompiledShader> shaders = {});
+bool BarycentricPatches(const State& state, std::span<const CompiledShader> shaders);
 bool ValidateSampleLocations(const Context& context, const State& state);
 
 void ValidateShaderPair(const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment);
@@ -126,7 +142,7 @@ bool ShadersWriteMemory(std::span<const CompiledShader> shaders);
 bool DeclaresEarlyFragmentTests(const ShaderRecompiler::RecompileResult& program);
 void ValidateSampleMask(const Context& context, const State& state, std::span<const CompiledShader> shaders);
 // Returns the color attachment locations the pixel shader writes.
-std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing = false, bool imageInt64Atomics = false, bool geometryShader = false, bool sampleRateShading = false, bool bufferInt64Atomics = false, bool shaderClipDistance = false, bool shaderCullDistance = false, bool viewportIndexLayer = false, bool runtimeDescriptorArray = false, bool demoteToHelperInvocation = false);
+std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing = false, bool imageInt64Atomics = false, bool geometryShader = false, bool sampleRateShading = false, bool bufferInt64Atomics = false, bool shaderClipDistance = false, bool shaderCullDistance = false, bool viewportIndexLayer = false, bool runtimeDescriptorArray = false, bool demoteToHelperInvocation = false, bool shaderTerminateInvocation = false);
 
 }
 

core/libs/prx/libSceAgcDriver/Graphics/include/PipelineCache.hpp

@@ -18,13 +18,14 @@ class PipelineCache {
 public:
     static constexpr int SaveIntervalSeconds = 10;
 
-    PipelineCache(const Context& context, const VkPhysicalDeviceProperties& properties);
+    PipelineCache(const Context& context, const VkPhysicalDeviceProperties& properties, bool loadFromFile = true);
     ~PipelineCache();
 
     PipelineCache(const PipelineCache&) = delete;
     PipelineCache& operator=(const PipelineCache&) = delete;
     VkPipelineCache Handle() const { return cache; }
     PipelineCacheSynchronization& Synchronization() { return synchronization; }
+    void SaveNow();
 
 private:
     struct FileHeader {
@@ -54,6 +55,7 @@ private:
     VkPipelineCache cache = VK_NULL_HANDLE;
     PipelineCacheSynchronization synchronization;
     std::filesystem::path path;
+    bool holdsFile = false;
     std::size_t loadedBytes = 0;
     std::size_t settledBytes = 0;
     std::optional<FileHeader> settledFile;

core/libs/prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp

@@ -0,0 +1,104 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PIPELINEJOURNAL_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PIPELINEJOURNAL_HPP
+
+#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp"
+#include "Recompiler.hpp"
+#include <cstddef>
+#include <cstdint>
+#include <filesystem>
+#include <functional>
+#include <memory>
+#include <span>
+#include <vector>
+
+namespace AgcDriver::Graphics {
+
+// What a graphics pipeline is built from besides the device: the stage results (named by their
+// run-stable identity), the vertex input layout, the descriptor set layout keys and the state
+// fields the Pipeline constructor reads, with every other State field cleared
+// (CanonicalPipelineState). The pipeline store keys on exactly this, and the journal records it so
+// the next run builds the same VkPipeline before the title asks for it. A generated stage (the
+// rect-list control and evaluation pair, the barycentric geometry stage) is built from the vertex
+// and fragment results and the device's limits; its identity is in the key like any other stage's,
+// so a replayed pipeline built from another device's generated module is missed, not used.
+struct GraphicsPipelineRecipe {
+    struct Stage {
+        ShaderRecompiler::ShaderStage stage;
+        ShaderRecompiler::PipelineIdentity identity;
+        std::uint32_t pushConstantOffset;
+        bool generated;
+    };
+    VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+    std::vector<Stage> stages;
+    VertexInputLayout vertexInput;
+    std::vector<std::uint32_t> mainLayoutKey;
+    std::vector<std::uint32_t> fragmentLayoutKey;
+    bool splitSets = false;
+    State state;
+};
+
+[[nodiscard]] State CanonicalPipelineState(const State& state);
+[[nodiscard]] bool GeneratedPipelineStage(const State& state, ShaderRecompiler::ShaderStage stage);
+[[nodiscard]] GraphicsPipelineRecipe MakeGraphicsPipelineRecipe(const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout);
+[[nodiscard]] std::vector<std::byte> EncodeGraphicsPipelineRecipe(const GraphicsPipelineRecipe& recipe);
+[[nodiscard]] bool DecodeGraphicsPipelineRecipe(std::span<const std::byte> bytes, GraphicsPipelineRecipe& recipe);
+
+struct PipelineJournalCounters {
+    std::uint64_t loadedRecipes = 0;
+    std::uint64_t loadedModules = 0;
+    std::uint64_t loadedBytes = 0;
+    std::uint64_t droppedRecords = 0;
+    double loadMs = 0;
+    std::uint64_t replayedGraphics = 0;
+    std::uint64_t replayedCompute = 0;
+    std::uint64_t replayPresent = 0;
+    std::uint64_t replaySkipped = 0;
+    std::uint64_t replayRejected = 0;
+    std::uint64_t replayFailed = 0;
+    double replayMs = 0;
+    std::uint32_t replayThreads = 0;
+    std::uint64_t recordedGraphics = 0;
+    std::uint64_t recordedCompute = 0;
+    std::uint64_t recordedModules = 0;
+    std::uint64_t writtenBytes = 0;
+    std::uint64_t writeFailures = 0;
+};
+
+// A decimal switch of the pipeline store or journal, or its default when unset; anything else throws.
+[[nodiscard]] std::uint64_t PipelineSwitch(const char* name, std::uint64_t fallback);
+[[nodiscard]] bool PipelineJournalEnabled();
+// True on a replay thread while it builds an entry: what it builds is already in the file.
+[[nodiscard]] bool PipelineJournalReplaying();
+[[nodiscard]] std::filesystem::path PipelineJournalPath();
+[[nodiscard]] PipelineJournalCounters PipelineJournalTotals();
+void RecordGraphicsPipeline(const GraphicsPipelineRecipe& recipe, std::span<const CompiledShader> shaders);
+void RecordComputePipeline(const ShaderRecompiler::RecompileResult& shader, bool pushConstants);
+void FlushPipelineJournal();
+void ReportPipelineJournal();
+
+// Cancel stops the threads taking further entries (the pipeline each is building completes);
+// Wait joins them. A device cancels its replay before it is replaced or destroyed.
+class PipelineJournalReplay {
+public:
+    virtual ~PipelineJournalReplay() = default;
+    virtual void Cancel() = 0;
+    virtual void Wait() = 0;
+    [[nodiscard]] virtual bool Done() const = 0;
+};
+
+struct PipelineJournalHooks {
+    std::function<bool(const ShaderRecompiler::RecompileResult& shader, bool pushConstants)> compute;
+};
+
+// The journal belongs to the device that wrote it, named by the same properties as the driver's
+// pipeline cache file; a replay for another device starts the file over.
+[[nodiscard]] std::uint64_t PipelineJournalDeviceFingerprint(const VkPhysicalDeviceProperties& properties);
+[[nodiscard]] std::shared_ptr<PipelineJournalReplay> StartPipelineJournalReplay(const Context& context, std::uint64_t deviceFingerprint, PipelineJournalHooks hooks);
+
+}
+
+#endif

core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp

@@ -4,6 +4,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
 #include <array>
 #include <atomic>
+#include <functional>
 #include <memory>
 #include <cstddef>
 #include <cstdint>
@@ -32,7 +33,13 @@ struct OptimizedPipeline {
     PFN_vkDestroyPipeline destroy = nullptr;
     std::atomic<VkPipeline> handle{VK_NULL_HANDLE};
     std::atomic<bool> released{false};
+    // The relink of a pipeline the journal replay built waits for the first draw that binds it
+    // (Schedule), so the relink thread serves the title's own pipelines first and a replayed
+    // pipeline the title never draws is never relinked.
+    std::atomic<bool> deferred{false};
+    std::function<void()> relink;
     void Release() noexcept;
+    void Schedule();
 };
 
 VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr<OptimizedPipeline>* optimized = nullptr, bool partsOnly = false, std::span<VkShaderModule> modules = {});
@@ -41,6 +48,9 @@ void WaitForOptimizedPipelines(VkDevice device);
 
 void ClearPipelineLibraries(VkDevice device);
 
+void StopPipelineLibraryWorker();
+unsigned PipelineLibraryWorkers();
+
 struct PipelineLibraryCounters {
     std::array<std::uint64_t, 4> built{};
     std::uint64_t linked = 0;

core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp

@@ -752,6 +752,8 @@ private:
     bool syncThroughUnlocked(std::uint64_t address, std::uint64_t end, int source, const void* site);
     // `source` is the CountSync source the wait is attributed to.
     void finish(std::unique_ptr<Batch> batch, bool wait, int source, bool releaseNow = false);
+    void discard(std::unique_ptr<Batch> batch) noexcept;
+    void removeLabels(Batch& batch);
     void release(Batch& batch) noexcept;
     static bool overlaps(const Batch& batch, std::uint64_t address, std::uint64_t end);
     static std::optional<ReadKind> readOverlap(const Batch& batch, std::uint64_t address, std::uint64_t end);