perf(agc): cheaper shader-memory captures per draw

The per-draw capture of a shader's memory (the SRT walk and the guest pages it reads) cost about
15 us per draw packet in GTA V Performance mode; it now costs about 6.6.

- The SRT evaluator memoizes per plan value in a thread's reused, stamped array instead of a hash
  table allocated and grown on every walk (evaluate 5.3 -> 3.6 us per capture).
- ExtractPlan also stores the plan's values as 32-byte records with argument ids, and the walk
  evaluates that compact copy: the IR walk spent most of its time in cache misses (3.6 -> 2.7 us).
  The evaluator is a template over the node view, so both walks share every opcode's semantics, memo,
  cycle and failure rules. A plan that cannot be expressed compactly keeps the IR walk.
  APS5_VERIFY_SRT_COMPACT=1 walks both and throws on any difference.
- ShaderMemory's valid/read/recent marks are arrays of 64-bit words scanned with countr_zero instead
  of bitsets tested bit by bit (about 1 us per draw packet).
- Captured pages come from a per-thread pool, are found through a short vector with a last-page
  shortcut, and a page no recorded GPU work writes gets the flush hook a whole-page read would make,
  then has each 64-byte block copied when a word of it is first read (about 0.3 us per packet).
  Pages with pending GPU writes and unmapped pages are read as before.
- APS5_PROFILE_DRAW splits the capture time into SRT evaluation and materialization, and prints the
  [capture] line once per 10 s instead of every 500 captures (it cost about 2 us per capture).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Co-Authored-By: Gjon Gojcevic <[email protected]>

core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp

@@ -2,6 +2,7 @@
 #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SHADERMEMORY_HPP
 
 #include "Recompiler.hpp"
+#include <algorithm>
 #include <array>
 #include <bitset>
 #include <cstddef>
@@ -56,6 +57,9 @@ public:
     using WaitedMsProvider = double (*)();
     static void SetWaitedMsProvider(WaitedMsProvider provider);
     explicit ShaderMemory(std::span<const ShaderRecompiler::MemoryRegion> initial, PendingWriteQuery pendingWrite = nullptr, PendingWriteObserver observe = nullptr, HookWaitCounter hookWaits = nullptr);
+    ~ShaderMemory();
+    ShaderMemory(const ShaderMemory&) = delete;
+    ShaderMemory& operator=(const ShaderMemory&) = delete;
     // Returns what the capture resolved (plan, snapshot, specialization) for
     // ShaderRecompiler::Recompile(request, capture), which then skips its own materialization. One
     // result per call: the draw path captures several stages on one ShaderMemory. With `handle`
@@ -76,12 +80,30 @@ private:
     static constexpr std::size_t PageBytes = 4096;
     static constexpr std::size_t PageWords = PageBytes / sizeof(std::uint32_t);
 
+    class WordMask {
+    public:
+        [[nodiscard]] bool Test(std::size_t index) const { return (bits[index / 64u] >> (index % 64u) & 1u) != 0u; }
+        void Set(std::size_t index) { bits[index / 64u] |= std::uint64_t{1} << (index % 64u); }
+        void SetAll() { bits.fill(~std::uint64_t{0}); }
+        void SetBlock(std::size_t index) { bits[index / 64u] |= ((std::uint64_t{1} << BlockWords) - 1u) << (index % 64u / BlockWords * BlockWords); }
+        void Reset() { bits.fill(0u); }
+        [[nodiscard]] bool None() const;
+        [[nodiscard]] bool All() const { return std::all_of(bits.begin(), bits.end(), [](std::uint64_t word) { return word == ~std::uint64_t{0}; }); }
+        template<typename TRun>
+        void ForEachRun(TRun&& run) const;
+
+    private:
+        std::array<std::uint64_t, PageWords / 64u> bits{};
+    };
+
+    static constexpr std::size_t BlockWords = 16;
     struct Page {
-        std::array<std::uint32_t, PageWords> words{};
-        std::bitset<PageWords> valid;
-        std::bitset<PageWords> read;
-        std::bitset<PageWords> recent;
+        std::array<std::uint32_t, PageWords> words;
+        WordMask valid;
+        WordMask read;
+        WordMask recent;
         bool wordwise = false;
+        bool lazy = false;
     };
 
     static bool read(void* context, std::uint64_t address, std::uint32_t* value);
@@ -94,7 +116,9 @@ private:
     // Regions given at construction (the registered shader's code and header), referenced as given:
     // the caller keeps them alive for as long as the capture is used.
     std::map<std::uint64_t, std::span<const std::byte>> initial;
-    std::map<std::uint64_t, Page> pages;
+    std::vector<std::pair<std::uint64_t, Page*>> pages;
+    std::uint64_t lastBase = ~std::uint64_t{0};
+    Page* lastPage = nullptr;
     PendingWriteQuery pendingWrite = nullptr;
     PendingWriteObserver observe = nullptr;
     HookWaitCounter hookWaits = nullptr;

core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp

@@ -1,16 +1,19 @@
 #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libc/include/HostThreadLocal.hpp"
 #include "Optimization/RequestMemoryView.hpp"
 #include "Optimization/ResourceMaterializer.hpp"
 #include "Optimization/ResourceProgram.hpp"
 #include <algorithm>
 #include <atomic>
+#include <bit>
 #include <chrono>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <limits>
+#include <memory>
 #include <stdexcept>
 #include <vector>
 
@@ -81,6 +84,14 @@ double WaitedMs() {
     return provider != nullptr ? provider() : 0.0;
 }
 
+template<typename TPage>
+std::vector<std::unique_ptr<TPage>>& pagePool() {
+    struct PagePoolStorage {};
+    return HostThreadLocal<std::vector<std::unique_ptr<TPage>>, PagePoolStorage>();
+}
+
+constexpr std::size_t PagePoolLimit = 64;
+
 }
 
 void ShaderMemory::SetWaitedMsProvider(WaitedMsProvider provider) {
@@ -94,10 +105,64 @@ ShaderMemory::ShaderMemory(std::span<const ShaderRecompiler::MemoryRegion> regio
     }
 }
 
+ShaderMemory::~ShaderMemory() {
+    auto& pool = pagePool<Page>();
+    for (const auto& [base, page] : pages) {
+        if (pool.size() < PagePoolLimit) pool.emplace_back(page);
+        else delete page;
+    }
+}
+
+bool ShaderMemory::WordMask::None() const {
+    return std::all_of(bits.begin(), bits.end(), [](std::uint64_t word) { return word == 0u; });
+}
+
+template<typename TRun>
+void ShaderMemory::WordMask::ForEachRun(TRun&& run) const {
+    constexpr std::size_t words = PageWords / 64u;
+    std::size_t index = 0;
+    while (index < PageWords) {
+        auto word = index / 64u;
+        auto set = bits[word] & (~std::uint64_t{0} << (index % 64u));
+        while (set == 0u) {
+            if (++word == words) return;
+            set = bits[word];
+        }
+        const auto first = word * 64u + static_cast<std::size_t>(std::countr_zero(set));
+        auto clear = ~bits[word] & (~std::uint64_t{0} << (first % 64u));
+        while (clear == 0u && ++word < words) clear = ~bits[word];
+        const auto end = word == words ? PageWords : word * 64u + static_cast<std::size_t>(std::countr_zero(clear));
+        run(first, end);
+        index = end;
+    }
+}
+
 ShaderMemory::Page& ShaderMemory::page(std::uint64_t base) {
-    const auto found = pages.find(base);
-    if (found != pages.end()) return found->second;
-    auto& page = pages[base];
+    if (base == lastBase) return *lastPage;
+    auto position = pages.begin();
+    while (position != pages.end() && position->first < base) ++position;
+    if (position != pages.end() && position->first == base) {
+        lastBase = base;
+        lastPage = position->second;
+        return *lastPage;
+    }
+    auto& pool = pagePool<Page>();
+    std::unique_ptr<Page> owned;
+    if (pool.empty()) {
+        owned = std::make_unique<Page>();
+    } else {
+        owned = std::move(pool.back());
+        pool.pop_back();
+    }
+    owned->valid.Reset();
+    owned->read.Reset();
+    owned->recent.Reset();
+    owned->wordwise = false;
+    owned->lazy = false;
+    auto& page = *owned;
+    pages.insert(position, {base, owned.release()});
+    lastBase = base;
+    lastPage = &page;
     ++CaptureTotals().pages;
     // A page is either mapped whole or read word by word where the guest mapped less than a page,
     // or where recorded GPU work writes into it (a whole-page read would wait for that work).
@@ -108,9 +173,9 @@ ShaderMemory::Page& ShaderMemory::page(std::uint64_t base) {
             return page;
         }
         const auto started = CaptureProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
-        GuestMemory::Read(base, std::as_writable_bytes(std::span(page.words)), sizeof(std::uint32_t));
+        GuestMemory::FlushGpuWrites(base, PageBytes);
         if (CaptureProfiled()) CaptureTotals().pageReadNanoseconds += NanosecondsSince(started);
-        page.valid.set();
+        page.lazy = true;
     }
     return page;
 }
@@ -136,7 +201,12 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val
     }
     auto& page = self.page(address & ~static_cast<std::uint64_t>(PageBytes - 1));
     const auto index = static_cast<std::size_t>((address % PageBytes) / sizeof(*value));
-    if (!page.valid.test(index)) {
+    if (!page.valid.Test(index) && page.lazy) {
+        const auto first = index / BlockWords * BlockWords;
+        std::memcpy(&page.words[first], reinterpret_cast<const void*>((address & ~static_cast<std::uint64_t>(PageBytes - 1)) + first * sizeof(std::uint32_t)), BlockWords * sizeof(std::uint32_t));
+        page.valid.SetBlock(index);
+    }
+    if (!page.valid.Test(index)) {
         std::uint32_t word = 0;
         auto policy = PendingWrite::Sync;
         if (page.wordwise) {
@@ -193,10 +263,10 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val
         }
         if (CaptureProfiled()) CaptureTotals().wordReadNanoseconds += NanosecondsSince(fetchStarted);
         page.words[index] = word;
-        page.valid.set(index);
+        page.valid.Set(index);
     }
-    page.read.set(index);
-    page.recent.set(index);
+    page.read.Set(index);
+    page.recent.Set(index);
     *value = page.words[index];
     return true;
 }
@@ -254,7 +324,7 @@ bool ShaderMemory::accessible(void* context, std::uint64_t address, std::uint64_
         const auto base = address & ~static_cast<std::uint64_t>(PageBytes - 1);
         const auto pageEnd = std::min<std::uint64_t>(base + PageBytes, end);
         const auto& page = self.page(base);
-        if (!page.wordwise && !page.valid.all() && !GuestMemory::Accessible(reinterpret_cast<const void*>(address), static_cast<std::size_t>(pageEnd - address))) return false;
+        if (!page.wordwise && !page.valid.All() && !GuestMemory::Accessible(reinterpret_cast<const void*>(address), static_cast<std::size_t>(pageEnd - address))) return false;
         address = pageEnd;
     }
     return true;
@@ -309,7 +379,11 @@ std::shared_ptr<const ShaderRecompiler::ResourceCapture> ShaderMemory::capture(c
         totals.hookWaitNanoseconds += static_cast<std::uint64_t>((WaitedMs() - waitedBefore) * 1e6);
         std::uint64_t initialBytes = 0;
         for (const auto& [address, bytes] : initial) initialBytes += bytes.size();
-        if (++totals.captures % 500 == 0) {
+        ++totals.captures;
+        static std::atomic<std::chrono::steady_clock::rep> nextReport{0};
+        const auto now = std::chrono::steady_clock::now().time_since_epoch().count();
+        auto due = nextReport.load(std::memory_order_relaxed);
+        if (now >= due && nextReport.compare_exchange_strong(due, now + std::chrono::duration_cast<std::chrono::steady_clock::duration>(std::chrono::seconds(10)).count(), std::memory_order_relaxed)) {
             const auto captures = static_cast<double>(totals.captures.load());
             const auto captureNs = totals.captureNanoseconds.load();
             const auto resolveNs = totals.resolveNanoseconds.load();
@@ -317,8 +391,10 @@ std::shared_ptr<const ShaderRecompiler::ResourceCapture> ShaderMemory::capture(c
             const auto wordNs = totals.wordReadNanoseconds.load();
             const auto specializationNs = ShaderRecompiler::ResourceMaterializer::SpecializationNanoseconds();
             const auto hookNs = totals.hookWaitNanoseconds.load();
+            const auto materializeNs = ShaderRecompiler::ResourceMaterializer::MaterializeNanoseconds();
+            const auto evaluateNs = ShaderRecompiler::ResourceMaterializer::EvaluateNanoseconds();
             const auto walkNs = captureNs > resolveNs + pageNs + wordNs + specializationNs ? captureNs - resolveNs - pageNs - wordNs - specializationNs : 0;
-            AgcDriver::ProfilePrint_nid_no_patch( "[capture] %llu captures: %llu word reads, %llu pages fetched (%llu read word by word over pending GPU writes: %llu words waited %.2f s, %llu read raw on evidence, %llu verified with %llu mismatches, %llu served from known values, %llu hook reads not observed: no GPU wait), capture (plan lookup + materialize) %.1f s, %llu KiB registered code this capture; sub-phases (s, us per capture): source resolve %.2f/%.1f, walk %.2f/%.1f, specialization (every materialize) %.2f/%.1f, page reads %.2f/%.1f, word reads %.2f/%.1f, hook GPU waits inside %.2f/%.1f; source handle memo %llu hits, %llu resolves\n", static_cast<unsigned long long>(totals.captures.load()), static_cast<unsigned long long>(totals.reads.load()), static_cast<unsigned long long>(totals.pages.load()), static_cast<unsigned long long>(totals.pagesWordwise.load()), static_cast<unsigned long long>(totals.wordWaits.load()), totals.wordWaitNanoseconds.load() / 1e9, static_cast<unsigned long long>(totals.wordsRaw.load()), static_cast<unsigned long long>(totals.wordsVerified.load()), static_cast<unsigned long long>(totals.wordMismatches.load()), static_cast<unsigned long long>(totals.wordsKnown.load()), static_cast<unsigned long long>(totals.observationsSkipped.load()), captureNs / 1e9, static_cast<unsigned long long>(initialBytes / 1024), resolveNs / 1e9, resolveNs / captures / 1e3, walkNs / 1e9, walkNs / captures / 1e3, specializationNs / 1e9, specializationNs / captures / 1e3, pageNs / 1e9, pageNs / captures / 1e3, wordNs / 1e9, wordNs / captures / 1e3, hookNs / 1e9, hookNs / captures / 1e3, static_cast<unsigned long long>(totals.handleHits.load()), static_cast<unsigned long long>(totals.handleMisses.load()));
+            AgcDriver::ProfilePrint_nid_no_patch( "[capture] %llu captures: %llu word reads, %llu pages fetched (%llu read word by word over pending GPU writes: %llu words waited %.2f s, %llu read raw on evidence, %llu verified with %llu mismatches, %llu served from known values, %llu hook reads not observed: no GPU wait), capture (plan lookup + materialize) %.1f s, %llu KiB registered code this capture; sub-phases (s, us per capture): source resolve %.2f/%.1f, walk %.2f/%.1f, specialization (every materialize) %.2f/%.1f, page reads %.2f/%.1f, word reads %.2f/%.1f, hook GPU waits inside %.2f/%.1f, materialize %.2f/%.1f (SRT evaluate %.2f/%.1f); source handle memo %llu hits, %llu resolves\n", static_cast<unsigned long long>(totals.captures.load()), static_cast<unsigned long long>(totals.reads.load()), static_cast<unsigned long long>(totals.pages.load()), static_cast<unsigned long long>(totals.pagesWordwise.load()), static_cast<unsigned long long>(totals.wordWaits.load()), totals.wordWaitNanoseconds.load() / 1e9, static_cast<unsigned long long>(totals.wordsRaw.load()), static_cast<unsigned long long>(totals.wordsVerified.load()), static_cast<unsigned long long>(totals.wordMismatches.load()), static_cast<unsigned long long>(totals.wordsKnown.load()), static_cast<unsigned long long>(totals.observationsSkipped.load()), captureNs / 1e9, static_cast<unsigned long long>(initialBytes / 1024), resolveNs / 1e9, resolveNs / captures / 1e3, walkNs / 1e9, walkNs / captures / 1e3, specializationNs / 1e9, specializationNs / captures / 1e3, pageNs / 1e9, pageNs / captures / 1e3, wordNs / 1e9, wordNs / captures / 1e3, hookNs / 1e9, hookNs / captures / 1e3, materializeNs / 1e9, materializeNs / captures / 1e3, evaluateNs / 1e9, evaluateNs / captures / 1e3, static_cast<unsigned long long>(totals.handleHits.load()), static_cast<unsigned long long>(totals.handleMisses.load()));
         }
     }
     return capture;
@@ -329,20 +405,13 @@ std::vector<ShaderRecompiler::MemoryRegion> ShaderMemory::Regions() const {
     result.reserve(initial.size() + pages.size());
     auto next = initial.begin();
     // Both maps are ordered by address and never overlap, so a merge keeps the result sorted.
-    for (const auto& [base, page] : pages) {
+    for (const auto& [base, pagePointer] : pages) {
+        const auto& page = *pagePointer;
         while (next != initial.end() && next->first < base) {
             result.push_back({next->first, next->second});
             ++next;
         }
-        for (std::size_t index = 0; index < PageWords;) {
-            if (!page.read.test(index)) {
-                ++index;
-                continue;
-            }
-            const auto first = index;
-            while (index < PageWords && page.read.test(index)) ++index;
-            result.push_back({base + first * sizeof(std::uint32_t), std::as_bytes(std::span(page.words).subspan(first, index - first))});
-        }
+        page.read.ForEachRun([&](std::size_t first, std::size_t end) { result.push_back({base + first * sizeof(std::uint32_t), std::as_bytes(std::span(page.words).subspan(first, end - first))}); });
     }
     for (; next != initial.end(); ++next) result.push_back({next->first, next->second});
     return result;
@@ -350,18 +419,11 @@ std::vector<ShaderRecompiler::MemoryRegion> ShaderMemory::Regions() const {
 
 std::vector<ShaderRecompiler::MemoryRegion> ShaderMemory::TakeRecentRegions() {
     std::vector<ShaderRecompiler::MemoryRegion> result;
-    for (auto& [base, page] : pages) {
-        if (page.recent.none()) continue;
-        for (std::size_t index = 0; index < PageWords;) {
-            if (!page.recent.test(index)) {
-                ++index;
-                continue;
-            }
-            const auto first = index;
-            while (index < PageWords && page.recent.test(index)) ++index;
-            result.push_back({base + first * sizeof(std::uint32_t), std::as_bytes(std::span(page.words).subspan(first, index - first))});
-        }
-        page.recent.reset();
+    for (auto& [base, pagePointer] : pages) {
+        auto& page = *pagePointer;
+        if (page.recent.None()) continue;
+        page.recent.ForEachRun([&](std::size_t first, std::size_t end) { result.push_back({base + first * sizeof(std::uint32_t), std::as_bytes(std::span(page.words).subspan(first, end - first))}); });
+        page.recent.Reset();
     }
     return result;
 }

core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp

@@ -521,7 +521,7 @@ void verifySignedMinimum() {
         SrtRuntime runtime{data};
         Detail::Evaluator evaluator(plan, runtime);
         std::uint32_t result = 0;
-        require(evaluator.Evaluate(&minimum, result) && result == expected, "SRT signed minimum used unsigned ordering or failed evaluation");
+        require(evaluator.Evaluate(Detail::IrNode(&minimum), result) && result == expected, "SRT signed minimum used unsigned ordering or failed evaluation");
     }
 }
 

core/shader/recompiler/CMakeLists.txt

@@ -246,6 +246,8 @@ function(add_shader_recompiler target runtimeTarget)
             MaterializeShader
             RegistrationResult
             ResolveSource
+            ResourceMaterializer::EvaluateNanoseconds
+            ResourceMaterializer::MaterializeNanoseconds
             ResourceMaterializer::SpecializationNanoseconds
             RuntimeAbi::RequireVersion
             RuntimeAbi::HeapCapacity

core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp

@@ -130,6 +130,26 @@ private:
     std::size_t capacity = 0;
 };
 
+struct CompactPlanValue {
+    std::uint64_t immediate = 0;
+    std::uint64_t flags = 0;
+    std::uint32_t firstArgument = 0;
+    std::uint32_t registerIndex = 0;
+    IrType type = IrType::Void;
+    IrOpcode opcode = IrOpcode::Void;
+    std::uint8_t argumentCount = 0;
+    bool hasImmediate = false;
+};
+
+struct CompactResourcePlan {
+    static constexpr std::uint32_t NoValue = 0xffffffffu;
+    std::vector<CompactPlanValue> values;
+    std::vector<std::uint32_t> arguments;
+    std::vector<std::uint32_t> sourceDwords;
+    std::vector<std::uint32_t> srtReads;
+    std::vector<std::uint32_t> conditions;
+};
+
 struct IrResourcePlan {
     IrShaderStage stage = IrShaderStage::Unknown;
     std::uint64_t shaderHash = 0;
@@ -149,6 +169,8 @@ struct IrResourcePlan {
     // One byte per srtReads slot, 1 when the CPU walk never consumes the slot's value (see
     // Detail::ComputePureFlatSlots): a driver may reuse a capture whose words differ only there.
     std::vector<std::uint8_t> pureFlatSlots;
+    bool denseValueIds = false;
+    CompactResourcePlan compact;
     bool requiresSpecializationMemory = false;
     bool srtPlanComplete = false;
     bool resourceTrackingComplete = false;

core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp

@@ -58,8 +58,11 @@ public:
     static const char* UnnormalizedSampleMismatch(const ImageResource& image);
     static ResolvedImageTables ResolveImageTables(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint64_t memoKey = 0);
     [[nodiscard]] IrResourcePlan ExtractPlan(const IrProgram& program) const;
+    static void CompactPlan(IrResourcePlan& plan);
     void Materialize(const IrResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot) const;
     static std::uint64_t SpecializationNanoseconds();
+    static std::uint64_t EvaluateNanoseconds();
+    static std::uint64_t MaterializeNanoseconds();
 };
 
 }

core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp

@@ -5,6 +5,8 @@
 #include "Optimization/SrtWalker.hpp"
 
 #include <cstdint>
+#include <cstring>
+#include <stdexcept>
 #include <span>
 #include <vector>
 
@@ -71,13 +73,107 @@ struct InaccessibleRead {
     std::uint64_t address = 0;
 };
 
-class Evaluator {
+class DenseValues {
 public:
-    Evaluator(const IrResourcePlan& program, const SrtRuntime& runtime, std::span<const std::uint8_t> cleanFlatSlots = {}, Evaluator* cleanEvaluator = nullptr, IrValue* activeMask = nullptr) : _program(program), _runtime(runtime), _cleanFlatSlots(cleanFlatSlots), _cleanEvaluator(cleanEvaluator), _activeMask(activeMask != nullptr ? activeMask->Resolve() : nullptr) {}
+    struct Slot {
+        std::uint64_t stamp = 0;
+        std::uint64_t value = 0;
+    };
+    explicit DenseValues(std::size_t values);
+    ~DenseValues();
+    DenseValues(const DenseValues&) = delete;
+    DenseValues& operator=(const DenseValues&) = delete;
+    Slot& At(std::uint32_t id) { return _slots[id]; }
+    [[nodiscard]] std::size_t Size() const { return _size; }
+    [[nodiscard]] std::uint64_t Done() const { return _stamp; }
+    [[nodiscard]] std::uint64_t Visiting() const { return _stamp + 1u; }
+
+private:
+    std::vector<Slot>* _array = nullptr;
+    Slot* _slots = nullptr;
+    std::size_t _size = 0;
+    std::uint64_t _stamp = 0;
+};
+
+class IrNode {
+public:
+    IrNode() = default;
+    explicit IrNode(IrValue* value) : _value(value->Resolve()) {}
+    static IrNode From(const IrResourcePlan&, IrValue* value) { return IrNode(value); }
+    static IrNode SrtRead(const IrResourcePlan& program, std::uint32_t slot) { return IrNode(program.srtReads[slot].value); }
+    static IrNode SourceDword(const IrResourcePlan& program, std::uint32_t source, std::uint32_t dword) { return IrNode(program.descriptorSources[source].dwords[dword]); }
+    static bool Condition(const IrResourcePlan& program, std::uint32_t block, IrNode& condition) {
+        if (program.controlFlow[block].condition == nullptr) return false;
+        condition = IrNode(program.controlFlow[block].condition);
+        return true;
+    }
+    [[nodiscard]] IrOpcode Opcode() const { return _value->Opcode(); }
+    [[nodiscard]] IrType Type() const { return _value->Type(); }
+    [[nodiscard]] bool HasImmediate() const { return _value->HasImmediate(); }
+    [[nodiscard]] std::uint64_t ImmediateBits() const { return _value->ImmediateU64(); }
+    [[nodiscard]] std::size_t ArgumentCount() const { return _value->ArgumentCount(); }
+    [[nodiscard]] IrNode Argument(std::size_t index) const { return IrNode(_value->Argument(index)); }
+    template<typename TValue>
+    [[nodiscard]] TValue Flags() const { return _value->Flags<TValue>(); }
+    [[nodiscard]] std::uint32_t RegisterIndex() const { return _value->Register().index; }
+    [[nodiscard]] std::uint32_t Id() const { return _value->Id(); }
+    [[nodiscard]] IrValue* Value() const { return _value; }
+    bool operator==(const IrNode& other) const { return _value == other._value; }
+
+private:
+    IrValue* _value = nullptr;
+};
+
+class CompactNode {
+public:
+    CompactNode() = default;
+    CompactNode(const IrResourcePlan& program, std::uint32_t id) : _program(&program), _id(id) {}
+    static CompactNode From(const IrResourcePlan& program, IrValue* value) { return {program, value->Resolve()->Id()}; }
+    static CompactNode SrtRead(const IrResourcePlan& program, std::uint32_t slot) { return {program, program.compact.srtReads[slot]}; }
+    static CompactNode SourceDword(const IrResourcePlan& program, std::uint32_t source, std::uint32_t dword) { return {program, program.compact.sourceDwords[source * 8u + dword]}; }
+    static bool Condition(const IrResourcePlan& program, std::uint32_t block, CompactNode& condition) {
+        if (program.compact.conditions[block] == CompactResourcePlan::NoValue) return false;
+        condition = {program, program.compact.conditions[block]};
+        return true;
+    }
+    [[nodiscard]] IrOpcode Opcode() const { return entry().opcode; }
+    [[nodiscard]] IrType Type() const { return entry().type; }
+    [[nodiscard]] bool HasImmediate() const { return entry().hasImmediate; }
+    [[nodiscard]] std::uint64_t ImmediateBits() const { return entry().immediate; }
+    [[nodiscard]] std::size_t ArgumentCount() const { return entry().argumentCount; }
+    [[nodiscard]] CompactNode Argument(std::size_t index) const {
+        const auto& value = entry();
+        if (index >= value.argumentCount) throw std::out_of_range("IrValue::Argument index is out of range");
+        return {*_program, _program->compact.arguments[value.firstArgument + index]};
+    }
+    template<typename TValue>
+    [[nodiscard]] TValue Flags() const {
+        TValue result{};
+        std::memcpy(&result, &entry().flags, sizeof(result));
+        return result;
+    }
+    [[nodiscard]] std::uint32_t RegisterIndex() const { return entry().registerIndex; }
+    [[nodiscard]] std::uint32_t Id() const { return _id; }
+    [[nodiscard]] IrValue* Value() const { return const_cast<IrValue*>(&_program->valueStorage[_id]); }
+    bool operator==(const CompactNode& other) const { return _id == other._id; }
+
+private:
+    [[nodiscard]] const CompactPlanValue& entry() const { return _program->compact.values[_id]; }
+
+    const IrResourcePlan* _program = nullptr;
+    std::uint32_t _id = 0;
+};
+
+template<typename TNode>
+class BasicEvaluator {
+public:
+    BasicEvaluator(const IrResourcePlan& program, const SrtRuntime& runtime, std::span<const std::uint8_t> cleanFlatSlots = {}, BasicEvaluator* cleanEvaluator = nullptr, const TNode* activeMask = nullptr) : _program(program), _runtime(runtime), _cleanFlatSlots(cleanFlatSlots), _cleanEvaluator(cleanEvaluator), _activeMask(activeMask != nullptr ? *activeMask : TNode{}), _masked(activeMask != nullptr), _dense(program.denseValueIds ? program.valueStorage.Size() : 0u) {}
+    BasicEvaluator(const BasicEvaluator&) = delete;
+    BasicEvaluator& operator=(const BasicEvaluator&) = delete;
 
     static bool MayEvaluate(const IrResourcePlan& program, IrValue* value);
-    bool Evaluate(IrValue* value, std::uint32_t& result);
-    bool EvaluateWide(IrValue* raw, std::uint64_t& result);
+    bool Evaluate(TNode value, std::uint32_t& result);
+    bool EvaluateWide(TNode value, std::uint64_t& result);
     void ReportInaccessibleReads(InaccessibleRead* sink) { _inaccessible = sink; }
     void ReportNullRootReads(std::uint32_t* sink) { _nullRoots = sink; }
 
@@ -85,23 +181,28 @@ private:
     static float Float32(std::uint64_t bits);
     static std::uint64_t Float32Bits(float value);
 
-    bool Arg(IrValue& inst, std::size_t index, std::uint64_t& result);
-    bool EvaluatePhi(IrValue& inst, std::uint64_t& result);
-    bool EvaluateExtract(IrValue& inst, std::uint64_t& result);
-    bool EvaluateRawRead(IrValue& inst, std::uint64_t& result);
-    bool EvaluateInst(IrValue& inst, std::uint64_t& result);
+    bool Arg(const TNode& inst, std::size_t index, std::uint64_t& result);
+    bool EvaluatePhi(const TNode& inst, std::uint64_t& result);
+    bool EvaluateExtract(const TNode& inst, std::uint64_t& result);
+    bool EvaluateRawRead(const TNode& inst, std::uint64_t& result);
+    bool EvaluateInst(const TNode& inst, std::uint64_t& result);
 
     const IrResourcePlan& _program;
     const SrtRuntime& _runtime;
     std::span<const std::uint8_t> _cleanFlatSlots;
-    Evaluator* _cleanEvaluator = nullptr;
-    IrValue* _activeMask = nullptr;
+    BasicEvaluator* _cleanEvaluator = nullptr;
+    TNode _activeMask;
+    bool _masked = false;
     InaccessibleRead* _inaccessible = nullptr;
     std::uint32_t* _nullRoots = nullptr;
     EvaluatedValues _cache;
     std::vector<IrValue*> _visiting;
+    DenseValues _dense;
 };
 
+using Evaluator = BasicEvaluator<IrNode>;
+using CompactEvaluator = BasicEvaluator<CompactNode>;
+
 }
 
 #endif

core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtInstructionPredicates.hpp

@@ -6,6 +6,7 @@
 namespace ShaderRecompiler::Detail {
 
 bool IsRawRead(const IrResourcePlan& program, const IrValue& inst);
+bool IsRawRead(const IrResourcePlan& program, IrOpcode op, std::uint32_t memoryIndex);
 bool LoadedFromMemory(const IrValue& handle);
 bool IsDescriptorHandle(IrOpcode opcode);
 bool IsRuntimeSelect(IrOpcode opcode);

core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp

@@ -35,6 +35,8 @@ namespace {
 using ImageTableAbi::PoisonReason;
 
 std::atomic<std::uint64_t> specializationNanoseconds{0};
+std::atomic<std::uint64_t> evaluateNanoseconds{0};
+std::atomic<std::uint64_t> materializeNanoseconds{0};
 
 bool MaterializeProfiled() {
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
@@ -787,7 +789,9 @@ void materializeSnapshot(const IrResourcePlan& plan, const SrtRuntime& runtime,
     snapshot.userData.assign(runtime.userData.begin(), runtime.userData.begin() + plan.userDataCount);
 
     std::vector<DescriptorValue> values;
+    const auto evaluateStarted = MaterializeProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
     walker.EvaluateRuntimeSources(plan, plan.materializationSources, runtime, values, snapshot.flattenedSrt, plan.cleanFlatSlots, activeSources, &snapshot.srtPoison, &snapshot.nullRootReads);
+    if (MaterializeProfiled()) evaluateNanoseconds.fetch_add(static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - evaluateStarted).count()), std::memory_order_relaxed);
 
     std::size_t cursor = 0;
     if (values.size() < plan.info.buffers.size()) {
@@ -1525,7 +1529,7 @@ void ownPlanValues(IrResourcePlan& plan) {
     }
     plan.valueStorage.Reserve(order.size());
     for (const auto* value : order) {
-        auto* clone = plan.valueStorage.Create(value->Opcode(), value->Type(), value->Id());
+        auto* clone = plan.valueStorage.Create(value->Opcode(), value->Type(), static_cast<std::uint32_t>(plan.valueStorage.Size()));
         clone->SetFlags(value->Flags<std::uint64_t>());
         if (value->HasImmediate()) clone->SetImmediateU64(value->ImmediateU64());
         clone->SetRegister(value->Register());
@@ -1555,6 +1559,59 @@ void ownPlanValues(IrResourcePlan& plan) {
     for (auto* root : roots) {
         if (*root != nullptr) *root = clones.at(*root);
     }
+    plan.denseValueIds = true;
+}
+
+void buildCompactPlan(IrResourcePlan& plan) {
+    if (!plan.denseValueIds || plan.valueStorage.Size() >= CompactResourcePlan::NoValue) return;
+    CompactResourcePlan compact;
+    const auto idOf = [&](const IrValue* value, std::uint32_t& id) {
+        if (value == nullptr) return false;
+        const auto* resolved = value->Resolve();
+        id = resolved->Id();
+        return id < plan.valueStorage.Size() && &plan.valueStorage[id] == resolved;
+    };
+    try {
+        compact.values.reserve(plan.valueStorage.Size());
+        for (const auto& stored : plan.valueStorage) {
+            const auto* value = &stored;
+            if (value->ArgumentCount() > 0xffu) return;
+            CompactPlanValue entry;
+            entry.immediate = value->ImmediateU64();
+            entry.flags = value->Flags<std::uint64_t>();
+            entry.firstArgument = static_cast<std::uint32_t>(compact.arguments.size());
+            entry.registerIndex = value->Register().index;
+            entry.type = value->Type();
+            entry.opcode = value->Opcode();
+            entry.argumentCount = static_cast<std::uint8_t>(value->ArgumentCount());
+            entry.hasImmediate = value->HasImmediate();
+            for (std::size_t index = 0; index < value->ArgumentCount(); index++) {
+                std::uint32_t id = 0;
+                if (!idOf(value->Argument(index), id)) return;
+                compact.arguments.push_back(id);
+            }
+            compact.values.push_back(entry);
+        }
+        compact.sourceDwords.assign(plan.descriptorSources.size() * 8u, CompactResourcePlan::NoValue);
+        for (std::size_t source = 0; source < plan.descriptorSources.size(); source++) {
+            const auto& descriptor = plan.descriptorSources[source];
+            if (descriptor.dwordCount > 8u) return;
+            for (std::uint32_t dword = 0; dword < descriptor.dwordCount; dword++) {
+                if (!idOf(descriptor.dwords[dword], compact.sourceDwords[source * 8u + dword])) return;
+            }
+        }
+        compact.srtReads.resize(plan.srtReads.size());
+        for (std::size_t slot = 0; slot < plan.srtReads.size(); slot++) {
+            if (!idOf(plan.srtReads[slot].value, compact.srtReads[slot])) return;
+        }
+        compact.conditions.assign(plan.controlFlow.size(), CompactResourcePlan::NoValue);
+        for (std::size_t block = 0; block < plan.controlFlow.size(); block++) {
+            if (plan.controlFlow[block].condition != nullptr && !idOf(plan.controlFlow[block].condition, compact.conditions[block])) return;
+        }
+    } catch (const std::exception&) {
+        return;
+    }
+    plan.compact = std::move(compact);
 }
 
 }
@@ -1611,9 +1668,15 @@ IrResourcePlan ResourceMaterializer::ExtractPlan(const IrProgram& program) const
     }
     plan.pureFlatSlots = Detail::ComputePureFlatSlots(plan);
     plan.guardedSrtSlots = Detail::ComputeGuardedFlatSlots(plan);
+    buildCompactPlan(plan);
     return plan;
 }
 
+void ResourceMaterializer::CompactPlan(IrResourcePlan& plan) {
+    plan.compact = {};
+    buildCompactPlan(plan);
+}
+
 void ResourceMaterializer::Materialize(const IrResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot) const {
     const IrResourcePlan& plan = program;
     if (!plan.resourceTrackingComplete) {
@@ -1622,6 +1685,7 @@ void ResourceMaterializer::Materialize(const IrResourcePlan& program, const SrtR
     if (plan.requiresSpecializationMemory && runtime.readMemory == nullptr) {
         throw std::runtime_error("ResourceMaterializer::Materialize requires runtime memory access for image tables");
     }
+    const auto materializeStarted = MaterializeProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
     SrtWalker walker;
     ResourceSnapshot nextSnapshot;
     std::vector<std::uint8_t> activeSources;
@@ -1639,10 +1703,19 @@ void ResourceMaterializer::Materialize(const IrResourcePlan& program, const SrtR
     if (MaterializeProfiled()) specializationNanoseconds.fetch_add(static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - started).count()), std::memory_order_relaxed);
     snapshot = std::move(nextSnapshot);
     reportTables();
+    if (MaterializeProfiled()) materializeNanoseconds.fetch_add(static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - materializeStarted).count()), std::memory_order_relaxed);
 }
 
 std::uint64_t ResourceMaterializer::SpecializationNanoseconds() {
     return specializationNanoseconds.load(std::memory_order_relaxed);
 }
 
+std::uint64_t ResourceMaterializer::EvaluateNanoseconds() {
+    return evaluateNanoseconds.load(std::memory_order_relaxed);
+}
+
+std::uint64_t ResourceMaterializer::MaterializeNanoseconds() {
+    return materializeNanoseconds.load(std::memory_order_relaxed);
+}
+
 }

core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp

@@ -5,6 +5,7 @@
 #include <cstdio>
 #include <cstdlib>
 #include <algorithm>
+#include <stdexcept>
 #include <string>
 #include <utility>
 
@@ -43,9 +44,8 @@ const DescriptorSource* Source(const IrResourcePlan& program, std::uint32_t sour
     return &program.descriptorSources[source];
 }
 
-}
-
-bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span<const std::uint32_t> sources, const SrtRuntime& runtime, std::vector<DescriptorValue>& results, std::vector<std::uint32_t>& flat, bool evaluateFlat, std::span<const std::uint8_t> cleanFlatSlots, std::vector<std::uint8_t>& activeSources, std::vector<SrtReadPoison>* poison, std::uint32_t* nullRootReads) {
+template<typename TNode>
+bool evaluateRuntimeSources(const IrResourcePlan& program, std::span<const std::uint32_t> sources, const SrtRuntime& runtime, std::vector<DescriptorValue>& results, std::vector<std::uint32_t>& flat, bool evaluateFlat, std::span<const std::uint8_t> cleanFlatSlots, std::vector<std::uint8_t>& activeSources, std::vector<SrtReadPoison>* poison, std::uint32_t* nullRootReads) {
     failureReason().clear();
     static const bool debug = std::getenv("APS5_SRT_DEBUG") != nullptr;
     if (debug) {
@@ -59,8 +59,8 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span<const s
     }
     SrtRuntime cleanRuntime = runtime;
     cleanRuntime.readMemory = runtime.readSpecializationMemory;
-    Evaluator cleanEvaluator(program, cleanRuntime);
-    Evaluator evaluator(program, runtime, cleanFlatSlots, &cleanEvaluator);
+    BasicEvaluator<TNode> cleanEvaluator(program, cleanRuntime);
+    BasicEvaluator<TNode> evaluator(program, runtime, cleanFlatSlots, &cleanEvaluator);
     std::vector<std::uint8_t> active;
     if (evaluateFlat) {
         active.assign(program.descriptorSources.size() + program.info.images.size(), 1u);
@@ -95,8 +95,9 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span<const s
                 active[program.descriptorSources.size() + image] = 1u;
             }
             std::uint32_t condition = 0;
+            TNode conditionValue{};
             const auto nullRootsBefore = conditionNullRoots;
-            const bool cleanEvaluable = block.condition != nullptr && runtime.readSpecializationMemory != nullptr && cleanEvaluator.Evaluate(block.condition, condition) && conditionNullRoots == nullRootsBefore;
+            const bool cleanEvaluable = TNode::Condition(program, index, conditionValue) && runtime.readSpecializationMemory != nullptr && cleanEvaluator.Evaluate(conditionValue, condition) && conditionNullRoots == nullRootsBefore;
             if (cleanEvaluable) {
                 pending.push_back(block.successors[condition != 0u ? 0u : 1u]);
             } else {
@@ -129,7 +130,7 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span<const s
         if (!evaluateFlat || active[sourceIndex]) {
             for (std::uint32_t index = 0; index < source->dwordCount; index++) {
                 inaccessible = {};
-                if (!evaluator.Evaluate(source->dwords[index], value.dwords[index])) {
+                if (!evaluator.Evaluate(TNode::SourceDword(program, sourceIndex, index), value.dwords[index])) {
                     if (inaccessible.read != nullptr) {
                         if (debug) std::fprintf(stderr, "[srt] descriptor source %u reads inaccessible 0x%llx at pc 0x%x: zero words\n", sourceIndex, static_cast<unsigned long long>(inaccessible.address), inaccessible.read->Flags<MemoryFlags>().pc);
                         value.dwords.fill(0u);
@@ -152,7 +153,8 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span<const s
     std::vector<SrtReadPoison> poisoned;
     if (evaluateFlat) {
         flattened.resize(program.srtReads.size());
-        for (const auto& read : program.srtReads) {
+        for (std::uint32_t slot = 0; slot < program.srtReads.size(); ++slot) {
+            const auto& read = program.srtReads[slot];
             const bool clean = read.flatOffset < cleanFlatSlots.size() && cleanFlatSlots[read.flatOffset] != 0u;
             auto& selected = clean ? cleanEvaluator : evaluator;
             // A pure slot's raw read is reachable from no root, so it was not evaluated (nor
@@ -161,11 +163,11 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span<const s
             auto* trace = runtime.readTrace;
             const bool pure = trace != nullptr && read.flatOffset < program.pureFlatSlots.size() && program.pureFlatSlots[read.flatOffset] != 0u;
             if (pure) {
-                trace->leaf = read.value->Resolve();
+                trace->leaf = TNode::SrtRead(program, slot).Value();
                 trace->leafSlot = read.flatOffset;
             }
             inaccessible = {};
-            const bool evaluated = read.flatOffset < flattened.size() && selected.Evaluate(read.value, flattened[read.flatOffset]);
+            const bool evaluated = read.flatOffset < flattened.size() && selected.Evaluate(TNode::SrtRead(program, slot), flattened[read.flatOffset]);
             if (pure) trace->leaf = nullptr;
             if (!evaluated && inaccessible.read != nullptr) {
                 if (debug) std::fprintf(stderr, "[srt] flat offset %u reads inaccessible 0x%llx at pc 0x%x: poisoned\n", read.flatOffset, static_cast<unsigned long long>(inaccessible.address), inaccessible.read->Flags<MemoryFlags>().pc);
@@ -201,6 +203,53 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span<const s
     return true;
 }
 
+void prefetchCompactPlan(const CompactResourcePlan& compact) {
+    const auto prefetch = [](const void* data, std::size_t bytes) {
+        const auto* bytesData = static_cast<const char*>(data);
+        for (std::size_t offset = 0; offset < std::min<std::size_t>(bytes, 16384u); offset += 64) __builtin_prefetch(bytesData + offset);
+    };
+    prefetch(compact.values.data(), compact.values.size() * sizeof(CompactPlanValue));
+    prefetch(compact.arguments.data(), compact.arguments.size() * sizeof(std::uint32_t));
+    prefetch(compact.sourceDwords.data(), compact.sourceDwords.size() * sizeof(std::uint32_t));
+    prefetch(compact.srtReads.data(), compact.srtReads.size() * sizeof(std::uint32_t));
+}
+
+std::string describeDifference(const std::vector<DescriptorValue>& compactResults, const std::vector<DescriptorValue>& irResults, const std::vector<std::uint32_t>& compactFlat, const std::vector<std::uint32_t>& irFlat) {
+    for (std::size_t index = 0; index < std::min(compactResults.size(), irResults.size()); ++index) {
+        for (std::size_t dword = 0; dword < compactResults[index].dwords.size(); ++dword) {
+            if (compactResults[index].dwords[dword] != irResults[index].dwords[dword]) return "source " + std::to_string(index) + " dword " + std::to_string(dword) + ": compact " + std::to_string(compactResults[index].dwords[dword]) + ", IR " + std::to_string(irResults[index].dwords[dword]);
+        }
+    }
+    for (std::size_t index = 0; index < std::min(compactFlat.size(), irFlat.size()); ++index) {
+        if (compactFlat[index] != irFlat[index]) return "flat offset " + std::to_string(index) + ": compact " + std::to_string(compactFlat[index]) + ", IR " + std::to_string(irFlat[index]);
+    }
+    return "sizes " + std::to_string(compactResults.size()) + "/" + std::to_string(irResults.size()) + " sources, " + std::to_string(compactFlat.size()) + "/" + std::to_string(irFlat.size()) + " flat words";
+}
+
+}
+
+bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span<const std::uint32_t> sources, const SrtRuntime& runtime, std::vector<DescriptorValue>& results, std::vector<std::uint32_t>& flat, bool evaluateFlat, std::span<const std::uint8_t> cleanFlatSlots, std::vector<std::uint8_t>& activeSources, std::vector<SrtReadPoison>* poison, std::uint32_t* nullRootReads) {
+    if (program.compact.values.empty()) return evaluateRuntimeSources<IrNode>(program, sources, runtime, results, flat, evaluateFlat, cleanFlatSlots, activeSources, poison, nullRootReads);
+    prefetchCompactPlan(program.compact);
+    const bool evaluated = evaluateRuntimeSources<CompactNode>(program, sources, runtime, results, flat, evaluateFlat, cleanFlatSlots, activeSources, poison, nullRootReads);
+    static const bool verify = std::getenv("APS5_VERIFY_SRT_COMPACT") != nullptr;
+    if (verify) {
+        const auto compactReason = RuntimeSourceFailureReason();
+        SrtRuntime untraced = runtime;
+        untraced.readTrace = nullptr;
+        std::vector<DescriptorValue> irResults;
+        std::vector<std::uint32_t> irFlat = flat;
+        std::vector<std::uint8_t> irActive;
+        std::vector<SrtReadPoison> irPoison;
+        std::uint32_t irNullRoots = 0;
+        const bool irEvaluated = evaluateRuntimeSources<IrNode>(program, sources, untraced, irResults, irFlat, evaluateFlat, cleanFlatSlots, irActive, poison != nullptr ? &irPoison : nullptr, nullRootReads != nullptr ? &irNullRoots : nullptr);
+        if (evaluated != irEvaluated) throw std::runtime_error("SRT compact walk " + std::string(evaluated ? "succeeded" : "failed (" + compactReason + ")") + " where the IR walk " + (irEvaluated ? "succeeded" : "failed (" + RuntimeSourceFailureReason() + ")"));
+        if (evaluated && (results != irResults || (evaluateFlat && flat != irFlat) || activeSources != irActive)) throw std::runtime_error("SRT compact walk differs from the IR walk at " + describeDifference(results, irResults, flat, irFlat));
+        if (!evaluated) failureReason() = compactReason;
+    }
+    return evaluated;
+}
+
 const std::string& RuntimeSourceFailureReason() {
     return failureReason();
 }

core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp

@@ -2,6 +2,7 @@
 #include "Optimization/SrtWalker/SrtAddressArithmetic.hpp"
 #include "Optimization/SrtWalker/SrtInstructionPredicates.hpp"
 #include "IntermediateRepresentation/IrBuilder.hpp"
+#include "prx/libc/include/HostThreadLocal.hpp"
 
 #include <cstdio>
 #include <cstdlib>
@@ -10,6 +11,7 @@
 #include <bit>
 #include <cmath>
 #include <cstring>
+#include <memory>
 #include <optional>
 #include <unordered_map>
 #include <utility>
@@ -17,7 +19,44 @@
 
 namespace ShaderRecompiler::Detail {
 
-bool Evaluator::Evaluate(IrValue* value, std::uint32_t& result) {
+namespace {
+
+struct DenseValuePool {
+    std::vector<std::unique_ptr<std::vector<DenseValues::Slot>>> arrays;
+    std::vector<std::vector<DenseValues::Slot>*> free;
+    std::uint64_t stamp = 0;
+};
+
+DenseValuePool& denseValuePool() {
+    struct DenseValuePoolStorage {};
+    return HostThreadLocal<DenseValuePool, DenseValuePoolStorage>();
+}
+
+}
+
+DenseValues::DenseValues(std::size_t values) : _size(values) {
+    if (values == 0u) return;
+    auto& pool = denseValuePool();
+    if (pool.free.empty()) {
+        pool.arrays.push_back(std::make_unique<std::vector<Slot>>());
+        _array = pool.arrays.back().get();
+    } else {
+        _array = pool.free.back();
+        pool.free.pop_back();
+    }
+    if (_array->size() < values) _array->resize(values);
+    _slots = _array->data();
+    pool.stamp += 2u;
+    _stamp = pool.stamp;
+}
+
+DenseValues::~DenseValues() {
+    if (_array == nullptr) return;
+    denseValuePool().free.push_back(_array);
+}
+
+template<typename TNode>
+bool BasicEvaluator<TNode>::Evaluate(TNode value, std::uint32_t& result) {
     std::uint64_t wide = 0;
     if (!EvaluateWide(value, wide)) {
         return false;
@@ -26,63 +65,90 @@ bool Evaluator::Evaluate(IrValue* value, std::uint32_t& result) {
     return true;
 }
 
-bool Evaluator::EvaluateWide(IrValue* raw, std::uint64_t& result) {
-    IrValue* value = raw->Resolve();
-    if (value->HasImmediate()) {
-        switch (value->Type()) {
-            case IrType::Bool: result = value->ImmediateBool() ? 1u : 0u; return true;
-            case IrType::U8: result = value->ImmediateU8(); return true;
-            case IrType::U16: result = value->ImmediateU16(); return true;
-            case IrType::U32: result = value->ImmediateU32(); return true;
-            case IrType::U64: result = value->ImmediateU64(); return true;
-            case IrType::F32: result = Float32Bits(value->ImmediateF32()); return true;
+template<typename TNode>
+bool BasicEvaluator<TNode>::EvaluateWide(TNode value, std::uint64_t& result) {
+    if (value.HasImmediate()) {
+        const auto bits = value.ImmediateBits();
+        switch (value.Type()) {
+            case IrType::Bool: result = bits != 0u ? 1u : 0u; return true;
+            case IrType::U8: result = static_cast<std::uint8_t>(bits); return true;
+            case IrType::U16: result = static_cast<std::uint16_t>(bits); return true;
+            case IrType::U32: result = static_cast<std::uint32_t>(bits); return true;
+            case IrType::U64: result = bits; return true;
+            case IrType::F32: result = Float32Bits(std::bit_cast<float>(static_cast<std::uint32_t>(bits))); return true;
             default: return false;
         }
     }
-    if (value->Opcode() == IrOpcode::Void) {
+    if (value.Opcode() == IrOpcode::Void) {
         return false;
     }
-    IrValue* inst = value;
-    if (_activeMask != nullptr && IsRuntimeSelect(inst->Opcode()) && inst->ArgumentCount() == 3 && inst->Argument(0)->Resolve() == _activeMask) {
-        return EvaluateWide(inst->Argument(1), result);
+    const TNode& inst = value;
+    if (_masked && IsRuntimeSelect(inst.Opcode()) && inst.ArgumentCount() == 3 && inst.Argument(0) == _activeMask) {
+        return EvaluateWide(inst.Argument(1), result);
     }
-    if (_cache.Find(inst, result)) {
+    if (inst.Id() < _dense.Size()) {
+        auto& slot = _dense.At(inst.Id());
+        if (slot.stamp == _dense.Done()) {
+            result = slot.value;
+            return true;
+        }
+        if (slot.stamp == _dense.Visiting()) {
+            return false;
+        }
+        slot.stamp = _dense.Visiting();
+        std::uint64_t out = 0;
+        if (!EvaluateInst(inst, out)) {
+            slot.stamp = 0;
+            static const bool debug = std::getenv("APS5_SRT_DEBUG") != nullptr;
+            if (debug) std::fprintf(stderr, "[srt] cannot evaluate %s (%zu arguments)\n", std::string(IrOpcodeName(inst.Opcode())).c_str(), inst.ArgumentCount());
+            return false;
+        }
+        slot = {_dense.Done(), out};
+        result = out;
         return true;
     }
-    if (std::find(_visiting.begin(), _visiting.end(), inst) != _visiting.end()) {
+    if (_cache.Find(inst.Value(), result)) {
+        return true;
+    }
+    if (std::find(_visiting.begin(), _visiting.end(), inst.Value()) != _visiting.end()) {
         return false;
     }
-    _visiting.push_back(inst);
+    _visiting.push_back(inst.Value());
     std::uint64_t out = 0;
-    const bool evaluated = EvaluateInst(*inst, out);
+    const bool evaluated = EvaluateInst(inst, out);
     _visiting.pop_back();
     if (!evaluated) {
         static const bool debug = std::getenv("APS5_SRT_DEBUG") != nullptr;
-        if (debug) std::fprintf(stderr, "[srt] cannot evaluate %s (%zu arguments)\n", std::string(IrOpcodeName(inst->Opcode())).c_str(), inst->ArgumentCount());
+        if (debug) std::fprintf(stderr, "[srt] cannot evaluate %s (%zu arguments)\n", std::string(IrOpcodeName(inst.Opcode())).c_str(), inst.ArgumentCount());
         return false;
     }
-    _cache.Insert(inst, out);
+    _cache.Insert(inst.Value(), out);
     result = out;
     return true;
 }
 
-float Evaluator::Float32(std::uint64_t bits) { return std::bit_cast<float>(static_cast<std::uint32_t>(bits)); }
+template<typename TNode>
+float BasicEvaluator<TNode>::Float32(std::uint64_t bits) { return std::bit_cast<float>(static_cast<std::uint32_t>(bits)); }
 
-std::uint64_t Evaluator::Float32Bits(float value) { return std::bit_cast<std::uint32_t>(value); }
+template<typename TNode>
+std::uint64_t BasicEvaluator<TNode>::Float32Bits(float value) { return std::bit_cast<std::uint32_t>(value); }
 
-bool Evaluator::Arg(IrValue& inst, std::size_t index, std::uint64_t& result) { return EvaluateWide(inst.Argument(index), result); }
+template<typename TNode>
+bool BasicEvaluator<TNode>::Arg(const TNode& inst, std::size_t index, std::uint64_t& result) { return EvaluateWide(inst.Argument(index), result); }
 
-bool Evaluator::EvaluatePhi(IrValue& inst, std::uint64_t& result) {
-    IrValue* value = ResolveInvariantPhi(_program, &inst);
-    return value != nullptr && EvaluateWide(value, result);
+template<typename TNode>
+bool BasicEvaluator<TNode>::EvaluatePhi(const TNode& inst, std::uint64_t& result) {
+    IrValue* value = ResolveInvariantPhi(_program, inst.Value());
+    return value != nullptr && EvaluateWide(TNode::From(_program, value), result);
 }
 
-bool Evaluator::EvaluateExtract(IrValue& inst, std::uint64_t& result) {
-    IrValue* index = inst.Argument(1)->Resolve();
-    if (!index->HasImmediate() || index->Type() != IrType::U32) {
+template<typename TNode>
+bool BasicEvaluator<TNode>::EvaluateExtract(const TNode& inst, std::uint64_t& result) {
+    const TNode index = inst.Argument(1);
+    if (!index.HasImmediate() || index.Type() != IrType::U32) {
         return false;
     }
-    const auto component = index->ImmediateU32();
+    const auto component = static_cast<std::uint32_t>(index.ImmediateBits());
     if (component >= 2u) {
         return false;
     }
@@ -94,17 +160,17 @@ bool Evaluator::EvaluateExtract(IrValue& inst, std::uint64_t& result) {
         result = static_cast<std::uint32_t>(packed >> (component * 32u));
         return true;
     }
-    IrValue* source = inst.Argument(0)->Resolve();
-    if (source->Opcode() == IrOpcode::Void) {
+    const TNode source = inst.Argument(0);
+    if (source.Opcode() == IrOpcode::Void) {
         return false;
     }
-    if (source->Opcode() == IrOpcode::CompositeConstructU32x2) {
-        return EvaluateWide(source->Argument(component), result);
+    if (source.Opcode() == IrOpcode::CompositeConstructU32x2) {
+        return EvaluateWide(source.Argument(component), result);
     }
-    if (source->Opcode() == IrOpcode::IAddCarry32) {
+    if (source.Opcode() == IrOpcode::IAddCarry32) {
         std::uint64_t lhs = 0;
         std::uint64_t rhs = 0;
-        if (!Arg(*source, 0, lhs) || !Arg(*source, 1, rhs)) {
+        if (!Arg(source, 0, lhs) || !Arg(source, 1, rhs)) {
             return false;
         }
         const auto sum = static_cast<std::uint64_t>(static_cast<std::uint32_t>(lhs)) + static_cast<std::uint32_t>(rhs);
@@ -114,20 +180,21 @@ bool Evaluator::EvaluateExtract(IrValue& inst, std::uint64_t& result) {
     return false;
 }
 
-bool Evaluator::EvaluateRawRead(IrValue& inst, std::uint64_t& result) {
-    const auto flags = inst.Flags<MemoryFlags>();
+template<typename TNode>
+bool BasicEvaluator<TNode>::EvaluateRawRead(const TNode& inst, std::uint64_t& result) {
+    const auto flags = inst.template Flags<MemoryFlags>();
     if (flags.index >= _program.memoryInfo.size()) {
         return false;
     }
     const auto& mem = _program.memoryInfo[flags.index];
-    IrValue* handle = inst.Argument(0)->Resolve();
-    if (handle->Opcode() == IrOpcode::Void) {
+    const TNode handle = inst.Argument(0);
+    if (handle.Opcode() == IrOpcode::Void) {
         return false;
     }
     std::uint64_t low = 0;
     std::uint64_t high = 0;
     std::uint64_t offset = 0;
-    if (!Arg(*handle, 0, low) || !Arg(*handle, 1, high) || !Arg(inst, 1, offset)) {
+    if (!Arg(handle, 0, low) || !Arg(handle, 1, high) || !Arg(inst, 1, offset)) {
         return false;
     }
     const auto base = ((high << 32u) | static_cast<std::uint32_t>(low)) & AddressMask;
@@ -136,7 +203,7 @@ bool Evaluator::EvaluateRawRead(IrValue& inst, std::uint64_t& result) {
     if (inst.Opcode() == IrOpcode::ReadConstBuffer) {
         std::uint64_t records = 0;
         std::uint64_t word3 = 0;
-        if (handle->ArgumentCount() != 4u || !Arg(*handle, 2, records) || !Arg(*handle, 3, word3)) {
+        if (handle.ArgumentCount() != 4u || !Arg(handle, 2, records) || !Arg(handle, 3, word3)) {
             return false;
         }
         if (immediate < 0) {
@@ -157,17 +224,17 @@ bool Evaluator::EvaluateRawRead(IrValue& inst, std::uint64_t& result) {
             return false;
         }
     }
-    if (_nullRoots != nullptr && base == 0 && !LoadedFromMemory(*handle)) {
+    if (_nullRoots != nullptr && base == 0 && !LoadedFromMemory(*handle.Value())) {
         ++*_nullRoots;
         result = 0;
         return true;
     }
-    if (_inaccessible != nullptr && _runtime.accessible != nullptr && !_runtime.accessible(_runtime.userContext, address, sizeof(std::uint32_t)) && LoadedFromMemory(*handle)) {
-        *_inaccessible = {&inst, address};
+    if (_inaccessible != nullptr && _runtime.accessible != nullptr && !_runtime.accessible(_runtime.userContext, address, sizeof(std::uint32_t)) && LoadedFromMemory(*handle.Value())) {
+        *_inaccessible = {inst.Value(), address};
         return false;
     }
     if (auto* trace = _runtime.readTrace; trace != nullptr) {
-        if (&inst == trace->leaf) trace->leaves.emplace_back(trace->leafSlot, address);
+        if (inst.Value() == trace->leaf) trace->leaves.emplace_back(trace->leafSlot, address);
         else trace->otherReads.push_back(address);
     }
     std::uint32_t word = 0;
@@ -274,7 +341,8 @@ std::optional<bool> evaluableAlone(const IrResourcePlan& program, const IrValue&
 
 }
 
-bool Evaluator::MayEvaluate(const IrResourcePlan& program, IrValue* value) {
+template<typename TNode>
+bool BasicEvaluator<TNode>::MayEvaluate(const IrResourcePlan& program, IrValue* value) {
     std::unordered_map<const IrValue*, bool> known;
     const auto requirements = [&program](IrValue* current) {
         std::vector<IrValue*> required;
@@ -326,7 +394,8 @@ bool Evaluator::MayEvaluate(const IrResourcePlan& program, IrValue* value) {
     return known.at(value->Resolve());
 }
 
-bool Evaluator::EvaluateInst(IrValue& inst, std::uint64_t& result) {
+template<typename TNode>
+bool BasicEvaluator<TNode>::EvaluateInst(const TNode& inst, std::uint64_t& result) {
     std::uint64_t a = 0;
     std::uint64_t b = 0;
     std::uint64_t c = 0;
@@ -334,7 +403,7 @@ bool Evaluator::EvaluateInst(IrValue& inst, std::uint64_t& result) {
     const auto ternary = [&]() { return Arg(inst, 0, a) && Arg(inst, 1, b) && Arg(inst, 2, c); };
     switch (inst.Opcode()) {
         case IrOpcode::GetUserData: {
-            const auto reg = RegIndex(static_cast<ScalarReg>(inst.Argument(0)->Register().index));
+            const auto reg = RegIndex(static_cast<ScalarReg>(inst.Argument(0).RegisterIndex()));
             if (reg < _program.userDataBase || reg - _program.userDataBase >= _runtime.userData.size()) {
                 return false;
             }
@@ -344,7 +413,8 @@ bool Evaluator::EvaluateInst(IrValue& inst, std::uint64_t& result) {
         case IrOpcode::GetShaderBase: result = _runtime.shaderBase; return true;
         case IrOpcode::Phi: return EvaluatePhi(inst, result);
         case IrOpcode::ReadFirstLane: {
-            Evaluator active(_program, _runtime, _cleanFlatSlots, _cleanEvaluator, inst.Argument(1));
+            const TNode mask = inst.Argument(1);
+            BasicEvaluator active(_program, _runtime, _cleanFlatSlots, _cleanEvaluator, &mask);
             active.ReportInaccessibleReads(_inaccessible);
             active.ReportNullRootReads(_nullRoots);
             return active.EvaluateWide(inst.Argument(0), result);
@@ -360,18 +430,19 @@ bool Evaluator::EvaluateInst(IrValue& inst, std::uint64_t& result) {
             result = static_cast<std::uint32_t>(a) | (static_cast<std::uint64_t>(static_cast<std::uint32_t>(b)) << 32u);
             return true;
         case IrOpcode::ReadConst: {
-            IrValue* slot = inst.Argument(1)->Resolve();
-            if (!slot->HasImmediate() || slot->Type() != IrType::U32 || slot->ImmediateU32() >= _program.srtReads.size()) {
+            const TNode slotValue = inst.Argument(1);
+            const auto slot = static_cast<std::uint32_t>(slotValue.ImmediateBits());
+            if (!slotValue.HasImmediate() || slotValue.Type() != IrType::U32 || slot >= _program.srtReads.size()) {
                 return false;
             }
-            if (slot->ImmediateU32() < _cleanFlatSlots.size() && _cleanFlatSlots[slot->ImmediateU32()] != 0u && _cleanEvaluator != nullptr) {
-                return _cleanEvaluator->EvaluateWide(_program.srtReads[slot->ImmediateU32()].value, result);
+            if (slot < _cleanFlatSlots.size() && _cleanFlatSlots[slot] != 0u && _cleanEvaluator != nullptr) {
+                return _cleanEvaluator->EvaluateWide(TNode::SrtRead(_program, slot), result);
             }
-            return EvaluateWide(_program.srtReads[slot->ImmediateU32()].value, result);
+            return EvaluateWide(TNode::SrtRead(_program, slot), result);
         }
         case IrOpcode::LoadAddressU32:
         case IrOpcode::ReadConstBuffer:
-            if (IsRawRead(_program, inst)) {
+            if (IsRawRead(_program, inst.Opcode(), inst.template Flags<MemoryFlags>().index)) {
                 return EvaluateRawRead(inst, result);
             }
             break;
@@ -657,4 +728,7 @@ bool Evaluator::EvaluateInst(IrValue& inst, std::uint64_t& result) {
     return false;
 }
 
+template class BasicEvaluator<IrNode>;
+template class BasicEvaluator<CompactNode>;
+
 }

core/shader/recompiler/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp

@@ -6,11 +6,13 @@
 namespace ShaderRecompiler::Detail {
 
 bool IsRawRead(const IrResourcePlan& program, const IrValue& inst) {
-    const auto op = inst.Opcode();
+    return IsRawRead(program, inst.Opcode(), inst.Flags<MemoryFlags>().index);
+}
+
+bool IsRawRead(const IrResourcePlan& program, IrOpcode op, std::uint32_t index) {
     if (op != IrOpcode::LoadAddressU32 && op != IrOpcode::ReadConstBuffer) {
         return false;
     }
-    const auto index = inst.Flags<MemoryFlags>().index;
     if (index >= program.memoryInfo.size()) {
         return false;
     }

core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp

@@ -29,7 +29,7 @@ void SrtWalker::EvaluateUniformValues(const IrResourcePlan& program, std::span<I
     clean.readMemory = runtime.readSpecializationMemory != nullptr ? runtime.readSpecializationMemory : +[](void*, std::uint64_t, std::uint32_t*) { return false; };
     Detail::Evaluator evaluator(program, clean);
     for (std::size_t i = 0; i < values.size(); ++i) {
-        if (!evaluator.Evaluate(values[i], results[i])) {
+        if (!evaluator.Evaluate(Detail::IrNode(values[i]), results[i])) {
             throw std::runtime_error("SrtWalker::EvaluateUniformValues failed to evaluate a uniform value");
         }
     }

core/shader/recompiler/ShaderDiskCache.cpp

@@ -145,7 +145,7 @@ static_assert(fieldCount<SamplerResource>() == 8, "SamplerResource changed: upda
 static_assert(fieldCount<SampledResourcePair>() == 3, "SampledResourcePair changed: update the info encoder");
 static_assert(fieldCount<StageInput>() == 5, "StageInput changed: update the info encoder");
 static_assert(fieldCount<StageOutput>() == 4, "StageOutput changed: update the info encoder");
-static_assert(fieldCount<IrResourcePlan>() == 21, "IrResourcePlan changed: update EncodePlan and DecodePlan");
+static_assert(fieldCount<IrResourcePlan>() == 23, "IrResourcePlan changed: update EncodePlan and DecodePlan");
 static_assert(fieldCount<GuestRegister>() == 2, "GuestRegister changed: update the plan value encoder");
 static_assert(fieldCount<MemoryInfo>() == 31, "MemoryInfo changed: update the plan encoder");
 static_assert(fieldCount<DescriptorSource>() == 3, "DescriptorSource changed: update the plan encoder");
@@ -1057,6 +1057,7 @@ bool encodePlan(Writer& writer, const IrResourcePlan& plan) {
     writer.Value(plan.srtGuardOffset);
     writer.Values(std::span<const std::uint8_t>(plan.cleanFlatSlots));
     writer.Values(std::span<const std::uint8_t>(plan.pureFlatSlots));
+    writer.Value(plan.denseValueIds);
     writer.Value(plan.requiresSpecializationMemory);
     writer.Value(plan.srtPlanComplete);
     writer.Value(plan.resourceTrackingComplete);
@@ -1212,6 +1213,7 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) {
     reader.Value(plan.srtGuardOffset);
     reader.Values(plan.cleanFlatSlots);
     reader.Values(plan.pureFlatSlots);
+    reader.Value(plan.denseValueIds);
     reader.Value(plan.requiresSpecializationMemory);
     reader.Value(plan.srtPlanComplete);
     reader.Value(plan.resourceTrackingComplete);
@@ -1230,12 +1232,15 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) {
     if (!std::ranges::all_of(plan.info.buffers, [&](const BufferResource& buffer) { return knownSource(buffer.source); })) return false;
     if (!std::ranges::all_of(plan.info.images, [&](const ImageResource& image) { return knownSource(image.source); })) return false;
     if (!std::ranges::all_of(plan.info.samplers, [&](const SamplerResource& sampler) { return knownSource(sampler.source); })) return false;
-    return std::ranges::all_of(plan.controlFlow, [&](const ResourceBlock& block) {
+    const bool blocksValid = std::ranges::all_of(plan.controlFlow, [&](const ResourceBlock& block) {
         return (block.condition == nullptr || block.successors.size() == 2u) &&
                std::ranges::all_of(block.successors, [&](std::uint32_t successor) { return successor < plan.controlFlow.size(); }) &&
                std::ranges::all_of(block.sources, knownSource) &&
                std::ranges::all_of(block.images, [&](std::uint32_t image) { return image < plan.info.images.size(); });
     });
+    if (!blocksValid) return false;
+    ResourceMaterializer::CompactPlan(plan);
+    return true;
 }
 
 constexpr std::string_view NeutralSwitches[] = {

core/shader/recompiler/ShaderDiskCache.hpp

@@ -13,7 +13,7 @@
 
 namespace ShaderRecompiler::ShaderDiskCache {
 
-inline constexpr std::uint32_t FormatVersion = 35;
+inline constexpr std::uint32_t FormatVersion = 36;
 
 enum class LoadStatus {
     Loaded,