Merge pull request #9401 from tibzejoker/perf/grow-pipeline-store

perf(agc): size the pipeline store to the title's working set

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -2753,6 +2753,11 @@ if(BUILD_TESTING)
     set_tests_properties(agc_pipeline_journal_long_paths PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_pipeline_library_clear_waits_for_build COMMAND agc_driver_graphics_tests --pipeline-library-clear-waits-for-build)
     add_test(NAME agc_pipeline_store_lazy_buffer_pool COMMAND agc_driver_graphics_tests --pipeline-store-lazy-buffer-pool)
+    add_test(NAME agc_pipeline_store_growth COMMAND agc_driver_graphics_tests --pipeline-store-growth)
+    add_test(NAME agc_pipeline_store_fixed_bound COMMAND agc_driver_graphics_tests --pipeline-store-growth)
+    add_test(NAME agc_pipeline_store_replay_growth COMMAND agc_driver_graphics_tests --pipeline-store-replay-growth)
+    set_tests_properties(agc_pipeline_store_growth agc_pipeline_store_fixed_bound agc_pipeline_store_replay_growth PROPERTIES TIMEOUT 60 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    set_tests_properties(agc_pipeline_store_fixed_bound PROPERTIES ENVIRONMENT "APS5_PIPELINE_STORE_ENTRIES=1024")
     set_tests_properties(agc_pipeline_journal PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     set_tests_properties(agc_pipeline_library_clear_waits_for_build PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     set_tests_properties(agc_pipeline_store_lazy_buffer_pool PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2760,7 +2765,7 @@ if(BUILD_TESTING)
         add_test(NAME agc_pipeline_journal_extended_paths COMMAND agc_driver_graphics_tests --pipeline-journal-extended-paths)
         set_tests_properties(agc_pipeline_journal_extended_paths PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
         set_tests_properties(agc_pipeline_journal_long_paths agc_pipeline_journal_extended_paths PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
-        set_tests_properties(agc_pipeline_journal agc_pipeline_library_clear_waits_for_build agc_pipeline_store_lazy_buffer_pool PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(agc_pipeline_journal agc_pipeline_library_clear_waits_for_build agc_pipeline_store_lazy_buffer_pool agc_pipeline_store_growth agc_pipeline_store_fixed_bound agc_pipeline_store_replay_growth PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     set_tests_properties(agc_driver_sparse_buffer_holes PROPERTIES TIMEOUT 30)
     if(WIN32)
@@ -3997,6 +4002,6 @@ endif()
 
 if(BUILD_TESTING)
     get_directory_property(agcRegisteredTests TESTS)
-    list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal(_long_paths|_extended_paths)?|agc_driver_pipeline_journal)$")
+    list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal(_long_paths|_extended_paths)?|agc_driver_pipeline_journal|agc_pipeline_store_replay_growth)$")
     set_property(TEST ${agcRegisteredTests} APPEND PROPERTY ENVIRONMENT "APS5_NO_PIPELINE_JOURNAL=1")
 endif()

core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp

@@ -109,8 +109,10 @@ private:
 // stages and every pipeline state field. A configuration whose stage has no module gets a private
 // pipeline. A miss builds the pipeline outside the store's mutex (a concurrent build of the same
 // key keeps the first inserted) and records its recipe in the pipeline journal. Entries are
-// LRU-bounded (APS5_PIPELINE_STORE_ENTRIES, 1024) and only evicted once no recorded draw holds
-// them. Debug aid: APS5_NO_PIPELINE_CACHE=1 builds one per draw as before.
+// LRU-bounded and only evicted once no recorded draw holds them. The bound starts at 1024 and
+// doubles up to 4096 for a journal replay that does not fit and while pipelines it evicted keep
+// coming back; it never shrinks during a run. APS5_PIPELINE_STORE_ENTRIES fixes it. Debug aid:
+// APS5_NO_PIPELINE_CACHE=1 builds one per draw as before.
 std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
 // `created` reports whether this call built the pipeline (false: served from the store or private).
 std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, bool* created = nullptr);
@@ -124,6 +126,8 @@ struct PipelineStoreCounters {
     std::uint64_t bound = 0;
 };
 PipelineStoreCounters PipelineStoreTotals();
+// Whether the store may still raise its bound (not fixed by APS5_PIPELINE_STORE_ENTRIES, below 4096).
+bool PipelineStoreCanGrow();
 // The pipeline store's key of a configuration (empty when the configuration gets a private pipeline).
 void BuildPipelineStoreKey(std::vector<std::byte>& key, const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout);
 struct DescribedSetLayouts {

core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp

@@ -1127,6 +1127,14 @@ struct PipelineStore {
     std::uint64_t evicted = 0;
     PipelineStoreCounters totals;
     std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now();
+    std::size_t capacity = 0;
+    bool fixedCapacity = false;
+    std::list<std::uint64_t> evictedOrder;
+    std::unordered_map<std::uint64_t, std::list<std::uint64_t>::iterator> evictedKeys;
+    std::size_t evictionsSinceCheck = 0;
+    std::size_t returnsSinceCheck = 0;
+    static constexpr std::size_t BaseEntries = 1024;
+    static constexpr std::size_t MaximumEntries = 4096;
 };
 
 // Never destroyed: the pipelines belong to a device that may already be gone when statics die, and
@@ -1155,9 +1163,41 @@ std::list<PipelineStore::Entry>::iterator abandon(PipelineStore& store, std::lis
     return store.entries.erase(it);
 }
 
-std::size_t storeBound() {
-    static const auto bound = static_cast<std::size_t>(std::max<std::uint64_t>(1u, PipelineSwitch("APS5_PIPELINE_STORE_ENTRIES", 1024u)));
-    return bound;
+std::size_t storeBound(PipelineStore& store) {
+    if (store.capacity == 0) {
+        const auto set = PipelineSwitch("APS5_PIPELINE_STORE_ENTRIES", 0u);
+        store.fixedCapacity = set != 0;
+        store.capacity = store.fixedCapacity ? static_cast<std::size_t>(set) : PipelineStore::BaseEntries;
+    }
+    return store.capacity;
+}
+
+void noteStoreEviction(PipelineStore& store, std::uint64_t hash) {
+    if (store.fixedCapacity) return;
+    if (const auto found = store.evictedKeys.find(hash); found != store.evictedKeys.end()) store.evictedOrder.erase(found->second);
+    store.evictedOrder.push_front(hash);
+    store.evictedKeys[hash] = store.evictedOrder.begin();
+    while (store.evictedOrder.size() > store.capacity) {
+        store.evictedKeys.erase(store.evictedOrder.back());
+        store.evictedOrder.pop_back();
+    }
+    if (++store.evictionsSinceCheck < store.capacity / 32u) return;
+    const bool churning = store.returnsSinceCheck * 2u > store.evictionsSinceCheck;
+    store.evictionsSinceCheck = 0;
+    store.returnsSinceCheck = 0;
+    if (!churning || store.capacity >= PipelineStore::MaximumEntries) return;
+    const auto previous = store.capacity;
+    store.capacity = std::min(store.capacity * 2u, PipelineStore::MaximumEntries);
+    std::fprintf(stderr, "[pipecache] bound %zu -> %zu pipelines: pipelines it evicted keep coming back\n", previous, store.capacity);
+}
+
+void noteStoreReturn(PipelineStore& store, std::uint64_t hash) {
+    if (store.fixedCapacity) return;
+    const auto found = store.evictedKeys.find(hash);
+    if (found == store.evictedKeys.end()) return;
+    store.evictedOrder.erase(found->second);
+    store.evictedKeys.erase(found);
+    ++store.returnsSinceCheck;
 }
 
 void reportPipelines(PipelineStore& store) {
@@ -1229,12 +1269,12 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
         }
         ++store.misses;
         ++store.totals.misses;
+        noteStoreReturn(store, hash);
     }
     // Built outside the mutex: a draw that finds its pipeline resident is not held behind another
     // thread's build (the journal replay builds here too), and a device's driver compiles in
     // parallel when several threads build at once.
     auto pipeline = std::make_shared<Pipeline>(context, state, vertexInput, resources, shaders, attachmentLayout);
-    const auto bound = storeBound();
     const bool replaying = PipelineJournalReplaying();
     {
         std::lock_guard lock(store.mutex);
@@ -1252,7 +1292,10 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
         }
         // A replayed pipeline enters at the least recently used end and never displaces a draw's:
         // the title's own pipelines evict it first, and a full store takes none.
-        if (replaying && store.entries.size() >= bound) return pipeline;
+        if (replaying && store.entries.size() >= storeBound(store)) {
+            if (store.fixedCapacity || store.capacity >= PipelineStore::MaximumEntries) return pipeline;
+            store.capacity = std::min(store.capacity * 2u, PipelineStore::MaximumEntries);
+        }
         if (replaying) {
             store.entries.push_front({context.device, context.bufferPool, context.bufferPool != nullptr, hash, std::vector<std::byte>(key), pipeline});
             store.index[hash] = store.entries.begin();
@@ -1260,12 +1303,14 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
             store.entries.push_back({context.device, context.bufferPool, context.bufferPool != nullptr, hash, std::vector<std::byte>(key), pipeline});
             store.index[hash] = std::prev(store.entries.end());
         }
-        while (store.entries.size() > bound) {
+        while (store.entries.size() > storeBound(store)) {
             // Only an entry no recorded draw still holds may go (Kept keeps its shared_ptr until the fence).
             const auto victim = std::find_if(store.entries.begin(), store.entries.end(), [](const PipelineStore::Entry& entry) { return entry.pipeline.use_count() == 1; });
             if (victim == store.entries.end()) break;
+            const auto evictedHash = victim->hash;
             store.index.erase(victim->hash);
             store.entries.erase(victim);
+            noteStoreEviction(store, evictedHash);
             ++store.evicted;
             ++store.totals.evicted;
         }
@@ -1284,10 +1329,16 @@ PipelineStoreCounters PipelineStoreTotals() {
     std::lock_guard lock(store.mutex);
     auto totals = store.totals;
     totals.cached = store.entries.size();
-    totals.bound = storeBound();
+    totals.bound = storeBound(store);
     return totals;
 }
 
+bool PipelineStoreCanGrow() {
+    auto& store = Pipelines();
+    std::lock_guard lock(store.mutex);
+    return storeBound(store) < PipelineStore::MaximumEntries && !store.fixedCapacity;
+}
+
 void ClearCachedPipelines(VkDevice device) {
     auto& store = Pipelines();
     std::lock_guard lock(store.mutex);
@@ -1299,6 +1350,13 @@ void ClearCachedPipelines(VkDevice device) {
         store.index.erase(it->hash);
         it = store.entries.erase(it);
     }
+    if (store.entries.empty() && !store.fixedCapacity) {
+        store.capacity = 0;
+        store.evictedOrder.clear();
+        store.evictedKeys.clear();
+        store.evictionsSinceCheck = 0;
+        store.returnsSinceCheck = 0;
+    }
     ClearPipelineLibraries(device);
     ReleasePixelFunctionLds(device);
 }

core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp

@@ -1352,7 +1352,7 @@ private:
         std::call_once(loadOnce, [&] {
             try {
                 auto journal = Journal::Instance().Load();
-                const auto limit = configured("APS5_PIPELINE_JOURNAL_ENTRIES", 1024u);
+                const auto limit = configured("APS5_PIPELINE_JOURNAL_ENTRIES", 4096u);
                 if (journal.entries.size() > limit) journal.entries.erase(journal.entries.begin(), journal.entries.end() - static_cast<std::ptrdiff_t>(limit));
                 loaded = std::move(journal);
             } catch (const std::exception& error) {
@@ -1438,7 +1438,7 @@ private:
             return;
         }
         const auto store = PipelineStoreTotals();
-        if (store.cached >= store.bound) {
+        if (store.cached >= store.bound && !PipelineStoreCanGrow()) {
             counters.skipped.fetch_add(1, std::memory_order_relaxed);
             return;
         }

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -11452,6 +11452,117 @@ void pipelineStoreLazyBufferPoolTests() {
     Require(mock.live == 0, "lazy buffer pool pipeline tests leaked Vulkan objects");
 }
 
+// The pipeline store's bound without a GPU: 1536 configurations cycled through the default 1024
+// entries keep coming back after their eviction, so the bound doubles and a later pass finds every
+// pipeline resident; a bound set with APS5_PIPELINE_STORE_ENTRIES never moves.
+void pipelineStoreGrowthTests(bool fixed) {
+    using namespace AgcDriver::Graphics;
+    mock = MockVulkan{};
+    auto context = mockContext();
+    context.device = makeHandle<VkDevice>();
+    context.graphicsPipelineLibrary = true;
+    context.pipelineLinkTimeOptimization = false;
+    context.limits.framebufferNoAttachmentsSampleCounts = VK_SAMPLE_COUNT_1_BIT;
+    ShaderRecompiler::RecompileResult vertex, fragment;
+    vertex.spirv = makeModule({});
+    fragment.spirv = makeModule({.fragment = true});
+    vertex.variantId = 301;
+    fragment.variantId = 302;
+    const std::array<CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, PipelinePushSlotBytes}}};
+    std::array<std::uint32_t, 1> mainKey{41}, fragmentKey{42};
+    const PipelineSetLayouts resources{makeHandle<VkDescriptorSetLayout>(), makeHandle<VkDescriptorSetLayout>(), true, mainKey, fragmentKey};
+    constexpr std::uint32_t configurations = 1536;
+    const auto pass = [&](std::uint32_t count) {
+        std::uint32_t built = 0;
+        for (std::uint32_t configuration = 0; configuration < count; ++configuration) {
+            VertexInputLayout input;
+            input.bindings.push_back({0, 4u * (configuration + 1u), VK_VERTEX_INPUT_RATE_VERTEX});
+            bool created = false;
+            static_cast<void>(CachedPipeline(context, State{}, input, resources, shaders, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, &created));
+            built += created ? 1u : 0u;
+        }
+        return built;
+    };
+    std::uint32_t lastBuilt = 0;
+    for (int round = 0; round < 6; ++round) lastBuilt = pass(configurations);
+    const auto grown = PipelineStoreTotals();
+    if (fixed) {
+        Require(grown.bound == 1024, "a bound set with APS5_PIPELINE_STORE_ENTRIES changed");
+        Require(lastBuilt == configurations, "a fixed 1024-entry bound kept pipelines of a 1536-configuration cycle");
+    } else {
+        Require(grown.bound == 2048, "the bound did not grow to 2048 while evicted pipelines kept coming back");
+        Require(lastBuilt == 0, "a pass over 1536 configurations still rebuilt pipelines after the bound grew");
+    }
+    ClearCachedPipelines(context.device);
+    Require(mock.live == 0, "pipeline store growth tests leaked Vulkan objects");
+}
+
+// A journal longer than the store's starting bound: 1032 recorded pipelines replayed into an empty
+// store all stay resident (the replay grows the bound instead of dropping what does not fit).
+void pipelineStoreReplayGrowthTests() {
+    using namespace AgcDriver::Graphics;
+    const auto root = std::filesystem::temp_directory_path() / ("aps5-pipeline-store-replay-" + std::to_string(std::random_device{}()));
+    std::filesystem::create_directories(root);
+    struct Cleanup {
+        std::filesystem::path directory;
+        ~Cleanup() {
+            ShaderRecompiler::ShaderDiskCache::Shutdown();
+            std::error_code error;
+            std::filesystem::remove_all(directory, error);
+        }
+    } cleanup{root};
+#ifdef _WIN32
+    _putenv_s("ANYPS5_NO_SHADER_CACHE", "0");
+    _wputenv_s(L"ANYPS5_SHADER_CACHE_DIR", root.c_str());
+    _putenv_s("APS5_PIPELINE_JOURNAL_THREADS", "1");
+#else
+    setenv("ANYPS5_NO_SHADER_CACHE", "0", 1);
+    setenv("ANYPS5_SHADER_CACHE_DIR", root.string().c_str(), 1);
+    setenv("APS5_PIPELINE_JOURNAL_THREADS", "1", 1);
+#endif
+    Require(PipelineJournalEnabled(), "the journal is disabled with a shader cache directory set");
+    mock = MockVulkan{};
+    auto context = mockContext();
+    context.device = makeHandle<VkDevice>();
+    context.graphicsPipelineLibrary = true;
+    context.pipelineLinkTimeOptimization = false;
+    context.limits.framebufferNoAttachmentsSampleCounts = VK_SAMPLE_COUNT_1_BIT;
+    ShaderRecompiler::RecompileResult vertex, fragment;
+    vertex.spirv = makeModule({});
+    fragment.spirv = makeModule({.fragment = true});
+    vertex.variantId = 401;
+    fragment.variantId = 402;
+    const std::array<CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, PipelinePushSlotBytes}}};
+    auto cache = std::make_unique<DescriptorCache>(context);
+    context.descriptorCache = cache.get();
+    const auto layouts = DescribeSetLayouts(context, shaders);
+    const auto resources = layouts.View();
+    constexpr std::uint64_t fingerprint = 0x5eed'0000'0000'0002ull;
+    {
+        auto absent = StartPipelineJournalReplay(context, fingerprint, {});
+        Require(absent != nullptr, "the first replay did not start");
+        absent->Wait();
+    }
+    constexpr std::uint32_t configurations = 1032;
+    for (std::uint32_t configuration = 0; configuration < configurations; ++configuration) {
+        VertexInputLayout input;
+        input.bindings.push_back({0, 4u * (configuration + 1u), VK_VERTEX_INPUT_RATE_VERTEX});
+        static_cast<void>(CachedPipeline(context, State{}, input, resources, shaders));
+    }
+    FlushPipelineJournal();
+    ClearCachedPipelines(context.device);
+    Require(PipelineStoreTotals().cached == 0, "clearing the device left pipelines in the store");
+    auto replay = StartPipelineJournalReplay(context, fingerprint, {});
+    Require(replay != nullptr, "the replay did not start");
+    replay->Wait();
+    const auto totals = PipelineStoreTotals();
+    Require(totals.cached == configurations, "a replay of " + std::to_string(configurations) + " recorded pipelines kept " + std::to_string(totals.cached) + " in the store (bound " + std::to_string(totals.bound) + ")");
+    ClearCachedPipelines(context.device);
+    context.descriptorCache = nullptr;
+    cache.reset();
+    Require(mock.live == 0, "pipeline store replay growth tests leaked Vulkan objects");
+}
+
 void pipelineJournalTests(bool longPath = false, bool extended = false) {
     using namespace AgcDriver::Graphics;
     using Stage = ShaderRecompiler::ShaderStage;
@@ -11883,6 +11994,16 @@ int main(int argc, char** argv) {
             std::cout << "pipeline library clear waits for active part builds\n";
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--pipeline-store-replay-growth") {
+            pipelineStoreReplayGrowthTests();
+            std::cout << "a replayed journal longer than the store bound stays resident\n";
+            return 0;
+        }
+        if (argc == 2 && std::string_view(argv[1]) == "--pipeline-store-growth") {
+            pipelineStoreGrowthTests(std::getenv("APS5_PIPELINE_STORE_ENTRIES") != nullptr);
+            std::cout << "pipeline store bound follows the evicted pipelines that come back\n";
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--pipeline-store-lazy-buffer-pool") {
             pipelineStoreLazyBufferPoolTests();
             std::cout << "cached pipelines survive lazy buffer pool creation\n";

core/libs/prx/libSceAgcDriver/tests/Recorder.cpp

@@ -50,6 +50,7 @@
 #endif
 #include <io.h>
 #include <windows.h>
+#include <psapi.h>
 #else
 #include <fcntl.h>
 #include <sys/mman.h>
@@ -67,6 +68,7 @@
 #include <cstring>
 #include <filesystem>
 #include <fstream>
+#include <iomanip>
 #include <future>
 #include <iostream>
 #include <memory>
@@ -322,6 +324,7 @@ public:
                 }
                 return false;
             };
+            memoryBudget = hasExtension(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME);
             auto bytes = AgcDriver::QueryBdaByteFeatures(context.physical, function<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2"), available);
             auto address = AgcDriver::QueryBdaFeatures(context.physical, function<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2"), available);
             const auto queues = function<PFN_vkGetPhysicalDeviceQueueFamilyProperties>("vkGetPhysicalDeviceQueueFamilyProperties");
@@ -516,6 +519,21 @@ public:
         return properties;
     }
     void WaitQueue() const { Check(context.Function<PFN_vkQueueWaitIdle>("vkQueueWaitIdle")(context.queue), "vkQueueWaitIdle"); }
+    struct HeapUsage {
+        std::uint64_t deviceLocal = 0;
+        std::uint64_t host = 0;
+    };
+    std::optional<HeapUsage> MemoryUsage() const {
+        if (!memoryBudget) return std::nullopt;
+        VkPhysicalDeviceMemoryBudgetPropertiesEXT budget{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
+        VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, &budget};
+        function<PFN_vkGetPhysicalDeviceMemoryProperties2>("vkGetPhysicalDeviceMemoryProperties2")(context.physical, &properties);
+        HeapUsage usage;
+        for (std::uint32_t heap = 0; heap < properties.memoryProperties.memoryHeapCount; ++heap) {
+            ((properties.memoryProperties.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0 ? usage.deviceLocal : usage.host) += budget.heapUsage[heap];
+        }
+        return usage;
+    }
 
 private:
     template<typename TFunction>
@@ -545,6 +563,7 @@ private:
     bool graphics = false;
     std::mutex settleMutex;
     bool nullDescriptors = false;
+    bool memoryBudget = false;
 };
 
 using Kind = Recorder::ReadKind;
@@ -10112,6 +10131,98 @@ bool vertexTargetRangeTests(const Device& device, Recorder& recorder, bool impor
     return true;
 }
 
+std::uint64_t processPrivateBytes() {
+#ifdef _WIN32
+    PROCESS_MEMORY_COUNTERS_EX counters{};
+    counters.cb = sizeof(counters);
+    if (!K32GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PROCESS_MEMORY_COUNTERS*>(&counters), sizeof(counters))) return 0;
+    return counters.PrivateUsage;
+#else
+    std::ifstream status("/proc/self/status");
+    std::string line;
+    while (std::getline(status, line)) {
+        if (line.rfind("RssAnon:", 0) == 0) return std::stoull(line.substr(8)) * 1024u;
+    }
+    return 0;
+#endif
+}
+
+// What one graphics pipeline the store keeps costs: `count` pipelines, each with its own fragment
+// module (so its own fragment shader library), built through CachedPipeline and held, after their
+// optimized relinks; process private bytes and VK_EXT_memory_budget heap usage before and after,
+// and after the store is cleared.
+int pipelineStoreMemoryBench(const Device& device, std::uint32_t count) {
+    if (!device.Graphics() || count == 0) {
+        std::cout << "skipped, the pipeline store memory bench requires a graphics queue and at least one pipeline\n";
+        return 77;
+    }
+    std::lock_guard gpu(GpuMutex());
+    const auto& context = device.GetContext();
+    struct Cleanup {
+        VkDevice device;
+        ~Cleanup() { ClearCachedPipelines(device); }
+    } cleanup{context.device};
+    ClearCachedPipelines(context.device);
+    ShaderRecompiler::RecompileResult vertex;
+    vertex.variantId = 21;
+    vertex.spirv = std::vector<std::uint32_t>(std::begin(TRIANGLE_vert_SPV), std::end(TRIANGLE_vert_SPV));
+    std::vector<ShaderRecompiler::RecompileResult> fragments(count + 1u);
+    for (std::uint32_t index = 0; index <= count; ++index) {
+        fragments[index].variantId = 1000u + index;
+        fragments[index].spirv = std::vector<std::uint32_t>(std::begin(TRIANGLE_frag_SPV), std::end(TRIANGLE_frag_SPV));
+        fragments[index].spirv[2] = 0x50535430u + index;
+    }
+    State state{};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.hasColorTarget = true;
+    state.color.format = VK_FORMAT_R8G8B8A8_UNORM;
+    state.colors.push_back(state.color);
+    state.blend.colorWriteMask = 15;
+    state.blends.push_back(state.blend);
+    const std::array first{CompiledShader{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, CompiledShader{ShaderRecompiler::ShaderStage::Fragment, &fragments[count], 0}};
+    ShaderResources resources(context, first, state.color, static_cast<std::uint32_t>(state.colors.size()), 0, 0);
+    const VertexInputLayout input;
+    std::vector<std::shared_ptr<Pipeline>> held;
+    held.reserve(count + 1u);
+    const auto build = [&](std::uint32_t index) {
+        const std::array shaders{CompiledShader{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, CompiledShader{ShaderRecompiler::ShaderStage::Fragment, &fragments[index], 0}};
+        held.push_back(CachedPipeline(context, state, input, resources, shaders, VK_IMAGE_LAYOUT_GENERAL));
+    };
+    struct Sample {
+        std::uint64_t privateBytes;
+        std::optional<Device::HeapUsage> heaps;
+    };
+    const auto sample = [&] {
+        WaitForOptimizedPipelines(context.device);
+        return Sample{processPrivateBytes(), device.MemoryUsage()};
+    };
+    build(count);
+    const auto before = sample();
+    const auto started = std::chrono::steady_clock::now();
+    for (std::uint32_t index = 0; index < count; ++index) build(index);
+    const auto built = std::chrono::steady_clock::now();
+    const auto after = sample();
+    const auto linked = std::chrono::steady_clock::now();
+    const auto totals = PipelineStoreTotals();
+    held.clear();
+    ClearCachedPipelines(context.device);
+    const auto cleared = sample();
+    const auto mib = [](std::uint64_t later, std::uint64_t earlier) { return (static_cast<double>(later) - static_cast<double>(earlier)) / (1024.0 * 1024.0); };
+    const auto perPipeline = [&](std::uint64_t later, std::uint64_t earlier) { return (static_cast<double>(later) - static_cast<double>(earlier)) / 1024.0 / count; };
+    const char* linking = !context.graphicsPipelineLibrary ? "whole pipelines" : context.pipelineLinkTimeOptimization ? "libraries and optimized relinks" : "libraries, fast links only";
+    std::cout << std::fixed << std::setprecision(1);
+    std::cout << "Pipeline store memory: " << count << " graphics pipelines (one fragment module each, " << linking << "), " << totals.cached << " in the store, built in "
+              << std::chrono::duration<double>(built - started).count() << " s, relinked in " << std::chrono::duration<double>(linked - built).count() << " s\n";
+    std::cout << "  process private bytes: +" << mib(after.privateBytes, before.privateBytes) << " MiB = " << perPipeline(after.privateBytes, before.privateBytes) << " KiB per pipeline; after clearing the store " << mib(cleared.privateBytes, before.privateBytes) << " MiB above the start\n";
+    if (before.heaps && after.heaps && cleared.heaps) {
+        std::cout << "  VK_EXT_memory_budget heap usage: device-local +" << mib(after.heaps->deviceLocal, before.heaps->deviceLocal) << " MiB = " << perPipeline(after.heaps->deviceLocal, before.heaps->deviceLocal) << " KiB per pipeline, host +" << mib(after.heaps->host, before.heaps->host) << " MiB = " << perPipeline(after.heaps->host, before.heaps->host)
+                  << " KiB per pipeline; after clearing the store device-local " << mib(cleared.heaps->deviceLocal, before.heaps->deviceLocal) << " MiB, host " << mib(cleared.heaps->host, before.heaps->host) << " MiB above the start\n";
+    } else {
+        std::cout << "  VK_EXT_memory_budget unavailable: no device heap usage\n";
+    }
+    return 0;
+}
+
 void pipelineCacheTests(const Device& device, bool benchmark) {
     std::thread establishThreadedProcess([] {});
     establishThreadedProcess.join();
@@ -11331,6 +11442,7 @@ int main(int argc, char** argv) {
             pipelineLibraryTests(device);
             return 0;
         }
+        if ((argc == 2 || argc == 3) && std::string_view(argv[1]) == "--benchmark-pipeline-store-memory") return pipelineStoreMemoryBench(device, argc == 3 ? static_cast<std::uint32_t>(std::stoul(argv[2])) : 4096u);
         if (argc == 2 && (std::string_view(argv[1]) == "--benchmark-pipeline-cache" || std::string_view(argv[1]) == "--pipeline-cache-only")) {
             pipelineCacheTests(device, std::string_view(argv[1]) == "--benchmark-pipeline-cache");
             return 0;

docs/dev/ARCHITECTURE.md

@@ -124,7 +124,7 @@ flowchart LR
 
 - [`Recompiler.cpp`](../../core/shader/recompiler/Recompiler.cpp) runs the stages in this order. With `ANYPS5_ENABLE_SPIRV_TOOLS`, the SPIR-V is also validated and optimized with SPIRV-Tools.
 - `ShaderRecompiler::Recompile` keeps compiled variants in memory, and `ShaderDiskCache` stores them on disk so later runs reuse them. Before the writable cache, a miss is also looked up in read-only prebuilt packs and loose entry folders under `shader_cache/prebuilt`; pack entries are ordinary encoded cache entries behind a sorted key-hash index, validated by magic, compiler source version and index checksum when mapped, and a miss or rejected entry falls through to the writable cache and the compiler. Cache format 35 also stores the finite stencil-reference set, its replay-safety proof and reserved push-data word in the artifact and binding layouts. Entries from formats 31 to 34 are rejected.
-- The Vulkan pipelines are journaled as well ([`PipelineJournal.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp)): a graphics pipeline the store builds (`CachedPipeline`) or a compute pipeline a dispatch builds is appended to `pipelines.journal` next to the shader cache entries, as the materialized result of each stage (the specialized SPIR-V, bindings and push constants, through `ShaderDiskCache::EncodeResult`) plus the recipe the pipeline was built from (`GraphicsPipelineRecipe`: the stages' identities, vertex input layout, descriptor set layout keys and the state fields the `Pipeline` constructor reads, `CanonicalPipelineState`). When a `VulkanDevice` is created, the journal is loaded and its pipelines are built on a pool of `hardware_concurrency() / 2` threads (`APS5_PIPELINE_JOURNAL_THREADS`) while the title loads. The load parses the file a record at a time and keeps it open, and a replay thread reads an entry's modules from it when it builds that entry (a record that no longer reads back as loaded rejects the entry), so the replay holds the modules of the entries being built rather than the file, so the first draw or dispatch of a configuration seen on an earlier run finds its pipeline resident. The pipeline stores key on `RecompileResult::Identity`, a hash of the module words, specialization constants, subgroup width and bindings that is the same on every run (`SharedSpirv::ContentHash`, computed once per shared module), rather than on the process-lifetime variant ids, and a graphics pipeline is built outside the store's mutex, so the replay threads and the queue workers build in parallel. A draw that arrives before its replayed entry builds the pipeline itself; the first inserted wins. A replayed entry enters the store at its least recently used end and a full store takes none, so the title's own pipelines are never evicted for a replayed one, and the optimized relink of a replayed pipeline (`OptimizedPipeline::Schedule`) waits for the first draw that binds it, so the relink thread serves the title's pipelines first. A device created without a window starts its replay at its first draw or dispatch rather than at creation, since `Driver::AttachWindow` usually replaces it before the title draws. The journal records a pipeline once (modules and recipes are deduplicated by content), keeps at most `APS5_PIPELINE_JOURNAL_MAX_MIB` (1024), replays the first `APS5_PIPELINE_JOURNAL_ENTRIES` recorded (1024, the pipeline store's bound, `APS5_PIPELINE_STORE_ENTRIES`; a run reaches those first), and is disabled with `APS5_NO_PIPELINE_JOURNAL=1` or without a shader cache directory. The file header names the device (`PipelineJournalDeviceFingerprint`, the properties the driver's pipeline cache file is named by), so a journal written for another GPU or driver is started over rather than replayed; appends and reads hold `pipelines.journal.lock` against another process sharing the directory, and a device cancels its replay before it is replaced (`PrepareForReplacement`) or destroyed, so a replaced headless device never inserts entries the window's device then sweeps. `APS5_PROFILE_DRAW=1` prints a `[pipeline-journal]` summary when the replay finishes and at teardown.
+- The Vulkan pipelines are journaled as well ([`PipelineJournal.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp)): a graphics pipeline the store builds (`CachedPipeline`) or a compute pipeline a dispatch builds is appended to `pipelines.journal` next to the shader cache entries, as the materialized result of each stage (the specialized SPIR-V, bindings and push constants, through `ShaderDiskCache::EncodeResult`) plus the recipe the pipeline was built from (`GraphicsPipelineRecipe`: the stages' identities, vertex input layout, descriptor set layout keys and the state fields the `Pipeline` constructor reads, `CanonicalPipelineState`). When a `VulkanDevice` is created, the journal is loaded and its pipelines are built on a pool of `hardware_concurrency() / 2` threads (`APS5_PIPELINE_JOURNAL_THREADS`) while the title loads. The load parses the file a record at a time and keeps it open, and a replay thread reads an entry's modules from it when it builds that entry (a record that no longer reads back as loaded rejects the entry), so the replay holds the modules of the entries being built rather than the file, so the first draw or dispatch of a configuration seen on an earlier run finds its pipeline resident. The pipeline stores key on `RecompileResult::Identity`, a hash of the module words, specialization constants, subgroup width and bindings that is the same on every run (`SharedSpirv::ContentHash`, computed once per shared module), rather than on the process-lifetime variant ids, and a graphics pipeline is built outside the store's mutex, so the replay threads and the queue workers build in parallel. A draw that arrives before its replayed entry builds the pipeline itself; the first inserted wins. A replayed entry enters the store at its least recently used end and a full store takes none, so the title's own pipelines are never evicted for a replayed one, and the optimized relink of a replayed pipeline (`OptimizedPipeline::Schedule`) waits for the first draw that binds it, so the relink thread serves the title's pipelines first. A device created without a window starts its replay at its first draw or dispatch rather than at creation, since `Driver::AttachWindow` usually replaces it before the title draws. The journal records a pipeline once (modules and recipes are deduplicated by content), keeps at most `APS5_PIPELINE_JOURNAL_MAX_MIB` (1024), replays the last `APS5_PIPELINE_JOURNAL_ENTRIES` recorded (4096, the pipeline store's largest bound: a replayed pipeline that finds the store full grows its bound from 1024 instead of being dropped), and is disabled with `APS5_NO_PIPELINE_JOURNAL=1` or without a shader cache directory. The file header names the device (`PipelineJournalDeviceFingerprint`, the properties the driver's pipeline cache file is named by), so a journal written for another GPU or driver is started over rather than replayed; appends and reads hold `pipelines.journal.lock` against another process sharing the directory, and a device cancels its replay before it is replaced (`PrepareForReplacement`) or destroyed, so a replaced headless device never inserts entries the window's device then sweeps. `APS5_PROFILE_DRAW=1` prints a `[pipeline-journal]` summary when the replay finishes and at teardown.
 - A compute shader whose LDS plus its lock dword exceeds the host's `maxComputeSharedMemorySize` is not given a `Workgroup` array. `Recompile` sets `ShaderInfo::sharedMemoryBytes` for it and the SPIR-V backend addresses the LDS in a coherent storage buffer at descriptor set 1 (`WorkgroupMemoryDescriptorSet`), with a slice of `RecompileResult::workgroupMemoryDwords` per workgroup, indexed by `WorkgroupID` and `NumWorkGroups`. `VulkanDevice` keeps one such buffer, sized from the dispatch dimensions, and binds it after the resource set; programs that fit keep using shared memory.
 - `Driver::RegisterShader` splits shader preparation in two. `PlanRegistered` runs on the registering guest thread: it validates the registers and entry point, decodes the code and builds the recompile request, so invalid registrations still fail there. The RDNA to IR translation and resource plan (`PrepareShader`) run on a background pool of `hardware_concurrency() / 2` detached threads (at least 2, 64 MiB stacks).
 - A thread that needs the GPU mutex, the committer's next item or the end of the committer's queue spins before it blocks (`HandoffSpin.hpp`): `GpuMutexType::lock` retries `try_lock` for the window, the committer polls its item count, and `DrawPipeline::Drain` polls it for twice the window, so a hand-off that completes within the window costs no kernel wake-up. The window is 150 us when the process may run on more than 8 logical processors (its affinity mask, via `CpuTopology`) and 0 otherwise, where the spinning threads would compete with the holder; `APS5_HANDOFF_SPIN_US=<n>` sets it (0 turns it off, at most 10000). Ordering and the lock's semantics are unchanged.

docs/dev/ENVIRONMENT.md

@@ -353,11 +353,11 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti
 | `APS5_PIPELINE_DRAIN_FILLS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp` |  | runs queue-0 buffer fills through the draw pipeline to keep their order; any value stores them in place instead |
 | `APS5_PIPELINE_DRAIN_LABELS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp` |  | answers a queue-0 wait from the draw pipeline's own pending labels while it is busy; any value drains the pipeline at the wait |
 | `APS5_PIPELINE_DRAIN_SUBMISSIONS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp` |  | lets a submission end without submitting its recorded work, so it joins a later submit with the pipeline's queue; any value drains the pipeline and submits at the end of the submission |
-| `APS5_PIPELINE_JOURNAL_ENTRIES` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp` | 1024 | number of journal entries loaded at most; a longer journal keeps its last entries, the ones recorded last |
+| `APS5_PIPELINE_JOURNAL_ENTRIES` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp` | 4096 | number of journal entries loaded at most; a longer journal keeps its last entries, the ones recorded last |
 | `APS5_PIPELINE_JOURNAL_MAX_MIB` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp` | 1024 | size cap in MiB on the pipeline journal; the file is rewritten down to three quarters of the cap when it is reached, keeping the newest pipelines and the modules they name and leaving room for later records |
 | `APS5_PIPELINE_JOURNAL_THREADS` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp` |  | threads that replay the journal's pipelines, 0 disables the replay; unset uses half the hardware threads, at least one |
 | `APS5_PIPELINE_STATS` | string | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` |  | captures and prints driver statistics for each new pipeline, requiring `VK_KHR_pipeline_executable_properties`; `1` enables |
-| `APS5_PIPELINE_STORE_ENTRIES` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp` | 1024 | LRU bound on the entries of the pipeline store; an entry is evicted only once no recorded draw holds it |
+| `APS5_PIPELINE_STORE_ENTRIES` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp` | adaptive | fixes the LRU bound on the entries of the pipeline store, which otherwise starts at 1024 and doubles up to 4096 for a journal replay that does not fit and while evicted pipelines keep coming back; an entry is evicted only once no recorded draw holds it |
 | `APS5_PM4_RING` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/SubmissionRing.cpp` | 16 | number of submissions the PM4 ring keeps for dumps, from 0 to 4096; 0 disables the ring |
 | `APS5_POLL_REAP` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp` |  | reaps completions on every poll tick |
 | `APS5_POLL_TRY_EACH` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp` |  | tries the poll service on every tick instead of at the poll interval |

docs/dev/debt/libSceAgcDriver.md

@@ -116,7 +116,7 @@
 - [AGC buffer allocations](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp) retry host/device out-of-memory once after reclaiming idle pool buffers and fully unused slab blocks. Live buffers and blocks with live slots are retained. Live images and other retained resources have no global memory bound; allocation failure still throws when reclaim cannot satisfy the request, except for the device buffers and images that the "Host placement after a device-memory refusal" entry places in host memory.
 - [Storage-cache pressure and idle device buffers](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp): the one-second idle observation interval is a retention choice, not a console rule; buffers reused less often can be recreated under pressure. Before a storage-cache eviction, independently allocated device-tier buffers unused throughout that interval can be released, oldest first, for the cache's shortfall below its existing ceiling. Live buffers, host tiers and slab allocations are not reclaimed this way. Reclamation requires exactly one device-local heap because retained buffers do not record their heap. The heap budget is read again after an actual release; a delayed driver report can still cause an eviction. This is not a shared budget for all retained GPU resources, and its effect on titles with intermittent large buffer use is unmeasured.
 - [Host placement after a device-memory refusal](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp) (libSceAgcDriver): when the driver refuses a device-local allocation with `VK_ERROR_OUT_OF_DEVICE_MEMORY` after the pool's trim and the texture-cache release, device buffers and images (textures, storage textures, block, repacked, depth, depth plane and stencil copies, render, depth and multisampled targets and attachment proxies) are allocated in a host memory type their requirements allow. A host-placed device buffer leaves the pool on release. A host-placed image keeps that placement until it is destroyed and is never moved back to device memory, so every access to it crosses PCIe for its lifetime. The placements are counted in a rate-limited `[bufferpool]` line with the cumulative MiB, not the resident bytes. The 1x1 denormal clear probe still allocates device memory directly and throws on a refusal.
-- [Pipeline journal](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp) (libSceAgcDriver): the journal is append-only until it reaches `APS5_PIPELINE_JOURNAL_MAX_MIB`, when it is rewritten to three quarters of the cap with the newest pipelines and their modules; the rewrite streams kept records; a replay thread reads an entry's modules when it builds that entry. Only the last `APS5_PIPELINE_JOURNAL_ENTRIES` pipelines are replayed, and the store evicts the least recently used entry past `APS5_PIPELINE_STORE_ENTRIES`: a title with more configurations rebuilds the rest at the draw. A compute pipeline is replayed over the set layout `DescribeSetLayouts` derives from the result's bindings, checked against its `ShaderResources` layout only under `APS5_VERIFY_SET_LAYOUTS=1`. Recording and replay are best effort: a record whose write fails is lost and an entry whose build fails is skipped, both counted in the `[pipeline-journal]` line. The file header's format number guards the recipe codec and the store key.
+- [Pipeline journal](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp) (libSceAgcDriver): resident graphics pipelines are bounded by count, not bytes: the store grows from 1024 to 4096 entries for a replay that does not fit and for evicted pipelines that keep coming back, and the last `APS5_PIPELINE_JOURNAL_ENTRIES` (4096) records are all replayed, so a pipeline the title never asks for again costs its memory until teardown. `agc_driver_recorder_tests --benchmark-pipeline-store-memory [count]` gives a floor with the triangle shaders: about 59 KiB host and no device memory per pipeline on an RTX 4090, 100 KiB host and 8 KiB device-local on a UHD 770; larger shaders cost more. The file is rewritten to three quarters of `APS5_PIPELINE_JOURNAL_MAX_MIB` when full. A compute pipeline is replayed over the layout `DescribeSetLayouts` derives, checked only under `APS5_VERIFY_SET_LAYOUTS=1`. The header's format number guards the recipe codec and the store key.
 - [Graphics pipeline libraries](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp) (libSceAgcDriver): a pipeline is fast-linked at the draw and relinked with `VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT` on a background thread, and binds the optimized one from the next draw on. Draws recorded before that use the fast link, whose vertex stage keeps the outputs the fragment stage does not read: on RADV (Radeon 660M, Mesa 26.2.4) 346 of 483 Astro's Playroom vertex shaders paired with a fragment stage that reads none of them used more VGPRs (median 8 more, up to 2.5 times), and on an RTX 4090 13 of 15 PPSA36082 vertex shaders up to 68 instead of 38 (#2242). The optimized link compiles to the same code as a whole pipeline
 - [Fragment shader interlock pipelines](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp) (libSceAgcDriver): a pipeline whose fragment shader declares a fragment shader interlock capability is never built from graphics pipeline libraries, on every driver, so it is compiled whole at the draw with no fast link to bridge the compile. RADV's fast link ([mesa#16653](https://gitlab.freedesktop.org/mesa/mesa/-/work_items/16653), Mesa 26.2.5, RX 7900 GRE) jumps to the PS epilog without waiting for the shader's stores, and the epilog's `done` export ends the GFX11 primitive-ordered section, so the next overlapping wave reads the old value: `agc_driver_pixel_interlock` lost read-modify-writes in 108 of 120 loaded runs under `APS5_NO_PIPELINE_LTO=1`. On an RTX 4090 (NVIDIA 617.42, Windows) the fast link kept the ordering in 240 of 240 loaded runs; other drivers are not measured.
 - [Prebuilt pipeline libraries](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver): each registered vertex or pixel shader in registration form gets its pre-rasterization or fragment-shader library built on one background thread, with a fixed state the draw may not use: triangle list, counter-clockwise front face, no depth clamp, depth bias or conservative rasterization, first provoking vertex, one sample. Libraries are kept until the device is torn down; on Astro's Playroom 1948 of them added 0.44 GB of peak memory on RADV.

docs/user/USAGE.md

@@ -320,7 +320,7 @@ Recompiled shaders and the Vulkan pipeline cache are kept in a `shader_cache/` d
 
 ### Pipeline journal
 
-Besides the recompiled shaders, the shader cache directory holds `pipelines.journal`: every Vulkan graphics pipeline the pipeline store kept and every compute pipeline a run built, with the specialized shader modules they were built from. The next run of the title builds those pipelines on background threads while it loads, so a draw or dispatch the earlier run already made does not compile its pipeline when it is first reached. The journal belongs to the GPU and driver that wrote it, like the driver's pipeline cache file next to it: a run on another GPU or driver version starts it over, and the driver's own pipeline cache decides how fast each pipeline is rebuilt. Two runs sharing the directory take turns on `pipelines.journal.lock`; a run that cannot lock it records nothing. `APS5_NO_PIPELINE_JOURNAL=1` disables recording and replay; `APS5_PIPELINE_JOURNAL_THREADS` sets the replay threads (half the CPU threads by default; `0` records without replaying), `APS5_PIPELINE_JOURNAL_ENTRIES` how many pipelines are replayed (the first recorded, which a run reaches first) and `APS5_PIPELINE_STORE_ENTRIES` how many graphics pipelines stay resident (1024 each), and `APS5_PIPELINE_JOURNAL_MAX_MIB` (1024) the file size at which recording stops. Deleting the file loses nothing but the head start. With `APS5_PROFILE_DRAW=1` a `[pipeline-journal]` line reports what was loaded, replayed and recorded.
+Besides the recompiled shaders, the shader cache directory holds `pipelines.journal`: every Vulkan graphics pipeline the pipeline store kept and every compute pipeline a run built, with the specialized shader modules they were built from. The next run of the title builds those pipelines on background threads while it loads, so a draw or dispatch the earlier run already made does not compile its pipeline when it is first reached. The journal belongs to the GPU and driver that wrote it, like the driver's pipeline cache file next to it: a run on another GPU or driver version starts it over, and the driver's own pipeline cache decides how fast each pipeline is rebuilt. Two runs sharing the directory take turns on `pipelines.journal.lock`; a run that cannot lock it records nothing. `APS5_NO_PIPELINE_JOURNAL=1` disables recording and replay; `APS5_PIPELINE_JOURNAL_THREADS` sets the replay threads (half the CPU threads by default; `0` records without replaying), `APS5_PIPELINE_JOURNAL_ENTRIES` how many pipelines are replayed (4096, the last recorded) and `APS5_PIPELINE_STORE_ENTRIES` how many graphics pipelines stay resident (by default the store grows from 1024 up to 4096 to hold the replay and the pipelines that keep coming back), and `APS5_PIPELINE_JOURNAL_MAX_MIB` (1024) the file size at which recording stops. Deleting the file loses nothing but the head start. With `APS5_PROFILE_DRAW=1` a `[pipeline-journal]` line reports what was loaded, replayed and recorded.
 
 ### Draw pipeline