diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 3b02737fc2..04b0e5a7aa 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -2541,6 +2541,12 @@ if(BUILD_TESTING) add_test(NAME agc_shader_frontend_pair_off COMMAND agc_shader_memory_tests --frontend-pair) set_tests_properties(agc_shader_frontend_pair PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") set_tests_properties(agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_FRONTEND_PAIR=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_result_memo_budget COMMAND agc_shader_memory_tests --result-memo-budget) + set_tests_properties(agc_shader_result_memo_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_MASKED_RESULT_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_result_memo_budget_threads COMMAND agc_shader_memory_tests --result-memo-budget-threads) + set_tests_properties(agc_shader_result_memo_budget_threads PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_MASKED_RESULT_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_result_memo_budget_masked COMMAND agc_shader_memory_tests --result-memo-budget-masked) + set_tests_properties(agc_shader_result_memo_budget_masked PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_shader_layout_pair COMMAND agc_shader_memory_tests --layout-pair) add_test(NAME agc_shader_layout_pair_off COMMAND agc_shader_memory_tests --layout-pair) set_tests_properties(agc_shader_layout_pair PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") @@ -2548,7 +2554,7 @@ if(BUILD_TESTING) add_test(NAME agc_shader_entry_lane_proof_budget COMMAND agc_shader_memory_tests --entry-lane-proof-budget) set_tests_properties(agc_shader_entry_lane_proof_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") if(WIN32) - set_tests_properties(agc_shader_bindless agc_shader_selected_records_image_limit agc_shader_selected_records_pair_limit agc_shader_selected_records_direct_image_limit agc_shader_selected_records_direct_pair_limit agc_shader_srt_read_patching agc_shader_type_zero_image_materialization agc_shader_frontend_pair agc_shader_frontend_pair_off agc_shader_layout_pair agc_shader_layout_pair_off agc_shader_entry_lane_proof_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + set_tests_properties(agc_shader_bindless agc_shader_selected_records_image_limit agc_shader_selected_records_pair_limit agc_shader_selected_records_direct_image_limit agc_shader_selected_records_direct_pair_limit agc_shader_srt_read_patching agc_shader_type_zero_image_materialization agc_shader_frontend_pair agc_shader_frontend_pair_off agc_shader_layout_pair agc_shader_layout_pair_off agc_shader_result_memo_budget agc_shader_result_memo_budget_threads agc_shader_result_memo_budget_masked agc_shader_entry_lane_proof_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() foreach(agcTest IN ITEMS agc_vertex_input_tests agc_mesh_output_budget_tests agc_shared_write_tests agc_locks_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vertex_input_validation_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_barrier_batch_tests agc_shader_disk_cache_tests agc_screenshot_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests agc_formatted_buffer_byte_stores_tests ${agcExecutionTests} agc_fma_mix_once_tests agc_f16_single_convert_tests) if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests" OR agcTest STREQUAL "agc_driver_pipeline_optimized_readback_tests" OR agcTest STREQUAL "agc_driver_rcp_hardware_tests") diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 6ee1fdace0..2a31594300 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -436,6 +437,90 @@ void verifySpecializedModuleRelease() { require(shared.spirv.data() == again.spirv.data(), "specialized module release: a second result of a live module copied its words"); } +const std::vector& resultMemoPixelCode() { + static const std::vector code{0xbf800000u, 0x7e000200u, 0xf800180fu, 0u, 0xbf810000u}; + return code; +} + +bool compileResultMemoPixel(std::uint32_t value) { + using namespace ShaderRecompiler; + std::array color{value}; + auto request = FrontendRequest(resultMemoPixelCode(), color); + request.shader.stage = ShaderStage::Fragment; + request.context.compute.reset(); + request.context.pixel = ShaderPixelStageInfo{}; + request.context.pixel->wave32 = true; + request.context.pixel->targetOutputMode[0] = 9u; + request.context.waveSize = 32u; + request.useCache = true; + AgcDriver::ShaderMemory memory({}); + const auto capture = memory.Capture(request); + const auto regions = memory.Regions(); + request.context.memory = regions; + bool hit = false; + static_cast(Recompile(request, *capture, &hit)); + return hit; +} + +void verifyResultMemoBudgetThreads() { + using namespace ShaderRecompiler; + require(!compileResultMemoPixel(1u) && compileResultMemoPixel(1u), "result memo budget threads: the memo did not keep a result"); + const auto one = ResultMemoBytes(); + const auto budget = 3u * one; + SetResultMemoBudget(budget); + static_cast(compileResultMemoPixel(2u)); + static_cast(compileResultMemoPixel(3u)); + require(ResultMemoBytes() == budget, "result memo budget threads: three results did not fill a budget of three"); + std::vector late; + SetResultMemoTrimHook([&] { + const auto before = ResultMemoBytes(); + for (const auto value : {5u, 6u}) late.emplace_back([value] { static_cast(compileResultMemoPixel(value)); }); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (ResultMemoBytes() < before + 2u * one && std::chrono::steady_clock::now() < deadline) std::this_thread::yield(); + }); + static_cast(compileResultMemoPixel(4u)); + for (auto& thread : late) thread.join(); + require(late.size() == 2u, "result memo budget threads: the fourth result did not start a trim"); + require(ResultMemoBytes() <= budget, "result memo budget threads: results inserted during a trim left the memo over its budget"); + SetResultMemoBudget(0u); +} + +void verifyResultMemoBudgetMasked() { + using namespace ShaderRecompiler; + require(ResultMemoBytes() == 0u, "result memo budget masked: the memo held results before the first compile"); + require(!compileResultMemoPixel(1u), "result memo budget masked: the first compile hit the memo"); + const auto first = ResultMemoBytes(); + require(!compileResultMemoPixel(2u), "result memo budget masked: a new snapshot hit the memo"); + const auto second = ResultMemoBytes() - first; + require(second != 0u && second < first, std::string("result memo budget masked: the masked table's entry was not counted (first result " + std::to_string(first) + " bytes, patched result " + std::to_string(second) + ")").c_str()); + SetResultMemoBudget(1u); + require(ResultMemoBytes() == 0u, std::string("result memo budget masked: a trim to one byte left " + std::to_string(ResultMemoBytes()) + " bytes").c_str()); + SetResultMemoBudget(0u); + require(!compileResultMemoPixel(3u), "result memo budget masked: a snapshot after the trim hit the memo"); + require(ResultMemoBytes() == first, std::string("result memo budget masked: the trim left a masked entry to patch from (" + std::to_string(ResultMemoBytes()) + " bytes after one compile, " + std::to_string(first) + " for a full one)").c_str()); +} + +void verifyResultMemoBudget() { + using namespace ShaderRecompiler; + const auto compile = [](std::uint32_t value) { return compileResultMemoPixel(value); }; + require(ResultMemoBytes() == 0u, "result memo budget: the memo held results before the first compile"); + require(!compile(1u) && compile(1u), "result memo budget: the memo did not keep a result"); + const auto one = ResultMemoBytes(); + require(one != 0u, "result memo budget: a memo'd result counted no bytes"); + const auto budget = 3u * one; + SetResultMemoBudget(budget); + for (const auto value : {2u, 3u, 4u}) { + require(!compile(value), "result memo budget: a new snapshot hit the memo"); + require(ResultMemoBytes() <= budget, "result memo budget: the memo holds more than its budget"); + } + require(ResultMemoBytes() < budget, "result memo budget: a fourth result over a budget of three was not trimmed"); + require(compile(4u) && compile(3u), "result memo budget: the most recently used results were dropped"); + require(!compile(1u), "result memo budget: the least recently used result outlived the trim"); + SetResultMemoBudget(0u); + for (const auto value : {5u, 6u, 7u}) static_cast(compile(value)); + require(ResultMemoBytes() > budget, "result memo budget: clearing the budget kept bounding the memo"); +} + ShaderRecompiler::RecompileRequest LayoutPairRequest(std::span code, std::span userData, bool registration, std::uint32_t wordLimit) { using namespace ShaderRecompiler; auto request = FrontendRequest(code, userData); @@ -8485,6 +8570,21 @@ int main(int argc, char** argv) { std::cout << "Prepared frontend reuse preserves full and partial-thread programs\n"; return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--result-memo-budget") { + verifyResultMemoBudget(); + std::cout << "The result memo keeps its most recently used results within its byte budget\n"; + return 0; + } + if (argc == 2 && std::string_view(argv[1]) == "--result-memo-budget-masked") { + verifyResultMemoBudgetMasked(); + std::cout << "The result memo budget counts and trims the masked table's entries\n"; + return 0; + } + if (argc == 2 && std::string_view(argv[1]) == "--result-memo-budget-threads") { + verifyResultMemoBudgetThreads(); + std::cout << "The result memo ends within its byte budget when results are inserted during a trim\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--layout-pair") { verifyLayoutPair(); std::cout << "Both push-constant layouts of a pixel shader compile from one program unchanged\n"; diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 5d4ecc297a..3aa81cb5d7 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -369,6 +369,7 @@ struct ResultMemoEntry { std::shared_ptr result; std::uint64_t used = 0; std::shared_ptr patch; + std::uint64_t bytes = 0; }; struct EmissionFailure { @@ -399,7 +400,6 @@ struct SourceEntry : std::enable_shared_from_this { std::vector emissionFailures; std::unique_ptr memo; std::unique_ptr maskedMemo; - std::uint64_t memoClock = 0; }; namespace { @@ -1455,8 +1455,9 @@ ResultMemoEntry* memoSet(const std::unique_ptr& table, std::uin } struct ResultMemoCounters { - std::atomic hits{0}, patched{0}, misses{0}, evictions{0}, populateNanoseconds{0}, patchNanoseconds{0}, mismatches{0}; + std::atomic hits{0}, patched{0}, misses{0}, evictions{0}, trimmed{0}, populateNanoseconds{0}, patchNanoseconds{0}, mismatches{0}; std::atomic lastReport{0}; + std::atomic clock{0}, heldBytes{0}, heldEntries{0}; }; void applyMemoPatch(RecompileResult& result, const MemoPatch& patch, const ResourceSnapshot& snapshot) { @@ -1503,6 +1504,123 @@ ResultMemoCounters& resultMemoCounters() { return counters; } +std::uint64_t nextMemoUse() { + return resultMemoCounters().clock.fetch_add(1, std::memory_order_relaxed) + 1u; +} + +template +std::uint64_t vectorBytes(const std::vector& values) { + return static_cast(values.capacity()) * sizeof(T); +} + +std::uint64_t vectorBytes(const std::vector& values) { + return static_cast(values.capacity() + 7u) / 8u; +} + +std::uint64_t descriptorBindingBytes(const DescriptorBinding& binding) { + return vectorBytes(binding.guestDescriptor) + vectorBytes(binding.imageShapes) + vectorBytes(binding.imageSamplers) + vectorBytes(binding.bufferExtent) + vectorBytes(binding.samplerDepthCompare) + vectorBytes(binding.imageWritten) + vectorBytes(binding.imageDepthCompare) + vectorBytes(binding.imageAtomic) + vectorBytes(binding.imageAtomic64) + vectorBytes(binding.bufferAtomic) + vectorBytes(binding.bufferWritten) + vectorBytes(binding.samplerUnnormalized) + vectorBytes(binding.samplerIntegerBorder) + vectorBytes(binding.samplerCoordinatesIgnored) + vectorBytes(binding.imageUnnormalized) + vectorBytes(binding.imageUnnormalizedBaseLevel) + vectorBytes(binding.bufferRead); +} + +std::uint64_t resultBytes(const RecompileResult& result) { + const auto bytes = [](const auto& values) { return vectorBytes(values); }; + auto total = static_cast(sizeof(RecompileResult)) + bytes(result.vertexInputPatches) + bytes(result.runtimeImageResources) + bytes(result.vertexInputs) + bytes(result.parameterExports) + bytes(result.fragmentParameters) + bytes(result.stencilReferences) + bytes(result.shaderDataUserWords) + bytes(result.specialization) + bytes(result.bindings) + bytes(result.pushConstants) + bytes(result.vertexAttributes) + bytes(result.imageTablePoison) + bytes(result.imageTableRanges) + bytes(result.imageTableReadRanges); + for (const auto& ranges : result.imageTableReadRanges) total += bytes(ranges); + for (const auto& binding : result.bindings) total += descriptorBindingBytes(binding); + return total; +} + +std::uint64_t patchBytes(const MemoPatch& patch) { + auto total = static_cast(sizeof(MemoPatch)) + vectorBytes(patch.buffers) + vectorBytes(patch.userWords); + for (const auto& buffer : patch.buffers) total += vectorBytes(buffer.second); + return total; +} + +std::atomic& resultMemoBudgetBytes() { + static std::atomic budget{[] { + const char* value = std::getenv("APS5_RESULT_MEMO_MB"); + const std::uint64_t megabytes = value == nullptr || *value == '\0' ? 256u : std::strtoull(value, nullptr, 10); + return megabytes << 20u; + }()}; + return budget; +} + +std::uint64_t resultMemoBudget() { + return resultMemoBudgetBytes().load(std::memory_order_relaxed); +} + +struct ResultMemoRegistry { + std::mutex mutex; + std::vector> sources; +}; + +ResultMemoRegistry& resultMemoRegistry() { + static ResultMemoRegistry registry; + return registry; +} + +std::mutex& resultMemoTrimMutex() { + static std::mutex mutex; + return mutex; +} + +std::function& resultMemoTrimHook() { + static std::function hook; + return hook; +} + +void trimResultMemo(std::uint64_t budget) { + std::lock_guard trim(resultMemoTrimMutex()); + auto& counters = resultMemoCounters(); + for (auto held = counters.heldBytes.load(std::memory_order_relaxed); held > budget; held = counters.heldBytes.load(std::memory_order_relaxed)) { + if (auto hook = std::exchange(resultMemoTrimHook(), nullptr)) hook(); + std::vector> sources; + { + auto& registry = resultMemoRegistry(); + std::lock_guard lock(registry.mutex); + sources.reserve(registry.sources.size()); + for (const auto& weak : registry.sources) { + if (auto source = weak.lock()) sources.push_back(std::move(source)); + } + } + std::vector> stamps; + for (const auto& source : sources) { + std::lock_guard lock(source->mutex); + for (const auto* table : {source->memo.get(), source->maskedMemo.get()}) { + if (table == nullptr) continue; + for (const auto& entry : *table) { + if (entry.result != nullptr) stamps.emplace_back(entry.used, entry.bytes); + } + } + } + if (stamps.empty()) return; + std::sort(stamps.begin(), stamps.end()); + const auto excess = held - budget / 4u * 3u; + std::uint64_t freed = 0; + std::uint64_t threshold = 0; + for (const auto& [used, bytes] : stamps) { + threshold = used; + freed += bytes; + if (freed >= excess) break; + } + std::uint64_t dropped = 0; + for (const auto& source : sources) { + std::lock_guard lock(source->mutex); + for (auto* table : {source->memo.get(), source->maskedMemo.get()}) { + if (table == nullptr) continue; + for (auto& entry : *table) { + if (entry.result == nullptr || entry.used > threshold) continue; + counters.heldBytes.fetch_sub(entry.bytes, std::memory_order_relaxed); + counters.heldEntries.fetch_sub(1, std::memory_order_relaxed); + counters.trimmed.fetch_add(1, std::memory_order_relaxed); + entry = {}; + ++dropped; + } + } + } + if (dropped == 0) return; + } +} + void verifyPatchedResult(const RecompileResult& patched, const RecompileResult& full, std::uint64_t variantId) { const auto* difference = patchedResultDifference(patched, full); if (difference == nullptr) return; @@ -1526,11 +1644,12 @@ void reportResultMemo() { const auto patched = counters.patched.exchange(0, std::memory_order_relaxed); const auto misses = counters.misses.exchange(0, std::memory_order_relaxed); const auto evictions = counters.evictions.exchange(0, std::memory_order_relaxed); + const auto trimmed = counters.trimmed.exchange(0, std::memory_order_relaxed); const auto populate = counters.populateNanoseconds.exchange(0, std::memory_order_relaxed); const auto patching = counters.patchNanoseconds.exchange(0, std::memory_order_relaxed); const auto mismatches = counters.mismatches.exchange(0, std::memory_order_relaxed); const auto total = static_cast(hits + patched + misses); - std::fprintf(stderr, "[recompile] result memo (10 s): %llu hits, %llu patched, %llu misses (%.1f%% hits, %.1f%% patched), Populate %.1f us per miss / %.1f ms in total, patch %.1f us each / %.1f ms in total, %llu evictions, %llu patch mismatches\n", static_cast(hits), static_cast(patched), static_cast(misses), total != 0.0 ? 100.0 * static_cast(hits) / total : 0.0, total != 0.0 ? 100.0 * static_cast(patched) / total : 0.0, misses != 0 ? static_cast(populate) / 1000.0 / static_cast(misses) : 0.0, static_cast(populate) / 1e6, patched != 0 ? static_cast(patching) / 1000.0 / static_cast(patched) : 0.0, static_cast(patching) / 1e6, static_cast(evictions), static_cast(mismatches)); + std::fprintf(stderr, "[recompile] result memo (10 s): %llu hits, %llu patched, %llu misses (%.1f%% hits, %.1f%% patched), Populate %.1f us per miss / %.1f ms in total, patch %.1f us each / %.1f ms in total, %llu evictions, %llu patch mismatches, %llu trimmed; %llu results held, %.1f MiB\n", static_cast(hits), static_cast(patched), static_cast(misses), total != 0.0 ? 100.0 * static_cast(hits) / total : 0.0, total != 0.0 ? 100.0 * static_cast(patched) / total : 0.0, misses != 0 ? static_cast(populate) / 1000.0 / static_cast(misses) : 0.0, static_cast(populate) / 1e6, patched != 0 ? static_cast(patching) / 1000.0 / static_cast(patched) : 0.0, static_cast(patching) / 1e6, static_cast(evictions), static_cast(mismatches), static_cast(trimmed), static_cast(counters.heldEntries.load(std::memory_order_relaxed)), static_cast(counters.heldBytes.load(std::memory_order_relaxed)) / 1048576.0); } // Everything materializeResult reads besides the variant: the snapshot (the descriptor words, the @@ -1644,7 +1763,7 @@ std::shared_ptr materializePreparedMemoized(SourceEntry& for (std::size_t way = 0; way < ResultMemoWays; ++way) { auto& entry = set[way]; if (entry.result == nullptr || entry.variantId != variantId || entry.hash != hash) continue; - entry.used = ++source.memoClock; + entry.used = nextMemoUse(); counters.hits.fetch_add(1, std::memory_order_relaxed); if (memoHit != nullptr) *memoHit = true; reportResultMemo(); @@ -1663,7 +1782,7 @@ std::shared_ptr materializePreparedMemoized(SourceEntry& for (std::size_t way = 0; way < ResultMemoWays; ++way) { auto& entry = set[way]; if (entry.result == nullptr || entry.variantId != variantId || entry.hash != maskedHash) continue; - entry.used = ++source.memoClock; + entry.used = nextMemoUse(); donor = entry.result; donorPatch = entry.patch; break; @@ -1687,20 +1806,35 @@ std::shared_ptr materializePreparedMemoized(SourceEntry& } result->cacheHit = cacheHit; std::shared_ptr shared = std::move(result); + const auto bytes = resultBytes(*shared); const auto insert = [&](std::unique_ptr& table, std::uint64_t at, std::uint64_t key, std::shared_ptr recipe, bool count) { - if (table == nullptr) table = std::make_unique(); + if (table == nullptr) { + table = std::make_unique(); + if (source.memo == nullptr || source.maskedMemo == nullptr) { + auto& registry = resultMemoRegistry(); + std::lock_guard registryLock(registry.mutex); + registry.sources.push_back(source.weak_from_this()); + } + } auto* set = memoSet(table, at); auto* victim = set; for (std::size_t way = 0; way < ResultMemoWays; ++way) { auto& entry = set[way]; if (entry.result != nullptr && entry.variantId == variantId && entry.hash == key) { - entry.used = ++source.memoClock; + entry.used = nextMemoUse(); return entry.result; } if (entry.used < victim->used) victim = &entry; } - if (count && victim->result != nullptr) counters.evictions.fetch_add(1, std::memory_order_relaxed); - *victim = {variantId, key, shared, ++source.memoClock, std::move(recipe)}; + if (victim->result != nullptr) { + if (count) counters.evictions.fetch_add(1, std::memory_order_relaxed); + counters.heldBytes.fetch_sub(victim->bytes, std::memory_order_relaxed); + counters.heldEntries.fetch_sub(1, std::memory_order_relaxed); + } + const auto held = bytes + (recipe != nullptr ? patchBytes(*recipe) : 0u); + *victim = {variantId, key, shared, nextMemoUse(), std::move(recipe), held}; + counters.heldBytes.fetch_add(held, std::memory_order_relaxed); + counters.heldEntries.fetch_add(1, std::memory_order_relaxed); return shared; }; { @@ -1708,6 +1842,7 @@ std::shared_ptr materializePreparedMemoized(SourceEntry& shared = insert(source.memo, index, hash, nullptr, true); if (patch != nullptr && patch->valid) static_cast(insert(source.maskedMemo, maskedIndex, maskedHash, patch, false)); } + if (const auto budget = resultMemoBudget(); budget != 0 && counters.heldBytes.load(std::memory_order_relaxed) > budget) trimResultMemo(budget); reportResultMemo(); return shared; } @@ -1940,4 +2075,18 @@ std::shared_ptr Recompile(const RecompileRequest& request return recompileReporting(request, [&] { return RecompileImpl(request, capture, memoHit); }); } +void SetResultMemoBudget(std::uint64_t bytes) { + resultMemoBudgetBytes().store(bytes, std::memory_order_relaxed); + if (bytes != 0 && resultMemoCounters().heldBytes.load(std::memory_order_relaxed) > bytes) trimResultMemo(bytes); +} + +void SetResultMemoTrimHook(std::function hook) { + std::lock_guard lock(resultMemoTrimMutex()); + resultMemoTrimHook() = std::move(hook); +} + +std::uint64_t ResultMemoBytes() { + return resultMemoCounters().heldBytes.load(std::memory_order_relaxed); +} + } diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 212e6b2f8d..0f51f9a795 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -826,6 +827,9 @@ struct MeshOutputBudget { struct ResourceCapture; [[nodiscard]] std::shared_ptr Recompile(const RecompileRequest& request, const ResourceCapture& capture, bool* memoHit = nullptr); +void SetResultMemoBudget(std::uint64_t bytes); +[[nodiscard]] std::uint64_t ResultMemoBytes(); +void SetResultMemoTrimHook(std::function hook); // Debug aid (see DebugProbe in Translation/TranslationContext.hpp): the APS5_PROBE register probe is // only applied while a driver has it active, so it can be limited to one dispatch; the recompile diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index aca1cb4eea..ce6863e337 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -1380,6 +1380,7 @@ constexpr std::string_view NeutralSwitches[] = { "APS5_NO_RESULT_MEMO", "APS5_NO_TABLE_RESOLUTION_REUSE", "APS5_NO_TABLE_SNAPSHOT_REUSE", + "APS5_RESULT_MEMO_MB", }; const std::vector& switchKey() { diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index d54b9c8fd1..91f7fdd94d 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -51,6 +51,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_WINDOW_NARROWING` | presence | `core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp` | | indexes every slot of the shader's vector window instead of only the slots the program uses | | `APS5_PROBE` | string | `core/shader/recompiler/Translation/src/DispatchInstructions.cpp` | | pc:vgpr[:shift] probe that copies that VGPR into v255 after the instruction so image stores write it as a per-lane trace | | `APS5_RAYTRACING` | string | `core/shader/recompiler/Translation/src/DispatchInstructions.cpp` | | selects the image_bvh_intersect_ray behaviour: strict throws and miss writes 0xffffffff | +| `APS5_RESULT_MEMO_MB` | integer | `core/shader/recompiler/Recompiler.cpp` | 256 | MiB the result memos of all shader sources may hold before the least recently used results are dropped (0 keeps them until their set evicts them) | | `APS5_SAVEEXEC_WRITE_FIRST` | presence | `core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp` | | writes the saved exec to the destination before reading the source in s_*_saveexec | | `APS5_SINGLE_LANE` | string | `core/shader/recompiler/Recompiler.cpp` | | hex code addresses (comma separated, or all) of wave64 programs kept at one guest lane per invocation | | `APS5_SRT_DEBUG` | presence | `core/shader/recompiler/Optimization/src/RequestMemoryView.cpp` | | prints extra SRT descriptor-walk debug output, such as guest reads outside the request memory |