mirror of
https://github.com/boykopovar/AnyPS5.git
synced 2026-10-27 00:28:14 +00:00
Merge pull request #10423 from meh7an/perf/shader-masked-result-memo
perf(shader): reuse a memoized result by patching its copied words
This commit is contained in:
4 files changed
+186
-38
No files matched your search
@@ -2478,12 +2478,14 @@ if(BUILD_TESTING)
|
||||
set_tests_properties(agc_shader_srt_read_patching PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
|
||||
add_test(NAME agc_shader_table_snapshot_reuse COMMAND agc_shader_memory_tests --table-snapshot-reuse)
|
||||
set_tests_properties(agc_shader_table_snapshot_reuse PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
|
||||
add_test(NAME agc_shader_table_snapshot_reuse_unmasked COMMAND agc_shader_memory_tests --table-snapshot-reuse)
|
||||
set_tests_properties(agc_shader_table_snapshot_reuse_unmasked PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_MASKED_RESULT_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
|
||||
add_test(NAME agc_driver_image_table_vouched_lookups COMMAND agc_driver_image_table_cache_bounds_tests --vouched-lookups)
|
||||
set_tests_properties(agc_driver_image_table_vouched_lookups PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_WRITE_WATCH_IMPORTS=watch" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60)
|
||||
add_test(NAME agc_driver_image_table_vouch_queued_draws COMMAND agc_driver_image_table_cache_bounds_tests --queued-draws)
|
||||
set_tests_properties(agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60)
|
||||
if(WIN32)
|
||||
set_tests_properties(agc_shader_table_snapshot_reuse agc_driver_image_table_vouched_lookups agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
|
||||
set_tests_properties(agc_shader_table_snapshot_reuse agc_shader_table_snapshot_reuse_unmasked agc_driver_image_table_vouched_lookups agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
|
||||
endif()
|
||||
add_test(NAME agc_shader_type_zero_image_materialization COMMAND agc_shader_memory_tests --type-zero-image-materialization)
|
||||
set_tests_properties(agc_shader_type_zero_image_materialization PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
|
||||
|
||||
@@ -2339,7 +2339,10 @@ void verifyTableSnapshotReuse() {
|
||||
const auto otherBuffer = CompileTable(request);
|
||||
std::copy(outputV.begin(), outputV.end(), guest.srt.begin() + 12);
|
||||
const auto resolvedShared = ResourceMaterializer::TableReuse();
|
||||
require(otherBuffer.result != compiled.result && otherBuffer.result->variantId == compiled.result->variantId && resolvedShared.resolutionsShared - resolvedBefore.resolutionsShared == 1u && resolvedShared.resolutions == resolvedBefore.resolutions && TableMap(*otherBuffer.result) == map && TableBinding(*otherBuffer.result, DescriptorRole::ImageTable)->guestDescriptor == images->guestDescriptor, "table snapshot reuse: a result for another buffer did not reuse the unchanged table's resolution");
|
||||
const bool resolves = std::getenv("APS5_NO_MASKED_RESULT_MEMO") != nullptr || std::getenv("APS5_VERIFY_MASKED_RESULT_MEMO") != nullptr;
|
||||
require(otherBuffer.result != compiled.result && otherBuffer.result->variantId == compiled.result->variantId && resolvedShared.resolutionsShared - resolvedBefore.resolutionsShared == (resolves ? 1u : 0u) && resolvedShared.resolutions == resolvedBefore.resolutions && TableMap(*otherBuffer.result) == map && TableBinding(*otherBuffer.result, DescriptorRole::ImageTable)->guestDescriptor == images->guestDescriptor, "table snapshot reuse: a result for another buffer did not reuse the unchanged table's resolution");
|
||||
const auto* otherBuffers = TableBinding(*otherBuffer.result, DescriptorRole::GuestBuffers);
|
||||
require(otherBuffers != nullptr && std::search(otherBuffers->guestDescriptor.begin(), otherBuffers->guestDescriptor.end(), otherOutputV.begin(), otherOutputV.end()) != otherBuffers->guestDescriptor.end(), "table snapshot reuse: a result for another buffer does not hold that buffer's descriptor");
|
||||
const auto storedWord = heap[5u * 8u];
|
||||
heap[5u * 8u] = heap[6u * 8u];
|
||||
const auto restoredMap = map;
|
||||
|
||||
@@ -353,11 +353,22 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request, ShaderPreparat
|
||||
// A materialized result of one variant over one snapshot (Recompile(request, capture)): the
|
||||
// shared immutable object every later capture that reproduces the snapshot receives, so Populate
|
||||
// and the per-request copy run once per distinct snapshot.
|
||||
struct MemoPatch {
|
||||
static constexpr std::uint32_t NoBinding = ~0u;
|
||||
std::vector<std::pair<std::uint32_t, std::vector<std::uint32_t>>> buffers;
|
||||
std::vector<std::uint32_t> userWords;
|
||||
std::uint32_t shaderData = NoBinding;
|
||||
std::uint32_t flattenedSrt = NoBinding;
|
||||
bool pushData = false;
|
||||
bool valid = false;
|
||||
};
|
||||
|
||||
struct ResultMemoEntry {
|
||||
std::uint64_t variantId = 0;
|
||||
std::uint64_t hash = 0;
|
||||
std::shared_ptr<const RecompileResult> result;
|
||||
std::uint64_t used = 0;
|
||||
std::shared_ptr<const MemoPatch> patch;
|
||||
};
|
||||
|
||||
struct EmissionFailure {
|
||||
@@ -387,6 +398,7 @@ struct SourceEntry : std::enable_shared_from_this<SourceEntry> {
|
||||
std::vector<std::shared_ptr<const CompiledVariant>> variants;
|
||||
std::vector<EmissionFailure> emissionFailures;
|
||||
std::unique_ptr<ResultMemoTable> memo;
|
||||
std::unique_ptr<ResultMemoTable> maskedMemo;
|
||||
std::uint64_t memoClock = 0;
|
||||
};
|
||||
|
||||
@@ -1008,7 +1020,28 @@ const ResourceSnapshot& mergeTargetImage(const CompiledVariant& variant, const R
|
||||
return merged;
|
||||
}
|
||||
|
||||
RecompileResult materializeResult(const CompiledVariant& variant, const RecompileRequest& request, const ResourceSnapshot& captured, bool registrationDefaults = false) {
|
||||
void recordMemoPatch(MemoPatch& patch, const DescriptorBindingPlan& plan, const CompiledVariant& variant) {
|
||||
const auto& layout = variant.bindings.layout;
|
||||
patch = {};
|
||||
for (std::size_t index = 0; index < plan.bindings.size(); ++index) {
|
||||
const auto& binding = plan.bindings[index];
|
||||
const auto position = static_cast<std::uint32_t>(index);
|
||||
if (binding.descriptor.role == DescriptorRole::GuestBuffers) {
|
||||
patch.buffers.push_back({position, binding.resources});
|
||||
} else if (binding.descriptor.role == DescriptorRole::FlattenedSrt) {
|
||||
patch.flattenedSrt = position;
|
||||
} else if (binding.descriptor.role == DescriptorRole::ShaderData) {
|
||||
patch.shaderData = position;
|
||||
}
|
||||
}
|
||||
patch.pushData = layout.UsesPushData();
|
||||
if (patch.pushData || plan.storageData) {
|
||||
for (const auto reg : layout.userDataRegisters) patch.userWords.push_back(reg - variant.info.userDataBase);
|
||||
}
|
||||
patch.valid = layout.userDataRegisters.size() <= layout.DispatchThreadLimitDword() && (patch.userWords.empty() || patch.pushData || patch.shaderData != MemoPatch::NoBinding);
|
||||
}
|
||||
|
||||
RecompileResult materializeResult(const CompiledVariant& variant, const RecompileRequest& request, const ResourceSnapshot& captured, bool registrationDefaults = false, MemoPatch* patch = nullptr) {
|
||||
ResourceSnapshot merged;
|
||||
const auto& snapshot = mergeTargetImage(variant, request, captured, registrationDefaults, merged);
|
||||
RecompileResult result;
|
||||
@@ -1162,6 +1195,7 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil
|
||||
const auto& tables = *resolved;
|
||||
BindingAllocationResult bindings;
|
||||
DescriptorBindingBuilder{}.Populate(bindings, variant.bindings, *bindingPlan, variant.info.userDataBase, snapshot, partialThreads(request), &tables);
|
||||
if (patch != nullptr) recordMemoPatch(*patch, *bindingPlan, variant);
|
||||
result.workgroupMemoryDwords = WorkgroupMemoryStrideDwords(variant.info.info);
|
||||
result.bindings = std::move(bindings.bindings);
|
||||
if (std::ranges::none_of(result.bindings, [](const DescriptorBinding& binding) { return binding.role == DescriptorRole::BdaPagetable || binding.role == DescriptorRole::FaultBuffer; })) result.bdaAbiVersion = 0u;
|
||||
@@ -1357,21 +1391,78 @@ bool ResultMemo() {
|
||||
return resultMemo;
|
||||
}
|
||||
|
||||
ResultMemoEntry* memoSet(SourceEntry& source, std::uint64_t index) {
|
||||
if (source.memo == nullptr) return nullptr;
|
||||
return source.memo->data() + static_cast<std::size_t>((index ^ (index >> 32u)) % ResultMemoSets) * ResultMemoWays;
|
||||
bool MaskedResultMemo() {
|
||||
static const bool masked = std::getenv("APS5_NO_MASKED_RESULT_MEMO") == nullptr;
|
||||
return masked;
|
||||
}
|
||||
|
||||
bool VerifyMaskedResultMemo() {
|
||||
static const bool verify = std::getenv("APS5_VERIFY_MASKED_RESULT_MEMO") != nullptr;
|
||||
return verify;
|
||||
}
|
||||
|
||||
ResultMemoEntry* memoSet(const std::unique_ptr<ResultMemoTable>& table, std::uint64_t index) {
|
||||
if (table == nullptr) return nullptr;
|
||||
return table->data() + static_cast<std::size_t>((index ^ (index >> 32u)) % ResultMemoSets) * ResultMemoWays;
|
||||
}
|
||||
|
||||
struct ResultMemoCounters {
|
||||
std::atomic<std::uint64_t> hits{0}, misses{0}, evictions{0}, populateNanoseconds{0};
|
||||
std::atomic<std::uint64_t> hits{0}, patched{0}, misses{0}, evictions{0}, populateNanoseconds{0}, patchNanoseconds{0}, mismatches{0};
|
||||
std::atomic<std::int64_t> lastReport{0};
|
||||
};
|
||||
|
||||
void applyMemoPatch(RecompileResult& result, const MemoPatch& patch, const ResourceSnapshot& snapshot) {
|
||||
for (const auto& [binding, resources] : patch.buffers) {
|
||||
auto& words = result.bindings.at(binding).guestDescriptor;
|
||||
if (words.size() != resources.size() * 4u) throw std::runtime_error("a patched guest buffer binding changed its width");
|
||||
for (std::size_t element = 0; element < resources.size(); ++element) std::copy_n(snapshot.buffers.at(resources[element]).dwords.begin(), 4u, words.begin() + element * 4u);
|
||||
}
|
||||
if (!patch.userWords.empty()) {
|
||||
if (patch.pushData) {
|
||||
if (patch.userWords.size() * sizeof(std::uint32_t) > result.pushConstants.size()) throw std::runtime_error("patched user data lies outside the push data");
|
||||
for (std::size_t word = 0; word < patch.userWords.size(); ++word) std::memcpy(result.pushConstants.data() + word * sizeof(std::uint32_t), &snapshot.userData.at(patch.userWords[word]), sizeof(std::uint32_t));
|
||||
} else {
|
||||
auto& words = result.bindings.at(patch.shaderData).guestDescriptor;
|
||||
if (patch.userWords.size() > words.size()) throw std::runtime_error("patched user data lies outside the shader data");
|
||||
for (std::size_t word = 0; word < patch.userWords.size(); ++word) words[word] = snapshot.userData.at(patch.userWords[word]);
|
||||
}
|
||||
}
|
||||
if (patch.flattenedSrt != MemoPatch::NoBinding) result.bindings.at(patch.flattenedSrt).guestDescriptor = snapshot.flattenedSrt;
|
||||
result.poisonedSrtReads = static_cast<std::uint32_t>(snapshot.srtPoison.size()) + snapshot.nullRootReads;
|
||||
}
|
||||
|
||||
const char* patchedResultDifference(const RecompileResult& patched, const RecompileResult& full) {
|
||||
if (patched.bindings.size() != full.bindings.size()) return "binding count";
|
||||
if (patched.pushConstants != full.pushConstants) return "push constants";
|
||||
if (patched.specializationId != full.specializationId || patched.spirv.ContentHash() != full.spirv.ContentHash()) return "module";
|
||||
if (patched.poisonedSrtReads != full.poisonedSrtReads) return "poisoned SRT reads";
|
||||
if (patched.shaderDataUserWords != full.shaderDataUserWords) return "shader data user words";
|
||||
if (patched.specialization.size() != full.specialization.size()) return "specialization";
|
||||
for (std::size_t index = 0; index < patched.specialization.size(); ++index) {
|
||||
if (patched.specialization[index].id != full.specialization[index].id || patched.specialization[index].value != full.specialization[index].value) return "specialization";
|
||||
}
|
||||
for (std::size_t index = 0; index < patched.bindings.size(); ++index) {
|
||||
const auto& left = patched.bindings[index];
|
||||
const auto& right = full.bindings[index];
|
||||
if (left.role != right.role || left.binding != right.binding || left.count != right.count) return "binding shape";
|
||||
if (left.guestDescriptor != right.guestDescriptor) return "binding words";
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ResultMemoCounters& resultMemoCounters() {
|
||||
static ResultMemoCounters counters;
|
||||
return counters;
|
||||
}
|
||||
|
||||
void verifyPatchedResult(const RecompileResult& patched, const RecompileResult& full, std::uint64_t variantId) {
|
||||
const auto* difference = patchedResultDifference(patched, full);
|
||||
if (difference == nullptr) return;
|
||||
resultMemoCounters().mismatches.fetch_add(1, std::memory_order_relaxed);
|
||||
static std::atomic<std::uint32_t> printed{0};
|
||||
if (const auto index = printed.fetch_add(1, std::memory_order_relaxed); index < 20u) std::fprintf(stderr, "[recompile] masked result memo mismatch %u: variant %016llx differs in its %s\n", index + 1u, static_cast<unsigned long long>(variantId), difference);
|
||||
}
|
||||
|
||||
void reportResultMemo() {
|
||||
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
|
||||
if (!profile) return;
|
||||
@@ -1384,16 +1475,20 @@ void reportResultMemo() {
|
||||
}
|
||||
if (now - last < 10'000'000'000ll || !counters.lastReport.compare_exchange_strong(last, now, std::memory_order_relaxed)) return;
|
||||
const auto hits = counters.hits.exchange(0, std::memory_order_relaxed);
|
||||
const auto patched = counters.patched.exchange(0, std::memory_order_relaxed);
|
||||
const auto misses = counters.misses.exchange(0, std::memory_order_relaxed);
|
||||
const auto evictions = counters.evictions.exchange(0, std::memory_order_relaxed);
|
||||
const auto populate = counters.populateNanoseconds.exchange(0, std::memory_order_relaxed);
|
||||
std::fprintf(stderr, "[recompile] result memo (10 s): %llu hits, %llu misses (%.1f%% hits), Populate %.1f us per miss / %.1f ms in total, %llu evictions\n", static_cast<unsigned long long>(hits), static_cast<unsigned long long>(misses), hits + misses != 0 ? 100.0 * static_cast<double>(hits) / static_cast<double>(hits + misses) : 0.0, misses != 0 ? static_cast<double>(populate) / 1000.0 / static_cast<double>(misses) : 0.0, static_cast<double>(populate) / 1e6, static_cast<unsigned long long>(evictions));
|
||||
const auto patching = counters.patchNanoseconds.exchange(0, std::memory_order_relaxed);
|
||||
const auto mismatches = counters.mismatches.exchange(0, std::memory_order_relaxed);
|
||||
const auto total = static_cast<double>(hits + patched + misses);
|
||||
std::fprintf(stderr, "[recompile] result memo (10 s): %llu hits, %llu patched, %llu misses (%.1f%% hits, %.1f%% patched), Populate %.1f us per miss / %.1f ms in total, patch %.1f us each / %.1f ms in total, %llu evictions, %llu patch mismatches\n", static_cast<unsigned long long>(hits), static_cast<unsigned long long>(patched), static_cast<unsigned long long>(misses), total != 0.0 ? 100.0 * static_cast<double>(hits) / total : 0.0, total != 0.0 ? 100.0 * static_cast<double>(patched) / total : 0.0, misses != 0 ? static_cast<double>(populate) / 1000.0 / static_cast<double>(misses) : 0.0, static_cast<double>(populate) / 1e6, patched != 0 ? static_cast<double>(patching) / 1000.0 / static_cast<double>(patched) : 0.0, static_cast<double>(patching) / 1e6, static_cast<unsigned long long>(evictions), static_cast<unsigned long long>(mismatches));
|
||||
}
|
||||
|
||||
// Everything materializeResult reads besides the variant: the snapshot (the descriptor words, the
|
||||
// flattened SRT, the user data, the uniform fill) and, for the vertex family, the V# table the
|
||||
// attributes are resolved from.
|
||||
std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapshot& snapshot) {
|
||||
std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapshot& snapshot, bool masked) {
|
||||
std::uint64_t hash = 0xcbf29ce484222325ull;
|
||||
const auto mix = [&](std::uint64_t value) {
|
||||
hash ^= value;
|
||||
@@ -1411,11 +1506,28 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh
|
||||
for (std::uint32_t i = 0; i < value.dwordCount && i < value.dwords.size(); ++i) mix(value.dwords[i]);
|
||||
}
|
||||
};
|
||||
mixDescriptors(snapshot.buffers);
|
||||
if (masked) {
|
||||
mix(snapshot.buffers.size());
|
||||
for (const auto& value : snapshot.buffers) {
|
||||
mix(value.dwordCount);
|
||||
mix(value.availableDwordMask);
|
||||
const bool present = value.dwords[2] != 0u && (value.dwords[0] != 0u || (value.dwords[1] & 0xffffu) != 0u);
|
||||
mix(present ? 4u | (value.dwords[0] & 3u) : 0u);
|
||||
mix(value.dwords[1] & 0xffff0000u);
|
||||
for (std::uint32_t i = 3; i < value.dwordCount && i < value.dwords.size(); ++i) mix(value.dwords[i]);
|
||||
}
|
||||
} else {
|
||||
mixDescriptors(snapshot.buffers);
|
||||
}
|
||||
mixDescriptors(snapshot.images);
|
||||
mixDescriptors(snapshot.samplers);
|
||||
mixWords(snapshot.flattenedSrt);
|
||||
mixWords(snapshot.userData);
|
||||
if (masked) {
|
||||
mix(snapshot.flattenedSrt.size());
|
||||
mix(snapshot.userData.size());
|
||||
} else {
|
||||
mixWords(snapshot.flattenedSrt);
|
||||
mixWords(snapshot.userData);
|
||||
}
|
||||
mix(static_cast<std::uint64_t>(snapshot.uniformFill.kind));
|
||||
mix(snapshot.uniformFill.resource);
|
||||
for (const auto stride : snapshot.uniformFill.groupStride) mix(stride);
|
||||
@@ -1474,16 +1586,16 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh
|
||||
// as good). `memoHit` reports the hit.
|
||||
std::shared_ptr<const RecompileResult> materializePreparedMemoized(SourceEntry& source, const std::shared_ptr<const CompiledVariant>& variant, const RecompileRequest& request, const ResourceSnapshot& snapshot, bool cacheHit, bool* memoHit) {
|
||||
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
|
||||
const auto hash = snapshotHash(request, snapshot);
|
||||
std::uint64_t index = 0;
|
||||
const auto variantId = variant->artifact.variantId;
|
||||
const auto hash = snapshotHash(request, snapshot, false);
|
||||
const auto index = (variantId * 0x9e3779b97f4a7c15ull) ^ hash;
|
||||
auto& counters = resultMemoCounters();
|
||||
{
|
||||
std::lock_guard lock(source.mutex);
|
||||
index = (variant->artifact.variantId * 0x9e3779b97f4a7c15ull) ^ hash;
|
||||
if (auto* set = memoSet(source, index)) {
|
||||
if (auto* set = memoSet(source.memo, index)) {
|
||||
for (std::size_t way = 0; way < ResultMemoWays; ++way) {
|
||||
auto& entry = set[way];
|
||||
if (entry.result == nullptr || entry.variantId != variant->artifact.variantId || entry.hash != hash) continue;
|
||||
if (entry.result == nullptr || entry.variantId != variantId || entry.hash != hash) continue;
|
||||
entry.used = ++source.memoClock;
|
||||
counters.hits.fetch_add(1, std::memory_order_relaxed);
|
||||
if (memoHit != nullptr) *memoHit = true;
|
||||
@@ -1492,32 +1604,61 @@ std::shared_ptr<const RecompileResult> materializePreparedMemoized(SourceEntry&
|
||||
}
|
||||
}
|
||||
}
|
||||
const auto started = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
|
||||
auto result = std::make_shared<RecompileResult>(materializeResult(*variant, request, snapshot));
|
||||
result->cacheHit = cacheHit;
|
||||
if (profile) counters.populateNanoseconds.fetch_add(static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - started).count()), std::memory_order_relaxed);
|
||||
counters.misses.fetch_add(1, std::memory_order_relaxed);
|
||||
std::shared_ptr<const RecompileResult> shared = std::move(result);
|
||||
{
|
||||
const bool masking = MaskedResultMemo();
|
||||
const auto maskedHash = masking ? snapshotHash(request, snapshot, true) : 0u;
|
||||
const auto maskedIndex = (variantId * 0x9e3779b97f4a7c15ull) ^ maskedHash;
|
||||
std::shared_ptr<const RecompileResult> donor;
|
||||
std::shared_ptr<const MemoPatch> donorPatch;
|
||||
if (masking) {
|
||||
std::lock_guard lock(source.mutex);
|
||||
if (source.memo == nullptr) source.memo = std::make_unique<ResultMemoTable>();
|
||||
auto* set = memoSet(source, index);
|
||||
auto* victim = set;
|
||||
bool present = false;
|
||||
for (std::size_t way = 0; way < ResultMemoWays && !present; ++way) {
|
||||
auto& entry = set[way];
|
||||
if (entry.result != nullptr && entry.variantId == variant->artifact.variantId && entry.hash == hash) {
|
||||
if (auto* set = memoSet(source.maskedMemo, maskedIndex)) {
|
||||
for (std::size_t way = 0; way < ResultMemoWays; ++way) {
|
||||
auto& entry = set[way];
|
||||
if (entry.result == nullptr || entry.variantId != variantId || entry.hash != maskedHash) continue;
|
||||
entry.used = ++source.memoClock;
|
||||
shared = entry.result;
|
||||
present = true;
|
||||
} else if (entry.used < victim->used) {
|
||||
victim = &entry;
|
||||
donor = entry.result;
|
||||
donorPatch = entry.patch;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!present) {
|
||||
if (victim->result != nullptr) counters.evictions.fetch_add(1, std::memory_order_relaxed);
|
||||
*victim = {variant->artifact.variantId, hash, shared, ++source.memoClock};
|
||||
}
|
||||
const auto started = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
|
||||
std::shared_ptr<RecompileResult> result;
|
||||
std::shared_ptr<MemoPatch> patch;
|
||||
if (donor != nullptr) {
|
||||
result = std::make_shared<RecompileResult>(*donor);
|
||||
applyMemoPatch(*result, *donorPatch, snapshot);
|
||||
if (VerifyMaskedResultMemo()) verifyPatchedResult(*result, materializeResult(*variant, request, snapshot), variantId);
|
||||
if (profile) counters.patchNanoseconds.fetch_add(static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - started).count()), std::memory_order_relaxed);
|
||||
counters.patched.fetch_add(1, std::memory_order_relaxed);
|
||||
} else {
|
||||
if (masking) patch = std::make_shared<MemoPatch>();
|
||||
result = std::make_shared<RecompileResult>(materializeResult(*variant, request, snapshot, false, patch.get()));
|
||||
if (profile) counters.populateNanoseconds.fetch_add(static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - started).count()), std::memory_order_relaxed);
|
||||
counters.misses.fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
result->cacheHit = cacheHit;
|
||||
std::shared_ptr<const RecompileResult> shared = std::move(result);
|
||||
const auto insert = [&](std::unique_ptr<ResultMemoTable>& table, std::uint64_t at, std::uint64_t key, std::shared_ptr<const MemoPatch> recipe, bool count) {
|
||||
if (table == nullptr) table = std::make_unique<ResultMemoTable>();
|
||||
auto* set = memoSet(table, at);
|
||||
auto* victim = set;
|
||||
for (std::size_t way = 0; way < ResultMemoWays; ++way) {
|
||||
auto& entry = set[way];
|
||||
if (entry.result != nullptr && entry.variantId == variantId && entry.hash == key) {
|
||||
entry.used = ++source.memoClock;
|
||||
return entry.result;
|
||||
}
|
||||
if (entry.used < victim->used) victim = &entry;
|
||||
}
|
||||
if (count && victim->result != nullptr) counters.evictions.fetch_add(1, std::memory_order_relaxed);
|
||||
*victim = {variantId, key, shared, ++source.memoClock, std::move(recipe)};
|
||||
return shared;
|
||||
};
|
||||
{
|
||||
std::lock_guard lock(source.mutex);
|
||||
shared = insert(source.memo, index, hash, nullptr, true);
|
||||
if (patch != nullptr && patch->valid) static_cast<void>(insert(source.maskedMemo, maskedIndex, maskedHash, patch, false));
|
||||
}
|
||||
reportResultMemo();
|
||||
return shared;
|
||||
|
||||
@@ -39,6 +39,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti
|
||||
| `APS5_NO_CODE_HASH_KEY` | presence | `core/shader/recompiler/CacheKey.hpp` | | appends every code word to the cache key instead of hashing the code |
|
||||
| `APS5_NO_FAILURE_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | rebuilds a program whose compilation failed instead of reusing the memoized failure |
|
||||
| `APS5_NO_IMAGE_TABLES` | presence | `core/shader/recompiler/Optimization/src/ResourceTracker.cpp` | | disables image tables, leaving table-loaded descriptors unplanned |
|
||||
| `APS5_NO_MASKED_RESULT_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | materializes every result memo miss instead of patching the buffer descriptors, user data and flattened SRT of a cached result whose snapshot differs only in those words |
|
||||
| `APS5_NO_PERF_DISK_PLANS` | presence | `core/shader/recompiler/Recompiler.cpp` | | disables the disk cache of planning results |
|
||||
| `APS5_NO_PERF_DOM_INTERSECTION` | presence | `core/shader/recompiler/ControlFlow/src/Structurizer.cpp` | | computes dominator intersections into a fresh vector instead of intersecting in place |
|
||||
| `APS5_NO_PERF_ITERATION_SELECTION` | presence | `core/shader/recompiler/ControlFlow/src/Structurizer.cpp` | | recomputes the post-dominator sets when finding a loop's iteration-selection merge instead of reusing them |
|
||||
@@ -55,6 +56,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti
|
||||
| `APS5_TRACE_BDA` | presence | `core/shader/recompiler/Optimization/src/ResourceTracker.cpp` | | prints the first few accesses that make a program take the address-based path |
|
||||
| `APS5_TRACE_BUFFER_WRITTEN` | presence | `core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp` | | prints how many guest buffer elements a program may store to and how many it only loads |
|
||||
| `APS5_TRACE_IMAGE_TABLE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | traces image-table resolution |
|
||||
| `APS5_VERIFY_MASKED_RESULT_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | also materializes every patched result memo hit, prints the first twenty whose patched result differs and counts them in the result memo report of `APS5_PROFILE_DRAW` |
|
||||
| `APS5_VERIFY_SRT_COMPACT` | presence | `core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp` | | walks every SRT descriptor a second time through the IR evaluator and throws when the two walks differ |
|
||||
| `APS5_VERIFY_SRT_TAPE` | presence | `core/shader/recompiler/Optimization/src/SrtWalker/SrtTape.cpp` | | walks a plan's runtime sources a second time through the evaluator, adopts its answers where the two differ and prints the first twenty differences |
|
||||
| `APS5_WAVE_LDS_SUBGROUP` | presence | `core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp` | | restores subgroup-scope LDS barriers for single-wave workgroups instead of workgroup scope |
|
||||
|
||||
Reference in new issue
Block a user