diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index a420eade76..c693fdafd9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -1401,6 +1401,10 @@ void verifyInvocationIsolation() { if ((descriptor[0] & 3u) == (original[0] & 3u) && (descriptor[1] & 0xffff0000u) == (original[1] & 0xffff0000u) && descriptor[3] == original[3] && (descriptor[0] != 0u || (descriptor[1] & 0xffffu) != 0u)) { require(first.spirv.data() == changed.spirv.data() && first.PipelineVariantId() == changed.PipelineVariantId(), "buffer address or size rebuilt the specialized module"); } + if (descriptor[2] == 0u) { + require(first.spirv.data() == changed.spirv.data() && first.PipelineVariantId() == changed.PipelineVariantId(), "a null V# did not specialize with the bound slot's substitute"); + continue; + } ComputeRequest fresh(false); fresh.userData = descriptor; requireSameResult(Recompile(fresh.request), changed, "reused binding plan against fresh compilation"); diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_runtime_formatted_store_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_runtime_formatted_store_tests.cmake index b304bfb333..d9dac7c6a9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_runtime_formatted_store_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_runtime_formatted_store_tests.cmake @@ -1,5 +1,6 @@ if(BUILD_TESTING) set_tests_properties(agc_driver_runtime_formatted_store PROPERTIES TIMEOUT 30) + set_tests_properties(agc_driver_null_buffer_substitutes PROPERTIES TIMEOUT 30) foreach(mode IN ITEMS x1 x2 x3 x1_d16 x2_d16 x3_d16 x4_d16) string(REGEX REPLACE "_d16$" "" components "${mode}") set(arguments "--${components}") diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/NullBufferSubstitutes.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/NullBufferSubstitutes.cpp new file mode 100644 index 0000000000..9e094d1c38 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/NullBufferSubstitutes.cpp @@ -0,0 +1,153 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "Optimization/RequestMemoryView.hpp" +#include "Optimization/ResourceProgram.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Lanes = 32; +constexpr std::uint32_t InputBytes = Lanes * 4u; +constexpr std::uint32_t Fill = 0xdeadbeefu; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t Format16UInt = 11; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +constexpr std::array LoadCode(std::uint32_t marker) { + return {0x34020082u, 0xe0001000u, 0x80000201u, 0xe0301000u, 0x80000301u, 0xbf8c0000u, + 0xe0101000u, 0x80000201u, 0xe0701000u, 0x80000301u, + 0xe0701000u, 0x80010201u, 0xe0701080u + marker * 4u, 0x80010301u, 0xbf800000u | marker, 0xbf810000u}; +} + +enum class Mode { Direct, Captured, Prepared }; + +alignas(256) constexpr std::array Codes{LoadCode(0u), LoadCode(1u), LoadCode(2u), LoadCode(3u), LoadCode(4u), LoadCode(5u)}; + +std::array Descriptor(const void* data, std::uint32_t records, std::uint32_t format, std::uint32_t stride = 0u) { + const auto address = reinterpret_cast(data); + const auto oobSelect = stride != 0u ? 0u : 3u; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), records, (0x01016facu & ~(0x7fu << 12u)) | (format << 12u) | (oobSelect << 28u)}; +} + +constexpr std::array GameNull{0x00000000u, 0x00080050u, 0x00000000u, 0x00005204u}; +constexpr std::array ZeroNull{0u, 0u, 0u, 0u}; + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +struct Outcome { + std::uint64_t pipeline = 0; + std::vector spirv; +}; + +Outcome Run(AgcDriver::VulkanDevice& device, std::span code, std::uint32_t marker, const std::array& optional, const std::string& name, std::uint32_t mask, Mode mode) { + for (std::uint32_t lane = 0; lane < Lanes; ++lane) Input[lane] = 0x9e3779b9u * (lane + 1u) ^ 0x01234567u; + Output.fill(Fill); + std::vector userData(optional.begin(), optional.end()); + const auto output = Descriptor(Output.data(), sizeof(Output), Format32UInt); + userData.insert(userData.end(), output.begin(), output.end()); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Lanes, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {Lanes, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = true; + const auto result = [&] { + if (mode == Mode::Direct) return ShaderRecompiler::Recompile(request); + ShaderRecompiler::RequestMemoryView memoryView(request.context.memory); + const auto runtime = memoryView.MakeRuntime(request.context.userData, request.shader.codeAddress); + if (mode == Mode::Captured) return *ShaderRecompiler::Recompile(request, *ShaderRecompiler::CaptureResources(request, runtime)); + const auto handle = ShaderRecompiler::PrepareShader(request); + const auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(request, handle); + Require(invocation.has_value(), "the prepared shader refused its own request"); + return *invocation->Materialize(*invocation->Capture(runtime)); + }(); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); + const bool present = optional[2] != 0u && (optional[0] != 0u || (optional[1] & 0xffffu) != 0u); + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + Require(Input[lane] == (0x9e3779b9u * (lane + 1u) ^ 0x01234567u), name + ": an absent store changed its backing memory"); + const auto formatted = present ? Input[lane] & mask : 0u; + const auto raw = present ? Input[lane] : 0u; + Require(Output[lane] == formatted, name + ": lane " + std::to_string(lane) + " buffer_load_format_x is " + Hex(Output[lane]) + ", expected " + Hex(formatted)); + Require(Output[Lanes + marker + lane] == raw, name + ": lane " + std::to_string(lane) + " buffer_load_dword is " + Hex(Output[Lanes + marker + lane]) + ", expected " + Hex(raw)); + } + Outcome outcome; + outcome.pipeline = result.PipelineVariantId(); + outcome.spirv = result.spirv.Copy(); + Require(outcome.pipeline != 0u && !outcome.spirv.empty(), name + ": no specialized module"); + return outcome; +} + +void BoundThenNull(AgcDriver::VulkanDevice& device, Mode mode) { + const auto marker = static_cast(mode) * 2u; + const auto& code = Codes[marker]; + const auto bound = Run(device, code, marker, Descriptor(Input.data(), InputBytes, Format32UInt), "bound", 0xffffffffu, mode); + const auto absent = Run(device, code, marker, GameNull, "null V#", 0xffffffffu, mode); + Require(absent.pipeline == bound.pipeline && absent.spirv == bound.spirv, "a null V# in an optional slot made another module than the bound slot"); + const auto zero = Run(device, code, marker, ZeroNull, "zero V#", 0xffffffffu, mode); + Require(zero.pipeline == bound.pipeline, "a zero V# made another module than the bound slot"); + auto zeroAddress = Descriptor(nullptr, InputBytes, Format32UInt); + const auto addressless = Run(device, code, marker, zeroAddress, "zero address with records", 0xffffffffu, mode); + Require(addressless.pipeline == bound.pipeline, "a zero-address V# did not reuse the bound module"); + auto zeroRecords = Descriptor(Input.data(), 0u, Format16UInt, InputBytes); + const auto backedNull = Run(device, code, marker, zeroRecords, "zero records with backing memory", 0xffffffffu, mode); + Require(backedNull.pipeline == bound.pipeline, "a zero-record V# did not reuse the bound module"); + const auto again = Run(device, code, marker, Descriptor(Input.data(), InputBytes, Format32UInt), "bound again", 0xffffffffu, mode); + Require(again.pipeline == bound.pipeline, "the bound slot changed modules after a null V#"); + const auto reformatted = Run(device, code, marker, Descriptor(Input.data(), 1u, Format16UInt, InputBytes), "bound 16_UINT structured stride 128", 0xffffu, mode); + Require(reformatted.pipeline != bound.pipeline && reformatted.spirv != bound.spirv, "a bound slot with another format and stride shared the substitute's module"); + const auto absentAfter = Run(device, code, marker, GameNull, "null V# after another format", 0xffffffffu, mode); + Require(absentAfter.pipeline == bound.pipeline, "a null V# did not keep the first bound slot's substitute"); +} + +void NullThenBound(AgcDriver::VulkanDevice& device, Mode mode) { + const auto marker = static_cast(mode) * 2u + 1u; + const auto& code = Codes[marker]; + const auto first = Run(device, code, marker, GameNull, "null V# before any bound slot", 0xffffffffu, mode); + const auto bound = Run(device, code, marker, Descriptor(Input.data(), InputBytes, Format32UInt), "bound after null", 0xffffffffu, mode); + Require(first.pipeline != bound.pipeline, "a slot never seen bound did not specialize as absent"); + const auto repeated = Run(device, code, marker, GameNull, "same null V# after learning", 0xffffffffu, mode); + Require(repeated.pipeline == bound.pipeline && repeated.spirv == bound.spirv, "the full-result memo retained the null module from before learning"); + const auto learned = Run(device, code, marker, ZeroNull, "zero V# after the slot was bound", 0xffffffffu, mode); + Require(learned.pipeline == bound.pipeline && learned.spirv == bound.spirv, "a null V# did not take the substitute learned from the bound slot"); +} + +} + +int main(int argc, char** argv) { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + std::printf("null buffer substitutes registrationModules=%d\n", device->Target().registrationModules); + for (const auto mode : {Mode::Direct, Mode::Captured, Mode::Prepared}) { + if (argc > 1 && ((std::string(argv[1]) == "--captured" && mode != Mode::Captured) || (std::string(argv[1]) == "--prepared" && mode != Mode::Prepared))) continue; + BoundThenNull(*device, mode); + NullThenBound(*device, mode); + std::printf("null buffer substitutes mode=%u passed\n", static_cast(mode)); + } + std::puts("null buffer substitute tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/CompiledVariant.hpp b/core/shader/recompiler/CompiledVariant.hpp index 00b780914d..adc493521b 100644 --- a/core/shader/recompiler/CompiledVariant.hpp +++ b/core/shader/recompiler/CompiledVariant.hpp @@ -4,14 +4,42 @@ #include "Recompiler.hpp" #include "IntermediateRepresentation/IrMetadata/CompiledShaderInfo.hpp" #include "Optimization/include/Optimization/BindingAllocator.hpp" +#include "Optimization/include/Optimization/DescriptorBindingBuilder.hpp" +#include +#include +#include +#include +#include namespace ShaderRecompiler { +class BufferSubstituteTable { +public: + explicit BufferSubstituteTable(std::vector slots) : slots(std::move(slots)) {} + + [[nodiscard]] std::vector Slots() const { + std::lock_guard lock(mutex); + return slots; + } + + void Learn(std::size_t slot, const BufferSubstitute& value) { + std::lock_guard lock(mutex); + if (slot >= slots.size() || slots[slot].known) return; + slots[slot] = value; + slots[slot].known = true; + } + +private: + mutable std::mutex mutex; + std::vector slots; +}; + struct CompiledVariant { BindingLayout layout; CompiledShaderInfo info; CompiledBindingLayout bindings; CompiledShaderArtifact artifact; + std::shared_ptr substitutes; }; } diff --git a/core/shader/recompiler/Optimization/include/Optimization/DescriptorBindingBuilder.hpp b/core/shader/recompiler/Optimization/include/Optimization/DescriptorBindingBuilder.hpp index 9d48693aba..cc0a71caf7 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/DescriptorBindingBuilder.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/DescriptorBindingBuilder.hpp @@ -27,6 +27,18 @@ struct PreparedImageMetadata { RuntimeAbi::ResourceMetadata metadata; }; +struct BufferSubstitute { + std::uint32_t word1 = 0; + std::uint32_t word3 = 0; + std::uint32_t misalignment = 0; + bool known = false; + + bool operator==(const BufferSubstitute&) const = default; +}; + +bool BufferDescriptorPresent(const DescriptorValue& descriptor); +BufferSubstitute BufferSpecializationFields(const DescriptorValue& descriptor, std::span substitutes, std::size_t index); + struct DescriptorBindingPlan { std::vector bindings; std::vector imageMetadata; @@ -37,7 +49,7 @@ struct DescriptorBindingPlan { class DescriptorBindingBuilder { public: - DescriptorBindingPlan Prepare(const IrBindingLayout& layout, const ShaderInfo& info, IrShaderStage stage, const ResourceSnapshot& snapshot, std::span exportMappings = {}, bool registrationModules = false, bool fragmentDescriptorSet = false) const; + DescriptorBindingPlan Prepare(const IrBindingLayout& layout, const ShaderInfo& info, IrShaderStage stage, const ResourceSnapshot& snapshot, std::span exportMappings = {}, bool registrationModules = false, bool fragmentDescriptorSet = false, std::span substitutes = {}) const; DescriptorBindingPlan Select(const DescriptorBindingPlan& plan, std::span bindings) const; void Populate(BindingAllocationResult& allocation, const CompiledBindingLayout& compiled, const DescriptorBindingPlan& plan, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads, const ResolvedImageTables* tables = nullptr) const; void Populate(BindingAllocationResult& allocation, const IrProgram& program, const ResourceSnapshot& snapshot, const std::array& partialThreads, std::span exportMappings = {}) const; diff --git a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp index 576ac75a42..4c89147090 100644 --- a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp +++ b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp @@ -305,7 +305,17 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con allocation.specialization = std::move(plan.specialization); } -DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& layout, const ShaderInfo& info, IrShaderStage stage, const ResourceSnapshot& snapshot, std::span exportMappings, bool registrationModules, bool fragmentDescriptorSet) const { +bool BufferDescriptorPresent(const DescriptorValue& descriptor) { + return descriptor.dwords[2] != 0u && (descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffffu) != 0u); +} + +BufferSubstitute BufferSpecializationFields(const DescriptorValue& descriptor, std::span substitutes, std::size_t index) { + const bool present = BufferDescriptorPresent(descriptor); + if (!present && !substitutes.empty()) return index < substitutes.size() && substitutes[index].known ? substitutes[index] : BufferSubstitute{}; + return {descriptor.dwords[1] & 0xffff0000u, descriptor.dwords[3], present ? descriptor.dwords[0] & 3u : 0u, present}; +} + +DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& layout, const ShaderInfo& info, IrShaderStage stage, const ResourceSnapshot& snapshot, std::span exportMappings, bool registrationModules, bool fragmentDescriptorSet, std::span substitutes) const { if (stage == IrShaderStage::Pixel && exportMappings.size() != 8u) fail("fragment runtime export mappings are missing"); if (info.images.size() > ShaderInfo::MaxImages) fail("runtime image count exceeds the static capacity"); DescriptorBindingPlan plan; @@ -331,25 +341,25 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l const auto& descriptor = snapshot.buffers.at(index); if (descriptor.dwordCount != 4u) fail("buffer specialization requires four descriptor words"); const auto& buffer = info.buffers[index]; - const auto format = (descriptor.dwords[3] >> 12u) & 0x7fu; - const bool present = descriptor.dwords[2] != 0u && (descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffffu) != 0u); - if (buffer.descriptorFormatted && (format > static_cast(IrBufferFormat::Format32_32_32_32Float) || (present && format == 0u))) fail("buffer specialization has an unsupported format"); - const auto misalignment = present ? descriptor.dwords[0] & 3u : 0u; + const auto fields = BufferSpecializationFields(descriptor, substitutes, index); + const auto format = (fields.word3 >> 12u) & 0x7fu; + if (buffer.descriptorFormatted && (format > static_cast(IrBufferFormat::Format32_32_32_32Float) || (fields.known && format == 0u))) fail("buffer specialization has an unsupported format"); + const auto misalignment = fields.misalignment; auto alignment = static_cast(buffer.typedAlignment); if (buffer.descriptorFormatted && format != 0u) alignment = std::max(alignment, std::min(GetFormatInfo(static_cast(format)).byteSize, 4u)); if (alignment == 0u || misalignment % alignment != 0u) fail("is a formatted buffer access through a V# whose base is not aligned to its element"); - if (registrationModules && buffer.descriptorFormatted) plan.specialization.push_back({PipelineSpecialization::BufferBase + index * PipelineSpecialization::BufferWords + 1u, descriptor.dwords[3]}); + if (registrationModules && buffer.descriptorFormatted) plan.specialization.push_back({PipelineSpecialization::BufferBase + index * PipelineSpecialization::BufferWords + 1u, fields.word3}); if (registrationModules) plan.specialization.push_back({PipelineSpecialization::BufferBase + index * PipelineSpecialization::BufferWords + 3u, misalignment}); if (!registrationModules) { const auto first = PipelineSpecialization::BufferBase + index * PipelineSpecialization::BufferWords; - plan.specialization.push_back({first, descriptor.dwords[1] & 0xffff0000u}); - plan.specialization.push_back({first + 1u, descriptor.dwords[3]}); + plan.specialization.push_back({first, fields.word1}); + plan.specialization.push_back({first + 1u, fields.word3}); plan.specialization.push_back({first + 3u, misalignment}); } if (buffer.formattedReadMask != 0u) { for (std::uint32_t component = 0; component < 4u; ++component) { if ((buffer.formattedReadMask & (1u << component)) == 0u) continue; - const auto selector = (descriptor.dwords[3] >> (component * 3u)) & 7u; + const auto selector = (fields.word3 >> (component * 3u)) & 7u; if (selector == 2u || selector == 3u) fail("buffer specialization has a reserved component selector"); } } diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 515f054b5a..e4e7ee0a41 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "ControlFlow/include/ControlFlow/GraphBuilder.hpp" #include "ControlFlow/include/ControlFlow/Structurizer.hpp" @@ -478,6 +479,7 @@ bool diskPlans() { } bool sameLayout(const BindingLayout& left, const BindingLayout& right); +void attachSubstitutes(CompiledVariant& variant); std::shared_ptr getSource(const RecompileRequest& request, ShaderPreparationContext* preparation = nullptr, bool reclaim = true, const SourceHandle* preparedHandle = nullptr) { static PrimedSharedMutex mutex; @@ -983,7 +985,12 @@ SpecializedSpirv registrationModule(std::uint64_t variantId, std::spanmodule, std::move(built)}; } -std::shared_ptr preparedBindingPlan(const CompiledVariant& variant, const ResourceSnapshot& snapshot, std::span exports, bool fragmentDescriptorSet) { +bool bufferSubstitutesEnabled() { + static const bool enabled = std::getenv("APS5_NO_BUFFER_SUBSTITUTES") == nullptr; + return enabled; +} + +std::shared_ptr preparedBindingPlan(const CompiledVariant& variant, const ResourceSnapshot& snapshot, std::span exports, bool fragmentDescriptorSet, std::span substitutes) { struct BindingPlanKeyStorage {}; auto& key = HostThreadLocal, BindingPlanKeyStorage>(); key.clear(); @@ -991,10 +998,10 @@ std::shared_ptr preparedBindingPlan(const CompiledVariant& for (std::size_t index = 0; index < variant.info.info.buffers.size(); ++index) { const auto& descriptor = snapshot.buffers.at(index); key.push_back(descriptor.dwordCount); - key.push_back(descriptor.dwords[1] & 0xffff0000u); - key.push_back(descriptor.dwords[3]); - const bool present = descriptor.dwords[2] != 0u && (descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffffu) != 0u); - key.push_back(present ? 4u | (descriptor.dwords[0] & 3u) : 0u); + const auto fields = BufferSpecializationFields(descriptor, substitutes, index); + key.push_back(fields.word1); + key.push_back(fields.word3); + key.push_back(fields.known ? 4u | fields.misalignment : 0u); } for (std::size_t index = 0; index < variant.info.info.images.size(); ++index) { const auto& descriptor = snapshot.images.at(index); @@ -1024,12 +1031,16 @@ std::shared_ptr preparedBindingPlan(const CompiledVariant& } std::call_once(plan->ready, [&] { try { - plan->bindings = DescriptorBindingBuilder{}.Prepare(variant.bindings.layout, variant.info.info, variant.info.stage, snapshot, exports, variant.artifact.registrationModule, fragmentDescriptorSet); + plan->bindings = DescriptorBindingBuilder{}.Prepare(variant.bindings.layout, variant.info.info, variant.info.stage, snapshot, exports, variant.artifact.registrationModule, fragmentDescriptorSet, substitutes); } catch (...) { plan->failure = std::current_exception(); } }); if (plan->failure) std::rethrow_exception(plan->failure); + for (std::size_t index = 0; index < substitutes.size(); ++index) { + const auto& descriptor = snapshot.buffers.at(index); + if (!substitutes[index].known && BufferDescriptorPresent(descriptor)) variant.substitutes->Learn(index, BufferSpecializationFields(descriptor, {}, index)); + } return plan; } @@ -1106,7 +1117,9 @@ void recordMemoPatch(MemoPatch& patch, const DescriptorBindingPlan& plan, const patch.valid = layout.userDataRegisters.size() <= layout.DispatchThreadLimitDword() && (patch.userWords.empty() || patch.pushData || patch.shaderData != MemoPatch::NoBinding); } -RecompileResult materializeResult(const CompiledVariant& variant, const RecompileRequest& request, const ResourceSnapshot& captured, bool registrationDefaults = false, MemoPatch* patch = nullptr) { +RecompileResult materializeResult(const CompiledVariant& variant, const RecompileRequest& request, const ResourceSnapshot& captured, bool registrationDefaults = false, MemoPatch* patch = nullptr, const std::vector* substituteSnapshot = nullptr) { + const auto localSubstitutes = substituteSnapshot == nullptr && bufferSubstitutesEnabled() && variant.substitutes != nullptr ? variant.substitutes->Slots() : std::vector{}; + const auto substitutes = substituteSnapshot != nullptr ? std::span(*substituteSnapshot) : std::span(localSubstitutes); ResourceSnapshot merged; const auto& snapshot = mergeTargetImage(variant, request, captured, registrationDefaults, merged); RecompileResult result; @@ -1147,7 +1160,7 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil const auto index = reg - request.context.userDataBaseRegister; result.userDataRegisterReads |= reg >= request.context.userDataBaseRegister && index < 64u ? std::uint64_t{1} << index : ~std::uint64_t{0}; } - const auto plan = preparedBindingPlan(variant, snapshot, request.context.pixel ? std::span(request.context.pixel->targetExportMapping) : std::span{}, request.target.registrationModules); + const auto plan = preparedBindingPlan(variant, snapshot, request.context.pixel ? std::span(request.context.pixel->targetExportMapping) : std::span{}, request.target.registrationModules, substitutes); const DescriptorBindingPlan* bindingPlan = &plan->bindings; std::shared_ptr entry; struct LocalModuleKeyStorage {}; @@ -1315,6 +1328,18 @@ bool sameLayout(const BindingLayout& left, const BindingLayout& right) { return left.descriptorSet == right.descriptorSet && left.firstBinding == right.firstBinding && left.pushConstantOffsetBytes == right.pushConstantOffsetBytes && left.pushConstantSizeBytes == right.pushConstantSizeBytes; } +void attachSubstitutes(CompiledVariant& variant) { + const auto slots = variant.info.info.buffers.size(); + if (slots == 0u || variant.artifact.spirv.empty()) return; + const auto hash = variant.artifact.spirv.ContentHash(); + static std::mutex mutex; + static std::map, std::shared_ptr> tables; + std::lock_guard lock(mutex); + auto& table = tables[{hash.low, hash.high, slots}]; + if (table == nullptr) table = std::make_shared(std::vector(slots)); + variant.substitutes = table; +} + } SharedSpirv InternSpirv(std::vector words) { @@ -1395,6 +1420,7 @@ std::shared_ptr findOrCompileVariant(SourceEntry& source, if (ShaderDiskCache::Load(diskKey, loaded)) { loaded.layout = request.layout; loaded.artifact.variantId = nextVariantId(); + attachSubstitutes(loaded); variant = std::make_shared(std::move(loaded)); } } @@ -1402,13 +1428,14 @@ std::shared_ptr findOrCompileVariant(SourceEntry& source, static const bool layoutPair = std::getenv("APS5_NO_PERF_LAYOUT_PAIR") == nullptr; std::exception_ptr emissionFailure; try { - if (layoutPair && preparation != nullptr) { - variant = std::make_shared(PreparedInterface::Compile(source, request, *preparation, &emissionFailure)); - } else { + auto compiled = [&] { + if (layoutPair && preparation != nullptr) return PreparedInterface::Compile(source, request, *preparation, &emissionFailure); auto program = source.program != nullptr ? std::move(*source.program) : PrepareResourceProgram(request, preparation); source.program.reset(); - variant = std::make_shared(compileVariant(request, std::move(program), &emissionFailure)); - } + return compileVariant(request, std::move(program), &emissionFailure); + }(); + attachSubstitutes(compiled); + variant = std::make_shared(std::move(compiled)); } catch (...) { if (emissionFailure != nullptr && FailureMemo()) source.emissionFailures.push_back({request.shader.codeAddress, request.layout, emissionFailure}); throw; @@ -1686,7 +1713,7 @@ void reportResultMemo() { // Everything materializeResult reads besides the variant: the snapshot (the descriptor words, the // flattened SRT, the user data, the uniform fill) and, for the vertex family, the V# table the // attributes are resolved from. -std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapshot& snapshot, bool masked) { +std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapshot& snapshot, bool masked, std::span substitutes = {}) { std::uint64_t hash = 0xcbf29ce484222325ull; const auto mix = [&](std::uint64_t value) { hash ^= value; @@ -1706,16 +1733,24 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh }; if (masked) { mix(snapshot.buffers.size()); - for (const auto& value : snapshot.buffers) { + for (std::size_t index = 0; index < snapshot.buffers.size(); ++index) { + const auto& value = snapshot.buffers[index]; mix(value.dwordCount); mix(value.availableDwordMask); - const bool present = value.dwords[2] != 0u && (value.dwords[0] != 0u || (value.dwords[1] & 0xffffu) != 0u); - mix(present ? 4u | (value.dwords[0] & 3u) : 0u); - mix(value.dwords[1] & 0xffff0000u); - for (std::uint32_t i = 3; i < value.dwordCount && i < value.dwords.size(); ++i) mix(value.dwords[i]); + const auto fields = BufferSpecializationFields(value, substitutes, index); + mix(fields.known ? 4u | fields.misalignment : 0u); + mix(fields.word1); + if (value.dwordCount > 3u) mix(fields.word3); + for (std::uint32_t i = 4; i < value.dwordCount && i < value.dwords.size(); ++i) mix(value.dwords[i]); } } else { mixDescriptors(snapshot.buffers); + for (std::size_t index = 0; index < substitutes.size() && index < snapshot.buffers.size(); ++index) { + const auto fields = BufferSpecializationFields(snapshot.buffers[index], substitutes, index); + mix(fields.word1); + mix(fields.word3); + mix(fields.known ? 4u | fields.misalignment : 0u); + } } mixDescriptors(snapshot.images); mixDescriptors(snapshot.samplers); @@ -1797,7 +1832,8 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh std::shared_ptr materializePreparedMemoized(SourceEntry& source, const std::shared_ptr& variant, const RecompileRequest& request, const ResourceSnapshot& snapshot, bool cacheHit, bool* memoHit) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; const auto variantId = variant->artifact.variantId; - const auto hash = snapshotHash(request, snapshot, false); + const auto substitutes = bufferSubstitutesEnabled() && variant->substitutes != nullptr ? variant->substitutes->Slots() : std::vector{}; + const auto hash = snapshotHash(request, snapshot, false, substitutes); const auto index = (variantId * 0x9e3779b97f4a7c15ull) ^ hash; auto& counters = resultMemoCounters(); { @@ -1815,7 +1851,7 @@ std::shared_ptr materializePreparedMemoized(SourceEntry& } } const bool masking = MaskedResultMemo(); - const auto maskedHash = masking ? snapshotHash(request, snapshot, true) : 0u; + const auto maskedHash = masking ? snapshotHash(request, snapshot, true, substitutes) : 0u; const auto maskedIndex = (variantId * 0x9e3779b97f4a7c15ull) ^ maskedHash; std::shared_ptr donor; std::shared_ptr donorPatch; @@ -1838,12 +1874,12 @@ std::shared_ptr materializePreparedMemoized(SourceEntry& if (donor != nullptr) { result = std::make_shared(*donor); applyMemoPatch(*result, *donorPatch, snapshot); - if (VerifyMaskedResultMemo()) verifyPatchedResult(*result, materializeResult(*variant, request, snapshot), variantId); + if (VerifyMaskedResultMemo()) verifyPatchedResult(*result, materializeResult(*variant, request, snapshot, false, nullptr, &substitutes), variantId); if (profile) counters.patchNanoseconds.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()), std::memory_order_relaxed); counters.patched.fetch_add(1, std::memory_order_relaxed); } else { if (masking) patch = std::make_shared(); - result = std::make_shared(materializeResult(*variant, request, snapshot, false, patch.get())); + result = std::make_shared(materializeResult(*variant, request, snapshot, false, patch.get(), &substitutes)); if (profile) counters.populateNanoseconds.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()), std::memory_order_relaxed); counters.misses.fetch_add(1, std::memory_order_relaxed); } diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index a564f91563..f2ccae1808 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -1389,6 +1389,7 @@ constexpr std::string_view NeutralSwitches[] = { "APS5_NO_PERF_DOM_INTERSECTION", "APS5_NO_PERF_ITERATION_SELECTION", "APS5_NO_PERF_LAYOUT_PAIR", + "APS5_NO_BUFFER_SUBSTITUTES", "APS5_NO_RESULT_MEMO", "APS5_NO_TABLE_RESOLUTION_REUSE", "APS5_NO_TABLE_SNAPSHOT_REUSE", @@ -1507,6 +1508,7 @@ public: if (DecodeEntry(bytes, key, *persisted, std::move(mapped)) != LoadStatus::Loaded) return nullptr; persisted->layout = variant.layout; persisted->artifact.variantId = variant.artifact.variantId; + persisted->substitutes = variant.substitutes; return persisted; } diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 5ce9bf15a8..d5afcc5294 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -41,6 +41,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_IMAGE_TABLE_STRICT` | string | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | sets image-table materialization to its strict form; 0 leaves it off | | `APS5_NO_ATOMIC_BARRIER` | presence | `core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvFormatConvert.cpp` | | drops the device-scope acquire/release memory barrier emitted after atomics | | `APS5_NO_ATOMIC_ZERO_SKIP` | presence | `core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp` | | restores plain buffer atomics instead of skipping identity-operand atomics | +| `APS5_NO_BUFFER_SUBSTITUTES` | presence | `core/shader/recompiler/Recompiler.cpp` | | specializes a null buffer V# with its own words instead of the fields its slot had the first time a dispatch of the same program bound it | | `APS5_NO_CODE_HASH_KEY` | presence | `core/shader/recompiler/CacheKey.hpp` | | appends every code word to the cache key instead of hashing the code | | `APS5_NO_FAILURE_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | rebuilds a program whose compilation failed instead of reusing the memoized failure | | `APS5_NO_IMAGE_TABLES` | presence | `core/shader/recompiler/Optimization/src/ResourceTracker.cpp` | | disables image tables, leaving table-loaded descriptors unplanned |