From 895002149d5a7857cc3d81f30e4809e9b5be7d0f Mon Sep 17 00:00:00 2001 From: ThatsLeo Date: Tue, 13 Oct 2026 17:26:51 +0200 Subject: [PATCH] perf(shader): map cached SPIR-V from the shader cache files A cached variant kept its SPIR-V in memory for the whole run. On Linux and macOS the words are now a view into the mapped cache file, and a freshly compiled variant is written and mapped back, so the kernel can drop them. --- .../Execution/src/VulkanDevice.cpp | 2 +- .../tests/Float16Conversion.cpp | 2 +- .../prx/libSceAgcDriver/tests/Graphics.cpp | 4 +- core/libs/prx/libSceAgcDriver/tests/Mesh.cpp | 2 +- .../libSceAgcDriver/tests/ShaderDiskCache.cpp | 23 +++- .../libSceAgcDriver/tests/ShaderMemory.cpp | 48 +++---- .../tests/SpirvBranchTargets.cpp | 2 +- .../tests/VertexLayerExport.cpp | 4 +- .../tests/execution/F64FmaSubnormals.cpp | 2 +- .../tests/execution/LayeredVolumeDraw.cpp | 2 +- .../tests/execution/ZeroStrideVertexFetch.cpp | 4 +- core/shader/recompiler/Recompiler.cpp | 5 +- core/shader/recompiler/Recompiler.hpp | 27 ++-- core/shader/recompiler/ShaderDiskCache.cpp | 127 +++++++++++++++--- core/shader/recompiler/ShaderDiskCache.hpp | 5 +- .../recompiler/VertexInputSpecialization.hpp | 2 +- 16 files changed, 190 insertions(+), 71 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 45ccc24d26..9b536a4ef0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -4278,7 +4278,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec const auto subgroupSize = state->computeWave32 && (shader.hostSubgroupSize == 32u || shader.hostSubgroupSize > state->maxComputeSubgroupSize) ? std::min(shader.hostSubgroupSize, state->maxComputeSubgroupSize) : 0u; if (const char* directory = std::getenv("APS5_DUMP_COMPUTE_PIPELINES"); directory != nullptr && *directory != '\0' && CaptureComputePipeline(shader.spirv.size())) { ComputePipelineCapture capture; - capture.spirv = shader.spirv; + capture.spirv = shader.spirv.Copy(); ComputeDescriptorSetLayout resourcesLayout; resourcesLayout.flags = resources->LayoutUpdateAfterBind() ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT : 0; const auto layoutBindings = resources->LayoutBindings(); diff --git a/core/libs/prx/libSceAgcDriver/tests/Float16Conversion.cpp b/core/libs/prx/libSceAgcDriver/tests/Float16Conversion.cpp index 4035a58490..baf3155a63 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Float16Conversion.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Float16Conversion.cpp @@ -42,7 +42,7 @@ std::vector recompile(std::span capabilities request.useCache = false; auto result = Recompile(request); require(!result.spirv.empty(), "v_cvt_f16_f32 did not recompile"); - return std::move(result.spirv); + return result.spirv.Copy(); } struct Usage { diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index c0d04b1dd2..8219f52b4d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -4727,7 +4727,7 @@ void aliasedBufferViewTests() { Require(declared == views, "the pixel shader declared the guest buffers as" + names + ", not as the expected " + std::to_string(views.size()) + " aliased variables"); const std::array shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &pixel, 0}}}; AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false, false, false, false, false, true); - auto unaliased = pixel.spirv.Words(); + auto unaliased = pixel.spirv.Copy(); for (std::size_t at = 5; at < unaliased.size() && (unaliased[at] >> 16u) != 0; at += unaliased[at] >> 16u) { if (static_cast(unaliased[at] & 0xffffu) == spv::OpDecorate && unaliased[at + 2] == spv::DecorationAliased) unaliased[at + 2] = spv::DecorationRestrict; } @@ -5077,7 +5077,7 @@ void unorm10_11_11TargetTests() { const auto packed = ShaderRecompiler::Recompile(request); const auto unpacked = ShaderRecompiler::Recompile(plain); Require(packed.variantId == unpacked.variantId && packed.variantId == ShaderRecompiler::GetPreparedArtifact(*prepared).variantId, "the packed draw did not reuse the registered artifact"); - Require(packed.PipelineVariantId() != unpacked.PipelineVariantId() && packed.spirv.Words() != unpacked.spirv.Words(), "the export packing did not specialize the module"); + Require(packed.PipelineVariantId() != unpacked.PipelineVariantId() && !(packed.spirv == unpacked.spirv), "the export packing did not specialize the module"); Require(ShaderRecompiler::Recompile(request).PipelineVariantId() == packed.PipelineVariantId(), "the packed specialization was not reused"); ShaderRecompiler::RecompileResult vertex; vertex.spirv = makeModule({}); diff --git a/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp b/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp index 47db4246d4..ac2fff1057 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp @@ -144,7 +144,7 @@ struct MeshDraw { std::uint32_t stagesRegister = 0x20u; }; -bool ReadsNumWorkgroups(const std::vector& words) { +bool ReadsNumWorkgroups(std::span words) { constexpr std::uint32_t OpDecorate = 71; constexpr std::uint32_t OpAccessChain = 65; constexpr std::uint32_t OpLoad = 61; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 7fad973a6c..184c778f5e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -69,7 +69,7 @@ bool sameBindings(const std::vector& left, const std::vector< void requireSameArtifact(const CompiledShaderArtifact& left, const CompiledShaderArtifact& right, const char* what) { const std::string prefix = std::string(what) + ": "; - require(left.spirv.Words() == right.spirv.Words(), prefix + "SPIR-V differs"); + require(left.spirv == right.spirv, prefix + "SPIR-V differs"); require(left.bdaAbiVersion == right.bdaAbiVersion, prefix + "BDA ABI version differs"); require(left.runtimeAbiVersion == right.runtimeAbiVersion, prefix + "runtime ABI version differs"); require(left.memoryOffsetDword == right.memoryOffsetDword, prefix + "memory offset differs"); @@ -595,6 +595,24 @@ void verifyStore() { require(ShaderDiskCache::Load(key, loaded), "a rewritten entry does not load"); } +void verifyPersist() { + SampleRequest sample; + const auto key = sample.Key(); + const auto variant = sampleVariant(); + const auto persisted = ShaderDiskCache::Persist(key, variant); +#ifdef _WIN32 + require(persisted == nullptr, "an entry was persisted where the cache does not map entries"); +#else + require(persisted != nullptr, "a persisted entry was not read back"); + requireSameVariant(variant, *persisted, "persist round trip"); + const auto path = ShaderDiskCache::EntryDirectory() / ShaderDiskCache::EntryName(key); + require(WriteFileAtomically(path, std::vector(64, std::byte{0x5a})), "cannot replace the persisted entry"); + std::error_code error; + std::filesystem::remove(path, error); + requireSameArtifact(variant.artifact, persisted->artifact, "a persisted entry whose file was replaced and removed"); +#endif +} + struct ComputeRequest { std::vector code{0xe0700000u, 0x80000000u, 0xbf810000u}; std::array userData{0x10000000u, 0x00000000u, 0x40u, 0x00027facu}; @@ -945,7 +963,7 @@ void verifyVertexTypeSpecialization() { } } require(checked == 4u, "vertex specialization lost attribute loads"); - require(artifact.spirv.Words() == original, "vertex specialization mutated the shared template"); + require(std::ranges::equal(artifact.spirv.Words(), original), "vertex specialization mutated the shared template"); } } auto invalid = artifact; @@ -1858,6 +1876,7 @@ int main(int argc, char** argv) { verifyPlanRoundTrip(); verifyPlanKey(); verifyStore(); + verifyPersist(); verifyAcrossProcesses(argv[0]); verifyPlanAcrossProcesses(argv[0]); verifyEmissionFailureMemo(); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index a52274cfa9..2a915f5c4d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -750,7 +750,7 @@ void verifyBindlessTable() { const std::array capabilities{29u, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; const std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; // A wave64 workgroup on a 32-wide host is held by one subgroup (two lanes per invocation). - const auto makeRequest = [&](const std::vector& code) { + const auto makeRequest = [&](std::span code) { RecompileRequest request{}; request.shader = {ShaderStage::Compute, 0x20000u, code, 0, {}}; request.context.waveSize = 64; @@ -843,7 +843,7 @@ void verifyBindlessTable() { bool nonUniform = false; bool switched = false; }; - const auto scan = [&](const std::vector& words) { + const auto scan = [&](std::span words) { SpirvScan result; for (std::size_t cursor = 5; cursor < words.size();) { const auto count = words[cursor] >> 16u; @@ -976,7 +976,7 @@ void verifyDescriptorPhis() { const std::array userData{static_cast(srtAddress), static_cast(srtAddress >> 32u)}; const std::array capabilities{29u, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; const std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; - const auto makeRequest = [&](const std::vector& code, std::uint32_t waveSize = 32u) { + const auto makeRequest = [&](std::span code, std::uint32_t waveSize = 32u) { RecompileRequest request{}; request.shader = {ShaderStage::Compute, 0x21000u, code, 0, {}}; request.context.waveSize = waveSize; @@ -993,7 +993,7 @@ void verifyDescriptorPhis() { request.layout.pushConstantSizeBytes = 128; return request; }; - const auto countOps = [](const std::vector& words, std::uint32_t opcode) { + const auto countOps = [](std::span words, std::uint32_t opcode) { std::size_t count = 0; for (std::size_t cursor = 5; cursor < words.size();) { const auto length = words[cursor] >> 16u; @@ -1009,7 +1009,7 @@ void verifyDescriptorPhis() { const std::vector imageCode{0xf4000400u, 0xfa000050u, 0xf4080500u, 0xfa000020u, 0xbf8cc07fu, 0xbf068010u, 0xbf850003u, 0xf40c0200u, 0xfa000000u, 0xbf820002u, 0xf40c0200u, 0xfa000060u, 0xf4080600u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf810000u}; const std::vector dynamicCode{0xf40c0200u, 0xfa000000u, 0xf4000400u, 0xfa000050u, 0xbf8cc07fu, 0xbf068010u, 0xbf850003u, 0xf4080500u, 0xfa000020u, 0xbf820002u, 0xf4080500u, 0x20000000u, 0xf4080600u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf810000u}; - const auto compile = [&](const std::vector& code, std::size_t images, std::size_t samplers) { + const auto compile = [&](std::span code, std::size_t images, std::size_t samplers) { auto request = makeRequest(code); const auto plan = GetResourcePlan(request); require(plan->info.images.size() == images && plan->info.samplers.size() == samplers && plan->info.sampledPairs.size() == 2u, "descriptor Phi: the edges were not given one resource each"); @@ -1092,7 +1092,7 @@ void verifyDescriptorPhis() { const std::array prologue{0xf4080500u, 0xfa000020u, 0xf4080600u, 0xfa000040u, 0xf40c0200u, 0xfa000000u, 0xbf8cc07fu}; const std::array epilogue{0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf810000u}; - const auto planImages = [&](const std::vector& body) { + const auto planImages = [&](std::span body) { std::vector code(prologue.begin(), prologue.end()); code.insert(code.end(), body.begin(), body.end()); code.insert(code.end(), epilogue.begin(), epilogue.end()); @@ -1305,14 +1305,14 @@ void verifyHalfWaveReduction() { request.useCache = false; return Recompile(request).spirv; }; - const auto reductions = [](const std::vector& spirv, spv::Op opcode) { + const auto reductions = [](std::span spirv, spv::Op opcode) { std::uint32_t count = 0; for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) { if ((spirv[cursor] & 0xffffu) == static_cast(opcode) && (spirv[cursor] >> 16u) == 6u && spirv[cursor + 4u] == spv::GroupOperationReduce) ++count; } return count; }; - const auto halves = [](const std::vector& spirv, spv::Op opcode, std::uint32_t identity) { + const auto halves = [](std::span spirv, spv::Op opcode, std::uint32_t identity) { std::map definitions; std::vector reduces; for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) { @@ -1498,7 +1498,7 @@ void verifyNarrowVertexWave() { const auto prepares = [&prepared](std::span code, std::uint32_t subgroupSize, std::uint32_t userDataBase = 8u) { return !prepared(code, subgroupSize, userDataBase).empty(); }; - const auto reductions = [](const std::vector& spirv) { + const auto reductions = [](std::span spirv) { std::uint32_t count = 0; for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) { if ((spirv[cursor] & 0xffffu) == spv::OpGroupNonUniformBitwiseOr && (spirv[cursor] >> 16u) == 6u && spirv[cursor + 4u] == spv::GroupOperationReduce) ++count; @@ -1559,8 +1559,8 @@ void verifyNarrowVertexWave() { require(first->source != second->source, "narrow vertex wave: the program prepared for 32 and for 64 lanes shares one translated source"); require(MatchesPreparedShader(narrowRequest, narrowHandle) && MatchesPreparedShader(wideRequest, wideHandle), "narrow vertex wave: a prepared program did not match its own request"); require(!MatchesPreparedShader(narrowRequest, wideHandle) && !MatchesPreparedShader(wideRequest, narrowHandle), "narrow vertex wave: a program prepared for one subgroup width matched a request for the other"); - narrow[narrowFirst ? 1u : 0u] = GetPreparedArtifact(narrowHandle).spirv; - wide[narrowFirst ? 1u : 0u] = GetPreparedArtifact(wideHandle).spirv; + narrow[narrowFirst ? 1u : 0u] = GetPreparedArtifact(narrowHandle).spirv.Copy(); + wide[narrowFirst ? 1u : 0u] = GetPreparedArtifact(wideHandle).spirv.Copy(); } require(narrow[0] == narrow[1] && wide[0] == wide[1], "narrow vertex wave: preparing for one subgroup width after the other gave another program than preparing it first"); require(narrow[0] != wide[0], "narrow vertex wave: s3 of a wave64 vertex program was the same on 32 and 64 lanes"); @@ -1589,7 +1589,7 @@ void verifySingleLaneVertexWave() { const auto compiled = [&vertex](std::span code, std::uint32_t subgroupSize, bool vertexSubgroups) { return Recompile(vertex(code, subgroupSize, vertexSubgroups)).spirv; }; - const auto usesSubgroups = [](const std::vector& spirv) { + const auto usesSubgroups = [](std::span spirv) { for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) { const auto op = spirv[cursor] & 0xffffu; if (op == spv::OpCapability && spirv[cursor + 1u] >= spv::CapabilityGroupNonUniform && spirv[cursor + 1u] <= spv::CapabilityGroupNonUniformQuad) return true; @@ -1598,7 +1598,7 @@ void verifySingleLaneVertexWave() { } return false; }; - const auto hasConstant = [](const std::vector& spirv, std::uint32_t value) { + const auto hasConstant = [](std::span spirv, std::uint32_t value) { for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) { if ((spirv[cursor] & 0xffffu) == spv::OpConstant && (spirv[cursor] >> 16u) == 4u && spirv[cursor + 3u] == value) return true; } @@ -2307,7 +2307,7 @@ void verifyComputedTexelOffsets() { }; const auto computed = program(0x100u, 0u); const auto constant = program(0xffu, 0x3fu | (1u << 8u)); - const auto recompile = [&](const std::vector& code, bool offsets) { + const auto recompile = [&](std::span code, bool offsets) { RecompileRequest request{}; request.shader = {ShaderStage::Compute, 0x30000u, code, 0, {}}; request.target = BufferTarget(); @@ -2327,7 +2327,7 @@ void verifyComputedTexelOffsets() { request.context.memory = memory.Regions(); return Recompile(request).spirv; }; - const auto sampleOperands = [](const std::vector& words) { + const auto sampleOperands = [](std::span words) { std::uint32_t mask = 0; bool gatherExtended = false; for (std::size_t cursor = 5; cursor < words.size();) { @@ -2412,7 +2412,7 @@ void verifyInt64AtomicCapabilities() { 0u, 0u, 0u, 0u}; const std::vector bufferAtomic{0xe1705000u, 0x80012803u, 0xbf810000u}; const std::vector imageAtomic{0xf03c0308u, 0x00020a08u, 0xbf810000u}; - const auto compile = [&](const std::vector& code, spv::Capability added) { + const auto compile = [&](std::span code, spv::Capability added) { const std::array capabilities{spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, static_cast(added)}; RecompileRequest request{}; request.shader = {ShaderStage::Compute, 0x58000u, code, 0, {}}; @@ -2431,7 +2431,7 @@ void verifyInt64AtomicCapabilities() { request.useCache = false; return Recompile(request).spirv; }; - const auto declares = [](const std::vector& words, spv::Capability capability) { + const auto declares = [](std::span words, spv::Capability capability) { for (std::size_t cursor = 5; cursor < words.size();) { const auto count = words[cursor] >> 16u; require(count != 0 && count <= words.size() - cursor, "64-bit atomics: truncated SPIR-V instruction"); @@ -2488,7 +2488,7 @@ void verifySpecializedBufferAtomicBindings() { require(tables == 0, "specialized buffer atomic: a buffer_atomic_inc through a V# the program specializes binds the address tables"); require(faults == 1 && result.bdaAbiVersion == BdaAbi::Version, "specialized buffer atomic: the program has no fault buffer for its unaligned atomic"); require(atomicElements == 1, "specialized buffer atomic: the V# element is not reported as atomic"); - const std::vector& words = result.spirv; + const auto words = result.spirv.Words(); std::uint32_t faultFunction = 0; std::size_t faultCalls = 0; bool physical = false; @@ -2672,7 +2672,7 @@ void verifyUnnormalizedSamplers() { }; const std::array capabilities{1u, static_cast(spv::CapabilityImageGatherExtended), static_cast(spv::CapabilityMinLod)}; std::uint64_t nextAddress = 0x40000u; - const auto recompile = [&](const std::vector& code, const std::array& data, std::uint64_t address = 0u, bool offsets = false) { + const auto recompile = [&](std::span code, const std::array& data, std::uint64_t address = 0u, bool offsets = false) { RecompileRequest request{}; request.shader = {ShaderStage::Compute, address != 0u ? address : (nextAddress += 0x1000u), code, 0, {}}; request.context.waveSize = 32; @@ -2700,7 +2700,7 @@ void verifyUnnormalizedSamplers() { const auto proven = [&](const RecompileResult& result, bool expected) { return flags(result, DescriptorRole::GuestSamplers) == std::vector{expected} && flags(result, DescriptorRole::GuestImages) == std::vector{expected}; }; - const auto auditSpirv = [](const std::vector& words) { + const auto auditSpirv = [](std::span words) { std::size_t samples = 0; for (std::size_t cursor = 5; cursor < words.size();) { const auto count = words[cursor] >> 16u; @@ -2722,7 +2722,7 @@ void verifyUnnormalizedSamplers() { auditSpirv(accepted.spirv.Words()); const auto plain = recompile(lz, userData(normalized), 0x40000u); require(proven(plain, false), "unnormalized samplers: a normalized S# was flagged"); - require(plain.variantId == accepted.variantId && plain.spirv.Words() == accepted.spirv.Words(), "unnormalized samplers: the unnormalized S# compiled another variant"); + require(plain.variantId == accepted.variantId && plain.spirv == accepted.spirv, "unnormalized samplers: the unnormalized S# compiled another variant"); for (const std::uint32_t mimg : {0xf0900f08u, 0xf0800f08u, 0xf0940f08u, 0xf0840f08u, 0xf0880f08u}) { const auto result = recompile(program(mimg), userData(unnormalized)); @@ -2756,7 +2756,7 @@ void verifyUnnormalizedSamplers() { code.insert(code.end(), {0x7e040280u, 0x7e060280u, 0x7e080280u, 0x7e0a0280u, 0x7e0c0280u, mimg, 0x00400801u, 0xe0700000u, 0x80030800u, 0xbf810000u}); return code; }; - const auto foldedSamples = [](const std::vector& words) { + const auto foldedSamples = [](std::span words) { std::vector exact; std::vector sums; std::vector coordinates; @@ -2771,7 +2771,7 @@ void verifyUnnormalizedSamplers() { } return !coordinates.empty() && std::ranges::all_of(coordinates, [&](std::uint32_t id) { return std::ranges::count(sums, id) != 0 && std::ranges::count(exact, id) != 0; }); }; - const auto sampleMasks = [](const std::vector& words) { + const auto sampleMasks = [](std::span words) { std::vector masks; for (std::size_t cursor = 5; cursor < words.size();) { const auto count = words[cursor] >> 16u; @@ -2831,7 +2831,7 @@ void verifyUnnormalizedSamplers() { pixel.target.subgroupSize = 64; pixel.layout.pushConstantSizeBytes = 128; pixel.useCache = false; - const auto implicitSamples = [](const std::vector& words) { + const auto implicitSamples = [](std::span words) { std::size_t samples = 0; for (std::size_t cursor = 5; cursor < words.size();) { const auto count = words[cursor] >> 16u; diff --git a/core/libs/prx/libSceAgcDriver/tests/SpirvBranchTargets.cpp b/core/libs/prx/libSceAgcDriver/tests/SpirvBranchTargets.cpp index dda775da27..3add236bdb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/SpirvBranchTargets.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/SpirvBranchTargets.cpp @@ -41,7 +41,7 @@ std::vector recompile(std::span code) { request.useCache = false; auto result = Recompile(request); require(!result.spirv.empty(), "the shader did not recompile"); - return std::move(result.spirv); + return result.spirv.Copy(); } void requireDistinctTargets(const char* name, std::span code) { diff --git a/core/libs/prx/libSceAgcDriver/tests/VertexLayerExport.cpp b/core/libs/prx/libSceAgcDriver/tests/VertexLayerExport.cpp index 27a7785a5a..1b8f715065 100644 --- a/core/libs/prx/libSceAgcDriver/tests/VertexLayerExport.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/VertexLayerExport.cpp @@ -16,7 +16,7 @@ struct Module { std::uint32_t version = 0; }; -Module inspect(const std::vector& words) { +Module inspect(std::span words) { Module module; if (words.size() > 1) module.version = words[1]; std::vector layerIds; @@ -32,7 +32,7 @@ Module inspect(const std::vector& words) { return module; } -bool has(const std::vector& values, std::uint32_t value) { +bool has(std::span values, std::uint32_t value) { for (const auto entry : values) { if (entry == value) return true; } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64FmaSubnormals.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64FmaSubnormals.cpp index cb9823b3a0..b00770b08e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64FmaSubnormals.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64FmaSubnormals.cpp @@ -135,7 +135,7 @@ std::vector Recompile(bool float64DenormPreserve) { request.target.float64DenormPreserve = float64DenormPreserve; request.layout = {0, 0, 0, 128}; request.useCache = false; - return ShaderRecompiler::Recompile(request).spirv; + return ShaderRecompiler::Recompile(request).spirv.Copy(); } bool HasDenormPreserve64(std::span words) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LayeredVolumeDraw.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LayeredVolumeDraw.cpp index 8f419f2164..20ab53b94c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LayeredVolumeDraw.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LayeredVolumeDraw.cpp @@ -137,7 +137,7 @@ private: std::byte* block = nullptr; }; -bool DeclaresLayer(const std::vector& words) { +bool DeclaresLayer(std::span words) { for (std::size_t at = 5; at < words.size();) { const auto length = words[at] >> 16u; const auto opcode = words[at] & 0xffffu; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ZeroStrideVertexFetch.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ZeroStrideVertexFetch.cpp index a308a40635..5715c39d27 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ZeroStrideVertexFetch.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ZeroStrideVertexFetch.cpp @@ -78,8 +78,8 @@ void Draw(AgcDriver::VulkanDevice& device, const Case& value) { const auto vertexResult = ShaderRecompiler::Recompile(vertexRequest); Require(vertexResult.vertexAttributes.size() == 1u, std::string(value.what) + ": the fetch did not become a vertex attribute"); if (device.Target().registrationModules) { - if (registrationModule.empty()) registrationModule = vertexResult.spirv.Words(); - Require(vertexResult.spirv.Words() == registrationModule, std::string(value.what) + ": registration modules: the V# specialized the vertex module"); + if (registrationModule.empty()) registrationModule = vertexResult.spirv.Copy(); + Require(std::ranges::equal(vertexResult.spirv.Words(), registrationModule), std::string(value.what) + ": registration modules: the V# specialized the vertex module"); } const auto vertexPush = static_cast(vertexResult.pushConstants.size()); diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 46f382ce18..2ce957d6f9 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -958,7 +958,10 @@ std::shared_ptr findOrCompileVariant(SourceEntry& source, if (emissionFailure != nullptr && FailureMemo()) source.emissionFailures.push_back({request.shader.codeAddress, request.layout, emissionFailure}); throw; } - if (disk) ShaderDiskCache::Store(std::move(diskKey), variant); + if (disk) { + if (auto persisted = ShaderDiskCache::Persist(diskKey, *variant)) variant = std::move(persisted); + else ShaderDiskCache::Store(std::move(diskKey), variant); + } } source.program.reset(); source.variants.push_back(variant); diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 9a22de5259..08de057b2d 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -3,6 +3,7 @@ #include "RuntimeAbi.hpp" #include "PipelineSpecialization.hpp" +#include #include #include #include @@ -434,39 +435,43 @@ class SharedSpirv { public: SharedSpirv() = default; SharedSpirv(std::vector words) : words(std::make_shared>(std::move(words))) {} + SharedSpirv(std::shared_ptr owner, std::span view) : owner(std::move(owner)), view(view) {} SharedSpirv& operator=(std::vector other) { words = std::make_shared>(std::move(other)); + owner.reset(); + view = {}; return *this; } - [[nodiscard]] const std::vector& Words() const { return words ? *words : Empty(); } - operator const std::vector&() const { return Words(); } + [[nodiscard]] std::span Words() const { return words ? std::span(*words) : view; } + [[nodiscard]] std::vector Copy() const { return {Words().begin(), Words().end()}; } [[nodiscard]] std::size_t size() const { return Words().size(); } [[nodiscard]] bool empty() const { return Words().empty(); } [[nodiscard]] const std::uint32_t* data() const { return Words().data(); } [[nodiscard]] std::uint32_t* data() { return Mutable().data(); } [[nodiscard]] const std::uint32_t& operator[](std::size_t index) const { return Words()[index]; } [[nodiscard]] std::uint32_t& operator[](std::size_t index) { return Mutable()[index]; } - [[nodiscard]] std::vector::const_iterator begin() const { return Words().begin(); } - [[nodiscard]] std::vector::const_iterator end() const { return Words().end(); } + [[nodiscard]] std::span::iterator begin() const { return Words().begin(); } + [[nodiscard]] std::span::iterator end() const { return Words().end(); } [[nodiscard]] std::vector::iterator begin() { return Mutable().begin(); } [[nodiscard]] std::vector::iterator end() { return Mutable().end(); } void resize(std::size_t count) { Mutable().resize(count); } std::vector::iterator insert(std::vector::const_iterator where, std::initializer_list values) { return Mutable().insert(where, values); } - friend bool operator==(const SharedSpirv& left, const SharedSpirv& right) { return left.words == right.words || left.Words() == right.Words(); } + friend bool operator==(const SharedSpirv& left, const SharedSpirv& right) { return std::ranges::equal(left.Words(), right.Words()); } private: - static const std::vector& Empty() { - static const std::vector empty; - return empty; - } - // Copy on write: a holder whose words are shared gets its own copy before the first write. std::vector& Mutable() { - if (words == nullptr || words.use_count() != 1) words = std::make_shared>(Words()); + if (words == nullptr || words.use_count() != 1) { + words = std::make_shared>(Copy()); + owner.reset(); + view = {}; + } return *words; } std::shared_ptr> words; + std::shared_ptr owner; + std::span view; }; struct VertexInput { diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 8d934a1716..36d4e68724 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -14,6 +14,12 @@ #elif defined(__APPLE__) #include #endif +#ifndef _WIN32 +#include +#include +#include +#include +#endif #include "ShaderCacheVersion.hpp" #include #include @@ -77,9 +83,9 @@ std::filesystem::path ShaderCacheDirectory() { namespace ShaderRecompiler::ShaderDiskCache { #if defined(__linux__) && defined(__x86_64__) && defined(__GLIBCXX__) -static_assert(sizeof(CompiledShaderArtifact) == 200, "CompiledShaderArtifact changed: update the artifact encoder"); +static_assert(sizeof(CompiledShaderArtifact) == 232, "CompiledShaderArtifact changed: update the artifact encoder"); static_assert(sizeof(ShaderInvocation) == 112, "ShaderInvocation changed: update the invocation encoder"); -static_assert(sizeof(RecompileResult) == 320, "RecompileResult changed: update EncodeResult and DecodeResult"); +static_assert(sizeof(RecompileResult) == 352, "RecompileResult changed: update EncodeResult and DecodeResult"); static_assert(sizeof(DescriptorBinding) == 568, "DescriptorBinding changed: update the binding encoder"); static_assert(sizeof(VertexAttribute) == 32, "VertexAttribute changed: update the attribute encoder"); static_assert(sizeof(VertexInput) == 16, "VertexInput changed: update the vertex input encoder"); @@ -158,6 +164,38 @@ struct FileHeader { std::uint64_t keyHash; std::uint64_t payloadHash; }; + +std::size_t PayloadOffset(std::size_t keyBytes) { + constexpr std::size_t alignment = 8; + return (sizeof(FileHeader) + keyBytes + alignment - 1) / alignment * alignment; +} + +#ifndef _WIN32 +class MappedFile { +public: + MappedFile(const void* data, std::size_t size) : data(data), size(size) {} + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + ~MappedFile() { munmap(const_cast(data), size); } + + [[nodiscard]] std::span Bytes() const { return {static_cast(data), size}; } + +private: + const void* data; + std::size_t size; +}; + +std::shared_ptr MapFile(const std::filesystem::path& path) { + const int descriptor = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (descriptor < 0) return nullptr; + struct stat status{}; + void* data = MAP_FAILED; + if (fstat(descriptor, &status) == 0 && status.st_size > 0) data = mmap(nullptr, static_cast(status.st_size), PROT_READ, MAP_PRIVATE, descriptor, 0); + close(descriptor); + if (data == MAP_FAILED) return nullptr; + return std::make_shared(data, static_cast(status.st_size)); +} +#endif static_assert(sizeof(FileHeader) == 48 && std::is_trivially_copyable_v); class Writer { @@ -220,7 +258,7 @@ private: class Reader { public: - explicit Reader(std::span bytes) : bytes(bytes) {} + explicit Reader(std::span bytes, std::shared_ptr owner = nullptr) : bytes(bytes), owner(std::move(owner)) {} [[nodiscard]] bool Ok() const { return ok; } [[nodiscard]] bool Done() const { return ok && position == bytes.size(); } @@ -271,6 +309,18 @@ public: } } + SharedSpirv Spirv() { + const auto count = Count(sizeof(std::uint32_t)); + const auto* first = bytes.data() + position; + position += count * sizeof(std::uint32_t); + if (owner != nullptr && reinterpret_cast(first) % alignof(std::uint32_t) == 0) { + return SharedSpirv(owner, std::span(reinterpret_cast(first), count)); + } + std::vector words(count); + if (count != 0) std::memcpy(words.data(), first, count * sizeof(std::uint32_t)); + return SharedSpirv(std::move(words)); + } + void Flags(std::vector& flags) { const auto count = Count(1); flags.assign(count, false); @@ -316,6 +366,7 @@ private: } std::span bytes; + std::shared_ptr owner; std::size_t position = 0; bool ok = true; }; @@ -415,9 +466,7 @@ void encodeArtifact(Writer& writer, const CompiledShaderArtifact& result) { void decodeArtifact(Reader& reader, CompiledShaderArtifact& result) { reader.Value(result.runtimeAbiVersion); - std::vector words; - reader.Values(words); - result.spirv = std::move(words); + result.spirv = reader.Spirv(); reader.Value(result.bdaAbiVersion); reader.Value(result.memoryOffsetDword); reader.Value(result.shaderDataDwords); @@ -1114,10 +1163,11 @@ std::string hex(std::uint64_t value) { std::vector wrapEntry(std::span key, std::span payload) { const FileHeader header{FileMagic, FormatVersion, SourceVersion(), key.size(), payload.size(), HashBytes(key), HashBytes(payload)}; - std::vector file(sizeof(header) + key.size() + payload.size()); + const auto payloadOffset = PayloadOffset(key.size()); + std::vector file(payloadOffset + payload.size()); std::memcpy(file.data(), &header, sizeof(header)); std::memcpy(file.data() + sizeof(header), key.data(), key.size()); - if (!payload.empty()) std::memcpy(file.data() + sizeof(header) + key.size(), payload.data(), payload.size()); + if (!payload.empty()) std::memcpy(file.data() + payloadOffset, payload.data(), payload.size()); return file; } @@ -1127,9 +1177,13 @@ LoadStatus unwrapEntry(std::span file, std::span body || header.payloadBytes != body - header.keyBytes) return LoadStatus::Rejected; + if (header.keyBytes > body) return LoadStatus::Rejected; + const auto payloadOffset = PayloadOffset(static_cast(header.keyBytes)); + if (payloadOffset > file.size() || header.payloadBytes != file.size() - payloadOffset) return LoadStatus::Rejected; const auto storedKey = file.subspan(sizeof(header), static_cast(header.keyBytes)); - payload = file.subspan(sizeof(header) + static_cast(header.keyBytes)); + const auto padding = file.subspan(sizeof(header) + storedKey.size(), payloadOffset - sizeof(header) - storedKey.size()); + if (std::ranges::any_of(padding, [](std::byte value) { return value != std::byte{0}; })) return LoadStatus::Rejected; + payload = file.subspan(payloadOffset); if (HashBytes(storedKey) != header.keyHash) return LoadStatus::Rejected; if (storedKey.size() != key.size() || !std::equal(storedKey.begin(), storedKey.end(), key.begin())) return LoadStatus::KeyMismatch; if (HashBytes(payload) != header.payloadHash) return LoadStatus::Rejected; @@ -1150,11 +1204,11 @@ public: [[nodiscard]] const std::filesystem::path& Directory() const { return directory; } bool Load(std::span key, CompiledVariant& variant) { - return load(key, [&](std::span file) { return DecodeEntry(file, key, variant); }, hits, misses, loadFailures); + return load(key, [&](std::span file, std::shared_ptr owner) { return DecodeEntry(file, key, variant, std::move(owner)); }, hits, misses, loadFailures); } bool LoadPlan(std::span key, IrResourcePlan& plan) { - return load(key, [&](std::span file) { return DecodePlanEntry(file, key, plan); }, planHits, planMisses, planLoadFailures); + return load(key, [&](std::span file, std::shared_ptr) { return DecodePlanEntry(file, key, plan); }, planHits, planMisses, planLoadFailures); } void Store(std::vector key, std::shared_ptr variant) { @@ -1165,6 +1219,28 @@ public: enqueue({std::move(key), nullptr, std::move(plan)}); } + std::shared_ptr Persist(std::span key, const CompiledVariant& variant) { +#ifdef _WIN32 + static_cast(key); + static_cast(variant); + return nullptr; +#else + const auto path = directory / EntryName(key); + const auto file = EncodeEntry(key, variant); + if (!WriteFileAtomically(path, file)) return nullptr; + writes.fetch_add(1, std::memory_order_relaxed); + bytesWritten.fetch_add(file.size(), std::memory_order_relaxed); + auto mapped = MapFile(path); + if (mapped == nullptr) return nullptr; + auto persisted = std::make_shared(); + const auto bytes = mapped->Bytes(); + if (DecodeEntry(bytes, key, *persisted, std::move(mapped)) != LoadStatus::Loaded) return nullptr; + persisted->layout = variant.layout; + persisted->artifact.variantId = variant.artifact.variantId; + return persisted; +#endif + } + void Flush() { std::unique_lock lock(mutex); idle.wait(lock, [&] { return jobs.empty() && !writing; }); @@ -1200,13 +1276,23 @@ private: thread_local std::vector* fileSlot = nullptr; auto& file = ThreadOwned(fileSlot); bool loaded = false; - if (!ReadWholeFile(directory / name, file)) { + std::span bytes; + std::shared_ptr owner; +#ifndef _WIN32 + if (auto mapped = MapFile(directory / name)) { + bytes = mapped->Bytes(); + owner = std::move(mapped); + } +#endif + const bool present = owner != nullptr || ReadWholeFile(directory / name, file); + if (owner == nullptr) bytes = file; + if (!present) { missCount.fetch_add(1, std::memory_order_relaxed); } else { - switch (decode(std::span(file))) { + switch (decode(bytes, std::move(owner))) { case LoadStatus::Loaded: hitCount.fetch_add(1, std::memory_order_relaxed); - bytesRead.fetch_add(file.size(), std::memory_order_relaxed); + bytesRead.fetch_add(bytes.size(), std::memory_order_relaxed); loaded = true; break; case LoadStatus::Absent: @@ -1215,7 +1301,7 @@ private: break; case LoadStatus::Rejected: { const auto failures = failureCount.fetch_add(1, std::memory_order_relaxed); - if (failures < 8) std::fprintf(stderr, "[shader-disk-cache] rejected %s (%zu bytes): truncated, corrupt or another format; recompiling\n", name.c_str(), file.size()); + if (failures < 8) std::fprintf(stderr, "[shader-disk-cache] rejected %s (%zu bytes): truncated, corrupt or another format; recompiling\n", name.c_str(), bytes.size()); break; } } @@ -1414,11 +1500,11 @@ std::vector EncodeEntry(std::span key, const Compile return wrapEntry(key, payload); } -LoadStatus DecodeEntry(std::span file, std::span key, CompiledVariant& variant) { +LoadStatus DecodeEntry(std::span file, std::span key, CompiledVariant& variant, std::shared_ptr owner) { std::span payload; const auto status = unwrapEntry(file, key, payload); if (status != LoadStatus::Loaded) return status; - Reader reader(payload); + Reader reader(payload, std::move(owner)); CompiledVariant decoded; decodeArtifact(reader, decoded.artifact); decodeInfo(reader, decoded.info); @@ -1484,6 +1570,11 @@ void StorePlan(std::vector key, std::shared_ptr if (instance.Enabled() && plan != nullptr) instance.StorePlan(std::move(key), std::move(plan)); } +std::shared_ptr Persist(std::span key, const CompiledVariant& variant) { + auto& instance = store(); + return instance.Enabled() ? instance.Persist(key, variant) : nullptr; +} + void Flush() { auto& instance = store(); if (instance.Enabled()) instance.Flush(); diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 530ccae9c4..327eb2f30a 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -13,7 +13,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 30; +inline constexpr std::uint32_t FormatVersion = 31; enum class LoadStatus { Loaded, @@ -46,7 +46,7 @@ void EncodeResult(const RecompileResult& result, std::vector& out); [[nodiscard]] bool DecodeResult(std::span bytes, RecompileResult& result); [[nodiscard]] std::vector EncodeEntry(std::span key, const CompiledVariant& variant); -[[nodiscard]] LoadStatus DecodeEntry(std::span file, std::span key, CompiledVariant& variant); +[[nodiscard]] LoadStatus DecodeEntry(std::span file, std::span key, CompiledVariant& variant, std::shared_ptr owner = nullptr); void BuildPlanKey(const RecompileRequest& request, std::uint32_t hostSubgroupSize, std::vector& key); [[nodiscard]] bool EncodePlan(const IrResourcePlan& plan, std::vector& out); @@ -59,6 +59,7 @@ void BuildPlanKey(const RecompileRequest& request, std::uint32_t hostSubgroupSiz [[nodiscard]] bool Load(std::span key, CompiledVariant& variant); void Store(std::vector key, std::shared_ptr variant); +[[nodiscard]] std::shared_ptr Persist(std::span key, const CompiledVariant& variant); [[nodiscard]] bool LoadPlan(std::span key, IrResourcePlan& plan); void StorePlan(std::vector key, std::shared_ptr plan); void Flush(); diff --git a/core/shader/recompiler/VertexInputSpecialization.hpp b/core/shader/recompiler/VertexInputSpecialization.hpp index 2b2722eed6..12053a78a5 100644 --- a/core/shader/recompiler/VertexInputSpecialization.hpp +++ b/core/shader/recompiler/VertexInputSpecialization.hpp @@ -16,7 +16,7 @@ namespace ShaderRecompiler { inline SharedSpirv SpecializeVertexInputTypes(const CompiledShaderArtifact& artifact, std::span classes) { if (artifact.vertexInputPatches.empty()) return artifact.spirv; - auto words = artifact.spirv.Words(); + auto words = artifact.spirv.Copy(); for (const auto& patch : artifact.vertexInputPatches) { if (patch.location >= classes.size() || classes[patch.location] >= patch.values.size() || patch.word >= words.size()) throw std::runtime_error("invalid prepared vertex type patch"); words[patch.word] = patch.values[classes[patch.location]];