diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 1a8234e8f7..fab9d851c4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -2481,6 +2481,9 @@ void verifyBdaReadFallbackFunctions() { std::size_t mainProbes = 0; std::size_t spanCalls = 0; std::array readerLoads{}; + std::size_t lookupFaults = 0; + std::size_t lookupVariables = 0; + std::size_t readerLookups = 0; for (std::size_t cursor = 5; cursor < five.size();) { const auto length = five[cursor] >> 16u; require(length != 0 && length <= five.size() - cursor, "BDA read functions: truncated SPIR-V instruction"); @@ -2501,13 +2504,19 @@ void verifyBdaReadFallbackFunctions() { if (callee == span) ++spanCalls; } if (op == spv::OpLoad && function == reader) ++readerLoads[length > 4u && (five[cursor + 4] & spv::MemoryAccessVolatileMask) != 0u]; + if (op == spv::OpFunctionCall && function == "get_bda_pointer" && names[five[cursor + 3]] == "record_bda_fault") ++lookupFaults; + if (op == spv::OpVariable && function == "get_bda_pointer") ++lookupVariables; + if (op == spv::OpFunctionCall && function == reader && names[five[cursor + 3]] == "get_bda_pointer") ++readerLookups; cursor += length; } require(definitions["record_bda_fault"] == 1u && definitions[reader] == 1u && definitions[span] == 1u, "BDA read functions: the fault, byte read and span read functions are not defined"); for (const auto& [name, count] : definitions) require(count == 1u, "BDA read functions: a function is defined twice"); require(compareExchanges == 1u, "BDA read functions: a fault is recorded outside record_bda_fault"); require(mainLookups == 0u, "BDA read functions: a read site looks up its bytes inline"); - require(readerLoads[coherent] == 4u && readerLoads[!coherent] == 0u, "BDA read functions: the byte loads do not keep the access's coherence"); + require(readerLoads[coherent] == 1u && readerLoads[!coherent] == 0u, "BDA read functions: the byte loads do not keep the access's coherence"); + require(lookupFaults <= 4u, "BDA read functions: the page-table lookup records its faults from more than 4 call sites"); + require(lookupVariables == 0u, "BDA read functions: the page-table lookup keeps its search bounds in function variables"); + require(readerLookups == 1u, "BDA read functions: the byte read does not look up its bytes once, in a loop"); require(mainProbes == 0u, "BDA read functions: a read site probes or reads bytes outside its span read function"); require(spanCalls == 5u, "BDA read functions: a read site does not call its span read function once"); require(five.size() - one.size() < 4u * 150u, "BDA read functions: a read site takes 150 SPIR-V words or more"); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp index 0b308f556c..21dda89e68 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp @@ -17,6 +17,7 @@ void DefineBdaFaultFunction(SpirvEmitterState& state); void DefineBdaDwordReadFunctions(SpirvEmitterState& state); void DefineBdaSpanReadFunctions(SpirvEmitterState& state); void RecordBdaFault(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); +void RecordBdaFaultReason(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, std::uint32_t reason); void RecordBdaFaultWords(SpirvEmitterState& state, std::uint32_t addressLow, std::uint32_t addressHigh, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); void ReturnBdaFailureIf(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& target); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp index ca8c1a2678..4b0d60d921 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp @@ -75,6 +75,11 @@ void RecordBdaFault(SpirvEmitterState& state, std::uint32_t address, std::uint32 state.module.AddFunction(spv::OpFunctionCall, TypeVoid(state), state.module.AllocateId(), state.bdaFaultFunction, ConstantU32(state, static_cast(reason)), address, bytes, instruction); } +void RecordBdaFaultReason(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, std::uint32_t reason) { + if (state.bdaFaultFunction == 0 || !state.program.Info().usesDma) throw std::runtime_error("BDA fault function is missing"); + state.module.AddFunction(spv::OpFunctionCall, TypeVoid(state), state.module.AllocateId(), state.bdaFaultFunction, reason, address, bytes, instruction); +} + void RecordBdaFaultWords(SpirvEmitterState& state, std::uint32_t addressLow, std::uint32_t addressHigh, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason) { if (state.program.Info().usesDma) { const auto u64 = TypeScalarU64(state); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp index 248ff205c0..b48e8775b5 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp @@ -2,6 +2,9 @@ #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" #include +#include +#include +#include namespace ShaderRecompiler { @@ -24,11 +27,6 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r const auto boolean = TypeBool(state); const auto constant = [&](std::uint32_t value) { return ConstantU32(state, value); }; const auto binary = [&](std::uint32_t op, std::uint32_t type, std::uint32_t left, std::uint32_t right) { return Binary(state, op, type, left, right); }; - const auto load = [&](std::uint32_t pointer) { - const auto value = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, u32, value, pointer); - return value; - }; const auto function = state.module.AllocateId(); state.module.AddName(function, name); state.module.AddFunction(spv::OpFunction, u64, function, spv::FunctionControlDontInlineMask, state.module.Type(spv::OpTypeFunction, u64, u64, u32, u32)); @@ -39,11 +37,6 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r state.module.AddFunction(spv::OpFunctionParameter, u32, bytes); state.module.AddFunction(spv::OpFunctionParameter, u32, instruction); EmitLabel(state, state.module.AllocateId()); - const auto low = state.module.AllocateId(); - const auto high = state.module.AllocateId(); - const auto pointer = TypePointer(state, spv::StorageClassFunction, u32); - state.module.AddFunction(spv::OpVariable, pointer, low, spv::StorageClassFunction); - state.module.AddFunction(spv::OpVariable, pointer, high, spv::StorageClassFunction); const auto fail = [&](std::uint32_t condition, BdaAbi::FaultReason reason) { if (recordFaults) { ReturnBdaFailureIf(state, condition, address, bytes, instruction, reason); @@ -51,29 +44,53 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r ReturnBdaZeroIf(state, condition); } }; + const auto failFirst = [&](std::initializer_list> checks) { + auto reason = constant(0u); + for (auto check = std::rbegin(checks); check != std::rend(checks); ++check) { + reason = Select(state, u32, check->first, constant(static_cast(check->second)), reason); + } + const auto failed = binary(spv::OpINotEqual, boolean, reason, constant(0u)); + if (!recordFaults) { + ReturnBdaZeroIf(state, failed); + return; + } + const auto failedLabel = state.module.AllocateId(); + const auto next = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelectionMerge, next, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, failed, failedLabel, next); + EmitLabel(state, failedLabel); + RecordBdaFaultReason(state, address, bytes, instruction, reason); + state.module.AddFunction(spv::OpReturnValue, BdaConstant(state, 0u)); + EmitLabel(state, next); + }; const auto length = state.module.AllocateId(); state.module.AddFunction(spv::OpArrayLength, u32, length, state.bdaPagetableVariable, 0u); fail(binary(spv::OpULessThan, boolean, length, constant(4)), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpINotEqual, boolean, BdaLoadWord(state, constant(0)), constant(BdaAbi::Version)), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpINotEqual, boolean, BdaLoadWord(state, constant(2)), constant(sizeof(BdaAbi::Range))), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpINotEqual, boolean, BdaLoadWord(state, constant(3)), constant(0)), BdaAbi::FaultReason::InvalidTable); const auto count = BdaLoadWord(state, constant(1)); const auto available = binary(spv::OpISub, u32, length, constant(4)); - fail(binary(spv::OpINotEqual, boolean, binary(spv::OpBitwiseAnd, u32, available, constant(7)), constant(0)), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpINotEqual, boolean, count, binary(spv::OpShiftRightLogical, u32, available, constant(3))), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpIEqual, boolean, bytes, constant(0)), BdaAbi::FaultReason::Overflow); const auto end = binary(spv::OpIAdd, u64, address, Unary(state, spv::OpUConvert, u64, bytes)); - fail(binary(spv::OpULessThanEqual, boolean, end, address), BdaAbi::FaultReason::Overflow); - state.module.AddFunction(spv::OpStore, low, constant(0)); - state.module.AddFunction(spv::OpStore, high, count); + failFirst({ + {binary(spv::OpINotEqual, boolean, BdaLoadWord(state, constant(0)), constant(BdaAbi::Version)), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpINotEqual, boolean, BdaLoadWord(state, constant(2)), constant(sizeof(BdaAbi::Range))), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpINotEqual, boolean, BdaLoadWord(state, constant(3)), constant(0)), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpINotEqual, boolean, binary(spv::OpBitwiseAnd, u32, available, constant(7)), constant(0)), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpINotEqual, boolean, count, binary(spv::OpShiftRightLogical, u32, available, constant(3))), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpIEqual, boolean, bytes, constant(0)), BdaAbi::FaultReason::Overflow}, + {binary(spv::OpULessThanEqual, boolean, end, address), BdaAbi::FaultReason::Overflow}, + }); const auto header = state.module.AllocateId(); const auto body = state.module.AllocateId(); const auto continuation = state.module.AllocateId(); const auto merge = state.module.AllocateId(); + const auto entryLabel = state.currentLabel; state.module.AddFunction(spv::OpBranch, header); EmitLabel(state, header); - const auto lower = load(low); - const auto upper = load(high); + const auto lower = state.module.AllocateId(); + const auto upper = state.module.AllocateId(); + const auto newLow = state.module.AllocateId(); + const auto newHigh = state.module.AllocateId(); + state.module.AddFunction(spv::OpPhi, u32, lower, constant(0), entryLabel, newLow, continuation); + state.module.AddFunction(spv::OpPhi, u32, upper, count, entryLabel, newHigh, continuation); const auto search = binary(spv::OpULessThan, boolean, lower, upper); state.module.AddFunction(spv::OpLoopMerge, merge, continuation, spv::LoopControlMaskNone); state.module.AddFunction(spv::OpBranchConditional, search, body, merge); @@ -81,17 +98,13 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r const auto midpoint = binary(spv::OpIAdd, u32, lower, binary(spv::OpShiftRightLogical, u32, binary(spv::OpISub, u32, upper, lower), constant(1))); const auto entry = binary(spv::OpIAdd, u32, constant(4), binary(spv::OpIMul, u32, midpoint, constant(8))); const auto before = binary(spv::OpULessThan, boolean, address, BdaLoadAddress(state, entry)); - const auto newLow = state.module.AllocateId(); - const auto newHigh = state.module.AllocateId(); state.module.AddFunction(spv::OpSelect, u32, newLow, before, lower, binary(spv::OpIAdd, u32, midpoint, constant(1))); state.module.AddFunction(spv::OpSelect, u32, newHigh, before, midpoint, upper); - state.module.AddFunction(spv::OpStore, low, newLow); - state.module.AddFunction(spv::OpStore, high, newHigh); state.module.AddFunction(spv::OpBranch, continuation); EmitLabel(state, continuation); state.module.AddFunction(spv::OpBranch, header); EmitLabel(state, merge); - const auto index = load(low); + const auto index = lower; fail(binary(spv::OpIEqual, boolean, index, constant(0)), BdaAbi::FaultReason::Unmapped); const auto selected = binary(spv::OpIAdd, u32, constant(4), binary(spv::OpIMul, u32, binary(spv::OpISub, u32, index, constant(1)), constant(8))); const auto at = [&](std::uint32_t offset) { return binary(spv::OpIAdd, u32, selected, constant(offset)); }; @@ -99,16 +112,18 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r const auto finish = BdaLoadAddress(state, at(2)); const auto base = BdaLoadAddress(state, at(4)); const auto permissions = BdaLoadWord(state, at(6)); - fail(binary(spv::OpINotEqual, boolean, BdaLoadWord(state, at(7)), constant(0)), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpUGreaterThanEqual, boolean, begin, finish), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpUGreaterThan, boolean, end, finish), BdaAbi::FaultReason::Unmapped); - fail(binary(spv::OpINotEqual, boolean, binary(spv::OpBitwiseAnd, u32, permissions, constant(permission)), constant(permission)), BdaAbi::FaultReason::Permission); - fail(binary(spv::OpIEqual, boolean, base, BdaConstant(state, 0)), BdaAbi::FaultReason::InvalidTable); const auto offset = binary(spv::OpISub, u64, address, begin); const auto result = binary(spv::OpIAdd, u64, base, offset); - fail(binary(spv::OpULessThan, boolean, result, base), BdaAbi::FaultReason::Overflow); const auto deviceEnd = binary(spv::OpIAdd, u64, result, Unary(state, spv::OpUConvert, u64, bytes)); - fail(binary(spv::OpULessThanEqual, boolean, deviceEnd, result), BdaAbi::FaultReason::Overflow); + failFirst({ + {binary(spv::OpINotEqual, boolean, BdaLoadWord(state, at(7)), constant(0)), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpUGreaterThanEqual, boolean, begin, finish), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpUGreaterThan, boolean, end, finish), BdaAbi::FaultReason::Unmapped}, + {binary(spv::OpINotEqual, boolean, binary(spv::OpBitwiseAnd, u32, permissions, constant(permission)), constant(permission)), BdaAbi::FaultReason::Permission}, + {binary(spv::OpIEqual, boolean, base, BdaConstant(state, 0)), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpULessThan, boolean, result, base), BdaAbi::FaultReason::Overflow}, + {binary(spv::OpULessThanEqual, boolean, deviceEnd, result), BdaAbi::FaultReason::Overflow}, + }); state.module.AddFunction(spv::OpReturnValue, result); state.module.AddFunction(spv::OpFunctionEnd); return function; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp index f07e159eef..a3d8ee606a 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp @@ -66,16 +66,41 @@ std::uint32_t EmitBdaByte(SpirvEmitterState& state, std::uint32_t guest, std::ui return value; } -std::uint32_t EmitBdaBytes(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, std::uint32_t accessMask) { - auto result = ConstantU32(state, 0u); - for (std::uint32_t byte = 0; byte < bytes; ++byte) { - const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), address, BdaConstant(state, byte)); - const auto value = EmitBdaByte(state, guest, instruction, accessMask); - result = Binary(state, spv::OpBitwiseOr, TypeU32(state), result, Binary(state, spv::OpShiftLeftLogical, TypeU32(state), value, ConstantU32(state, byte * 8u))); - } +std::uint32_t EmitBdaByteLoop(SpirvEmitterState& state, std::uint32_t address, std::uint32_t count, std::uint32_t instruction, std::uint32_t accessMask) { + const auto u32 = TypeU32(state); + const auto entry = state.currentLabel; + const auto header = state.module.AllocateId(); + const auto body = state.module.AllocateId(); + const auto continuation = state.module.AllocateId(); + const auto merge = state.module.AllocateId(); + const auto byte = state.module.AllocateId(); + const auto result = state.module.AllocateId(); + const auto nextByte = state.module.AllocateId(); + const auto nextResult = state.module.AllocateId(); + state.module.AddFunction(spv::OpBranch, header); + EmitLabel(state, header); + state.module.AddFunction(spv::OpPhi, u32, byte, ConstantU32(state, 0u), entry, nextByte, continuation); + state.module.AddFunction(spv::OpPhi, u32, result, ConstantU32(state, 0u), entry, nextResult, continuation); + const auto more = Binary(state, spv::OpULessThan, TypeBool(state), byte, count); + state.module.AddFunction(spv::OpLoopMerge, merge, continuation, spv::LoopControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, more, body, merge); + EmitLabel(state, body); + const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), address, Unary(state, spv::OpUConvert, TypeScalarU64(state), byte)); + const auto value = EmitBdaByte(state, guest, instruction, accessMask); + state.module.AddFunction(spv::OpBitwiseOr, u32, nextResult, result, Binary(state, spv::OpShiftLeftLogical, u32, value, Binary(state, spv::OpShiftLeftLogical, u32, byte, ConstantU32(state, 3u)))); + state.module.AddFunction(spv::OpBranch, continuation); + EmitLabel(state, continuation); + state.module.AddFunction(spv::OpIAdd, u32, nextByte, byte, ConstantU32(state, 1u)); + state.module.AddFunction(spv::OpBranch, header); + EmitLabel(state, merge); return result; } +std::uint32_t EmitBdaBytes(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, std::uint32_t accessMask) { + if (bytes == 1u) return EmitBdaByte(state, address, instruction, accessMask); + return EmitBdaByteLoop(state, address, ConstantU32(state, bytes), instruction, accessMask); +} + // The dwords of a wide load that the program extracts (all of them when the composite is used // whole). Dead-code elimination has dropped the unused extracts, as it dropped unused dword loads // when every dword was its own instruction. @@ -180,26 +205,48 @@ std::uint32_t DefineBdaSpanReadFunction(SpirvEmitterState& state, bool coherent) } return composite(loaded); }, [&] { - std::array bytes{}; - for (std::uint32_t dword = 0; dword < 4u; ++dword) { - bytes[dword] = EmitValueOrZeroIfCondition(state, isUsed(dword), [&] { - const auto isFirst = Binary(state, spv::OpIEqual, boolean, first, ConstantU32(state, dword)); - const auto dwordAddress = EmitValueIfElse(state, isFirst, u64, [&] { return firstAddress; }, [&] { - return AddBdaSignedOffset(state, address, Binary(state, spv::OpIAdd, u32, offset, ConstantU32(state, dword * 4u)), instruction); - }); - const auto pair = state.module.AllocateId(); - state.module.AddFunction(spv::OpFunctionCall, TypeU32Vector(state, 2), pair, reader, dwordAddress, instruction); - if (stops) { - const auto stopped = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeExtract, u32, stopped, pair, 1u); - StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, boolean, stopped, zero)); - } - const auto value = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeExtract, u32, value, pair, 0u); - return value; + const auto entry = state.currentLabel; + const auto header = state.module.AllocateId(); + const auto body = state.module.AllocateId(); + const auto continuation = state.module.AllocateId(); + const auto merge = state.module.AllocateId(); + const auto dword = state.module.AllocateId(); + const auto gathered = state.module.AllocateId(); + const auto nextDword = state.module.AllocateId(); + const auto nextGathered = state.module.AllocateId(); + const auto zeros = state.module.Constant(spv::OpConstantComposite, values, zero, zero, zero, zero); + state.module.AddFunction(spv::OpBranch, header); + EmitLabel(state, header); + state.module.AddFunction(spv::OpPhi, u32, dword, zero, entry, nextDword, continuation); + state.module.AddFunction(spv::OpPhi, values, gathered, zeros, entry, nextGathered, continuation); + const auto more = Binary(state, spv::OpULessThan, boolean, dword, ConstantU32(state, 4u)); + state.module.AddFunction(spv::OpLoopMerge, merge, continuation, spv::LoopControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, more, body, merge); + EmitLabel(state, body); + const auto used32 = Binary(state, spv::OpINotEqual, boolean, Binary(state, spv::OpBitwiseAnd, u32, used, Binary(state, spv::OpShiftLeftLogical, u32, ConstantU32(state, 1u), dword)), zero); + const auto value = EmitValueOrZeroIfCondition(state, used32, [&] { + const auto isFirst = Binary(state, spv::OpIEqual, boolean, first, dword); + const auto dwordAddress = EmitValueIfElse(state, isFirst, u64, [&] { return firstAddress; }, [&] { + return AddBdaSignedOffset(state, address, Binary(state, spv::OpIAdd, u32, offset, Binary(state, spv::OpShiftLeftLogical, u32, dword, ConstantU32(state, 2u))), instruction); }); - } - return composite(bytes); + const auto pair = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, TypeU32Vector(state, 2), pair, reader, dwordAddress, instruction); + if (stops) { + const auto stopped = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, u32, stopped, pair, 1u); + StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, boolean, stopped, zero)); + } + const auto loaded = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, u32, loaded, pair, 0u); + return loaded; + }); + state.module.AddFunction(spv::OpVectorInsertDynamic, values, nextGathered, gathered, value, dword); + state.module.AddFunction(spv::OpBranch, continuation); + EmitLabel(state, continuation); + state.module.AddFunction(spv::OpIAdd, u32, nextDword, dword, ConstantU32(state, 1u)); + state.module.AddFunction(spv::OpBranch, header); + EmitLabel(state, merge); + return gathered; }); auto returned = read; if (stops) {