From 2d3a970da71512cfffc73f299bd771ddf5a09eb7 Mon Sep 17 00:00:00 2001 From: EdouardSence <103503538+EdouardSence@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:51:11 +0200 Subject: [PATCH] fix(shader): keep each BDA page-table walk to one inlined copy per read RADV inlines every SPIR-V function, DontInline included, so every get_bda_pointer call site becomes its own copy of the page-table search, and NIR's copy propagation of variables grows with each copy. The lookup now keeps its search bounds in phis and records its header and range faults from one call site each, picking the first failing check; a byte read and the byte fallback of a span read loop over one lookup instead of unrolling one per byte or dword. Co-Authored-By: Claude Opus 5.5 --- .../libSceAgcDriver/tests/ShaderMemory.cpp | 11 ++- .../include/SpirvBackend/SpirvBda.hpp | 1 + .../SpirvBackend/src/SpirvBdaFault.cpp | 5 + .../SpirvBackend/src/SpirvBdaLookup.cpp | 81 ++++++++------- .../SpirvBackend/src/SpirvBdaRead.cpp | 99 ++++++++++++++----- 5 files changed, 137 insertions(+), 60 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 1a8234e8f7..fab9d851c4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -2481,6 +2481,9 @@ void verifyBdaReadFallbackFunctions() { std::size_t mainProbes = 0; std::size_t spanCalls = 0; std::array readerLoads{}; + std::size_t lookupFaults = 0; + std::size_t lookupVariables = 0; + std::size_t readerLookups = 0; for (std::size_t cursor = 5; cursor < five.size();) { const auto length = five[cursor] >> 16u; require(length != 0 && length <= five.size() - cursor, "BDA read functions: truncated SPIR-V instruction"); @@ -2501,13 +2504,19 @@ void verifyBdaReadFallbackFunctions() { if (callee == span) ++spanCalls; } if (op == spv::OpLoad && function == reader) ++readerLoads[length > 4u && (five[cursor + 4] & spv::MemoryAccessVolatileMask) != 0u]; + if (op == spv::OpFunctionCall && function == "get_bda_pointer" && names[five[cursor + 3]] == "record_bda_fault") ++lookupFaults; + if (op == spv::OpVariable && function == "get_bda_pointer") ++lookupVariables; + if (op == spv::OpFunctionCall && function == reader && names[five[cursor + 3]] == "get_bda_pointer") ++readerLookups; cursor += length; } require(definitions["record_bda_fault"] == 1u && definitions[reader] == 1u && definitions[span] == 1u, "BDA read functions: the fault, byte read and span read functions are not defined"); for (const auto& [name, count] : definitions) require(count == 1u, "BDA read functions: a function is defined twice"); require(compareExchanges == 1u, "BDA read functions: a fault is recorded outside record_bda_fault"); require(mainLookups == 0u, "BDA read functions: a read site looks up its bytes inline"); - require(readerLoads[coherent] == 4u && readerLoads[!coherent] == 0u, "BDA read functions: the byte loads do not keep the access's coherence"); + require(readerLoads[coherent] == 1u && readerLoads[!coherent] == 0u, "BDA read functions: the byte loads do not keep the access's coherence"); + require(lookupFaults <= 4u, "BDA read functions: the page-table lookup records its faults from more than 4 call sites"); + require(lookupVariables == 0u, "BDA read functions: the page-table lookup keeps its search bounds in function variables"); + require(readerLookups == 1u, "BDA read functions: the byte read does not look up its bytes once, in a loop"); require(mainProbes == 0u, "BDA read functions: a read site probes or reads bytes outside its span read function"); require(spanCalls == 5u, "BDA read functions: a read site does not call its span read function once"); require(five.size() - one.size() < 4u * 150u, "BDA read functions: a read site takes 150 SPIR-V words or more"); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp index 0b308f556c..21dda89e68 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp @@ -17,6 +17,7 @@ void DefineBdaFaultFunction(SpirvEmitterState& state); void DefineBdaDwordReadFunctions(SpirvEmitterState& state); void DefineBdaSpanReadFunctions(SpirvEmitterState& state); void RecordBdaFault(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); +void RecordBdaFaultReason(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, std::uint32_t reason); void RecordBdaFaultWords(SpirvEmitterState& state, std::uint32_t addressLow, std::uint32_t addressHigh, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); void ReturnBdaFailureIf(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& target); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp index ca8c1a2678..4b0d60d921 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp @@ -75,6 +75,11 @@ void RecordBdaFault(SpirvEmitterState& state, std::uint32_t address, std::uint32 state.module.AddFunction(spv::OpFunctionCall, TypeVoid(state), state.module.AllocateId(), state.bdaFaultFunction, ConstantU32(state, static_cast(reason)), address, bytes, instruction); } +void RecordBdaFaultReason(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, std::uint32_t reason) { + if (state.bdaFaultFunction == 0 || !state.program.Info().usesDma) throw std::runtime_error("BDA fault function is missing"); + state.module.AddFunction(spv::OpFunctionCall, TypeVoid(state), state.module.AllocateId(), state.bdaFaultFunction, reason, address, bytes, instruction); +} + void RecordBdaFaultWords(SpirvEmitterState& state, std::uint32_t addressLow, std::uint32_t addressHigh, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason) { if (state.program.Info().usesDma) { const auto u64 = TypeScalarU64(state); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp index 248ff205c0..b48e8775b5 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp @@ -2,6 +2,9 @@ #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" #include +#include +#include +#include namespace ShaderRecompiler { @@ -24,11 +27,6 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r const auto boolean = TypeBool(state); const auto constant = [&](std::uint32_t value) { return ConstantU32(state, value); }; const auto binary = [&](std::uint32_t op, std::uint32_t type, std::uint32_t left, std::uint32_t right) { return Binary(state, op, type, left, right); }; - const auto load = [&](std::uint32_t pointer) { - const auto value = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, u32, value, pointer); - return value; - }; const auto function = state.module.AllocateId(); state.module.AddName(function, name); state.module.AddFunction(spv::OpFunction, u64, function, spv::FunctionControlDontInlineMask, state.module.Type(spv::OpTypeFunction, u64, u64, u32, u32)); @@ -39,11 +37,6 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r state.module.AddFunction(spv::OpFunctionParameter, u32, bytes); state.module.AddFunction(spv::OpFunctionParameter, u32, instruction); EmitLabel(state, state.module.AllocateId()); - const auto low = state.module.AllocateId(); - const auto high = state.module.AllocateId(); - const auto pointer = TypePointer(state, spv::StorageClassFunction, u32); - state.module.AddFunction(spv::OpVariable, pointer, low, spv::StorageClassFunction); - state.module.AddFunction(spv::OpVariable, pointer, high, spv::StorageClassFunction); const auto fail = [&](std::uint32_t condition, BdaAbi::FaultReason reason) { if (recordFaults) { ReturnBdaFailureIf(state, condition, address, bytes, instruction, reason); @@ -51,29 +44,53 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r ReturnBdaZeroIf(state, condition); } }; + const auto failFirst = [&](std::initializer_list> checks) { + auto reason = constant(0u); + for (auto check = std::rbegin(checks); check != std::rend(checks); ++check) { + reason = Select(state, u32, check->first, constant(static_cast(check->second)), reason); + } + const auto failed = binary(spv::OpINotEqual, boolean, reason, constant(0u)); + if (!recordFaults) { + ReturnBdaZeroIf(state, failed); + return; + } + const auto failedLabel = state.module.AllocateId(); + const auto next = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelectionMerge, next, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, failed, failedLabel, next); + EmitLabel(state, failedLabel); + RecordBdaFaultReason(state, address, bytes, instruction, reason); + state.module.AddFunction(spv::OpReturnValue, BdaConstant(state, 0u)); + EmitLabel(state, next); + }; const auto length = state.module.AllocateId(); state.module.AddFunction(spv::OpArrayLength, u32, length, state.bdaPagetableVariable, 0u); fail(binary(spv::OpULessThan, boolean, length, constant(4)), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpINotEqual, boolean, BdaLoadWord(state, constant(0)), constant(BdaAbi::Version)), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpINotEqual, boolean, BdaLoadWord(state, constant(2)), constant(sizeof(BdaAbi::Range))), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpINotEqual, boolean, BdaLoadWord(state, constant(3)), constant(0)), BdaAbi::FaultReason::InvalidTable); const auto count = BdaLoadWord(state, constant(1)); const auto available = binary(spv::OpISub, u32, length, constant(4)); - fail(binary(spv::OpINotEqual, boolean, binary(spv::OpBitwiseAnd, u32, available, constant(7)), constant(0)), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpINotEqual, boolean, count, binary(spv::OpShiftRightLogical, u32, available, constant(3))), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpIEqual, boolean, bytes, constant(0)), BdaAbi::FaultReason::Overflow); const auto end = binary(spv::OpIAdd, u64, address, Unary(state, spv::OpUConvert, u64, bytes)); - fail(binary(spv::OpULessThanEqual, boolean, end, address), BdaAbi::FaultReason::Overflow); - state.module.AddFunction(spv::OpStore, low, constant(0)); - state.module.AddFunction(spv::OpStore, high, count); + failFirst({ + {binary(spv::OpINotEqual, boolean, BdaLoadWord(state, constant(0)), constant(BdaAbi::Version)), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpINotEqual, boolean, BdaLoadWord(state, constant(2)), constant(sizeof(BdaAbi::Range))), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpINotEqual, boolean, BdaLoadWord(state, constant(3)), constant(0)), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpINotEqual, boolean, binary(spv::OpBitwiseAnd, u32, available, constant(7)), constant(0)), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpINotEqual, boolean, count, binary(spv::OpShiftRightLogical, u32, available, constant(3))), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpIEqual, boolean, bytes, constant(0)), BdaAbi::FaultReason::Overflow}, + {binary(spv::OpULessThanEqual, boolean, end, address), BdaAbi::FaultReason::Overflow}, + }); const auto header = state.module.AllocateId(); const auto body = state.module.AllocateId(); const auto continuation = state.module.AllocateId(); const auto merge = state.module.AllocateId(); + const auto entryLabel = state.currentLabel; state.module.AddFunction(spv::OpBranch, header); EmitLabel(state, header); - const auto lower = load(low); - const auto upper = load(high); + const auto lower = state.module.AllocateId(); + const auto upper = state.module.AllocateId(); + const auto newLow = state.module.AllocateId(); + const auto newHigh = state.module.AllocateId(); + state.module.AddFunction(spv::OpPhi, u32, lower, constant(0), entryLabel, newLow, continuation); + state.module.AddFunction(spv::OpPhi, u32, upper, count, entryLabel, newHigh, continuation); const auto search = binary(spv::OpULessThan, boolean, lower, upper); state.module.AddFunction(spv::OpLoopMerge, merge, continuation, spv::LoopControlMaskNone); state.module.AddFunction(spv::OpBranchConditional, search, body, merge); @@ -81,17 +98,13 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r const auto midpoint = binary(spv::OpIAdd, u32, lower, binary(spv::OpShiftRightLogical, u32, binary(spv::OpISub, u32, upper, lower), constant(1))); const auto entry = binary(spv::OpIAdd, u32, constant(4), binary(spv::OpIMul, u32, midpoint, constant(8))); const auto before = binary(spv::OpULessThan, boolean, address, BdaLoadAddress(state, entry)); - const auto newLow = state.module.AllocateId(); - const auto newHigh = state.module.AllocateId(); state.module.AddFunction(spv::OpSelect, u32, newLow, before, lower, binary(spv::OpIAdd, u32, midpoint, constant(1))); state.module.AddFunction(spv::OpSelect, u32, newHigh, before, midpoint, upper); - state.module.AddFunction(spv::OpStore, low, newLow); - state.module.AddFunction(spv::OpStore, high, newHigh); state.module.AddFunction(spv::OpBranch, continuation); EmitLabel(state, continuation); state.module.AddFunction(spv::OpBranch, header); EmitLabel(state, merge); - const auto index = load(low); + const auto index = lower; fail(binary(spv::OpIEqual, boolean, index, constant(0)), BdaAbi::FaultReason::Unmapped); const auto selected = binary(spv::OpIAdd, u32, constant(4), binary(spv::OpIMul, u32, binary(spv::OpISub, u32, index, constant(1)), constant(8))); const auto at = [&](std::uint32_t offset) { return binary(spv::OpIAdd, u32, selected, constant(offset)); }; @@ -99,16 +112,18 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r const auto finish = BdaLoadAddress(state, at(2)); const auto base = BdaLoadAddress(state, at(4)); const auto permissions = BdaLoadWord(state, at(6)); - fail(binary(spv::OpINotEqual, boolean, BdaLoadWord(state, at(7)), constant(0)), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpUGreaterThanEqual, boolean, begin, finish), BdaAbi::FaultReason::InvalidTable); - fail(binary(spv::OpUGreaterThan, boolean, end, finish), BdaAbi::FaultReason::Unmapped); - fail(binary(spv::OpINotEqual, boolean, binary(spv::OpBitwiseAnd, u32, permissions, constant(permission)), constant(permission)), BdaAbi::FaultReason::Permission); - fail(binary(spv::OpIEqual, boolean, base, BdaConstant(state, 0)), BdaAbi::FaultReason::InvalidTable); const auto offset = binary(spv::OpISub, u64, address, begin); const auto result = binary(spv::OpIAdd, u64, base, offset); - fail(binary(spv::OpULessThan, boolean, result, base), BdaAbi::FaultReason::Overflow); const auto deviceEnd = binary(spv::OpIAdd, u64, result, Unary(state, spv::OpUConvert, u64, bytes)); - fail(binary(spv::OpULessThanEqual, boolean, deviceEnd, result), BdaAbi::FaultReason::Overflow); + failFirst({ + {binary(spv::OpINotEqual, boolean, BdaLoadWord(state, at(7)), constant(0)), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpUGreaterThanEqual, boolean, begin, finish), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpUGreaterThan, boolean, end, finish), BdaAbi::FaultReason::Unmapped}, + {binary(spv::OpINotEqual, boolean, binary(spv::OpBitwiseAnd, u32, permissions, constant(permission)), constant(permission)), BdaAbi::FaultReason::Permission}, + {binary(spv::OpIEqual, boolean, base, BdaConstant(state, 0)), BdaAbi::FaultReason::InvalidTable}, + {binary(spv::OpULessThan, boolean, result, base), BdaAbi::FaultReason::Overflow}, + {binary(spv::OpULessThanEqual, boolean, deviceEnd, result), BdaAbi::FaultReason::Overflow}, + }); state.module.AddFunction(spv::OpReturnValue, result); state.module.AddFunction(spv::OpFunctionEnd); return function; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp index f07e159eef..a3d8ee606a 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp @@ -66,16 +66,41 @@ std::uint32_t EmitBdaByte(SpirvEmitterState& state, std::uint32_t guest, std::ui return value; } -std::uint32_t EmitBdaBytes(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, std::uint32_t accessMask) { - auto result = ConstantU32(state, 0u); - for (std::uint32_t byte = 0; byte < bytes; ++byte) { - const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), address, BdaConstant(state, byte)); - const auto value = EmitBdaByte(state, guest, instruction, accessMask); - result = Binary(state, spv::OpBitwiseOr, TypeU32(state), result, Binary(state, spv::OpShiftLeftLogical, TypeU32(state), value, ConstantU32(state, byte * 8u))); - } +std::uint32_t EmitBdaByteLoop(SpirvEmitterState& state, std::uint32_t address, std::uint32_t count, std::uint32_t instruction, std::uint32_t accessMask) { + const auto u32 = TypeU32(state); + const auto entry = state.currentLabel; + const auto header = state.module.AllocateId(); + const auto body = state.module.AllocateId(); + const auto continuation = state.module.AllocateId(); + const auto merge = state.module.AllocateId(); + const auto byte = state.module.AllocateId(); + const auto result = state.module.AllocateId(); + const auto nextByte = state.module.AllocateId(); + const auto nextResult = state.module.AllocateId(); + state.module.AddFunction(spv::OpBranch, header); + EmitLabel(state, header); + state.module.AddFunction(spv::OpPhi, u32, byte, ConstantU32(state, 0u), entry, nextByte, continuation); + state.module.AddFunction(spv::OpPhi, u32, result, ConstantU32(state, 0u), entry, nextResult, continuation); + const auto more = Binary(state, spv::OpULessThan, TypeBool(state), byte, count); + state.module.AddFunction(spv::OpLoopMerge, merge, continuation, spv::LoopControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, more, body, merge); + EmitLabel(state, body); + const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), address, Unary(state, spv::OpUConvert, TypeScalarU64(state), byte)); + const auto value = EmitBdaByte(state, guest, instruction, accessMask); + state.module.AddFunction(spv::OpBitwiseOr, u32, nextResult, result, Binary(state, spv::OpShiftLeftLogical, u32, value, Binary(state, spv::OpShiftLeftLogical, u32, byte, ConstantU32(state, 3u)))); + state.module.AddFunction(spv::OpBranch, continuation); + EmitLabel(state, continuation); + state.module.AddFunction(spv::OpIAdd, u32, nextByte, byte, ConstantU32(state, 1u)); + state.module.AddFunction(spv::OpBranch, header); + EmitLabel(state, merge); return result; } +std::uint32_t EmitBdaBytes(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, std::uint32_t accessMask) { + if (bytes == 1u) return EmitBdaByte(state, address, instruction, accessMask); + return EmitBdaByteLoop(state, address, ConstantU32(state, bytes), instruction, accessMask); +} + // The dwords of a wide load that the program extracts (all of them when the composite is used // whole). Dead-code elimination has dropped the unused extracts, as it dropped unused dword loads // when every dword was its own instruction. @@ -180,26 +205,48 @@ std::uint32_t DefineBdaSpanReadFunction(SpirvEmitterState& state, bool coherent) } return composite(loaded); }, [&] { - std::array bytes{}; - for (std::uint32_t dword = 0; dword < 4u; ++dword) { - bytes[dword] = EmitValueOrZeroIfCondition(state, isUsed(dword), [&] { - const auto isFirst = Binary(state, spv::OpIEqual, boolean, first, ConstantU32(state, dword)); - const auto dwordAddress = EmitValueIfElse(state, isFirst, u64, [&] { return firstAddress; }, [&] { - return AddBdaSignedOffset(state, address, Binary(state, spv::OpIAdd, u32, offset, ConstantU32(state, dword * 4u)), instruction); - }); - const auto pair = state.module.AllocateId(); - state.module.AddFunction(spv::OpFunctionCall, TypeU32Vector(state, 2), pair, reader, dwordAddress, instruction); - if (stops) { - const auto stopped = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeExtract, u32, stopped, pair, 1u); - StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, boolean, stopped, zero)); - } - const auto value = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeExtract, u32, value, pair, 0u); - return value; + const auto entry = state.currentLabel; + const auto header = state.module.AllocateId(); + const auto body = state.module.AllocateId(); + const auto continuation = state.module.AllocateId(); + const auto merge = state.module.AllocateId(); + const auto dword = state.module.AllocateId(); + const auto gathered = state.module.AllocateId(); + const auto nextDword = state.module.AllocateId(); + const auto nextGathered = state.module.AllocateId(); + const auto zeros = state.module.Constant(spv::OpConstantComposite, values, zero, zero, zero, zero); + state.module.AddFunction(spv::OpBranch, header); + EmitLabel(state, header); + state.module.AddFunction(spv::OpPhi, u32, dword, zero, entry, nextDword, continuation); + state.module.AddFunction(spv::OpPhi, values, gathered, zeros, entry, nextGathered, continuation); + const auto more = Binary(state, spv::OpULessThan, boolean, dword, ConstantU32(state, 4u)); + state.module.AddFunction(spv::OpLoopMerge, merge, continuation, spv::LoopControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, more, body, merge); + EmitLabel(state, body); + const auto used32 = Binary(state, spv::OpINotEqual, boolean, Binary(state, spv::OpBitwiseAnd, u32, used, Binary(state, spv::OpShiftLeftLogical, u32, ConstantU32(state, 1u), dword)), zero); + const auto value = EmitValueOrZeroIfCondition(state, used32, [&] { + const auto isFirst = Binary(state, spv::OpIEqual, boolean, first, dword); + const auto dwordAddress = EmitValueIfElse(state, isFirst, u64, [&] { return firstAddress; }, [&] { + return AddBdaSignedOffset(state, address, Binary(state, spv::OpIAdd, u32, offset, Binary(state, spv::OpShiftLeftLogical, u32, dword, ConstantU32(state, 2u))), instruction); }); - } - return composite(bytes); + const auto pair = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, TypeU32Vector(state, 2), pair, reader, dwordAddress, instruction); + if (stops) { + const auto stopped = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, u32, stopped, pair, 1u); + StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, boolean, stopped, zero)); + } + const auto loaded = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, u32, loaded, pair, 0u); + return loaded; + }); + state.module.AddFunction(spv::OpVectorInsertDynamic, values, nextGathered, gathered, value, dword); + state.module.AddFunction(spv::OpBranch, continuation); + EmitLabel(state, continuation); + state.module.AddFunction(spv::OpIAdd, u32, nextDword, dword, ConstantU32(state, 1u)); + state.module.AddFunction(spv::OpBranch, header); + EmitLabel(state, merge); + return gathered; }); auto returned = read; if (stops) {