Merge pull request #7922 from ThatsLeo/feat/shader-spanned-uniformity

perf(shader): span single-wave wave64 compute by default after a uniformity check

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -2073,10 +2073,8 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         byteFeatures.pNext = &subgroupSizeFeatures;
         APS5_LOG_OUT("Wave32 programs run on subgroups of %u: compute %d, mesh %d", 32u, state->computeWave32, state->meshWave32);
     }
-    if (std::getenv("APS5_SPANNED_WAVE") != nullptr) {
-        state->spannedWaves = state->computeFullSubgroups;
-        APS5_LOG_OUT("APS5_SPANNED_WAVE: single-wave wave64 compute spans two full 32-lane subgroups: %d", state->spannedWaves);
-    }
+    state->spannedWaves = state->computeFullSubgroups && std::getenv("APS5_NO_SPANNED_WAVE") == nullptr;
+    APS5_LOG_OUT("single-wave wave64 compute spans two full 32-lane subgroups: %d", state->spannedWaves);
     timelineFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES_KHR};
     timelineFeatures.timelineSemaphore = VK_TRUE;
     if (state->timelineSemaphores) {

core/shader/recompiler/CMakeLists.txt

@@ -144,6 +144,7 @@ function(add_shader_recompiler target runtimeTarget)
             ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvBufferFormat.cpp
             ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvRuntimeBuffer.cpp
             ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvSpecialization.cpp
+            ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvSpannedUniformity.cpp
 
             ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp
             ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvMemory/SpirvConstants.cpp
@@ -365,6 +366,23 @@ function(add_shader_recompiler target runtimeTarget)
         target_link_libraries(${target} PRIVATE SPIRV-Tools-opt SPIRV-Tools-static)
     endif()
     target_compile_definitions(${target} PUBLIC ANYPS5_ENABLE_SPIRV_TOOLS=$<BOOL:${ANYPS5_ENABLE_SPIRV_TOOLS}>)
+    if(BUILD_TESTING AND NOT TARGET spanned_uniformity_tests)
+        add_test_executable(spanned_uniformity_tests
+            "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/tests/SpannedUniformityTests.cpp"
+            "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvSpannedUniformity.cpp")
+        target_include_directories(spanned_uniformity_tests PRIVATE
+            "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/include"
+            ${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR})
+        set_target_properties(spanned_uniformity_tests PROPERTIES CXX_EXTENSIONS OFF)
+        configure_windows_unwind(spanned_uniformity_tests)
+        add_test(NAME spanned_uniformity COMMAND spanned_uniformity_tests)
+        set_tests_properties(spanned_uniformity PROPERTIES TIMEOUT 20)
+        if(MINGW)
+            target_link_libraries(spanned_uniformity_tests PRIVATE libc)
+            set_tests_properties(spanned_uniformity PROPERTIES ENVIRONMENT_MODIFICATION
+                    "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        endif()
+    endif()
     if(BUILD_TESTING AND ANYPS5_ENABLE_SPIRV_TOOLS AND NOT ANYPS5_USE_SYSTEM_DEPS AND NOT TARGET spirv_validator_exit_tests)
         add_test_executable(spirv_validator_exit_tests "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/tests/SpirvValidatorExit.cpp")
         target_link_libraries(spirv_validator_exit_tests PRIVATE SPIRV-Tools-static)

core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvSpannedUniformity.hpp

@@ -0,0 +1,15 @@
+#ifndef CORE_SHADER_RECOMPILER_SPIRVSPANNEDUNIFORMITY_HPP
+#define CORE_SHADER_RECOMPILER_SPIRVSPANNEDUNIFORMITY_HPP
+
+#include <cstdint>
+#include <optional>
+#include <span>
+#include <string>
+
+namespace ShaderRecompiler {
+
+[[nodiscard]] std::optional<std::string> SpannedBarrierInDivergentFlow(std::span<const std::uint32_t> words);
+
+}
+
+#endif

core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp

@@ -1,5 +1,6 @@
 #include "SpirvBackend/SpirvBda.hpp"
 #include "SpirvBackend/SpirvEmitter.hpp"
+#include "SpirvBackend/SpirvSpannedUniformity.hpp"
 #include "SpirvBackend/SpirvEmitterHelpers.hpp"
 #include "SpirvBackend/SpirvEmitterState.hpp"
 #include "SpirvBackend/SpirvFlowEmitter.hpp"
@@ -328,12 +329,16 @@ std::vector<std::uint32_t> SpirvEmitter::Emit(const IrProgram& program, const Sh
     EmitModuleHeader(state, bindings);
     EmitProgram(state);
     state.module.EmitEntryPoint(ExecutionModelForStage(state.program.Resources().stage), state.mainFunc, "main", state.interfaceVariables);
+    auto words = state.module.Finalize();
+    if (state.spannedWave) {
+        if (auto reason = SpannedBarrierInDivergentFlow(words)) throw SpannedWaveUnsupported(*reason);
+    }
     if (info != nullptr) {
         info->fullSubgroups = state.spannedWave;
         info->functionLdsBuffer = state.pixelFunctionLdsBuffer;
         info->meshWindowBuffer = state.meshWindowBuffer;
     }
-    return state.module.Finalize();
+    return words;
 }
 
 }

core/shader/recompiler/SpirvBackend/src/SpirvInstructionOperands.hpp

@@ -0,0 +1,231 @@
+#ifndef CORE_SHADER_RECOMPILER_SPIRVINSTRUCTIONOPERANDS_HPP
+#define CORE_SHADER_RECOMPILER_SPIRVINSTRUCTIONOPERANDS_HPP
+
+#define SPV_ENABLE_UTILITY_CODE
+#include <spirv/unified1/spirv.hpp>
+#undef SPV_ENABLE_UTILITY_CODE
+#include <cstdint>
+#include <initializer_list>
+
+namespace ShaderRecompiler::SpirvInstructions {
+
+template<typename TInstruction>
+spv::Op Opcode(const TInstruction& instruction) {
+    return instruction.empty() ? spv::OpNop : static_cast<spv::Op>(instruction[0] & 0xffffu);
+}
+
+template<typename TInstruction>
+std::uint32_t Result(const TInstruction& instruction) {
+    if (instruction.empty()) return 0u;
+    bool result = false;
+    bool type = false;
+    spv::HasResultAndType(Opcode(instruction), &result, &type);
+    return result ? instruction.at(type ? 2u : 1u) : 0u;
+}
+
+template<typename TInstruction, typename TVisitor>
+void VisitIdOperands(const TInstruction& instruction, std::uint32_t switchLiteralWords, const TVisitor& visit) {
+    const auto size = instruction.size();
+    const auto from = [&](std::size_t first) {
+        for (auto index = first; index < size; ++index) visit(index);
+    };
+    const auto only = [&](std::initializer_list<std::size_t> indices) {
+        for (const auto index : indices) {
+            if (index < size) visit(index);
+        }
+    };
+    constexpr std::uint32_t knownMemoryAccess = static_cast<std::uint32_t>(spv::MemoryAccessVolatileMask) | static_cast<std::uint32_t>(spv::MemoryAccessAlignedMask) |
+        static_cast<std::uint32_t>(spv::MemoryAccessNontemporalMask) | static_cast<std::uint32_t>(spv::MemoryAccessMakePointerAvailableMask) |
+        static_cast<std::uint32_t>(spv::MemoryAccessMakePointerVisibleMask) | static_cast<std::uint32_t>(spv::MemoryAccessNonPrivatePointerMask);
+    const auto memoryOperands = [&](std::size_t index) {
+        while (index < size) {
+            const auto mask = instruction[index++];
+            if ((mask & ~knownMemoryAccess) != 0u) {
+                from(index);
+                return;
+            }
+            if ((mask & static_cast<std::uint32_t>(spv::MemoryAccessAlignedMask)) != 0u) ++index;
+            if ((mask & static_cast<std::uint32_t>(spv::MemoryAccessMakePointerAvailableMask)) != 0u && index < size) visit(index++);
+            if ((mask & static_cast<std::uint32_t>(spv::MemoryAccessMakePointerVisibleMask)) != 0u && index < size) visit(index++);
+        }
+    };
+    switch (Opcode(instruction)) {
+    case spv::OpNop:
+    case spv::OpCapability:
+    case spv::OpExtension:
+    case spv::OpExtInstImport:
+    case spv::OpMemoryModel:
+    case spv::OpSource:
+    case spv::OpSourceContinued:
+    case spv::OpSourceExtension:
+    case spv::OpString:
+    case spv::OpName:
+    case spv::OpMemberName:
+    case spv::OpModuleProcessed:
+    case spv::OpDecorate:
+    case spv::OpMemberDecorate:
+    case spv::OpDecorateString:
+    case spv::OpMemberDecorateString:
+    case spv::OpTypeVoid:
+    case spv::OpTypeBool:
+    case spv::OpTypeInt:
+    case spv::OpTypeFloat:
+    case spv::OpTypeSampler:
+    case spv::OpConstantTrue:
+    case spv::OpConstantFalse:
+    case spv::OpConstant:
+    case spv::OpConstantSampler:
+    case spv::OpConstantNull:
+    case spv::OpSpecConstantTrue:
+    case spv::OpSpecConstantFalse:
+    case spv::OpSpecConstant:
+    case spv::OpLabel:
+    case spv::OpNoLine:
+    case spv::OpReturn:
+    case spv::OpUnreachable:
+    case spv::OpKill:
+    case spv::OpFunctionEnd:
+        return;
+    case spv::OpTypeVector:
+    case spv::OpTypeMatrix:
+    case spv::OpTypeImage:
+        only({2});
+        return;
+    case spv::OpTypePointer:
+        only({3});
+        return;
+    case spv::OpTypeForwardPointer:
+    case spv::OpExecutionMode:
+    case spv::OpLine:
+    case spv::OpSelectionMerge:
+        only({1});
+        return;
+    case spv::OpSpecConstantOp:
+        only({1});
+        from(4);
+        return;
+    case spv::OpVariable:
+    case spv::OpFunction:
+        only({1, 4});
+        return;
+    case spv::OpLoad:
+        only({1, 3});
+        memoryOperands(4);
+        return;
+    case spv::OpStore:
+    case spv::OpCopyMemory:
+        only({1, 2});
+        memoryOperands(3);
+        return;
+    case spv::OpCopyMemorySized:
+        only({1, 2, 3});
+        memoryOperands(4);
+        return;
+    case spv::OpArrayLength:
+    case spv::OpCompositeExtract:
+        only({1, 3});
+        return;
+    case spv::OpCompositeInsert:
+    case spv::OpVectorShuffle:
+        only({1, 3, 4});
+        return;
+    case spv::OpExtInst:
+        only({1, 3});
+        from(5);
+        return;
+    case spv::OpEntryPoint: {
+        only({2});
+        std::size_t index = 3;
+        while (index < size) {
+            const auto word = instruction[index++];
+            if ((word & 0xffu) == 0u || (word & 0xff00u) == 0u || (word & 0xff0000u) == 0u || (word & 0xff000000u) == 0u) break;
+        }
+        from(index);
+        return;
+    }
+    case spv::OpExecutionModeId:
+    case spv::OpDecorateId:
+        only({1});
+        from(3);
+        return;
+    case spv::OpLoopMerge:
+        only({1, 2});
+        return;
+    case spv::OpBranchConditional:
+        only({1, 2, 3});
+        return;
+    case spv::OpSwitch:
+        only({1, 2});
+        for (std::size_t index = 3; index + switchLiteralWords < size; index += switchLiteralWords + 1u) visit(index + switchLiteralWords);
+        return;
+    case spv::OpImageSampleImplicitLod:
+    case spv::OpImageSampleExplicitLod:
+    case spv::OpImageSampleProjImplicitLod:
+    case spv::OpImageSampleProjExplicitLod:
+    case spv::OpImageFetch:
+    case spv::OpImageRead:
+    case spv::OpImageSparseSampleImplicitLod:
+    case spv::OpImageSparseSampleExplicitLod:
+    case spv::OpImageSparseSampleProjImplicitLod:
+    case spv::OpImageSparseSampleProjExplicitLod:
+    case spv::OpImageSparseFetch:
+    case spv::OpImageSparseRead:
+        only({1, 3, 4});
+        from(6);
+        return;
+    case spv::OpImageSampleDrefImplicitLod:
+    case spv::OpImageSampleDrefExplicitLod:
+    case spv::OpImageSampleProjDrefImplicitLod:
+    case spv::OpImageSampleProjDrefExplicitLod:
+    case spv::OpImageGather:
+    case spv::OpImageDrefGather:
+    case spv::OpImageSparseSampleDrefImplicitLod:
+    case spv::OpImageSparseSampleDrefExplicitLod:
+    case spv::OpImageSparseSampleProjDrefImplicitLod:
+    case spv::OpImageSparseSampleProjDrefExplicitLod:
+    case spv::OpImageSparseGather:
+    case spv::OpImageSparseDrefGather:
+        only({1, 3, 4, 5});
+        from(7);
+        return;
+    case spv::OpImageWrite:
+        only({1, 2, 3});
+        from(5);
+        return;
+    case spv::OpGroupIAdd:
+    case spv::OpGroupFAdd:
+    case spv::OpGroupFMin:
+    case spv::OpGroupUMin:
+    case spv::OpGroupSMin:
+    case spv::OpGroupFMax:
+    case spv::OpGroupUMax:
+    case spv::OpGroupSMax:
+    case spv::OpGroupNonUniformBallotBitCount:
+    case spv::OpGroupNonUniformIAdd:
+    case spv::OpGroupNonUniformFAdd:
+    case spv::OpGroupNonUniformIMul:
+    case spv::OpGroupNonUniformFMul:
+    case spv::OpGroupNonUniformSMin:
+    case spv::OpGroupNonUniformUMin:
+    case spv::OpGroupNonUniformFMin:
+    case spv::OpGroupNonUniformSMax:
+    case spv::OpGroupNonUniformUMax:
+    case spv::OpGroupNonUniformFMax:
+    case spv::OpGroupNonUniformBitwiseAnd:
+    case spv::OpGroupNonUniformBitwiseOr:
+    case spv::OpGroupNonUniformBitwiseXor:
+    case spv::OpGroupNonUniformLogicalAnd:
+    case spv::OpGroupNonUniformLogicalOr:
+    case spv::OpGroupNonUniformLogicalXor:
+        only({1, 3});
+        from(5);
+        return;
+    default:
+        from(1);
+        return;
+    }
+}
+
+}
+
+#endif

core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp

@@ -1568,7 +1568,15 @@ std::uint32_t EmitReadConst(SpirvValueEmitContext& ctx, const IrValue& inst) {
             EmitIfCondition(state, poisoned, [&] {
                 const auto record = Binary(state, spv::OpIAdd, u32, ConstantU32(state, records), Binary(state, spv::OpIMul, u32, Binary(state, spv::OpISub, u32, flag, ConstantU32(state, 1u)), ConstantU32(state, 3u)));
                 const auto stage = ConstantU32(state, static_cast<std::uint32_t>(resources.stage) << BdaAbi::SrtGuardStageShift);
-                state.module.AddFunction(spv::OpAtomicStore, BdaWord(state, state.faultBufferVariable, ConstantU32(state, BdaAbi::SrtGuardWord)), ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), Binary(state, spv::OpBitwiseOr, u32, record, stage));
+                const auto word = BdaWord(state, state.faultBufferVariable, ConstantU32(state, BdaAbi::SrtGuardWord));
+                const auto scope = ConstantU32(state, spv::ScopeDevice);
+                const auto relaxed = ConstantU32(state, spv::MemorySemanticsMaskNone);
+                const auto value = Binary(state, spv::OpBitwiseOr, u32, record, stage);
+                if (state.bdaStopsInvocations) {
+                    state.module.AddFunction(spv::OpAtomicStore, word, scope, relaxed, value);
+                } else {
+                    state.module.AddFunction(spv::OpAtomicCompareExchange, u32, state.module.AllocateId(), word, scope, relaxed, relaxed, value, ConstantU32(state, 0u));
+                }
             });
             StopBdaInvocationIf(state, poisoned);
         }

core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp

@@ -0,0 +1,545 @@
+#include "SpirvInstructionOperands.hpp"
+#include "SpirvBackend/SpirvSpannedUniformity.hpp"
+#include <algorithm>
+#include <stdexcept>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+namespace ShaderRecompiler {
+namespace {
+
+using Instruction = std::vector<std::uint32_t>;
+using SpirvInstructions::Opcode;
+using SpirvInstructions::Result;
+using SpirvInstructions::VisitIdOperands;
+
+struct Block {
+    std::uint32_t label = 0u;
+    std::vector<Instruction> body;
+};
+
+struct Function {
+    std::uint32_t id = 0u;
+    std::vector<std::uint32_t> parameters;
+    std::vector<Block> blocks;
+};
+
+struct Module {
+    std::unordered_map<std::uint32_t, std::uint32_t> builtins;
+    std::unordered_map<std::uint32_t, std::uint32_t> storageClasses;
+    std::unordered_map<std::uint32_t, std::uint32_t> pointerClasses;
+    std::unordered_map<std::uint32_t, std::uint32_t> constants;
+    std::vector<Function> functions;
+};
+
+Module Parse(std::span<const std::uint32_t> words) {
+    if (words.size() < 5u) throw std::runtime_error("SPIR-V module has no header");
+    Module module;
+    Function* function = nullptr;
+    Block* block = nullptr;
+    for (std::size_t cursor = 5u; cursor < words.size();) {
+        const auto count = words[cursor] >> 16u;
+        if (count == 0u || cursor + count > words.size()) throw std::runtime_error("SPIR-V instruction runs past the module");
+        Instruction instruction(words.begin() + static_cast<std::ptrdiff_t>(cursor), words.begin() + static_cast<std::ptrdiff_t>(cursor + count));
+        cursor += count;
+        switch (Opcode(instruction)) {
+        case spv::OpDecorate:
+            if (instruction.size() >= 4u && instruction[2] == spv::DecorationBuiltIn) module.builtins[instruction[1]] = instruction[3];
+            break;
+        case spv::OpTypePointer:
+            if (instruction.size() >= 4u) module.pointerClasses[instruction[1]] = instruction[2];
+            break;
+        case spv::OpConstant:
+            if (instruction.size() >= 4u) module.constants[instruction[2]] = instruction[3];
+            break;
+        case spv::OpVariable:
+            if (instruction.size() >= 4u) module.storageClasses[instruction[2]] = instruction[3];
+            if (block != nullptr) block->body.push_back(std::move(instruction));
+            break;
+        case spv::OpFunction:
+            function = &module.functions.emplace_back();
+            function->id = instruction.at(2);
+            break;
+        case spv::OpFunctionParameter:
+            if (function != nullptr) function->parameters.push_back(instruction.at(2));
+            break;
+        case spv::OpLabel:
+            if (function == nullptr) throw std::runtime_error("SPIR-V label outside a function");
+            block = &function->blocks.emplace_back();
+            block->label = instruction.at(1);
+            break;
+        case spv::OpFunctionEnd:
+            function = nullptr;
+            block = nullptr;
+            break;
+        default:
+            if (block != nullptr) block->body.push_back(std::move(instruction));
+            break;
+        }
+    }
+    return module;
+}
+
+bool VaryingBuiltin(std::uint32_t builtin) {
+    switch (builtin) {
+    case spv::BuiltInLocalInvocationId:
+    case spv::BuiltInGlobalInvocationId:
+    case spv::BuiltInLocalInvocationIndex:
+    case spv::BuiltInSubgroupId:
+    case spv::BuiltInSubgroupLocalInvocationId:
+    case spv::BuiltInSubgroupEqMask:
+    case spv::BuiltInSubgroupGeMask:
+    case spv::BuiltInSubgroupGtMask:
+    case spv::BuiltInSubgroupLeMask:
+    case spv::BuiltInSubgroupLtMask:
+        return true;
+    default:
+        return false;
+    }
+}
+
+bool PerInvocationResult(spv::Op op) {
+    const auto raw = static_cast<std::uint32_t>(op);
+    if (raw >= spv::OpAtomicExchange && raw <= spv::OpAtomicXor) return true;
+    if (raw >= spv::OpGroupAll && raw <= spv::OpGroupSMax) return true;
+    if (raw >= spv::OpGroupNonUniformElect && raw <= spv::OpGroupNonUniformQuadSwap) return true;
+    switch (op) {
+    case spv::OpAtomicFlagTestAndSet:
+    case spv::OpAtomicFMinEXT:
+    case spv::OpAtomicFMaxEXT:
+    case spv::OpAtomicFAddEXT:
+    case spv::OpSubgroupBallotKHR:
+    case spv::OpSubgroupFirstInvocationKHR:
+    case spv::OpSubgroupAllKHR:
+    case spv::OpSubgroupAnyKHR:
+    case spv::OpSubgroupAllEqualKHR:
+    case spv::OpSubgroupReadInvocationKHR:
+    case spv::OpGroupNonUniformRotateKHR:
+        return true;
+    default:
+        return false;
+    }
+}
+
+bool PrivateStorage(std::uint32_t storageClass) {
+    return storageClass == spv::StorageClassFunction || storageClass == spv::StorageClassPrivate;
+}
+
+std::uint32_t MemoryClass(std::uint32_t storageClass) {
+    switch (storageClass) {
+    case spv::StorageClassUniform:
+    case spv::StorageClassPhysicalStorageBuffer:
+        return spv::StorageClassStorageBuffer;
+    default:
+        return storageClass;
+    }
+}
+
+bool AtomicWrite(spv::Op op) {
+    const auto raw = static_cast<std::uint32_t>(op);
+    return (raw >= spv::OpAtomicExchange && raw <= spv::OpAtomicXor) || op == spv::OpAtomicFlagTestAndSet || op == spv::OpAtomicFMinEXT || op == spv::OpAtomicFMaxEXT ||
+        op == spv::OpAtomicFAddEXT;
+}
+
+bool ImageRead(spv::Op op) {
+    return op == spv::OpImageRead || op == spv::OpImageFetch || op == spv::OpImageSparseRead || op == spv::OpImageSparseFetch;
+}
+
+class Analysis {
+public:
+    explicit Analysis(const Module& module) : module(module) {
+        for (const auto& function : module.functions) {
+            for (const auto& block : function.blocks) {
+                for (const auto& instruction : block.body) {
+                    const auto op = Opcode(instruction);
+                    if (op == spv::OpControlBarrier) barriers.insert(function.id);
+                    bool hasResult = false;
+                    bool hasType = false;
+                    spv::HasResultAndType(op, &hasResult, &hasType);
+                    if (hasResult && hasType && instruction.size() > 2u) types[instruction[2]] = instruction[1];
+                }
+            }
+        }
+        for (const auto& function : module.functions) {
+            for (const auto& block : function.blocks) {
+                for (const auto& instruction : block.body) {
+                    const auto pointer = WrittenPointer(instruction);
+                    if (pointer != 0u) written.insert(ClassOf(pointer));
+                    if (Opcode(instruction) == spv::OpImageWrite) written.insert(spv::StorageClassImage);
+                }
+            }
+        }
+        for (bool changed = true; changed;) {
+            changed = false;
+            for (const auto& function : module.functions) {
+                if (barriers.contains(function.id)) continue;
+                for (const auto& block : function.blocks) {
+                    for (const auto& instruction : block.body) {
+                        if (Opcode(instruction) == spv::OpFunctionCall && instruction.size() > 3u && barriers.contains(instruction[3]) && barriers.insert(function.id).second) changed = true;
+                    }
+                }
+            }
+        }
+    }
+
+    std::optional<std::string> Run() {
+        if (barriers.empty()) return std::nullopt;
+        std::vector<Flow> flows;
+        flows.reserve(module.functions.size());
+        for (const auto& function : module.functions) flows.push_back(BuildFlow(function));
+        for (;;) {
+            const auto before = Size();
+            report.reset();
+            for (std::size_t index = 0u; index < module.functions.size(); ++index) Visit(module.functions[index], flows[index]);
+            if (Size() == before) return report;
+        }
+    }
+
+private:
+    struct Flow {
+        std::vector<std::uint32_t> order;
+        std::unordered_map<std::uint32_t, std::size_t> index;
+        std::unordered_map<std::uint32_t, std::vector<std::uint32_t>> successors;
+        std::unordered_map<std::uint32_t, std::uint32_t> idom;
+        std::unordered_map<std::uint32_t, std::vector<std::uint32_t>> children;
+    };
+
+    static std::uint32_t WrittenPointer(const Instruction& instruction) {
+        const auto op = Opcode(instruction);
+        if ((op == spv::OpStore || op == spv::OpAtomicStore || op == spv::OpAtomicFlagClear || op == spv::OpCopyMemory) && instruction.size() > 1u) return instruction[1];
+        if (AtomicWrite(op) && instruction.size() > 3u) return instruction[3];
+        return 0u;
+    }
+
+    std::uint32_t ClassOf(std::uint32_t pointer) const {
+        if (const auto variable = module.storageClasses.find(pointer); variable != module.storageClasses.end()) return MemoryClass(variable->second);
+        const auto type = types.find(pointer);
+        if (type == types.end()) return UnknownClass;
+        const auto storage = module.pointerClasses.find(type->second);
+        return storage == module.pointerClasses.end() ? UnknownClass : MemoryClass(storage->second);
+    }
+
+    bool Written(std::uint32_t storageClass) const {
+        return written.contains(UnknownClass) || written.contains(storageClass);
+    }
+
+    bool Fence(const Instruction& instruction) const {
+        if (Opcode(instruction) != spv::OpControlBarrier || instruction.size() < 4u) return false;
+        const auto scope = module.constants.find(instruction[1]);
+        const auto semantics = module.constants.find(instruction[3]);
+        return scope != module.constants.end() && semantics != module.constants.end() && scope->second <= spv::ScopeWorkgroup &&
+            (semantics->second & spv::MemorySemanticsWorkgroupMemoryMask) != 0u;
+    }
+
+    bool Fenced(bool fenced, const Instruction& instruction) const {
+        if (Fence(instruction)) return true;
+        if (Opcode(instruction) == spv::OpFunctionCall) return false;
+        const auto pointer = WrittenPointer(instruction);
+        if (pointer == 0u) return fenced;
+        const auto storageClass = ClassOf(pointer);
+        return fenced && storageClass != spv::StorageClassWorkgroup && storageClass != UnknownClass;
+    }
+
+    bool Racy(std::uint32_t pointer, bool fenced) const {
+        const auto storageClass = ClassOf(pointer);
+        if (storageClass != UnknownClass && (PrivateStorage(storageClass) || storageClass == spv::StorageClassInput)) return false;
+        if (!Written(storageClass)) return false;
+        return !fenced || storageClass != spv::StorageClassWorkgroup;
+    }
+
+    std::unordered_map<std::uint32_t, bool> FencedEntries(const Function& function, const Flow& flow, const std::unordered_map<std::uint32_t, const Block*>& blocks) const {
+        std::unordered_map<std::uint32_t, std::vector<std::uint32_t>> predecessors;
+        for (const auto node : flow.order) {
+            for (const auto successor : flow.successors.at(node)) predecessors[successor].push_back(node);
+        }
+        std::unordered_map<std::uint32_t, bool> entries;
+        std::unordered_map<std::uint32_t, bool> exits;
+        for (const auto node : flow.order) exits[node] = true;
+        const auto root = function.blocks.front().label;
+        for (bool changed = true; changed;) {
+            changed = false;
+            for (const auto node : flow.order) {
+                bool fenced = node != root;
+                for (const auto predecessor : predecessors[node]) fenced = fenced && exits.at(predecessor);
+                entries[node] = fenced;
+                for (const auto& instruction : blocks.at(node)->body) fenced = Fenced(fenced, instruction);
+                if (exits[node] != fenced) {
+                    exits[node] = fenced;
+                    changed = true;
+                }
+            }
+        }
+        return entries;
+    }
+
+    std::size_t Size() const {
+        return nonUniform.size() + storedNonUniform.size() + varyingFunctions.size();
+    }
+
+    bool NonUniform(std::uint32_t id) const {
+        return nonUniform.contains(id);
+    }
+
+    static std::vector<std::uint32_t> Successors(const Block& block) {
+        if (block.body.empty()) return {};
+        const auto& terminator = block.body.back();
+        switch (Opcode(terminator)) {
+        case spv::OpBranch: return {terminator.at(1)};
+        case spv::OpBranchConditional: return {terminator.at(2), terminator.at(3)};
+        case spv::OpSwitch: {
+            std::vector<std::uint32_t> targets{terminator.at(2)};
+            for (std::size_t operand = 4u; operand < terminator.size(); operand += 2u) targets.push_back(terminator[operand]);
+            return targets;
+        }
+        default: return {};
+        }
+    }
+
+    static Flow BuildFlow(const Function& function) {
+        Flow flow;
+        for (const auto& block : function.blocks) flow.successors[block.label] = Successors(block);
+        if (function.blocks.empty()) return flow;
+        std::vector<std::uint32_t> postorder;
+        std::unordered_set<std::uint32_t> seen{function.blocks.front().label};
+        std::vector<std::pair<std::uint32_t, std::size_t>> stack{{function.blocks.front().label, 0u}};
+        while (!stack.empty()) {
+            auto& [node, next] = stack.back();
+            const auto& successors = flow.successors[node];
+            if (next == successors.size()) {
+                postorder.push_back(node);
+                stack.pop_back();
+                continue;
+            }
+            const auto successor = successors[next++];
+            if (seen.insert(successor).second) stack.emplace_back(successor, 0u);
+        }
+        flow.order.assign(postorder.rbegin(), postorder.rend());
+        for (std::size_t position = 0u; position < flow.order.size(); ++position) flow.index[flow.order[position]] = position;
+        std::unordered_map<std::uint32_t, std::vector<std::uint32_t>> predecessors;
+        for (const auto node : flow.order) {
+            for (const auto successor : flow.successors[node]) predecessors[successor].push_back(node);
+        }
+        const auto root = flow.order.front();
+        flow.idom[root] = root;
+        const auto intersect = [&](std::uint32_t left, std::uint32_t right) {
+            while (left != right) {
+                while (flow.index[left] > flow.index[right]) left = flow.idom[left];
+                while (flow.index[right] > flow.index[left]) right = flow.idom[right];
+            }
+            return left;
+        };
+        for (bool changed = true; changed;) {
+            changed = false;
+            for (std::size_t position = 1u; position < flow.order.size(); ++position) {
+                const auto node = flow.order[position];
+                std::uint32_t dominator = 0u;
+                for (const auto predecessor : predecessors[node]) {
+                    if (!flow.idom.contains(predecessor)) continue;
+                    dominator = dominator == 0u ? predecessor : intersect(predecessor, dominator);
+                }
+                if (dominator != 0u && flow.idom[node] != dominator) {
+                    flow.idom[node] = dominator;
+                    changed = true;
+                }
+            }
+        }
+        for (const auto node : flow.order) {
+            if (node != root) flow.children[flow.idom[node]].push_back(node);
+        }
+        return flow;
+    }
+
+    static bool Exits(spv::Op op) {
+        return op == spv::OpReturn || op == spv::OpReturnValue || op == spv::OpKill || op == spv::OpTerminateInvocation || op == spv::OpUnreachable;
+    }
+
+    static std::vector<std::uint32_t> Reachable(const Flow& flow, std::uint32_t from) {
+        std::vector<std::uint32_t> blocks;
+        std::unordered_set<std::uint32_t> seen{from};
+        std::vector<std::uint32_t> pending{from};
+        while (!pending.empty()) {
+            const auto node = pending.back();
+            pending.pop_back();
+            blocks.push_back(node);
+            if (const auto found = flow.successors.find(node); found != flow.successors.end()) {
+                for (const auto successor : found->second) {
+                    if (seen.insert(successor).second) pending.push_back(successor);
+                }
+            }
+        }
+        return blocks;
+    }
+
+    static std::vector<std::uint32_t> Construct(const Flow& flow, std::uint32_t header, std::uint32_t merge) {
+        std::vector<std::uint32_t> blocks;
+        std::vector<std::uint32_t> pending;
+        if (const auto found = flow.children.find(header); found != flow.children.end()) pending = found->second;
+        while (!pending.empty()) {
+            const auto node = pending.back();
+            pending.pop_back();
+            if (node == merge) continue;
+            blocks.push_back(node);
+            if (const auto found = flow.children.find(node); found != flow.children.end()) pending.insert(pending.end(), found->second.begin(), found->second.end());
+        }
+        return blocks;
+    }
+
+    std::uint32_t BaseVariable(std::uint32_t pointer, const std::unordered_map<std::uint32_t, std::uint32_t>& chains) const {
+        for (std::size_t depth = 0u; depth < 64u; ++depth) {
+            const auto found = chains.find(pointer);
+            if (found == chains.end()) break;
+            pointer = found->second;
+        }
+        return pointer;
+    }
+
+    void Visit(const Function& function, const Flow& flow) {
+        if (function.blocks.empty()) return;
+        std::unordered_map<std::uint32_t, const Block*> blocks;
+        std::unordered_map<std::uint32_t, std::uint32_t> chains;
+        for (const auto& block : function.blocks) {
+            blocks[block.label] = &block;
+            for (const auto& instruction : block.body) {
+                const auto op = Opcode(instruction);
+                if ((op == spv::OpAccessChain || op == spv::OpInBoundsAccessChain || op == spv::OpPtrAccessChain) && instruction.size() > 3u) chains[instruction[2]] = instruction[3];
+            }
+        }
+        const auto fencedEntries = FencedEntries(function, flow, blocks);
+        for (const auto parameter : function.parameters) nonUniform.insert(parameter);
+        std::unordered_set<std::uint32_t> divergent;
+        std::unordered_set<std::uint32_t> divergentMerges;
+        for (auto position = flow.order.size(); position-- > 0u;) {
+            const auto header = flow.order[position];
+            const auto& body = blocks.at(header)->body;
+            if (body.size() < 2u) continue;
+            const auto& mergeInstruction = body[body.size() - 2u];
+            const auto mergeOp = Opcode(mergeInstruction);
+            if (mergeOp != spv::OpSelectionMerge && mergeOp != spv::OpLoopMerge) continue;
+            const auto merge = mergeInstruction.at(1);
+            const auto& terminator = body.back();
+            const auto terminatorOp = Opcode(terminator);
+            bool split = (terminatorOp == spv::OpBranchConditional || terminatorOp == spv::OpSwitch) && NonUniform(terminator.at(1));
+            const auto construct = Construct(flow, header, merge);
+            if (mergeOp == spv::OpLoopMerge && !split) {
+                for (const auto node : construct) {
+                    const auto& successors = flow.successors.at(node);
+                    if (std::find(successors.begin(), successors.end(), merge) == successors.end()) continue;
+                    const auto& exit = blocks.at(node)->body.back();
+                    const auto exitOp = Opcode(exit);
+                    if (divergent.contains(node) || ((exitOp == spv::OpBranchConditional || exitOp == spv::OpSwitch) && NonUniform(exit.at(1)))) {
+                        split = true;
+                        break;
+                    }
+                }
+            }
+            if (!split) {
+                const auto continueTarget = mergeOp == spv::OpLoopMerge ? mergeInstruction.at(2) : 0u;
+                for (const auto node : construct) {
+                    const auto& nodeBody = blocks.at(node)->body;
+                    if (nodeBody.empty()) continue;
+                    const auto& exit = nodeBody.back();
+                    const auto exitOp = Opcode(exit);
+                    if ((exitOp != spv::OpBranchConditional && exitOp != spv::OpSwitch) || !NonUniform(exit.at(1))) continue;
+                    if (nodeBody.size() >= 2u && (Opcode(nodeBody[nodeBody.size() - 2u]) == spv::OpSelectionMerge || Opcode(nodeBody[nodeBody.size() - 2u]) == spv::OpLoopMerge)) continue;
+                    const auto& successors = flow.successors.at(node);
+                    if (std::find(successors.begin(), successors.end(), merge) != successors.end() || (continueTarget != 0u && std::find(successors.begin(), successors.end(), continueTarget) != successors.end())) {
+                        split = true;
+                        break;
+                    }
+                }
+            }
+            if (!split) continue;
+            divergent.insert(construct.begin(), construct.end());
+            if (mergeOp == spv::OpLoopMerge) divergent.insert(header);
+            divergentMerges.insert(merge);
+            const std::unordered_set<std::uint32_t> inside(construct.begin(), construct.end());
+            bool exits = false;
+            bool escapes = false;
+            for (const auto node : construct) {
+                exits = exits || (!blocks.at(node)->body.empty() && Exits(Opcode(blocks.at(node)->body.back())));
+                for (const auto successor : flow.successors.at(node)) escapes = escapes || (successor != merge && successor != header && !inside.contains(successor));
+            }
+            if (exits) {
+                const auto after = Reachable(flow, merge);
+                divergent.insert(after.begin(), after.end());
+            } else if (escapes) {
+                const auto after = Construct(flow, merge, 0u);
+                divergent.insert(merge);
+                divergent.insert(after.begin(), after.end());
+            }
+        }
+        for (const auto label : flow.order) {
+            const bool inDivergentFlow = divergent.contains(label);
+            bool fenced = fencedEntries.at(label);
+            for (const auto& instruction : blocks.at(label)->body) {
+                const auto op = Opcode(instruction);
+                const bool fencedBefore = fenced;
+                fenced = Fenced(fenced, instruction);
+                if (op == spv::OpStore && instruction.size() >= 3u) {
+                    const auto variable = BaseVariable(instruction[1], chains);
+                    const auto storage = module.storageClasses.find(variable);
+                    if (storage != module.storageClasses.end() && PrivateStorage(storage->second) && (inDivergentFlow || NonUniform(instruction[1]) || NonUniform(instruction[2]))) storedNonUniform.insert(variable);
+                    continue;
+                }
+                if (inDivergentFlow && !report && (op == spv::OpControlBarrier || (op == spv::OpFunctionCall && instruction.size() > 3u && barriers.contains(instruction[3])))) {
+                    report = std::string(op == spv::OpControlBarrier ? "an OpControlBarrier" : "a call reaching an OpControlBarrier") + " in block %" + std::to_string(label) + " of function %" + std::to_string(function.id) + " sits in control flow a per-invocation value decides";
+                }
+                const auto result = Result(instruction);
+                if (result == 0u || NonUniform(result)) continue;
+                bool varying = false;
+                switch (op) {
+                case spv::OpLoad: {
+                    const auto variable = BaseVariable(instruction.at(3), chains);
+                    const auto builtin = module.builtins.find(variable);
+                    const auto storage = module.storageClasses.find(variable);
+                    varying = NonUniform(instruction[3]) || (builtin != module.builtins.end() && VaryingBuiltin(builtin->second)) || storedNonUniform.contains(variable) ||
+                        (storage != module.storageClasses.end() && storage->second == spv::StorageClassInput && builtin == module.builtins.end()) || Racy(instruction[3], fencedBefore);
+                    break;
+                }
+                case spv::OpAtomicLoad:
+                    varying = NonUniform(instruction.at(3)) || Racy(instruction[3], fencedBefore);
+                    break;
+                case spv::OpPhi:
+                    varying = inDivergentFlow || divergentMerges.contains(label);
+                    for (std::size_t operand = 3u; !varying && operand < instruction.size(); operand += 2u) varying = NonUniform(instruction[operand]);
+                    break;
+                case spv::OpFunctionCall:
+                    varying = instruction.size() > 3u && varyingFunctions.contains(instruction[3]);
+                    for (std::size_t operand = 4u; !varying && operand < instruction.size(); ++operand) varying = NonUniform(instruction[operand]);
+                    break;
+                default:
+                    varying = PerInvocationResult(op) || (ImageRead(op) && Written(spv::StorageClassImage));
+                    if (!varying) VisitIdOperands(instruction, 1u, [&](std::size_t operand) { varying = varying || NonUniform(instruction[operand]); });
+                    break;
+                }
+                if (!varying) continue;
+                nonUniform.insert(result);
+                if (op != spv::OpFunctionCall) varyingFunctions.insert(function.id);
+            }
+        }
+    }
+
+    static constexpr std::uint32_t UnknownClass = 0xffffffffu;
+
+    const Module& module;
+    std::unordered_map<std::uint32_t, std::uint32_t> types;
+    std::unordered_set<std::uint32_t> written;
+    std::unordered_set<std::uint32_t> barriers;
+    std::unordered_set<std::uint32_t> nonUniform;
+    std::unordered_set<std::uint32_t> storedNonUniform;
+    std::unordered_set<std::uint32_t> varyingFunctions;
+    std::optional<std::string> report;
+};
+
+}
+
+std::optional<std::string> SpannedBarrierInDivergentFlow(std::span<const std::uint32_t> words) {
+    bool barrier = false;
+    for (std::size_t cursor = 5u; !barrier && cursor < words.size(); cursor += std::max(words[cursor] >> 16u, 1u)) barrier = (words[cursor] & 0xffffu) == spv::OpControlBarrier;
+    if (!barrier) return std::nullopt;
+    const auto module = Parse(words);
+    return Analysis(module).Run();
+}
+
+}

core/shader/recompiler/SpirvBackend/src/SpirvSpecialization.cpp

@@ -2,6 +2,7 @@
 #include <spirv/unified1/spirv.hpp>
 #undef SPV_ENABLE_UTILITY_CODE
 #include "SpirvBackend/SpirvSpecialization.hpp"
+#include "SpirvInstructionOperands.hpp"
 #include <algorithm>
 #include <array>
 #include <initializer_list>
@@ -68,17 +69,9 @@ struct ArenaAllocator {
 
 using Instruction = std::vector<std::uint32_t, ArenaAllocator<std::uint32_t>>;
 
-spv::Op Opcode(const Instruction& instruction) {
-    return instruction.empty() ? spv::OpNop : static_cast<spv::Op>(instruction[0] & 0xffffu);
-}
-
-std::uint32_t Result(const Instruction& instruction) {
-    if (instruction.empty()) return 0u;
-    bool result = false;
-    bool type = false;
-    spv::HasResultAndType(Opcode(instruction), &result, &type);
-    return result ? instruction.at(type ? 2u : 1u) : 0u;
-}
+using SpirvInstructions::Opcode;
+using SpirvInstructions::Result;
+using SpirvInstructions::VisitIdOperands;
 
 Instruction Make(spv::Op op, std::initializer_list<std::uint32_t> operands) {
     Instruction result{(static_cast<std::uint32_t>(operands.size() + 1u) << 16u) | op};
@@ -273,209 +266,6 @@ bool VisitInputs(const Instruction& instruction, const TVisitor& visit) {
     return true;
 }
 
-template<typename TVisitor>
-void VisitIdOperands(const Instruction& instruction, std::uint32_t switchLiteralWords, const TVisitor& visit) {
-    const auto size = instruction.size();
-    const auto from = [&](std::size_t first) {
-        for (auto index = first; index < size; ++index) visit(index);
-    };
-    const auto only = [&](std::initializer_list<std::size_t> indices) {
-        for (const auto index : indices) {
-            if (index < size) visit(index);
-        }
-    };
-    constexpr std::uint32_t knownMemoryAccess = static_cast<std::uint32_t>(spv::MemoryAccessVolatileMask) | static_cast<std::uint32_t>(spv::MemoryAccessAlignedMask) |
-        static_cast<std::uint32_t>(spv::MemoryAccessNontemporalMask) | static_cast<std::uint32_t>(spv::MemoryAccessMakePointerAvailableMask) |
-        static_cast<std::uint32_t>(spv::MemoryAccessMakePointerVisibleMask) | static_cast<std::uint32_t>(spv::MemoryAccessNonPrivatePointerMask);
-    const auto memoryOperands = [&](std::size_t index) {
-        while (index < size) {
-            const auto mask = instruction[index++];
-            if ((mask & ~knownMemoryAccess) != 0u) {
-                from(index);
-                return;
-            }
-            if ((mask & static_cast<std::uint32_t>(spv::MemoryAccessAlignedMask)) != 0u) ++index;
-            if ((mask & static_cast<std::uint32_t>(spv::MemoryAccessMakePointerAvailableMask)) != 0u && index < size) visit(index++);
-            if ((mask & static_cast<std::uint32_t>(spv::MemoryAccessMakePointerVisibleMask)) != 0u && index < size) visit(index++);
-        }
-    };
-    switch (Opcode(instruction)) {
-    case spv::OpNop:
-    case spv::OpCapability:
-    case spv::OpExtension:
-    case spv::OpExtInstImport:
-    case spv::OpMemoryModel:
-    case spv::OpSource:
-    case spv::OpSourceContinued:
-    case spv::OpSourceExtension:
-    case spv::OpString:
-    case spv::OpName:
-    case spv::OpMemberName:
-    case spv::OpModuleProcessed:
-    case spv::OpDecorate:
-    case spv::OpMemberDecorate:
-    case spv::OpDecorateString:
-    case spv::OpMemberDecorateString:
-    case spv::OpTypeVoid:
-    case spv::OpTypeBool:
-    case spv::OpTypeInt:
-    case spv::OpTypeFloat:
-    case spv::OpTypeSampler:
-    case spv::OpConstantTrue:
-    case spv::OpConstantFalse:
-    case spv::OpConstant:
-    case spv::OpConstantSampler:
-    case spv::OpConstantNull:
-    case spv::OpSpecConstantTrue:
-    case spv::OpSpecConstantFalse:
-    case spv::OpSpecConstant:
-    case spv::OpLabel:
-    case spv::OpNoLine:
-    case spv::OpReturn:
-    case spv::OpUnreachable:
-    case spv::OpKill:
-    case spv::OpFunctionEnd:
-        return;
-    case spv::OpTypeVector:
-    case spv::OpTypeMatrix:
-    case spv::OpTypeImage:
-        only({2});
-        return;
-    case spv::OpTypePointer:
-        only({3});
-        return;
-    case spv::OpTypeForwardPointer:
-    case spv::OpExecutionMode:
-    case spv::OpLine:
-    case spv::OpSelectionMerge:
-        only({1});
-        return;
-    case spv::OpSpecConstantOp:
-        only({1});
-        from(4);
-        return;
-    case spv::OpVariable:
-    case spv::OpFunction:
-        only({1, 4});
-        return;
-    case spv::OpLoad:
-        only({1, 3});
-        memoryOperands(4);
-        return;
-    case spv::OpStore:
-    case spv::OpCopyMemory:
-        only({1, 2});
-        memoryOperands(3);
-        return;
-    case spv::OpCopyMemorySized:
-        only({1, 2, 3});
-        memoryOperands(4);
-        return;
-    case spv::OpArrayLength:
-    case spv::OpCompositeExtract:
-        only({1, 3});
-        return;
-    case spv::OpCompositeInsert:
-    case spv::OpVectorShuffle:
-        only({1, 3, 4});
-        return;
-    case spv::OpExtInst:
-        only({1, 3});
-        from(5);
-        return;
-    case spv::OpEntryPoint: {
-        only({2});
-        std::size_t index = 3;
-        while (index < size) {
-            const auto word = instruction[index++];
-            if ((word & 0xffu) == 0u || (word & 0xff00u) == 0u || (word & 0xff0000u) == 0u || (word & 0xff000000u) == 0u) break;
-        }
-        from(index);
-        return;
-    }
-    case spv::OpExecutionModeId:
-    case spv::OpDecorateId:
-        only({1});
-        from(3);
-        return;
-    case spv::OpLoopMerge:
-        only({1, 2});
-        return;
-    case spv::OpBranchConditional:
-        only({1, 2, 3});
-        return;
-    case spv::OpSwitch:
-        only({1, 2});
-        for (std::size_t index = 3; index + switchLiteralWords < size; index += switchLiteralWords + 1u) visit(index + switchLiteralWords);
-        return;
-    case spv::OpImageSampleImplicitLod:
-    case spv::OpImageSampleExplicitLod:
-    case spv::OpImageSampleProjImplicitLod:
-    case spv::OpImageSampleProjExplicitLod:
-    case spv::OpImageFetch:
-    case spv::OpImageRead:
-    case spv::OpImageSparseSampleImplicitLod:
-    case spv::OpImageSparseSampleExplicitLod:
-    case spv::OpImageSparseSampleProjImplicitLod:
-    case spv::OpImageSparseSampleProjExplicitLod:
-    case spv::OpImageSparseFetch:
-    case spv::OpImageSparseRead:
-        only({1, 3, 4});
-        from(6);
-        return;
-    case spv::OpImageSampleDrefImplicitLod:
-    case spv::OpImageSampleDrefExplicitLod:
-    case spv::OpImageSampleProjDrefImplicitLod:
-    case spv::OpImageSampleProjDrefExplicitLod:
-    case spv::OpImageGather:
-    case spv::OpImageDrefGather:
-    case spv::OpImageSparseSampleDrefImplicitLod:
-    case spv::OpImageSparseSampleDrefExplicitLod:
-    case spv::OpImageSparseSampleProjDrefImplicitLod:
-    case spv::OpImageSparseSampleProjDrefExplicitLod:
-    case spv::OpImageSparseGather:
-    case spv::OpImageSparseDrefGather:
-        only({1, 3, 4, 5});
-        from(7);
-        return;
-    case spv::OpImageWrite:
-        only({1, 2, 3});
-        from(5);
-        return;
-    case spv::OpGroupIAdd:
-    case spv::OpGroupFAdd:
-    case spv::OpGroupFMin:
-    case spv::OpGroupUMin:
-    case spv::OpGroupSMin:
-    case spv::OpGroupFMax:
-    case spv::OpGroupUMax:
-    case spv::OpGroupSMax:
-    case spv::OpGroupNonUniformBallotBitCount:
-    case spv::OpGroupNonUniformIAdd:
-    case spv::OpGroupNonUniformFAdd:
-    case spv::OpGroupNonUniformIMul:
-    case spv::OpGroupNonUniformFMul:
-    case spv::OpGroupNonUniformSMin:
-    case spv::OpGroupNonUniformUMin:
-    case spv::OpGroupNonUniformFMin:
-    case spv::OpGroupNonUniformSMax:
-    case spv::OpGroupNonUniformUMax:
-    case spv::OpGroupNonUniformFMax:
-    case spv::OpGroupNonUniformBitwiseAnd:
-    case spv::OpGroupNonUniformBitwiseOr:
-    case spv::OpGroupNonUniformBitwiseXor:
-    case spv::OpGroupNonUniformLogicalAnd:
-    case spv::OpGroupNonUniformLogicalOr:
-    case spv::OpGroupNonUniformLogicalXor:
-        only({1, 3});
-        from(5);
-        return;
-    default:
-        from(1);
-        return;
-    }
-}
-
 struct ScalarType {
     std::uint32_t width;
     bool boolean;

core/shader/recompiler/tests/SpannedUniformityTests.cpp

@@ -0,0 +1,314 @@
+#include "SpirvBackend/SpirvSpannedUniformity.hpp"
+#include <spirv/unified1/spirv.hpp>
+#include <cstdio>
+#include <cstdlib>
+#include <functional>
+#include <initializer_list>
+#include <vector>
+
+namespace {
+
+enum Id : std::uint32_t {
+    Void = 1u,
+    Bool,
+    Uint,
+    FunctionType,
+    InputPointer,
+    Index,
+    Two,
+    Three,
+    Semantics,
+    WorkgroupPointer,
+    WorkgroupArrayType,
+    WorkgroupArrayPointer,
+    Exchange,
+    FunctionPointer,
+    True,
+    Main,
+    Helper,
+    StoragePointer,
+    Storage,
+    PhysicalPointer,
+    First = 100u,
+};
+
+class Module {
+public:
+    Module() {
+        words = {spv::MagicNumber, 0x00010300u, 0u, 1000u, 0u};
+        emit(spv::OpCapability, {spv::CapabilityShader});
+        emit(spv::OpMemoryModel, {spv::AddressingModelLogical, spv::MemoryModelGLSL450});
+        emit(spv::OpEntryPoint, {spv::ExecutionModelGLCompute, Main, 0x6e69616du, 0u, Index});
+        emit(spv::OpExecutionMode, {Main, spv::ExecutionModeLocalSize, 64u, 1u, 1u});
+        emit(spv::OpDecorate, {Index, spv::DecorationBuiltIn, spv::BuiltInLocalInvocationIndex});
+        emit(spv::OpTypeVoid, {Void});
+        emit(spv::OpTypeBool, {Bool});
+        emit(spv::OpTypeInt, {Uint, 32u, 0u});
+        emit(spv::OpTypeFunction, {FunctionType, Void});
+        emit(spv::OpTypePointer, {InputPointer, spv::StorageClassInput, Uint});
+        emit(spv::OpVariable, {InputPointer, Index, spv::StorageClassInput});
+        emit(spv::OpConstant, {Uint, Two, 2u});
+        emit(spv::OpConstant, {Uint, Three, 3u});
+        emit(spv::OpConstant, {Uint, Semantics, 264u});
+        emit(spv::OpConstantTrue, {Bool, True});
+        emit(spv::OpTypePointer, {WorkgroupPointer, spv::StorageClassWorkgroup, Uint});
+        emit(spv::OpTypeArray, {WorkgroupArrayType, Uint, Three});
+        emit(spv::OpTypePointer, {WorkgroupArrayPointer, spv::StorageClassWorkgroup, WorkgroupArrayType});
+        emit(spv::OpVariable, {WorkgroupArrayPointer, Exchange, spv::StorageClassWorkgroup});
+        emit(spv::OpTypePointer, {FunctionPointer, spv::StorageClassFunction, Uint});
+        emit(spv::OpTypePointer, {StoragePointer, spv::StorageClassStorageBuffer, Uint});
+        emit(spv::OpVariable, {StoragePointer, Storage, spv::StorageClassStorageBuffer});
+        emit(spv::OpTypePointer, {PhysicalPointer, spv::StorageClassPhysicalStorageBuffer, Uint});
+    }
+
+    void emit(spv::Op op, std::initializer_list<std::uint32_t> operands) {
+        words.push_back((static_cast<std::uint32_t>(operands.size() + 1u) << 16u) | op);
+        words.insert(words.end(), operands);
+    }
+
+    void function(std::uint32_t id) {
+        emit(spv::OpFunction, {Void, id, spv::FunctionControlMaskNone, FunctionType});
+    }
+
+    void barrier() {
+        emit(spv::OpControlBarrier, {Two, Two, Semantics});
+    }
+
+    void lane(std::uint32_t result) {
+        emit(spv::OpLoad, {Uint, result, Index});
+    }
+
+    void end() {
+        emit(spv::OpReturn, {});
+        emit(spv::OpFunctionEnd, {});
+    }
+
+    std::vector<std::uint32_t> words;
+};
+
+bool failed = false;
+
+void expect(const char* name, const Module& module, bool rejected) {
+    const auto reason = ShaderRecompiler::SpannedBarrierInDivergentFlow(module.words);
+    if (reason.has_value() == rejected) return;
+    std::fprintf(stderr, "%s: expected %s, got %s\n", name, rejected ? "a rejection" : "no rejection", reason ? reason->c_str() : "none");
+    failed = true;
+}
+
+Module selection(bool perLane) {
+    Module module;
+    module.function(Main);
+    module.emit(spv::OpLabel, {First});
+    module.lane(First + 1u);
+    module.emit(spv::OpULessThan, {Bool, First + 2u, perLane ? First + 1u : Two, Three});
+    module.emit(spv::OpSelectionMerge, {First + 4u, spv::SelectionControlMaskNone});
+    module.emit(spv::OpBranchConditional, {First + 2u, First + 3u, First + 4u});
+    module.emit(spv::OpLabel, {First + 3u});
+    module.barrier();
+    module.emit(spv::OpBranch, {First + 4u});
+    module.emit(spv::OpLabel, {First + 4u});
+    module.barrier();
+    module.end();
+    return module;
+}
+
+Module loop(bool laneExit, bool barrierInSelection) {
+    Module module;
+    module.function(Main);
+    module.emit(spv::OpLabel, {First});
+    module.lane(First + 1u);
+    module.emit(spv::OpULessThan, {Bool, First + 2u, First + 1u, Three});
+    module.emit(spv::OpBranch, {First + 3u});
+    module.emit(spv::OpLabel, {First + 3u});
+    module.emit(spv::OpLoopMerge, {First + 6u, First + 5u, spv::LoopControlMaskNone});
+    module.emit(spv::OpBranch, {First + 4u});
+    module.emit(spv::OpLabel, {First + 4u});
+    if (barrierInSelection) {
+        module.emit(spv::OpSelectionMerge, {First + 8u, spv::SelectionControlMaskNone});
+        module.emit(spv::OpBranchConditional, {First + 2u, First + 7u, First + 8u});
+        module.emit(spv::OpLabel, {First + 7u});
+        module.emit(spv::OpBranch, {First + 8u});
+        module.emit(spv::OpLabel, {First + 8u});
+    }
+    module.barrier();
+    module.emit(spv::OpBranch, {First + 5u});
+    module.emit(spv::OpLabel, {First + 5u});
+    module.emit(spv::OpBranchConditional, {laneExit ? First + 2u : True, First + 3u, First + 6u});
+    module.emit(spv::OpLabel, {First + 6u});
+    module.end();
+    return module;
+}
+
+Module call(bool perLane) {
+    Module module;
+    module.function(Helper);
+    module.emit(spv::OpLabel, {First + 10u});
+    module.barrier();
+    module.end();
+    module.function(Main);
+    module.emit(spv::OpLabel, {First});
+    module.lane(First + 1u);
+    module.emit(spv::OpULessThan, {Bool, First + 2u, perLane ? First + 1u : Two, Three});
+    module.emit(spv::OpSelectionMerge, {First + 4u, spv::SelectionControlMaskNone});
+    module.emit(spv::OpBranchConditional, {First + 2u, First + 3u, First + 4u});
+    module.emit(spv::OpLabel, {First + 3u});
+    module.emit(spv::OpFunctionCall, {Void, First + 5u, Helper});
+    module.emit(spv::OpBranch, {First + 4u});
+    module.emit(spv::OpLabel, {First + 4u});
+    module.end();
+    return module;
+}
+
+Module early(bool perLane, spv::Op exit) {
+    Module module;
+    module.function(Main);
+    module.emit(spv::OpLabel, {First});
+    module.lane(First + 1u);
+    module.emit(spv::OpULessThan, {Bool, First + 2u, perLane ? First + 1u : Two, Three});
+    module.emit(spv::OpBranch, {First + 10u});
+    module.emit(spv::OpLabel, {First + 10u});
+    module.emit(spv::OpLoopMerge, {First + 13u, First + 12u, spv::LoopControlMaskNone});
+    module.emit(spv::OpBranch, {First + 11u});
+    module.emit(spv::OpLabel, {First + 11u});
+    module.emit(spv::OpSelectionMerge, {First + 4u, spv::SelectionControlMaskNone});
+    module.emit(spv::OpBranchConditional, {First + 2u, First + 3u, First + 4u});
+    module.emit(spv::OpLabel, {First + 3u});
+    if (exit == spv::OpReturn) module.emit(spv::OpReturn, {});
+    else module.emit(spv::OpBranch, {First + 12u});
+    module.emit(spv::OpLabel, {First + 4u});
+    module.barrier();
+    module.emit(spv::OpBranch, {First + 12u});
+    module.emit(spv::OpLabel, {First + 12u});
+    module.emit(spv::OpBranchConditional, {True, First + 10u, First + 13u});
+    module.emit(spv::OpLabel, {First + 13u});
+    module.end();
+    return module;
+}
+
+Module unmergedContinue(bool perLane) {
+    Module module;
+    module.function(Main);
+    module.emit(spv::OpLabel, {First});
+    module.lane(First + 1u);
+    module.emit(spv::OpULessThan, {Bool, First + 2u, perLane ? First + 1u : Two, Three});
+    module.emit(spv::OpBranch, {First + 10u});
+    module.emit(spv::OpLabel, {First + 10u});
+    module.emit(spv::OpLoopMerge, {First + 13u, First + 12u, spv::LoopControlMaskNone});
+    module.emit(spv::OpBranch, {First + 11u});
+    module.emit(spv::OpLabel, {First + 11u});
+    module.emit(spv::OpBranchConditional, {First + 2u, First + 12u, First + 14u});
+    module.emit(spv::OpLabel, {First + 14u});
+    module.barrier();
+    module.emit(spv::OpBranch, {First + 12u});
+    module.emit(spv::OpLabel, {First + 12u});
+    module.emit(spv::OpBranchConditional, {True, First + 10u, First + 13u});
+    module.emit(spv::OpLabel, {First + 13u});
+    module.end();
+    return module;
+}
+
+Module joined(const std::function<void(Module&)>& condition) {
+    Module module;
+    module.function(Main);
+    module.emit(spv::OpLabel, {First});
+    module.emit(spv::OpVariable, {FunctionPointer, First + 20u, spv::StorageClassFunction});
+    module.lane(First + 1u);
+    condition(module);
+    module.emit(spv::OpSelectionMerge, {First + 9u, spv::SelectionControlMaskNone});
+    module.emit(spv::OpBranchConditional, {First + 8u, First + 7u, First + 9u});
+    module.emit(spv::OpLabel, {First + 7u});
+    module.barrier();
+    module.emit(spv::OpBranch, {First + 9u});
+    module.emit(spv::OpLabel, {First + 9u});
+    module.end();
+    return module;
+}
+
+void laneSplit(Module& module, std::uint32_t merge) {
+    module.emit(spv::OpULessThan, {Bool, First + 2u, First + 1u, Three});
+    module.emit(spv::OpSelectionMerge, {merge, spv::SelectionControlMaskNone});
+    module.emit(spv::OpBranchConditional, {First + 2u, First + 3u, merge});
+    module.emit(spv::OpLabel, {First + 3u});
+}
+
+}
+
+int main() {
+    expect("a barrier under a lane comparison", selection(true), true);
+    expect("a barrier under a constant comparison", selection(false), false);
+    expect("a barrier in a loop a lane value leaves", loop(true, false), true);
+    expect("a barrier in a uniform loop after a per-lane selection", loop(false, true), false);
+    expect("a call reaching a barrier under a lane comparison", call(true), true);
+    expect("a barrier after a per-lane return", early(true, spv::OpReturn), true);
+    expect("a barrier after a per-lane continue", early(true, spv::OpBranch), true);
+    expect("a barrier after a uniform return", early(false, spv::OpReturn), false);
+    expect("a barrier after a per-lane continue without a selection merge", unmergedContinue(true), true);
+    expect("a barrier after a uniform continue without a selection merge", unmergedContinue(false), false);
+    expect("a call reaching a barrier under a constant comparison", call(false), false);
+    expect("a barrier under a phi joining a per-lane selection", joined([](Module& module) {
+        laneSplit(module, First + 4u);
+        module.emit(spv::OpBranch, {First + 4u});
+        module.emit(spv::OpLabel, {First + 4u});
+        module.emit(spv::OpPhi, {Bool, First + 8u, True, First + 3u, True, First});
+    }), true);
+    expect("a barrier under a variable stored on one side of a per-lane selection", joined([](Module& module) {
+        module.emit(spv::OpStore, {First + 20u, Two});
+        laneSplit(module, First + 4u);
+        module.emit(spv::OpStore, {First + 20u, Three});
+        module.emit(spv::OpBranch, {First + 4u});
+        module.emit(spv::OpLabel, {First + 4u});
+        module.emit(spv::OpLoad, {Uint, First + 5u, First + 20u});
+        module.emit(spv::OpIEqual, {Bool, First + 8u, First + 5u, Two});
+    }), true);
+    expect("a barrier under an atomic result", joined([](Module& module) {
+        module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, Two});
+        module.emit(spv::OpAtomicIAdd, {Uint, First + 6u, First + 5u, Two, Semantics, Three});
+        module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two});
+    }), true);
+    expect("a barrier under a workgroup word read at a uniform index", joined([](Module& module) {
+        module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, Two});
+        module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u});
+        module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two});
+    }), false);
+    expect("a barrier under a workgroup word read at a uniform index after a store", joined([](Module& module) {
+        module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, Two});
+        module.emit(spv::OpStore, {First + 5u, Three});
+        module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u});
+        module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two});
+    }), true);
+    expect("a barrier under a workgroup word read at a uniform index after a store and a barrier", joined([](Module& module) {
+        module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, Two});
+        module.emit(spv::OpStore, {First + 5u, Three});
+        module.barrier();
+        module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u});
+        module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two});
+    }), false);
+    expect("a barrier under a workgroup word read at a uniform index after a store on one path past the barrier", joined([](Module& module) {
+        module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, Two});
+        module.barrier();
+        module.emit(spv::OpSelectionMerge, {First + 4u, spv::SelectionControlMaskNone});
+        module.emit(spv::OpBranchConditional, {True, First + 3u, First + 4u});
+        module.emit(spv::OpLabel, {First + 3u});
+        module.emit(spv::OpStore, {First + 5u, Three});
+        module.emit(spv::OpBranch, {First + 4u});
+        module.emit(spv::OpLabel, {First + 4u});
+        module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u});
+        module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two});
+    }), true);
+    expect("a barrier under a buffer word read at a uniform index", joined([](Module& module) {
+        module.emit(spv::OpLoad, {Uint, First + 6u, Storage});
+        module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two});
+    }), false);
+    expect("a barrier under a buffer word read at a uniform index after a store through a device address", joined([](Module& module) {
+        module.emit(spv::OpConvertUToPtr, {PhysicalPointer, First + 10u, Two});
+        module.emit(spv::OpStore, {First + 10u, Three});
+        module.emit(spv::OpLoad, {Uint, First + 6u, Storage});
+        module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two});
+    }), true);
+    expect("a barrier under a workgroup word read at a lane index", joined([](Module& module) {
+        module.emit(spv::OpAccessChain, {WorkgroupPointer, First + 5u, Exchange, First + 1u});
+        module.emit(spv::OpLoad, {Uint, First + 6u, First + 5u});
+        module.emit(spv::OpIEqual, {Bool, First + 8u, First + 6u, Two});
+    }), true);
+    return failed ? EXIT_FAILURE : EXIT_SUCCESS;
+}

docs/dev/ENVIRONMENT.md

@@ -350,6 +350,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti
 | `APS5_NO_SNAPSHOT_CHECK` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` |  | skips the live compare that proves a captured snapshot still equals the guest bytes |
 | `APS5_NO_SORTED_SNAPSHOT_LOOKUP` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` |  | scans the snapshot's regions in AddSnapshot instead of searching them |
 | `APS5_NO_SPACE_STAMP_ONCE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` |  | stamps every direct import of a cached address space on each hit instead of once per space |
+| `APS5_NO_SPANNED_WAVE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` |  | keeps single-wave wave64 compute programs at two guest lanes per invocation instead of spanning two full 32-lane subgroups |
 | `APS5_NO_SPARSE_HEAP_MIRROR` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` |  | disables sparse heap mirrors, mirroring only whole heap ranges |
 | `APS5_NO_STAGE_A_IMAGES` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` |  | leaves every image element to the stage-B full lookup instead of a stage-A record |
 | `APS5_NO_STORAGE_DEDUPE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` |  | looks each identical consecutive storage descriptor up, and refreshes its image, separately |
@@ -419,7 +420,6 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti
 | `APS5_SKIP_SHADER` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Diagnostics.cpp` | 0 | hex program address of a shader whose draws and dispatches are dropped |
 | `APS5_SKIP_SHADER_FILE` | path | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Diagnostics.cpp` |  | file of whitespace-separated hex program addresses (`#` starts a comment) whose draws and dispatches are dropped; re-read within 500 ms of a change; a token that is not a whole 64-bit hex number is reported and ignored |
 | `APS5_SNAPSHOT_CHECK_FATAL` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` |  | throws on the first snapshot that differs from registered memory instead of only counting them |
-| `APS5_SPANNED_WAVE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` |  | spans a single-wave wave64 compute program over two full 32-lane subgroups |
 | `APS5_SPIRV_OPT` | string | `core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp` |  | selects the SPIR-V optimization pipeline: `full` restores the performance pipeline, `none` skips optimization |
 | `APS5_SRGB_ATTACHMENT_PROXY` | string | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` |  | forces the sRGB color-attachment proxy format; `1` forces it |
 | `APS5_SRGB_SHADER_DECODE` | string | `core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp` |  | `1` forces shader sRGB decode for the sRGB 8-bit formats; `copy` does the same, except that 8 and 8_8 sRGB sample an RGBA8 sRGB copy when the device can blit to RGBA8; any other value has no effect |

docs/dev/debt/shader.md

@@ -221,7 +221,7 @@
 - [Flag structurization](../../../core/shader/recompiler/ControlFlow/src/Structurizer.cpp) (shader recompiler) - a graph that every structurizer attempt refuses, irreducible and non-terminating graphs included, is rebuilt with one reach flag per block and one again flag per loop (`structurizeWithFlags`): each block keeps one copy and runs behind a guard on its flag, and each strongly connected component becomes a loop whose single latch repeats it while a back edge set its flag. The output has about 8 times the input's blocks and one route variable per block, and ends in an empty Return that never runs. Only blocks that start the top-level order unconditionally run unguarded, so resource planning also sees the static paths that skip a guarded block and can plan extra descriptor candidates for them. The attempts still run first. Shared merge splitting refuses an attempt once the graph has grown past 16 times its block count at the start of splitting (at least 64 blocks); succeeding attempts grew at most 10.6 times on random graphs of up to 128 blocks, and an attempt that would succeed only beyond the budget leaves the graph to a later attempt or to this construction. Splitting that adds one block per round stays within that budget until its 4 rounds per block run out, so the attempts can still take more than 10 s (9 of 2,000 random graphs of up to 128 blocks, 63 s at most). `APS5_FORCE_STRUCTURIZER_FALLBACK=1` builds every graph this way, for tests
 - [Shared loop latches](../../../core/shader/recompiler/ControlFlow/src/Structurizer.cpp) (shader recompiler) - a loop whose single latch holds instructions and is reached by one edge gets an empty continue block, so its continue construct is that one block. A latch that several `continue` paths share keeps its instructions in the continue construct, since isolating it would clone them into each path. When such a latch emits more than one SPIR-V block, as a memory or image access does, SPIRV-Cross (MoltenVK) folds the construct into a `for` increment and drops the header's `OpPhi` updates, so the loop never advances on Metal. No title is known to reach this case.
 - [Scalar buffer descriptors](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) - the PS5's gfx1013 is not measured, nor is a V# with a null base. Scalar loads use only the base address, stride and record count: the fourth word is normalized to zero for resources accessed exclusively by scalar loads, and a [V# read at run time](../../../core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp) does not fault on its type for a scalar load. Measured on RDNA2 (gfx1036, AMD Windows driver 32.0.21030.31) with `s_buffer_load_dword` and `_dwordx4` through the [hardware oracle](../HW_ORACLE.md): every `OOB_SELECT`, `TYPE`, `FORMAT` 0, `ADD_TID_ENABLE`, `INDEX_STRIDE`, a zero and an all-ones fourth word read as 0x31016fac does; the range is `NUM_RECORDS` bytes with stride 0, else `STRIDE` × `NUM_RECORDS`, whatever `SWIZZLE_ENABLE`; a dword not wholly in range reads 0. Vector and mixed accesses retain descriptor validation.
-- [Spanned wave64 compute](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp) (shader recompiler) - `APS5_SPANNED_WAVE=1`, off by default, runs a single-wave wave64 compute program at one lane per invocation on a 32-lane host subgroup and exchanges ballots, shuffles and reductions between the two host subgroups through workgroup memory and barriers. The driver turns it on only where compute subgroups can be pinned at 32 full lanes (`subgroupSizeControl` and `computeFullSubgroups`), passes it as `SpirvTarget::spannedWaves` (part of the cache key and the serialized request), and creates the pipelines of spanned programs with `REQUIRE_FULL_SUBGROUPS`, since the lane number `SubgroupId * 32 + SubgroupLocalInvocationId` needs two full subgroups. Those barriers are correct only where every invocation of the workgroup reaches them, which holds for the guest's wave-uniform control flow. The emulated 64-bit LDS atomics ([LockedLdsUpdate](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp)) run only on the lanes the exec mask keeps, so their lock loop ballots within its own host subgroup, as its elect already does. Any other cross-lane exchange emitted inside per-lane control flow would hang a spanned wave in the same way, and the emitter does not reject one: there is no uniformity check. The only check so far is manual: disassemble the module (`spirv-dis`), list the selection and loop constructs that enclose each `OpControlBarrier`, and read the condition of each enclosing selection. In Ghost of Yōtei's program 0x80003af600 (PPSA26344), 108 of 111 barriers sat only in loops on wave-uniform conditions and 3 inside per-invocation conditionals, all in the lock loop. Other programs and titles have not been scanned. An earlier, cheaper ballot (one barrier instead of three) also lost the device for a reason that is not understood, which may be another barrier placement problem. Measured on an RTX 4090 (Windows): an offline replay of that program and two Yōtei runs that previously lost the device (NVIDIA TDR) now run past it, and [LdsMaskWaterfall.cpp](../../../core/libs/prx/libSceAgcDriver/tests/execution/LdsMaskWaterfall.cpp) runs both modes and covers a half-inactive wave. Stray (PPSA02101) on Intel ANV and a Demon's Souls (PPSA01342) request replay showed no fault, before this fix
+- [Spanned wave64 compute](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp) (shader recompiler) - a spanned wave exchanges ballots, shuffles and reductions between its two host subgroups through workgroup barriers, which hang the device when not every invocation reaches them. [SpannedBarrierInDivergentFlow](../../../core/shader/recompiler/SpirvBackend/src/SpirvSpannedUniformity.cpp) sends a program back to two lanes per invocation when a barrier sits in control flow that a per-invocation value decides. It counts a load at a uniform address as uniform only from memory the program never writes, or from workgroup memory a barrier fences on every path since the last write; writes by other dispatches or queues are not considered. An earlier one-barrier ballot also lost the device for a reason that is not understood. `APS5_NO_SPANNED_WAVE` turns spanning off.
 - [Guest memory load guards](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - a guest load whose lane is inactive or whose address is out of bounds is still performed and its result replaced by 0 with `OpSelect`, instead of a selection per guard, when the load can be pointed at a location nothing writes: scratch reads its own element 0; storage buffers (direct buffers, GDS) read element 0xfffffffe, which `robustBufferAccess2` turns into a zero load with no memory access, so the extra read cannot race with a write the guarded program would not have raced with ([Vulkan memory model](https://docs.vulkan.org/spec/latest/appendices/memorymodel.html#data-race)). Without `robustBufferAccess2` ([VulkanDevice.cpp](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) enables it when the device has it) those loads keep their selections; LDS loads, stores, atomics, formatted and physical-address (BDA) accesses always do. LDS keeps the branch because NVIDIA's compiler predicates a branched LDS load at no cost, while the select form costs about 2.5 instructions per dword and splits a `ds_read_b128` into four loads (RTX 4060 measurements by @karanvora2599 in #4048); Intel's static counts favour the select only for wide reads, and run time was not measured on either. The aligned high word of a misaligned-capable buffer read re-reads the low word's own element. A selection guard costs four blocks per lane half and the host compiler's time grows with the block count: PPSA11142's merged PSP vertex program (3176 ISA words, wave64) emitted 16,983 blocks and 1,710 conditional branches, 5,907 and 868 with the selects. `agc_guarded_memory_loads` traces the emitted loads in both device modes. When the device also reports `robustStorageBufferAccessSizeAlignment` 1, a direct buffer load leaves out the comparison with the descriptor's `OpArrayLength` (which reads the descriptor) and the device returns the zero past the descriptor range: the host rounds a guest buffer's or a draw snapshot's descriptor range of 4 bytes or more down to whole dwords, which no direct access reaches past, and the misaligned-base high word keeps its comparison with the length. The guest bound and the guards of stores, atomics, GDS, LDS and BDA are unchanged. The flag is part of the shader cache key and `APS5_NO_EXACT_ROBUST_LOADS` (any value) keeps the comparison. Hardware treatment of a 32-bit load that straddles the end of a 1 to 3 byte range is not measured.
 - [Merged hull programs](../../../core/shader/recompiler/Optimization/src/TessellationLowering.cpp) (registered binary type 3) run whole in the hull stage, one invocation per output control point. On the console hull lane i keeps local lane i's registers, which with fewer input than output control points stop matching the patch's control points from i = inputs on; here invocation i holds control point min(i, inputs - 1) of its own patch. A wave-mask branch (`s_cbranch_exec[n]z`, `s_cbranch_vcc[n]z`) tests the invocation's own lane, so it can skip a block, scalar instructions included, that the console runs for the wave; `v_readfirstlane_b32` returns the invocation's own value, not the first active lane's, and another lane's register throws. The register layout (`s3`, `v0`-`v5`, as for the separate halves) is not measured on hardware. s[0:1] hold the hull user-data address (SH 0x102/0x103) when a draw sets it and zero otherwise, so a program reading it unset faults at the null address.
 - [SPIR-V instruction length and deferred phi validation](../../../core/shader/recompiler/SpirvBackend/src/SpirvModule.cpp) (shader recompiler) - SPIR-V instruction word count boundaries and deferred phi patch tokens validate bounds against logical layout rules; console GPU driver responses to oversized instructions or unlinked phi tokens remain unmeasured. Instruction emission enforces the 65,535 word limit (16-bit word count field), and deferred phi patching verifies token registration.