diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/VectorWindowNarrowing.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/VectorWindowNarrowing.cpp new file mode 100644 index 0000000000..73b15eb93a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/VectorWindowNarrowing.cpp @@ -0,0 +1,120 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t MaxThreads = 64; +constexpr std::uint32_t Results = 16; +constexpr std::uint32_t Checked = 6; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020086, 0x34060088, 0x4a140680, 0x4a160681, 0x4a180682, 0x4a1a0683, 0x4a1c0684, 0x4a1e0685, + 0x4a200686, 0x4a220687, 0x4a240688, 0x4a260689, 0x4a28068a, 0x4a2a068b, 0x4a2c068c, 0x4a2e068d, + 0x4a30068e, 0x4a32068f, 0x4a340690, 0x4a360691, 0x4a380692, 0x4a3a0693, 0x4a3c0694, 0x4a3e0695, + 0x4a400696, 0x4a420697, 0x4a440698, 0x4a460699, 0x4a48069a, 0x4a4a069b, 0x4a4c069c, 0x4a4e069d, + 0x4a50069e, 0x4a52069f, 0x877c8308, 0x7e08870a, 0xbefc0309, 0x7e0a870a, 0x7e148504, 0x877c8308, + 0x7e148505, 0xe0701000, 0x80010401, 0xe0701004, 0x80010501, 0xe0701008, 0x80010c01, 0xe070100c, + 0x80010d01, 0xe0701010, 0x80011e01, 0xe0701014, 0x80012901, 0xbf810000, +}; + +constexpr const char* Names[Checked] = {"v4", "v5", "v12", "v13", "v30", "v41"}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +std::uint32_t FunctionVariables(std::span words) { + std::uint32_t count = 0; + for (std::size_t at = 5; at < words.size();) { + const auto length = words[at] >> 16u; + if (length == 0u) break; + if ((words[at] & 0xffffu) == spv::OpVariable && at + 3u < words.size() && words[at + 3u] == spv::StorageClassFunction) ++count; + at += length; + } + return count; +} + +void Run(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, std::uint32_t threads, std::uint32_t bounded, std::uint32_t data) { + Output.fill(0xdeadbeefu); + std::vector userData(10, 0u); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(output.begin(), output.end(), userData.begin() + 4); + userData[8] = bounded; + userData[9] = data; + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + Require(FunctionVariables(result.spirv.Words()) > 8u, "vector window narrowing: wave" + std::to_string(waveSize) + " window was not split into separate slot variables"); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(std::uint32_t waveSize, std::uint32_t threads, std::uint32_t bounded, std::uint32_t data) { + for (std::uint32_t tid = 0; tid < threads; ++tid) { + std::array window{}; + for (std::uint32_t k = 0; k < window.size(); ++k) window[k] = (tid << 8u) + k; + const std::uint32_t v4 = window[bounded & 3u]; + const std::uint32_t v5 = window[data]; + window[data] = v4; + window[bounded & 3u] = v5; + const std::array expected{v4, v5, window[2], window[3], window[20], window[31]}; + const std::uint32_t* out = &Output[tid * Results]; + for (std::uint32_t index = 0; index < Checked; ++index) { + Require(out[index] == expected[index], "vector window narrowing: wave" + std::to_string(waveSize) + " data index " + std::to_string(data) + " lane " + std::to_string(tid) + " " + Names[index] + " is " + Hex(out[index]) + ", expected " + Hex(expected[index])); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (device->Target().subgroupSize < 32u) { + std::printf("skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize); + return VulkanTestSkipped; + } + for (const std::uint32_t data : {20u, 2u, 31u}) { + Run(*device, 32, 32, 7u, data); + Check(32, 32, 7u, data); + Run(*device, 64, 64, 7u, data); + Check(64, 64, 7u, data); + } + std::puts("vector window narrowing tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index f5d6a95d9e..30676e608e 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -153,6 +153,8 @@ struct SpirvEmitterState { std::uint32_t numWorkgroupsVariable = 0; std::array scratchVariable {}; std::array vectorWindowVariable {}; + std::uint32_t vectorWindowIndexed = 0; + std::array, 2> vectorWindowSlots {}; std::array imageVariables {}; std::uint32_t samplerVariable = 0; std::uint32_t mainFunc = 0; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterRegisters.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterRegisters.cpp index f24270d401..f38c0fe047 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterRegisters.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterRegisters.cpp @@ -1,7 +1,10 @@ #include "SpirvBackend/SpirvAluEmitter.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include "SpirvBackend/SpirvEmitterInstructions.hpp" +#include +#include #include +#include namespace ShaderRecompiler { @@ -45,21 +48,89 @@ void EmitSetVectorRegister(SpirvValueEmitContext& ctx, const IrValue& inst) { EmitUnreachable(ctx, inst); } -std::uint32_t VectorWindowElement(SpirvValueEmitContext& ctx, const IrValue& inst) { - auto& state = ctx.state; +namespace { + +std::uint32_t VectorWindowElement(SpirvEmitterState& state, std::uint32_t index) { const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, TypePointer(state, spv::StorageClassFunction, TypeU32(state)), pointer, state.vectorWindowVariable.at(state.laneHalf), ctx.Arg(inst, 0u)); + state.module.AddFunction(spv::OpAccessChain, TypePointer(state, spv::StorageClassFunction, TypeU32(state)), pointer, state.vectorWindowVariable.at(state.laneHalf), index); return pointer; } +std::uint32_t AccessNarrowedWindow(SpirvEmitterState& state, std::uint32_t index, const std::function& access) { + const auto& slots = state.vectorWindowSlots.at(state.laneHalf); + const auto arrayLabel = state.module.AllocateId(); + const auto slotsLabel = state.module.AllocateId(); + const auto slotsMerge = state.module.AllocateId(); + const auto merge = state.module.AllocateId(); + const auto inArray = Binary(state, spv::OpULessThan, TypeBool(state), index, ConstantU32(state, state.vectorWindowIndexed)); + state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, inArray, arrayLabel, slotsLabel); + EmitLabel(state, arrayLabel); + const auto fromArray = access(VectorWindowElement(state, index)); + const auto arrayExit = state.currentLabel; + state.module.AddFunction(spv::OpBranch, merge); + EmitLabel(state, slotsLabel); + std::vector labels(slots.size()); + std::vector words{spv::OpSwitch, index, slotsMerge}; + for (std::size_t slot = 0; slot < slots.size(); ++slot) { + labels[slot] = state.module.AllocateId(); + words.push_back(state.vectorWindowIndexed + static_cast(slot)); + words.push_back(labels[slot]); + } + state.module.AddFunction(spv::OpSelectionMerge, slotsMerge, spv::SelectionControlMaskNone); + state.module.AddFunction(std::span(words)); + std::vector incoming; + for (std::size_t slot = 0; slot < slots.size(); ++slot) { + EmitLabel(state, labels[slot]); + incoming.push_back(access(slots[slot])); + incoming.push_back(labels[slot]); + state.module.AddFunction(spv::OpBranch, slotsMerge); + } + incoming.push_back(ConstantU32(state, 0u)); + incoming.push_back(slotsLabel); + EmitLabel(state, slotsMerge); + const auto fromSlots = state.module.AllocateId(); + std::vector phi{spv::OpPhi, TypeU32(state), fromSlots}; + phi.insert(phi.end(), incoming.begin(), incoming.end()); + state.module.AddFunction(std::span(phi)); + state.module.AddFunction(spv::OpBranch, merge); + EmitLabel(state, merge); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpPhi, TypeU32(state), value, fromArray, arrayExit, fromSlots, slotsMerge); + return value; +} + +std::uint32_t DirectWindowElement(SpirvValueEmitContext& ctx, const IrValue& inst) { + auto& state = ctx.state; + const IrValue* index = inst.Argument(0)->Resolve(); + if (state.vectorWindowIndexed == 0u) return VectorWindowElement(state, ctx.Arg(inst, 0u)); + if (!index->HasImmediate()) return 0u; + if (index->ImmediateU32() < state.vectorWindowIndexed) return VectorWindowElement(state, ctx.Arg(inst, 0u)); + return state.vectorWindowSlots.at(state.laneHalf).at(index->ImmediateU32() - state.vectorWindowIndexed); +} + +} + void EmitLoadVectorWindow(SpirvValueEmitContext& ctx, const IrValue& inst) { - const auto value = ctx.state.module.AllocateId(); - ctx.state.module.AddFunction(spv::OpLoad, TypeU32(ctx.state), value, VectorWindowElement(ctx, inst)); - ctx.Define(inst, value); + auto& state = ctx.state; + const auto load = [&state](std::uint32_t pointer) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); + return value; + }; + const auto pointer = DirectWindowElement(ctx, inst); + ctx.Define(inst, pointer != 0u ? load(pointer) : AccessNarrowedWindow(state, ctx.Arg(inst, 0u), load)); } void EmitStoreVectorWindow(SpirvValueEmitContext& ctx, const IrValue& inst) { - ctx.state.module.AddFunction(spv::OpStore, VectorWindowElement(ctx, inst), ctx.Arg(inst, 1u)); + auto& state = ctx.state; + const auto value = ctx.Arg(inst, 1u); + const auto store = [&state, value](std::uint32_t pointer) { + state.module.AddFunction(spv::OpStore, pointer, value); + return ConstantU32(state, 0u); + }; + if (const auto pointer = DirectWindowElement(ctx, inst); pointer != 0u) (void)store(pointer); + else (void)AccessNarrowedWindow(state, ctx.Arg(inst, 0u), store); } void EmitGetGotoVariable(SpirvValueEmitContext& ctx, const IrValue& inst) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp index f4c1da3f2b..641413c954 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp @@ -5,6 +5,11 @@ #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include #include +#include +#include +#include +#include +#include #include #include @@ -41,6 +46,170 @@ std::uint32_t DefineInterfaceVariable(SpirvEmitterState& state, std::uint32_t ty return variable; } +namespace { + +class WindowIndexBounds { +public: + std::optional Of(const IrValue& input) { + const IrValue* value = input.Resolve(); + if (value->HasImmediate()) return value->ImmediateU32(); + if (const auto found = memo.find(value); found != memo.end()) return found->second; + if (!active.insert(value).second || active.size() > 256u) return std::nullopt; + const auto bound = compute(*value); + active.erase(value); + memo.emplace(value, bound); + return bound; + } + +private: + using Bound = std::optional; + + Bound maxOf(std::initializer_list bounds) { + std::uint64_t result = 0; + for (const auto& bound : bounds) { + if (!bound) return std::nullopt; + result = std::max(result, *bound); + } + return result; + } + + static std::vector reachingStores(const IrValue& load, std::uint32_t slot) { + std::vector values; + std::unordered_set visited; + std::vector> pending {{load.Parent(), &load}}; + while (!pending.empty()) { + const auto [block, before] = pending.back(); + pending.pop_back(); + const auto& instructions = block->Instructions(); + auto it = instructions.rbegin(); + if (before != nullptr) it = std::next(std::find(instructions.rbegin(), instructions.rend(), before)); + bool stored = false; + for (; it != instructions.rend() && !stored; ++it) { + if ((*it)->Opcode() != IrOpcode::StoreVectorWindow) continue; + const IrValue* index = (*it)->Argument(0)->Resolve(); + if (!index->HasImmediate() || index->ImmediateU32() != slot) continue; + values.push_back((*it)->Argument(1)); + stored = true; + } + if (stored) continue; + for (IrBlock* predecessor : block->Predecessors()) { + if (visited.insert(predecessor).second) pending.emplace_back(predecessor, nullptr); + } + } + return values; + } + + Bound compute(const IrValue& value) { + const auto arg = [&](std::size_t index) { return Of(*value.Argument(index)); }; + const auto immediate = [&](std::size_t index) -> std::optional { + const IrValue* operand = value.Argument(index)->Resolve(); + if (!operand->HasImmediate()) return std::nullopt; + return operand->ImmediateU32(); + }; + const auto fits = [](std::uint64_t bound) -> Bound { return bound <= 0xffffffffull ? Bound(bound) : std::nullopt; }; + switch (value.Opcode()) { + case IrOpcode::BitFieldUExtract: { + const auto count = immediate(2); + if (count && *count < 32u) return (std::uint64_t {1} << *count) - 1u; + return arg(0); + } + case IrOpcode::BitwiseAnd32: { + const auto lhs = arg(0); + const auto rhs = arg(1); + if (lhs && rhs) return std::min(*lhs, *rhs); + return lhs ? lhs : rhs; + } + case IrOpcode::UMin32: { + const auto lhs = arg(0); + const auto rhs = arg(1); + if (!lhs || !rhs) return std::nullopt; + return std::min(*lhs, *rhs); + } + case IrOpcode::UMax32: + return maxOf({arg(0), arg(1)}); + case IrOpcode::BitwiseOr32: { + const auto both = maxOf({arg(0), arg(1)}); + if (!both) return std::nullopt; + std::uint64_t mask = 0; + while (mask < *both) mask = mask << 1u | 1u; + return mask; + } + case IrOpcode::IAdd32: { + const auto lhs = arg(0); + const auto rhs = arg(1); + if (!lhs || !rhs) return std::nullopt; + return fits(*lhs + *rhs); + } + case IrOpcode::ShiftRightLogical32: { + const auto shift = immediate(1); + const auto bound = arg(0); + if (!bound) return std::nullopt; + return shift ? *bound >> (*shift & 31u) : *bound; + } + case IrOpcode::ShiftLeftLogical32: { + const auto shift = immediate(1); + const auto bound = arg(0); + if (!bound || !shift) return std::nullopt; + return fits(*bound << (*shift & 31u)); + } + case IrOpcode::ConvertU32U16: + return 0xffffu; + case IrOpcode::Select: + case IrOpcode::SelectU32: + { + if (value.Argument(0)->Resolve()->Opcode() == IrOpcode::GetExec || value.Argument(2)->Resolve()->Opcode() == IrOpcode::LoadVectorWindow) return arg(1); + const auto lhs = arg(1); + const auto rhs = arg(2); + if (lhs && rhs) return std::max(*lhs, *rhs); + return lhs ? lhs : rhs; + } + case IrOpcode::ReadFirstLane: + return arg(0); + case IrOpcode::Identity: + case IrOpcode::Phi: { + Bound bound; + for (std::size_t index = 0; index < value.ArgumentCount(); ++index) { + if (const auto incoming = arg(index)) bound = std::max(bound.value_or(0u), *incoming); + } + return bound; + } + case IrOpcode::LoadVectorWindow: { + const IrValue* index = value.Argument(0)->Resolve(); + if (!index->HasImmediate()) return std::nullopt; + Bound bound = 0u; + for (const IrValue* stored : reachingStores(value, index->ImmediateU32())) { + bound = maxOf({bound, Of(*stored)}); + if (!bound) return std::nullopt; + } + return bound; + } + default: + return std::nullopt; + } + } + + std::unordered_map memo; + std::unordered_set active; +}; + +std::uint32_t IndexedWindowSlots(const IrProgram& program) { + const auto window = program.Metadata().vectorWindowSize; + WindowIndexBounds bounds; + std::uint64_t reached = 0; + for (const auto& block : program.Blocks()) { + for (const IrValue* inst : block->Instructions()) { + if (inst->Opcode() != IrOpcode::LoadVectorWindow && inst->Opcode() != IrOpcode::StoreVectorWindow) continue; + const IrValue* index = inst->Argument(0)->Resolve(); + if (index->HasImmediate()) continue; + const auto bound = bounds.Of(*index); + if (bound && *bound < window / 2u) reached = std::max(reached, *bound + 1u); + } + } + return static_cast(reached); +} + +} + void DefineModule(SpirvEmitterState& state) { state.interfaceVariables.reserve(state.program.Info().inputs.size() + state.program.Info().outputs.size()); DefineInputs(state); @@ -55,9 +224,14 @@ void DefineModule(SpirvEmitterState& state) { state.scratchVariable.at(half) = state.module.AllocateId(); } } - if (state.program.Metadata().vectorWindowSize != 0u) { + if (const auto window = state.program.Metadata().vectorWindowSize; window != 0u) { + static const bool narrowWindow = std::getenv("APS5_NO_WINDOW_NARROWING") == nullptr; + state.vectorWindowIndexed = narrowWindow ? IndexedWindowSlots(state.program) : 0u; for (std::uint32_t half = 0; half < state.laneCount; half++) { state.vectorWindowVariable.at(half) = state.module.AllocateId(); + if (state.vectorWindowIndexed == 0u) continue; + auto& slots = state.vectorWindowSlots.at(half); + for (std::uint32_t slot = state.vectorWindowIndexed; slot < window; slot++) slots.push_back(state.module.AllocateId()); } } state.mainFunc = state.module.AllocateId(); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp index 410f5b1580..752082de85 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp @@ -1053,10 +1053,14 @@ void EmitProgram(SpirvEmitterState& state) { } } if (const auto window = state.program.Metadata().vectorWindowSize; window != 0u) { - const auto array = state.module.Type(spv::OpTypeArray, TypeU32(state), ConstantU32(state, window)); + const auto indexed = state.vectorWindowIndexed != 0u ? state.vectorWindowIndexed : window; + const auto array = state.module.Type(spv::OpTypeArray, TypeU32(state), ConstantU32(state, indexed)); const auto zero = state.module.Constant(spv::OpConstantNull, array); for (std::uint32_t half = 0; half < state.laneCount; half++) { state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, array), state.vectorWindowVariable.at(half), spv::StorageClassFunction, zero); + for (const auto slot : state.vectorWindowSlots.at(half)) { + state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, TypeU32(state)), slot, spv::StorageClassFunction, ConstantU32(state, 0u)); + } } } if (state.pixelValidMaskVariable != 0u) {