diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index 9a4a6e63eb..8567f73b0f 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -29,9 +29,11 @@ #ifdef __GLIBC__ #include #endif +#include #include #include #include +#include "Optimization/ShaderStageInputInfo.hpp" #include "RdnaDecoder/RdnaInstructionDecoder.hpp" namespace AgcDriver::DriverDetail { @@ -538,14 +540,48 @@ std::vector ReadUserDataCallee(std::uint64_t address, std::uint32 throw std::runtime_error("its callee at " + hexText(address) + " does not return within " + std::to_string(limitWords * 4u) + " bytes"); } +std::vector ReadGuestBytes(std::uint64_t address, std::uint64_t bytes, const char* what) { + if (address > UINTPTR_MAX - bytes || !GuestMemory::Accessible(reinterpret_cast(static_cast(address)), static_cast(bytes))) { + throw std::runtime_error(std::string("its ") + what + " at " + hexText(address) + " + " + hexText(bytes) + " is not readable"); + } + std::vector data(static_cast(bytes)); + GuestMemory::Read(address, std::as_writable_bytes(std::span(data))); + return data; +} + +std::vector ReadTableCallees(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::TableCall& call) { + constexpr std::uint64_t limitBytes = 64u << 20u; + constexpr std::size_t limitCallees = 64u; + const auto slot = call.userDataRegister - request.context.userDataBaseRegister; + const auto pointer = static_cast(request.context.userData[slot]) | static_cast(request.context.userData[slot + 1u]) << 32u; + const auto words = ReadGuestBytes(pointer + call.descriptorOffset, 16u, "call table descriptor"); + ShaderRecompiler::ShaderBufferResource descriptor{}; + std::memcpy(descriptor.fields.data(), words.data(), words.size()); + const auto bytes = descriptor.GetSize(); + if (bytes > limitBytes) throw std::runtime_error("its call table of " + std::to_string(bytes) + " bytes is larger than the " + std::to_string(limitBytes) + " bytes that are searched for callees"); + const auto table = bytes == 0u ? std::vector{} : ReadGuestBytes(descriptor.Base48(), bytes, "call table"); + std::set targets; + for (std::uint64_t offset = call.entryOffset; offset + 8u <= bytes; offset += call.entryStride) { + std::uint64_t target = 0; + std::memcpy(&target, table.data() + offset, sizeof(target)); + if (target != 0u) targets.insert(target); + if (call.entryStride == 0u) break; + } + if (targets.size() > limitCallees) throw std::runtime_error("its call table holds " + std::to_string(targets.size()) + " distinct targets; at most " + std::to_string(limitCallees) + " are linked"); + std::vector callees; + for (const auto target : targets) callees.push_back({target, ReadUserDataCallee(target, call.linkRegister)}); + return callees; +} + std::vector LinkUserDataCallees(const ShaderRecompiler::RecompileRequest& request) { if (request.shader.stage != ShaderRecompiler::ShaderStage::Compute) return {}; const auto code = request.shader.code; if (std::none_of(code.begin(), code.end(), [](std::uint32_t word) { return (word & 0xff80ff00u) == 0xbe802100u; })) return {}; const auto program = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code); - const ShaderRecompiler::SwappcInfo swappc{false, request.context.userDataBaseRegister, static_cast(request.context.userData.size())}; + const ShaderRecompiler::SwappcInfo swappc{false, request.context.userDataBaseRegister, static_cast(request.context.userData.size()), request.context.waveSize}; const auto calls = ShaderRecompiler::FindUserDataCalls(program, swappc); - if (calls.empty()) return {}; + const auto tableCalls = ShaderRecompiler::FindTableCalls(program, swappc); + if (calls.empty() && tableCalls.empty()) return {}; std::vector> callees; for (const auto& call : calls) { const auto slot = call.userDataRegister - swappc.userDataBaseRegister; @@ -557,7 +593,15 @@ std::vector LinkUserDataCallees(const ShaderRecompiler::Recompile std::to_string(call.userDataRegister + 1u) + "] at program counter " + hexText(call.programCounter) + ", but " + error.what()); } } - return ShaderRecompiler::LinkUserDataCalls(program, calls, callees); + std::vector> tableCallees; + for (const auto& call : tableCalls) { + try { + tableCallees.push_back(ReadTableCallees(request, call)); + } catch (const std::exception& error) { + throw std::runtime_error("AGC driver: compute shader " + hexText(request.shader.codeAddress) + " calls a target loaded from a table at program counter " + hexText(call.programCounter) + ", but " + error.what()); + } + } + return ShaderRecompiler::LinkCalls(program, calls, callees, tableCalls, tableCallees); } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp index f4f4750f14..148b5e4f55 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp @@ -275,6 +275,10 @@ void BdaResources::CheckFault() const { throw std::runtime_error(message.str()); } std::ostringstream message; + if (report.reason == ShaderRecompiler::BdaAbi::FaultReason::UnlinkedCall) { + message << "shader stage " << report.stage << " called address 0x" << std::hex << report.address << " at program counter 0x" << report.instruction << ", which no linked callee of that call matches"; + throw std::runtime_error(message.str()); + } message << "BDA access failed: address=0x" << std::hex << report.address << " instruction=0x" << report.instruction << std::dec << " bytes=" << report.bytes << " stage=" << report.stage << " reason=" << static_cast(report.reason); if (report.reason == ShaderRecompiler::BdaAbi::FaultReason::Permission && table != nullptr) { const auto bytes = table->Bytes(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp index c4fb6eba1f..ece322e968 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp @@ -611,8 +611,8 @@ void DeferredRegistration() { commands.insert(commands.end(), {0xc0031500u, 1, 1, 1, 0x8041}); Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; sceAgcDriverSubmitAcb(0x20, &packet); - ExpectFailure([] { AgcDriverWaitIdle_nid_postfix(); }, "not statically resolvable"); - ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "not statically resolvable"); + AgcDriverWaitIdle_nid_postfix(); + AgcDriverShutdown_nid_postfix(); } void DeferredUndecodableRegistration() { diff --git a/core/libs/tests/ScalarCall.cpp b/core/libs/tests/ScalarCall.cpp index e797b7aeba..39aba26f5a 100644 --- a/core/libs/tests/ScalarCall.cpp +++ b/core/libs/tests/ScalarCall.cpp @@ -117,7 +117,7 @@ static void CheckUserDataCalls() { Require(calls.size() == 2u); Require(calls[0].programCounter == 0x10u && calls[0].linkRegister == 14u && calls[0].userDataRegister == 0u); Require(calls[1].programCounter == 0x1cu && calls[1].linkRegister == 14u && calls[1].userDataRegister == 2u); - ExpectThrow("not statically resolvable", [&] { static_cast(GraphBuilder{}.Build(program, &swappc)); }); + Require(GraphBuilder{}.Build(program, &swappc).unlinkedCallProgramCounters == std::vector{0x10u, 0x1cu}); auto overwritten = caller; overwritten[1] = 0xbe800380u; @@ -151,10 +151,8 @@ static void CheckUserDataCalls() { Require(std::equal(addFive.begin(), addFive.end(), linked.begin() + 12) && std::equal(addSixtyFour.begin(), addSixtyFour.end(), linked.begin() + 14)); auto readsPc = caller; readsPc[1] = 0xbe941f00u; - ExpectThrow("cannot have user-data callees linked", [&] { - const auto readsPcProgram = RdnaInstructionDecoder{}.Decode(readsPc); - static_cast(LinkUserDataCalls(readsPcProgram, FindUserDataCalls(readsPcProgram, swappc), callees)); - }); + const auto readsPcProgram = RdnaInstructionDecoder{}.Decode(readsPc); + Require(LinkUserDataCalls(readsPcProgram, FindUserDataCalls(readsPcProgram, swappc), callees).size() == 17u); const auto linkedProgram = RdnaInstructionDecoder{}.Decode(linked); const auto cfg = GraphBuilder{}.Build(linkedProgram, &swappc); @@ -181,6 +179,92 @@ static void CheckUserDataCalls() { ExpectThrow("is shared with another call", [&] { static_cast(GraphBuilder{}.Build(RdnaInstructionDecoder{}.Decode(sameCopy), &swappc)); }); } +static std::vector TableCaller() { + return {0x8f148414u, 0xf4080406u, 0xfa000270u, 0xbf8cc07fu, 0xf4280208u, 0x28000000u, 0xbf8cc07fu, 0xbe8e0308u, 0xbe8f0309u, 0xbe8e210eu, 0xbf810000u}; +} + +static ControlFlowGraph BuildCode(std::span code, const SwappcInfo& swappc) { + return GraphBuilder{}.Build(RdnaInstructionDecoder{}.Decode(code), &swappc); +} + +static IrProgram TranslateCode(std::span code, const SwappcInfo& swappc) { + const auto program = RdnaInstructionDecoder{}.Decode(code); + const auto cfg = GraphBuilder{}.Build(program, &swappc); + ShaderComputeInputInfo computeInfo{}; + TranslateOptions options{}; + options.stage = ShaderStageKind::Compute; + options.inputInfo.compute = &computeInfo; + options.userDataCount = swappc.userDataCount; + return InstructionTranslator{}.Translate(program, cfg, options); +} + +static void CheckTableCalls() { + const SwappcInfo swappc{false, 0u, 16u}; + const auto caller = TableCaller(); + const auto program = RdnaInstructionDecoder{}.Decode(caller); + const auto calls = FindTableCalls(program, swappc); + Require(calls.size() == 1u); + Require(calls[0].programCounter == 0x24u && calls[0].linkRegister == 14u && calls[0].targetRegister == 14u && calls[0].userDataRegister == 12u); + Require(calls[0].descriptorOffset == 0x270u && calls[0].entryOffset == 0u && calls[0].entryStride == 16u); + Require(FindUserDataCalls(program, swappc).empty()); + const auto table = [&](std::size_t word, std::uint32_t value, std::uint32_t userDataCount = 16u) { + auto code = caller; + code[word] = value; + return FindTableCalls(RdnaInstructionDecoder{}.Decode(code), SwappcInfo{false, 0u, userDataCount}); + }; + Require(table(0u, 0x8f148114u).empty()); + Require(table(2u, 0x28000270u).empty()); + Require(table(8u, 0xbe8f030au).empty()); + Require(table(10u, 0xbf810000u, 12u).empty()); + Require(table(5u, 0x28000008u).size() == 1u && table(5u, 0x28000008u)[0].entryOffset == 8u); + Require(table(0u, 0xbe940380u).empty()); + auto readsLane = caller; + readsLane.insert(readsLane.begin(), {0xd4c2000bu, 0x00020300u}); + const auto readsLaneProgram = RdnaInstructionDecoder{}.Decode(readsLane); + Require(FindTableCalls(readsLaneProgram, SwappcInfo{false, 0u, 16u, 32u}).size() == 1u); + Require(FindTableCalls(readsLaneProgram, SwappcInfo{false, 0u, 16u, 64u}).empty()); + + Require(BuildCode(caller, swappc).unlinkedCallProgramCounters == std::vector{0x24u}); + const auto unlinked = TranslateCode(caller, swappc); + Require(unlinked.Info().usesFaultBuffer && ProgramToString(unlinked).find("UnlinkedCall") != std::string::npos); + + const std::vector first{0xbf068000u, 0x4a020285u, 0xbe80200eu}; + const std::vector second{0x4a0202c0u, 0xbf068001u, 0xbe80200eu}; + const std::vector> callees{{{0x1122334455ull, first}, {0x66778899aaull, second}}}; + const auto linked = LinkCalls(program, {}, {}, calls, callees); + Require(linked.size() == 59u && linked[9] == 0xbf820001u && linked[11] == 0xbf850010u && linked[58] == 0xbf9f0000u); + Require(linked[12] == 0xbf07ff0eu && linked[13] == 0x22334455u && linked[14] == 0xbf850004u && linked[15] == 0xbf07ff0fu && linked[16] == 0x11u && linked[17] == 0xbf850001u && linked[18] == 0xbf82001bu); + Require(linked[20] == 0x778899aau && linked[26] == 0xbf078080u && linked[27] == 0xbf820016u && linked[34] == 0xbf068080u && linked[35] == 0xbf82000au && linked[44] == 0xbf068080u && linked[45] == 0xbf820004u); + Require(linked[46] == 0xbb0e0005u && linked[47] == 0xbf82ffdau && linked[48] == 0xbb0e0006u && linked[49] == 0xbf82ffd8u && linked[50] == 0xbe8e210eu && linked[51] == 0xbf82ffd6u); + Require(std::equal(first.begin(), first.end(), linked.begin() + 52) && std::equal(second.begin(), second.end(), linked.begin() + 55)); + const auto cfg = BuildCode(linked, swappc); + const auto block = [&](std::uint32_t word) { return cfg.FindBlockByProgramCounter(word * 4u).id; }; + Require(cfg.FindBlockByProgramCounter(46u * 4u).terminator.trueBlock == block(52u)); + Require(cfg.FindBlockByProgramCounter(52u * 4u).terminator.trueBlock == block(47u)); + Require(cfg.FindBlockByProgramCounter(48u * 4u).terminator.trueBlock == block(55u)); + Require(cfg.FindBlockByProgramCounter(55u * 4u).terminator.trueBlock == block(49u)); + Require(cfg.unlinkedCallProgramCounters == std::vector{50u * 4u}); + static_cast(TranslateCode(linked, swappc)); + + std::vector large(0x8000u, 0xbf800000u); + const std::vector> far{{{0x10ull, large}, {0x20ull, first}}}; + ExpectThrow("out of s_call_b64 range", [&] { static_cast(LinkCalls(program, {}, {}, calls, far)); }); + + const std::vector jumpTable{ + 0xbe8c1f00u, 0x800cff0cu, 0x00000058u, 0x820d800du, 0x83928202u, 0xd4c40016u, 0x02000500u, 0x8f128212u, 0xf4000386u, 0x24000000u, + 0xbf8cc07fu, 0x7d820003u, 0x808c0e0cu, 0x828d800du, 0xbefd210cu, 0x7e020281u, 0xbf820003u, 0x7e020282u, 0xbf820001u, 0x7e020283u, + 0xe0700000u, 0x80000100u, 0xbf810000u, 0x00000020u, 0x00000018u, 0x00000010u}; + ExpectThrow("jumps through a table in its code", [&] { static_cast(LinkCalls(RdnaInstructionDecoder{}.Decode(jumpTable), {}, {}, {}, {})); }); + const std::array compare{0xbf068008u}; + const auto comparesLink = Build({DecodeRdnaSopc(0u, compare, 0u), Call(4u, 8u, 1u), Sopp(8u, 1u), Sop1(12u, 0u, 0x20u, 8u)}); + Require(comparesLink.FindBlockByProgramCounter(0u).terminator.trueBlock == comparesLink.FindBlockByProgramCounter(12u).id); + const auto readsFirst = Build({Sop1(0u, 0u, 3u, 8u), Call(4u, 8u, 1u), Sopp(8u, 1u), Sop1(12u, 0u, 0x20u, 8u)}); + Require(readsFirst.FindBlockByProgramCounter(0u).terminator.trueBlock == readsFirst.FindBlockByProgramCounter(12u).id); + ExpectThrow("escapes the constant-offset call/return model", [] { + Build({Sop1(0u, 0u, 3u, 8u), Call(4u, 8u, 2u), Sopp(8u, 2u, 0xfffdu), Sopp(12u, 1u), Sop1(16u, 0u, 0x20u, 8u)}); + }); +} + static void CheckRejections() { ExpectThrow("invalid instruction boundary", [] { Build({Call(0u, 8u, 16u), Sopp(4u, 1u)}); @@ -280,6 +364,7 @@ int main(int argc, char** argv) { CheckCallAfterEnd(); CheckRejections(); CheckUserDataCalls(); + CheckTableCalls(); CheckGuards(""); } else { Require(argc == 2); diff --git a/core/libs/tests/ScalarSwappc.cpp b/core/libs/tests/ScalarSwappc.cpp index 6ab8dc4f05..ba414a1c35 100644 --- a/core/libs/tests/ScalarSwappc.cpp +++ b/core/libs/tests/ScalarSwappc.cpp @@ -126,16 +126,14 @@ static void CheckFetchCallAtEntry() { } static void CheckThrows() { - ExpectThrow("computed/data-dependent s_swappc_b64", [] { - Build(Program({ - Sop1(0x00u, 8u, 0x21u, 4u), - Sopp(0x04u, 0x01u), - })); - }); + Require(Build(Program({ + Sop1(0x00u, 8u, 0x21u, 4u), + Sopp(0x04u, 0x01u), + })).unlinkedCallProgramCounters == std::vector{0x00u}); ExpectThrow("escapes the constant-offset call/return model", [] { Build(Program({ Sop1(0x00u, 4u, 0x1fu, 0u), - Sop2Literal(0x04u, 4u, 0x00u, 4u, 0x18u), + Sop2Literal(0x04u, 4u, 0x00u, 4u, 0x20u), Sop2Literal(0x0cu, 5u, 0x04u, 5u, 0u), Sop1(0x14u, 8u, 0x21u, 4u), Sop1(0x18u, 8u, 0x03u, 8u), @@ -166,7 +164,7 @@ static void CheckThrows() { Sop1(0x24u, 0u, 0x20u, 8u), })); }); - ExpectThrow("is shared with another call", [] { + ExpectThrow("control flow enters the call region", [] { Build(Program({ Sop1(0x00u, 4u, 0x1fu, 0u), Sop2Literal(0x04u, 4u, 0x00u, 4u, 0x18u), @@ -188,17 +186,17 @@ static void CheckThrows() { Sop1(0x1cu, 0u, 0x20u, 8u), })); }); - ExpectThrow("computed/data-dependent s_swappc_b64", [] { + { SwappcInfo info; info.fetchCallAllowed = true; info.userDataBaseRegister = 16u; info.userDataCount = 8u; - Build(Program({ + Require(Build(Program({ Sopp(0x00u, 0x00u), Sop1(0x04u, 2u, 0x21u, 4u), Sopp(0x08u, 0x01u), - }), &info); - }); + }), &info).unlinkedCallProgramCounters == std::vector{0x04u}); + } ExpectThrow("must live in the fetch-shader code", [] { SwappcInfo info; info.fetchCallAllowed = true; diff --git a/core/shader/recompiler/BdaAbi.hpp b/core/shader/recompiler/BdaAbi.hpp index 0b055a525e..7a79928c76 100644 --- a/core/shader/recompiler/BdaAbi.hpp +++ b/core/shader/recompiler/BdaAbi.hpp @@ -28,7 +28,7 @@ struct Range { }; enum class FaultState : std::uint32_t { Empty, Writing, Ready }; -enum class FaultReason : std::uint32_t { Unmapped = 1, Permission, Overflow, InvalidTable, InvalidRectangle, LoopLimit, Unaligned, InvalidDescriptor, ImageTable }; +enum class FaultReason : std::uint32_t { Unmapped = 1, Permission, Overflow, InvalidTable, InvalidRectangle, LoopLimit, Unaligned, InvalidDescriptor, ImageTable, UnlinkedCall }; struct Fault { FaultState state; diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index 3068aca764..08b8bbb703 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -260,6 +260,8 @@ function(add_shader_recompiler target runtimeTarget) FindUserDataCalls UserDataCalleeLength LinkUserDataCalls + FindTableCalls + LinkCalls WriteFileAtomically ) set(shaderCacheManifest) diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp index d93905ba94..3199579688 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp @@ -107,6 +107,7 @@ struct ControlFlowGraph { std::vector components; std::vector codeTableLoadProgramCounters; std::vector codeTableLoads; + std::vector unlinkedCallProgramCounters; std::uint32_t entryBlock = InvalidControlFlowId; bool hasFetchCall = false; std::uint32_t fetchCallProgramCounter = 0; diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp index 27bbdae67d..19f95de746 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp @@ -23,6 +23,7 @@ struct SwappcInfo { bool fetchCallAllowed = false; std::uint32_t userDataBaseRegister = 0; std::uint32_t userDataCount = 0; + std::uint32_t waveSize = 0; }; struct SwappcCall { @@ -41,10 +42,28 @@ struct UserDataCall { std::uint32_t userDataRegister = 0; }; +struct TableCall { + std::uint32_t programCounter = 0; + std::uint32_t linkRegister = 0; + std::uint32_t targetRegister = 0; + std::uint32_t userDataRegister = 0; + std::uint32_t descriptorOffset = 0; + std::uint32_t entryOffset = 0; + std::uint32_t entryStride = 0; +}; + +struct TableCallee { + std::uint64_t target = 0; + std::vector code; +}; + [[nodiscard]] std::optional UnresolvableSwappcTarget(const RdnaProgram& program, const SwappcInfo* swappc); [[nodiscard]] std::vector FindUserDataCalls(const RdnaProgram& program, const SwappcInfo& swappc); [[nodiscard]] std::optional UserDataCalleeLength(std::span callee, std::uint32_t linkRegister); [[nodiscard]] std::vector LinkUserDataCalls(const RdnaProgram& program, std::span calls, std::span> callees); +[[nodiscard]] std::vector FindTableCalls(const RdnaProgram& program, const SwappcInfo& swappc); +[[nodiscard]] std::vector LinkCalls(const RdnaProgram& program, std::span userDataCalls, std::span> userDataCallees, + std::span tableCalls, std::span> tableCallees); class GraphBuilder { public: diff --git a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp index 7a487915c8..71656b69bb 100644 --- a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp +++ b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp @@ -2,6 +2,7 @@ #include "ControlFlow/ControlFlowHelpers.hpp" #include "RdnaDecoder/RdnaInstructionDecoder.hpp" #include +#include #include #include #include @@ -257,7 +258,7 @@ bool resolveJumpTable(const RdnaProgram& program, std::uint32_t index, BoundedJu return false; } -bool writesScalar(const RdnaInstruction& instruction, std::uint32_t index) { +bool writesScalar(const RdnaInstruction& instruction, std::uint32_t index, std::uint32_t waveSize = 0u) { std::uint32_t count = instruction.dataDwordCount; switch (instruction.family) { case RdnaInstructionFamily::SOP1: @@ -273,7 +274,7 @@ bool writesScalar(const RdnaInstruction& instruction, std::uint32_t index) { case RdnaInstructionFamily::VOP3: case RdnaInstructionFamily::VOP3P: case RdnaInstructionFamily::VOPC: - count = 2u; + count = instruction.op == RdnaOpcode::VReadlaneB32 || instruction.op == RdnaOpcode::VReadfirstlaneB32 || waveSize == 32u ? 1u : 2u; break; case RdnaInstructionFamily::SOPC: case RdnaInstructionFamily::SOPP: @@ -293,11 +294,11 @@ bool writesScalar(const RdnaInstruction& instruction, std::uint32_t index) { return covers(instruction.destination) || covers(instruction.destination2); } -bool findLastWriter(const RdnaProgram& program, std::uint32_t end, std::initializer_list indices, std::uint32_t& writer) { +bool findLastWriter(const RdnaProgram& program, std::uint32_t end, std::initializer_list indices, std::uint32_t& writer, std::uint32_t waveSize = 0u) { for (auto position = end; position-- > 0u;) { const auto& instruction = program.instructions[position]; if (IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SSetpcB64 || instruction.op == RdnaOpcode::SEndpgm) return false; - if (std::ranges::any_of(indices, [&](std::uint32_t index) { return writesScalar(instruction, index); })) { + if (std::ranges::any_of(indices, [&](std::uint32_t index) { return writesScalar(instruction, index, waveSize); })) { writer = position; return true; } @@ -475,7 +476,298 @@ bool isFetchCall(const RdnaProgram& program, std::uint32_t index, const SwappcIn return swappc != nullptr && swappc->fetchCallAllowed && positional && (userDataPair || outsideProgram); } -std::vector analyzeSwappcCalls(const RdnaProgram& program, const SwappcInfo* swappc) { +bool overwritesScalar(const RdnaInstruction& instruction, std::uint32_t reg, std::uint32_t waveSize) { + const auto first = scalarIndex(instruction.destination); + switch (instruction.family) { + case RdnaInstructionFamily::SMEM: + return isScalarLoad(instruction.op) && first != NoScalarRegister && reg >= first && reg - first < instruction.dataDwordCount; + case RdnaInstructionFamily::SOP1: + switch (instruction.op) { + case RdnaOpcode::SMovreldB32: + case RdnaOpcode::SMovreldB64: + case RdnaOpcode::SMovrelsd2B32: + case RdnaOpcode::SCmovB32: + case RdnaOpcode::SCmovB64: + case RdnaOpcode::SBitset0B32: + case RdnaOpcode::SBitset0B64: + case RdnaOpcode::SBitset1B32: + case RdnaOpcode::SBitset1B64: + case RdnaOpcode::SSwappcB64: + case RdnaOpcode::SSetpcB64: return false; + case RdnaOpcode::SMovB64: + case RdnaOpcode::SGetpcB64: return first != NoScalarRegister && reg >= first && reg - first < 2u; + default: return first != NoScalarRegister && reg == first; + } + case RdnaInstructionFamily::SOP2: return first != NoScalarRegister && reg == first; + case RdnaInstructionFamily::SOPK: return instruction.op == RdnaOpcode::SMovkI32 && reg == first; + case RdnaInstructionFamily::VOP1: + case RdnaInstructionFamily::VOP2: + case RdnaInstructionFamily::VOP3: + case RdnaInstructionFamily::VOP3P: + case RdnaInstructionFamily::VOPC: { + const std::uint32_t count = instruction.op != RdnaOpcode::VReadlaneB32 && instruction.op != RdnaOpcode::VReadfirstlaneB32 && waveSize == 64u ? 2u : 1u; + return std::ranges::any_of(std::array{&instruction.destination, &instruction.destination2}, [&](const RdnaOperand* operand) { + const auto slot = scalarIndex(*operand); + return slot != NoScalarRegister && reg >= slot && reg - slot < count; + }); + } + default: return false; + } +} + +bool readsVectorSources64(RdnaOpcode op) { + switch (op) { + case RdnaOpcode::VMadU64U32: + case RdnaOpcode::VMadI64I32: + case RdnaOpcode::VFmaF64: + case RdnaOpcode::VAddF64: + case RdnaOpcode::VMulF64: + case RdnaOpcode::VMinF64: + case RdnaOpcode::VMaxF64: + case RdnaOpcode::VLdexpF64: + case RdnaOpcode::VCvtI32F64: + case RdnaOpcode::VCvtF64I32: + case RdnaOpcode::VCvtF32F64: + case RdnaOpcode::VCvtF64F32: + case RdnaOpcode::VCvtU32F64: + case RdnaOpcode::VCvtF64U32: + case RdnaOpcode::VTruncF64: + case RdnaOpcode::VCeilF64: + case RdnaOpcode::VRndneF64: + case RdnaOpcode::VFloorF64: + case RdnaOpcode::VFrexpExpI32F64: + case RdnaOpcode::VFrexpMantF64: + case RdnaOpcode::VFractF64: + case RdnaOpcode::VRcpF64: + case RdnaOpcode::VRsqF64: + case RdnaOpcode::VSqrtF64: + case RdnaOpcode::VTrigPreopF64: + case RdnaOpcode::VDivScaleF64: + case RdnaOpcode::VDivFmasF64: + case RdnaOpcode::VDivFixupF64: + case RdnaOpcode::VLshlrevB64: + case RdnaOpcode::VLshrrevB64: + case RdnaOpcode::VAshrrevI64: + case RdnaOpcode::VCmpEqI64: + case RdnaOpcode::VCmpLtU64: + case RdnaOpcode::VCmpEqU64: + case RdnaOpcode::VCmpGtU64: + case RdnaOpcode::VCmpNeU64: + case RdnaOpcode::VCmpxNeI64: + case RdnaOpcode::VCmpxNeU64: + case RdnaOpcode::VCmpFI64: + case RdnaOpcode::VCmpLtI64: + case RdnaOpcode::VCmpLeI64: + case RdnaOpcode::VCmpGtI64: + case RdnaOpcode::VCmpNeI64: + case RdnaOpcode::VCmpGeI64: + case RdnaOpcode::VCmpTI64: + case RdnaOpcode::VCmpxFI64: + case RdnaOpcode::VCmpxLtI64: + case RdnaOpcode::VCmpxEqI64: + case RdnaOpcode::VCmpxLeI64: + case RdnaOpcode::VCmpxGtI64: + case RdnaOpcode::VCmpxGeI64: + case RdnaOpcode::VCmpxTI64: + case RdnaOpcode::VCmpFU64: + case RdnaOpcode::VCmpLeU64: + case RdnaOpcode::VCmpGeU64: + case RdnaOpcode::VCmpTU64: + case RdnaOpcode::VCmpxFU64: + case RdnaOpcode::VCmpxLtU64: + case RdnaOpcode::VCmpxEqU64: + case RdnaOpcode::VCmpxLeU64: + case RdnaOpcode::VCmpxGtU64: + case RdnaOpcode::VCmpxGeU64: + case RdnaOpcode::VCmpxTU64: + case RdnaOpcode::VCmpFF64: + case RdnaOpcode::VCmpLtF64: + case RdnaOpcode::VCmpEqF64: + case RdnaOpcode::VCmpLeF64: + case RdnaOpcode::VCmpGtF64: + case RdnaOpcode::VCmpLgF64: + case RdnaOpcode::VCmpGeF64: + case RdnaOpcode::VCmpOF64: + case RdnaOpcode::VCmpUF64: + case RdnaOpcode::VCmpNgeF64: + case RdnaOpcode::VCmpNlgF64: + case RdnaOpcode::VCmpNgtF64: + case RdnaOpcode::VCmpNleF64: + case RdnaOpcode::VCmpNeqF64: + case RdnaOpcode::VCmpNltF64: + case RdnaOpcode::VCmpTruF64: + case RdnaOpcode::VCmpClassF64: + case RdnaOpcode::VCmpxFF64: + case RdnaOpcode::VCmpxLtF64: + case RdnaOpcode::VCmpxEqF64: + case RdnaOpcode::VCmpxLeF64: + case RdnaOpcode::VCmpxGtF64: + case RdnaOpcode::VCmpxLgF64: + case RdnaOpcode::VCmpxGeF64: + case RdnaOpcode::VCmpxOF64: + case RdnaOpcode::VCmpxUF64: + case RdnaOpcode::VCmpxNgeF64: + case RdnaOpcode::VCmpxNlgF64: + case RdnaOpcode::VCmpxNgtF64: + case RdnaOpcode::VCmpxNleF64: + case RdnaOpcode::VCmpxNeqF64: + case RdnaOpcode::VCmpxNltF64: + case RdnaOpcode::VCmpxTruF64: + case RdnaOpcode::VCmpxClassF64: + return true; + default: return false; + } +} + +bool readsScalarDword(const RdnaInstruction& instruction, std::uint32_t reg, std::uint32_t waveSize) { + if (instruction.op == RdnaOpcode::SMovrelsB32 || instruction.op == RdnaOpcode::SMovrelsB64) return true; + const auto overlaps = [&](const RdnaOperand& operand, std::uint32_t count) { + const auto slot = scalarIndex(operand); + return slot != NoScalarRegister && slot <= reg && reg - slot < count; + }; + std::uint32_t source0Count = 1u, source1Count = 1u, source2Count = 1u, source3Count = 1u; + switch (instruction.family) { + case RdnaInstructionFamily::SMEM: + switch (instruction.op) { + case RdnaOpcode::SBufferLoadDword: + case RdnaOpcode::SBufferLoadDwordx2: + case RdnaOpcode::SBufferLoadDwordx4: + case RdnaOpcode::SBufferLoadDwordx8: + case RdnaOpcode::SBufferLoadDwordx16: source0Count = 4u; break; + default: source0Count = 2u; break; + } + break; + case RdnaInstructionFamily::MUBUF: + case RdnaInstructionFamily::MTBUF: source1Count = 4u; break; + case RdnaInstructionFamily::MIMG: + source1Count = instruction.imageR128 ? 4u : 8u; + source2Count = 4u; + break; + case RdnaInstructionFamily::FLAT: + source0Count = 2u; + source1Count = 2u; + break; + case RdnaInstructionFamily::SOPK: + if (instruction.op != RdnaOpcode::SMovkI32 && instruction.op != RdnaOpcode::SCallB64 && overlaps(instruction.destination, 1u)) return true; + break; + case RdnaInstructionFamily::SOP1: + case RdnaInstructionFamily::SOP2: + case RdnaInstructionFamily::SOPC: + switch (instruction.op) { + case RdnaOpcode::SBcnt0I32B64: + case RdnaOpcode::SBcnt1I32B64: + case RdnaOpcode::SFf0I32B64: + case RdnaOpcode::SFf1I32B64: + case RdnaOpcode::SFlbitI32B64: + case RdnaOpcode::SFlbitI32I64: + case RdnaOpcode::SCmpEqU64: + case RdnaOpcode::SCmpLgU64: + case RdnaOpcode::SBitcmp0B64: + case RdnaOpcode::SBitcmp1B64: source0Count = source1Count = 2u; break; + default: source0Count = source1Count = instruction.family == RdnaInstructionFamily::SOPC ? 1u : std::max(instruction.dataDwordCount, 1u); break; + } + break; + case RdnaInstructionFamily::VOP1: + case RdnaInstructionFamily::VOP2: + case RdnaInstructionFamily::VOP3: + case RdnaInstructionFamily::VOP3P: + case RdnaInstructionFamily::VOPC: + source0Count = source1Count = source2Count = source3Count = readsVectorSources64(instruction.op) ? 2u : 1u; + switch (instruction.op) { + case RdnaOpcode::VCndmaskB32: + case RdnaOpcode::VAddcU32: + case RdnaOpcode::VSubCoCiU32: + case RdnaOpcode::VSubrevCoCiU32: + case RdnaOpcode::VDivFmasF32: + case RdnaOpcode::VDivFmasF64: source2Count = std::max(source2Count, waveSize == 32u ? 1u : 2u); break; + default: break; + } + break; + default: break; + } + switch (instruction.op) { + case RdnaOpcode::SBitset0B32: + case RdnaOpcode::SBitset0B64: + case RdnaOpcode::SBitset1B32: + case RdnaOpcode::SBitset1B64: + case RdnaOpcode::SCmovB32: + case RdnaOpcode::SCmovB64: + if (overlaps(instruction.destination, 2u)) return true; + break; + default: break; + } + return overlaps(instruction.source0, source0Count) || overlaps(instruction.source1, source1Count) || overlaps(instruction.source2, source2Count) || overlaps(instruction.source3, source3Count); +} + +template +void requireReturnAddressesStayInCalls(const RdnaProgram& program, const std::vector& calls, const std::vector& unlinked, std::uint32_t waveSize, const TError& escapeError) { + std::set links; + for (const auto& call : calls) { + if (!call.fetch) links.insert(call.linkRegister); + } + const auto size = static_cast(program.instructions.size()); + const auto successors = [&](std::uint32_t index) { + std::vector next; + const auto& instruction = program.instructions[index]; + if (instruction.op == RdnaOpcode::SEndpgm || instruction.op == RdnaOpcode::SCodeEnd) return next; + if (IsDirectBranchOpcode(instruction.op)) { + next.push_back(instructionIndexOfProgramCounter(program, instruction.branchTarget)); + if (IsConditionalBranchOpcode(instruction.op)) next.push_back(index + 1u); + } else if (instruction.op == RdnaOpcode::SSwappcB64 || instruction.op == RdnaOpcode::SCallB64) { + const SwappcCall* call = findCallAt(calls, index); + next.push_back(call == nullptr || call->fetch ? index + 1u : call->targetIndex); + } else if (instruction.op == RdnaOpcode::SSetpcB64) { + std::uint32_t target = 0; + if (const SwappcCall* call = findReturnAt(calls, index); call != nullptr) { + next.push_back(call->callIndex + 1u); + } else if (resolveSetpcTarget(program, index, target)) { + next.push_back(instructionIndexOfProgramCounter(program, target)); + } + } else { + next.push_back(index + 1u); + } + std::erase_if(next, [&](std::uint32_t successor) { return successor >= size; }); + return next; + }; + for (const auto link : links) { + const auto writesLink = [&](std::uint32_t index) { + const SwappcCall* call = findCallAt(calls, index); + return (call != nullptr && call->linkRegister == link) || (std::ranges::find(unlinked, index) != unlinked.end() && scalarIndex(program.instructions[index].destination) == link); + }; + std::vector holds(size, 0u); + std::vector work; + for (std::uint32_t index = 0; index < size; ++index) { + if (writesLink(index)) work.push_back(index); + } + const auto after = [&](std::uint32_t index) -> std::uint8_t { + if (writesLink(index)) return 3u; + std::uint8_t value = holds[index]; + if (overwritesScalar(program.instructions[index], link, waveSize)) value &= 2u; + if (overwritesScalar(program.instructions[index], link + 1u, waveSize)) value &= 1u; + return value; + }; + while (!work.empty()) { + const auto index = work.back(); + work.pop_back(); + const auto value = after(index); + for (const auto successor : successors(index)) { + if ((holds[successor] | value) == holds[successor]) continue; + holds[successor] |= value; + work.push_back(successor); + } + } + for (std::uint32_t index = 0; index < size; ++index) { + const auto& instruction = program.instructions[index]; + if (holds[index] == 0u || writesLink(index) || instruction.family == RdnaInstructionFamily::SOPC || findReturnAt(calls, index) != nullptr) continue; + const bool inside = std::ranges::any_of(calls, [&](const SwappcCall& call) { return !call.fetch && call.linkRegister == link && index >= call.targetIndex && index <= call.returnIndex; }); + if (inside || !(((holds[index] & 1u) != 0u && readsScalarDword(instruction, link, waveSize)) || ((holds[index] & 2u) != 0u && readsScalarDword(instruction, link + 1u, waveSize)))) continue; + const auto owner = std::ranges::find_if(calls, [&](const SwappcCall& call) { return !call.fetch && call.linkRegister == link; }); + throw escapeError(*owner, instruction); + } + } +} + +std::vector analyzeSwappcCalls(const RdnaProgram& program, const SwappcInfo* swappc, std::vector& unlinked) { std::vector calls; for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { const auto& instruction = program.instructions[index]; @@ -502,11 +794,8 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa continue; } if (!staticTarget || call.targetIndex == InvalidControlFlowId) { - if (std::ranges::any_of(calls, [&](const SwappcCall& other) { return other.linkRegister == call.linkRegister; })) { - throw std::invalid_argument("unclosable scalar call/return pairing at program counter " + toHexString(instruction.programCounter) + - ": link register s[" + std::to_string(call.linkRegister) + "] is shared with another call"); - } - throw std::invalid_argument("computed/data-dependent s_swappc_b64 call target at program counter " + toHexString(instruction.programCounter) + " is not statically resolvable"); + unlinked.push_back(index); + continue; } call.targetProgramCounter = target; calls.push_back(call); @@ -574,7 +863,8 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa continue; } const bool outsideRegion = regionEnd != InvalidControlFlowId && (index < call.targetIndex || index > regionEnd); - if (touchesLinkRegister(instruction, call.linkRegister) && !(outsideRegion && onlyWritesLinkRegister(instruction, call.linkRegister))) { + if (outsideRegion) continue; + if (touchesLinkRegister(instruction, call.linkRegister)) { throw escapeError(call, instruction); } } @@ -609,14 +899,12 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa if (std::ranges::any_of(calls, [&](const SwappcCall& other) { return other.callIndex == index && other.linkRegister == call.linkRegister; })) continue; const auto& instruction = program.instructions[index]; const auto region = std::ranges::find_if(calls, [&](const SwappcCall& other) { return other.linkRegister == call.linkRegister && index >= other.targetIndex && index <= other.returnIndex; }); - if (region != calls.end()) { - if (index != region->returnIndex && touchesLinkRegister(instruction, call.linkRegister)) throw escapeError(*region, instruction); - } else if (touchesLinkRegister(instruction, call.linkRegister) && !onlyWritesLinkRegister(instruction, call.linkRegister)) { - throw escapeError(call, instruction); - } + if (region != calls.end() && index != region->returnIndex && touchesLinkRegister(instruction, call.linkRegister)) throw escapeError(*region, instruction); } } + requireReturnAddressesStayInCalls(program, calls, unlinked, swappc != nullptr ? swappc->waveSize : 0u, escapeError); + for (std::size_t first = 0; first < calls.size(); ++first) { if (!calls[first].fetch && calls[first].callIndex >= calls[first].targetIndex && calls[first].callIndex <= calls[first].returnIndex) { throw std::invalid_argument("recursive scalar call at program counter " + toHexString(program.instructions[calls[first].callIndex].programCounter) + @@ -664,7 +952,7 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa } if (instruction.op == RdnaOpcode::SSwappcB64 || instruction.op == RdnaOpcode::SCallB64) { const SwappcCall* call = findCallAt(calls, index); - transfers.push_back(Transfer{index, call->fetch ? index + 1u : call->targetIndex}); + transfers.push_back(Transfer{index, call == nullptr || call->fetch ? index + 1u : call->targetIndex}); continue; } if (instruction.op != RdnaOpcode::SEndpgm && instruction.op != RdnaOpcode::SCodeEnd) transfers.push_back(Transfer{index, index + 1u}); @@ -958,23 +1246,75 @@ std::optional UnresolvableSwappcTarget(const RdnaProgram& program return std::nullopt; } -std::vector FindUserDataCalls(const RdnaProgram& program, const SwappcInfo& swappc) { +namespace { + +std::set branchLabels(const RdnaProgram& program) { std::set labels; for (const auto& instruction : program.instructions) { if (IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SCallB64) labels.insert(instruction.branchTarget); } - const auto userData = [&](std::uint32_t reg) { return reg >= swappc.userDataBaseRegister && reg - swappc.userDataBaseRegister < swappc.userDataCount; }; - const auto unchangedUntil = [&](std::uint32_t reg, std::uint32_t reader) { - for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { - if (!writesScalar(program.instructions[index], reg)) continue; - if (index <= reader) return false; - for (std::uint32_t later = index; later < program.instructions.size(); ++later) { - const auto& instruction = program.instructions[later]; - if ((IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SCallB64) && instruction.branchTarget <= program.instructions[reader].programCounter) return false; - } + return labels; +} + +bool unchangedUntil(const RdnaProgram& program, std::uint32_t reg, std::uint32_t reader, std::uint32_t waveSize) { + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + if (!writesScalar(program.instructions[index], reg, waveSize)) continue; + if (index <= reader) return false; + for (std::uint32_t later = index; later < program.instructions.size(); ++later) { + const auto& instruction = program.instructions[later]; + if ((IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SCallB64) && instruction.branchTarget <= program.instructions[reader].programCounter) return false; } - return true; - }; + } + return true; +} + +bool isScalarBufferLoad(RdnaOpcode op) { + switch (op) { + case RdnaOpcode::SBufferLoadDword: + case RdnaOpcode::SBufferLoadDwordx2: + case RdnaOpcode::SBufferLoadDwordx4: + case RdnaOpcode::SBufferLoadDwordx8: + case RdnaOpcode::SBufferLoadDwordx16: return true; + default: return false; + } +} + +bool loadedDword(const RdnaProgram& program, std::uint32_t reg, std::uint32_t before, std::uint32_t waveSize, std::uint32_t& load, std::uint32_t& dword) { + for (std::uint32_t copies = 0; copies < 4u; ++copies) { + std::uint32_t writer = 0; + if (!findLastWriter(program, before, {reg}, writer, waveSize)) return false; + const auto& instruction = program.instructions[writer]; + const auto destination = scalarIndex(instruction.destination); + const auto source = scalarIndex(instruction.source0); + if (instruction.op == RdnaOpcode::SMovB32 && destination == reg && source != NoScalarRegister) { + reg = source; + } else if (instruction.op == RdnaOpcode::SMovB64 && destination != NoScalarRegister && reg - destination < 2u && source != NoScalarRegister) { + reg = source + (reg - destination); + } else if (isScalarBufferLoad(instruction.op) && destination != NoScalarRegister && reg >= destination && reg - destination < instruction.dataDwordCount) { + load = writer; + dword = reg - destination; + return true; + } else { + return false; + } + before = writer; + } + return false; +} + +std::uint32_t branchWord(std::uint32_t opcode, std::size_t from, std::size_t to, const char* what) { + const auto offset = static_cast(to) - static_cast(from) - 1; + if (offset < -0x8000 || offset > 0x7fff) throw std::invalid_argument(std::string(what) + " lands out of the 16-bit branch range"); + return opcode | (static_cast(offset) & 0xffffu); +} + +} + + +std::vector FindUserDataCalls(const RdnaProgram& program, const SwappcInfo& swappc) { + const auto labels = branchLabels(program); + const auto userData = [&](std::uint32_t reg) { return reg >= swappc.userDataBaseRegister && reg - swappc.userDataBaseRegister < swappc.userDataCount; }; + const auto unchanged = [&](std::uint32_t reg, std::uint32_t reader) { return unchangedUntil(program, reg, reader, swappc.waveSize); }; std::vector calls; for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { const auto& instruction = program.instructions[index]; @@ -985,7 +1325,7 @@ std::vector FindUserDataCalls(const RdnaProgram& program, const Sw const auto source = scalarIndex(instruction.source0); if (link == NoScalarRegister || source == NoScalarRegister || source % 2u != 0u) continue; std::uint32_t pair = NoScalarRegister; - if (source != link && userData(source) && userData(source + 1u) && unchangedUntil(source, index) && unchangedUntil(source + 1u, index)) { + if (source != link && userData(source) && userData(source + 1u) && unchanged(source, index) && unchanged(source + 1u, index)) { pair = source; } else { std::uint32_t low = 0, high = 0; @@ -1003,7 +1343,7 @@ std::vector FindUserDataCalls(const RdnaProgram& program, const Sw from = scalarIndex(lowWriter.source0); if (from != NoScalarRegister && !isScalar(highWriter.source0, from + 1u)) from = NoScalarRegister; } - if (!sameBlock || from == NoScalarRegister || from % 2u != 0u || !userData(from) || !userData(from + 1u) || !unchangedUntil(from, low) || !unchangedUntil(from + 1u, high)) continue; + if (!sameBlock || from == NoScalarRegister || from % 2u != 0u || !userData(from) || !userData(from + 1u) || !unchanged(from, low) || !unchanged(from + 1u, high)) continue; pair = from; } calls.push_back({instruction.programCounter, link, pair}); @@ -1077,32 +1417,130 @@ std::optional UserDataCalleeLength(std::span } std::vector LinkUserDataCalls(const RdnaProgram& program, std::span calls, std::span> callees) { - if (calls.size() != callees.size()) throw std::logic_error("user-data calls and callees differ in number"); - for (const auto& instruction : program.instructions) { - if (instruction.op == RdnaOpcode::SGetpcB64) { - throw std::invalid_argument("a program that reads its program counter (s_getpc_b64 at program counter " + toHexString(instruction.programCounter) + ") cannot have user-data callees linked after it"); + return LinkCalls(program, calls, callees, {}, {}); +} + +std::vector FindTableCalls(const RdnaProgram& program, const SwappcInfo& swappc) { + const auto labels = branchLabels(program); + const auto userData = [&](std::uint32_t reg) { return reg >= swappc.userDataBaseRegister && reg - swappc.userDataBaseRegister < swappc.userDataCount; }; + const auto sameBlock = [&](std::uint32_t first, std::uint32_t last) { + return std::none_of(labels.begin(), labels.end(), [&](std::uint32_t label) { + return label > program.instructions[first].programCounter && label <= program.instructions[last].programCounter; + }); + }; + std::vector calls; + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + const auto& instruction = program.instructions[index]; + if (instruction.op != RdnaOpcode::SSwappcB64) continue; + std::uint32_t ignored = 0; + if (resolveSetpcTarget(program, index, ignored)) continue; + const auto link = scalarIndex(instruction.destination); + const auto target = scalarIndex(instruction.source0); + if (link == NoScalarRegister || link % 2u != 0u || link > 104u || target == NoScalarRegister || target % 2u != 0u || target > 104u) continue; + std::uint32_t lowLoad = 0, lowDword = 0, highLoad = 0, highDword = 0; + if (!loadedDword(program, target, index, swappc.waveSize, lowLoad, lowDword) || !loadedDword(program, target + 1u, index, swappc.waveSize, highLoad, highDword)) continue; + if (lowLoad != highLoad || highDword != lowDword + 1u || !sameBlock(lowLoad, index)) continue; + const auto& load = program.instructions[lowLoad]; + const auto descriptor = scalarIndex(load.source0); + if (load.source0.kind != RdnaOperandKind::ScalarRegister || descriptor % 4u != 0u || load.memoryOffset >= 0x100000u || (load.memoryOffset & 3u) != 0u) continue; + std::uint32_t descriptorLoad = 0; + if (!findLastWriter(program, lowLoad, {descriptor, descriptor + 1u, descriptor + 2u, descriptor + 3u}, descriptorLoad, swappc.waveSize) || !sameBlock(descriptorLoad, index)) continue; + const auto& table = program.instructions[descriptorLoad]; + const auto pointer = scalarIndex(table.source0); + if (table.op != RdnaOpcode::SLoadDwordx4 || !isScalar(table.destination, descriptor) || table.source0.kind != RdnaOperandKind::ScalarRegister || table.source1.kind != RdnaOperandKind::Null || + table.memoryOffset >= 0x100000u || (table.memoryOffset & 3u) != 0u || !userData(pointer) || !userData(pointer + 1u) || + !unchangedUntil(program, pointer, descriptorLoad, swappc.waveSize) || !unchangedUntil(program, pointer + 1u, descriptorLoad, swappc.waveSize)) continue; + std::uint32_t stride = 0; + if (load.source1.kind != RdnaOperandKind::Null) { + const auto offset = scalarIndex(load.source1); + std::uint32_t shift = 0, shiftIndex = 0; + if (offset == NoScalarRegister || !findLastWriter(program, lowLoad, {offset}, shiftIndex, swappc.waveSize) || !sameBlock(shiftIndex, index)) continue; + const auto& shifted = program.instructions[shiftIndex]; + if (shifted.op != RdnaOpcode::SLshlB32 || !isScalar(shifted.destination, offset) || !isImmediate(shifted.source1, shift) || shift < 2u || shift > 24u) continue; + stride = 1u << shift; + } + calls.push_back({instruction.programCounter, link, target, pointer, table.memoryOffset, load.memoryOffset + lowDword * 4u, stride}); + } + return calls; +} + +std::vector LinkCalls(const RdnaProgram& program, std::span userDataCalls, std::span> userDataCallees, + std::span tableCalls, std::span> tableCallees) { + if (userDataCalls.size() != userDataCallees.size()) throw std::logic_error("user-data calls and callees differ in number"); + if (tableCalls.size() != tableCallees.size()) throw std::logic_error("table calls and callees differ in number"); + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + BoundedJumpTable table; + if (program.instructions[index].op == RdnaOpcode::SSetpcB64 && resolveBoundedJumpTable(program, index, table)) { + throw std::invalid_argument("a program that jumps through a table in its code (s_setpc_b64 at program counter " + toHexString(program.instructions[index].programCounter) + ") cannot have callees linked over the words after its last instruction"); } } const auto end = instructionEndProgramCounter(program.instructions.back()) / 4u; std::vector linked(program.code.begin(), program.code.begin() + end); - for (std::size_t call = 0; call < calls.size(); ++call) { - const auto index = instructionIndexOfProgramCounter(program, calls[call].programCounter); + const auto callWord = [&](std::uint32_t programCounter, const char* kind) { + const auto index = instructionIndexOfProgramCounter(program, programCounter); if (index == InvalidControlFlowId || program.instructions[index].op != RdnaOpcode::SSwappcB64 || program.instructions[index].wordCount != 1u) { - throw std::logic_error("user-data call does not name an s_swappc_b64"); + throw std::logic_error(std::string(kind) + " call does not name an s_swappc_b64"); } - const auto word = calls[call].programCounter / 4u; + return programCounter / 4u; + }; + for (std::size_t call = 0; call < userDataCalls.size(); ++call) { + const auto word = callWord(userDataCalls[call].programCounter, "user-data"); const auto offset = static_cast(linked.size()) - static_cast(word) - 1; - if (offset > 0x7fff) throw std::invalid_argument("user-data callee of the call at program counter " + toHexString(calls[call].programCounter) + " lands out of s_call_b64 range"); - linked[word] = 0xbb000000u | (calls[call].linkRegister << 16u) | static_cast(offset); - linked.insert(linked.end(), callees[call].begin(), callees[call].end()); + if (offset > 0x7fff) throw std::invalid_argument("user-data callee of the call at program counter " + toHexString(userDataCalls[call].programCounter) + " lands out of s_call_b64 range"); + linked[word] = 0xbb000000u | (userDataCalls[call].linkRegister << 16u) | static_cast(offset); + linked.insert(linked.end(), userDataCallees[call].begin(), userDataCallees[call].end()); + } + for (std::size_t call = 0; call < tableCalls.size(); ++call) { + const auto& table = tableCalls[call]; + const auto& callees = tableCallees[call]; + const auto word = callWord(table.programCounter, "table"); + const auto at = "the table callees of the call at program counter " + toHexString(table.programCounter); + const auto original = linked[word]; + const auto count = callees.size(); + const auto dispatch = linked.size(); + const auto zero = dispatch + 1u; + const auto one = zero + count * 7u + 2u; + const auto calls = one + count * 8u + 2u; + const auto fault = calls + count * 2u; + const auto branch = [&](std::uint32_t opcode, std::size_t target) { linked.push_back(branchWord(opcode, linked.size(), target, at.c_str())); }; + linked[word] = branchWord(0xbf820000u, word, dispatch, at.c_str()); + branch(0xbf850000u, one); + for (const bool sccOne : {false, true}) { + for (std::size_t callee = 0; callee < count; ++callee) { + const auto next = linked.size() + (sccOne ? 8u : 7u); + linked.insert(linked.end(), {0xbf07ff00u | table.targetRegister, static_cast(callees[callee].target)}); + branch(0xbf850000u, next); + linked.insert(linked.end(), {0xbf07ff00u | (table.targetRegister + 1u), static_cast(callees[callee].target >> 32u)}); + branch(0xbf850000u, next); + if (sccOne) linked.push_back(0xbf068080u); + branch(0xbf820000u, calls + callee * 2u); + } + linked.push_back(sccOne ? 0xbf068080u : 0xbf078080u); + branch(0xbf820000u, fault); + } + for (std::size_t callee = 0; callee < count; ++callee) { + linked.push_back(0u); + branch(0xbf820000u, word + 1u); + } + linked.push_back(original); + branch(0xbf820000u, word + 1u); + for (std::size_t callee = 0; callee < count; ++callee) { + const auto site = calls + callee * 2u; + const auto offset = static_cast(linked.size()) - static_cast(site) - 1; + if (offset > 0x7fff) throw std::invalid_argument(at + " land out of s_call_b64 range"); + linked[site] = 0xbb000000u | (table.linkRegister << 16u) | static_cast(offset); + linked.insert(linked.end(), callees[callee].code.begin(), callees[callee].code.end()); + } } linked.push_back(0xbf9f0000u); return linked; } ControlFlowGraph GraphBuilder::Build(const RdnaProgram& program, const SwappcInfo* swappc) const { - const std::vector calls = analyzeSwappcCalls(program, swappc); + std::vector unlinked; + const std::vector calls = analyzeSwappcCalls(program, swappc, unlinked); ControlFlowGraph graph; + for (const auto index : unlinked) graph.unlinkedCallProgramCounters.push_back(program.instructions[index].programCounter); graph.blocks = splitIntoBlocks(program, calls); linkBlocks(graph.blocks, program, calls); for (const auto& call : calls) { diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp index 6ed27287ab..fd43cc3a09 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp @@ -73,6 +73,7 @@ enum class IrOpcode : std::uint16_t { Void, Reference, ReferenceU32, + UnlinkedCall, GetUserData, GetShaderBase, ShaderClock, diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp index b61b111ffd..8776466709 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp @@ -114,6 +114,7 @@ constexpr std::array(IrOpcode::Count)> Meta makeMeta("Void", Void), makeMeta("Reference", Void, U1), makeMeta("ReferenceU32", Void, U32), + makeMeta("UnlinkedCall", Void, U32, U32, U32), makeMeta("GetUserData", U32, ScalarReg), makeMeta("GetShaderBase", U64), makeMeta("ShaderClock", U64), @@ -888,6 +889,7 @@ bool IrOpcodeHasSideEffects(IrOpcode opcode) { case IrOpcode::Discard: case IrOpcode::Reference: case IrOpcode::ReferenceU32: + case IrOpcode::UnlinkedCall: case IrOpcode::SetTessellationAttribute: case IrOpcode::InterpolateHostP2: case IrOpcode::SetThreadBitScalarRegister: diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp index 0ad1aefdb9..51a49b08bb 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp @@ -220,6 +220,7 @@ std::uint32_t EmitFPNeg32(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitFPCmpClass32(SpirvEmitterState& state, std::uint32_t value, std::uint32_t mask); void EmitReference(SpirvValueEmitContext& context); void EmitReferenceU32(SpirvValueEmitContext& context); +void EmitUnlinkedCall(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t low, std::uint32_t high, std::uint32_t programCounter); void EmitControlNop(SpirvValueEmitContext& context); void EmitWaitcnt(SpirvValueEmitContext& context); void EmitSendmsg(SpirvValueEmitContext& context); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index 769ec4c329..456a79b761 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -293,6 +293,7 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) { case IrOpcode::Void: return Invoke(EmitVoid, ctx, inst); case IrOpcode::Reference: return Invoke(EmitReference, ctx, inst); case IrOpcode::ReferenceU32: return Invoke(EmitReferenceU32, ctx, inst); + case IrOpcode::UnlinkedCall: return Invoke(EmitUnlinkedCall, ctx, inst); case IrOpcode::GetUserData: return Invoke(EmitGetUserData, ctx, inst); case IrOpcode::GetShaderBase: return Invoke(EmitGetShaderBase, ctx, inst); case IrOpcode::ShaderClock: return Invoke(EmitShaderClock, ctx, inst); @@ -874,6 +875,13 @@ void EmitReferenceU32(SpirvValueEmitContext& context) { EmitVoid(context); } +void EmitUnlinkedCall(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t low, std::uint32_t high, std::uint32_t programCounter) { + auto& state = ctx.state; + if (state.faultBufferVariable == 0) ctx.Fail(inst, "calls an unlinked target without a fault buffer"); + RecordBdaFaultWords(state, low, high, ConstantU32(state, 0u), programCounter, BdaAbi::FaultReason::UnlinkedCall); + StopBdaInvocationIf(state, ConstantBool(state, true)); +} + void EmitControlNop(SpirvValueEmitContext& context) { EmitVoid(context); } diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index 711f219057..f5c21e95ca 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -27,6 +27,7 @@ public: void SetNativeFloat16(bool enabled) { nativeFloat16 = enabled; } void AddBranchCondition(const BasicBlock& source, BlockInfo& info); void TranslateCodeTableLoad(const RdnaInstruction& instruction, const ControlFlowGraph::CodeTableLoad& table); + void TranslateUnlinkedCall(const RdnaInstruction& instruction); private: struct AddressOperands { diff --git a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp index 18bb250332..9c67377bf0 100644 --- a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp +++ b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp @@ -283,6 +283,13 @@ void TranslationContext::sSwappcB64(const RdnaInstruction& inst) { writeU32Pair(inst.destination, extractU64(link)); } +void TranslationContext::TranslateUnlinkedCall(const RdnaInstruction& inst) { + const std::array target = readU32Pair(inst.source0); + (void)ir.Emit(IrOpcode::UnlinkedCall, IrType::Void, {&target[0].Value(), &target[1].Value(), &ir.Constant(inst.programCounter)}); + program.Info().usesFaultBuffer = true; + sSwappcB64(inst); +} + void TranslationContext::sCselectB32(const RdnaInstruction& inst) { const IrU32 trueValue = readU32(sourceAt(inst, 0u)); const IrU32 falseValue = readU32(sourceAt(inst, 1u)); diff --git a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp index 8bd53ac8db..a138ba9fa3 100644 --- a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp +++ b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp @@ -86,6 +86,10 @@ const ShaderWorkgroupInputInfo* shaderWorkgroupInput(ShaderStageKind stage, cons } } +bool isUnlinkedCall(const ControlFlowGraph& cfg, std::uint32_t programCounter) { + return std::find(cfg.unlinkedCallProgramCounters.begin(), cfg.unlinkedCallProgramCounters.end(), programCounter) != cfg.unlinkedCallProgramCounters.end(); +} + bool isCodeTableLoad(const ControlFlowGraph& cfg, std::uint32_t programCounter) { return std::find(cfg.codeTableLoadProgramCounters.begin(), cfg.codeTableLoadProgramCounters.end(), programCounter) != cfg.codeTableLoadProgramCounters.end(); } @@ -534,6 +538,10 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con context.TranslateCodeTableLoad(instruction, *table); continue; } + if (isUnlinkedCall(cfg, instruction.programCounter)) { + context.TranslateUnlinkedCall(instruction); + continue; + } if (options.embeddedFetch != nullptr) { const auto load = std::ranges::find(options.embeddedFetch->loads, instruction.programCounter, &EmbeddedFetchLoad::programCounter); if (load != options.embeddedFetch->loads.end()) { diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 979a7bfa7e..dd1f856efb 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -124,7 +124,8 @@ - [Table S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) DISABLE_CUBE_WRAP faults on selection without `VK_EXT_non_seamless_cube_map`: no runtime dimension check, including valid non-cube direct/table T#s. Unselected entries do not fault; with the feature this bit alone is accepted. Table T#/direct S# faults only for cubes needing the missing feature. - An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) whose record range overlaps a buffer or storage image written by its own dispatch faults (`table written by its dispatch`) when any entry is selected, even if the write misses the selected record. - An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) T# whose surface has no mapped byte faults when selected instead of becoming a zero texture as a direct T# does; what the console returns for it is not measured. -- [User-data callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is a user-data pair, or a copy of one made with `s_mov_b32` or `s_mov_b64` in the same block, calls the code at the address that user data holds. At dispatch the callee is read from guest memory up to its `s_setpc_b64` on the link pair, one copy of it is appended per call after the caller's last instruction, and the call is rewritten to `s_call_b64` to that copy. The words after the caller's last instruction are dropped; in the shaders seen they are metadata, and a caller that reads its program counter, which could address them, throws. A copy runs at another program counter than the callee, so a callee that reads its program counter, makes a call, ends the program, jumps through another pair, returns before code it branches to, spans more than 64 KiB or is not readable throws. Callees are read again whenever a dispatch misses the dispatch cache; a dispatch that hits it reuses the linked program, so a title that rewrote a callee in place at the same address would keep running the old code. +- [User-data callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is a user-data pair, or a copy of one made with `s_mov_b32` or `s_mov_b64` in the same block, calls the code at the address that user data holds. At dispatch the callee is read from guest memory up to its `s_setpc_b64` on the link pair, one copy of it is appended per call after the caller's last instruction, and the call is rewritten to `s_call_b64` to that copy. The words after the caller's last instruction are dropped from the recompiled code; in the shaders seen they are metadata or constants the caller reads relative to its program counter, and those reads go to the captured guest code, which linking does not change. A caller that jumps through a table among those words, which the control flow analysis reads from the recompiled code, throws. A copy runs at another program counter than the callee, so a callee that reads its program counter, makes a call, ends the program, jumps through another pair, returns before code it branches to, spans more than 64 KiB or is not readable throws. Callees are read again whenever a dispatch misses the dispatch cache; a dispatch that hits it reuses the linked program, so a title that rewrote a callee in place at the same address would keep running the old code. +- [Table callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is two consecutive dwords of an `s_buffer_load_dword*`, or a copy of them made with `s_mov_b32` or `s_mov_b64` in the same block, calls one of the targets that buffer holds when the load's buffer descriptor comes from an `s_load_dwordx4` without SOFFSET from a user-data pointer and the load's SOFFSET from an `s_lshl_b32` by 2 to 24 bits. At dispatch the descriptor and the buffer are read from guest memory, and every non-zero 64-bit value at the loaded offset plus a multiple of the shift's stride, below the descriptor's size (stride times num_records, or num_records bytes with stride 0), is read as a user-data callee and appended once per call. The call becomes a branch to a chain that compares the source pair with each target (`s_cmp_lg_u32` with a literal and `s_cbranch_scc1` per dword) and `s_call_b64`s the copy that matches; no match runs the original `s_swappc_b64` as an unlinked call. There is one chain for each value SCC has at the call, and the chain sets SCC back to that value (`s_cmp_eq_u32 0, 0` or `s_cmp_lg_u32 0, 0`) before the callee or the unlinked call runs. The searched range is the descriptor's whole size, which covers the range the load can reach whether it is checked in bytes or in strides; which one the PS5 uses for `s_buffer_load` was not measured. More than 64 distinct targets, a buffer above 64 MiB or a copy out of `s_call_b64` range throws. As with user-data callees, the buffer is read again only when a dispatch misses the dispatch cache, so a target a title adds to the buffer later is an unlinked call when a cached program runs. - [DB_RENDER_CONTROL resummarize draws](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - HTILE is not modeled, so a RESUMMARIZE_ENABLE draw without depth or stencil tests and without color writes changes no modeled state and draws nothing. A pure resummarize with pass-through D16/D32 depth, no depth writes or bounds test, no stencil plane, no color writes and no running pixel stage also draws nothing; other resummarize, copy or decompress draws throw. - [Comparison-sampled R32 float and R16 unorm textures](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) whose surface a storage image holds are sampled from a D32_SFLOAT or D16_UNORM image that the GPU refills from the storage image whenever its version moves, because Vulkan compares only depth formats. Only surfaces of at least 1 MiB (`APS5_DEPTH_COPY_MIN_KIB`) that are not 3D take the copy; smaller ones stay CPU snapshots compared against guest memory. With copies of small surfaces, one title hit an intermittent boot-time crash that was not explained. - [Null vertex buffer V#](../../../core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp) (libSceAgcDriver) - a V# whose four dwords are all zero is bound to a zeroed vertex buffer of the attribute's size with stride 0, so every component of its attribute reads 0. This follows the V# fields in the [AMD RDNA2 ISA](https://gpuopen.com/download/rdna2-shader-instruction-set-architecture.pdf) (`NUM_RECORDS` 0, `DST_SEL` 0 selects 0), not measured on hardware. diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 92d3cea3b1..8d2b3ce2c8 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -122,16 +122,17 @@ - [Raw `buffer_load_dwordx2`](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) is one 64-bit access on the hardware (with or without glc), so it never sees half of a `buffer_atomic_swap_x2`. Only a glc load that is 8-byte aligned and in bounds, in a program that has the 64-bit storage view (it uses 64-bit buffer atomics), is read with one 64-bit atomic load; every other dwordx2 load is two dword loads and can tear against a concurrent 64-bit write: loads without glc, and loads in a program without 64-bit buffer atomics while another shader writes the pair with `swap_x2`. - [Buffer loads without GLC or DLC](../../../core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp) (shader recompiler) are not coherent in a program that has GLC or DLC buffer accesses and no `buffer_gl0_inv`, `buffer_gl1_inv`, `s_gl1_inv` or `s_dcache_inv`: only its GLC and DLC loads, stores and atomics use the `Coherent` buffer view, so a plain load can return a value cached before another work-group's write. This follows the [LLVM AMDGPU memory model for GFX10](https://llvm.org/docs/AMDGPUUsage.html#memory-model-gfx10-gfx11), in which a load-acquire is a GLC DLC load followed by `buffer_gl1_inv` and `buffer_gl0_inv`; it is not measured on the console or on RDNA hardware. - [DS src2 operations](../../../core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp) (`ds_*_src2_*`, GFX10.1 only) read `DS[B]`, with `B = A + 4 * sext(offset1[6:0]:offset0)`, and then update `DS[A]` with the matching 32-bit atomic, so the read of `B` is not atomic with the update of `A`. The form that takes the offset from the address (`offset1[7]` set), the 64-bit forms and GDS throw: whether the LDS address keeps `A[31:17]` in that form, and whether the 64-bit forms scale the offset by 4 or 8, is not documented. `ds_min_src2_f32`, `ds_max_src2_f32` and `ds_add_src2_f32` throw: on RDNA2 (gfx1035, the oracle's hardware, which has no src2 opcodes) `ds_min_f32`, `ds_max_f32` and `ds_add_f32` ignore `IEEE_MODE` but depend on whether f32 denormals are flushed (28, 28 and 12 of 196 special rows differ), and that is not modelled for LDS float atomics yet (#981). -- [s_swappc_b64](../../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) writes the link pair with the same model as `s_getpc_b64`: the guest return address `pc + 4` as a byte offset over `GetShaderBase`. Any other use of the link pair (arithmetic, compare, hash, store) throws, apart from the overwrites and shared links that [static scalar calls](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) allow; constant-offset folding over the link is not modeled. The special encoding `s_swappc_b64` with SDST 125 still decodes as `s_setpc_b64`, which matches the hardware: measured on RDNA2 (gfx1036) in wave32 and wave64, it branches to the source pair and leaves a marker SGPR pair untouched, and with an SGPR destination the link is `PC + 4`. +- [s_swappc_b64](../../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) writes the link pair with the same model as `s_getpc_b64`: the guest return address `pc + 4` as a byte offset over `GetShaderBase`. Any other use of the link pair (arithmetic, hash, store) throws, apart from the overwrites, SOPC compares and shared links that [static scalar calls](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) allow; constant-offset folding over the link is not modeled. The special encoding `s_swappc_b64` with SDST 125 still decodes as `s_setpc_b64`, which matches the hardware: measured on RDNA2 (gfx1036) in wave32 and wave64, it branches to the source pair and leaves a marker SGPR pair untouched, and with an SGPR destination the link is `PC + 4`. - The [s_swappc_b64 fetch-shader call at vertex/local/tessellation/mesh entry](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) assumes the called fetch-shader blob ends in `s_setpc_b64` to the link pair (the GNM fetch-shader shape shadPS4's `EmitFetch` copies); the blob is not part of the shader program, so its return closes the pairing by assumption and any in-program reader of the link pair throws. Attribute fetches are inlined from the request's parsed fetch-shader plan through the same `GetAttribute` path as the embedded fetch analyzer, honoring the buffer resource `dst_sel`; an attribute writing more than four registers, or an entry call without a parsed plan, throws. The vertex/instance offset accumulator inside the fetch blob is not replayed for external blobs. - The scalar aperture sources [src_shared_base, src_shared_limit, src_private_base, src_private_limit](../../../core/shader/recompiler/RdnaDecoder/src/RdnaInstructionDecoder.cpp) (core/shader/recompiler, scalar source codes 235 to 238) read as the 64-bit apertures the flat routing uses: a base is the aperture's high dword (`SharedApertureHigh`, `PrivateApertureHigh`) over a low dword of 0, a limit is the same high dword over 0xffffffff. A 64-bit read (`s_mov_b64`, a 64-bit VALU source, a wave64 lane mask) gets the whole value and a 32-bit read gets the low dword. This follows LLVM's `SIRegisterInfo.td` (aperture registers are 64-bit, their low 32 bits are always 0 for a base and -1 for a limit) and `getSegmentAperture` (`AMDGPULegalizerInfo.cpp`, `SIISelLowering.cpp`: a 64-bit read whose high half is the aperture), the KFD's `MAKE_LDS_APP_LIMIT` (limit = base | 0xffffffff) and the gfx1030 and gfx1035 readouts on [#1190](https://github.com/boykopovar/AnyPS5/pull/1190) and [#3803](https://github.com/boykopovar/AnyPS5/issues/3803). Not measured on a console: the PS5's high dwords are the inferred values of the flat aperture bases entry, and the wave is taken to be in 64-bit address mode (PTR32 clear) - [EmitSetAttribute](../../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - a primitive with a vertex whose clip position is (0,0,0,0) is clipped away through an extra `gl_ClipDistance` element on vertex and tessellation evaluation shaders, matching what a Radeon 610M (RDNA2, RADV) and llvmpipe rasterise; NVIDIA 615.78.08 without it draws the vertex at the viewport centre. Measured for triangles (strip and list) and for a point at (0,0,0,0), which the Radeon 610M does not draw, NVIDIA draws at the viewport centre and llvmpipe at the top-left corner; lines with such a vertex are clipped by the same element and were not measured. Mesh, geometry and the generated rect-list and barycentric stages do not write it, a device without `shaderClipDistance` cannot compile vertex shaders (the compile throws), and guest clip distances are still refused by the pipeline validator. The PS5 rasteriser was not measured. - [PERSP_SAMPLE and LINEAR_SAMPLE](../../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) hold the pixel-centre barycentrics, like the centroid pairs. With one sample this is the sample position ([Vulkan standard sample locations](https://docs.vulkan.org/spec/latest/chapters/primsrast.html#primsrast-multisampling): (0.5, 0.5)); with more samples the console's per-sample weights are not computed and were not measured. - [LINE_STIPPLE_TEX](../../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) reads as 0: line stipple is not emulated. RADV enables the input only as the required pixel input of a shader that reads no interpolants, because it initializes the fewest VGPRs ([Mesa](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/vulkan/radv_shader.c) `radv_compute_spi_ps_input`), and never reads it -- [Static scalar calls](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) (`s_call_b64` and `s_swappc_b64`) require an ordinary even-aligned SGPR link pair (s0:s1 through s104:s105) and a later `s_setpc_b64` return reading that exact pair. A pair serves one call, or several calls whose regions are disjoint, each closed by the first return reading the pair after its target. Outside every region of its calls the pair may only be overwritten whole, by `s_mov_b32`, `s_mov_b64`, an SOP2 result or a scalar load, as the caller of a user-data callee does before each call. Regions must nest or be disjoint; recursion, other shared links, escaping link values (including overlapping tuples and resource descriptors), invalid targets, interleaved regions, cross-region branches or fall-through, relative SGPR reads and dynamic `s_setpc_b64` in a program with calls throw. The analysis checks all decoded instructions, including unreachable code. Arbitrary indirect calls and general call frames are not implemented. +- [Static scalar calls](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) (`s_call_b64` and `s_swappc_b64`) require an ordinary even-aligned SGPR link pair (s0:s1 through s104:s105) and a later `s_setpc_b64` return reading that exact pair. A pair serves one call, or several calls whose regions are disjoint, each closed by the first return reading the pair after its target. Outside every region of its calls the pair is free to use wherever no return address can reach: a dataflow over the decoded instructions follows each dword of the pair from the calls that write it until a scalar load, SALU or VALU result is known to overwrite it (a VALU mask counts as one SGPR unless the request says wave64), and an instruction there that may read a dword still holding a return address throws; SOPC compares are exempt, as they only set SCC. Regions must nest or be disjoint; recursion, other shared links, escaping link values (including overlapping tuples and resource descriptors), invalid targets, interleaved regions, cross-region branches or fall-through, relative SGPR reads and dynamic `s_setpc_b64` in a program with calls throw. The analysis checks all decoded instructions, including unreachable code. Arbitrary indirect calls and general call frames are not implemented. - [Relative vector addressing](../../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) (`v_movrels_b32`, `v_movreld_b32`, `v_movrelsd_b32`, `v_movrelsd_2_b32`, `v_swaprel_b32`) keeps every vector register from the lowest base register the program addresses relatively up to the last register the program names in a zero-initialised function-scope array per invocation (`LoadVectorWindow`/`StoreVectorWindow`, one element per register), so an indexed access is one load or store instead of a compare-and-select chain over the window; the host compiler decides whether that array stays in registers or spills to scratch, which is not measured. A register in the window the program never wrote reads 0, as an unwritten SSA register does; the console's value is undefined. M0 is unsigned: an index at or past the window reads `v0` and drops its write, and `v_swaprel_b32` writes neither side when either is out of range, as measured on RDNA2 (gfx1035, `agc_driver_relative_indexing`). The window ends at the last register the program names rather than at the shader's VGPR allocation, which the recompiler does not receive, so a relative write past the named registers but inside the allocation is dropped where the hardware keeps it for a later relative read. Killzone: Liberation's PSP vertex program (PPSA11142, 3176 ISA words, 104 relative accesses over 154 registers) went from 649,515 to 392,333 SPIR-V words with this. Pixel programs whose relative span contains the packed ancillary input instead keep that span in SSA and implement indexed operations with compare/select chains. This keeps ancillary definitions visible to field lowering and masked-write proofs; storing the unmodelled whole word in a private array would require composing unknown bits. A fold after masked-select elimination lowers fields whose consumers become visible only after redundant copies are removed. A whole-word use that the masked-select pass keeps gets the composed word described under the packed pixel ancillary word. Both representations include implicit entry inputs in the inferred register limit and retain unsigned M0, inactive-write and DPP update rules; wave32/wave64 native Vulkan readbacks cover relative reads, writes, split-offset swaps, inactive writes, DPP masks and out-of-range accesses, but console comparison and the performance cost of selecting over this span are unverified. - [s_call_b64](../../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) saves `PC + 4` over `GetShaderBase` and branches to `PC + 4 + signext(SIMM16) * 4`, following [AMD RDNA2 ISA](https://gpuopen.com/download/rdna2-shader-instruction-set-architecture.pdf) sections 3.6.3, 12.2 and 12.3. Forward callees after `s_endpgm` are decoded. Measured on RDNA2 (gfx1036) in wave32 and wave64 with SIMM16 = 3: the link pair receives `PC + 4`, the branch goes to `PC + 4 + signext(SIMM16) * 4`, and `s_setpc_b64` of the link returns to the instruction after the call. Execution on the PS5 (gfx1013) and title compatibility for these bounded scalar calls remain unverified; `s_rfe_b64` still throws in translation. - [BDA span reads](../../../core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp) (shader recompiler) - `read_bda_span` is one selection: one arm calls `load_bda_span`, a `DontInline` function that holds the in-place dword loads and calls nothing, and the other arm is the per-dword byte fallback. Two other shapes of the same reads each hang a group of NVIDIA drivers, where a dispatch never finishes and the device is lost. With the loads emitted in the arm itself, a dispatch whose lanes took both arms hung on an RTX 3060 and an RTX 4060 Laptop GPU (Windows, driver 591.86) and an RTX 5060 (Linux, driver 590.48.01) ([#3291](https://github.com/boykopovar/AnyPS5/issues/3291)). With the loads and the fallback as two selections, the first under the span's condition and the second under its negation, it hung on an RTX 4070 Laptop GPU (Windows, drivers 595.97 and 596.36), an RTX 5080 (Windows, driver 610.62) and an RTX 4090 (Linux, driver 610.57.04) ([#4975](https://github.com/boykopovar/AnyPS5/issues/4975)). The current shape is measured on the RTX 4090 with driver 610.57.04 only, where it does not hang with or without SPIRV-Tools; a shape with the loads in their own `DontInline` function was reported passing on driver 591.86 in #3291. The cause inside the driver is not known. - [SDWA source selects on `v_cvt_f32_ubyte1`, `v_cvt_f32_ubyte2` and `v_cvt_f32_ubyte3`](../../../core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp) (shader recompiler) - byte N is taken from source 0 after `SRC0_SEL` and `SRC0_SEXT` select and extend it, since SDWA supplies source 0 (RDNA 2 ISA guide 6.8 and 13.3.7) and the opcode reads `S0.u[8N+7:8N]` (12.8). Not measured on hardware. - [Flag structurization](../../../core/shader/recompiler/ControlFlow/src/Structurizer.cpp) (shader recompiler) - a graph that every structurizer attempt refuses, irreducible and non-terminating graphs included, is rebuilt with one reach flag per block and one again flag per loop (`structurizeWithFlags`): each block keeps one copy and runs behind a guard on its flag, and each strongly connected component becomes a loop whose single latch repeats it while a back edge set its flag. The output has about 8 times the input's blocks and one route variable per block, and ends in an empty Return that never runs. Only blocks that start the top-level order unconditionally run unguarded, so resource planning also sees the static paths that skip a guarded block and can plan extra descriptor candidates for them. The attempts still run first. Shared merge splitting refuses an attempt once the graph has grown past 16 times its block count at the start of splitting (at least 64 blocks); succeeding attempts grew at most 10.6 times on random graphs of up to 128 blocks, and an attempt that would succeed only beyond the budget leaves the graph to a later attempt or to this construction. Splitting that adds one block per round stays within that budget until its 4 rounds per block run out, so the attempts can still take more than 10 s (9 of 2,000 random graphs of up to 128 blocks, 63 s at most). `APS5_FORCE_STRUCTURIZER_FALLBACK=1` builds every graph this way, for tests - [Scalar buffer descriptors](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) use only the base address, stride and record count for resources accessed exclusively by scalar loads; their fourth word is normalized to zero, following AMD RDNA2 ISA section 7.2.2. Vector and mixed accesses retain descriptor validation. Native Vulkan readbacks cover ignored fourth-word fields on Windows/NVIDIA; their console behavior has not been measured. +- [Unlinked calls](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) - an `s_swappc_b64` whose target is neither resolved in the program nor a fetch-shader call records the `UnlinkedCall` fault with the target and the program counter, and stops the invocation, or continues after the call in programs whose faults do not stop invocations (see `bdaStopsInvocations`); the driver reports it like a BDA fault, naming the target: a dispatch that checks its faults when it completes throws, one whose check runs in a deferred write-back logs it and continues. The hardware would jump to the target, which is not modelled, so a shader with such a call compiles and fails only if it makes the call.