diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp index 5fa2834621..77fc5ac4d0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp @@ -23,6 +23,7 @@ struct DrawProgram { std::shared_ptr snapshot; std::size_t codeOffset = 0; + std::shared_ptr tailContinuation; }; struct DrawDecode { diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp index 51334acc72..82c05c7ff9 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp @@ -87,17 +87,19 @@ struct ShaderSnapshot { std::shared_ptr registeredState; std::shared_ptr> workgroupUse = std::make_shared>(0u); std::shared_ptr> positionVectorUse = std::make_shared>(0u); + std::optional userDataTailRegister; }; std::shared_ptr ReadRawShader(std::uint64_t address); +std::shared_ptr ReadUserDataTailContinuation(std::span code, std::uint32_t userDataBase, std::span userData); std::shared_ptr ProgramSnapshot(const ShaderRegistry& shaders, std::uint64_t address, std::initializer_list types); void NoteRawGraphicsDraw(); bool RawGraphicsDrawn(); bool ProgramUsesWorkgroup(const ShaderSnapshot& snapshot, std::size_t codeOffset); // The PA_CL_VS_OUT_CNTL position export bits a vertex program is compiled with: none for a program // without pos1..pos3 exports, so its artifact does not depend on them. -std::uint32_t PositionExportLayout(const ShaderSnapshot* snapshot, std::size_t codeOffset, std::span code, std::uint32_t paClVsOutCntl); +std::uint32_t PositionExportLayout(const ShaderSnapshot* snapshot, std::size_t codeOffset, std::span code, std::uint32_t paClVsOutCntl, const ShaderSnapshot* continuation = nullptr); bool VertexWorkgroupRequired(const QueueState& queue, const ShaderRegistry& registry); std::shared_ptr PrepareShaderWithDiagnostics(const ShaderRecompiler::RecompileRequest& request, ShaderRecompiler::ShaderPreparationContext* preparation = nullptr); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp index e51868b6aa..418de2eef5 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp @@ -145,7 +145,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe if (program.binary.stage == Stage::Fragment || roles[i] == Role::GeometryBack) return; decodeReads[i].clear(); vertexInfos[i] = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, &decodeReads[i], false, 8u - program.firstUserSgpr); - if (vertexInfos[i]) vertexInfos[i]->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, readRegister(queue.context, 0x207)) | graphics.layerExports; + if (vertexInfos[i]) vertexInfos[i]->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, readRegister(queue.context, 0x207), program.tailContinuation.get()) | graphics.layerExports; }; if (!registerKey) { for (std::size_t i = 0; i < programs.size(); ++i) decodeVertexInfo(i); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp index 7844679fd2..f2ccc7140f 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp @@ -120,7 +120,9 @@ void Driver::eraseDrawEntry(std::uint64_t key, const std::shared_ptr& bool DecodeProgramsCurrent(const DrawDecode& decode, const ShaderRegistry&) { return std::all_of(decode.programs.begin(), decode.programs.end(), [&](const DrawProgram& program) { - return !program.snapshot->header.empty() || ReadRawShader(program.snapshot->codeAddress + program.codeOffset * sizeof(std::uint32_t)) == program.snapshot; + if (!program.snapshot->header.empty()) return true; + return ReadRawShader(program.snapshot->codeAddress + program.codeOffset * sizeof(std::uint32_t)) == program.snapshot && + (program.tailContinuation == nullptr || ReadRawShader(program.tailContinuation->codeAddress) == program.tailContinuation); }); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp index 80b56997d2..101270bdbd 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp @@ -51,6 +51,7 @@ void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, userDataBase + i); result.userData.push_back((staticAbi ? ReadGraphicsRegister(queue.shader, userDataBase + i) : readUserData(queue.shader, userDataBase + i))); } + if (stage == Stage::Vertex && snapshot.header.empty() && snapshot.userDataTailRegister) result.tailContinuation = ReadUserDataTailContinuation(result.binary.code, result.firstUserSgpr, result.userData); return result; }; { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp index 8369224ac7..e7298e1f69 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp @@ -137,7 +137,7 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) { for (std::size_t i = 0; i < a.programs.size(); ++i) { const auto& x = a.programs[i]; const auto& y = b.programs[i]; - if (x.binary.stage != y.binary.stage || x.binary.codeAddress != y.binary.codeAddress || x.userDataBase != y.userDataBase || x.firstUserSgpr != y.firstUserSgpr || x.userData != y.userData || x.snapshot != y.snapshot || x.codeOffset != y.codeOffset) return false; + if (x.binary.stage != y.binary.stage || x.binary.codeAddress != y.binary.codeAddress || x.userDataBase != y.userDataBase || x.firstUserSgpr != y.firstUserSgpr || x.userData != y.userData || x.snapshot != y.snapshot || x.codeOffset != y.codeOffset || x.tailContinuation != y.tailContinuation) return false; } if (p.targetExportPacking != q.targetExportPacking || p.dualSourceBlend != q.dualSourceBlend || p.alphaToCoverage != q.alphaToCoverage || p.rasterPrimitive != q.rasterPrimitive) return false; return true; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index 4ad3671f97..9919737cf3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -91,6 +91,7 @@ std::shared_ptr ReadRawShader(std::uint64_t address) { std::as_writable_bytes(std::span(snapshot.code).subspan(previous)), alignof(std::uint32_t)); try { const auto decoded = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(snapshot.code); + if (const auto tail = ShaderRecompiler::FindUserDataTail(decoded, 0u, 106u)) snapshot.userDataTailRegister = tail->userDataRegister; const auto& last = decoded.instructions.back(); snapshot.code.resize(last.programCounter / sizeof(std::uint32_t) + last.wordCount); auto result = std::make_shared(std::move(snapshot)); @@ -120,6 +121,16 @@ std::shared_ptr ReadRawShader(std::uint64_t address) { throw std::runtime_error("AGC driver: raw shader program has no reachable end within mapped code or the size limit"); } +std::shared_ptr ReadUserDataTailContinuation(std::span code, std::uint32_t userDataBase, std::span userData) { + const auto program = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code); + const auto tail = ShaderRecompiler::FindUserDataTail(program, userDataBase, static_cast(userData.size())); + if (!tail) return nullptr; + const auto slot = tail->userDataRegister - userDataBase; + const auto address = userData[slot] | (static_cast(userData[slot + 1u]) << 32u); + require(address != 0 && (address & 3u) == 0u, "user-data tail continuation address is null or not dword aligned"); + return ReadRawShader(address); +} + std::shared_ptr ProgramSnapshot(const ShaderRegistry& shaders, std::uint64_t address, std::initializer_list types) { if (auto registered = RegisteredProgram(shaders, address, types)) return registered; require(shaders.empty(), "graphics program does not belong to a compatible registered shader"); @@ -168,10 +179,11 @@ bool ProgramUsesWorkgroup(const ShaderSnapshot& snapshot, std::size_t codeOffset return used; } -std::uint32_t PositionExportLayout(const ShaderSnapshot* snapshot, std::size_t codeOffset, std::span code, std::uint32_t paClVsOutCntl) { +std::uint32_t PositionExportLayout(const ShaderSnapshot* snapshot, std::size_t codeOffset, std::span code, std::uint32_t paClVsOutCntl, const ShaderSnapshot* continuation) { constexpr std::uint8_t Unknown = 0, Unused = 1, Used = 2; const auto bits = paClVsOutCntl & Graphics::PositionExportLayoutBits; if (bits == 0) return 0; + if (continuation != nullptr && PositionExportLayout(continuation, 0, continuation->code, bits) != 0) return bits; const bool cached = snapshot != nullptr && codeOffset == 0; if (cached) { const auto use = snapshot->positionVectorUse->load(std::memory_order_acquire); @@ -199,7 +211,22 @@ bool VertexWorkgroupRequired(const QueueState& queue, const ShaderRegistry& regi const auto address = (static_cast(low->second) << 8u) | (static_cast(high->second) << 40u); const auto snapshot = ProgramSnapshot(registry, address, {2}); require((address - snapshot->codeAddress) % sizeof(std::uint32_t) == 0, "graphics entry point is not dword aligned"); - return (snapshot->header.empty() || snapshot->type == 2) && ProgramUsesWorkgroup(*snapshot, (address - snapshot->codeAddress) / sizeof(std::uint32_t)); + const auto offset = (address - snapshot->codeAddress) / sizeof(std::uint32_t); + if (!snapshot->header.empty()) return snapshot->type == 2 && ProgramUsesWorkgroup(*snapshot, offset); + if (!snapshot->userDataTailRegister) return ProgramUsesWorkgroup(*snapshot, offset); + Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, 0x8b); + const auto resources = queue.shader.find(0x8b); + const auto rsrc2 = resources == queue.shader.end() ? 0u : resources->second; + const auto count = ((rsrc2 >> 1u) & 0x1fu) | (((rsrc2 >> 27u) & 1u) << 5u); + require(count <= 32, "graphics user SGPR count exceeds the register bank"); + std::vector userData; + for (std::uint32_t index = 0; index < count; ++index) { + Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, 0x8c + index); + const auto found = queue.shader.find(0x8c + index); + userData.push_back(found == queue.shader.end() ? 0u : found->second); + } + const auto continuation = ReadUserDataTailContinuation(std::span(snapshot->code).subspan(offset), 8u, userData); + return ProgramUsesWorkgroup(*snapshot, offset) || (continuation != nullptr && ProgramUsesWorkgroup(*continuation, 0u)); } struct ShaderPreparationTransaction::State { @@ -458,6 +485,15 @@ ShaderRecompiler::RectListShaders DrawRectangle(const ShaderSnapshot& front, con namespace { +std::vector LinkVertexUserDataTail(const ShaderRecompiler::RecompileRequest& request, const ShaderSnapshot& snapshot) { + if (request.shader.stage != ShaderRecompiler::ShaderStage::Vertex || !request.shader.header.empty() || !snapshot.userDataTailRegister) return {}; + const auto program = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(request.shader.code); + const auto tail = ShaderRecompiler::FindUserDataTail(program, request.context.userDataBaseRegister, static_cast(request.context.userData.size())); + if (!tail) return {}; + const auto continuation = ReadUserDataTailContinuation(request.shader.code, request.context.userDataBaseRegister, request.context.userData); + return ShaderRecompiler::LinkUserDataTail(program, *tail, ShaderRecompiler::RdnaInstructionDecoder{}.Decode(continuation->code)); +} + std::string hexText(std::uint64_t value) { char text[24]; std::snprintf(text, sizeof(text), "0x%llx", static_cast(value)); @@ -512,7 +548,8 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s require(codeOffset < snapshot.code.size(), "prepared shader code offset is outside the snapshot"); const auto code = std::span(snapshot.code).subspan(codeOffset); require(request.shader.code.data() == code.data() && request.shader.code.size() == code.size(), "prepared invocation does not refer to registered code"); - const auto linked = LinkUserDataCallees(request); + auto linked = LinkUserDataCallees(request); + if (linked.empty()) linked = LinkVertexUserDataTail(request, snapshot); auto source = request; if (!linked.empty()) source.shader.code = linked; auto invocationRequest = source; @@ -970,7 +1007,7 @@ std::vector PrepareGraphicsStages(const DrawDecode& decod const auto wave = fragment ? decoded.state.stages.fragmentWaveSize : decoded.state.stages.vertexWaveSize; std::optional vertex; if (!fragment) vertex = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, nullptr, true, 8u - program.firstUserSgpr); - if (vertex) vertex->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, decoded.state.paClVsOutCntl) | decoded.state.layerExports; + if (vertex) vertex->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, decoded.state.paClVsOutCntl, program.tailContinuation.get()) | decoded.state.layerExports; pushOffset = Graphics::StagePushOffset(pushOffset, program.binary.stage, fixedSlots); const auto offset = pushOffset; const auto room = (fixedSlots && fragment ? Graphics::PipelinePushSlotBytes : capacity) - pushOffset % Graphics::PipelinePushSlotBytes; diff --git a/core/libs/prx/libSceAgcDriver/tests/ControlFlow.cpp b/core/libs/prx/libSceAgcDriver/tests/ControlFlow.cpp index a78c75f9bc..46a79cf8de 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ControlFlow.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ControlFlow.cpp @@ -480,6 +480,38 @@ void verifyNullSwappc() { } +void VerifyUserDataTail() { + const RdnaInstructionDecoder decoder; + const std::vector code{0x7e000280u, 0xbe802008u, 0xbf9f0000u}; + const auto decoded = decoder.Decode(code); + const auto tail = FindUserDataTail(decoded, 8u, 2u); + require(tail && tail->programCounter == 4u && tail->userDataRegister == 8u, "unchanged user-data tail was not recognized"); + require(!FindUserDataTail(decoded, 9u, 2u) && !FindUserDataTail(decoded, 8u, 1u), "tail outside user data was accepted"); + const std::vector> refused{ + {0xbe880380u, 0xbe802008u, 0xbf9f0000u}, + {0xbe890380u, 0xbe802008u, 0xbf9f0000u}, + {0xbe880480u, 0xbe802008u, 0xbf9f0000u}, + {0xf4080200u, 0xfa000000u, 0xbe802008u, 0xbf9f0000u}, + {0xbf820000u, 0xbe802008u, 0xbf9f0000u}, + {0xbe802009u, 0xbf9f0000u}, + {0xbe802008u, 0xbf810000u}, + {0xbe802008u, 0xbf800000u, 0xbf9f0000u}, + }; + for (const auto& rejected : refused) require(!FindUserDataTail(decoder.Decode(rejected), 8u, 4u), "a modified, branching or malformed tail was accepted"); + const std::vector continuation{0xbf820001u, 0x7e000281u, 0xbf810000u}; + const auto linked = LinkUserDataTail(decoded, *tail, decoder.Decode(continuation)); + require(linked == std::vector({0x7e000280u, 0xbf820000u, 0xbf820001u, 0x7e000281u, 0xbf810000u}), "linking did not preserve relative branches and register operations"); + static_cast(GraphBuilder{}.Build(decoder.Decode(linked))); + const std::vector readsPc{0xbe8c1f00u, 0xbf810000u}; + bool refusedPc = false; + try { + static_cast(LinkUserDataTail(decoded, *tail, decoder.Decode(readsPc))); + } catch (const std::invalid_argument& error) { + refusedPc = std::string_view(error.what()).find("reads its program counter") != std::string_view::npos; + } + require(refusedPc, "a continuation reading its relocated PC was accepted"); +} + int main(int argc, char** argv) { try { verifyLoopMergeDoesNotEnterNestedSelection(); @@ -1279,6 +1311,7 @@ done: } try { verifyNullSwappc(); + VerifyUserDataTail(); verifyLargeLoopExitFallback(); verifyNestedExitForwarders(); } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp index 21a2806af9..721edbaa26 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp @@ -658,6 +658,39 @@ void testProgramWorkgroupUse() { check(!AgcDriver::DriverDetail::ProgramUsesWorkgroup(entry, 2) && AgcDriver::DriverDetail::ProgramUsesWorkgroup(entry, 0), "the workgroup check did not start at the program's entry"); } +void testVertexTailSnapshots() { + using namespace AgcDriver::DriverDetail; + alignas(256) static std::array prolog{0xbe802008u, 0xbf9f0000u}; + alignas(256) static std::array continuation{0xbf800000u, 0xbf810000u}; + const auto address = reinterpret_cast(prolog.data()); + const auto next = reinterpret_cast(continuation.data()); + const std::array data{static_cast(next), static_cast(next >> 32u)}; + const auto source = ReadRawShader(address); + check(source->userDataTailRegister == 8u, "raw tail metadata is missing"); + const auto target = ReadUserDataTailContinuation(source->code, 8u, data); + check(target && target->codeAddress == next, "tail did not read its user-data continuation"); + DrawDecode decode{}; + decode.programs.resize(1); + decode.programs[0].snapshot = source; + decode.programs[0].tailContinuation = target; + check(DecodeProgramsCurrent(decode, {}), "unchanged tail snapshot became stale"); + continuation[0] = 0xbf8a0000u; + check(!DecodeProgramsCurrent(decode, {}), "rewriting the continuation did not invalidate the draw cache"); + AgcDriver::QueueState queue{}; + queue.context[0x2d5] = 0x2000u; + queue.shader = {{0xc8, static_cast(address >> 8u)}, {0xc9, static_cast(address >> 40u)}, + {0x8b, 4u}, {0x8c, data[0]}, {0x8d, data[1]}}; + check(VertexWorkgroupRequired(queue, {}), "a barrier in the continuation was hidden from workgroup routing"); + queue.context[0x2d5] |= 0x02000000u; + check(!VertexWorkgroupRequired(queue, {}), "a passthrough program was rerouted to a workgroup"); + const std::array nullAddress{}; + check(expectFailure([&] { static_cast(ReadUserDataTailContinuation(source->code, 8u, nullAddress)); }).find("address is null") != std::string::npos, "null tail address was accepted"); + const std::array unaligned{data[0] + 1u, data[1]}; + check(expectFailure([&] { static_cast(ReadUserDataTailContinuation(source->code, 8u, unaligned)); }).find("dword aligned") != std::string::npos, "unaligned tail address was accepted"); + const std::array unmapped{0x1000u, 0u}; + check(!expectFailure([&] { static_cast(ReadUserDataTailContinuation(source->code, 8u, unmapped)); }).empty(), "unmapped tail address was accepted"); +} + void testRegisteredVertexWorkgroupRouting() { using namespace AgcDriver::DriverDetail; ShaderSnapshot snapshot{0x20000, 0x30000, 2, {0xd8340000u, 0x00000100u, 0xbf810000u, 0xbf810000u}, {}}; @@ -781,6 +814,7 @@ int main(int argc, char** argv) { #endif testProgramSnapshots(); testProgramWorkgroupUse(); + testVertexTailSnapshots(); testRegisteredVertexWorkgroupRouting(); testEvents(); testValidation(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/VertexUserDataTail.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/VertexUserDataTail.cpp new file mode 100644 index 0000000000..939a1722d3 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/VertexUserDataTail.cpp @@ -0,0 +1,142 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +constexpr std::byte Kept{0x40}; +using Image = std::array; +alignas(256) Image Pixels{}; +alignas(256) std::array Prolog{}; +alignas(256) std::array Reference{}; +alignas(256) std::array Continuation{0xbf840003u, 0x7e08020cu, 0xf80008cfu, 0x04030201u, 0xbf810000u}; +alignas(256) const std::array OtherContinuation{0xbf840003u, 0x7e0802f4u, 0xf80008cfu, 0x04030201u, 0xbf810000u}; +alignas(256) constexpr std::array PixelCode{0x7e000200u, 0x7e020280u, 0xf800180fu, 0x00010100u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u}; + +void Append(std::vector& commands, std::uint32_t opcode, std::initializer_list payload) { + commands.push_back(0xc0000000u | (static_cast(payload.size() - 1) << 16u) | (opcode << 8u)); + commands.insert(commands.end(), payload); +} + +Image Draw(const std::uint32_t* code, const std::uint32_t* continuation, std::uint32_t distanceLayout = 0) { + Pixels.fill(Kept); + const auto vertex = reinterpret_cast(code); + const auto pixel = reinterpret_cast(PixelCode.data()); + const auto next = reinterpret_cast(continuation); + const std::vector> context{ + {0x2d5, 0x2000}, + {0x1b6, 0}, {0x207, distanceLayout}, {0x200, 0}, {0x203, 0x800}, + {0x2dc, 0xaa00}, {0x2f8, 0}, {0x292, 2}, {0x293, 0}, + {0x80, 0}, {0x8d, 0}, {0x83, 0xffff}, {0x8c, 0xa}, + {0x2f9, 0x2d}, {0x313, 0x6000}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff}, + {0x206, 0x43f}, {0x204, 0x80000}, {0x205, 0x240}, + {0x8e, 0xf}, {0x8f, 0xf}, {0x202, 0xcc0010}, + {0x1c4, 0}, {0x1c5, 9}, {0x1c3, 4}, {0x31c, 0x28028}, + {0x31b, 0}, {0x31d, 0}, {0x3b0, (63u << 14u) | 3u}, + {0x3b8, 0x9000000}, {0x1e0, 0}, + {0xc, 0}, {0xd, 0x40040}, + {0x81, 0x80000000}, {0x82, 0x40040}, + {0x90, 0x80000000}, {0x91, 0x40040}, + {0x94, 0x80000000}, {0x95, 0x40040}, + {0x10f, std::bit_cast(32.0f)}, {0x110, std::bit_cast(32.0f)}, + {0x111, std::bit_cast(-2.0f)}, {0x112, std::bit_cast(2.0f)}, + {0x113, std::bit_cast(1.0f)}, {0x114, 0}, + {0xb4, 0}, {0xb5, std::bit_cast(1.0f)}, + {0x2fe, 0}, {0x302, 0}, {0x306, 0}, {0x30a, 0}, + }; + const std::vector> shader{ + {0xc8, static_cast(vertex >> 8u)}, {0xc9, static_cast(vertex >> 40u)}, {0x8b, 2u << 1u}, + {0x8c, static_cast(next)}, {0x8d, static_cast(next >> 32u)}, + {0x008, static_cast(pixel >> 8u)}, {0x009, static_cast(pixel >> 40u)}, {0x00b, 1u << 1u}, {0x00c, std::bit_cast(1.0f)}, + }; + std::vector commands; + Append(commands, 0x79, {0x242, 4}); + for (const auto& [offset, value] : context) Append(commands, 0x69, {offset, value}); + for (const auto& [offset, value] : shader) Append(commands, 0x76, {offset, value}); + const auto color = reinterpret_cast(Pixels.data()); + Append(commands, 0x69, {0x318, static_cast(color >> 8u)}); + Append(commands, 0x69, {0x390, static_cast(color >> 40u)}); + Append(commands, 0x2d, {3, 2}); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + Require(sceAgcDriverSubmitDcb(&packet) == 0, "vertex tail draw submission failed"); + AgcDriverWaitIdle_nid_postfix(); + AgcDriver::GuestMemory::FlushGpuWrites(color, Pixels.size()); + return Pixels; +} + +Image Check(std::span prefix, std::span continuation, std::uint32_t distanceLayout = 0, std::size_t continuationWords = 5) { + Prolog.fill(0xbf9f0000u); + Reference.fill(0xbf9f0000u); + std::copy(prefix.begin(), prefix.end(), Prolog.begin()); + Prolog[prefix.size()] = 0xbe802008u; + std::copy(prefix.begin(), prefix.end(), Reference.begin()); + std::copy_n(continuation.begin(), continuationWords, Reference.begin() + prefix.size()); + const auto expected = Draw(Reference.data(), continuation.data(), distanceLayout); + for (unsigned repeat = 0; repeat < 2u; ++repeat) { + const auto actual = Draw(Prolog.data(), continuation.data(), distanceLayout); + Require(actual == expected, "linked vertex tail pixels differ from the monolithic shader"); + } + return expected; +} + +void VertexTailDraws() { + std::vector prefix{ + 0x7e000d05u, 0x080200f4u, 0x10020300u, 0x100202f4u, 0x060202f3u, 0x060400f3u, 0x10040500u, 0x060404f3u, + 0x7e060280u, 0x7e0802f2u, 0xbe8c03ffu, 0x3f800000u, 0xbf060c0cu, + }; + const auto original = Check(prefix, Continuation); + const auto kept = std::count(original.begin(), original.end(), Kept); + Require(kept != 0 && kept != static_cast(original.size()), "reference triangle did not cross the render target"); + const auto changedAddress = Check(prefix, OtherContinuation); + Require(changedAddress != original, "changing the continuation did not change the reference pixels"); + Continuation[1] = OtherContinuation[1]; + Require(Check(prefix, Continuation) == changedAddress, "rewritten continuation kept the old pixels"); + Continuation[1] = 0x7e08020cu; + Require(Check(prefix, Continuation) == original, "restored continuation did not restore the pixels"); + alignas(256) const std::array clipContinuation{0xbf840005u, 0x7e08020cu, 0xf80000cfu, 0x04030201u, 0xf80008dfu, 0x01010101u, 0xbf810000u}; + const auto clipped = Check(prefix, clipContinuation, 0x00400001u, clipContinuation.size()); + Require(clipped != original && std::count(clipped.begin(), clipped.end(), Kept) > kept, "a clip distance exported by the continuation did not clip the triangle"); + const auto culled = Check(prefix, clipContinuation, 0x00400100u, clipContinuation.size()); + Require(culled == original, "a mixed-sign cull distance discarded the triangle"); + prefix.back() = 0xbf070c0cu; + const auto falseScc = Check(prefix, Continuation); + Require(std::ranges::all_of(falseScc, [](std::byte value) { return value == Kept; }), "SCC was not preserved across the tail jump"); + prefix.back() = 0xbf060c0cu; + prefix.push_back(0xbefe0480u); + const auto emptyExec = Check(prefix, Continuation); + Require(std::ranges::all_of(emptyExec, [](std::byte value) { return value == Kept; }), "EXEC was not preserved across the tail jump"); +} + +} + +int main() { + try { + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + VertexTailDraws(); + AgcDriverShutdown_nid_postfix(); + std::puts("vertex user-data tail tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + try { AgcDriverShutdown_nid_postfix(); } catch (const std::exception&) {} + return 1; + } +} diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index d4ff8cefaa..b3e04b8339 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -246,6 +246,8 @@ function(add_shader_recompiler target runtimeTarget) ShaderCacheDirectory SrgbDecodeBit SrgbUnormFormat + FindUserDataTail + LinkUserDataTail ThreadOwned UnresolvableSwappcTarget ValidateAndOptimizeSpirv diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp index ada6f1084e..0b7cb0fb7d 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp @@ -10,6 +10,14 @@ namespace ShaderRecompiler { +struct UserDataTail { + std::uint32_t programCounter; + std::uint32_t userDataRegister; +}; + +[[nodiscard]] std::optional FindUserDataTail(const RdnaProgram& program, std::uint32_t userDataBase, std::uint32_t userDataCount); +[[nodiscard]] std::vector LinkUserDataTail(const RdnaProgram& program, const UserDataTail& tail, const RdnaProgram& continuation); + struct SwappcInfo { bool fetchCallAllowed = false; std::uint32_t userDataBaseRegister = 0; diff --git a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp index 3b623565eb..48bdd33cd0 100644 --- a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp +++ b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp @@ -1011,6 +1011,34 @@ std::vector FindUserDataCalls(const RdnaProgram& program, const Sw return calls; } +std::optional FindUserDataTail(const RdnaProgram& program, std::uint32_t userDataBase, std::uint32_t userDataCount) { + if (program.instructions.size() < 2u || program.instructions.back().op != RdnaOpcode::SCodeEnd) return std::nullopt; + const auto& tail = program.instructions[program.instructions.size() - 2u]; + if (tail.op != RdnaOpcode::SSetpcB64 || tail.source0.kind != RdnaOperandKind::ScalarRegister) return std::nullopt; + const auto reg = tail.source0.reg; + if (reg % 2u != 0u || reg < userDataBase || userDataCount < 2u || reg - userDataBase >= userDataCount - 1u) return std::nullopt; + for (const auto& instruction : program.instructions) { + if (instruction.programCounter >= tail.programCounter) break; + if (IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SCallB64 || instruction.op == RdnaOpcode::SSwappcB64 || + instruction.op == RdnaOpcode::SSetpcB64 || instruction.op == RdnaOpcode::SEndpgm || instruction.op == RdnaOpcode::SCodeEnd || + writesScalar(instruction, reg) || writesScalar(instruction, reg + 1u)) return std::nullopt; + } + return UserDataTail{tail.programCounter, reg}; +} + +std::vector LinkUserDataTail(const RdnaProgram& program, const UserDataTail& tail, const RdnaProgram& continuation) { + const auto recognized = FindUserDataTail(program, tail.userDataRegister, 2u); + if (!recognized || recognized->programCounter != tail.programCounter) throw std::invalid_argument("user-data tail is not a linear jump through an unchanged register pair"); + for (const auto& instruction : continuation.instructions) { + if (instruction.op == RdnaOpcode::SGetpcB64) throw std::invalid_argument("user-data tail continuation reads its program counter, which a linked copy does not keep"); + } + const auto words = tail.programCounter / 4u; + std::vector linked(program.code.begin(), program.code.begin() + words); + linked.push_back(0xbf820000u); + linked.insert(linked.end(), continuation.code.begin(), continuation.code.end()); + return linked; +} + std::optional UserDataCalleeLength(std::span callee, std::uint32_t linkRegister) { std::uint32_t furthest = 0; for (std::uint32_t word = 0; word < callee.size();) { diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index de34303b91..5c91ef1580 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -18,6 +18,7 @@ ### Functional - [Cached source IR](../../../core/shader/recompiler/Recompiler.cpp) releases lazily allocated SSA construction tables after resource preparation and reclaims values unreachable from instructions, metadata and resource plans when retaining a pending source program. Immediate shader preparation skips reclamation because compilation consumes the program. Reclamation must account for every new IR pointer owner; a remaining external use throws. Live IDs are retained. Source and variant entries remain cached without an eviction bound. A pending source program is released after its first variant is obtained; memory beyond the measured title checkpoint and Linux performance remain unmeasured. +- [Vertex user-data tail jumps](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) link a linear raw vertex prolog's terminal `s_setpc_b64` through an unchanged user-data pair to its continuation. The instruction semantics follow AMD RDNA2 ISA section 12.3; the prolog convention follows Mesa `aco_select_vs_prolog.cpp` at `0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8`. Console hardware has not been measured. Computed pointers, branching prologs, fragment epilogs and continuations reading `s_getpc_b64` remain unsupported - The [resource tracker](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) plans a [descriptor source](../../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) used only behind a conditional branch as inactive when the branch condition reads only clean (specialization) memory and so is evaluable at capture time; an inactive source is bound as zero words and not validated. Other conditions (EXEC, VCC, SCC set by vector or non-clean data, loop headers) leave every source active, as before, and a branch on a value that changes between dispatches is a specialization key like any other clean slot. PPSA15246 has a copy kernel that selects a buffer or an image path from an SRT word and reads the same address as a V# or a T#; before, the untaken path's descriptor was validated and threw. Whether the console would also accept an invalid descriptor in a path it does not take is assumed, not measured - [MaskedSelectEliminator](../../../core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp) can discard inactive source lanes through `ReadFirstLane` only when its read mask implies the write mask and its result is unobserved outside the read mask. A loop-local read mask requires result uses in the same block without phi transport. The empty-EXEC lane-zero fallback follows AMD's [Vega ISA, section 12.8, V_READFIRSTLANE_B32, page 127](https://www.amd.com/content/dam/amd/en/documents/radeon-tech-docs/instruction-set-architectures/vega-shader-instruction-set-architecture.pdf) and the existing SPIR-V emitter. The optimization is covered by synthetic IR tests but has not been compared with console hardware; a proof that needs a read mask of a kind that is already being solved is refused.