feat(shader): link raw vertex prologs through user-data tails

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp

@@ -23,6 +23,7 @@ struct DrawProgram {
 
     std::shared_ptr<const ShaderSnapshot> snapshot;
     std::size_t codeOffset = 0;
+    std::shared_ptr<const ShaderSnapshot> tailContinuation;
 };
 
 struct DrawDecode {

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp

@@ -87,17 +87,19 @@ struct ShaderSnapshot {
     std::shared_ptr<const RegisteredShaderState> registeredState;
     std::shared_ptr<std::atomic<std::uint8_t>> workgroupUse = std::make_shared<std::atomic<std::uint8_t>>(0u);
     std::shared_ptr<std::atomic<std::uint8_t>> positionVectorUse = std::make_shared<std::atomic<std::uint8_t>>(0u);
+    std::optional<std::uint32_t> userDataTailRegister;
 };
 
 
 std::shared_ptr<const ShaderSnapshot> ReadRawShader(std::uint64_t address);
+std::shared_ptr<const ShaderSnapshot> ReadUserDataTailContinuation(std::span<const std::uint32_t> code, std::uint32_t userDataBase, std::span<const std::uint32_t> userData);
 std::shared_ptr<const ShaderSnapshot> ProgramSnapshot(const ShaderRegistry& shaders, std::uint64_t address, std::initializer_list<std::uint8_t> types);
 void NoteRawGraphicsDraw();
 bool RawGraphicsDrawn();
 bool ProgramUsesWorkgroup(const ShaderSnapshot& snapshot, std::size_t codeOffset);
 // The PA_CL_VS_OUT_CNTL position export bits a vertex program is compiled with: none for a program
 // without pos1..pos3 exports, so its artifact does not depend on them.
-std::uint32_t PositionExportLayout(const ShaderSnapshot* snapshot, std::size_t codeOffset, std::span<const std::uint32_t> code, std::uint32_t paClVsOutCntl);
+std::uint32_t PositionExportLayout(const ShaderSnapshot* snapshot, std::size_t codeOffset, std::span<const std::uint32_t> code, std::uint32_t paClVsOutCntl, const ShaderSnapshot* continuation = nullptr);
 bool VertexWorkgroupRequired(const QueueState& queue, const ShaderRegistry& registry);
 
 std::shared_ptr<const ShaderRecompiler::SourceHandle> PrepareShaderWithDiagnostics(const ShaderRecompiler::RecompileRequest& request, ShaderRecompiler::ShaderPreparationContext* preparation = nullptr);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp

@@ -145,7 +145,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         if (program.binary.stage == Stage::Fragment || roles[i] == Role::GeometryBack) return;
         decodeReads[i].clear();
         vertexInfos[i] = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, &decodeReads[i], false, 8u - program.firstUserSgpr);
-        if (vertexInfos[i]) vertexInfos[i]->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, readRegister(queue.context, 0x207)) | graphics.layerExports;
+        if (vertexInfos[i]) vertexInfos[i]->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, readRegister(queue.context, 0x207), program.tailContinuation.get()) | graphics.layerExports;
     };
     if (!registerKey) {
         for (std::size_t i = 0; i < programs.size(); ++i) decodeVertexInfo(i);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp

@@ -120,7 +120,9 @@ void Driver::eraseDrawEntry(std::uint64_t key, const std::shared_ptr<DrawEntry>&
 
 bool DecodeProgramsCurrent(const DrawDecode& decode, const ShaderRegistry&) {
     return std::all_of(decode.programs.begin(), decode.programs.end(), [&](const DrawProgram& program) {
-        return !program.snapshot->header.empty() || ReadRawShader(program.snapshot->codeAddress + program.codeOffset * sizeof(std::uint32_t)) == program.snapshot;
+        if (!program.snapshot->header.empty()) return true;
+        return ReadRawShader(program.snapshot->codeAddress + program.codeOffset * sizeof(std::uint32_t)) == program.snapshot &&
+            (program.tailContinuation == nullptr || ReadRawShader(program.tailContinuation->codeAddress) == program.tailContinuation);
     });
 }
 

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp

@@ -51,6 +51,7 @@ void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const
             Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, userDataBase + i);
             result.userData.push_back((staticAbi ? ReadGraphicsRegister(queue.shader, userDataBase + i) : readUserData(queue.shader, userDataBase + i)));
         }
+        if (stage == Stage::Vertex && snapshot.header.empty() && snapshot.userDataTailRegister) result.tailContinuation = ReadUserDataTailContinuation(result.binary.code, result.firstUserSgpr, result.userData);
         return result;
     };
     {

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp

@@ -137,7 +137,7 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) {
     for (std::size_t i = 0; i < a.programs.size(); ++i) {
         const auto& x = a.programs[i];
         const auto& y = b.programs[i];
-        if (x.binary.stage != y.binary.stage || x.binary.codeAddress != y.binary.codeAddress || x.userDataBase != y.userDataBase || x.firstUserSgpr != y.firstUserSgpr || x.userData != y.userData || x.snapshot != y.snapshot || x.codeOffset != y.codeOffset) return false;
+        if (x.binary.stage != y.binary.stage || x.binary.codeAddress != y.binary.codeAddress || x.userDataBase != y.userDataBase || x.firstUserSgpr != y.firstUserSgpr || x.userData != y.userData || x.snapshot != y.snapshot || x.codeOffset != y.codeOffset || x.tailContinuation != y.tailContinuation) return false;
     }
     if (p.targetExportPacking != q.targetExportPacking || p.dualSourceBlend != q.dualSourceBlend || p.alphaToCoverage != q.alphaToCoverage || p.rasterPrimitive != q.rasterPrimitive) return false;
     return true;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp

@@ -91,6 +91,7 @@ std::shared_ptr<const ShaderSnapshot> ReadRawShader(std::uint64_t address) {
             std::as_writable_bytes(std::span(snapshot.code).subspan(previous)), alignof(std::uint32_t));
         try {
             const auto decoded = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(snapshot.code);
+            if (const auto tail = ShaderRecompiler::FindUserDataTail(decoded, 0u, 106u)) snapshot.userDataTailRegister = tail->userDataRegister;
             const auto& last = decoded.instructions.back();
             snapshot.code.resize(last.programCounter / sizeof(std::uint32_t) + last.wordCount);
             auto result = std::make_shared<const ShaderSnapshot>(std::move(snapshot));
@@ -120,6 +121,16 @@ std::shared_ptr<const ShaderSnapshot> ReadRawShader(std::uint64_t address) {
     throw std::runtime_error("AGC driver: raw shader program has no reachable end within mapped code or the size limit");
 }
 
+std::shared_ptr<const ShaderSnapshot> ReadUserDataTailContinuation(std::span<const std::uint32_t> code, std::uint32_t userDataBase, std::span<const std::uint32_t> userData) {
+    const auto program = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code);
+    const auto tail = ShaderRecompiler::FindUserDataTail(program, userDataBase, static_cast<std::uint32_t>(userData.size()));
+    if (!tail) return nullptr;
+    const auto slot = tail->userDataRegister - userDataBase;
+    const auto address = userData[slot] | (static_cast<std::uint64_t>(userData[slot + 1u]) << 32u);
+    require(address != 0 && (address & 3u) == 0u, "user-data tail continuation address is null or not dword aligned");
+    return ReadRawShader(address);
+}
+
 std::shared_ptr<const ShaderSnapshot> ProgramSnapshot(const ShaderRegistry& shaders, std::uint64_t address, std::initializer_list<std::uint8_t> types) {
     if (auto registered = RegisteredProgram(shaders, address, types)) return registered;
     require(shaders.empty(), "graphics program does not belong to a compatible registered shader");
@@ -168,10 +179,11 @@ bool ProgramUsesWorkgroup(const ShaderSnapshot& snapshot, std::size_t codeOffset
     return used;
 }
 
-std::uint32_t PositionExportLayout(const ShaderSnapshot* snapshot, std::size_t codeOffset, std::span<const std::uint32_t> code, std::uint32_t paClVsOutCntl) {
+std::uint32_t PositionExportLayout(const ShaderSnapshot* snapshot, std::size_t codeOffset, std::span<const std::uint32_t> code, std::uint32_t paClVsOutCntl, const ShaderSnapshot* continuation) {
     constexpr std::uint8_t Unknown = 0, Unused = 1, Used = 2;
     const auto bits = paClVsOutCntl & Graphics::PositionExportLayoutBits;
     if (bits == 0) return 0;
+    if (continuation != nullptr && PositionExportLayout(continuation, 0, continuation->code, bits) != 0) return bits;
     const bool cached = snapshot != nullptr && codeOffset == 0;
     if (cached) {
         const auto use = snapshot->positionVectorUse->load(std::memory_order_acquire);
@@ -199,7 +211,22 @@ bool VertexWorkgroupRequired(const QueueState& queue, const ShaderRegistry& regi
     const auto address = (static_cast<std::uint64_t>(low->second) << 8u) | (static_cast<std::uint64_t>(high->second) << 40u);
     const auto snapshot = ProgramSnapshot(registry, address, {2});
     require((address - snapshot->codeAddress) % sizeof(std::uint32_t) == 0, "graphics entry point is not dword aligned");
-    return (snapshot->header.empty() || snapshot->type == 2) && ProgramUsesWorkgroup(*snapshot, (address - snapshot->codeAddress) / sizeof(std::uint32_t));
+    const auto offset = (address - snapshot->codeAddress) / sizeof(std::uint32_t);
+    if (!snapshot->header.empty()) return snapshot->type == 2 && ProgramUsesWorkgroup(*snapshot, offset);
+    if (!snapshot->userDataTailRegister) return ProgramUsesWorkgroup(*snapshot, offset);
+    Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, 0x8b);
+    const auto resources = queue.shader.find(0x8b);
+    const auto rsrc2 = resources == queue.shader.end() ? 0u : resources->second;
+    const auto count = ((rsrc2 >> 1u) & 0x1fu) | (((rsrc2 >> 27u) & 1u) << 5u);
+    require(count <= 32, "graphics user SGPR count exceeds the register bank");
+    std::vector<std::uint32_t> userData;
+    for (std::uint32_t index = 0; index < count; ++index) {
+        Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, 0x8c + index);
+        const auto found = queue.shader.find(0x8c + index);
+        userData.push_back(found == queue.shader.end() ? 0u : found->second);
+    }
+    const auto continuation = ReadUserDataTailContinuation(std::span(snapshot->code).subspan(offset), 8u, userData);
+    return ProgramUsesWorkgroup(*snapshot, offset) || (continuation != nullptr && ProgramUsesWorkgroup(*continuation, 0u));
 }
 
 struct ShaderPreparationTransaction::State {
@@ -458,6 +485,15 @@ ShaderRecompiler::RectListShaders DrawRectangle(const ShaderSnapshot& front, con
 
 namespace {
 
+std::vector<std::uint32_t> LinkVertexUserDataTail(const ShaderRecompiler::RecompileRequest& request, const ShaderSnapshot& snapshot) {
+    if (request.shader.stage != ShaderRecompiler::ShaderStage::Vertex || !request.shader.header.empty() || !snapshot.userDataTailRegister) return {};
+    const auto program = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(request.shader.code);
+    const auto tail = ShaderRecompiler::FindUserDataTail(program, request.context.userDataBaseRegister, static_cast<std::uint32_t>(request.context.userData.size()));
+    if (!tail) return {};
+    const auto continuation = ReadUserDataTailContinuation(request.shader.code, request.context.userDataBaseRegister, request.context.userData);
+    return ShaderRecompiler::LinkUserDataTail(program, *tail, ShaderRecompiler::RdnaInstructionDecoder{}.Decode(continuation->code));
+}
+
 std::string hexText(std::uint64_t value) {
     char text[24];
     std::snprintf(text, sizeof(text), "0x%llx", static_cast<unsigned long long>(value));
@@ -512,7 +548,8 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s
     require(codeOffset < snapshot.code.size(), "prepared shader code offset is outside the snapshot");
     const auto code = std::span(snapshot.code).subspan(codeOffset);
     require(request.shader.code.data() == code.data() && request.shader.code.size() == code.size(), "prepared invocation does not refer to registered code");
-    const auto linked = LinkUserDataCallees(request);
+    auto linked = LinkUserDataCallees(request);
+    if (linked.empty()) linked = LinkVertexUserDataTail(request, snapshot);
     auto source = request;
     if (!linked.empty()) source.shader.code = linked;
     auto invocationRequest = source;
@@ -970,7 +1007,7 @@ std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decod
         const auto wave = fragment ? decoded.state.stages.fragmentWaveSize : decoded.state.stages.vertexWaveSize;
         std::optional<ShaderRecompiler::ShaderVertexStageInfo> vertex;
         if (!fragment) vertex = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, nullptr, true, 8u - program.firstUserSgpr);
-        if (vertex) vertex->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, decoded.state.paClVsOutCntl) | decoded.state.layerExports;
+        if (vertex) vertex->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, decoded.state.paClVsOutCntl, program.tailContinuation.get()) | decoded.state.layerExports;
         pushOffset = Graphics::StagePushOffset(pushOffset, program.binary.stage, fixedSlots);
         const auto offset = pushOffset;
         const auto room = (fixedSlots && fragment ? Graphics::PipelinePushSlotBytes : capacity) - pushOffset % Graphics::PipelinePushSlotBytes;

core/libs/prx/libSceAgcDriver/tests/ControlFlow.cpp

@@ -480,6 +480,38 @@ void verifyNullSwappc() {
 
 }
 
+void VerifyUserDataTail() {
+    const RdnaInstructionDecoder decoder;
+    const std::vector<std::uint32_t> code{0x7e000280u, 0xbe802008u, 0xbf9f0000u};
+    const auto decoded = decoder.Decode(code);
+    const auto tail = FindUserDataTail(decoded, 8u, 2u);
+    require(tail && tail->programCounter == 4u && tail->userDataRegister == 8u, "unchanged user-data tail was not recognized");
+    require(!FindUserDataTail(decoded, 9u, 2u) && !FindUserDataTail(decoded, 8u, 1u), "tail outside user data was accepted");
+    const std::vector<std::vector<std::uint32_t>> refused{
+        {0xbe880380u, 0xbe802008u, 0xbf9f0000u},
+        {0xbe890380u, 0xbe802008u, 0xbf9f0000u},
+        {0xbe880480u, 0xbe802008u, 0xbf9f0000u},
+        {0xf4080200u, 0xfa000000u, 0xbe802008u, 0xbf9f0000u},
+        {0xbf820000u, 0xbe802008u, 0xbf9f0000u},
+        {0xbe802009u, 0xbf9f0000u},
+        {0xbe802008u, 0xbf810000u},
+        {0xbe802008u, 0xbf800000u, 0xbf9f0000u},
+    };
+    for (const auto& rejected : refused) require(!FindUserDataTail(decoder.Decode(rejected), 8u, 4u), "a modified, branching or malformed tail was accepted");
+    const std::vector<std::uint32_t> continuation{0xbf820001u, 0x7e000281u, 0xbf810000u};
+    const auto linked = LinkUserDataTail(decoded, *tail, decoder.Decode(continuation));
+    require(linked == std::vector<std::uint32_t>({0x7e000280u, 0xbf820000u, 0xbf820001u, 0x7e000281u, 0xbf810000u}), "linking did not preserve relative branches and register operations");
+    static_cast<void>(GraphBuilder{}.Build(decoder.Decode(linked)));
+    const std::vector<std::uint32_t> readsPc{0xbe8c1f00u, 0xbf810000u};
+    bool refusedPc = false;
+    try {
+        static_cast<void>(LinkUserDataTail(decoded, *tail, decoder.Decode(readsPc)));
+    } catch (const std::invalid_argument& error) {
+        refusedPc = std::string_view(error.what()).find("reads its program counter") != std::string_view::npos;
+    }
+    require(refusedPc, "a continuation reading its relocated PC was accepted");
+}
+
 int main(int argc, char** argv) {
     try {
         verifyLoopMergeDoesNotEnterNestedSelection();
@@ -1279,6 +1311,7 @@ done:
     }
     try {
         verifyNullSwappc();
+        VerifyUserDataTail();
         verifyLargeLoopExitFallback();
         verifyNestedExitForwarders();
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/Submit.cpp

@@ -658,6 +658,39 @@ void testProgramWorkgroupUse() {
     check(!AgcDriver::DriverDetail::ProgramUsesWorkgroup(entry, 2) && AgcDriver::DriverDetail::ProgramUsesWorkgroup(entry, 0), "the workgroup check did not start at the program's entry");
 }
 
+void testVertexTailSnapshots() {
+    using namespace AgcDriver::DriverDetail;
+    alignas(256) static std::array<std::uint32_t, 64> prolog{0xbe802008u, 0xbf9f0000u};
+    alignas(256) static std::array<std::uint32_t, 64> continuation{0xbf800000u, 0xbf810000u};
+    const auto address = reinterpret_cast<std::uintptr_t>(prolog.data());
+    const auto next = reinterpret_cast<std::uintptr_t>(continuation.data());
+    const std::array<std::uint32_t, 2> data{static_cast<std::uint32_t>(next), static_cast<std::uint32_t>(next >> 32u)};
+    const auto source = ReadRawShader(address);
+    check(source->userDataTailRegister == 8u, "raw tail metadata is missing");
+    const auto target = ReadUserDataTailContinuation(source->code, 8u, data);
+    check(target && target->codeAddress == next, "tail did not read its user-data continuation");
+    DrawDecode decode{};
+    decode.programs.resize(1);
+    decode.programs[0].snapshot = source;
+    decode.programs[0].tailContinuation = target;
+    check(DecodeProgramsCurrent(decode, {}), "unchanged tail snapshot became stale");
+    continuation[0] = 0xbf8a0000u;
+    check(!DecodeProgramsCurrent(decode, {}), "rewriting the continuation did not invalidate the draw cache");
+    AgcDriver::QueueState queue{};
+    queue.context[0x2d5] = 0x2000u;
+    queue.shader = {{0xc8, static_cast<std::uint32_t>(address >> 8u)}, {0xc9, static_cast<std::uint32_t>(address >> 40u)},
+        {0x8b, 4u}, {0x8c, data[0]}, {0x8d, data[1]}};
+    check(VertexWorkgroupRequired(queue, {}), "a barrier in the continuation was hidden from workgroup routing");
+    queue.context[0x2d5] |= 0x02000000u;
+    check(!VertexWorkgroupRequired(queue, {}), "a passthrough program was rerouted to a workgroup");
+    const std::array<std::uint32_t, 2> nullAddress{};
+    check(expectFailure([&] { static_cast<void>(ReadUserDataTailContinuation(source->code, 8u, nullAddress)); }).find("address is null") != std::string::npos, "null tail address was accepted");
+    const std::array<std::uint32_t, 2> unaligned{data[0] + 1u, data[1]};
+    check(expectFailure([&] { static_cast<void>(ReadUserDataTailContinuation(source->code, 8u, unaligned)); }).find("dword aligned") != std::string::npos, "unaligned tail address was accepted");
+    const std::array<std::uint32_t, 2> unmapped{0x1000u, 0u};
+    check(!expectFailure([&] { static_cast<void>(ReadUserDataTailContinuation(source->code, 8u, unmapped)); }).empty(), "unmapped tail address was accepted");
+}
+
 void testRegisteredVertexWorkgroupRouting() {
     using namespace AgcDriver::DriverDetail;
     ShaderSnapshot snapshot{0x20000, 0x30000, 2, {0xd8340000u, 0x00000100u, 0xbf810000u, 0xbf810000u}, {}};
@@ -781,6 +814,7 @@ int main(int argc, char** argv) {
 #endif
         testProgramSnapshots();
         testProgramWorkgroupUse();
+        testVertexTailSnapshots();
         testRegisteredVertexWorkgroupRouting();
         testEvents();
         testValidation();

core/libs/prx/libSceAgcDriver/tests/execution/VertexUserDataTail.cpp

@@ -0,0 +1,142 @@
+#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <exception>
+#include <initializer_list>
+#include <iostream>
+#include <span>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+constexpr std::byte Kept{0x40};
+using Image = std::array<std::byte, 1024>;
+alignas(256) Image Pixels{};
+alignas(256) std::array<std::uint32_t, 64> Prolog{};
+alignas(256) std::array<std::uint32_t, 64> Reference{};
+alignas(256) std::array<std::uint32_t, 64> Continuation{0xbf840003u, 0x7e08020cu, 0xf80008cfu, 0x04030201u, 0xbf810000u};
+alignas(256) const std::array<std::uint32_t, 64> OtherContinuation{0xbf840003u, 0x7e0802f4u, 0xf80008cfu, 0x04030201u, 0xbf810000u};
+alignas(256) constexpr std::array<std::uint32_t, 8> PixelCode{0x7e000200u, 0x7e020280u, 0xf800180fu, 0x00010100u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u};
+
+void Append(std::vector<std::uint32_t>& commands, std::uint32_t opcode, std::initializer_list<std::uint32_t> payload) {
+    commands.push_back(0xc0000000u | (static_cast<std::uint32_t>(payload.size() - 1) << 16u) | (opcode << 8u));
+    commands.insert(commands.end(), payload);
+}
+
+Image Draw(const std::uint32_t* code, const std::uint32_t* continuation, std::uint32_t distanceLayout = 0) {
+    Pixels.fill(Kept);
+    const auto vertex = reinterpret_cast<std::uintptr_t>(code);
+    const auto pixel = reinterpret_cast<std::uintptr_t>(PixelCode.data());
+    const auto next = reinterpret_cast<std::uintptr_t>(continuation);
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> context{
+        {0x2d5, 0x2000},
+        {0x1b6, 0}, {0x207, distanceLayout}, {0x200, 0}, {0x203, 0x800},
+        {0x2dc, 0xaa00}, {0x2f8, 0}, {0x292, 2}, {0x293, 0},
+        {0x80, 0}, {0x8d, 0}, {0x83, 0xffff}, {0x8c, 0xa},
+        {0x2f9, 0x2d}, {0x313, 0x6000}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff},
+        {0x206, 0x43f}, {0x204, 0x80000}, {0x205, 0x240},
+        {0x8e, 0xf}, {0x8f, 0xf}, {0x202, 0xcc0010},
+        {0x1c4, 0}, {0x1c5, 9}, {0x1c3, 4}, {0x31c, 0x28028},
+        {0x31b, 0}, {0x31d, 0}, {0x3b0, (63u << 14u) | 3u},
+        {0x3b8, 0x9000000}, {0x1e0, 0},
+        {0xc, 0}, {0xd, 0x40040},
+        {0x81, 0x80000000}, {0x82, 0x40040},
+        {0x90, 0x80000000}, {0x91, 0x40040},
+        {0x94, 0x80000000}, {0x95, 0x40040},
+        {0x10f, std::bit_cast<std::uint32_t>(32.0f)}, {0x110, std::bit_cast<std::uint32_t>(32.0f)},
+        {0x111, std::bit_cast<std::uint32_t>(-2.0f)}, {0x112, std::bit_cast<std::uint32_t>(2.0f)},
+        {0x113, std::bit_cast<std::uint32_t>(1.0f)}, {0x114, 0},
+        {0xb4, 0}, {0xb5, std::bit_cast<std::uint32_t>(1.0f)},
+        {0x2fe, 0}, {0x302, 0}, {0x306, 0}, {0x30a, 0},
+    };
+    const std::vector<std::pair<std::uint32_t, std::uint32_t>> shader{
+        {0xc8, static_cast<std::uint32_t>(vertex >> 8u)}, {0xc9, static_cast<std::uint32_t>(vertex >> 40u)}, {0x8b, 2u << 1u},
+        {0x8c, static_cast<std::uint32_t>(next)}, {0x8d, static_cast<std::uint32_t>(next >> 32u)},
+        {0x008, static_cast<std::uint32_t>(pixel >> 8u)}, {0x009, static_cast<std::uint32_t>(pixel >> 40u)}, {0x00b, 1u << 1u}, {0x00c, std::bit_cast<std::uint32_t>(1.0f)},
+    };
+    std::vector<std::uint32_t> commands;
+    Append(commands, 0x79, {0x242, 4});
+    for (const auto& [offset, value] : context) Append(commands, 0x69, {offset, value});
+    for (const auto& [offset, value] : shader) Append(commands, 0x76, {offset, value});
+    const auto color = reinterpret_cast<std::uintptr_t>(Pixels.data());
+    Append(commands, 0x69, {0x318, static_cast<std::uint32_t>(color >> 8u)});
+    Append(commands, 0x69, {0x390, static_cast<std::uint32_t>(color >> 40u)});
+    Append(commands, 0x2d, {3, 2});
+    Packet packet{commands.data(), static_cast<std::uint32_t>(commands.size()), 0, {}};
+    Require(sceAgcDriverSubmitDcb(&packet) == 0, "vertex tail draw submission failed");
+    AgcDriverWaitIdle_nid_postfix();
+    AgcDriver::GuestMemory::FlushGpuWrites(color, Pixels.size());
+    return Pixels;
+}
+
+Image Check(std::span<const std::uint32_t> prefix, std::span<const std::uint32_t> continuation, std::uint32_t distanceLayout = 0, std::size_t continuationWords = 5) {
+    Prolog.fill(0xbf9f0000u);
+    Reference.fill(0xbf9f0000u);
+    std::copy(prefix.begin(), prefix.end(), Prolog.begin());
+    Prolog[prefix.size()] = 0xbe802008u;
+    std::copy(prefix.begin(), prefix.end(), Reference.begin());
+    std::copy_n(continuation.begin(), continuationWords, Reference.begin() + prefix.size());
+    const auto expected = Draw(Reference.data(), continuation.data(), distanceLayout);
+    for (unsigned repeat = 0; repeat < 2u; ++repeat) {
+        const auto actual = Draw(Prolog.data(), continuation.data(), distanceLayout);
+        Require(actual == expected, "linked vertex tail pixels differ from the monolithic shader");
+    }
+    return expected;
+}
+
+void VertexTailDraws() {
+    std::vector<std::uint32_t> prefix{
+        0x7e000d05u, 0x080200f4u, 0x10020300u, 0x100202f4u, 0x060202f3u, 0x060400f3u, 0x10040500u, 0x060404f3u,
+        0x7e060280u, 0x7e0802f2u, 0xbe8c03ffu, 0x3f800000u, 0xbf060c0cu,
+    };
+    const auto original = Check(prefix, Continuation);
+    const auto kept = std::count(original.begin(), original.end(), Kept);
+    Require(kept != 0 && kept != static_cast<std::ptrdiff_t>(original.size()), "reference triangle did not cross the render target");
+    const auto changedAddress = Check(prefix, OtherContinuation);
+    Require(changedAddress != original, "changing the continuation did not change the reference pixels");
+    Continuation[1] = OtherContinuation[1];
+    Require(Check(prefix, Continuation) == changedAddress, "rewritten continuation kept the old pixels");
+    Continuation[1] = 0x7e08020cu;
+    Require(Check(prefix, Continuation) == original, "restored continuation did not restore the pixels");
+    alignas(256) const std::array<std::uint32_t, 7> clipContinuation{0xbf840005u, 0x7e08020cu, 0xf80000cfu, 0x04030201u, 0xf80008dfu, 0x01010101u, 0xbf810000u};
+    const auto clipped = Check(prefix, clipContinuation, 0x00400001u, clipContinuation.size());
+    Require(clipped != original && std::count(clipped.begin(), clipped.end(), Kept) > kept, "a clip distance exported by the continuation did not clip the triangle");
+    const auto culled = Check(prefix, clipContinuation, 0x00400100u, clipContinuation.size());
+    Require(culled == original, "a mixed-sign cull distance discarded the triangle");
+    prefix.back() = 0xbf070c0cu;
+    const auto falseScc = Check(prefix, Continuation);
+    Require(std::ranges::all_of(falseScc, [](std::byte value) { return value == Kept; }), "SCC was not preserved across the tail jump");
+    prefix.back() = 0xbf060c0cu;
+    prefix.push_back(0xbefe0480u);
+    const auto emptyExec = Check(prefix, Continuation);
+    Require(std::ranges::all_of(emptyExec, [](std::byte value) { return value == Kept; }), "EXEC was not preserved across the tail jump");
+}
+
+}
+
+int main() {
+    try {
+        {
+            const auto device = OpenVulkanTestDevice();
+            if (!device) return VulkanTestSkipped;
+        }
+        VertexTailDraws();
+        AgcDriverShutdown_nid_postfix();
+        std::puts("vertex user-data tail tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        try { AgcDriverShutdown_nid_postfix(); } catch (const std::exception&) {}
+        return 1;
+    }
+}

core/shader/recompiler/CMakeLists.txt

@@ -246,6 +246,8 @@ function(add_shader_recompiler target runtimeTarget)
             ShaderCacheDirectory
             SrgbDecodeBit
             SrgbUnormFormat
+            FindUserDataTail
+            LinkUserDataTail
             ThreadOwned
             UnresolvableSwappcTarget
             ValidateAndOptimizeSpirv

core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp

@@ -10,6 +10,14 @@
 
 namespace ShaderRecompiler {
 
+struct UserDataTail {
+    std::uint32_t programCounter;
+    std::uint32_t userDataRegister;
+};
+
+[[nodiscard]] std::optional<UserDataTail> FindUserDataTail(const RdnaProgram& program, std::uint32_t userDataBase, std::uint32_t userDataCount);
+[[nodiscard]] std::vector<std::uint32_t> LinkUserDataTail(const RdnaProgram& program, const UserDataTail& tail, const RdnaProgram& continuation);
+
 struct SwappcInfo {
     bool fetchCallAllowed = false;
     std::uint32_t userDataBaseRegister = 0;

core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp

@@ -1011,6 +1011,34 @@ std::vector<UserDataCall> FindUserDataCalls(const RdnaProgram& program, const Sw
     return calls;
 }
 
+std::optional<UserDataTail> FindUserDataTail(const RdnaProgram& program, std::uint32_t userDataBase, std::uint32_t userDataCount) {
+    if (program.instructions.size() < 2u || program.instructions.back().op != RdnaOpcode::SCodeEnd) return std::nullopt;
+    const auto& tail = program.instructions[program.instructions.size() - 2u];
+    if (tail.op != RdnaOpcode::SSetpcB64 || tail.source0.kind != RdnaOperandKind::ScalarRegister) return std::nullopt;
+    const auto reg = tail.source0.reg;
+    if (reg % 2u != 0u || reg < userDataBase || userDataCount < 2u || reg - userDataBase >= userDataCount - 1u) return std::nullopt;
+    for (const auto& instruction : program.instructions) {
+        if (instruction.programCounter >= tail.programCounter) break;
+        if (IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SCallB64 || instruction.op == RdnaOpcode::SSwappcB64 ||
+            instruction.op == RdnaOpcode::SSetpcB64 || instruction.op == RdnaOpcode::SEndpgm || instruction.op == RdnaOpcode::SCodeEnd ||
+            writesScalar(instruction, reg) || writesScalar(instruction, reg + 1u)) return std::nullopt;
+    }
+    return UserDataTail{tail.programCounter, reg};
+}
+
+std::vector<std::uint32_t> LinkUserDataTail(const RdnaProgram& program, const UserDataTail& tail, const RdnaProgram& continuation) {
+    const auto recognized = FindUserDataTail(program, tail.userDataRegister, 2u);
+    if (!recognized || recognized->programCounter != tail.programCounter) throw std::invalid_argument("user-data tail is not a linear jump through an unchanged register pair");
+    for (const auto& instruction : continuation.instructions) {
+        if (instruction.op == RdnaOpcode::SGetpcB64) throw std::invalid_argument("user-data tail continuation reads its program counter, which a linked copy does not keep");
+    }
+    const auto words = tail.programCounter / 4u;
+    std::vector<std::uint32_t> linked(program.code.begin(), program.code.begin() + words);
+    linked.push_back(0xbf820000u);
+    linked.insert(linked.end(), continuation.code.begin(), continuation.code.end());
+    return linked;
+}
+
 std::optional<std::uint32_t> UserDataCalleeLength(std::span<const std::uint32_t> callee, std::uint32_t linkRegister) {
     std::uint32_t furthest = 0;
     for (std::uint32_t word = 0; word < callee.size();) {

docs/dev/debt/shader.md

@@ -18,6 +18,7 @@
 ### Functional
 
 - [Cached source IR](../../../core/shader/recompiler/Recompiler.cpp) releases lazily allocated SSA construction tables after resource preparation and reclaims values unreachable from instructions, metadata and resource plans when retaining a pending source program. Immediate shader preparation skips reclamation because compilation consumes the program. Reclamation must account for every new IR pointer owner; a remaining external use throws. Live IDs are retained. Source and variant entries remain cached without an eviction bound. A pending source program is released after its first variant is obtained; memory beyond the measured title checkpoint and Linux performance remain unmeasured.
+- [Vertex user-data tail jumps](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) link a linear raw vertex prolog's terminal `s_setpc_b64` through an unchanged user-data pair to its continuation. The instruction semantics follow AMD RDNA2 ISA section 12.3; the prolog convention follows Mesa `aco_select_vs_prolog.cpp` at `0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8`. Console hardware has not been measured. Computed pointers, branching prologs, fragment epilogs and continuations reading `s_getpc_b64` remain unsupported
 
 - The [resource tracker](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) plans a [descriptor source](../../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) used only behind a conditional branch as inactive when the branch condition reads only clean (specialization) memory and so is evaluable at capture time; an inactive source is bound as zero words and not validated. Other conditions (EXEC, VCC, SCC set by vector or non-clean data, loop headers) leave every source active, as before, and a branch on a value that changes between dispatches is a specialization key like any other clean slot. PPSA15246 has a copy kernel that selects a buffer or an image path from an SRT word and reads the same address as a V# or a T#; before, the untaken path's descriptor was validated and threw. Whether the console would also accept an invalid descriptor in a path it does not take is assumed, not measured
 - [MaskedSelectEliminator](../../../core/shader/recompiler/Optimization/src/MaskedSelectEliminator.cpp) can discard inactive source lanes through `ReadFirstLane` only when its read mask implies the write mask and its result is unobserved outside the read mask. A loop-local read mask requires result uses in the same block without phi transport. The empty-EXEC lane-zero fallback follows AMD's [Vega ISA, section 12.8, V_READFIRSTLANE_B32, page 127](https://www.amd.com/content/dam/amd/en/documents/radeon-tech-docs/instruction-set-architectures/vega-shader-instruction-set-architecture.pdf) and the existing SPIR-V emitter. The optimization is covered by synthetic IR tests but has not been compared with console hardware; a proof that needs a read mask of a kind that is already being solved is refused.