diff --git a/core/libs/prx/libSceAgcDriver/Eq/include/Event.hpp b/core/libs/prx/libSceAgcDriver/Eq/include/Event.hpp index 9792696727..ffdff3df09 100644 --- a/core/libs/prx/libSceAgcDriver/Eq/include/Event.hpp +++ b/core/libs/prx/libSceAgcDriver/Eq/include/Event.hpp @@ -4,7 +4,7 @@ #include #include "SceTypes.hpp" -void AgcDriverDeliverEopInterrupt(std::uint32_t queue); +void AgcDriverDeliverEopInterrupt(std::uint32_t queue, std::uint32_t context); extern "C" int APS5_VABI sceAgcDriverAddEqEvent(KernelEqueue eq, int id, void* udata); extern "C" int APS5_VABI sceAgcDriverDeleteEqEvent(KernelEqueue eq, int id); diff --git a/core/libs/prx/libSceAgcDriver/Eq/src/Event.cpp b/core/libs/prx/libSceAgcDriver/Eq/src/Event.cpp index 339d258622..1254c09e3d 100644 --- a/core/libs/prx/libSceAgcDriver/Eq/src/Event.cpp +++ b/core/libs/prx/libSceAgcDriver/Eq/src/Event.cpp @@ -33,9 +33,11 @@ int AddEvent(KernelEqueue eq, std::uint32_t queue, std::uintptr_t ident, void* u event.event.ident = ident; event.event.filter = EvfiltGraphicsCore; event.event.flags = EV_ADD | EV_CLEAR; + event.event.data = static_cast(queue); event.event.udata = udata; - event.filter.triggerFunc = [](KernelEqueueEvent* e, void*) { - e->event.data = e->triggered ? e->event.data + 1 : 1; + event.filter.triggerFunc = [](KernelEqueueEvent* e, void* data) { + if (data == nullptr) throw std::runtime_error("graphics event triggered without its interrupt data"); + e->event.data = static_cast(*static_cast(data)); e->triggered = true; }; event.filter.resetFunc = [](KernelEqueueEvent* e) { @@ -67,14 +69,16 @@ int DeleteEvent(KernelEqueue eq, std::uintptr_t ident, const char* caller) { } -void AgcDriverDeliverEopInterrupt(std::uint32_t queue) { +void AgcDriverDeliverEopInterrupt(std::uint32_t queue, std::uint32_t context) { std::vector registrations; { std::lock_guard lock(g_mutex); registrations = g_registrations; } for (const auto& registration : registrations) { - if (registration.queue == queue) EqueueTriggerEvent_nid_postfix(registration.eq, registration.ident, EvfiltGraphicsCore, nullptr); + if (registration.queue != queue) continue; + std::uint64_t data = static_cast(static_cast(registration.queue)) | (static_cast(context) << 16u); + EqueueTriggerEvent_nid_postfix(registration.eq, registration.ident, EvfiltGraphicsCore, &data); } } diff --git a/core/libs/prx/libSceAgcDriver/Eq/src/Query.cpp b/core/libs/prx/libSceAgcDriver/Eq/src/Query.cpp index c0987dc2b7..dc00d7ad80 100644 --- a/core/libs/prx/libSceAgcDriver/Eq/src/Query.cpp +++ b/core/libs/prx/libSceAgcDriver/Eq/src/Query.cpp @@ -17,22 +17,14 @@ uint32_t APS5_VABI sceAgcDriverGetEqContextId(const KernelEvent* ev) { if (ev->filter != -14) { throw std::runtime_error(std::string(__func__) + ": not a graphics event"); } - if (ev->ident > std::numeric_limits::max()) { - throw std::runtime_error(std::string(__func__) + ": context id overflow"); - } - return static_cast(ev->ident); + return static_cast(static_cast(ev->data) >> 16u); } int APS5_VABI sceAgcDriverGetEqEventType(const KernelEvent* ev) { if (ev == nullptr || reinterpret_cast(ev) % alignof(KernelEvent) != 0) { throw std::runtime_error(std::string(__func__) + ": null or misaligned event"); } - if (ev->filter == -14) { - if (ev->ident > static_cast(std::numeric_limits::max())) { - throw std::runtime_error(std::string(__func__) + ": graphics event type overflow"); - } - return static_cast(ev->ident); - } + if (ev->filter == -14) return static_cast(static_cast(static_cast(ev->data))); if (ev->data < std::numeric_limits::min() || ev->data > std::numeric_limits::max()) { throw std::runtime_error(std::string(__func__) + ": event type overflow"); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index 6001bc181a..44c40e8b6b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -299,10 +299,10 @@ private: static std::vector> drawWriteRanges(const Graphics::State& graphics, std::span stages); void commitDraw(std::shared_ptr localDevice, std::uint32_t queue, const Graphics::State& graphics, const Pm4::DrawParameters& drawParameters, std::span stages, std::span snapshots, const std::shared_ptr& recipe, const std::vector>& recipeStages, std::uint64_t drawKey, VulkanDevice::DrawTraceContext trace = {}, std::uint64_t shape = 0); bool labelPacketQueues(std::span packet, std::uint32_t opcode); - bool enqueueLabelPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue); + bool enqueueLabelPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue, std::optional interrupt); bool enqueueDmaPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue, const QueueState& state); void commitDmaStore(std::uint32_t queue, std::uint64_t address, std::span bytes); - void commitLabel(std::uint32_t queue, std::uint64_t address, std::span bytes, bool endOfPipeInterrupt); + void commitLabel(std::uint32_t queue, std::uint64_t address, std::span bytes, std::optional interrupt); void submitDueAfterCommit(VulkanDevice* localDevice); std::optional newestWriterLocked(std::uint64_t begin, std::uint64_t end) const; std::optional newestWriter(std::uint64_t begin, std::uint64_t end); @@ -382,14 +382,15 @@ private: void recordQueuedLabelsByTry(std::uint32_t queue, std::atomic& counter); static void recordQueuedLabelsFromHook(); void flushBetweenPackets(std::uint32_t queue, std::uint32_t header, bool labelPacket); - bool preparePacketMemory(const Submission& submission, QueueState& queue, std::span packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& interruptDeferred, bool& drawPacket, bool& sampleDump); + bool preparePacketMemory(const Submission& submission, QueueState& queue, std::span packet, std::uint32_t header, std::uint32_t opcode, std::optional interrupt, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& interruptDeferred, bool& drawPacket, bool& sampleDump); void dumpSampleCounters(std::uint64_t address); template static void timed(double WorkerProfile::*bucket, TWork&& work); FrameTiming* frameTiming(); FrameTiming* includeTimingSubmission(const Submission& submission, bool firstSegment); void execute(const Submission& submission); - void deliverGuestEopInterrupt(std::uint32_t queue); + void deliverGuestEopInterrupt(std::uint32_t queue, std::uint32_t context); + static std::optional releaseInterrupt(std::span packet, std::uint32_t opcode, bool carriesFlip); void markCompleted(std::uint64_t serial); static const std::atomic*& workerQueued(); void reapCompletionLabels(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp index 163c5646cd..e9dadd3fd0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include namespace AgcDriver::DriverDetail { @@ -120,8 +122,18 @@ bool& drawnSinceSeal() { } -void Driver::deliverGuestEopInterrupt(std::uint32_t queue) { - AgcDriverDeliverEopInterrupt(queue); +void Driver::deliverGuestEopInterrupt(std::uint32_t queue, std::uint32_t context) { + AgcDriverDeliverEopInterrupt(queue, context); +} + +std::optional Driver::releaseInterrupt(std::span packet, std::uint32_t opcode, bool carriesFlip) { + if (opcode != 0x49 || carriesFlip || packet.size() < 3) return std::nullopt; + const auto interruptSelect = (packet[2] >> 24u) & 7u; + if (interruptSelect == 0 || interruptSelect == 3) return std::nullopt; + if (packet.size() < 8) throw std::runtime_error("AGC: an interrupting release has no interrupt context dword"); + const auto context = packet[7] & 0x0fffffffu; + if ((context & 0x08000000u) != 0) throw std::runtime_error("AGC: a release asks for typed interrupt context " + std::to_string(context) + ", which is not delivered as a graphics end-of-pipe event; not implemented"); + return context; } template @@ -283,7 +295,7 @@ void Driver::execute(const Submission& submission) { if (submission.queue != 0 && Graphics::Recorder::DispatchOverlapEnabled()) noteRingPacket(submission.queue, submission.serial, opcode, header); PerformanceTimer timing("Driver.Packet"); if (!predicated) noteLabelIssue(packet, submission.queue); - if (pipelined && !predicated && header != FlipPacketHeader && header != RenderingWaitPacketHeader && !submission.flips.contains(cursor) && (enqueueLabelPacket(packet, opcode, submission.queue) || enqueueDmaPacket(packet, opcode, submission.queue, queue))) { + if (pipelined && !predicated && header != FlipPacketHeader && header != RenderingWaitPacketHeader && !submission.flips.contains(cursor) && (enqueueLabelPacket(packet, opcode, submission.queue, releaseInterrupt(packet, opcode, false)) || enqueueDmaPacket(packet, opcode, submission.queue, queue))) { traceLabel(packet, submission.queue); recent.Record(cursor); cursor = nextCursor; @@ -374,7 +386,7 @@ void Driver::execute(const Submission& submission) { if (profilePackets) packetProfile.flushMs += std::chrono::duration(packetTimer.start - flushStart).count(); bool wroteOnGpu = false, endOfPipeInterrupt = false, interruptDeferred = false, drawPacket = false, sampleDump = false; - const bool drains = preparePacketMemory(submission, queue, packet, header, opcode, wroteOnGpu, endOfPipeInterrupt, interruptDeferred, drawPacket, sampleDump); + const bool drains = preparePacketMemory(submission, queue, packet, header, opcode, releaseInterrupt(packet, opcode, submission.flips.contains(cursor)), wroteOnGpu, endOfPipeInterrupt, interruptDeferred, drawPacket, sampleDump); traceLabel(packet, submission.queue); timing.Mark("prepare_memory"); @@ -398,7 +410,6 @@ void Driver::execute(const Submission& submission) { Pm4::Execute(packet, queue); if (const auto label = Pm4::DecodeLabelWrite(packet)) noteLabelStore(label->address, label->Bytes(), ++eventSerial, submission.queue); } - if (endOfPipeInterrupt && !interruptDeferred) deliverGuestEopInterrupt(submission.queue); } if (deviceUse.owns_lock()) deviceUse.unlock(); CheckFailure(); @@ -575,7 +586,7 @@ void Driver::execute(const Submission& submission) { if (const auto label = Pm4::DecodeLabelWrite(packet)) noteLabelStore(label->address, label->Bytes(), ++eventSerial, submission.queue); } } - if (endOfPipeInterrupt && !interruptDeferred) deliverGuestEopInterrupt(submission.queue); + if (endOfPipeInterrupt && !interruptDeferred) deliverGuestEopInterrupt(submission.queue, packet[7] & 0x0fffffffu); } if (drawPacket || (sampleDump && wroteOnGpu)) Graphics::Recorder::CountRecordedWork(); timing.Mark(header == RenderingWaitPacketHeader ? "rendering_wait" : opcode == 0x3c || opcode == 0x93 ? "memory_wait" : opcode == 0x15 || opcode == 0x16 ? "dispatch" : drawPacket ? "draw" : "execute"); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp index 887e45d265..906334c6fd 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp @@ -54,15 +54,14 @@ std::optional decodeCopy(std::span packet) } -bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue, std::span packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& interruptDeferred, bool& drawPacket, bool& sampleDump) { +bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue, std::span packet, std::uint32_t header, std::uint32_t opcode, std::optional interrupt, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& interruptDeferred, bool& drawPacket, bool& sampleDump) { static const bool drainAll = std::getenv("APS5_DRAIN_ALL") != nullptr; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; wroteOnGpu = false; bool orderedAlready = false; - const auto interruptSelect = opcode == 0x49 ? (packet[2] >> 24u) & 7u : 0u; - endOfPipeInterrupt = interruptSelect != 0 && interruptSelect != 3; + endOfPipeInterrupt = interrupt.has_value(); interruptDeferred = false; if (!drainAll && endOfPipeInterrupt) { const auto label = Pm4::DecodeLabelWrite(packet); @@ -94,7 +93,8 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue } if (reason == 0 || reason == 5) { const auto queueId = submission.queue; - interruptDeferred = localDevice->AfterRecordedWork([this, queueId] { deliverGuestEopInterrupt(queueId); }, submission.queue == 0); + const auto context = *interrupt; + interruptDeferred = localDevice->AfterRecordedWork([this, queueId, context] { deliverGuestEopInterrupt(queueId, context); }, submission.queue == 0); if (interruptDeferred && workOpen) localDevice->SubmitRecorded(submission.queue == 0); wroteOnGpu = true; } else if (reason == 1) { @@ -436,10 +436,9 @@ bool Driver::labelPacketQueues(std::span packet, std::uint3 return opcode == 0x49 && ((packet[2] >> 29u) == 0 || (packet[3] | (static_cast(packet[4]) << 32u)) == 0); } -bool Driver::enqueueLabelPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue) { +bool Driver::enqueueLabelPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue, std::optional interrupt) { if (!labelPacketQueues(packet, opcode)) return false; - const auto interruptSelect = opcode == 0x49 ? (packet[2] >> 24u) & 7u : 0u; - const bool endOfPipeInterrupt = interruptSelect != 0 && interruptSelect != 3; + const bool endOfPipeInterrupt = interrupt.has_value(); const auto label = Pm4::DecodeLabelWrite(packet); if (!label.has_value()) { if (opcode != 0x49) return false; @@ -462,10 +461,10 @@ bool Driver::enqueueLabelPacket(std::span packet, std::uint if (endOfPipeInterrupt) bumpEpoch(&EpochBumps::drains); auto stored = bytes; auto labelBytes = bytes; - auto enqueue =[this, queue, opcode, address, bytes = std::move(bytes), endOfPipeInterrupt, writes, stored = std::move(stored)]() mutable { - DrawPipeline::Queue0().Enqueue([this, queue, opcode, address, bytes = std::move(bytes), endOfPipeInterrupt] { + auto enqueue =[this, queue, opcode, address, bytes = std::move(bytes), interrupt, writes, stored = std::move(stored)]() mutable { + DrawPipeline::Queue0().Enqueue([this, queue, opcode, address, bytes = std::move(bytes), interrupt] { GuestMemory::SetCurrentPacket(opcode, queue); - commitLabel(queue, address, bytes, endOfPipeInterrupt); + commitLabel(queue, address, bytes, interrupt); }, std::move(writes), address, std::move(stored)); }; if (queue == 0 && DrawFinisher::Overlap() && DrawFinisher::Busy()) { @@ -481,7 +480,7 @@ bool Driver::enqueueLabelPacket(std::span packet, std::uint return true; } -void Driver::commitLabel(std::uint32_t queue, std::uint64_t address, std::span bytes, bool endOfPipeInterrupt) { +void Driver::commitLabel(std::uint32_t queue, std::uint64_t address, std::span bytes, std::optional interrupt) { GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); std::lock_guard gpuLock(GuestMemory::GpuMutex()); const auto localDevice = device.Load(); @@ -503,14 +502,15 @@ void Driver::commitLabel(std::uint32_t queue, std::uint64_t address, std::spanAfterRecordedWork([this, queue] { deliverGuestEopInterrupt(queue); }, queue == 0); + deferred = localDevice->AfterRecordedWork([this, queue, context] { deliverGuestEopInterrupt(queue, context); }, queue == 0); if (deferred && workOpen) localDevice->SubmitRecorded(queue == 0); } - if (!deferred) deliverGuestEopInterrupt(queue); + if (!deferred) deliverGuestEopInterrupt(queue, context); } submitDueAfterCommit(localDevice.get()); } diff --git a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp index f1dc731af9..c0ede60429 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp @@ -71,24 +71,20 @@ std::string expectFailure(TAction action) { void testEvents() { KernelEvent event{}; event.filter = -14; - event.ident = 0x40; - event.data = 123; - check(sceAgcDriverGetEqEventType(&event) == 0x40, "graphics event uses wrong field"); + event.ident = 0x20; + event.data = (0x38 << 16) | 0x20; + check(sceAgcDriverGetEqEventType(&event) == ((0x38 << 16) | 0x20), "graphics event type is not the event data"); + check(sceAgcDriverGetEqContextId(&event) == 0x38, "graphics event context id is not the data's upper bits"); + event.data = static_cast((std::uint64_t{0x0fffffff} << 16u) | 0x28); + check(sceAgcDriverGetEqContextId(&event) == 0x0fffffffu, "graphics event context id lost its upper bits"); + check(sceAgcDriverGetEqEventType(&event) == static_cast(0xffff0028u), "graphics event type is not the data's low word"); event.filter = -1; event.data = -17; check(sceAgcDriverGetEqEventType(&event) == -17, "non-graphics event uses wrong field"); event.data = std::numeric_limits::max(); expectFailure([&] { sceAgcDriverGetEqEventType(&event); }); - event.filter = -14; - event.ident = std::numeric_limits::max(); - expectFailure([&] { sceAgcDriverGetEqEventType(&event); }); expectFailure([] { sceAgcDriverGetEqEventType(nullptr); }); expectFailure([&] { sceAgcDriverGetEqEventType(reinterpret_cast(reinterpret_cast(&event) + 1)); }); - event.ident = 0x29; - check(sceAgcDriverGetEqContextId(&event) == 0x29, "graphics event context id uses wrong field"); - event.ident = std::numeric_limits::max(); - expectFailure([&] { sceAgcDriverGetEqContextId(&event); }); - event.ident = 1; event.filter = -1; expectFailure([&] { sceAgcDriverGetEqContextId(&event); }); expectFailure([] { sceAgcDriverGetEqContextId(nullptr); }); @@ -244,13 +240,24 @@ void testEndOfPipeInterrupts() { check(sceAgcDriverSubmitDcb(&packet) == 0 && sceAgcDriverSubmitDcb(&packet) == 0, "interrupt submit failed"); AgcDriverWaitIdle_nid_postfix(); check(owner->GetTriggeredEvents(events.data(), 2) == 1, "graphics end-of-pipe interrupt was not delivered to its queue only"); - check(events[0].filter == -14 && events[0].udata == &graphicsTag && events[0].data == 2 && sceAgcDriverGetEqEventType(events.data()) == 0, "graphics end-of-pipe event encoding is wrong"); + check(events[0].filter == -14 && events[0].udata == &graphicsTag && events[0].data == 0 && sceAgcDriverGetEqEventType(events.data()) == 0, "graphics end-of-pipe event encoding is wrong"); check(sceAgcDriverGetEqContextId(events.data()) == 0, "graphics end-of-pipe event reports another context"); check(owner->GetTriggeredEvents(events.data(), 2) == 0, "delivered interrupt was not cleared"); + words[7] = 0x38; + check(sceAgcDriverSubmitDcb(&packet) == 0, "interrupt submit with a context failed"); + AgcDriverWaitIdle_nid_postfix(); + check(owner->GetTriggeredEvents(events.data(), 2) == 1 && events[0].udata == &graphicsTag && events[0].data == (0x38 << 16), "graphics end-of-pipe event does not carry the release's interrupt context"); + check(sceAgcDriverGetEqContextId(events.data()) == 0x38 && sceAgcDriverGetEqEventType(events.data()) == (0x38 << 16), "graphics end-of-pipe event reports another context"); + words[7] = 0xf0000041u; + check(sceAgcDriverSubmitDcb(&packet) == 0, "interrupt submit with high context bits failed"); + AgcDriverWaitIdle_nid_postfix(); + check(owner->GetTriggeredEvents(events.data(), 2) == 1 && sceAgcDriverGetEqContextId(events.data()) == 0x41, "the interrupt context kept bits above its 28 bits"); + words[7] = 0x38; check(sceAgcDriverSubmitAcb(0x20, &packet) == 0, "compute interrupt submit failed"); AgcDriverWaitIdle_nid_postfix(); - check(owner->GetTriggeredEvents(events.data(), 2) == 1 && events[0].udata == &computeTag && sceAgcDriverGetEqEventType(events.data()) == 0x20, "compute end-of-pipe interrupt missing"); - check(sceAgcDriverGetEqContextId(events.data()) == 0x20, "compute end-of-pipe event reports another context"); + check(owner->GetTriggeredEvents(events.data(), 2) == 1 && events[0].udata == &computeTag && events[0].data == ((0x38 << 16) | 0x20) && sceAgcDriverGetEqEventType(events.data()) == ((0x38 << 16) | 0x20), "compute end-of-pipe interrupt missing"); + check(sceAgcDriverGetEqContextId(events.data()) == 0x38, "compute end-of-pipe event reports another context"); + words[7] = 0; words[2] = 0; check(sceAgcDriverSubmitDcb(&packet) == 0, "plain release submit failed"); AgcDriverWaitIdle_nid_postfix();