diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index 3fb9f473b3..0e96bcc002 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -86,6 +86,7 @@ private: friend class SampledReadScope; friend struct PendingView; friend struct PacketTimer; + friend struct DriverTestAccess; void stop(); Driver(); void rethrowFailure() const; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp index efa0e9d031..24e0fc7968 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp @@ -84,6 +84,7 @@ bool UnchangedCommittedSince(std::span& store); std::uint64_t StoreOwnBytes(std::uint64_t address, std::span source); bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generation); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 6562238c9e..52303ede78 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -17,6 +17,7 @@ #include #include #include +#include namespace AgcDriver { @@ -273,7 +274,7 @@ public: // `recipe`, when given, receives the Recipe a successful call built for its dispatch-cache // variant (design_cpu_final M4): only when the resource cache served or took the object // (reusable, cacheable) and recipes are on (APS5_NO_DISPATCH_RECIPE=1 builds none); else null. - void Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr prepared = nullptr, std::shared_ptr* recipe = nullptr); + void Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr prepared = nullptr, std::shared_ptr* recipe = nullptr, std::optional> htileClear = std::nullopt); struct IndirectOutcome { int cpuReason; double argumentReadMs; @@ -357,7 +358,7 @@ private: ShaderRecompiler::SpirvTarget buildTarget() const; Graphics::Context buildContext() const; // Body of Dispatch and DispatchIndirect: `arguments` 0 dispatches x, y, z groups. - IndirectOutcome dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipe, const std::array& threadGroup = {}); + IndirectOutcome dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipe, const std::array& threadGroup = {}, std::optional> htileClear = std::nullopt); // The stage-A pre-sync over `surfaces` (see PrepareDispatch): the serial waited for, 0 none. std::uint64_t presync(std::span surfaces); std::shared_ptr computePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index ccdc67b4aa..808000d470 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -16,7 +16,9 @@ #include "Optimization/ResourceProgram.hpp" #include #include +#include #include +#include namespace AgcDriver::DriverDetail { @@ -119,6 +121,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, } const bool pipelineCandidate = !resolving && !fromAhead && DispatchItemsEnabled() && DrawPipeline::Active() && indirectArguments == 0 && deferredLabels().labels.empty() && !matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute) && !MatchesStreamCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute); const bool partialLastGroup = partialGroups != std::array{}; + std::optional> htileClearCompletion; if (resolving) { if (indirectArguments == 0 && Graphics::MatchMultisampledSingleClear(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads)) return; if (!partialLastGroup && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return; @@ -139,7 +142,8 @@ void Driver::dispatch(QueueState& queue, std::span packet, if (const auto htile = Graphics::HtileDepthClearAddress(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads); htile != 0) { const auto threads = (packet[4] & 0x20u) != 0 ? static_cast(packet[1]) : static_cast(packet[1]) * compute.numThreads[0]; const auto nativeBytes = Graphics::HtileDepthClearBytes_nid_no_patch(std::span(snapshot.code).subspan(codeOffset), userData, compute, packet, indirectArguments); - Graphics::NoteHtileDepthClear(htile, static_cast(threads * 16u), nativeBytes); + const auto bytes = static_cast(threads * 16u); + if (Graphics::NoteHtileDepthClear(htile, bytes, nativeBytes)) htileClearCompletion = std::pair{htile, bytes}; } if (!partialLastGroup && indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) { pendingDispatchPhases().outcome = DispatchOutcome::CopyHle; @@ -462,18 +466,18 @@ void Driver::dispatch(QueueState& queue, std::span packet, const auto epoch = DrawPipeline::EpochToken().load(std::memory_order_relaxed); const auto queueId = submission.queue; itemCommits.fetch_add(1, std::memory_order_relaxed); - DrawPipeline::Queue0().Enqueue([this, localDevice = std::move(localDevice), result = compiledResult, snapshots = std::move(snapshots), liveWords = std::move(liveWords), groups, address, queueId, recipe, keepVariant, dataHit, epoch] { + DrawPipeline::Queue0().Enqueue([this, localDevice = std::move(localDevice), result = compiledResult, snapshots = std::move(snapshots), liveWords = std::move(liveWords), groups, address, queueId, recipe, keepVariant, dataHit, epoch, htileClearCompletion] { DrawPipeline::FollowEpoch(epoch); GuestMemory::SetCurrentPacket(0x15, queueId); const auto& compiledCommit = *result; const bool noteWrites = writeEvidenceEnabled() || traceCapSync(); try { - std::shared_ptr recipeHit = localDevice->PrepareRecipe(recipe, false); + std::shared_ptr recipeHit = htileClearCompletion.has_value() ? nullptr : localDevice->PrepareRecipe(recipe, false); std::shared_ptr builtRecipe; bool writersNoted = false; for (;;) { std::shared_ptr prepared; - if (recipeHit == nullptr || VulkanDevice::VerifyRecipes()) prepared = localDevice->PrepareDispatch(compiledCommit, snapshots); + if (recipeHit == nullptr || VulkanDevice::VerifyRecipes() || htileClearCompletion.has_value()) prepared = localDevice->PrepareDispatch(compiledCommit, snapshots); GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Dispatch); std::lock_guard gpuLock(GuestMemory::GpuMutex()); recordLabelsForPacket(localDevice.get(), queueId); @@ -491,7 +495,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, continue; } } else { - localDevice->Dispatch(compiledCommit, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared), &builtRecipe); + localDevice->Dispatch(compiledCommit, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared), htileClearCompletion.has_value() ? nullptr : &builtRecipe, htileClearCompletion); } break; } @@ -517,7 +521,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, std::shared_ptr recipeHit; if (cached && keepVariant != nullptr && !stampValidate()) { - recipeHit = localDevice->PrepareRecipe(keepVariant->recipe.load(std::memory_order_acquire), indirectArguments != 0); + recipeHit = htileClearCompletion.has_value() ? nullptr : localDevice->PrepareRecipe(keepVariant->recipe.load(std::memory_order_acquire), indirectArguments != 0); phaseTiming.Phase(PhaseRecipePrecheck); } @@ -595,7 +599,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, const auto outcome = localDevice->DispatchIndirect(compiled, indirectArguments, snapshots, address, std::move(prepared), attachTo != nullptr ? &builtRecipe : nullptr, threadGroup); countIndirect(outcome.cpuReason, outcome.argumentReadMs); } else { - localDevice->Dispatch(compiled, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared), attachTo != nullptr ? &builtRecipe : nullptr); + localDevice->Dispatch(compiled, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared), htileClearCompletion.has_value() ? nullptr : (attachTo != nullptr ? &builtRecipe : nullptr), htileClearCompletion); } } catch (const std::exception& error) { if (rethrow(error)) return; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp index abad85855e..0557a7af52 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp @@ -224,13 +224,17 @@ Driver::Driver() { void Driver::WaitIdle() { require(!onWorkerThread(), "worker cannot wait for itself"); - std::unique_lock lock(mutex); - const auto target = accepted; - ++idleWaiters; - changed.wait(lock, [&] { return failure != nullptr || stopping || completed >= target; }); - --idleWaiters; - rethrowFailure(); - checkStopping(); + { + std::unique_lock lock(mutex); + const auto target = accepted; + ++idleWaiters; + changed.wait(lock, [&] { return failure != nullptr || stopping || completed >= target; }); + --idleWaiters; + rethrowFailure(); + checkStopping(); + } + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (const auto localDevice = device.Load()) localDevice->WaitIdle(); } void Driver::CheckFailure() { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index cf5b1a1676..313498d4ca 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -971,8 +971,10 @@ struct WriteTracker { struct DriverPieces { std::uint32_t whole = 0; std::uint32_t dropped = 0; - std::uint32_t next = 0; std::array pieces{}; + std::uint32_t possibleWhole = 0; + std::uint32_t possibleDropped = 0; + std::array possiblePieces{}; bool covers(std::uint32_t begin, std::uint32_t end, std::uint64_t generation) const { if (generation == 0 || begin >= end || dropped > generation) return false; @@ -1025,7 +1027,7 @@ struct WriteTracker { #endif } - void stampWatchedPart(std::uint64_t address, std::uint64_t end, std::uint32_t stampGeneration) { + void stampWatchedPart(std::uint64_t address, std::uint64_t end, std::uint32_t stampGeneration, bool complete) { #ifdef _WIN32 address = std::max(address, base); end = std::min(end, base + size); @@ -1039,24 +1041,45 @@ struct WriteTracker { const auto stop = std::min(end, blockBegin(block) + WriteBlockBytes); if (!blockPartWatched(first, stop)) continue; stamp(block, stampGeneration, StampKind::Driver); - noteDriverStore(block, address, end, stampGeneration); + noteDriverStore(block, address, end, stampGeneration, complete); } } - void noteDriverStore(std::uint64_t block, std::uint64_t address, std::uint64_t end, std::uint32_t stampGeneration) { + void noteDriverStore(std::uint64_t block, std::uint64_t address, std::uint64_t end, std::uint32_t stampGeneration, bool complete) { const auto begin = blockBegin(block); const auto from = static_cast(std::max(address, begin) - begin); const auto to = static_cast(std::min(end, begin + WriteBlockBytes) - begin); if (from == 0 && to == WriteBlockBytes) { auto& entry = driverPieces[block]; - entry = DriverPieces{}; - entry.whole = stampGeneration; + if (complete) { + entry = DriverPieces{}; + entry.whole = stampGeneration; + } else { + entry.possibleWhole = stampGeneration; + entry.possibleDropped = 0; + entry.possiblePieces = {}; + } return; } auto& entry = driverPieces[block]; - auto& slot = entry.pieces[entry.next++ % entry.pieces.size()]; - entry.dropped = std::max(entry.dropped, slot.generation); - slot = {stampGeneration, from, to}; + auto& pieces = complete ? entry.pieces : entry.possiblePieces; + auto& dropped = complete ? entry.dropped : entry.possibleDropped; + DriverPiece* contained = nullptr; + for (auto& piece : pieces) { + if (piece.generation == 0 || piece.begin < from || piece.end > to) continue; + if (contained == nullptr) contained = &piece; + else piece = {}; + } + if (contained != nullptr) { + *contained = {stampGeneration, from, to}; + return; + } + auto* oldest = &pieces.front(); + for (auto& piece : pieces) { + if (piece.generation < oldest->generation) oldest = &piece; + } + dropped = std::max(dropped, oldest->generation); + *oldest = {stampGeneration, from, to}; } Leaf& leafFor(std::uint64_t block) { @@ -1167,10 +1190,13 @@ struct WriteTracker { continue; } const auto& entry = found->second; - if (entry.whole > generation || entry.dropped > generation) return true; + if (entry.whole > generation || entry.dropped > generation || entry.possibleWhole > generation || entry.possibleDropped > generation) return true; for (const auto& piece : entry.pieces) { if (piece.generation > generation && piece.begin < to && from < piece.end) return true; } + for (const auto& piece : entry.possiblePieces) { + if (piece.generation > generation && piece.begin < to && from < piece.end) return true; + } } return false; } @@ -1671,12 +1697,12 @@ std::uint64_t storeOwn(std::uint64_t address, std::size_t bytes, const std::func ++tracker.generation; if (!tracker.watched || !tracker.covers(stored.first, static_cast(stored.second - stored.first))) { tracker.noteUnstampedStore(stored.first, stored.second); - if (tracker.watched) tracker.stampWatchedPart(stored.first, stored.second, tracker.generation); + if (tracker.watched) tracker.stampWatchedPart(stored.first, stored.second, tracker.generation, true); return 0; } for (auto block = tracker.blockOf(stored.first); block <= tracker.blockOf(stored.second - 1); ++block) { tracker.stamp(block, tracker.generation, StampKind::Driver); - tracker.noteDriverStore(block, stored.first, stored.second, tracker.generation); + tracker.noteDriverStore(block, stored.first, stored.second, tracker.generation, true); } return tracker.generation; }; @@ -1709,7 +1735,7 @@ std::uint64_t StoreOwnBytes(std::uint64_t address, std::span so }); } -std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) { +static std::uint64_t markWritten(std::uint64_t address, std::size_t bytes, bool complete) { auto& tracker = Tracker(); auto lock = lockTracker(tracker); tracker.initialize(); @@ -1717,7 +1743,7 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) { ++tracker.generation; if (!tracker.watched || !tracker.covers(address, bytes)) { tracker.noteUnstampedStore(address, address + bytes); - if (tracker.watched) tracker.stampWatchedPart(address, address + bytes, tracker.generation); + if (tracker.watched) tracker.stampWatchedPart(address, address + bytes, tracker.generation, complete); return 0; } { @@ -1725,11 +1751,11 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) { const auto blockStop = (address + bytes + WriteBlockBytes - 1) & ~(std::uint64_t{WriteBlockBytes} - 1); StampRuns runs{tracker, StampKind::Fresh}; #ifdef _WIN32 - const bool complete = collectWindowsWrites(runs, blockFirst, blockStop, false); + const bool collected = collectWindowsWrites(runs, blockFirst, blockStop, false); #else - const bool complete = GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(static_cast(blockFirst), static_cast(blockStop - blockFirst), &stampWrittenRun, &runs); + const bool collected = GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(static_cast(blockFirst), static_cast(blockStop - blockFirst), &stampWrittenRun, &runs); #endif - if (!complete) { + if (!collected) { lock.unlock(); beforeUnwatch(blockFirst, static_cast(blockStop - blockFirst)); lock.lock(); @@ -1743,11 +1769,19 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) { const auto last = tracker.blockOf(address + bytes - 1); for (auto block = first; block <= last; ++block) { tracker.stamp(block, tracker.generation, StampKind::Driver); - tracker.noteDriverStore(block, address, address + bytes, tracker.generation); + tracker.noteDriverStore(block, address, address + bytes, tracker.generation, complete); } return tracker.generation; } +std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) { + return markWritten(address, bytes, true); +} + +std::uint64_t MarkPossiblyWritten(std::uint64_t address, std::size_t bytes) { + return markWritten(address, bytes, false); +} + bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generation) { auto& tracker = Tracker(); const auto lock = lockTracker(tracker); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 89ad6e3157..d52ba209aa 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -4406,6 +4406,7 @@ struct RecordedDispatch { // A template hit's revalidate time, charged to the [rescache] refresh accounting. double revalidateMs; DispatchTimer* timer; + std::optional> htileClear; // Whether the data refresh recorded anything. bool refreshed = false; bool resourceKeyMatched = false; @@ -4666,8 +4667,8 @@ std::shared_ptr VulkanDevice::PrepareRecipe(const std::shared_ptr snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipe) { - static_cast(dispatch(shader, x, y, z, 0, snapshots, programAddress, std::move(prepared), recipe)); +void VulkanDevice::Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipe, std::optional> htileClear) { + static_cast(dispatch(shader, x, y, z, 0, snapshots, programAddress, std::move(prepared), recipe, {}, htileClear)); } VulkanDevice::IndirectOutcome VulkanDevice::DispatchIndirect(const ShaderRecompiler::RecompileResult& shader, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipe, const std::array& threadGroup) { @@ -4928,21 +4929,27 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) { recorder.EndGpuTiming(trailingTiming); recorder.MarkCovered(dispatchedAccess); recordStep(PhaseRecordBind); + if (record.htileClear.has_value()) Graphics::ArmHtileDepthClear(record.htileClear->first, record.htileClear->second); resources.MarkGpuWrites(recorder); recordStep(PhaseRecordMarks); // Only copied written buffers (and BDA fault checks) need work once the GPU is done; without them // the batch can signal its labels from the GPU. - if (resources.NeedsCompletion() && resources.Reusable()) { + const bool completionNeeded = resources.NeedsCompletion() || record.htileClear.has_value(); + if (completionNeeded && resources.Reusable()) { auto kept = record.resources; - recorder.OnComplete([kept] { kept->CheckFaults(); }); - } else if (resources.NeedsCompletion()) { + recorder.OnComplete([kept, htileClear = record.htileClear] { + kept->CheckFaults(); + if (htileClear.has_value()) Graphics::CompleteHtileDepthClear(htileClear->first, htileClear->second); + }); + } else if (completionNeeded) { // Listed for DispatchIndirect until the write-back ran (a non-reusable object, so once). auto writers = state->copiedWriters; auto kept = record.resources; - recorder.OnComplete([kept, writers] { + recorder.OnComplete([kept, writers, htileClear = record.htileClear] { // Delisted before the write-back: one that fails must not keep indirect dispatches on the CPU. writers->erase(std::remove(writers->begin(), writers->end(), kept), writers->end()); kept->WriteBackBuffers(); + if (htileClear.has_value()) Graphics::CompleteHtileDepthClear(htileClear->first, htileClear->second); }); // Listed after the registration: a throw there leaves nothing that would pin the CPU path forever. if (resources.HasCopiedWrites()) writers->push_back(kept); @@ -5073,7 +5080,7 @@ std::shared_ptr VulkanDevice::computePipeline(const Grap return finish(std::move(objects), nullptr); } -VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipeOut, const std::array& threadGroup) { +VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipeOut, const std::array& threadGroup, std::optional> htileClear) { startJournalReplay(); for (const auto& binding : shader.bindings) { if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; @@ -5325,7 +5332,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec // works; the batch is waited for where the guest expects results (labels, flips, CPU reads). auto& recorder = *state->recorder; IndirectOutcome outcome{0, 0}; - RecordedDispatch record{&context, &shaders[0], resources, objects, pushStages, &pushBytes, x, y, z, arguments, nullptr, programAddress, fromCache && TemplateDataRefresh() ? RecordedDispatch::DataRefresh::Words : RecordedDispatch::DataRefresh::None, 0, revalidateMs, &timer}; + RecordedDispatch record{&context, &shaders[0], resources, objects, pushStages, &pushBytes, x, y, z, arguments, nullptr, programAddress, fromCache && TemplateDataRefresh() ? RecordedDispatch::DataRefresh::Words : RecordedDispatch::DataRefresh::None, 0, revalidateMs, &timer, htileClear}; record.resourceKeyMatched = fromCache; record.bindings = overlay; record.threadGroup = threadGroup; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp index f507ddd3a1..81fee305e1 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp @@ -54,7 +54,9 @@ bool DepthSurfaceOverlaps(VkDevice device, std::uint64_t address, std::uint64_t bool DepthSurfaceAt(const Context& context, std::span words, const GuestTextureResource& resource); std::uint64_t HtileDepthClearAddress(std::span code, std::span userData, const std::array& numThreads); std::size_t HtileDepthClearBytes_nid_no_patch(std::span code, std::span userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, std::span packet, std::uint64_t indirectArguments); -void NoteHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes, std::size_t nativeBytes = 0u); +bool NoteHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes, std::size_t nativeBytes = 0u); +void ArmHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes); +void CompleteHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes); struct MemoryValueFill { std::uint64_t address; std::size_t bytes; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp index 1da043f87f..03865c6a92 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp @@ -347,7 +347,7 @@ void BdaResources::markWrittenPages() const { Require(false, message.str()); } } - GuestMemory::MarkWritten(begin, bytes); + GuestMemory::MarkPossiblyWritten(begin, bytes); }), "invalid BDA written-page record"); for (std::uint32_t claim = 0; claim < count; ++claim) { const auto slot = words[Abi::WrittenListWord + claim]; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp index 9090b5952b..dd83557641 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp @@ -338,6 +338,7 @@ public: const auto clearLease = clearGeneration == 0u ? GuestAllocations::Lease{} : GuestAllocations::GuestAllocationsAcquireAll_nid_postfix(); Require(active->load(std::memory_order_acquire), "the native multisample depth owner was released"); Require(sampling.refusal.empty(), sampling.refusal); + Require(!clearPending, "native multisample depth clear has not been recorded"); for (const auto& range : sampling.initial) { Require(GuestMemory::Accessible(reinterpret_cast(range.address), range.bytes) && GuestMemory::Watched(range.address, range.bytes), "native multisample depth requires watched accessible guest memory"); if (const auto* recorder = RecorderFor(context)) Require(!recorder->PendingWriteOverlaps(range.address, range.bytes), "external GPU writes over native multisample depth are not implemented"); @@ -910,6 +911,7 @@ public: bool sampleable = true; bool samplingActive = false; DepthSamplingState sampling; + bool clearPending = false; std::uint64_t clearGeneration = 0u; std::shared_ptr owned; std::shared_ptr sampleShadow; @@ -2184,17 +2186,21 @@ std::size_t HtileDepthClearBytes_nid_no_patch(std::span cod return address <= std::numeric_limits::max() - bytes ? static_cast(bytes) : 0u; } -void NoteHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes, std::size_t nativeBytes) { +bool NoteHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes, std::size_t nativeBytes) { std::lock_guard lock(surfacesMutex()); forgetHtileReads(htileAddress, bytes); bool existing = false; + bool completionCandidate = false; for (const auto& surface : surfaces()) { if (!surface->active->load(std::memory_order_acquire)) continue; if (surface->samplingActive && surface->target.htileAddress == htileAddress) { const auto& metadata = surface->sampling.initial[1]; if (nativeBytes != bytes || nativeBytes < metadata.bytes || !GuestMemory::Accessible(reinterpret_cast(metadata.address), metadata.bytes) || !GuestMemory::Watched(metadata.address, metadata.bytes)) surface->sampling.Refuse("native multisample depth incomplete or unproven fast-clear transition is not implemented"); - else if (surface->clearGeneration != 0u) surface->sampling.Refuse("overlapping native multisample depth clear transitions are not implemented"); - else surface->clearGeneration = GuestMemory::CollectWritesUncached(metadata.address, metadata.bytes); + else if (surface->clearPending || surface->clearGeneration != 0u) surface->sampling.Refuse("overlapping native multisample depth clear transitions are not implemented"); + else { + surface->clearPending = true; + completionCandidate = true; + } existing = true; continue; } @@ -2204,6 +2210,30 @@ void NoteHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes, std::siz existing = true; } if (!existing) clearedHtiles()[htileAddress] = bytes; + return completionCandidate; +} + +void ArmHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes) { + std::lock_guard lock(surfacesMutex()); + for (const auto& surface : surfaces()) { + if (!surface->active->load(std::memory_order_acquire) || !surface->samplingActive || surface->target.htileAddress != htileAddress || !surface->clearPending) continue; + const auto& metadata = surface->sampling.initial[1]; + if (bytes < metadata.bytes) continue; + surface->clearGeneration = GuestMemory::CollectWritesUncached(metadata.address, metadata.bytes); + surface->clearPending = false; + if (surface->clearGeneration == 0u) surface->sampling.Refuse("native multisample depth clear requires tracked HTILE generations"); + } +} + +void CompleteHtileDepthClear(std::uint64_t htileAddress, std::size_t bytes) { + std::lock_guard lock(surfacesMutex()); + for (const auto& surface : surfaces()) { + if (!surface->active->load(std::memory_order_acquire) || !surface->samplingActive || surface->target.htileAddress != htileAddress || surface->clearGeneration == 0u) continue; + const auto& metadata = surface->sampling.initial[1]; + if (bytes < metadata.bytes) continue; + Require(GuestMemory::UnchangedSinceCollected(metadata.address, metadata.bytes, surface->clearGeneration), "CPU writes raced native multisample depth clear"); + GuestMemory::MarkWritten(metadata.address, metadata.bytes); + } } void ClearDepthSurfaces(VkDevice device) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index a1903262d7..c46494982e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -5173,7 +5173,7 @@ void GuestBufferMemory::MarkDirectWrites() const { const auto* found = owner(begin, end); if (found == nullptr) continue; const auto& region = *found; - if (region.direct != nullptr || (region.gpuCopy && region.copiedBack)) GuestMemory::MarkWritten(begin, static_cast(end - begin)); + if (region.direct != nullptr || (region.gpuCopy && region.copiedBack)) GuestMemory::MarkPossiblyWritten(begin, static_cast(end - begin)); } } @@ -5261,7 +5261,7 @@ void GuestBufferMemory::WriteBack() { if (region.direct == nullptr && !(region.gpuCopy && (region.copiedBack || region.deviceLocal)) && !(region.mirror != nullptr && !region.mirror->writable)) Recorder::NoteWrittenBack(begin, static_cast(end - begin)); if (region.direct != nullptr) { // The GPU wrote imported guest memory directly; page write watching did not see it. - GuestMemory::MarkWritten(begin, static_cast(end - begin)); + GuestMemory::MarkPossiblyWritten(begin, static_cast(end - begin)); continue; } if (region.mirror != nullptr) { @@ -5291,7 +5291,7 @@ void GuestBufferMemory::WriteBack() { if (region.copiedBack) { // The GPU copied the written sub-range back into the import; page write watching // did not see it. - GuestMemory::MarkWritten(begin, static_cast(end - begin)); + GuestMemory::MarkPossiblyWritten(begin, static_cast(end - begin)); continue; } static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; diff --git a/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp b/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp index e7ad973bba..267c945d64 100644 --- a/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp @@ -174,6 +174,21 @@ void CheckSharedBlock() { if (!Watched(outside, unwatched.size())) Require(StoredOver(outside, unwatched.size(), TrackerGeneration()), "an unwatched range reads as not stored over"); } +void CheckStoredOverUsesPageEvidence() { + constexpr std::size_t Page = 4096; + auto* memory = static_cast(AllocateWatched(2 * Block)); + const auto base = reinterpret_cast(memory); + std::memset(memory, 0x11, 2 * Block); + const auto before = CollectWritesUncached(base, 2 * Block); + Require(before != 0, "the stored-over page fixture was not collected"); + + static_cast(memory)[0] = 0x22; + const auto after = CollectWritesUncached(base, Block); + Require(after > before, "the page-local CPU write was not collected"); + Require(StoredOver(base, Page, before), "a collected CPU write on a page was not reported as stored over"); + Require(!StoredOver(base + Page, Page, before), "a page-local CPU write made an untouched page in the same block count as stored over"); +} + void CheckStoredAllSince() { constexpr std::size_t bytes = 4 * Block; auto* memory = static_cast(AllocateWatched(bytes)); @@ -217,6 +232,39 @@ void CheckStoredAllSince() { MarkWritten(base, 64); Require(StoredAllSince_nid_no_patch(base, 64, before), "a complete non-block-aligned range store is absent"); + before = TrackerGeneration(); + MarkPossiblyWritten(base + 2048, 16); + Require(StoredOver(base + 2048, 16, before), "a possible driver write did not invalidate its exact range"); + Require(!StoredOver(base + 2064, 16, before), "a possible driver write invalidated an adjacent range"); + Require(!UnchangedSince(base + 2048, 16, before), "a possible driver write left its exact range unchanged"); + Require(UnchangedSince(base + 2064, 16, before), "a possible driver write changed an adjacent range"); + Require(!StoredAllSince_nid_no_patch(base + 2048, 16, before), "a possible driver write proved complete byte coverage"); + for (unsigned repeat = 0; repeat < 6; ++repeat) MarkPossiblyWritten(base + 2048, 16); + Require(StoredOver(base + 2048, 16, before), "repeated possible writes lost their overlap evidence"); + Require(!StoredOver(base + 2064, 16, before), "repeated possible writes invalidated an adjacent range"); + Require(!StoredAllSince_nid_no_patch(base + 2048, 16, before), "repeated possible writes proved complete byte coverage"); + + before = TrackerGeneration(); + constexpr unsigned PossibleIntervals = 8; + auto beforeNewest = before; + for (unsigned interval = 0; interval < PossibleIntervals; ++interval) { + beforeNewest = TrackerGeneration(); + MarkPossiblyWritten(base + 4096 + static_cast(interval) * 32, 16); + } + Require(StoredOver(base + 4096, 16, before), "an evicted possible-write interval was forgotten instead of marking its block"); + Require(StoredOver(base + 4096 + 16, 16, before), "an evicted possible write did not mark the rest of its block"); + Require(StoredOver(base + 4096 + (PossibleIntervals - 1) * 32, 16, beforeNewest), "the newest possible-write interval was evicted"); + const auto beforeRecord = TrackerGeneration(); + MarkPossiblyWritten(base + 4096, 16); + Require(StoredOver(base + 4096, 16, beforeRecord), "a possible-write interval was not retained after being re-recorded"); + Require(!StoredAllSince_nid_no_patch(base + 4096, 16, before), "possible-write eviction or re-recording proved complete byte coverage"); + + before = TrackerGeneration(); + MarkWritten(base + 2560, 16); + MarkPossiblyWritten(base, Block); + Require(StoredAllSince_nid_no_patch(base + 2560, 16, before), "a possible whole-block range erased an exact partial-store proof"); + Require(!StoredAllSince_nid_no_patch(base, Block, before), "a possible whole-block range proved complete block coverage"); + before = TrackerGeneration(); MarkWritten(base + 512, 16); MarkWritten(base + 544, 16); @@ -348,6 +396,42 @@ void CheckSharedBlockProofs() { Require(!UnchangedSince(base + Block + 14 * page + 0x800, 16, beforeNeighbour), "a range in the page an own store dirtied reads as unchanged"); } +void CheckRepeatedPossibleStores() { + void* memory = AllocateWatched(3 * Block); + const auto base = reinterpret_cast(memory); + const auto boundary = base + 2 * Block - 0x9000; + std::memset(memory, 0x11, 3 * Block); + + const auto synced = CollectWrites(base, 3 * Block); + Require(synced != 0, "the test block is not collected"); + + const auto exact = TrackerGeneration(); + MarkWritten(base + Block, 0x100); + for (std::uint64_t store = 0; store < 5; ++store) MarkPossiblyWritten(boundary + store * 0x1000, 0x100); + Require(StoredAllSince_nid_no_patch(base + Block, 0x100, exact), "possible writes beyond the ring forgot an exact store's coverage"); + + void* fresh = AllocateWatched(2 * Block); + const auto freshBase = reinterpret_cast(fresh); + std::memset(fresh, 0x11, 2 * Block); + const auto freshSynced = CollectWrites(freshBase, 2 * Block); + const auto neighbour = freshBase + Block - 0x9000; + MarkPossiblyWritten(neighbour, 0x100); + const auto secondStored = MarkPossiblyWritten(neighbour + 0x1000, 0x100); + const auto thirdStored = MarkPossiblyWritten(neighbour + 0x2000, 0x100); + MarkPossiblyWritten(neighbour + 0x3000, 0x100); + const auto settled = TrackerGeneration(); + MarkPossiblyWritten(neighbour, 0x100); + MarkPossiblyWritten(neighbour, 0x100); + MarkPossiblyWritten(neighbour + 0x5000, 0x100); + Require(!StoredOver(freshBase, static_cast(neighbour - freshBase), secondStored), "a full possible ring evicted a newer piece instead of its oldest"); + Require(StoredOver(freshBase, static_cast(neighbour - freshBase), freshSynced), "an evicted possible write is not taken as a store over the rest of the block"); + + const auto partial = TrackerGeneration(); + MarkPossiblyWritten(neighbour + 0x5080, 0x100); + Require(StoredOver(neighbour + 0x5000, 0x10, settled) && StoredOver(neighbour + 0x5100, 0x10, partial), "a possible write that overlaps an earlier piece without containing it replaced it"); + Require(!StoredOver(freshBase, static_cast(neighbour - freshBase), thirdStored), "a possible write that overlaps without containing evicted a newer piece instead of the oldest"); +} + void CheckOwnStore() { void* memory = AllocateWatched(2 * Block); const auto base = reinterpret_cast(memory); @@ -1450,8 +1534,10 @@ int main() { } #endif CheckSharedBlock(); + CheckStoredOverUsesPageEvidence(); CheckStoredAllSince(); CheckSharedBlockProofs(); + CheckRepeatedPossibleStores(); CheckOwnStore(); CheckUnwatch(); CheckNewestStamp(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/NativeDepthBdaClear.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/NativeDepthBdaClear.cpp index d1564e3562..d6b99df4f7 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/NativeDepthBdaClear.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/NativeDepthBdaClear.cpp @@ -25,7 +25,9 @@ #include #include #include +#include #include +#include namespace AgcDriver { @@ -153,10 +155,12 @@ int RunNativeClear(AgcDriver::VulkanDevice& device) { device.WaitIdle(); target.clearDepth = 1.0f; Require(DepthSurfaceView(context, target) == view, "changing the native clear value replaced its attachment"); - NoteHtileDepthClear(target.htileAddress, metadataBytes, metadataBytes); + std::optional> htileClear; + if (NoteHtileDepthClear(target.htileAddress, metadataBytes, metadataBytes)) htileClear.emplace(target.htileAddress, metadataBytes); + Require(htileClear.has_value(), "the full BDA clear was not registered for completion"); const auto generation = Memory::TrackerGeneration(); userData[4] = 0xfffffff0u; - device.Dispatch(compile(), groups, 1u, 1u, {}, reinterpret_cast(code.data())); + device.Dispatch(compile(), groups, 1u, 1u, {}, reinterpret_cast(code.data()), nullptr, nullptr, htileClear); device.WaitIdle(); for (std::size_t offset = 0; offset < BlockBytes; offset += 4u) { std::uint32_t value = 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/NativeMultisampleDepth.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/NativeMultisampleDepth.cpp index cda5b9280a..d8bc68faf4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/NativeMultisampleDepth.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/NativeMultisampleDepth.cpp @@ -7,6 +7,9 @@ #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include "prx/libc/include/GuestArena.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/GuestWriteWatch.hpp" @@ -21,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +39,12 @@ #include #endif +namespace AgcDriver::DriverDetail { +struct DriverTestAccess { + static std::shared_ptr GetDevice() { return Driver::Get().device.Load(); } +}; +} + namespace { using namespace AgcDriver::Graphics; @@ -137,7 +147,7 @@ private: class TargetMemory { public: - explicit TargetMemory(std::size_t bytes) : bytes(bytes) { + explicit TargetMemory(std::size_t bytes, bool registerAllocation = false) : bytes(bytes), registered(registerAllocation) { #ifdef _WIN32 pointer = GuestArena::GuestArenaAllocate_nid_postfix(bytes, 65536u); GuestArena::GuestArenaCommit_nid_postfix(pointer, bytes, PAGE_READWRITE, bytes); @@ -153,8 +163,10 @@ public: #endif Require(pointer != nullptr, "multisample target fixture allocation failed"); std::memset(pointer, 0xa5, bytes); + if (registered) GuestAllocations::Mutation().Add(pointer, bytes, true, true, true); } ~TargetMemory() { + if (registered) GuestAllocations::Mutation().Remove(pointer); #ifdef _WIN32 GuestArena::GuestArenaReset_nid_postfix(pointer, bytes); GuestArena::GuestArenaRelease_nid_postfix(pointer, bytes); @@ -168,8 +180,49 @@ public: private: void* pointer = nullptr; std::size_t bytes; + bool registered = false; }; +alignas(256) constexpr std::array HtileMaskKernel{ + 0xd7460000u, 0x04010c06u, 0x34000084u, 0xdc388000u, 0x04020000u, 0xbf8c3f70u, 0xd7710008u, 0x00120805u, + 0xd7710009u, 0x00120a05u, 0xd771000au, 0x00120c05u, 0xd771000bu, 0x00120e05u, 0xdc788000u, 0x00020800u, + 0xbf810000u, +}; + +std::vector DriverPm4Packet(std::uint32_t opcode, std::initializer_list body) { + std::vector words{0xc0000000u | (static_cast(body.size() - 1u) << 16u) | (opcode << 8u)}; + words.insert(words.end(), body); + return words; +} + +Context SubmitDriverHtileClear(std::uint64_t address, std::unique_lock& gpuLock) { + const auto shader = reinterpret_cast(HtileMaskKernel.data()); + std::vector commands; + const auto append = [&commands](std::vector packet) { commands.insert(commands.end(), packet.begin(), packet.end()); }; + append(DriverPm4Packet(0x76u, {0x20cu, static_cast(shader >> 8u)})); + append(DriverPm4Packet(0x76u, {0x20du, static_cast(shader >> 40u) & 0xffu})); + append(DriverPm4Packet(0x76u, {0x207u, 64u})); + append(DriverPm4Packet(0x76u, {0x208u, 1u})); + append(DriverPm4Packet(0x76u, {0x209u, 1u})); + append(DriverPm4Packet(0x76u, {0x213u, (6u << 1u) | 0x80u})); + append(DriverPm4Packet(0x76u, {0x240u, 0u})); + append(DriverPm4Packet(0x76u, {0x241u, 0u})); + append(DriverPm4Packet(0x76u, {0x242u, static_cast(address)})); + append(DriverPm4Packet(0x76u, {0x243u, static_cast(address >> 32u)})); + append(DriverPm4Packet(0x76u, {0x244u, 0xfffc00f0u})); + append(DriverPm4Packet(0x76u, {0x245u, 0u})); + append(DriverPm4Packet(0x15u, {32u, 1u, 1u, 0x41u})); + Packet packet{commands.data(), static_cast(commands.size()), 0u, {}}; + Require(sceAgcDriverSubmitDcb(&packet) == 0, "native depth HTILE clear: PM4 submission failed"); + gpuLock.unlock(); + try { AgcDriverWaitIdle_nid_postfix(); } + catch (...) { gpuLock.lock(); throw; } + gpuLock.lock(); + const auto driverDevice = AgcDriver::DriverDetail::DriverTestAccess::GetDevice(); + Require(driverDevice != nullptr, "HTILE clear driver device is missing"); + return driverDevice->graphicsContext(); +} + template void TargetRefuses(Action action, std::string_view expected) { std::string message; @@ -600,7 +653,7 @@ void CheckDrawFeedback() { std::puts("Native depth feedback refused through the draw resource path"); } -enum class ClearFault { None, PartialStore, CpuWrite, DepthWrite, BadMetadata, UnprovenDispatch, PendingWrite, Overlap }; +enum class ClearFault { None, PartialStore, CpuWrite, DepthWrite, BadMetadata, UnprovenDispatch, PendingWrite, Overlap, PossibleStore, CompletedPossibleStore, DriverDispatch, ImportWindowWrite }; void CheckClearDispatch() { const std::array code{ @@ -609,7 +662,7 @@ void CheckClearDispatch() { 0xd771000a, 0x00120c05, 0xd771000b, 0x00120e05, 0xdc788000, 0x00020800, 0xbf810000, }; - const std::array userData{0u, 0u, 65536u, 0u, 0xf0u, 0xf0u}; + const std::array userData{0u, 0u, 65536u, 0u, 0xfffc00f0u, 0u}; const ShaderRecompiler::ShaderComputeStageInfo compute{{64u, 1u, 1u}, 0u, {true, false, false}, false, 1u}; const std::array packet{0u, 32u, 1u, 1u, 1u}; Require(HtileDepthClearBytes_nid_no_patch(code, userData, compute, packet, 0u) == 32768u, "complete HTILE mask dispatch was not recognized"); @@ -649,10 +702,10 @@ void CheckClearDispatch() { } } -void CheckDepthTarget(Context context, VkFormat format, std::uint32_t fragments, bool one, bool clearing = false, std::optional depthOnlyOverride = std::nullopt, ClearFault fault = ClearFault::None, bool iterate256 = true) { +void CheckDepthTarget(Context context, VkFormat format, std::uint32_t fragments, bool one, bool clearing, std::optional depthOnlyOverride, ClearFault fault, std::unique_lock& gpuLock, bool iterate256 = true) { const VkExtent2D extent{5u, 3u}; const auto bytes = static_cast(MultisampledDepthBytes_nid_no_patch(extent, format == VK_FORMAT_D16_UNORM ? 2u : 4u, fragments)); - TargetMemory memory(bytes), metadata(65536u); + TargetMemory memory(bytes), metadata(65536u, fault == ClearFault::DriverDispatch); const auto metadataBytes = HtileMetadataBytes(extent); const bool depthOnly = depthOnlyOverride.value_or(one); const std::uint32_t pattern = one ? (depthOnly ? 0xfffffff0u : 0xfffc00f0u) : 0u; @@ -770,17 +823,57 @@ void CheckDepthTarget(Context context, VkFormat format, std::uint32_t fragments, target.clearDepth = clearOne ? 1.0f : 0.0f; Require(DepthSurfaceView(context, target) == view, "updating the native clear value replaced its attachment"); const std::uint32_t clearPattern = clearOne ? (depthOnly ? 0xfffffff0u : 0xfffc00f0u) : 0u; - const auto nativeBytes = fault == ClearFault::UnprovenDispatch ? 0u : static_cast(metadataBytes); - NoteHtileDepthClear(target.htileAddress, static_cast(metadataBytes), nativeBytes); - if (fault == ClearFault::Overlap) NoteHtileDepthClear(target.htileAddress, static_cast(metadataBytes), nativeBytes); - const auto storedBytes = fault == ClearFault::PartialStore ? 4u : static_cast(metadataBytes); - AgcDriver::GuestMemory::StoreOwnBytes(target.htileAddress, storedBytes, [&] { - for (std::size_t i = 0u; i < storedBytes / 4u; ++i) std::memcpy(metadata.Data() + i * 4u, &clearPattern, 4u); - if (fault == ClearFault::BadMetadata) metadata.Data()[0] = std::byte{0x0f}; - }); - if (fault == ClearFault::CpuWrite) metadata.Data()[metadataBytes - 1u] ^= std::byte{0x01}; - if (fault == ClearFault::DepthWrite) memory.Data()[bytes - 1u] ^= std::byte{0x01}; - if (fault == ClearFault::PendingWrite) QueueTargetByteWrite(context, recorder, target.htileAddress, std::byte{0}); + if (fault == ClearFault::DriverDispatch && clearOne) { + struct ShutdownDriver { + ~ShutdownDriver() { + try { AgcDriverShutdown_nid_postfix(); } catch (...) {} + } + } shutdownDriver; + const auto generation = AgcDriver::GuestMemory::TrackerGeneration(); + const auto driverContext = SubmitDriverHtileClear(target.htileAddress, gpuLock); + recorder.Activate(); + Require(Recorder::Active() == &recorder, "driver idle did not restore the native depth fixture recorder"); + if (!AgcDriver::GuestMemory::Watched(target.htileAddress, metadataBytes)) { + Require(PrepareImportWatch(driverContext) == ImportWatch::Unwatch, + "the HTILE metadata range lost write-watch coverage on a device whose host imports stay watched"); + std::puts("skipped native HTILE write-coverage assertion: the write tracker does not cover the imported metadata range; the production depth-sampling path may still decline this clear"); + TargetRefuses([&] { Require(DepthSurfaceHolds(context, DecodeTextureResource(words), texture.get()), "native depth owner was not retained"); }, + "native multisample depth clear requires tracked HTILE generations"); + return; + } else { + const bool storedAll = AgcDriver::GuestMemory::StoredAllSince_nid_no_patch(target.htileAddress, metadataBytes, generation); + Require(storedAll, "driver idle returned before recording the completed HTILE dispatch"); + } + } else { + const auto nativeBytes = fault == ClearFault::UnprovenDispatch ? 0u : static_cast(metadataBytes); + if (fault == ClearFault::PartialStore || fault == ClearFault::PossibleStore || fault == ClearFault::CompletedPossibleStore) { + for (std::size_t i = 0u; i < metadataBytes / 4u; ++i) std::memcpy(metadata.Data() + i * 4u, &clearPattern, 4u); + } + const bool completionCandidate = NoteHtileDepthClear(target.htileAddress, static_cast(metadataBytes), nativeBytes); + if (fault == ClearFault::ImportWindowWrite) { + const auto beforeImport = AgcDriver::GuestMemory::TrackerGeneration(); + Require(AgcDriver::GuestMemory::ImportWatched(target.htileAddress, static_cast(metadataBytes), [&] { + metadata.Data()[0] ^= std::byte{0x01}; + return true; + }), "HTILE host import simulation failed"); + Require(AgcDriver::GuestMemory::CollectWritesUncached(target.htileAddress, static_cast(metadataBytes)) > beforeImport, "HTILE import window did not stamp its changed page"); + } + if (completionCandidate) ArmHtileDepthClear(target.htileAddress, static_cast(metadataBytes)); + if (fault == ClearFault::Overlap) NoteHtileDepthClear(target.htileAddress, static_cast(metadataBytes), nativeBytes); + const auto storedBytes = fault == ClearFault::PartialStore ? 4u : static_cast(metadataBytes); + if (fault == ClearFault::PossibleStore || fault == ClearFault::CompletedPossibleStore) { + const auto generation = AgcDriver::GuestMemory::TrackerGeneration(); + AgcDriver::GuestMemory::MarkPossiblyWritten(target.htileAddress, metadataBytes); + Require(!AgcDriver::GuestMemory::StoredAllSince_nid_no_patch(target.htileAddress, metadataBytes, generation), "a possible HTILE range proved complete coverage"); + if (fault == ClearFault::CompletedPossibleStore) CompleteHtileDepthClear(target.htileAddress, static_cast(metadataBytes)); + } else AgcDriver::GuestMemory::StoreOwnBytes(target.htileAddress, storedBytes, [&] { + for (std::size_t i = 0u; i < storedBytes / 4u; ++i) std::memcpy(metadata.Data() + i * 4u, &clearPattern, 4u); + if (fault == ClearFault::BadMetadata) metadata.Data()[0] = std::byte{0x0f}; + }); + if (fault == ClearFault::CpuWrite) metadata.Data()[metadataBytes - 1u] ^= std::byte{0x01}; + if (fault == ClearFault::DepthWrite) memory.Data()[bytes - 1u] ^= std::byte{0x01}; + if (fault == ClearFault::PendingWrite) QueueTargetByteWrite(context, recorder, target.htileAddress, std::byte{0}); + } Require(DepthSurfaceHolds(context, DecodeTextureResource(words), texture.get()), "completed native depth clear lost its sampled view"); recorder.Sync(); Require(DepthSurfaceView(context, target) == view && CachedSampledTexture(context, words) == texture, "completed native depth clear replaced an attachment or cached view"); @@ -1129,7 +1222,7 @@ int main(int argc, char** argv) { } CheckDrawFeedback(); const auto context = device->GetContext(); - const std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + std::unique_lock lock(AgcDriver::GuestMemory::GpuMutex()); Require(DepthSupported(context, VK_FORMAT_D32_SFLOAT, 4u), "the native depth regression requires sampled D32 with four samples"); if (argc == 2 && std::string_view(argv[1]) == "--clear-refusals") { for (const auto& [fault, expected] : std::array{ @@ -1137,10 +1230,11 @@ int main(int argc, char** argv) { std::pair{ClearFault::CpuWrite, "CPU writes raced"}, std::pair{ClearFault::DepthWrite, "guest writes"}, std::pair{ClearFault::BadMetadata, "does not prove its clear value"}, + std::pair{ClearFault::PossibleStore, "no complete tracked HTILE store"}, std::pair{ClearFault::UnprovenDispatch, "incomplete or unproven"}, std::pair{ClearFault::PendingWrite, "external GPU writes"}, std::pair{ClearFault::Overlap, "overlapping native multisample depth"}}) { - TargetRefuses([&] { CheckDepthTarget(context, VK_FORMAT_D32_SFLOAT, 4u, false, true, false, fault); }, expected); + TargetRefuses([&] { CheckDepthTarget(context, VK_FORMAT_D32_SFLOAT, 4u, false, true, false, fault, lock); }, expected); std::printf("Native depth clear fault=%u refused: %s\n", static_cast(fault), expected); } return 0; @@ -1152,10 +1246,16 @@ int main(int argc, char** argv) { for (const auto samples : {2u, 4u, 8u}) { if (!DepthSupported(context, format, samples)) { unavailable += 4u; continue; } for (const bool depthOnly : {false, true}) { - for (const bool one : {false, true}) { CheckDepthTarget(context, format, samples, one, true, depthOnly); ++checked; } + for (const bool one : {false, true}) { CheckDepthTarget(context, format, samples, one, true, depthOnly, ClearFault::None, lock); ++checked; } } } } + CheckDepthTarget(context, VK_FORMAT_D32_SFLOAT, 4u, false, true, false, ClearFault::CompletedPossibleStore, lock); + ++checked; + CheckDepthTarget(context, VK_FORMAT_D32_SFLOAT, 4u, false, true, false, ClearFault::ImportWindowWrite, lock); + ++checked; + CheckDepthTarget(context, VK_FORMAT_D32_SFLOAT, 4u, false, true, false, ClearFault::DriverDispatch, lock); + ++checked; std::printf("Native depth completed clear cases checked=%u unavailable=%u\n", checked, unavailable); return 0; } @@ -1176,7 +1276,7 @@ int main(int argc, char** argv) { for (const auto samples : {2u, 4u, 8u}) { if (!DepthSupported(context, format, samples)) { unavailable += 4u; continue; } for (const bool one : {false, true}) { - for (const bool iterate256 : {true, false}) { CheckDepthTarget(context, format, samples, one, false, std::nullopt, ClearFault::None, iterate256); ++checked; } + for (const bool iterate256 : {true, false}) { CheckDepthTarget(context, format, samples, one, false, std::nullopt, ClearFault::None, lock, iterate256); ++checked; } } CheckHtileRouting(context, format, samples); }