From 175d986c289014a277bde308c371e7bb01d463a3 Mon Sep 17 00:00:00 2001 From: thuanquang Date: Thu, 22 Oct 2026 16:54:35 +0700 Subject: [PATCH 1/3] perf(agc): create the host import of a linear color target in registered memory A linear color target got a resident image only when its memory already had a host import (#6892). Every other draw into it was synchronous: copied in, drawn, waited for on the CPU and copied back. #6892 only looked the import up because a first version that made it broke agc_driver_pixel_ancillary_word and agc_driver_pixel_fixed_point_position: the import over a short-lived block was compared, and the next block allocated at the same address stayed outside the write watch. A page committed again inside an excluded block is watched again since #6759. On a lookup miss the draw now makes the import itself, with the new ColorTarget requester, when its registered ranges are writable, the whole import run is write-watched and the import keeps it watched: a separate alias of shared memory, a direct import of private memory that settles on Windows (1 MiB or more, when the import probe keeps the watch and a settle queue exists), or an import the probe keeps watched on other systems, where a dma-buf import must stay watched too if the range has shared backing. Private pages that the write watch armed read-only count as writable when their guest protection is, because the import opens them (a resident direct-memory run the CPU has not stored to since its last collect). The gate's scan stops at the first page that decides a refusal, and a run of private memory whose direct import would be compared is refused after one page query. A refusal is remembered for its run until the registry, the committed pages (Windows) or the write watch change, or a shared alias is mapped over the run, so a target that stays synchronous costs a lookup per draw. importRunAt checks this after its remembered failures and refusals and before it retires or evicts anything, through the helpers its own decision now uses: importWatching (settled, watched host-pointer and dma-buf imports) and importPages (the Windows page scan). A target whose import would be compared stays synchronous, because a compared import makes every refresh of the resident image compare the whole target. A target over an import that already exists is resident as before, with ExistingHostImportFor as the fast path, and APS5_NO_RESIDENT_LINEAR=1 still keeps every linear target synchronous. This is the case draft #1191 (MichonGoddijn231849) was to bring back on top of #6892 once partial-block write-watch coverage existed. agc_driver_linear_target_import_creation covers a target in freshly registered, write-watched memory (the draw is recorded, the target is imported and stays watched), a block freed and allocated again at the same address (it is watched, and the next draw starts from the CPU's bytes), a resident direct-memory run armed read-only by the write watch (the draw is recorded and the target imported; Windows), the gate over armed and read-only private memory (a compared import of an armed run is refused without opening it, memory committed read-only behind a writable registry range is refused without a count, an armed run is imported where imports stay watched; Windows), a 64 KiB block whose import would be compared (no import, a synchronous draw, the block stays watched), a 1 MiB block the guest registered read-only (no import, a synchronous draw), a 1 MiB run with one 64 KiB block outside the write watch (refused, the target stays watched), and the same run once that block is committed again (the refused run is imported; Windows). On other systems the test forces ImportWatch::Watch, so the watched cases also run where imports are compared (lavapipe in CI). The import requester tests count the new requester and check that it makes no compared import, no import of a range the guest cannot write and no import of memory outside the write watch. Co-Authored-By: Claude --- .../Graphics/include/GuestBufferMemory.hpp | 4 +- .../prx/libSceAgcDriver/Graphics/src/Draw.cpp | 2 +- .../Graphics/src/GuestBufferMemory.cpp | 146 +++++- .../libSceAgcDriver/tests/BdaResources.cpp | 23 +- .../execution/LinearTargetImportCreation.cpp | 495 ++++++++++++++++++ docs/dev/ENVIRONMENT.md | 2 +- docs/dev/debt/libSceAgcDriver.md | 1 + 7 files changed, 642 insertions(+), 31 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetImportCreation.cpp diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index fa552ac524..618d713d48 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -74,8 +74,8 @@ ImportProbe ProbeDmaBufImportWriteProtection(const Context& context); ImportWatch PrepareImportWatch(const Context& context); void SetImportWatch(const Context& context, ImportWatch watch); -enum class ImportRequester : std::uint8_t { Map, Build, Access, Join, Segment }; -inline constexpr std::size_t ImportRequesterCount = 5; +enum class ImportRequester : std::uint8_t { Map, Build, Access, Join, Segment, ColorTarget }; +inline constexpr std::size_t ImportRequesterCount = 6; struct HostImportRequesterCounts { std::uint64_t made = 0; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index 339157e2b0..29c14ae5bc 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -2316,7 +2316,7 @@ bool prepareDraw(const Context& context, const State& state, const Pm4::DrawPara // Debug aid: APS5_NO_RESIDENT_TARGETS=1 copies every target in and out again. static const bool residentTargets = std::getenv("APS5_NO_RESIDENT_TARGETS") == nullptr; static const bool residentLinear = std::getenv("APS5_NO_RESIDENT_LINEAR") == nullptr; - const bool linearResident = residentLinear && color.tileMode == ColorTileMode::Linear && ExistingHostImportFor(context, color.address, colorLayout.Bytes()) != nullptr; + const bool linearResident = residentLinear && residentTargets && context.detiler != nullptr && color.tileMode == ColorTileMode::Linear && (ExistingHostImportFor(context, color.address, colorLayout.Bytes()) != nullptr || (GuestMemory::Watched(color.address, colorLayout.Bytes()) && HostImportFor(context, color.address, colorLayout.Bytes(), ImportRequester::ColorTarget) != nullptr)); if ((binding.gpuTiling || ((color.tileMode != ColorTileMode::Linear || linearResident) && context.detiler != nullptr)) && residentTargets) { // The lookup refreshes the image on every draw (StorageTexture::Refresh: FlushPending, // CollectWrites over the target's pages, the DCC key scan of TextureClearKeys, then diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 197979f33a..469f78a18b 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -172,6 +172,14 @@ struct ImportRefusal { std::uint64_t budget; }; +struct GateRefusal { + std::uint64_t base; + std::uint64_t bytes; + GuestAllocations::Mapped ranges; + std::uint64_t commits; + std::uint64_t watches; +}; + struct EvictingRefusal { std::uint64_t base; std::uint64_t bytes; @@ -242,6 +250,7 @@ struct HostImports { std::map, ImportFailure> failedBlocks; std::map, ImportRefusal> refusals; std::map, EvictingRefusal> evictingRefusals; + std::map, GateRefusal> gateRefusals; // Registry generation the imports were last reconciled with. std::uint64_t refreshedGeneration = 0; // Bumped whenever an import is dropped, so HostImport pointers taken under the lock earlier can be @@ -819,8 +828,12 @@ struct BackingFile { std::uint64_t offset = 0; }; +bool sharedBacking(const Context& context, std::uint64_t base, std::uint64_t bytes, BackingFile& backing) { + return context.dmaBufImport && GuestArena::GuestArenaSharedBacking_nid_postfix(static_cast(base), static_cast(bytes), &backing.file, &backing.offset); +} + bool sharedBacking(const Context& context, const HostImport& entry, BackingFile& backing) { - return context.dmaBufImport && GuestArena::GuestArenaSharedBacking_nid_postfix(static_cast(entry.base), static_cast(entry.bytes), &backing.file, &backing.offset); + return sharedBacking(context, entry.base, entry.bytes, backing); } const char* createBackingImport(const Context& context, HostImport& entry, BackingFile& backing, VkResult& failure) { @@ -1202,6 +1215,66 @@ void decideImportWatch(const Context& context, HostImports& state) { #endif } +struct ImportWatching { + bool settled = false; + bool hostPointer = false; + bool dmaBuf = false; +}; + +ImportWatching importWatching(const Context& context, HostImports& state, bool alias, bool writable, std::uint64_t bytes) { + decideImportWatch(context, state); +#ifdef _WIN32 + const bool settled = !alias && state.watchPrivateImports && writable && bytes >= settleFloorBytes; + const bool watched = alias || !state.unwatchImports || settled; + return {settled, watched, watched}; +#else + static_cast(alias); + static_cast(writable); + static_cast(bytes); + return {false, !state.unwatchImports, !state.unwatchDmaBufImports}; +#endif +} + +#ifdef _WIN32 +struct ImportPages { + bool committed = true; + bool writable = true; + bool opened = true; + bool readOnly = true; + bool shared = true; + MEMORY_BASIC_INFORMATION refused{}; +}; + +bool writableProtection(std::uint32_t protection) { + protection &= 0xffu; + return protection == PAGE_READWRITE || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_WRITECOPY; +} + +ImportPages importPages(std::uint64_t base, std::uint64_t bytes, bool opening = false) { + ImportPages pages; + for (std::uint64_t cursor = base; cursor < base + bytes;) { + MEMORY_BASIC_INFORMATION info{}; + if (VirtualQuery(reinterpret_cast(cursor), &info, sizeof(info)) == 0 || info.State != MEM_COMMIT || (info.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) { + pages.committed = false; + return pages; + } + const auto protection = info.Protect & 0xffu; + const bool pageWritable = (info.Type == MEM_PRIVATE || info.Type == MEM_IMAGE) && writableProtection(protection); + const bool pageReadOnly = info.Type != MEM_MAPPED && (protection == PAGE_READONLY || protection == PAGE_EXECUTE_READ); + std::uint32_t guestProtection = 0; + const bool armed = opening && info.Type == MEM_PRIVATE && pageReadOnly && GuestArena::GuestArenaProtection_nid_postfix(static_cast(cursor), &guestProtection) && writableProtection(guestProtection); + if (!pageReadOnly) pages.readOnly = false; + if (info.Type != MEM_MAPPED) pages.shared = false; + if (info.Type != MEM_MAPPED && !pageWritable && !pageReadOnly && pages.refused.BaseAddress == nullptr) pages.refused = info; + if (!pageWritable) pages.writable = false; + if (!pageWritable && !armed) pages.opened = false; + if (opening && !pages.opened && !pages.shared) return pages; + cursor = reinterpret_cast(info.BaseAddress) + info.RegionSize; + } + return pages; +} +#endif + std::shared_ptr leasedRangeOwner(const GuestAllocations::Lease& lease, std::uint64_t base) { const auto found = std::lower_bound(lease.begin(), lease.end(), base, [](const auto& range, std::uint64_t value) { return range->address < value; }); return found != lease.end() && (*found)->address == base ? *found : nullptr; @@ -1220,6 +1293,22 @@ struct ImportRun { std::vector> ranges; }; +bool importStaysWatched(const Context& context, HostImports& state, const ImportRun& run) { + if (std::any_of(run.ranges.begin(), run.ranges.end(), [](const auto& range) { return !range->writable; })) return false; + if (!GuestMemory::Watched(run.base, static_cast(run.bytes))) return false; +#ifdef _WIN32 + MEMORY_BASIC_INFORMATION first{}; + if (!importWatching(context, state, false, true, run.bytes).hostPointer && (VirtualQuery(reinterpret_cast(run.base), &first, sizeof(first)) == 0 || first.Type != MEM_MAPPED)) return false; + const auto pages = importPages(run.base, run.bytes, true); + if (!pages.committed || (!pages.opened && !pages.shared)) return false; + return importWatching(context, state, !pages.opened, pages.opened, run.bytes).hostPointer; +#else + const auto watching = importWatching(context, state, false, true, run.bytes); + BackingFile backing; + return watching.hostPointer && (watching.dmaBuf || !sharedBacking(context, run.base, run.bytes, backing)); +#endif +} + bool continuesRun(const GuestAllocations::Range& before, const GuestAllocations::Range& after) { return before.address + before.bytes == after.address && before.readable && after.readable && before.writable == after.writable && before.gpu == after.gpu; } @@ -1640,6 +1729,21 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import } state.evictingRefusals.erase(remembered); } + if (requester == ImportRequester::ColorTarget) { +#ifdef _WIN32 + const auto commits = GuestArena::GuestArenaCommitGeneration_nid_postfix(); +#else + const std::uint64_t commits = 0; +#endif + const auto watches = GuestMemory::UnwatchSerial(); + const std::pair key{base, bytes}; + if (const auto refused = state.gateRefusals.find(key); refused != state.gateRefusals.end() && refused->second.commits == commits && refused->second.watches == watches && sameRun(refused->second, run)) return nullptr; + if (!importStaysWatched(context, state, run)) { + state.gateRefusals.insert_or_assign(key, GateRefusal{base, bytes, GuestAllocations::Mapped(run.ranges.begin(), run.ranges.end()), commits, watches}); + return nullptr; + } + state.gateRefusals.erase(key); + } auto first = state.imports.lower_bound(base); if (first != state.imports.begin() && std::prev(first)->second.base + std::prev(first)->second.bytes > base) --first; std::uint64_t replaced = 0; @@ -1713,27 +1817,17 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import #ifdef _WIN32 // Drivers pin imported pages, so every page must be committed and accessible. const GuestArena::HostImportAccess importAccess(reinterpret_cast(base), static_cast(bytes)); - bool writable = true; - bool readOnly = true; - MEMORY_BASIC_INFORMATION refused{}; - for (std::uint64_t cursor = base; cursor < base + bytes;) { - MEMORY_BASIC_INFORMATION info{}; - if (VirtualQuery(reinterpret_cast(cursor), &info, sizeof(info)) == 0 || info.State != MEM_COMMIT || (info.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) return failedImport(); - const auto protection = info.Protect & 0xffu; - const bool pageWritable = (info.Type == MEM_PRIVATE || info.Type == MEM_IMAGE) && (protection == PAGE_READWRITE || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_WRITECOPY); - const bool pageReadOnly = info.Type != MEM_MAPPED && (protection == PAGE_READONLY || protection == PAGE_EXECUTE_READ); - if (!pageReadOnly) readOnly = false; - if (info.Type != MEM_MAPPED && !pageWritable && !pageReadOnly && refused.BaseAddress == nullptr) refused = info; - if (!pageWritable) writable = false; - cursor = reinterpret_cast(info.BaseAddress) + info.RegionSize; - } - if (!writable && readOnly) return failedImport(); + const auto pages = importPages(base, bytes); + if (!pages.committed) return failedImport(); + const bool writable = pages.writable; + if (!writable && pages.readOnly) return failedImport(); if (!writable) { - if (refused.BaseAddress != nullptr) { + if (pages.refused.BaseAddress != nullptr) { char text[256]; - std::snprintf(text, sizeof(text), "AGC graphics: host import of 0x%llx+0x%llx: memory at 0x%llx (type 0x%lx, protection 0x%lx) is neither read-write, read-only nor a shared mapping", static_cast(base), static_cast(bytes), reinterpret_cast(refused.BaseAddress), refused.Type, refused.Protect); + std::snprintf(text, sizeof(text), "AGC graphics: host import of 0x%llx+0x%llx: memory at 0x%llx (type 0x%lx, protection 0x%lx) is neither read-write, read-only nor a shared mapping", static_cast(base), static_cast(bytes), reinterpret_cast(pages.refused.BaseAddress), pages.refused.Type, pages.refused.Protect); throw std::runtime_error(text); } + std::erase_if(state.gateRefusals, [&](const auto& refusal) { return refusal.first.first < base + bytes && base < refusal.first.first + refusal.first.second; }); try { entry.alias = GuestArena::GuestArenaMapAlias_nid_postfix(static_cast(base), static_cast(bytes)); } catch (const std::runtime_error&) { @@ -1745,17 +1839,16 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import #else if (!GuestMemory::Accessible(reinterpret_cast(base), static_cast(bytes), true)) return failedImport(); #endif - decideImportWatch(context, state); VkResult result = VK_SUCCESS; const char* step = nullptr; #ifdef _WIN32 - const bool settled = entry.alias == nullptr && state.watchPrivateImports && writable && bytes >= settleFloorBytes; - const bool watchedHostPointer = entry.alias != nullptr || !state.unwatchImports || settled; - const bool watchedDmaBuf = watchedHostPointer; + const auto watching = importWatching(context, state, entry.alias != nullptr, writable, bytes); + const bool settled = watching.settled; #else - const bool watchedHostPointer = !state.unwatchImports; - const bool watchedDmaBuf = !state.unwatchDmaBufImports; + const auto watching = importWatching(context, state, false, true, bytes); #endif + const bool watchedHostPointer = watching.hostPointer; + const bool watchedDmaBuf = watching.dmaBuf; GuestMemory::ImportWatched(base, bytes, [&] { step = watchedDmaBuf == watchedHostPointer ? createImport(context, entry, result) : createHostPointerImport(context, entry, result); #ifdef _WIN32 @@ -1883,6 +1976,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc state.failedBlocks.clear(); state.refusals.clear(); state.evictingRefusals.clear(); + state.gateRefusals.clear(); state.ceiling = 0; state.allocationFailures = 0; state.allocationFailed = false; @@ -1900,6 +1994,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc const auto generation = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); if (generation == state.refreshedGeneration) return; state.refreshedGeneration = generation; + state.gateRefusals.clear(); const auto starts = [&](std::uint64_t address) { const auto index = leasedIndexContaining(lease, address); return index != lease.size() && lease[index]->address == address; @@ -3029,6 +3124,7 @@ void ClearHostImports(VkDevice device) { state.failedBlocks.clear(); state.refusals.clear(); state.evictingRefusals.clear(); + state.gateRefusals.clear(); state.ceiling = 0; state.allocationFailures = 0; state.allocationFailed = false; @@ -3252,6 +3348,7 @@ void SetImportWatch(const Context& context, ImportWatch watch) { state.watchDevice = context.device; state.unwatchImports = watch == ImportWatch::Unwatch; state.unwatchDmaBufImports = state.unwatchImports; + state.gateRefusals.clear(); } void SetHostImportWindow(std::uint64_t bytes) { @@ -3329,6 +3426,7 @@ const char* ImportRequesterName(ImportRequester requester) { case ImportRequester::Access: return "access"; case ImportRequester::Join: return "join"; case ImportRequester::Segment: return "segment"; + case ImportRequester::ColorTarget: return "color target"; } return "unknown"; } diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp index 860ba51020..1af667ae60 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp @@ -1038,7 +1038,7 @@ void importRequesterTests(const Context& context, const BdaTestAccess& access) { }; const auto counted = [](ImportRequester requester) { return HostImportCounters().requesters[static_cast(requester)]; }; const auto made = [&](ImportRequester requester) { return counted(requester).made; }; - Require(std::string(ImportRequesterName(ImportRequester::Map)) == "map" && std::string(ImportRequesterName(ImportRequester::Build)) == "build" && std::string(ImportRequesterName(ImportRequester::Access)) == "access" && std::string(ImportRequesterName(ImportRequester::Join)) == "join" && std::string(ImportRequesterName(ImportRequester::Segment)) == "segment", "an import requester has the wrong name"); + Require(std::string(ImportRequesterName(ImportRequester::Map)) == "map" && std::string(ImportRequesterName(ImportRequester::Build)) == "build" && std::string(ImportRequesterName(ImportRequester::Access)) == "access" && std::string(ImportRequesterName(ImportRequester::Join)) == "join" && std::string(ImportRequesterName(ImportRequester::Segment)) == "segment" && std::string(ImportRequesterName(ImportRequester::ColorTarget)) == "color target", "an import requester has the wrong name"); change(0, true); { GuestAllocations::Mapped ranges; @@ -1124,9 +1124,25 @@ void importRequesterTests(const Context& context, const BdaTestAccess& access) { SetHostImportWindow(0); ::operator delete(joined, std::align_val_t{window}); } + change(6, true); + change(7, true); + { + const auto decided = PrepareImportWatch(importing); + const auto before = counted(ImportRequester::ColorTarget); + SetImportWatch(importing, ImportWatch::Unwatch); + const auto* compared = HostImportFor(importing, address(6), 16, ImportRequester::ColorTarget); + const auto refused = counted(ImportRequester::ColorTarget); + SetImportWatch(importing, ImportWatch::Watch); + const auto* readOnly = HostImportFor(importing, address(7), 16, ImportRequester::ColorTarget); + const auto* outside = HostImportFor(importing, address(6), 16, ImportRequester::ColorTarget); + SetImportWatch(importing, decided); + Require(compared == nullptr && refused.made == before.made && refused.refused == before.refused && refused.failed == before.failed, "a color target's import was made where imports of its memory are compared"); + Require(readOnly == nullptr && !HostImportCovers(importing, address(7), piece), "a color target's import was made of registered memory the guest cannot write"); + Require(outside == nullptr && !HostImportCovers(importing, address(6), piece) && made(ImportRequester::ColorTarget) == before.made, "a color target's import was made of memory outside the write watch"); + } const auto counts = HostImportCounters(); Require(counts.live >= 4 && counts.liveBytes == HostImportLiveBytes(), "the live import counts disagree with the live bytes"); - for (std::size_t index = 0; index < 6; ++index) change(index, false); + for (std::size_t index = 0; index < pieces; ++index) change(index, false); ClearHostImports(importing.device); ::operator delete(block, std::align_val_t{piece}); } @@ -1138,8 +1154,9 @@ void hostImportReportTests() { counts.requesters[static_cast(ImportRequester::Map)] = {2, 1u << 20u, 0, 0, 0}; counts.requesters[static_cast(ImportRequester::Build)] = {5, 4u << 20u, 7, 7u << 19u, 1, 4}; counts.requesters[static_cast(ImportRequester::Segment)] = {1, 1u << 16u, 0, 0, 2}; + counts.requesters[static_cast(ImportRequester::ColorTarget)] = {1, 1u << 20u, 0, 0, 0}; const auto text = FormatHostImportCounts(counts); - Require(text == "3 live (3.0 MiB); made 8 (5.1 MiB), refused 7 (3.5 MiB, 4 remembered), failed 3; by requester (made/refused/failed): map 2/0/0, build 5/7/1, access 0/0/0, join 0/0/0, segment 1/0/2", "the host import report is wrong: " + text); + Require(text == "3 live (3.0 MiB); made 9 (6.1 MiB), refused 7 (3.5 MiB, 4 remembered), failed 3; by requester (made/refused/failed): map 2/0/0, build 5/7/1, access 0/0/0, join 0/0/0, segment 1/0/2, color target 1/0/0", "the host import report is wrong: " + text); } class PieceWindow { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetImportCreation.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetImportCreation.cpp new file mode 100644 index 0000000000..54c1f56ea7 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetImportCreation.cpp @@ -0,0 +1,495 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver { + +struct VulkanDeviceTestAccess { + static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); } +}; + +} + +namespace { + +using AgcDriver::Graphics::Require; +using AgcDriver::Graphics::StorageTexture; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 16; +constexpr std::size_t BlockBytes = 65536; +constexpr std::size_t SettledBytes = 16 * BlockBytes; +constexpr std::size_t ResidentBytes = 32 * BlockBytes; +constexpr std::size_t TargetBytes = std::size_t{Width} * Height * 16u; +constexpr std::size_t RowBytes = std::size_t{Width} * 16u; +constexpr std::uint32_t Rgba32FloatInfo = 0x8738u; +constexpr std::uint32_t LinearAttrib3 = 0x09000000u; +constexpr std::uint32_t Float32AbgrExport = 9; +constexpr VkColorComponentFlags RedGreen = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +constexpr std::array, 3> Triangle{{ + {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f} +}}; + +alignas(256) std::array PixelCode{}; + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +std::span Program() { + std::size_t at = 0; + PixelCode[at++] = 0x36000000u | (20u << 17u) | 0xffu; + PixelCode[at++] = 0x0000ffffu; + PixelCode[at++] = 0x2c000000u | (21u << 17u) | 0x90u; + PixelCode[at++] = 0x7e000c00u | (20u << 17u) | (256u + 20u); + PixelCode[at++] = 0x7e000c00u | (21u << 17u) | (256u + 21u); + PixelCode[at++] = 0x7e000280u | (22u << 17u); + PixelCode[at++] = 0x7e0002f2u | (23u << 17u); + PixelCode[at++] = 0xf800180fu; + PixelCode[at++] = 0x17161514u; + PixelCode[at++] = 0xbf810000u; + return std::span(PixelCode.data(), at); +} + +struct Block { + Block(std::size_t bytes, std::uintptr_t at, bool direct = false, std::size_t granule = 0, bool writable = true) : bytes(bytes), arena(AgcDriver::GuestMemory::WriteWatched()) { +#ifdef _WIN32 + if (arena && direct) { + data = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, 0x200000)); + Require(data != nullptr, "cannot allocate the direct memory of the color target"); + section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE | SEC_RESERVE, 0, static_cast(bytes), nullptr); + Require(section != nullptr, "cannot reserve the backing of the color target's direct memory"); + GuestArena::GuestArenaMapDirect_nid_postfix(data, bytes, section, 0, PAGE_READWRITE); + } else if (arena) { + data = static_cast(at != 0 ? GuestArena::GuestArenaAllocateAtOrAbove_nid_postfix(at, bytes, BlockBytes) : GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes)); + GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, granule != 0 ? granule : bytes); + } else { + data = static_cast(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + } +#else + static_cast(direct); + static_cast(granule); + if (arena && at != 0) { + void* raw = mmap(reinterpret_cast(at), bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(raw != MAP_FAILED, "cannot map the color target"); + data = static_cast(raw); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes); + } else if (arena) { + void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(raw != MAP_FAILED, "cannot map the color target"); + const auto begin = reinterpret_cast(raw); + const auto aligned = (begin + BlockBytes - 1) & ~(static_cast(BlockBytes) - 1); + if (aligned != begin) munmap(raw, aligned - begin); + if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast(aligned + bytes), begin + BlockBytes - aligned); + data = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes); + } else { + data = static_cast(std::aligned_alloc(BlockBytes, bytes)); + } +#endif + Require(data != nullptr, "cannot allocate the color target"); + GuestAllocations::Mutation mutation; + mutation.Add(data, bytes, true, writable, true); + } + ~Block() { + StorageTexture::FlushPending(Address(), bytes, nullptr, "test release"); + { + GuestAllocations::Mutation mutation; + mutation.Remove(data); + } +#ifdef _WIN32 + if (arena) { + GuestArena::GuestArenaReset_nid_postfix(data, bytes); + GuestArena::GuestArenaRelease_nid_postfix(data, bytes); + } else { + VirtualFree(data, 0, MEM_RELEASE); + } + if (section != nullptr) { + GuestArena::GuestArenaForgetSection_nid_postfix(section); + CloseHandle(section); + } +#else + if (arena) { + munmap(data, bytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes); + } else { + std::free(data); + } +#endif + } + Block(const Block&) = delete; + Block& operator=(const Block&) = delete; + std::uint64_t Address() const { return reinterpret_cast(data); } + bool Watched() const { return arena && AgcDriver::GuestMemory::Watched(Address(), bytes); } + std::size_t bytes; + bool arena; + std::uint8_t* data = nullptr; +#ifdef _WIN32 + HANDLE section = nullptr; +#endif +}; + +AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block) { + AgcDriver::Registers cx; + cx[0x318] = static_cast(block.Address() >> 8u); + cx[0x31b] = 0; + cx[0x31c] = Rgba32FloatInfo; + cx[0x31d] = 0; + cx[0x323] = 0; + cx[0x324] = 0; + cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u); + cx[0x3b8] = LinearAttrib3; + cx[0x390] = static_cast(block.Address() >> 40u); + auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0); + color.exportIndex = 0; + Require(color.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && color.bytes == TargetBytes, "the target is not a linear R32G32B32A32 target"); + return color; +} + +void Draw(AgcDriver::VulkanDevice& device, const Block& block) { + const auto color = DecodeTarget(block); + const auto target = device.Target(); + constexpr std::uint32_t waveSize = 64; + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast(Triangle.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + const auto pixelCode = Program(); + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionFixedPoint); + pixel.targetOutputMode.at(0) = Float32AbgrExport; + pixel.targetExportMapping.fill(0xe4u); + const std::vector pixelUserData(8, 0u); + const std::array pixelMemory{{{reinterpret_cast(pixelCode.data()), std::as_bytes(pixelCode)}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(pixelCode.data()), pixelCode, 0, {}}, + {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + target, + PixelPushLayout(vertexPush, target) + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} + }}; + + VkPipelineColorBlendAttachmentState blend{}; + blend.blendEnable = VK_FALSE; + blend.colorWriteMask = RedGreen; + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt}; + state.colors = {color}; + state.color = color; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blends = {blend}; + state.blend = blend; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(Triangle.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); +} + +void Seed(Block& block, const std::array& texel) { + for (std::size_t offset = 0; offset < TargetBytes; offset += sizeof(texel)) std::memcpy(block.data + offset, texel.data(), sizeof(texel)); +} + +void Check(const Block& block, const std::array& seed, const std::string& what) { + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + std::array texel{}; + std::memcpy(texel.data(), block.data + static_cast(y) * RowBytes + static_cast(x) * 16u, sizeof(texel)); + const std::array expected{static_cast(x), static_cast(y), seed[2], seed[3]}; + Require(texel == expected, what + ": pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") holds (" + std::to_string(texel[0]) + ", " + std::to_string(texel[1]) + ", " + std::to_string(texel[2]) + ", " + std::to_string(texel[3]) + ")"); + } + } +} + +void ExpectRecorded(const AgcDriver::Graphics::Context& context, const Block& block, const std::string& what) { + const auto* recorder = AgcDriver::Graphics::Recorder::Active(); + Require(recorder != nullptr && !recorder->Idle(), what + ": the draw into a linear target in registered, write-watched memory without a host import waited for the GPU instead of being recorded"); + Require(StorageTexture::FindPending(block.Address(), TargetBytes) != nullptr, what + ": the drawn linear target has no pending resident image"); + Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) != nullptr, what + ": the draw made no host import of the linear target"); + Require(block.Watched(), what + ": the linear target's host import took its memory out of the write watch"); +} + +void Created(AgcDriver::VulkanDevice& device) { + const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device); + constexpr std::array seed{100.0f, 200.0f, 300.0f, 400.0f}; + Block block(SettledBytes, 0); + Seed(block, seed); + Require(block.Watched(), "import created: the color target is not write-watched"); + Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) == nullptr, "import created: the color target has a host import before the draw"); + Draw(device, block); + ExpectRecorded(context, block, "import created"); + device.WaitIdle(); + Require(block.Watched(), "import created: the color target left the write watch after its store"); + Check(block, seed, "import created, after the device wait"); +} + +bool Reused(AgcDriver::VulkanDevice& device) { + const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device); + constexpr std::array first{100.0f, 200.0f, 300.0f, 400.0f}; + constexpr std::array second{500.0f, 600.0f, 700.0f, 800.0f}; + std::uint64_t address = 0; + { + Block block(SettledBytes, 0); + address = block.Address(); + Seed(block, first); + Draw(device, block); + ExpectRecorded(context, block, "reuse, first block"); + device.WaitIdle(); + Check(block, first, "reuse, first block"); + } + Block block(SettledBytes, static_cast(address)); + if (block.Address() != address) { + std::puts("linear target import reuse skipped: the second block was not mapped at the first one's address"); + return false; + } + Require(block.Watched(), "reuse: the block allocated at the freed block's address is not write-watched"); + Seed(block, second); + Require(block.Watched(), "reuse: the CPU's store took the second block out of the write watch"); + Draw(device, block); + ExpectRecorded(context, block, "reuse, second block"); + device.WaitIdle(); + Check(block, second, "reuse, the draw into the second block did not start from the CPU's bytes"); + return true; +} + +#ifdef _WIN32 +bool ArmedReadOnly(const Block& block) { + MEMORY_BASIC_INFORMATION memory{}; + return VirtualQuery(block.data, &memory, sizeof(memory)) == sizeof(memory) && memory.Type == MEM_PRIVATE && (memory.Protect & 0xffu) == PAGE_READONLY; +} + +bool Arm(const Block& block) { + for (int collect = 0; collect < 3; ++collect) static_cast(AgcDriver::GuestMemory::CollectWritesUncached(block.Address(), block.bytes)); + return ArmedReadOnly(block); +} + +AgcDriver::Graphics::HostImportRequesterCounts ColorTargetCounts() { + return AgcDriver::Graphics::HostImportCounters().requesters[static_cast(AgcDriver::Graphics::ImportRequester::ColorTarget)]; +} + +bool Armed(AgcDriver::VulkanDevice& device) { + const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device); + constexpr std::array seed{1300.0f, 1400.0f, 1500.0f, 1600.0f}; + Block block(ResidentBytes, 0, true); + Seed(block, seed); + if (!Arm(block)) { + std::puts("linear target import of armed direct memory skipped: the collects did not arm the resident run read-only"); + return false; + } + Require(block.Watched(), "armed direct memory: the color target is not write-watched"); + Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) == nullptr, "armed direct memory: the color target has a host import before the draw"); + Draw(device, block); + ExpectRecorded(context, block, "armed direct memory"); + device.WaitIdle(); + Require(block.Watched(), "armed direct memory: the color target left the write watch after its store"); + Check(block, seed, "armed direct memory, after the device wait"); + return true; +} + +bool ArmedGate(AgcDriver::VulkanDevice& device) { + using AgcDriver::Graphics::ImportRequester; + using AgcDriver::Graphics::ImportWatch; + const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device); + Block armed(ResidentBytes / 4, 0, true); + if (!Arm(armed)) { + std::puts("linear target import gate over armed direct memory skipped: the collects did not arm the resident run read-only"); + return false; + } + Block flexible(2 * BlockBytes, 0); + GuestArena::GuestArenaCommit_nid_postfix(flexible.data + BlockBytes, BlockBytes, PAGE_READONLY, BlockBytes); + const auto decided = AgcDriver::Graphics::PrepareImportWatch(context); + const auto before = ColorTargetCounts(); + AgcDriver::Graphics::SetImportWatch(context, ImportWatch::Unwatch); + const auto* compared = AgcDriver::Graphics::HostImportFor(context, armed.Address(), TargetBytes, ImportRequester::ColorTarget); + const bool stillArmed = ArmedReadOnly(armed); + AgcDriver::Graphics::SetImportWatch(context, ImportWatch::Watch); + const auto* unopened = AgcDriver::Graphics::HostImportFor(context, flexible.Address(), TargetBytes, ImportRequester::ColorTarget); + const auto refused = ColorTargetCounts(); + const auto* watched = AgcDriver::Graphics::HostImportFor(context, armed.Address(), TargetBytes, ImportRequester::ColorTarget); + const auto made = ColorTargetCounts(); + AgcDriver::Graphics::SetImportWatch(context, decided); + Require(compared == nullptr && stillArmed, "armed gate: a color target's import of armed direct memory under the settle floor was made where it would be compared, or deciding it opened the armed run"); + Require(unopened == nullptr && refused.made == before.made && refused.refused == before.refused && refused.failed == before.failed, "armed gate: a color target's import was made of read-only private memory that its import does not open, or a refused color target import was counted"); + Require(watched != nullptr && made.made == before.made + 1 && made.failed == before.failed, "armed gate: a color target's import of armed direct memory registered writable was not made where imports stay watched"); + return true; +} +#endif + +void OutsideWatch(AgcDriver::VulkanDevice& device) { + const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device); + constexpr std::array seed{1700.0f, 1800.0f, 1900.0f, 2000.0f}; + Block block(SettledBytes, 0); + Seed(block, seed); + AgcDriver::GuestMemory::Unwatch(block.Address() + SettledBytes - BlockBytes, BlockBytes); + Require(AgcDriver::GuestMemory::Watched(block.Address(), TargetBytes) && !block.Watched(), "run outside the write watch: the target is not watched, or the rest of its block still is"); + Draw(device, block); + const auto* recorder = AgcDriver::Graphics::Recorder::Active(); + Require(recorder == nullptr || recorder->Idle(), "run outside the write watch: a linear target whose import run has a block outside the write watch was drawn into a resident image"); + Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) == nullptr, "run outside the write watch: the draw made a host import over a block outside the write watch"); + Require(AgcDriver::GuestMemory::Watched(block.Address(), TargetBytes), "run outside the write watch: the draw took the target out of the write watch"); + device.WaitIdle(); + Check(block, seed, "run outside the write watch, after the device wait"); +} + +void ReadOnlyRegistry(AgcDriver::VulkanDevice& device) { + const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device); + constexpr std::array seed{2500.0f, 2600.0f, 2700.0f, 2800.0f}; + Block block(SettledBytes, 0, false, 0, false); + Seed(block, seed); + Require(block.Watched(), "range registered read-only: the color target is not write-watched"); + Draw(device, block); + const auto* recorder = AgcDriver::Graphics::Recorder::Active(); + Require(recorder == nullptr || recorder->Idle(), "range registered read-only: a linear target in memory the guest registered read-only was drawn into a resident image"); + Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) == nullptr, "range registered read-only: the draw made a host import of memory the guest registered read-only"); + device.WaitIdle(); + Check(block, seed, "range registered read-only, after the device wait"); +} + +#ifdef _WIN32 +void Rewatched(AgcDriver::VulkanDevice& device) { + const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device); + constexpr std::array seed{2100.0f, 2200.0f, 2300.0f, 2400.0f}; + Block block(SettledBytes, 0, false, BlockBytes); + Seed(block, seed); + auto* last = block.data + SettledBytes - BlockBytes; + AgcDriver::GuestMemory::Unwatch(block.Address() + SettledBytes - BlockBytes, BlockBytes); + Draw(device, block); + const auto* recorder = AgcDriver::Graphics::Recorder::Active(); + Require(recorder == nullptr || recorder->Idle(), "run back in the write watch: the draw before the block came back was drawn into a resident image"); + device.WaitIdle(); + GuestArena::GuestArenaReset_nid_postfix(last, BlockBytes); + GuestArena::GuestArenaCommit_nid_postfix(last, BlockBytes, PAGE_READWRITE, BlockBytes); + Require(block.Watched(), "run back in the write watch: committing the block again did not bring it back into the write watch"); + Draw(device, block); + ExpectRecorded(context, block, "run back in the write watch, a refused run whose block came back"); + device.WaitIdle(); + Check(block, seed, "run back in the write watch, after the device wait"); +} +#endif + +void Compared(AgcDriver::VulkanDevice& device) { + using AgcDriver::Graphics::ImportWatch; + const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device); + constexpr std::array seed{900.0f, 1000.0f, 1100.0f, 1200.0f}; + const auto decided = AgcDriver::Graphics::PrepareImportWatch(context); + AgcDriver::Graphics::SetImportWatch(context, ImportWatch::Unwatch); + Block block(BlockBytes, 0); + Seed(block, seed); + Require(block.Watched(), "compared import: the color target is not write-watched"); + Draw(device, block); + const auto* recorder = AgcDriver::Graphics::Recorder::Active(); + Require(recorder == nullptr || recorder->Idle(), "compared import: a linear target whose host import would be compared was drawn into a resident image"); + Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) == nullptr, "compared import: the draw made a host import that leaves the target's memory compared"); + Require(StorageTexture::FindPending(block.Address(), TargetBytes) == nullptr, "compared import: the drawn linear target has a pending resident image"); + Require(block.Watched(), "compared import: the draw took the color target out of the write watch"); + device.WaitIdle(); + Check(block, seed, "compared import, after the device wait"); + AgcDriver::Graphics::SetImportWatch(context, decided); +} + +} + +int main(int argc, char** argv) { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (!AgcDriver::GuestMemory::WriteWatched()) { + std::puts("skipped, guest memory is not write-watched"); + return VulkanTestSkipped; + } + const std::string_view only = argc > 1 ? argv[1] : ""; +#ifndef _WIN32 + const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(*device); + const auto decided = AgcDriver::Graphics::PrepareImportWatch(context); + AgcDriver::Graphics::SetImportWatch(context, AgcDriver::Graphics::ImportWatch::Watch); + struct Restore { + const AgcDriver::Graphics::Context& context; + AgcDriver::Graphics::ImportWatch decided; + ~Restore() { AgcDriver::Graphics::SetImportWatch(context, decided); } + } restore{context, decided}; +#endif + Block probe(SettledBytes, 0); + if (AgcDriver::Graphics::HostImportFor(AgcDriver::VulkanDeviceTestAccess::ContextOf(*device), probe.Address(), probe.bytes) == nullptr) { + std::puts("skipped, host imports are unavailable"); + return VulkanTestSkipped; + } + const bool watched = probe.Watched(); +#ifndef _WIN32 + Require(watched, "a host import of write-watched memory left the write watch although imports are set to stay watched"); +#endif + if (!watched) std::puts("linear target import creation: a host import of write-watched memory does not stay watched on this device; only the compared cases run"); + if (watched && (only.empty() || only == "--created")) Created(*device); + if (watched && (only.empty() || only == "--reused") && Reused(*device)) std::puts("linear target import reuse passed"); +#ifdef _WIN32 + if (watched && (only.empty() || only == "--armed") && Armed(*device)) std::puts("linear target import of armed direct memory passed"); + if ((only.empty() || only == "--armed-gate") && ArmedGate(*device)) std::puts("linear target import gate over armed and read-only private memory passed"); +#endif + if (watched && (only.empty() || only == "--outside-watch")) OutsideWatch(*device); + if (watched && (only.empty() || only == "--read-only-registry")) ReadOnlyRegistry(*device); +#ifdef _WIN32 + if (watched && (only.empty() || only == "--rewatched")) Rewatched(*device); +#endif + if (only.empty() || only == "--compared") Compared(*device); + std::puts("linear target import creation tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 694b495ef0..17b8198ea6 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -291,7 +291,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_RECORDER_SYNC_DRAWS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | gives the draws the recorder waits for their own batch instead of recording them with the wait | | `APS5_NO_RECORD_COPIED_DRAWS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | waits for copied-write draws at once instead of listing their write-back for later | | `APS5_NO_RECT_INDIRECT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp` | | sends an indirect rectangle-list draw down the CPU path instead of the GPU-side path | -| `APS5_NO_RESIDENT_LINEAR` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | drops the residency of a linear color target whose address already has a host import, so it is copied in and out again | +| `APS5_NO_RESIDENT_LINEAR` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | drops the residency of linear color targets, so a target whose address has a host import, or whose import a draw would make, is copied in and out again | | `APS5_NO_RESIDENT_PRESENT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | always presents through guest memory instead of the resident image | | `APS5_NO_RESIDENT_TARGETS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | copies every render target in and out again instead of keeping it resident | | `APS5_NO_RESOURCE_CACHE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | disables the shared resource cache, so every dispatch builds its resources fresh | diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 6bb62ca715..a45805af0e 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -111,6 +111,7 @@ - [Draw input snapshots](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver): `PrepareDrawBindings` caps read-only input buffer snapshots at 64 MiB (overridable via `APS5_DRAW_SNAPSHOT_MAX_MIB`). Larger inputs (streaming heaps, bindless pools) or draws exceeding the cumulative budget skip the per-draw snapshot and bind in place, avoiding multi-gigabyte CPU copy stalls (such as Ratchet & Clank: Rift Apart PPSA01474 stalling at frame ~380 copying 36 GB every 5 s). If the title rewrites those large ranges on the CPU before GPU completion, the in-place binding observes the modification, as verified by `drawSnapshotCapTests`. - [Pending storage-image results](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver), except drawn resident linear targets (next entry), are stored when something reads their memory, not when the GPU work that wrote them completes. A title that waits for that work, frees the image and reuses its memory can have the new data overwritten by the late store (#1970). A title's own CPU reads are not seen either, so a title that reads the results after waiting for a label reads stale memory. Explicit-size file writes (`write`, `pwrite`, `writev`, `pwritev`, AIO and `fwrite`) synchronize their source ranges through the [host-read hook](../../../core/libs/prx/libc/src/GuestArena.cpp); other host readers are not covered. `APS5_STORE_AT_FLIP=1` stores every pending result at each flip and `APS5_STORE_AT_LABEL=1` before each label the GPU writes; both store every pending image, not only the ones the waited-for work wrote, and neither is on by default. - [Resident linear color targets](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver): a CPU store inside a drawn scissor rectangle, on a pixel no primitive wrote, made before the target's store is overwritten by it, where the console keeps it. Recorded draws' results are stored before the graphics queue's next flip, label, end-of-pipe interrupt, drain or device wait, as the union of their scissor rectangles; a draw that brings a 17th rectangle that does not merge stores the target at once. A target with an attachment proxy, several mips or layers, a 3D or DCC surface, or results of a dispatch, fill or copy is stored after each draw. One that a fill or copy writes after its draws is stored whole at the next of those points, and GPU results in its CPU-written 64 KiB blocks are dropped. A store the draw pipeline's committer makes (label, end-of-pipe interrupt, device wait) skips a target whose range a queued draw still reads (`DrawPipeline::HeldReads`). +- [Linear color targets](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) with no host import are drawn synchronously (copied in, drawn, waited for, copied back), where the console renders into memory in stream order, unless the draw can make an import that keeps them write-watched. It makes one for a target in registered, writable memory only when the whole import run is write-watched and the import stays watched: a separate alias of shared memory, a direct import of 1 MiB or more of private memory that settles on Windows (when the import probe keeps the watch), or one the probe keeps watched elsewhere. Smaller private ranges on Windows, runs with memory outside the write watch and devices whose imports are compared stay synchronous: a compared import makes every refresh of the resident image compare the whole target. A target over an existing import is resident, compared or not. `APS5_NO_RESIDENT_LINEAR=1` keeps every linear target synchronous. - [`APS5_SKIP_REFUSED_DRAWS`](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp) (libSceAgcDriver) drops a draw whose graphics stage the recompiler refuses instead of failing the queue, so the frame lacks that draw, and the stage is compiled and refused again at each such draw. Compute refusals still fail. - [Pending storage-image results over memory outside the write watch](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) leave to the CPU the 64 KiB blocks whose bytes differ from the ones saved at the upload, the fill, the image copy, the previous store or the moment their range left the watch (`Unwatch`, or a host import that does not stay watched), and the blocks the watch saw the CPU write before that moment. A store made on the GPU saves its bytes through a host-visible copy that becomes the saved bytes when its batch completes; a store or a refresh that needs them earlier waits for that batch. A unit without a write generation saves the guest bytes when results land in it outside the watch. One whose range leaves the watch while its results are pending saves its bytes through the unwatch hook, which also runs when a failed collect takes a range out of the watch. A range that leaves the watch without the hook running over it (an import fallback that unwatched without saving) leaves its pending results without saved bytes; storing such results throws. A unit whose results were stored while its blocks were watched keeps only a generation; when its blocks leave the watch while nothing is pending, the hook saves nothing, and results that land in it afterwards have neither saved bytes nor a watch, so storing them throws too. - Windows host-pointer imports of the guest mapping itself are compared instead of write-watched, except writable imports of private `MEM_WRITE_WATCH` memory when the import probe keeps the write watch and a settle queue exists: those of 1 MiB or more stay watched, and smaller ones return to the watch once their settle has completed, unless a storage image overlaps them (see Pages under an import window): a driver can mark imported pages dirty after the synchronous import window. Shared memory imported through a separate writable alias retains protection-based tracking on the guest views; driver access to the import alias does not disarm those views. [GuestMemory](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) excludes every overlapping 64 KiB tracker block while its backing is retained. A full block replaced by fresh private backing regains tracking and gets a new generation; protection changes, partial replacements and shared remappings do not restore it. Excluded blocks keep their write-watch marks as a hint only (`UntrackedHint`): a storage image or a sampled texture snapshot over them compares its bytes again after a mark in its range, a driver store, a mapping change or a recorded GPU write instead of at every collect epoch or lookup, and a mark the driver makes without a CPU store costs one comparison. This preserves CPU edits through byte comparisons but limits snapshot reuse on excluded blocks to draws of one collect epoch and disables unit shadows for them; restoring tracking on import retirement without replacing the backing remains unimplemented. From 33a1577f84297115d910640d02e5176b1f072d8c Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Sun, 25 Oct 2026 12:57:08 +1300 Subject: [PATCH 2/3] test(libSceAgcDriver): use the public headers instead of local declarations --- .../tests/execution/LinearTargetCompletionPoints.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetCompletionPoints.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetCompletionPoints.cpp index 28c1d9aecc..8c8bd50ef9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetCompletionPoints.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetCompletionPoints.cpp @@ -1,3 +1,4 @@ +#include "prx/libkernel/include/Api.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" @@ -39,8 +40,6 @@ #include #include -extern "C" int APS5_VABI sceKernelCreateEqueue(KernelEqueue* eq, const char* name); -extern "C" int APS5_VABI sceKernelDeleteEqueue(KernelEqueue eq); namespace AgcDriver { From e1fa56c7bf35b51e0a7c52d86eae0abf64a390c8 Mon Sep 17 00:00:00 2001 From: thuanquang Date: Mon, 26 Oct 2026 03:29:04 +0700 Subject: [PATCH 3/3] perf(agc): drop the armed-page case from the color target import gate #9181 stopped write-protecting resident runs on Windows: the write watch no longer arms them read-only, and the import access opens nothing on them. The gate's case that counted armed read-only private pages as writable can no longer change a decision, and the tests that armed a resident run first could only skip. Remove the case: resident runs are read-write private pages and pass the writable check as they are. --armed becomes --direct, the same 2 MiB direct-memory run without the arming step. --armed-gate becomes --private-gate and keeps the checks that can still be reached: the direct run is refused under Unwatch and imported under Watch, and a block committed read-only while registered writable is refused without being counted. Co-Authored-By: Claude --- .../Graphics/src/GuestBufferMemory.cpp | 10 ++-- .../execution/LinearTargetImportCreation.cpp | 53 ++++++------------- 2 files changed, 19 insertions(+), 44 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index bb3d0feae5..f407bc9649 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -1289,7 +1289,6 @@ ImportWatching importWatching(const Context& context, HostImports& state, bool a struct ImportPages { bool committed = true; bool writable = true; - bool opened = true; bool readOnly = true; bool shared = true; MEMORY_BASIC_INFORMATION refused{}; @@ -1311,14 +1310,11 @@ ImportPages importPages(std::uint64_t base, std::uint64_t bytes, bool opening = const auto protection = info.Protect & 0xffu; const bool pageWritable = (info.Type == MEM_PRIVATE || info.Type == MEM_IMAGE) && writableProtection(protection); const bool pageReadOnly = info.Type != MEM_MAPPED && (protection == PAGE_READONLY || protection == PAGE_EXECUTE_READ); - std::uint32_t guestProtection = 0; - const bool armed = opening && info.Type == MEM_PRIVATE && pageReadOnly && GuestArena::GuestArenaProtection_nid_postfix(static_cast(cursor), &guestProtection) && writableProtection(guestProtection); if (!pageReadOnly) pages.readOnly = false; if (info.Type != MEM_MAPPED) pages.shared = false; if (info.Type != MEM_MAPPED && !pageWritable && !pageReadOnly && pages.refused.BaseAddress == nullptr) pages.refused = info; if (!pageWritable) pages.writable = false; - if (!pageWritable && !armed) pages.opened = false; - if (opening && !pages.opened && !pages.shared) return pages; + if (opening && !pages.writable && !pages.shared) return pages; cursor = reinterpret_cast(info.BaseAddress) + info.RegionSize; } return pages; @@ -1350,8 +1346,8 @@ bool importStaysWatched(const Context& context, HostImports& state, const Import MEMORY_BASIC_INFORMATION first{}; if (!importWatching(context, state, false, true, run.bytes).hostPointer && (VirtualQuery(reinterpret_cast(run.base), &first, sizeof(first)) == 0 || first.Type != MEM_MAPPED)) return false; const auto pages = importPages(run.base, run.bytes, true); - if (!pages.committed || (!pages.opened && !pages.shared)) return false; - return importWatching(context, state, !pages.opened, pages.opened, run.bytes).hostPointer; + if (!pages.committed || (!pages.writable && !pages.shared)) return false; + return importWatching(context, state, !pages.writable, pages.writable, run.bytes).hostPointer; #else const auto watching = importWatching(context, state, false, true, run.bytes); BackingFile backing; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetImportCreation.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetImportCreation.cpp index 54c1f56ea7..adb40038d9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetImportCreation.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetImportCreation.cpp @@ -309,65 +309,44 @@ bool Reused(AgcDriver::VulkanDevice& device) { } #ifdef _WIN32 -bool ArmedReadOnly(const Block& block) { - MEMORY_BASIC_INFORMATION memory{}; - return VirtualQuery(block.data, &memory, sizeof(memory)) == sizeof(memory) && memory.Type == MEM_PRIVATE && (memory.Protect & 0xffu) == PAGE_READONLY; -} - -bool Arm(const Block& block) { - for (int collect = 0; collect < 3; ++collect) static_cast(AgcDriver::GuestMemory::CollectWritesUncached(block.Address(), block.bytes)); - return ArmedReadOnly(block); -} - AgcDriver::Graphics::HostImportRequesterCounts ColorTargetCounts() { return AgcDriver::Graphics::HostImportCounters().requesters[static_cast(AgcDriver::Graphics::ImportRequester::ColorTarget)]; } -bool Armed(AgcDriver::VulkanDevice& device) { +void Direct(AgcDriver::VulkanDevice& device) { const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device); constexpr std::array seed{1300.0f, 1400.0f, 1500.0f, 1600.0f}; Block block(ResidentBytes, 0, true); Seed(block, seed); - if (!Arm(block)) { - std::puts("linear target import of armed direct memory skipped: the collects did not arm the resident run read-only"); - return false; - } - Require(block.Watched(), "armed direct memory: the color target is not write-watched"); - Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) == nullptr, "armed direct memory: the color target has a host import before the draw"); + Require(block.Watched(), "direct memory: the color target is not write-watched"); + Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) == nullptr, "direct memory: the color target has a host import before the draw"); Draw(device, block); - ExpectRecorded(context, block, "armed direct memory"); + ExpectRecorded(context, block, "direct memory"); device.WaitIdle(); - Require(block.Watched(), "armed direct memory: the color target left the write watch after its store"); - Check(block, seed, "armed direct memory, after the device wait"); - return true; + Require(block.Watched(), "direct memory: the color target left the write watch after its store"); + Check(block, seed, "direct memory, after the device wait"); } -bool ArmedGate(AgcDriver::VulkanDevice& device) { +void PrivateGate(AgcDriver::VulkanDevice& device) { using AgcDriver::Graphics::ImportRequester; using AgcDriver::Graphics::ImportWatch; const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device); - Block armed(ResidentBytes / 4, 0, true); - if (!Arm(armed)) { - std::puts("linear target import gate over armed direct memory skipped: the collects did not arm the resident run read-only"); - return false; - } + Block direct(ResidentBytes / 4, 0, true); Block flexible(2 * BlockBytes, 0); GuestArena::GuestArenaCommit_nid_postfix(flexible.data + BlockBytes, BlockBytes, PAGE_READONLY, BlockBytes); const auto decided = AgcDriver::Graphics::PrepareImportWatch(context); const auto before = ColorTargetCounts(); AgcDriver::Graphics::SetImportWatch(context, ImportWatch::Unwatch); - const auto* compared = AgcDriver::Graphics::HostImportFor(context, armed.Address(), TargetBytes, ImportRequester::ColorTarget); - const bool stillArmed = ArmedReadOnly(armed); + const auto* compared = AgcDriver::Graphics::HostImportFor(context, direct.Address(), TargetBytes, ImportRequester::ColorTarget); AgcDriver::Graphics::SetImportWatch(context, ImportWatch::Watch); - const auto* unopened = AgcDriver::Graphics::HostImportFor(context, flexible.Address(), TargetBytes, ImportRequester::ColorTarget); + const auto* readOnly = AgcDriver::Graphics::HostImportFor(context, flexible.Address(), TargetBytes, ImportRequester::ColorTarget); const auto refused = ColorTargetCounts(); - const auto* watched = AgcDriver::Graphics::HostImportFor(context, armed.Address(), TargetBytes, ImportRequester::ColorTarget); + const auto* watched = AgcDriver::Graphics::HostImportFor(context, direct.Address(), TargetBytes, ImportRequester::ColorTarget); const auto made = ColorTargetCounts(); AgcDriver::Graphics::SetImportWatch(context, decided); - Require(compared == nullptr && stillArmed, "armed gate: a color target's import of armed direct memory under the settle floor was made where it would be compared, or deciding it opened the armed run"); - Require(unopened == nullptr && refused.made == before.made && refused.refused == before.refused && refused.failed == before.failed, "armed gate: a color target's import was made of read-only private memory that its import does not open, or a refused color target import was counted"); - Require(watched != nullptr && made.made == before.made + 1 && made.failed == before.failed, "armed gate: a color target's import of armed direct memory registered writable was not made where imports stay watched"); - return true; + Require(compared == nullptr, "private gate: a color target's import of direct memory under the settle floor was made where it would be compared"); + Require(readOnly == nullptr && refused.made == before.made && refused.refused == before.refused && refused.failed == before.failed, "private gate: a color target's import was made of private memory committed read-only while registered writable, or a refused color target import was counted"); + Require(watched != nullptr && made.made == before.made + 1 && made.failed == before.failed, "private gate: a color target's import of direct memory registered writable was not made where imports stay watched"); } #endif @@ -477,8 +456,8 @@ int main(int argc, char** argv) { if (watched && (only.empty() || only == "--created")) Created(*device); if (watched && (only.empty() || only == "--reused") && Reused(*device)) std::puts("linear target import reuse passed"); #ifdef _WIN32 - if (watched && (only.empty() || only == "--armed") && Armed(*device)) std::puts("linear target import of armed direct memory passed"); - if ((only.empty() || only == "--armed-gate") && ArmedGate(*device)) std::puts("linear target import gate over armed and read-only private memory passed"); + if (watched && (only.empty() || only == "--direct")) Direct(*device); + if (only.empty() || only == "--private-gate") PrivateGate(*device); #endif if (watched && (only.empty() || only == "--outside-watch")) OutsideWatch(*device); if (watched && (only.empty() || only == "--read-only-registry")) ReadOnlyRegistry(*device);