Merge pull request #12378 from BunnyFl/perf/write-watch-page-exclusion

perf(agc): exclude only an import's own pages from the write watch and settle partial-block imports in the background

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -1803,6 +1803,23 @@ if(BUILD_TESTING)
                 ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
         endforeach()
         set_tests_properties(agc_hint_mapping_history_uncached PROPERTIES ENVIRONMENT "APS5_NO_COLLECT_MEMO=1")
+        foreach(mode IN ITEMS cached uncached)
+            add_test(NAME agc_hint_partial_block_store_${mode} COMMAND agc_driver_recorder_tests --hint-partial-block-store-only)
+            set_tests_properties(agc_hint_partial_block_store_${mode} PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30
+                ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        endforeach()
+        set_tests_properties(agc_hint_partial_block_store_uncached PROPERTIES ENVIRONMENT "APS5_NO_COLLECT_MEMO=1")
+        foreach(mode IN ITEMS cached uncached)
+            add_test(NAME agc_shared_block_rewatch_${mode} COMMAND agc_driver_recorder_tests --shared-block-rewatch-only)
+            set_tests_properties(agc_shared_block_rewatch_${mode} PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30
+                ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        endforeach()
+        set_tests_properties(agc_shared_block_rewatch_uncached PROPERTIES ENVIRONMENT "APS5_NO_COLLECT_MEMO=1")
+        foreach(kind IN ITEMS complete possible)
+            add_test(NAME agc_interior_watched_driver_write_${kind} COMMAND agc_driver_recorder_tests --interior-watched-driver-write-only ${kind})
+            set_tests_properties(agc_interior_watched_driver_write_${kind} PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30
+                ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        endforeach()
     endif()
     if(WIN32)
         add_test(NAME agc_partial_watch_restoration COMMAND agc_driver_recorder_tests --partial-watch-restoration-only)

core/libs/prx/libSceAgcDriver/Execution/include/WriteWatchCoverage.hpp

@@ -2,6 +2,7 @@
 #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_WRITEWATCHCOVERAGE_HPP
 
 #include <algorithm>
+#include <array>
 #include <cstddef>
 #include <cstdint>
 #include <vector>
@@ -42,14 +43,18 @@ public:
         std::size_t first = 0;
         std::size_t last = 0;
         if (!excludedBlocks(address, bytes, first, last)) return false;
+        const auto offset = std::max(address, base) - base;
+        const auto firstPage = offset / PageBytes;
+        const auto endPage = (std::min<std::uint64_t>(address + bytes - base, size) + PageBytes - 1) / PageBytes;
         bool changed = false;
         for (auto block = first; block <= last; ++block) {
-            const auto pages = (std::min<std::uint64_t>(BlockBytes, size - block * BlockBytes) + PageBytes - 1) / PageBytes;
-            const auto mask = pageMask(0, pages);
-            changed = changed || excluded[block] != mask;
-            auto& stamp = excludedGenerations[block];
-            stamp = std::max(stamp, std::max<std::uint64_t>(generation, 1));
-            excluded[block] = mask;
+            const auto from = block == first ? firstPage % PagesPerBlock : 0;
+            const auto to = block == last ? (endPage - 1) % PagesPerBlock + 1 : PagesPerBlock;
+            const auto mask = pageMask(from, to);
+            changed = changed || (excluded[block] & mask) != mask;
+            auto& stamps = excludedGenerations[block];
+            for (auto page = from; page < to; ++page) stamps[page] = std::max(stamps[page], std::max<std::uint64_t>(generation, 1));
+            excluded[block] |= mask;
         }
         return changed;
     }
@@ -72,12 +77,17 @@ public:
         const auto last = (endPage - 1) / PagesPerBlock;
         bool changed = false;
         for (auto block = first; block <= last; ++block) {
-            const auto mask = pageMask(block == first ? firstPage % PagesPerBlock : 0, block == last ? (endPage - 1) % PagesPerBlock + 1 : PagesPerBlock);
-            if ((excluded[block] & mask) != 0 && excludedGenerations.at(block) < generation) {
+            const auto from = block == first ? firstPage % PagesPerBlock : 0;
+            const auto to = block == last ? (endPage - 1) % PagesPerBlock + 1 : PagesPerBlock;
+            if ((excluded[block] & pageMask(from, to)) == 0) continue;
+            const auto& stamps = excludedGenerations.at(block);
+            for (auto page = from; page < to; ++page) {
+                const auto bit = static_cast<std::uint16_t>(1u << page);
+                if ((excluded[block] & bit) == 0 || stamps[page] >= generation) continue;
+                excluded[block] &= static_cast<std::uint16_t>(~bit);
                 changed = true;
-                excluded[block] &= static_cast<std::uint16_t>(~mask);
-                if (excluded[block] == 0) excludedGenerations.erase(block);
             }
+            if (excluded[block] == 0) excludedGenerations.erase(block);
         }
         return changed;
     }
@@ -106,7 +116,7 @@ private:
     std::uint64_t base = 0;
     std::size_t size = 0;
     std::vector<std::uint16_t> excluded;
-    std::unordered_map<std::uint64_t, std::uint64_t> excludedGenerations;
+    std::unordered_map<std::uint64_t, std::array<std::uint64_t, PagesPerBlock>> excludedGenerations;
 };
 
 }

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -1116,7 +1116,10 @@ struct WriteTracker {
 
     bool blockPartWatched(std::uint64_t first, std::uint64_t stop) const {
 #ifdef _WIN32
-        return covers(first, 1) || covers(stop - 1, 1);
+        for (auto page = first; page < stop; page = (page / WritePageBytes + 1) * WritePageBytes) {
+            if (covers(page, 1)) return true;
+        }
+        return false;
 #else
         return GuestWriteWatch::GuestWriteWatchOverlaps_nid_postfix(static_cast<std::uintptr_t>(first), static_cast<std::size_t>(stop - first));
 #endif
@@ -2150,12 +2153,17 @@ thread_local ThreadCollectMemo threadHintMemo;
 bool walkHints(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop) {
     auto cursor = first;
     bool marked = false;
+    StampRuns runs{tracker, StampKind::Cpu};
     while (cursor < stop) {
         std::size_t count = tracker.pages.size();
         if (!GuestArena::GuestArenaCollectWrites_nid_postfix(static_cast<std::uintptr_t>(cursor), static_cast<std::size_t>(stop - cursor), tracker.pages.data(), &count, true)) return false;
         if (count != 0 && !marked) ++tracker.generation;
         marked = marked || count != 0;
-        for (std::size_t i = 0; i < count; ++i) tracker.stampHint(tracker.blockOf(reinterpret_cast<std::uintptr_t>(tracker.pages[i])));
+        for (std::size_t i = 0; i < count; ++i) {
+            const auto page = reinterpret_cast<std::uintptr_t>(tracker.pages[i]);
+            tracker.stampHint(tracker.blockOf(page));
+            if (tracker.coverage.Covers(page, WritePageBytes)) stampWrittenRun(&runs, page, page + WritePageBytes);
+        }
         if (count < tracker.pages.size()) break;
         cursor = reinterpret_cast<std::uintptr_t>(tracker.pages[count - 1]) + WritePageBytes;
     }

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -2245,12 +2245,10 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
     state.pendingImports.insert(base);
 #ifdef _WIN32
     if (deferredSettles.load(std::memory_order_relaxed) && entry.alias == nullptr && state.watchPrivateImports && writable) {
-        if (base % settleBlockBytes != 0 || bytes % settleBlockBytes != 0) {
-            entry.settleOnDemand = entry.unwatched;
-        } else if (entry.unwatched) {
+        if (entry.unwatched) {
             state.pendingSettles.insert_or_assign(base, PendingSettle{});
             state.settleWanted.push_back(base);
-        } else if (settled && !GuestMemory::Watched(base, static_cast<std::size_t>(bytes))) {
+        } else if (base % settleBlockBytes == 0 && bytes % settleBlockBytes == 0 && settled && !GuestMemory::Watched(base, static_cast<std::size_t>(bytes))) {
             entry.unwatched = true;
             state.pendingSettles.insert_or_assign(base, PendingSettle{SettlePhase::Settled, 0});
             state.settleReady.push_back(base);

core/libs/prx/libSceAgcDriver/tests/Recorder.cpp

@@ -253,6 +253,113 @@ bool hintMappingHistoryTests() {
 #endif
 }
 
+bool hintPartialBlockStoreTests() {
+#ifdef _WIN32
+    constexpr std::size_t bytes = 131072;
+    constexpr std::size_t page = 4096;
+    auto* memory = static_cast<std::uint8_t*>(AllocateWatched(bytes, 65536));
+    if (memory == nullptr) return false;
+    GuestAllocations::Mutation().Add(memory, bytes, true, true, false);
+    struct Cleanup {
+        std::uint8_t* memory;
+        ~Cleanup() {
+            GuestAllocations::Mutation().Remove(memory);
+            ReleaseWatched(memory, bytes);
+        }
+    } cleanup{memory};
+    const auto address = reinterpret_cast<std::uint64_t>(memory);
+    std::memset(memory, 0x11, bytes);
+    AgcDriver::GuestMemory::BumpCollectEpoch();
+    const auto before = AgcDriver::GuestMemory::CollectWrites(address + 8 * page, page);
+    Require(before != 0, "the watched page has no collected write history");
+    AgcDriver::GuestMemory::Unwatch(address, 6 * page);
+    Require(!AgcDriver::GuestMemory::Watched(address, 6 * page) && AgcDriver::GuestMemory::Watched(address + 6 * page, 10 * page), "the partial-block import did not exclude only its own pages");
+    memory[8 * page] = 0x22;
+    AgcDriver::GuestMemory::BumpCollectEpoch();
+    Require(AgcDriver::GuestMemory::UntrackedHint(address + 4 * page, 6 * page) != 0, "a range across the import's end has no untracked hint");
+    Require(AgcDriver::GuestMemory::CollectWrites(address + 8 * page, page) != 0, "the watched page past the import stopped collecting CPU writes");
+    Require(!AgcDriver::GuestMemory::UnchangedSince(address + 8 * page, page, before), "a hint walk across a partial-block import consumed a CPU store to a watched page");
+    Require(memory[8 * page] == 0x22, "the hint walk changed guest bytes");
+    return true;
+#else
+    return false;
+#endif
+}
+
+bool sharedBlockRewatchTests() {
+#ifdef _WIN32
+    constexpr std::size_t bytes = 131072;
+    constexpr std::size_t page = 4096;
+    auto* memory = static_cast<std::uint8_t*>(AllocateWatched(bytes, 65536));
+    if (memory == nullptr) return false;
+    GuestAllocations::Mutation().Add(memory, bytes, true, true, false);
+    struct Cleanup {
+        std::uint8_t* memory;
+        ~Cleanup() {
+            GuestAllocations::Mutation().Remove(memory);
+            ReleaseWatched(memory, bytes);
+        }
+    } cleanup{memory};
+    const auto address = reinterpret_cast<std::uint64_t>(memory);
+    std::memset(memory, 0x11, bytes);
+    AgcDriver::GuestMemory::BumpCollectEpoch();
+    AgcDriver::GuestMemory::Unwatch(address, 6 * page);
+    AgcDriver::GuestMemory::Unwatch(address + 8 * page, 4 * page);
+    Require(!AgcDriver::GuestMemory::Watched(address, 6 * page) && !AgcDriver::GuestMemory::Watched(address + 8 * page, 4 * page), "two imports in one block were not both excluded");
+    Require(AgcDriver::GuestMemory::Watched(address + 6 * page, 2 * page) && AgcDriver::GuestMemory::Watched(address + 12 * page, 4 * page), "two imports in one block excluded pages outside both");
+    const auto before = AgcDriver::GuestMemory::CollectWrites(address + 6 * page, page);
+    Require(before != 0, "the page between the imports has no collected write history");
+    const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> first{{{address, address + 6 * page}}};
+    AgcDriver::GuestMemory::Rewatch(first);
+    Require(AgcDriver::GuestMemory::Watched(address, 8 * page), "settling the first import did not return its pages to the write watch");
+    for (std::size_t index = 8; index < 12; ++index) Require(!AgcDriver::GuestMemory::Watched(address + index * page, page), "settling the first import returned a page of the second import in its block to the write watch");
+    Require(!AgcDriver::GuestMemory::Watched(address, 16 * page), "settling one import proved its whole shared block watched");
+    memory[6 * page] = 0x22;
+    AgcDriver::GuestMemory::BumpCollectEpoch();
+    Require(AgcDriver::GuestMemory::CollectWrites(address + 6 * page, page) != 0 && !AgcDriver::GuestMemory::UnchangedSince(address + 6 * page, page, before), "a CPU store between the imports was lost after the first import settled");
+    const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> second{{{address + 8 * page, address + 12 * page}}};
+    AgcDriver::GuestMemory::Rewatch(second);
+    Require(AgcDriver::GuestMemory::Watched(address, 16 * page), "settling both imports did not return their shared block to the write watch");
+    Require(memory[6 * page] == 0x22, "rewatching changed guest bytes");
+    return true;
+#else
+    return false;
+#endif
+}
+
+bool interiorWatchedDriverWriteTests(bool complete) {
+#ifdef _WIN32
+    constexpr std::size_t bytes = 131072;
+    constexpr std::size_t page = 4096;
+    auto* memory = static_cast<std::uint8_t*>(AllocateWatched(bytes, 65536));
+    if (memory == nullptr) return false;
+    GuestAllocations::Mutation().Add(memory, bytes, true, true, false);
+    struct Cleanup {
+        std::uint8_t* memory;
+        ~Cleanup() {
+            GuestAllocations::Mutation().Remove(memory);
+            ReleaseWatched(memory, bytes);
+        }
+    } cleanup{memory};
+    const auto address = reinterpret_cast<std::uint64_t>(memory);
+    std::memset(memory, 0x11, bytes);
+    AgcDriver::GuestMemory::BumpCollectEpoch();
+    AgcDriver::GuestMemory::Unwatch(address, 6 * page);
+    AgcDriver::GuestMemory::Unwatch(address + 12 * page, 4 * page);
+    Require(!AgcDriver::GuestMemory::Watched(address, 6 * page) && !AgcDriver::GuestMemory::Watched(address + 12 * page, 4 * page), "both ends of the block were not excluded");
+    Require(AgcDriver::GuestMemory::Watched(address + 6 * page, 6 * page), "the pages between the excluded ends are not watched");
+    const auto before = AgcDriver::GuestMemory::CollectWrites(address + 8 * page, page);
+    Require(before != 0, "the watched interior page has no collected write history");
+    if (complete) AgcDriver::GuestMemory::MarkWritten(address, 16 * page);
+    else AgcDriver::GuestMemory::MarkPossiblyWritten(address, 16 * page);
+    Require(!AgcDriver::GuestMemory::UnchangedSince(address + 8 * page, page, before), complete ? "a complete driver write across a block with both ends excluded was not recorded on its watched interior page" : "a possible driver write across a block with both ends excluded was not recorded on its watched interior page");
+    return true;
+#else
+    (void)complete;
+    return false;
+#endif
+}
+
 void gpuTimingKeyTests() {
     constexpr std::uint64_t variant = 0x2;
     const auto key = Recorder::DrawTimingKey(variant);
@@ -2772,7 +2879,8 @@ bool settleInitialDccTests(const Device& device) {
                 gpu.lock();
                 HostImportFor(context, address, importBytes);
             }
-            Require(!Watched(address, importBytes), "(m) a deferred settle returned an import that is not a whole number of 64 KiB blocks to the write watch");
+            if (kind == Case::PartialBlock) Require(Watched(address, importBytes), "(m) a deferred settle did not return an import of whole and partial 64 KiB blocks to the write watch" + describe(ExistingHostImportFor(context, address, importBytes), address, importBytes));
+            else Require(!Watched(address, importBytes), "(m) a deferred settle returned an import that shares its last 64 KiB block with a compared import to the write watch");
         }
         const auto before = HostImportSettles();
         Require(!fmaskAlone || before.submitted == quiet.submitted, "(m) a settle read was submitted between the FMASK's import and the target's first use");
@@ -2794,7 +2902,8 @@ bool settleInitialDccTests(const Device& device) {
         }
         Require(refusal.empty(), std::string("(m) an initial multisampled DCC import over ") + name + " was refused: " + refusal);
         Require(!fmaskAlone || (after.submitted - before.submitted == 1 && after.rewatched - before.rewatched == 1), "(m) the first use over a pending FMASK settle alone submitted " + std::to_string(after.submitted - before.submitted) + " settle reads, not the FMASK's alone");
-        Require(after.submitted > before.submitted && after.rewatched > before.rewatched && Watched(address, importBytes) && settled != nullptr && !settled->unwatched, std::string("(m) the initial DCC import over ") + name + " did not settle the import and return it to the write watch");
+        const bool settledBefore = kind == Case::PartialBlock;
+        Require((settledBefore ? after.submitted == before.submitted : after.submitted > before.submitted && after.rewatched > before.rewatched) && Watched(address, importBytes) && settled != nullptr && !settled->unwatched, std::string("(m) the initial DCC import over ") + name + " did not settle the import and return it to the write watch");
         if (fmask != nullptr) {
             const auto* settledFmask = ExistingHostImportFor(context, target.fmaskAddress, fmaskBytes);
             Require(Watched(target.fmaskAddress, fmaskBytes) && settledFmask != nullptr && !settledFmask->unwatched, "(m) the FMASK's import did not settle back into the write watch" + describe(settledFmask, target.fmaskAddress, fmaskBytes));
@@ -12045,6 +12154,23 @@ int main(int argc, char** argv) {
             std::cout << "hint mapping history tests passed\n";
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--hint-partial-block-store-only") {
+            if (!hintPartialBlockStoreTests()) return 77;
+            std::cout << "hint partial-block store tests passed\n";
+            return 0;
+        }
+        if (argc == 2 && std::string_view(argv[1]) == "--shared-block-rewatch-only") {
+            if (!sharedBlockRewatchTests()) return 77;
+            std::cout << "shared-block rewatch tests passed\n";
+            return 0;
+        }
+        if (argc == 3 && std::string_view(argv[1]) == "--interior-watched-driver-write-only") {
+            const std::string_view kind = argv[2];
+            if (kind != "complete" && kind != "possible") return 2;
+            if (!interiorWatchedDriverWriteTests(kind == "complete")) return 77;
+            std::cout << "interior watched driver write tests passed\n";
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--partial-watch-restoration-only") {
             if (!partialWatchRestorationTests()) return 77;
             std::cout << "partial private write-watch restoration tests passed\n";

core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp

@@ -820,14 +820,16 @@ void CheckRewatchImportTail() {
     constexpr std::size_t bytes = 0xc4000;
     ImportFixture fixture(13 * Block, bytes);
     const auto import = fixture.Base();
+    const bool pastWatched = Watched(import + bytes, 4096);
     UnwatchImport(import, bytes);
+    Require(Watched(import + bytes, 4096) == pastWatched, "unwatching an import ending inside a block changed the page past it");
     PrintPageState("page past a 0xc4000-byte import", fixture.memory + bytes);
     const auto unwatched = UnwatchSerial();
     Rewatch(BlockSpan(import, bytes));
     Require(!Watched(import, bytes) && UnwatchSerial() == unwatched, "the 64 KiB span past the import's committed pages returned to the write watch");
     Rewatch(PageSpan(import, bytes));
     Require(Watched(import, bytes), "a settled import whose size is not a whole block did not return to the write watch");
-    Require(Watched(import + 12 * Block, bytes - 12 * Block) && !Watched(import, bytes + 1) && !Watched(import + bytes - 1, 2) && !Watched(import + bytes, 4096), "rewatching an import ending inside a block did not return exactly its pages");
+    Require(Watched(import + 12 * Block, bytes - 12 * Block) && Watched(import + bytes, 4096) == pastWatched && Watched(import, bytes + 1) == pastWatched, "rewatching an import ending inside a block changed the page past it");
     Require(UnwatchSerial() != unwatched && UntrackedHint(import, bytes) == 0, "a rewatched import still reads as unwatched");
     RequireRewatchedStores(import, bytes);
 }
@@ -838,13 +840,16 @@ void CheckRewatchUnalignedImport() {
     constexpr std::size_t committed = 0xc8000;
     ImportFixture fixture(13 * Block, committed);
     const auto import = fixture.Base() + offset;
+    const bool beforeWatched = Watched(fixture.Base(), offset);
+    const bool pastWatched = Watched(import + bytes, 4096);
     UnwatchImport(import, bytes);
+    Require(Watched(fixture.Base(), offset) == beforeWatched && Watched(import + bytes, 4096) == pastWatched, "unwatching an import inside its blocks changed the pages around it");
     PrintPageState("page past an import at block offset 0x1000", fixture.memory + committed);
     Rewatch(BlockSpan(import, bytes));
     Require(!Watched(import, bytes), "the 64 KiB span past the committed pages around an import returned to the write watch");
     Rewatch(PageSpan(import, bytes));
     Require(Watched(import, bytes), "a settled import that does not start on a block did not return to the write watch");
-    Require(!Watched(fixture.Base(), offset) && !Watched(import + bytes, 4096), "rewatching an import returned pages outside it");
+    Require(Watched(fixture.Base(), offset) == beforeWatched && Watched(import + bytes, 4096) == pastWatched, "rewatching an import changed the pages around it");
     RequireRewatchedStores(import, bytes);
 }
 

core/libs/prx/libSceAgcDriver/tests/WriteWatchCoverage.cpp

@@ -27,8 +27,11 @@ int main() {
         require(!coverage.ExcludedSpan(base - block, block, &spanBegin, &spanEnd) && !coverage.ExcludedSpan(base + 5 * block, block, &spanBegin, &spanEnd) && !coverage.ExcludedSpan(base, 0, &spanBegin, &spanEnd), "an outside or empty import has a span");
         require(coverage.Exclude(base + block + 4096, 1), "an imported page was not excluded");
         require(!coverage.Covers(base + block, block), "the imported block is still watched");
-        require(!coverage.Covers(base + block - 1, 2), "a range crossing into the import is watched");
+        require(coverage.Covers(base + block, 4096) && coverage.Covers(base + block + 2 * 4096, block - 2 * 4096), "an import unwatched pages of its block outside it");
+        require(!coverage.Covers(base + block + 4095, 2) && !coverage.Covers(base + block + 2 * 4096 - 1, 2), "a range crossing into the import is watched");
         require(coverage.Covers(base, block) && coverage.Covers(base + 2 * block, 2 * block + 1), "unrelated blocks lost write watching");
+        require(!coverage.Exclude(base + block + 4096, 1), "excluding an imported page again reports a change");
+        require(coverage.Exclude(base + block, block) && !coverage.Covers(base + block, 4096), "excluding the rest of an imported block reports no change");
         require(!coverage.Exclude(base + block, block), "excluding an imported block again reports a change");
         require(coverage.Exclude(base + 3 * block - 1, 2), "an import across a block boundary was not excluded");
         require(!coverage.Covers(base + 2 * block, block) && !coverage.Covers(base + 3 * block, block), "a boundary import left one block watched");
@@ -69,9 +72,9 @@ int main() {
         require(!coverage.Covers(base + 16383, 2) && !coverage.Covers(base + 32767, 2), "a boundary query included excluded pages");
         require(!coverage.Restore(base + 32768, 4096, 10), "a stale partial mapping restored an exclusion");
         coverage.Exclude(base + 4096, 1, 12);
-        require(!coverage.Covers(base + 16384, 16384), "a repeated import kept partial coverage from an older mapping");
-        require(!coverage.Restore(base + 16384, 16384, 11), "a delayed partial mapping restored a newer import");
-        require(coverage.Restore(base + 16384, 16384, 13), "fresh partial coverage was not restored after re-import");
+        require(!coverage.Covers(base + 4096, 4096) && coverage.Covers(base + 16384, 16384), "a re-import of one page changed the coverage of other pages");
+        require(!coverage.Restore(base + 4096, 4096, 11), "a delayed partial mapping restored a newer import");
+        require(coverage.Restore(base + 4096, 4096, 13) && coverage.Covers(base + 4096, 4096), "a fresh mapping did not restore a re-imported page");
         constexpr std::size_t page = 4096;
         for (std::size_t first = 0; first < block / page; ++first) {
             for (std::size_t count = 1; count <= block / page - first; ++count) {
@@ -104,12 +107,28 @@ int main() {
         coverage.Exclude(base, tail, 50);
         require(coverage.Restore(base, tail, 51), "an import ending inside a block did not restore its pages");
         require(coverage.Covers(base, tail) && coverage.Covers(base + 12 * block, tail - 12 * block), "an import ending inside a block is not watched");
-        require(!coverage.Covers(base, tail + 1) && !coverage.Covers(base + tail - 1, 2) && !coverage.Covers(base + tail, page) && !coverage.Covers(base, 13 * block), "a range past an import ending inside a block is watched");
-        require(coverage.Covers(base + tail - page, page) && !coverage.Covers(base + 13 * block - page, page) && coverage.Covers(base + 13 * block, block), "an import ending inside a block changed the pages after it");
+        require(coverage.Covers(base, 13 * block) && coverage.Covers(base + tail, page), "a restored import ending inside a block left pages excluded");
+        coverage.Exclude(base, tail, 52);
+        require(!coverage.Covers(base + tail - page, page) && !coverage.Covers(base + tail - 1, 2), "a range ending inside an import is watched");
+        require(coverage.Covers(base + tail, 13 * block - tail) && coverage.Covers(base + 13 * block, block), "an import ending inside a block unwatched the pages after it");
         coverage.Initialize(base, 14 * block);
         coverage.Exclude(base + page, tail, 60);
         require(coverage.Restore(base + page, tail, 61) && coverage.Covers(base + page, tail), "an import starting and ending inside blocks is not watched");
-        require(!coverage.Covers(base, page) && !coverage.Covers(base, tail) && !coverage.Covers(base + page + tail, page) && !coverage.Covers(base + page, tail + 1), "an import starting and ending inside blocks watched pages outside it");
+        coverage.Exclude(base + page, tail, 62);
+        require(coverage.Covers(base, page) && coverage.Covers(base + page + tail, page), "an import starting and ending inside blocks unwatched pages outside it");
+        require(!coverage.Covers(base, 2 * page) && !coverage.Covers(base + page + tail - 1, 2), "a range reaching into an import starting and ending inside blocks is watched");
+        coverage.Initialize(base, 2 * block);
+        coverage.Exclude(base + page, page, 70);
+        coverage.Exclude(base + 5 * page, page, 75);
+        require(coverage.Restore(base + page, page, 72) && coverage.Covers(base + page, page), "a newer import of another page in the block kept an older page excluded");
+        require(!coverage.Covers(base + 5 * page, page), "restoring an older page restored a newer import in the same block");
+        require(!coverage.Restore(base + 5 * page, page, 74) && !coverage.Covers(base + 5 * page, page), "a stale mapping restored a newer page in the same block");
+        require(coverage.Restore(base + 5 * page, page, 76) && coverage.Covers(base, 2 * block), "a fresh mapping did not restore the newer page");
+        coverage.Exclude(base + 3 * page, page, 80);
+        for (std::size_t index = 0; index < 2 * block / page; ++index) {
+            require(coverage.Covers(base + index * page, page) == (index != 3), "excluding one page changed another page of the arena");
+        }
+        require(coverage.ExcludedSpan(base + 3 * page, page, &spanBegin, &spanEnd) && spanBegin == base && spanEnd == base + block, "the span of an excluded page is not its whole block");
         std::cout << "write watch coverage passed\n";
         return 0;
     } catch (const std::exception& error) {

docs/dev/ARCHITECTURE.md

@@ -157,7 +157,7 @@ flowchart LR
 - Guest memory reaches the GPU through host imports (`VK_EXT_external_memory_host`) where the driver accepts them, and through copies otherwise ([`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp)). A registered range larger than the import window (64 MiB) is imported per window for buffer and fill accesses and whole for an address-based build. Live imports stay under a budget, with the least recently used import retired first (see TechnicalDebt, Host import windows and budget).
 - A guest protect or unmap that overlaps a plain host-pointer import retires the whole import before changing the guest range (`WaitForLeasesAndImports` in [`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp)). Later accesses import surviving registered ranges again; imports never keep the pages across a cut pinned in a trimmed state. `GuestAllocations::Range::backingSerial` distinguishes the original pages from replacements: untouched pieces of a split keep their serial, newly mapped or protected pieces get a fresh one, and every span passed to `RequireUnpinned` (including the other mappings of remapped direct memory and the resident runs touched by a 2 MiB unmap or remap) gets a fresh serial when the mutation ends.
 - A draw binds its vertex and index ranges in place from a host import, except with NVIDIA's Windows driver, which reads a host import of guest memory far slower than a buffer it allocated (Astro Bot at Crash Site on an RTX 5070 Ti: 164 against 45 ms of draws per frame), so they are copied there (`DrawInputsInPlaceUsable`, `Context::drawInputsInPlace`). `APS5_DRAW_INPUTS_IN_PLACE=1` binds them in place and `APS5_NO_DRAW_INPUTS_IN_PLACE=1` copies them on any device; the second wins.
-- On Windows, a host import of private `MEM_WRITE_WATCH` memory (singly mapped direct memory) stays on `GetWriteWatch` when `ProbeImportWriteProtection` shows the driver reports nothing written after the second GPU read and a GPU write, and a CPU store is collected (`decideImportWatch`). The driver marks every page of such an import written, asynchronously, until a GPU use of it completes (a read on the settle queue ends it even while main-queue work that used the import first is still running, and the first use of an earlier import of the same pages after that read marks nothing), so `settleImport` in [`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) copies 4 bytes out of the new import on a second queue of the graphics family, or on a queue of another copy-capable family when the graphics family has only one (`Context::settleQueue`, own pool and mutex) inside `ImportWatched`'s window and waits for it, for imports of 1 MiB or more; the main queue can't be used there because `importAllocation` holds the imports lock and `GpuMutex` comes before it in the lock order. A smaller import is compared when it is made, so a title that maps, imports and unmaps a small buffer around every submission does not block its mapping thread. Its 4-byte read is queued instead: the next import lookup that holds `GpuMutex` while the imports match the registry (`settleUnwatchedImports`) records the queued reads into one batch on the main queue, with barriers ordering earlier and later writes around the reads. Submission does not wait on the CPU, and once the batch's fence has signalled a later lookup returns the import's pages to the write watch (`GuestMemory::Rewatch` drops the marks of every 64 KiB block they touch and stamps those blocks as a CPU store; the rest of a block the import only partly covers stays unwatched), unless a live storage image overlaps those blocks. Such an import stays compared: an image that took its generation before the stamp would take the stamp for a CPU store over the results it produces after it, and drop them. A settled import whose range had already left the watch returns the same way, and an import retired while its read is in flight is kept with that batch and freed once the batch has completed, without waiting for it; a guest unmap of its memory waits for the batch instead. `APS5_NO_DEFERRED_SETTLES` (any value) keeps every import under 1 MiB compared. Imports through a section alias keep tracking on the guest views, and every other Windows host import is compared. A device without such a queue keeps the comparisons. On Linux the settle queue is made when the device imports dma-buf memory, and every dma-buf import of 1 MiB or more is settled right after it is made, for another reason: the kernel maps an import's pages for the GPU at the first submission that uses it, which took about 2 s for a 3 GiB import on RADV and held up the title's frame. The import's buffer is shared with the settle queue's family instead of aliased, since a sparse import is bound to many memory objects.
+- On Windows, a host import of private `MEM_WRITE_WATCH` memory (singly mapped direct memory) stays on `GetWriteWatch` when `ProbeImportWriteProtection` shows the driver reports nothing written after the second GPU read and a GPU write, and a CPU store is collected (`decideImportWatch`). The driver marks every page of such an import written, asynchronously, until a GPU use of it completes (a read on the settle queue ends it even while main-queue work that used the import first is still running, and the first use of an earlier import of the same pages after that read marks nothing), so `settleImport` in [`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) copies 4 bytes out of the new import on a second queue of the graphics family, or on a queue of another copy-capable family when the graphics family has only one (`Context::settleQueue`, own pool and mutex) inside `ImportWatched`'s window and waits for it, for imports of 1 MiB or more; the main queue can't be used there because `importAllocation` holds the imports lock and `GpuMutex` comes before it in the lock order. A smaller import is compared when it is made, so a title that maps, imports and unmaps a small buffer around every submission does not block its mapping thread. Its 4-byte read is queued instead: the next import lookup that holds `GpuMutex` while the imports match the registry (`settleUnwatchedImports`) records the queued reads into one batch on the main queue, with barriers ordering earlier and later writes around the reads. Submission does not wait on the CPU, and once the batch's fence has signalled a later lookup returns the import's pages to the write watch (`GuestMemory::Rewatch` drops the marks of every 64 KiB block they touch and stamps those blocks as a CPU store; an import only excludes its own pages, so the rest of a block it partly covers keeps its own state), unless a live storage image overlaps those blocks. Such an import stays compared: an image that took its generation before the stamp would take the stamp for a CPU store over the results it produces after it, and drop them. A settled import whose range had already left the watch returns the same way, and an import retired while its read is in flight is kept with that batch and freed once the batch has completed, without waiting for it; a guest unmap of its memory waits for the batch instead. `APS5_NO_DEFERRED_SETTLES` (any value) keeps every import under 1 MiB compared. Imports through a section alias keep tracking on the guest views, and every other Windows host import is compared. A device without such a queue keeps the comparisons. On Linux the settle queue is made when the device imports dma-buf memory, and every dma-buf import of 1 MiB or more is settled right after it is made, for another reason: the kernel maps an import's pages for the GPU at the first submission that uses it, which took about 2 s for a 3 GiB import on RADV and held up the title's frame. The import's buffer is shared with the settle queue's family instead of aliased, since a sparse import is bound to many memory objects.
 
 ### Log output
 

docs/dev/debt/libSceAgcDriver.md

@@ -153,9 +153,9 @@
 - [`APS5_SKIP_REFUSED_DRAWS`](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp) (libSceAgcDriver) drops a draw whose graphics stage the recompiler refuses instead of failing the queue, so the frame lacks that draw, and the stage is compiled and refused again at each such draw. Compute refusals still fail.
 - [Pending storage-image results over memory outside the write watch](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) leave to the CPU the 64 KiB blocks whose bytes differ from the ones saved at the upload, the fill, the image copy, the previous store or the moment their range left the watch (`Unwatch`, or a host import that does not stay watched or whose page walk fails), and the blocks the watch saw the CPU write before that moment. A store made on the GPU saves its bytes through a host-visible copy that becomes the saved bytes when its batch completes; a store or a refresh that needs them earlier waits for that batch. A unit without a write generation saves the guest bytes when results land in it outside the watch. One whose range leaves the watch while its results are pending saves its bytes through the unwatch hook, which also runs when a failed collect takes a range out of the watch. A range that leaves the watch without the hook running over it leaves its pending results without saved bytes; storing such results throws. A unit whose results were stored while its blocks were watched keeps only a generation; when its blocks leave the watch while nothing is pending, the hook saves nothing, and results that land in it afterwards have neither saved bytes nor a watch, so storing them throws too.
 - [Storage images over uncommitted guest memory](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) - a storage image whose guest range has no committed page when it is created throws in the `StorageTexture` constructor, which aborts the guest. Several titles reach this intermittently in runs of a few minutes, on AMD and NVIDIA alike; in two of the three logs, lines about storage images over released or rewritten memory (`memory no longer registered`, `guest memory changed while GPU results were pending`) come shortly before it. The log does not name the image's address. What the console does with a storage descriptor over memory that is not committed is unknown, and which allocation the guest recreates there was not found.
-- Windows host-pointer imports of the guest mapping itself are compared instead of write-watched, except writable imports of private `MEM_WRITE_WATCH` memory when the import probe keeps the write watch and a settle queue exists: those of 1 MiB or more stay watched, and smaller ones return to the watch once their settle has completed, unless a storage image overlaps them (see Pages under an import window): a driver can mark imported pages dirty after the synchronous import window. Shared memory imported through a separate writable alias retains protection-based tracking on the guest views; driver access to the import alias does not disarm those views. [GuestMemory](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) excludes every overlapping 64 KiB tracker block while its backing is retained. A full block replaced by fresh private backing regains tracking and gets a new generation; protection changes, partial replacements and shared remappings do not restore it. Excluded blocks keep their write-watch marks as a hint only (`UntrackedHint`): a storage image or a sampled texture snapshot over them compares its bytes again after a mark in its range, a driver store, a mapping change or a recorded GPU write instead of at every collect epoch or lookup, and a mark the driver makes without a CPU store costs one comparison. This preserves CPU edits through byte comparisons but limits snapshot reuse on excluded blocks to draws of one collect epoch and disables unit shadows for them; restoring tracking on import retirement without replacing the backing remains unimplemented.
-- [Pages under an import window](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) whose protection an import resolves count as stored only when their bytes change. A same-value CPU store during that window cannot be distinguished from driver activity and does not win over pending results. On Windows, NVIDIA 617.42 marks every private MEM_WRITE_WATCH page after import until a GPU use completes. The driver behavior was measured on this driver; other drivers are unverified. Imports of at least 1 MiB wait for a four-byte read on the separate settle queue inside the window. Smaller imports stay compared until a deferred read completes; a later lookup returns their pages to write watching, stamping their blocks as CPU stores; the rest of a partly covered block stays compared. Blocks overlapping a live storage image, blocks of an import retired before rewatching, and imports without a settle queue stay compared. Shared-memory imports retain their separate-alias tracking.
-- [Deferred host-import reads](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) use the main queue with barriers ordering earlier and later writes around the reads. Submission does not wait on the CPU. An initial multisampled DCC import needing a range whose import still awaits its read submits it if needed and waits under the GPU mutex (`WatchedAfterSettle`, #9724). Imports of partial 64 KiB blocks get no deferred read; only that path reads them, returning their pages as CPU stores unless a live, trimmed or parked compared import or storage image shares their blocks; one a recorder holds is unseen. The latency cost of placing batches behind main-queue work is unmeasured. The first-use test now checks that earlier main-queue reads finish before the import returns to write watching; the original two-queue canary measured driver marking while those reads were still in flight. The earlier-import canary still checks later use of another import of the same pages.
+- Windows host-pointer imports of the guest mapping itself are compared instead of write-watched, except writable imports of private `MEM_WRITE_WATCH` memory when the import probe keeps the watch and a settle queue exists: those of 1 MiB or more stay watched, smaller ones return to the watch after their settle unless a storage image overlaps them. Shared memory imported through a separate writable alias keeps protection-based tracking on its guest views. [GuestMemory](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) excludes the 4 KiB pages an import overlaps, each with the generation of its latest exclusion; a fresh private mapping restores the complete pages it covers, protection changes and shared remappings do not, nor does retiring an import (unimplemented). Blocks with excluded pages keep their write-watch marks as a hint only (`UntrackedHint`): byte comparisons keep CPU edits, but snapshot reuse lasts one collect epoch and unit shadows are off.
+- [Pages under an import window](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) whose protection an import resolves count as stored only when their bytes change. A same-value CPU store during that window cannot be distinguished from driver activity and does not win over pending results. On Windows, NVIDIA 617.42 marks every private MEM_WRITE_WATCH page after import until a GPU use completes. The driver behavior was measured on this driver; other drivers are unverified. Imports of at least 1 MiB wait for a four-byte read on the separate settle queue inside the window. Smaller imports stay compared until a deferred read completes; a later lookup returns their pages to write watching, stamping their blocks as CPU stores; the block's other pages keep their state. Blocks overlapping a live storage image, blocks of an import retired before rewatching, and imports without a settle queue stay compared. Shared-memory imports retain their separate-alias tracking.
+- [Deferred host-import reads](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) use the main queue with barriers ordering earlier and later writes around the reads. Submission does not wait on the CPU. An initial multisampled DCC import needing a range whose import still awaits its read submits it if needed and waits under the GPU mutex (`WatchedAfterSettle`, #9724). Imports of partial 64 KiB blocks are read the same way; that read or one on demand returns their pages as CPU stores unless a live, trimmed or parked compared import or storage image shares their blocks; one a recorder holds is unseen. The latency cost of placing batches behind main-queue work is unmeasured. The first-use test now checks that earlier main-queue reads finish before the import returns to write watching; the original two-queue canary measured driver marking while those reads were still in flight. The earlier-import canary still checks later use of another import of the same pages.
 - [Settled dma-buf imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): on Linux a dma-buf import of 1 MiB or more is read by a 4-byte copy on the settle queue right after it is made (`settleImport`), so the kernel maps its pages for the GPU there and not in the first submission that uses it (about 2 s for a 3 GiB import on RADV). The import's buffer is shared with the settle queue's family. An import made at map time settles on the guest thread that maps the range, inside its map call, so that thread waits while the kernel maps the pages: 0.47 to 2.2 s for the 3 GiB range of Hades (PPSA03355) and 0.49 to 4.3 s for the 4.5 GiB range of ASTRO BOT (PPSA21567) on RADV. Both titles map these ranges at start-up; a title that maps large ranges during gameplay stalls its mapping thread for as long. An import made from a submission settles under the GPU mutex, so its frame still pays for it; a smaller import, and every import on a device with no settle queue (one without a copy-capable queue besides the main one), pays in its first submission. A range imported through a host pointer, as lavapipe imports shared memory, is not settled
 - [Sparse dma-buf imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): on Linux a shared-backed range larger than one 64 MiB udmabuf buffer is imported as one sparse buffer bound to the 64 MiB buffers, on a queue of its own when the device has a second family that binds sparse memory. The path is decided before anything is imported, and a failure after that throws. A range whose offset in the backing file is not a multiple of 64 KiB starts its sparse buffer at the 64 KiB block below it (`HostImport::lead`). A range takes the single udmabuf import instead, which udmabuf's `size_limit_mb` refuses above 64 MiB by default so the range is copied, when the buffer's sparse block size is larger than 64 KiB (it is 64 KiB on RADV), when the sparse buffer does not accept every memory type a chunk import can have, when the backing file ends inside its last chunk, or when it is imported at map time on a device with no second family (the bind then needs the GPU mutex); the NVIDIA proprietary driver imports no dma-buf memory
 - [Presentation under the GPU mutex](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver) - on a device whose presentation family has no queue left after the main and settle queues, such as RADV, lavapipe and Intel's Windows driver (Arc Graphics on Meteor Lake, 101.9150), which expose a single graphics queue, `vkQueuePresentKHR` still runs on the main queue under `GuestMemory::GpuMutex`, so the draw committer and the queue workers wait for as long as the present blocks. How long that is on those drivers was not measured; on NVIDIA in FIFO mode it is about 6 ms per present. AMD's Windows driver (Radeon 860M) has one too: a Quake II port's trace had 0.3 ms presents (max 3.5), no `GpuMutex.wait.*` behind them. A device with a free queue in that family presents on it under its own mutex, after the GPU mutex is released. Closes when: traces on RADV, lavapipe and Intel's Windows driver also show no `GpuMutex.wait.*` behind `Vulkan.QueuePresent`.