Merge pull request #3874 from oneandonlydean/up4/import-window-compare

perf(agc): compare an import window's pages when the window closes

core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp

@@ -60,7 +60,7 @@ void WriteChanged(std::uint64_t address, std::span<const std::byte> current, std
 // UnchangedSince is always false, so callers fall back to comparing bytes.
 bool WriteWatched();
 void Unwatch(std::uint64_t address, std::size_t bytes);
-bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import);
+bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched = true);
 bool Watched(std::uint64_t address, std::size_t bytes);
 std::uint64_t CollectWrites(std::uint64_t address, std::size_t bytes);
 bool UnchangedSince(std::uint64_t address, std::size_t bytes, std::uint64_t generation);
@@ -116,7 +116,6 @@ bool UnchangedSinceCollected(std::uint64_t address, std::size_t bytes, std::uint
 // watched arena, or for a generation of 0, a block reads as changed.
 constexpr std::uint8_t BlockUnchanged = 0;
 constexpr std::uint8_t BlockWritten = 1;
-constexpr std::uint8_t BlockMaybeWritten = 2;
 bool ChangedBlocks(std::uint64_t address, std::size_t bytes, std::span<const std::uint64_t> generations, std::span<std::uint8_t> changed, std::span<std::uint8_t> cpu = {});
 
 // Serializes device work: draws, dispatches, presentation and the deferred write-backs below. The

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -722,6 +722,15 @@ constexpr std::size_t WritePagesPerBlock = WriteBlockBytes / WritePageBytes;
 
 enum class StampKind : std::uint8_t { Cpu, Driver, ImportWindow, Fresh };
 
+struct WindowCopy {
+    std::uint64_t first;
+    std::span<const std::byte> bytes;
+
+    bool Holds(std::uintptr_t page) const {
+        return std::memcmp(reinterpret_cast<const void*>(page), bytes.data() + (page - first), WritePageBytes) == 0;
+    }
+};
+
 #ifdef _WIN32
 void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_t generation);
 #endif
@@ -739,7 +748,6 @@ struct WriteTracker {
     // late label rule asks whether the CPU touched a label's block, and the driver's own GPU label
     // records in the same block (the title's per-job slots are 0x20 apart) must not count.
     std::vector<std::uint32_t> cpuBlocks;
-    std::vector<std::uint32_t> writtenBlocks;
     WriteWatchCoverage coverage;
 #else
     static constexpr std::size_t LeafBlocks = std::size_t{1} << 16;
@@ -747,7 +755,6 @@ struct WriteTracker {
     struct Leaf {
         std::array<std::uint32_t, LeafBlocks> blocks{};
         std::array<std::uint32_t, LeafBlocks> cpuBlocks{};
-        std::array<std::uint32_t, LeafBlocks> writtenBlocks{};
     };
     std::vector<std::unique_ptr<Leaf>> leaves;
 #endif
@@ -783,7 +790,6 @@ struct WriteTracker {
         if (!watched) return;
         blocks.assign(size / WriteBlockBytes + 1, 0);
         cpuBlocks.assign(size / WriteBlockBytes + 1, 0);
-        writtenBlocks.assign(size / WriteBlockBytes + 1, 0);
         coverage.Initialize(base, size);
         pages.resize(1u << 16);
         GuestArena::GuestArenaSetPrivateMappingObserver_nid_postfix(&watchPrivateMapping);
@@ -936,28 +942,17 @@ struct WriteTracker {
 #endif
     }
 
-    std::uint32_t writtenStampOf(std::uint64_t block) const {
-#ifdef _WIN32
-        return writtenBlocks[block];
-#else
-        const auto& leaf = leaves[block / LeafBlocks];
-        return leaf != nullptr ? leaf->writtenBlocks[block % LeafBlocks] : 0;
-#endif
-    }
-
     void stamp(std::uint64_t block, std::uint32_t stampGeneration, StampKind kind) {
         const bool cpu = kind == StampKind::Cpu || kind == StampKind::ImportWindow;
-        const bool written = kind != StampKind::ImportWindow;
+        const bool stored = kind != StampKind::ImportWindow;
 #ifdef _WIN32
-        blocks[block] = stampGeneration;
+        if (stored) blocks[block] = stampGeneration;
         if (cpu) cpuBlocks[block] = stampGeneration;
-        if (written) writtenBlocks[block] = stampGeneration;
 #else
         auto& leaf = leaves[block / LeafBlocks];
         if (leaf == nullptr) leaf = std::make_unique<Leaf>();
-        leaf->blocks[block % LeafBlocks] = stampGeneration;
+        if (stored) leaf->blocks[block % LeafBlocks] = stampGeneration;
         if (cpu) leaf->cpuBlocks[block % LeafBlocks] = stampGeneration;
-        if (written) leaf->writtenBlocks[block % LeafBlocks] = stampGeneration;
 #endif
     }
 };
@@ -1035,6 +1030,7 @@ struct StampRuns {
     WriteTracker& tracker;
     StampKind kind;
     std::size_t stamped = 0;
+    const WindowCopy* window = nullptr;
 };
 
 void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) {
@@ -1045,12 +1041,16 @@ void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) {
     ++runs.stamped;
     const auto generation = tracker.generation.load(std::memory_order_relaxed);
     if (kind != StampKind::Driver) tracker.noteCpuStore(begin, end, generation);
-    for (auto block = tracker.blockOf(begin); block <= tracker.blockOf(end - 1); ++block) tracker.stamp(block, generation, kind);
+    if (runs.window != nullptr) {
+        for (auto page = begin; page < end; page += WritePageBytes) tracker.stamp(tracker.blockOf(page), generation, runs.window->Holds(page) ? StampKind::ImportWindow : kind);
+    } else {
+        for (auto block = tracker.blockOf(begin); block <= tracker.blockOf(end - 1); ++block) tracker.stamp(block, generation, kind);
+    }
     collectDirtyRuns.fetch_add(1, std::memory_order_relaxed);
 }
 #endif
 
-bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, StampKind kind, bool armedOnly = false) {
+bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, StampKind kind, bool armedOnly = false, const WindowCopy* window = nullptr) {
     ++tracker.generation;
 #ifdef _WIN32
     static_cast<void>(armedOnly);
@@ -1072,7 +1072,7 @@ bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop,
         if (count != 0) dirty = true;
         for (ULONG_PTR i = 0; i < count; ++i) {
             const auto page = reinterpret_cast<std::uintptr_t>(tracker.pages[i]);
-            tracker.stamp(tracker.blockOf(page), tracker.generation, kind);
+            tracker.stamp(tracker.blockOf(page), tracker.generation, window != nullptr && window->Holds(page) ? StampKind::ImportWindow : kind);
             if (kind != StampKind::Driver) tracker.noteCpuStore(page, page + WritePageBytes, tracker.generation);
         }
         if (count < tracker.pages.size()) break;
@@ -1088,7 +1088,7 @@ bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop,
             if (!GuestArena::GuestArenaCollectWrites_nid_postfix(cursor, static_cast<std::size_t>(stop - cursor), tracker.pages.data(), &count, true)) return false;
             for (ULONG_PTR i = 0; i < count; ++i) {
                 const auto page = reinterpret_cast<std::uintptr_t>(tracker.pages[i]);
-                tracker.stamp(tracker.blockOf(page), tracker.generation, kind);
+                tracker.stamp(tracker.blockOf(page), tracker.generation, window != nullptr && window->Holds(page) ? StampKind::ImportWindow : kind);
                 if (kind != StampKind::Driver) tracker.noteCpuStore(page, page + WritePageBytes, tracker.generation);
             }
             if (count < tracker.pages.size()) break;
@@ -1098,7 +1098,7 @@ bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop,
     return true;
 #else
     const auto dirtyRuns = collectDirtyRuns.load(std::memory_order_relaxed);
-    StampRuns runs{tracker, kind};
+    StampRuns runs{tracker, kind, 0, window};
     const auto collect = armedOnly ? &GuestWriteWatch::GuestWriteWatchCollectArmed_nid_postfix : &GuestWriteWatch::GuestWriteWatchCollect_nid_postfix;
     const bool complete = collect(static_cast<std::uintptr_t>(first), static_cast<std::size_t>(stop - first), &stampWrittenRun, &runs);
     if (collectDirtyRuns.load(std::memory_order_relaxed) != dirtyRuns) collectDirty.fetch_add(1, std::memory_order_relaxed);
@@ -1208,7 +1208,7 @@ void Unwatch(std::uint64_t address, std::size_t bytes) {
     unwatchLocked(tracker, address, bytes);
 }
 
-bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import) {
+bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched) {
     if (bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return import();
     auto& tracker = Tracker();
     auto lock = lockTracker(tracker);
@@ -1221,9 +1221,21 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function
     const auto first = address & ~(page - 1);
     const auto stop = (address + bytes + page - 1) & ~(page - 1);
     const bool covered = tracker.covers(first, static_cast<std::size_t>(stop - first));
+    std::vector<std::byte> saved;
+    if (covered && staysWatched) {
+        CheckRange(reinterpret_cast<const void*>(first), static_cast<std::size_t>(stop - first), 1, false);
+        const auto* source = reinterpret_cast<const std::byte*>(first);
+        saved.assign(source, source + (stop - first));
+    }
     const bool before = covered && walkWrites(tracker, first, stop, StampKind::Cpu, true);
     if (!import()) return false;
-    if (!before || !walkWrites(tracker, first, stop, StampKind::ImportWindow, true)) unwatchLocked(tracker, first, static_cast<std::size_t>(stop - first));
+    if (!staysWatched) {
+        if (!before || !walkWrites(tracker, first, stop, StampKind::ImportWindow, true)) unwatchLocked(tracker, first, static_cast<std::size_t>(stop - first));
+        unwatchLocked(tracker, address, bytes);
+        return true;
+    }
+    const WindowCopy window{first, saved};
+    if (!before || !walkWrites(tracker, first, stop, StampKind::Cpu, true, &window)) unwatchLocked(tracker, first, static_cast<std::size_t>(stop - first));
     return true;
 }
 
@@ -1355,7 +1367,7 @@ bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generati
     const auto first = tracker.blockOf(address);
     const auto last = tracker.blockOf(end - 1);
     for (auto block = first; block <= last; ++block) {
-        if (tracker.writtenStampOf(block) <= generation) continue;
+        if (tracker.stampOf(block) <= generation) continue;
         if (const auto found = tracker.cpuPages.find(block); found != tracker.cpuPages.end()) {
             const auto begin = tracker.blockBegin(block);
             const auto from = std::max(address, begin) - begin;
@@ -1426,7 +1438,7 @@ bool WrittenSince(std::uint64_t address, std::size_t bytes, std::uint64_t genera
     const auto first = tracker.blockOf(address);
     const auto last = tracker.blockOf(address + bytes - 1);
     for (auto block = first; block <= last; ++block) {
-        if (tracker.writtenStampOf(block) > generation) return true;
+        if (tracker.stampOf(block) > generation) return true;
     }
     return false;
 }
@@ -1444,7 +1456,7 @@ bool ChangedBlocks(std::uint64_t address, std::size_t bytes, std::span<const std
         const auto k = block - first;
         if (k >= generations.size() || k >= changed.size()) break;
         const auto generation = generations[k];
-        changed[k] = generation == 0 || tracker.writtenStampOf(block) > generation ? BlockWritten : tracker.stampOf(block) > generation ? BlockMaybeWritten : BlockUnchanged;
+        changed[k] = generation == 0 || tracker.stampOf(block) > generation ? BlockWritten : BlockUnchanged;
         if (k < cpu.size()) cpu[k] = generation == 0 || tracker.cpuStampOf(block) > generation ? 1 : 0;
     }
     return true;

core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp

@@ -52,13 +52,6 @@ ImportProbe ProbeImportWriteProtection(const Context& context);
 ImportProbe ProbeDmaBufImportWriteProtection(const Context& context);
 ImportWatch PrepareImportWatch(const Context& context);
 void SetImportWatch(const Context& context, ImportWatch watch);
-// Whether a host import that leaves its range write-watched can stamp the range's pages when it is
-// made (the device's probe saw pages written at the import): its import window then stamps them as
-// a possible CPU store, which a storage image resolves only by comparing with its guest-byte copy.
-// Decides the device's import watch first; false without host imports or a write watch, true when
-// the imports were not probed. The overload without a device reads the last decision (true before one).
-bool WatchedImportsStampPages(const Context& context);
-bool WatchedImportsStampPages();
 
 // The host import of the registered allocation containing [address, address + bytes), made on demand
 // (alignment and budget permitting), or null. Bytes at `address` are at `address - import->base` in

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -23,6 +23,7 @@
 #include <array>
 #include <atomic>
 #include <tuple>
+#include <utility>
 #include <functional>
 #include <bit>
 #include <condition_variable>
@@ -169,10 +170,6 @@ struct HostImports {
     VkDevice watchDevice = VK_NULL_HANDLE;
     bool unwatchImports = false;
     bool unwatchDmaBufImports = false;
-    // Whether making an import of each kind was seen to stamp the imported pages (the probe's
-    // writtenAtImport): its import window then stamps a range the import leaves watched.
-    bool importsStampPages = true;
-    bool dmaBufImportsStampPages = true;
 };
 
 HostImports& Imports() {
@@ -481,20 +478,39 @@ const char* createChunkedDmaBufImport(const Context& context, HostImport& entry,
 }
 #endif
 
+#ifndef _WIN32
+struct BackingFile {
+    BackingFile() = default;
+    BackingFile(const BackingFile&) = delete;
+    BackingFile& operator=(const BackingFile&) = delete;
+    ~BackingFile() {
+        if (file >= 0) close(file);
+    }
+    int file = -1;
+    std::uint64_t offset = 0;
+};
+
+bool sharedBacking(const Context& context, const HostImport& entry, BackingFile& backing) {
+    return context.dmaBufImport && GuestArena::GuestArenaSharedBacking_nid_postfix(static_cast<std::uintptr_t>(entry.base), static_cast<std::size_t>(entry.bytes), &backing.file, &backing.offset);
+}
+
+const char* createBackingImport(const Context& context, HostImport& entry, BackingFile& backing, VkResult& failure) {
+    const int file = std::exchange(backing.file, -1);
+    bool fits = false;
+    VkResult chunkFailure = VK_SUCCESS;
+    if (createChunkedDmaBufImport(context, entry, file, backing.offset, fits, chunkFailure) == nullptr && fits) {
+        close(file);
+        return nullptr;
+    }
+    return createDmaBufImport(context, entry, file, backing.offset, failure);
+}
+#endif
+
 const char* createImport(const Context& context, HostImport& entry, VkResult& failure) {
     const char* step = createHostPointerImport(context, entry, failure);
 #ifndef _WIN32
-    int file = -1;
-    std::uint64_t offset = 0;
-    if (step != nullptr && context.dmaBufImport && GuestArena::GuestArenaSharedBacking_nid_postfix(static_cast<std::uintptr_t>(entry.base), static_cast<std::size_t>(entry.bytes), &file, &offset)) {
-        bool fits = false;
-        VkResult chunkFailure = VK_SUCCESS;
-        if (createChunkedDmaBufImport(context, entry, file, offset, fits, chunkFailure) == nullptr && fits) {
-            close(file);
-            return nullptr;
-        }
-        step = createDmaBufImport(context, entry, file, offset, failure);
-    }
+    BackingFile backing;
+    if (step != nullptr && sharedBacking(context, entry, backing)) step = createBackingImport(context, entry, backing, failure);
 #endif
     return step;
 }
@@ -520,21 +536,14 @@ void decideImportWatch(const Context& context, HostImports& state) {
     state.watchDevice = context.device;
     state.unwatchImports = true;
     state.unwatchDmaBufImports = false;
-    state.importsStampPages = false;
-    state.dmaBufImportsStampPages = false;
     if (context.hostImportAlignment != 0 && GuestMemory::WriteWatched()) std::fprintf(stderr, "[write-watch] Windows direct host imports use comparisons; separate shared aliases retain guest write tracking\n");
 #else
     const auto request = importWatchRequest();
     state.watchDevice = context.device;
     state.unwatchImports = false;
     state.unwatchDmaBufImports = false;
-    state.importsStampPages = false;
-    state.dmaBufImportsStampPages = false;
     if (context.hostImportAlignment == 0 || !GuestMemory::WriteWatched()) return;
     if (request == ImportWatchRequest::Watch) {
-        // Not probed: whether an import stamps its pages is unknown.
-        state.importsStampPages = true;
-        state.dmaBufImportsStampPages = true;
         std::fprintf(stderr, "[write-watch] host imports stay watched (APS5_WRITE_WATCH_IMPORTS=watch)\n");
         return;
     }
@@ -550,11 +559,9 @@ void decideImportWatch(const Context& context, HostImports& state) {
         std::fprintf(stderr, "[write-watch] host imports resolve write protection: unknown (probe failed at %s, %d); imported ranges are compared\n", probe.failure, static_cast<int>(probe.result));
     } else {
         state.unwatchImports = probe.writtenAfterSubmit != 0 || probe.writtenAfterFill != 0;
-        state.importsStampPages = probe.writtenAtImport != 0;
         std::fprintf(stderr, "[write-watch] host imports resolve write protection: %s (%u of %u scratch pages written after a GPU read, %u after a GPU write, %u after the import); imported ranges %s\n", state.unwatchImports ? "yes" : "no", probe.writtenAfterSubmit, probe.pages, probe.writtenAfterFill, probe.writtenAtImport, state.unwatchImports ? "are compared" : "stay watched");
     }
     state.unwatchDmaBufImports = state.unwatchImports;
-    state.dmaBufImportsStampPages = state.importsStampPages;
     if (!context.dmaBufImport) return;
     const auto dmaBuf = ProbeDmaBufImportWriteProtection(context);
     if (dmaBuf.failure != nullptr) {
@@ -563,7 +570,6 @@ void decideImportWatch(const Context& context, HostImports& state) {
         return;
     }
     state.unwatchDmaBufImports = dmaBuf.writtenAfterSubmit != 0 || dmaBuf.writtenAfterFill != 0 || dmaBuf.writtenByCpu == 0;
-    state.dmaBufImportsStampPages = dmaBuf.writtenAtImport != 0;
     std::fprintf(stderr, "[write-watch] dma-buf imports keep write protection: %s (%u of %u scratch pages written after a GPU read, %u after a GPU write, %u after the import, %u seen after a CPU store); imported ranges %s\n", state.unwatchDmaBufImports ? "no" : "yes", dmaBuf.writtenAfterSubmit, dmaBuf.pages, dmaBuf.writtenAfterFill, dmaBuf.writtenAtImport, dmaBuf.writtenByCpu, state.unwatchDmaBufImports ? "are compared" : "stay watched");
 #endif
 }
@@ -745,15 +751,26 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
     decideImportWatch(context, state);
     VkResult result = VK_SUCCESS;
     const char* step = nullptr;
-    GuestMemory::ImportWatched(base, bytes, [&] {
-        step = createImport(context, entry, result);
-        return step == nullptr;
-    });
-    entry.unwatched = step == nullptr && (entry.dmaBuf ? state.unwatchDmaBufImports : state.unwatchImports);
 #ifdef _WIN32
-    if (entry.alias != nullptr) entry.unwatched = false;
+    const bool watchedHostPointer = entry.alias != nullptr || !state.unwatchImports;
+    const bool watchedDmaBuf = watchedHostPointer;
+#else
+    const bool watchedHostPointer = !state.unwatchImports;
+    const bool watchedDmaBuf = !state.unwatchDmaBufImports;
 #endif
-    if (entry.unwatched) GuestMemory::Unwatch(base, bytes);
+    GuestMemory::ImportWatched(base, bytes, [&] {
+        step = watchedDmaBuf == watchedHostPointer ? createImport(context, entry, result) : createHostPointerImport(context, entry, result);
+        return step == nullptr;
+    }, watchedHostPointer);
+#ifndef _WIN32
+    if (BackingFile backing; step != nullptr && watchedDmaBuf != watchedHostPointer && sharedBacking(context, entry, backing)) {
+        GuestMemory::ImportWatched(base, bytes, [&] {
+            step = createBackingImport(context, entry, backing, result);
+            return step == nullptr;
+        }, watchedDmaBuf);
+    }
+#endif
+    entry.unwatched = step == nullptr && !(entry.dmaBuf ? watchedDmaBuf : watchedHostPointer);
     if (step != nullptr) {
 #ifdef _WIN32
         GuestArena::GuestArenaUnmapAlias_nid_postfix(entry.alias);
@@ -1971,27 +1988,6 @@ ImportWatch PrepareImportWatch(const Context& context) {
     return state.unwatchImports ? ImportWatch::Unwatch : ImportWatch::Watch;
 }
 
-namespace {
-
-bool stampsWatchedPages(const HostImports& state) {
-    return (!state.unwatchImports && state.importsStampPages) || (!state.unwatchDmaBufImports && state.dmaBufImportsStampPages);
-}
-
-}
-
-bool WatchedImportsStampPages(const Context& context) {
-    auto& state = Imports();
-    std::lock_guard lock(state.mutex);
-    decideImportWatch(context, state);
-    return stampsWatchedPages(state);
-}
-
-bool WatchedImportsStampPages() {
-    auto& state = Imports();
-    std::lock_guard lock(state.mutex);
-    return state.watchDevice == VK_NULL_HANDLE || stampsWatchedPages(state);
-}
-
 void SetImportWatch(const Context& context, ImportWatch watch) {
     auto& state = Imports();
     std::lock_guard lock(state.mutex);

core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp

@@ -1559,7 +1559,7 @@ bool StorageTexture::Refresh() {
         if (!GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast<std::size_t>(guestBytes), alias->layerGeneration, aliasStamped)) alias = nullptr;
         for (std::uint32_t unit = 0; alias != nullptr && unit < trackedLayers; ++unit) {
             if (!alias->layerPending[unit]) continue;
-            if (alias->layerGeneration[unit] == 0 || aliasStamped[unit] == GuestMemory::BlockMaybeWritten || (aliasStamped[unit] == GuestMemory::BlockUnchanged && layerPending[unit])) {
+            if (alias->layerGeneration[unit] == 0 || (aliasStamped[unit] == GuestMemory::BlockUnchanged && layerPending[unit])) {
                 alias = nullptr;
                 break;
             }
@@ -1600,13 +1600,6 @@ bool StorageTexture::Refresh() {
             stampedBlocks.assign(generations.size(), 0);
             cpuBlocks.assign(generations.size(), 0);
             tracked = GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast<std::size_t>(guestBytes), generations, stampedBlocks, cpuBlocks);
-            for (std::uint32_t unit = 0; tracked && blockUnits && unit < trackedLayers; ++unit) {
-                if (stampedBlocks[unit] != GuestMemory::BlockMaybeWritten) continue;
-                std::array<std::uint8_t, 1> block{};
-                if (!compareUntracked(layerBegin(unit), static_cast<std::size_t>(layerBytes(unit)), block) || block[0] != GuestMemory::BlockUnchanged) continue;
-                stampedBlocks[unit] = GuestMemory::BlockUnchanged;
-                cpuBlocks[unit] = 0;
-            }
             compared = !tracked && compareUntracked(descriptor.baseAddress, static_cast<std::size_t>(guestBytes), stampedBlocks, true);
             if (compared) cpuBlocks = stampedBlocks;
         };
@@ -1978,10 +1971,8 @@ void StorageTexture::upload(const std::vector<bool>* layers) {
         // The clear is recorded into the open batch like a direct upload (the image kept by it): a
         // batch of its own submitted the recorder's work first and waited for all of it, 25-40 ms
         // under the GPU mutex at the movie stage. APS5_NO_RECORDED_CLEAR=1 waits as before.
-        // Watched memory needs no copy unless an import window can stamp it: Refresh then compares
-        // the stamped units with the copy (WatchedImportsStampPages).
         static_cast<void>(HostImportFor(context, descriptor.baseAddress, static_cast<std::size_t>(guestBytes)));
-        if (GuestMemory::Watched(descriptor.baseAddress, static_cast<std::size_t>(guestBytes)) && !WatchedImportsStampPages(context)) originalValid = false;
+        if (GuestMemory::Watched(descriptor.baseAddress, static_cast<std::size_t>(guestBytes))) originalValid = false;
         else captureGuestBytes(nullptr);
         forgetBorrowed(0, trackedLayers);
         stampLayers(false);

core/libs/prx/libSceAgcDriver/tests/Recorder.cpp

@@ -3888,9 +3888,7 @@ void importWindowTests(const Device& device, Recorder& recorder) {
         void* block;
         ~Release() { ReleaseWatched(block, bytes); }
     } release{block};
-    const auto probe = ProbeImportWriteProtection(base);
-    Require(probe.failure == nullptr, "(w) the import probe failed");
-    const bool importWrites = probe.writtenAtImport != 0;
+    Require(ProbeImportWriteProtection(base).failure == nullptr, "(w) the import probe failed");
     const auto decided = PrepareImportWatch(base);
     if (decided == ImportWatch::Unwatch) {
         std::cout << "host imports are compared, not watched: the import window not tested\n";
@@ -3959,16 +3957,11 @@ void importWindowTests(const Device& device, Recorder& recorder) {
         }
         Require(!import->unwatched && Watched(address, bytes), "(w) an import under the watch decision left the watch");
         CollectWritesUncached(address, surfaceBytes);
-        if (importWrites) {
-            Require(!UnchangedSince(address, surfaceBytes, cached), "(w) a cache over the pages the import reported sees no change");
-            Require(image->Generation() != 0 && !UnchangedSince(address, surfaceBytes, image->Generation()), "(w) the storage image took its generation after the import window");
-            std::array<std::uint8_t, surfaceBytes / 65536> changed{};
-            const std::array<std::uint64_t, surfaceBytes / 65536> generations{cached, cached, cached, cached};
-            Require(ChangedBlocks(address, surfaceBytes, generations, changed) && std::all_of(changed.begin(), changed.end(), [](std::uint8_t value) { return value == BlockMaybeWritten; }), "(w) the pages the import reported read as written by the CPU");
-            Require(!WrittenSince(address, surfaceBytes, cached), "(w) the import window reads as a CPU store");
-        } else {
-            std::cout << "the import reports no pages here: only the pending results of the import window checked\n";
-        }
+        Require(UnchangedSince(address, surfaceBytes, cached), "(w) an import that stored nothing reads as a change to a cache taken before it");
+        Require(image->Generation() != 0 && UnchangedSince(address, surfaceBytes, image->Generation()), "(w) an import that stored nothing reads as a change to the pending storage image");
+        std::array<std::uint8_t, surfaceBytes / 65536> changed{};
+        const std::array<std::uint64_t, surfaceBytes / 65536> generations{cached, cached, cached, cached};
+        Require(ChangedBlocks(address, surfaceBytes, generations, changed) && std::all_of(changed.begin(), changed.end(), [](std::uint8_t value) { return value == BlockUnchanged; }), "(w) the pages the import resolved read as written");
         const auto after = CollectWritesUncached(address, surfaceBytes);
         Require(after != 0 && UnchangedSince(address, surfaceBytes, after), "(w) the imported range is not watched again after the import window");
         image->Refresh();

core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
 #include "prx/libc/include/GuestArena.hpp"
 #include "prx/libc/include/GuestWriteWatch.hpp"
 #ifdef _WIN32
@@ -413,6 +414,95 @@ void CheckCollectBoundaries() {
 }
 }
 
+void CheckImportWindow() {
+    void* memory = AllocateWatched(3 * Block);
+    const auto base = reinterpret_cast<std::uint64_t>(memory);
+    auto* bytes = static_cast<volatile std::uint8_t*>(memory);
+    std::memset(memory, 0x11, 3 * Block);
+    const auto synced = CollectWritesUncached(base, 3 * Block);
+    Require(synced != 0, "the import window block is not collected");
+    Require(ImportWatched(base, 3 * Block, [&] {
+        bytes[0x100] = 0x11;
+        bytes[Block + 0x2000] = 0x22;
+        return true;
+    }), "the import window import failed");
+    Require(Watched(base, 3 * Block), "an import window left its range unwatched");
+    const std::array<std::uint64_t, 3> generations{synced, synced, synced};
+    std::array<std::uint8_t, 3> changed{};
+    std::array<std::uint8_t, 3> cpu{};
+    Require(ChangedBlocks(base, 3 * Block, generations, changed, cpu), "the import window block is not tracked");
+    Require(changed[0] == BlockUnchanged && UnchangedSince(base, Block, synced), "a page an import window resolved without changing its bytes reads as written");
+    Require(changed[1] == BlockWritten && WrittenSince(base + Block, Block, synced) && cpu[1] == 1, "a store of new bytes within an import window was missed");
+    Require(changed[2] == BlockUnchanged && cpu[2] == 0, "a block the import window did not touch reads as written");
+    Require(cpu[0] == 1, "a page an import window resolved no longer counts as possibly touched by the CPU");
+    const auto after = CollectWritesUncached(base, 3 * Block);
+    bytes[0x100] = 0x33;
+    static_cast<void>(CollectWritesUncached(base, 3 * Block));
+    Require(!UnchangedSince(base, Block, after), "a store after the import window was missed");
+}
+
+void ProtectPage(std::uint64_t address, bool readable) {
+#ifdef _WIN32
+    DWORD previous = 0;
+    Require(VirtualProtect(reinterpret_cast<void*>(address), 4096, readable ? PAGE_READWRITE : PAGE_NOACCESS, &previous) != 0, "cannot change the protection of a test page");
+#else
+    Require(mprotect(reinterpret_cast<void*>(address), 4096, readable ? PROT_READ | PROT_WRITE : PROT_NONE) == 0, "cannot change the protection of a test page");
+#endif
+    GuestAllocations::GuestAllocationsInvalidate_nid_postfix(static_cast<std::uintptr_t>(address), 4096);
+}
+
+void CheckUnwatchedImport() {
+    void* memory = AllocateWatched(3 * Block);
+    const auto base = reinterpret_cast<std::uint64_t>(memory);
+    auto* bytes = static_cast<volatile std::uint8_t*>(memory);
+    std::memset(memory, 0x11, 3 * Block);
+    const auto synced = CollectWritesUncached(base, 3 * Block);
+    Require(synced != 0, "the unwatched import block is not collected");
+    const auto imported = base + Block / 2;
+    const std::size_t importedBytes = Block + Block / 2;
+    const auto unreadable = base + Block + 0x3000;
+    ProtectPage(unreadable, false);
+    bool threw = false;
+    bool ran = false;
+    try {
+        static_cast<void>(ImportWatched(imported, importedBytes, [&] {
+            ran = true;
+            return true;
+        }));
+    } catch (const std::runtime_error&) {
+        threw = true;
+    }
+    Require(threw && !ran, "an import that stays watched did not read its range before the import");
+    threw = false;
+    try {
+        Require(ImportWatched(imported, importedBytes, [&] {
+            bytes[Block / 2 + 0x100] = 0x22;
+            ran = true;
+            return true;
+        }, false), "an import that leaves the watch failed");
+    } catch (const std::runtime_error&) {
+        threw = true;
+    }
+    Require(!threw && ran, "an import that leaves the watch read its range");
+    ProtectPage(unreadable, true);
+    Require(!Watched(imported, importedBytes) && !Watched(imported, 4096) && !Watched(base + Block, Block), "an import that leaves the watch is still watched");
+    Require(Watched(base + 2 * Block, Block), "an import that leaves the watch unwatched a block outside its range");
+    Require(UnchangedSince(base + 2 * Block, Block, synced), "an import that leaves the watch stamped a block outside its range");
+#ifndef _WIN32
+    Require(Watched(base, Block / 2), "an import that leaves the watch unwatched the pages before its range");
+    const std::array<std::uint64_t, 1> generations{synced};
+    std::array<std::uint8_t, 1> changed{};
+    std::array<std::uint8_t, 1> cpu{};
+    Require(ChangedBlocks(base, Block / 2, generations, changed, cpu), "the pages before an import that leaves the watch are not tracked");
+    Require(changed[0] == BlockUnchanged && UnchangedSince(base, Block / 2, synced) && !WrittenSince(base, Block / 2, synced), "a store to a page that left the watch reads as a store to the watched pages of its block");
+    Require(cpu[0] == 1, "a page resolved while an import left the watch no longer counts as possibly touched by the CPU");
+    const auto after = CollectWritesUncached(base, Block / 2);
+    bytes[0x100] = 0x33;
+    static_cast<void>(CollectWritesUncached(base, Block / 2));
+    Require(after != 0 && !UnchangedSince(base, Block / 2, after), "a store beside an import that left the watch was missed");
+#endif
+}
+
 int main() {
     try {
         if (!WriteWatched()) {
@@ -435,6 +525,8 @@ int main() {
 #endif
         CheckVersionRanges();
         CheckCollectBoundaries();
+        CheckImportWindow();
+        CheckUnwatchedImport();
     } catch (const std::exception& error) {
         std::cerr << "write tracking test failed: " << error.what() << "\n";
         return 1;

core/libs/prx/libSceAgcDriver/tests/execution/DccKeysFirstWrite.cpp

@@ -229,8 +229,7 @@ void Run(AgcDriver::VulkanDevice& device, std::uint8_t* block) {
     Require(AgcDriver::Graphics::StorageImageServesKeys(*image, AddressOf(copied)), "the written image does not serve the copied keys");
     Require(image->UploadedKeys() == DccKeys::Uncompressed && image->FilledKeys() == DccKeys::Uncompressed, "the written image still holds the clear it was uploaded under");
     const bool stillWatched = AgcDriver::GuestMemory::Watched(surface, SurfaceBytes);
-    const bool keepsCopy = !stillWatched || AgcDriver::Graphics::WatchedImportsStampPages();
-    Require(image->GuestSnapshotValid() == keepsCopy, !keepsCopy ? "a fast-cleared upload of write-watched memory read its guest bytes" : stillWatched ? "a fast-cleared upload of write-watched memory that an import window can stamp kept no guest bytes to compare" : "a fast-cleared upload of untracked memory kept no guest bytes to compare");
+    Require(image->GuestSnapshotValid() == !stillWatched, stillWatched ? "a fast-cleared upload of write-watched memory read its guest bytes" : "a fast-cleared upload of untracked memory kept no guest bytes to compare");
     const auto pendingVersion = image->Version();
     for (int repeat = 0; repeat < 3; ++repeat) {
         Require(image->Refresh(), "an unchanged pending image was uploaded again");

docs/dev/debt/libSceAgcDriver.md

@@ -42,7 +42,7 @@
 - [Draw input snapshots](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver): `PrepareDrawBindings` caps read-only input buffer snapshots at 64 MiB (overridable via `APS5_DRAW_SNAPSHOT_MAX_MIB`). Larger inputs (streaming heaps, bindless pools) or draws exceeding the cumulative budget skip the per-draw snapshot and bind in place, avoiding multi-gigabyte CPU copy stalls (such as Ratchet & Clank: Rift Apart PPSA01474 stalling at frame ~380 copying 36 GB every 5 s). If the title rewrites those large ranges on the CPU before GPU completion, the in-place binding observes the modification, as verified by `drawSnapshotCapTests`.
 - [Pending storage-image results](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) are stored when something reads their memory, not when the GPU work that wrote them completes. A title that waits for that work, frees the image and reuses its memory can have the new data overwritten by the late store (#1970). `APS5_STORE_AT_FLIP=1` stores every pending result at each flip; storing them at the title's GPU waits (labels, fences) is unimplemented.
 - Windows host-pointer imports of the guest mapping itself are compared instead of write-watched: a driver can mark imported pages dirty after the synchronous import window. Shared memory imported through a separate writable alias retains protection-based tracking on the guest views; driver access to the import alias does not disarm those views. [GuestMemory](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) excludes every overlapping 64 KiB tracker block while its backing is retained. A full block replaced by fresh private backing regains tracking and gets a new generation; protection changes, partial replacements and shared remappings do not restore it. This preserves CPU edits through byte comparisons but disables snapshot reuse and unit shadows for excluded blocks; restoring tracking on import retirement without replacing the backing and measuring the comparison cost remain unimplemented.
-- [Storage-image units under a Linux import window](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) that the window stamped as a possible CPU store (`BlockMaybeWritten`) are compared with the bytes the image was uploaded from or written back as, and are unchanged when those still match. A CPU store of the bytes already there within the window cannot be told apart from an import that resolves the write protection without storing, as the NVIDIA driver (610.57.04) does for every imported page, so such a store does not win over pending results. Images tracked by layer instead of by 64 KiB unit still read the window as a change and upload again.
+- [Pages under an import window](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) (libSceAgcDriver) whose write protection was resolved while a host import that leaves its range watched was made are compared with the bytes they held before the import and count as stored over only when those differ. A CPU store of the bytes already there within the window cannot be told apart from an import that resolves the write protection without storing, as the NVIDIA driver (610.57.04) does for every imported page, so such a store does not win over pending results. Where host-pointer and dma-buf imports differ in whether they stay watched, a failed host-pointer import and the dma-buf import after it are two windows, and the pages the failed one resolved count as stored over.
 - [Unnormalized S# coordinates](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) - `FORCE_UNNORMALIZED` uses a Vulkan `unnormalizedCoordinates` sampler, so the S# LOD range, LOD bias and mip filter are not applied, which is exact only on a one-level view.
 - An [unnormalized S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) throws with unequal minification and magnification filters, anisotropy, `TRUNC_COORD`, `MC_COORD_TRUNC` or X/Y clamping other than last-texel or border.
 - [Background link-time-optimized relink of library pipelines](../../../core/libs/prx/libSceAgcDriver/Execution/include/PipelineLinkOptimization.hpp) (libSceAgcDriver) is skipped on every GPU under NVIDIA's proprietary Vulkan driver (`VK_DRIVER_ID_NVIDIA_PROPRIETARY`), so those pipelines keep their fast-linked form. The optimized link loses the device when bound on an RTX 3080 (Ampere, #2866) and an RTX 4060 (Ada, #3434); an RTX 5060 and an RTX 5080 (Blackwell) on the same driver do not, so the scope is wider than the evidence. No standard Vulkan query tells the architecture apart, and the driver version that fixes it is unknown. `APS5_NO_PIPELINE_LTO` disables the relink on every driver.