Merge pull request #9879 from thuanquang/fix/linear-target-import-creation

perf(agc): create the host import of a linear color target in registered memory

core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp

@@ -75,8 +75,8 @@ ImportProbe ProbeDmaBufImportWriteProtection(const Context& context);
 ImportWatch PrepareImportWatch(const Context& context);
 void SetImportWatch(const Context& context, ImportWatch watch);
 
-enum class ImportRequester : std::uint8_t { Map, Build, Access, Join, Segment };
-inline constexpr std::size_t ImportRequesterCount = 5;
+enum class ImportRequester : std::uint8_t { Map, Build, Access, Join, Segment, ColorTarget };
+inline constexpr std::size_t ImportRequesterCount = 6;
 
 struct HostImportRequesterCounts {
     std::uint64_t made = 0;

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -2566,7 +2566,7 @@ bool prepareDraw(const Context& context, const State& state, const Pm4::DrawPara
         // Debug aid: APS5_NO_RESIDENT_TARGETS=1 copies every target in and out again.
         static const bool residentTargets = std::getenv("APS5_NO_RESIDENT_TARGETS") == nullptr;
         static const bool residentLinear = std::getenv("APS5_NO_RESIDENT_LINEAR") == nullptr;
-        const bool linearResident = residentLinear && color.tileMode == ColorTileMode::Linear && ExistingHostImportFor(context, color.address, colorLayout.Bytes()) != nullptr;
+        const bool linearResident = residentLinear && residentTargets && context.detiler != nullptr && color.tileMode == ColorTileMode::Linear && (ExistingHostImportFor(context, color.address, colorLayout.Bytes()) != nullptr || (GuestMemory::Watched(color.address, colorLayout.Bytes()) && HostImportFor(context, color.address, colorLayout.Bytes(), ImportRequester::ColorTarget) != nullptr));
         if ((binding.gpuTiling || ((color.tileMode != ColorTileMode::Linear || linearResident) && context.detiler != nullptr)) && residentTargets) {
             // The lookup refreshes the image on every draw (StorageTexture::Refresh: FlushPending,
             // CollectWrites over the target's pages, the DCC key scan of TextureClearKeys, then

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -179,6 +179,14 @@ struct ImportRefusal {
     std::uint64_t budget;
 };
 
+struct GateRefusal {
+    std::uint64_t base;
+    std::uint64_t bytes;
+    GuestAllocations::Mapped ranges;
+    std::uint64_t commits;
+    std::uint64_t watches;
+};
+
 struct EvictingRefusal {
     std::uint64_t base;
     std::uint64_t bytes;
@@ -250,6 +258,7 @@ struct HostImports {
     std::map<std::pair<std::uint64_t, std::uint64_t>, ImportFailure> failedBlocks;
     std::map<std::pair<std::uint64_t, std::uint64_t>, ImportRefusal> refusals;
     std::map<std::pair<std::uint64_t, std::uint64_t>, EvictingRefusal> evictingRefusals;
+    std::map<std::pair<std::uint64_t, std::uint64_t>, GateRefusal> gateRefusals;
     // Registry generation the imports were last reconciled with.
     std::uint64_t refreshedGeneration = 0;
     // Bumped whenever an import is dropped, so HostImport pointers taken under the lock earlier can be
@@ -828,8 +837,12 @@ struct BackingFile {
     std::uint64_t offset = 0;
 };
 
+bool sharedBacking(const Context& context, std::uint64_t base, std::uint64_t bytes, BackingFile& backing) {
+    return context.dmaBufImport && GuestArena::GuestArenaSharedBacking_nid_postfix(static_cast<std::uintptr_t>(base), static_cast<std::size_t>(bytes), &backing.file, &backing.offset);
+}
+
 bool sharedBacking(const Context& context, const HostImport& entry, BackingFile& backing) {
-    return context.dmaBufImport && GuestArena::GuestArenaSharedBacking_nid_postfix(static_cast<std::uintptr_t>(entry.base), static_cast<std::size_t>(entry.bytes), &backing.file, &backing.offset);
+    return sharedBacking(context, entry.base, entry.bytes, backing);
 }
 
 const char* createBackingImport(const Context& context, HostImport& entry, BackingFile& backing, VkResult& failure) {
@@ -1252,6 +1265,62 @@ void decideImportWatch(const Context& context, HostImports& state) {
 #endif
 }
 
+struct ImportWatching {
+    bool settled = false;
+    bool hostPointer = false;
+    bool dmaBuf = false;
+};
+
+ImportWatching importWatching(const Context& context, HostImports& state, bool alias, bool writable, std::uint64_t bytes) {
+    decideImportWatch(context, state);
+#ifdef _WIN32
+    const bool settled = !alias && state.watchPrivateImports && writable && bytes >= settleFloorBytes;
+    const bool watched = alias || !state.unwatchImports || settled;
+    return {settled, watched, watched};
+#else
+    static_cast<void>(alias);
+    static_cast<void>(writable);
+    static_cast<void>(bytes);
+    return {false, !state.unwatchImports, !state.unwatchDmaBufImports};
+#endif
+}
+
+#ifdef _WIN32
+struct ImportPages {
+    bool committed = true;
+    bool writable = true;
+    bool readOnly = true;
+    bool shared = true;
+    MEMORY_BASIC_INFORMATION refused{};
+};
+
+bool writableProtection(std::uint32_t protection) {
+    protection &= 0xffu;
+    return protection == PAGE_READWRITE || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_WRITECOPY;
+}
+
+ImportPages importPages(std::uint64_t base, std::uint64_t bytes, bool opening = false) {
+    ImportPages pages;
+    for (std::uint64_t cursor = base; cursor < base + bytes;) {
+        MEMORY_BASIC_INFORMATION info{};
+        if (VirtualQuery(reinterpret_cast<const void*>(cursor), &info, sizeof(info)) == 0 || info.State != MEM_COMMIT || (info.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) {
+            pages.committed = false;
+            return pages;
+        }
+        const auto protection = info.Protect & 0xffu;
+        const bool pageWritable = (info.Type == MEM_PRIVATE || info.Type == MEM_IMAGE) && writableProtection(protection);
+        const bool pageReadOnly = info.Type != MEM_MAPPED && (protection == PAGE_READONLY || protection == PAGE_EXECUTE_READ);
+        if (!pageReadOnly) pages.readOnly = false;
+        if (info.Type != MEM_MAPPED) pages.shared = false;
+        if (info.Type != MEM_MAPPED && !pageWritable && !pageReadOnly && pages.refused.BaseAddress == nullptr) pages.refused = info;
+        if (!pageWritable) pages.writable = false;
+        if (opening && !pages.writable && !pages.shared) return pages;
+        cursor = reinterpret_cast<std::uint64_t>(info.BaseAddress) + info.RegionSize;
+    }
+    return pages;
+}
+#endif
+
 std::shared_ptr<const GuestAllocations::Range> leasedRangeOwner(const GuestAllocations::Lease& lease, std::uint64_t base) {
     const auto found = std::lower_bound(lease.begin(), lease.end(), base, [](const auto& range, std::uint64_t value) { return range->address < value; });
     return found != lease.end() && (*found)->address == base ? *found : nullptr;
@@ -1270,6 +1339,22 @@ struct ImportRun {
     std::vector<std::shared_ptr<const GuestAllocations::Range>> ranges;
 };
 
+bool importStaysWatched(const Context& context, HostImports& state, const ImportRun& run) {
+    if (std::any_of(run.ranges.begin(), run.ranges.end(), [](const auto& range) { return !range->writable; })) return false;
+    if (!GuestMemory::Watched(run.base, static_cast<std::size_t>(run.bytes))) return false;
+#ifdef _WIN32
+    MEMORY_BASIC_INFORMATION first{};
+    if (!importWatching(context, state, false, true, run.bytes).hostPointer && (VirtualQuery(reinterpret_cast<const void*>(run.base), &first, sizeof(first)) == 0 || first.Type != MEM_MAPPED)) return false;
+    const auto pages = importPages(run.base, run.bytes, true);
+    if (!pages.committed || (!pages.writable && !pages.shared)) return false;
+    return importWatching(context, state, !pages.writable, pages.writable, run.bytes).hostPointer;
+#else
+    const auto watching = importWatching(context, state, false, true, run.bytes);
+    BackingFile backing;
+    return watching.hostPointer && (watching.dmaBuf || !sharedBacking(context, run.base, run.bytes, backing));
+#endif
+}
+
 bool continuesRun(const GuestAllocations::Range& before, const GuestAllocations::Range& after) {
     return before.address + before.bytes == after.address && before.readable && after.readable && before.writable == after.writable && before.gpu == after.gpu;
 }
@@ -1843,6 +1928,21 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
         }
         state.evictingRefusals.erase(remembered);
     }
+    if (requester == ImportRequester::ColorTarget) {
+#ifdef _WIN32
+        const auto commits = GuestArena::GuestArenaCommitGeneration_nid_postfix();
+#else
+        const std::uint64_t commits = 0;
+#endif
+        const auto watches = GuestMemory::UnwatchSerial();
+        const std::pair key{base, bytes};
+        if (const auto refused = state.gateRefusals.find(key); refused != state.gateRefusals.end() && refused->second.commits == commits && refused->second.watches == watches && sameRun(refused->second, run)) return nullptr;
+        if (!importStaysWatched(context, state, run)) {
+            state.gateRefusals.insert_or_assign(key, GateRefusal{base, bytes, GuestAllocations::Mapped(run.ranges.begin(), run.ranges.end()), commits, watches});
+            return nullptr;
+        }
+        state.gateRefusals.erase(key);
+    }
     auto first = state.imports.lower_bound(base);
     if (first != state.imports.begin() && std::prev(first)->second.base + std::prev(first)->second.bytes > base) --first;
     std::uint64_t replaced = 0;
@@ -1917,27 +2017,17 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
 #ifdef _WIN32
     // Drivers pin imported pages, so every page must be committed and accessible.
     const GuestArena::HostImportAccess importAccess(reinterpret_cast<void*>(base), static_cast<std::size_t>(bytes));
-    bool writable = true;
-    bool readOnly = true;
-    MEMORY_BASIC_INFORMATION refused{};
-    for (std::uint64_t cursor = base; cursor < base + bytes;) {
-        MEMORY_BASIC_INFORMATION info{};
-        if (VirtualQuery(reinterpret_cast<const void*>(cursor), &info, sizeof(info)) == 0 || info.State != MEM_COMMIT || (info.Protect & (PAGE_NOACCESS | PAGE_GUARD)) != 0) return failedImport();
-        const auto protection = info.Protect & 0xffu;
-        const bool pageWritable = (info.Type == MEM_PRIVATE || info.Type == MEM_IMAGE) && (protection == PAGE_READWRITE || protection == PAGE_EXECUTE_READWRITE || protection == PAGE_WRITECOPY || protection == PAGE_EXECUTE_WRITECOPY);
-        const bool pageReadOnly = info.Type != MEM_MAPPED && (protection == PAGE_READONLY || protection == PAGE_EXECUTE_READ);
-        if (!pageReadOnly) readOnly = false;
-        if (info.Type != MEM_MAPPED && !pageWritable && !pageReadOnly && refused.BaseAddress == nullptr) refused = info;
-        if (!pageWritable) writable = false;
-        cursor = reinterpret_cast<std::uint64_t>(info.BaseAddress) + info.RegionSize;
-    }
-    if (!writable && readOnly) return failedImport();
+    const auto pages = importPages(base, bytes);
+    if (!pages.committed) return failedImport();
+    const bool writable = pages.writable;
+    if (!writable && pages.readOnly) return failedImport();
     if (!writable) {
-        if (refused.BaseAddress != nullptr) {
+        if (pages.refused.BaseAddress != nullptr) {
             char text[256];
-            std::snprintf(text, sizeof(text), "AGC graphics: host import of 0x%llx+0x%llx: memory at 0x%llx (type 0x%lx, protection 0x%lx) is neither read-write, read-only nor a shared mapping", static_cast<unsigned long long>(base), static_cast<unsigned long long>(bytes), reinterpret_cast<unsigned long long>(refused.BaseAddress), refused.Type, refused.Protect);
+            std::snprintf(text, sizeof(text), "AGC graphics: host import of 0x%llx+0x%llx: memory at 0x%llx (type 0x%lx, protection 0x%lx) is neither read-write, read-only nor a shared mapping", static_cast<unsigned long long>(base), static_cast<unsigned long long>(bytes), reinterpret_cast<unsigned long long>(pages.refused.BaseAddress), pages.refused.Type, pages.refused.Protect);
             throw std::runtime_error(text);
         }
+        std::erase_if(state.gateRefusals, [&](const auto& refusal) { return refusal.first.first < base + bytes && base < refusal.first.first + refusal.first.second; });
         try {
             entry.alias = GuestArena::GuestArenaMapAlias_nid_postfix(static_cast<std::uintptr_t>(base), static_cast<std::size_t>(bytes));
         } catch (const std::runtime_error&) {
@@ -1949,17 +2039,16 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
 #else
     if (!GuestMemory::Accessible(reinterpret_cast<const void*>(base), static_cast<std::size_t>(bytes), true)) return failedImport();
 #endif
-    decideImportWatch(context, state);
     VkResult result = VK_SUCCESS;
     const char* step = nullptr;
 #ifdef _WIN32
-    const bool settled = entry.alias == nullptr && state.watchPrivateImports && writable && bytes >= settleFloorBytes;
-    const bool watchedHostPointer = entry.alias != nullptr || !state.unwatchImports || settled;
-    const bool watchedDmaBuf = watchedHostPointer;
+    const auto watching = importWatching(context, state, entry.alias != nullptr, writable, bytes);
+    const bool settled = watching.settled;
 #else
-    const bool watchedHostPointer = !state.unwatchImports;
-    const bool watchedDmaBuf = !state.unwatchDmaBufImports;
+    const auto watching = importWatching(context, state, false, true, bytes);
 #endif
+    const bool watchedHostPointer = watching.hostPointer;
+    const bool watchedDmaBuf = watching.dmaBuf;
     GuestMemory::ImportWatched(base, bytes, [&] {
         step = watchedDmaBuf == watchedHostPointer ? createImport(context, entry, result) : createHostPointerImport(context, entry, result);
 #ifdef _WIN32
@@ -2089,6 +2178,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
         state.failedBlocks.clear();
         state.refusals.clear();
         state.evictingRefusals.clear();
+        state.gateRefusals.clear();
         state.ceiling = 0;
         state.allocationFailures = 0;
         state.allocationFailed = false;
@@ -2107,6 +2197,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
     const auto generation = GuestAllocations::GuestAllocationsGeneration_nid_postfix();
     if (generation == state.refreshedGeneration) return;
     state.refreshedGeneration = generation;
+    state.gateRefusals.clear();
     const auto starts = [&](std::uint64_t address) {
         const auto index = leasedIndexContaining(lease, address);
         return index != lease.size() && lease[index]->address == address;
@@ -3488,6 +3579,7 @@ void ClearHostImports(VkDevice device) {
     state.failedBlocks.clear();
     state.refusals.clear();
     state.evictingRefusals.clear();
+    state.gateRefusals.clear();
     state.ceiling = 0;
     state.allocationFailures = 0;
     state.allocationFailed = false;
@@ -3711,6 +3803,7 @@ void SetImportWatch(const Context& context, ImportWatch watch) {
     state.watchDevice = context.device;
     state.unwatchImports = watch == ImportWatch::Unwatch;
     state.unwatchDmaBufImports = state.unwatchImports;
+    state.gateRefusals.clear();
 }
 
 void SetHostImportWindow(std::uint64_t bytes) {
@@ -3792,6 +3885,7 @@ const char* ImportRequesterName(ImportRequester requester) {
         case ImportRequester::Access: return "access";
         case ImportRequester::Join: return "join";
         case ImportRequester::Segment: return "segment";
+        case ImportRequester::ColorTarget: return "color target";
     }
     return "unknown";
 }

core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp

@@ -1130,7 +1130,7 @@ void importAtMapOverBudgetTests(const Context& context) {
 void importRequesterTests(const Context& context, const BdaTestAccess& access) {
     ClearHostImports(context.device);
     constexpr std::size_t piece = 65536;
-    constexpr std::size_t pieces = 8;
+    constexpr std::size_t pieces = 11;
     auto* block = static_cast<std::uint8_t*>(::operator new(pieces * piece, std::align_val_t{piece}));
     std::memset(block, 0x2d, pieces * piece);
     auto importing = context;
@@ -1144,7 +1144,7 @@ void importRequesterTests(const Context& context, const BdaTestAccess& access) {
     };
     const auto counted = [](ImportRequester requester) { return HostImportCounters().requesters[static_cast<std::size_t>(requester)]; };
     const auto made = [&](ImportRequester requester) { return counted(requester).made; };
-    Require(std::string(ImportRequesterName(ImportRequester::Map)) == "map" && std::string(ImportRequesterName(ImportRequester::Build)) == "build" && std::string(ImportRequesterName(ImportRequester::Access)) == "access" && std::string(ImportRequesterName(ImportRequester::Join)) == "join" && std::string(ImportRequesterName(ImportRequester::Segment)) == "segment", "an import requester has the wrong name");
+    Require(std::string(ImportRequesterName(ImportRequester::Map)) == "map" && std::string(ImportRequesterName(ImportRequester::Build)) == "build" && std::string(ImportRequesterName(ImportRequester::Access)) == "access" && std::string(ImportRequesterName(ImportRequester::Join)) == "join" && std::string(ImportRequesterName(ImportRequester::Segment)) == "segment" && std::string(ImportRequesterName(ImportRequester::ColorTarget)) == "color target", "an import requester has the wrong name");
     const auto mappedRange = [&](std::size_t index) {
         GuestAllocations::Mapped ranges;
         const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
@@ -1245,9 +1245,27 @@ void importRequesterTests(const Context& context, const BdaTestAccess& access) {
         SetHostImportWindow(0);
         ::operator delete(joined, std::align_val_t{window});
     }
+    change(10, true);
+    change(9, true);
+    {
+        const auto decided = PrepareImportWatch(importing);
+        const auto before = counted(ImportRequester::ColorTarget);
+        SetImportWatch(importing, ImportWatch::Unwatch);
+        const auto* compared = HostImportFor(importing, address(10), 16, ImportRequester::ColorTarget);
+        const auto refused = counted(ImportRequester::ColorTarget);
+        SetImportWatch(importing, ImportWatch::Watch);
+        const auto* readOnly = HostImportFor(importing, address(9), 16, ImportRequester::ColorTarget);
+        const auto* outside = HostImportFor(importing, address(10), 16, ImportRequester::ColorTarget);
+        SetImportWatch(importing, decided);
+        Require(compared == nullptr && refused.made == before.made && refused.refused == before.refused && refused.failed == before.failed, "a color target's import was made where imports of its memory are compared");
+        Require(readOnly == nullptr && !HostImportCovers(importing, address(9), piece), "a color target's import was made of registered memory the guest cannot write");
+        Require(outside == nullptr && !HostImportCovers(importing, address(10), piece) && made(ImportRequester::ColorTarget) == before.made, "a color target's import was made of memory outside the write watch");
+    }
     const auto counts = HostImportCounters();
     Require(counts.live >= 4 && counts.liveBytes == HostImportLiveBytes(), "the live import counts disagree with the live bytes");
     for (std::size_t index = 0; index < 6; ++index) change(index, false);
+    change(10, false);
+    change(9, false);
     {
         GuestAllocations::Mutation mutation;
         mutation.Remove(block + 6 * piece);
@@ -1263,8 +1281,9 @@ void hostImportReportTests() {
     counts.requesters[static_cast<std::size_t>(ImportRequester::Map)] = {2, 1u << 20u, 0, 0, 0};
     counts.requesters[static_cast<std::size_t>(ImportRequester::Build)] = {5, 4u << 20u, 7, 7u << 19u, 1, 4};
     counts.requesters[static_cast<std::size_t>(ImportRequester::Segment)] = {1, 1u << 16u, 0, 0, 2};
+    counts.requesters[static_cast<std::size_t>(ImportRequester::ColorTarget)] = {1, 1u << 20u, 0, 0, 0};
     const auto text = FormatHostImportCounts(counts);
-    Require(text == "3 live (3.0 MiB); made 8 (5.1 MiB), refused 7 (3.5 MiB, 4 remembered), failed 3; by requester (made/refused/failed): map 2/0/0, build 5/7/1, access 0/0/0, join 0/0/0, segment 1/0/2", "the host import report is wrong: " + text);
+    Require(text == "3 live (3.0 MiB); made 9 (6.1 MiB), refused 7 (3.5 MiB, 4 remembered), failed 3; by requester (made/refused/failed): map 2/0/0, build 5/7/1, access 0/0/0, join 0/0/0, segment 1/0/2, color target 1/0/0", "the host import report is wrong: " + text);
 }
 
 class PieceWindow {

core/libs/prx/libSceAgcDriver/tests/execution/LinearTargetImportCreation.cpp

@@ -0,0 +1,474 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <span>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace AgcDriver {
+
+struct VulkanDeviceTestAccess {
+    static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); }
+};
+
+}
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using AgcDriver::Graphics::StorageTexture;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Width = 64;
+constexpr std::uint32_t Height = 16;
+constexpr std::size_t BlockBytes = 65536;
+constexpr std::size_t SettledBytes = 16 * BlockBytes;
+constexpr std::size_t ResidentBytes = 32 * BlockBytes;
+constexpr std::size_t TargetBytes = std::size_t{Width} * Height * 16u;
+constexpr std::size_t RowBytes = std::size_t{Width} * 16u;
+constexpr std::uint32_t Rgba32FloatInfo = 0x8738u;
+constexpr std::uint32_t LinearAttrib3 = 0x09000000u;
+constexpr std::uint32_t Float32AbgrExport = 9;
+constexpr VkColorComponentFlags RedGreen = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT;
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+constexpr std::array<std::array<float, 4>, 3> Triangle{{
+    {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f}
+}};
+
+alignas(256) std::array<std::uint32_t, 16> PixelCode{};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+std::span<const std::uint32_t> Program() {
+    std::size_t at = 0;
+    PixelCode[at++] = 0x36000000u | (20u << 17u) | 0xffu;
+    PixelCode[at++] = 0x0000ffffu;
+    PixelCode[at++] = 0x2c000000u | (21u << 17u) | 0x90u;
+    PixelCode[at++] = 0x7e000c00u | (20u << 17u) | (256u + 20u);
+    PixelCode[at++] = 0x7e000c00u | (21u << 17u) | (256u + 21u);
+    PixelCode[at++] = 0x7e000280u | (22u << 17u);
+    PixelCode[at++] = 0x7e0002f2u | (23u << 17u);
+    PixelCode[at++] = 0xf800180fu;
+    PixelCode[at++] = 0x17161514u;
+    PixelCode[at++] = 0xbf810000u;
+    return std::span<const std::uint32_t>(PixelCode.data(), at);
+}
+
+struct Block {
+    Block(std::size_t bytes, std::uintptr_t at, bool direct = false, std::size_t granule = 0, bool writable = true) : bytes(bytes), arena(AgcDriver::GuestMemory::WriteWatched()) {
+#ifdef _WIN32
+        if (arena && direct) {
+            data = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, 0x200000));
+            Require(data != nullptr, "cannot allocate the direct memory of the color target");
+            section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE | SEC_RESERVE, 0, static_cast<DWORD>(bytes), nullptr);
+            Require(section != nullptr, "cannot reserve the backing of the color target's direct memory");
+            GuestArena::GuestArenaMapDirect_nid_postfix(data, bytes, section, 0, PAGE_READWRITE);
+        } else if (arena) {
+            data = static_cast<std::uint8_t*>(at != 0 ? GuestArena::GuestArenaAllocateAtOrAbove_nid_postfix(at, bytes, BlockBytes) : GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes));
+            GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, granule != 0 ? granule : bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        }
+#else
+        static_cast<void>(direct);
+        static_cast<void>(granule);
+        if (arena && at != 0) {
+            void* raw = mmap(reinterpret_cast<void*>(at), bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map the color target");
+            data = static_cast<std::uint8_t*>(raw);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else if (arena) {
+            void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+            Require(raw != MAP_FAILED, "cannot map the color target");
+            const auto begin = reinterpret_cast<std::uintptr_t>(raw);
+            const auto aligned = (begin + BlockBytes - 1) & ~(static_cast<std::uintptr_t>(BlockBytes) - 1);
+            if (aligned != begin) munmap(raw, aligned - begin);
+            if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast<void*>(aligned + bytes), begin + BlockBytes - aligned);
+            data = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes);
+        } else {
+            data = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, bytes));
+        }
+#endif
+        Require(data != nullptr, "cannot allocate the color target");
+        GuestAllocations::Mutation mutation;
+        mutation.Add(data, bytes, true, writable, true);
+    }
+    ~Block() {
+        StorageTexture::FlushPending(Address(), bytes, nullptr, "test release");
+        {
+            GuestAllocations::Mutation mutation;
+            mutation.Remove(data);
+        }
+#ifdef _WIN32
+        if (arena) {
+            GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+            GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+        } else {
+            VirtualFree(data, 0, MEM_RELEASE);
+        }
+        if (section != nullptr) {
+            GuestArena::GuestArenaForgetSection_nid_postfix(section);
+            CloseHandle(section);
+        }
+#else
+        if (arena) {
+            munmap(data, bytes);
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes);
+        } else {
+            std::free(data);
+        }
+#endif
+    }
+    Block(const Block&) = delete;
+    Block& operator=(const Block&) = delete;
+    std::uint64_t Address() const { return reinterpret_cast<std::uintptr_t>(data); }
+    bool Watched() const { return arena && AgcDriver::GuestMemory::Watched(Address(), bytes); }
+    std::size_t bytes;
+    bool arena;
+    std::uint8_t* data = nullptr;
+#ifdef _WIN32
+    HANDLE section = nullptr;
+#endif
+};
+
+AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block) {
+    AgcDriver::Registers cx;
+    cx[0x318] = static_cast<std::uint32_t>(block.Address() >> 8u);
+    cx[0x31b] = 0;
+    cx[0x31c] = Rgba32FloatInfo;
+    cx[0x31d] = 0;
+    cx[0x323] = 0;
+    cx[0x324] = 0;
+    cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u);
+    cx[0x3b8] = LinearAttrib3;
+    cx[0x390] = static_cast<std::uint32_t>(block.Address() >> 40u);
+    auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
+    color.exportIndex = 0;
+    Require(color.tileMode == AgcDriver::Graphics::ColorTileMode::Linear && color.bytes == TargetBytes, "the target is not a linear R32G32B32A32 target");
+    return color;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, const Block& block) {
+    const auto color = DecodeTarget(block);
+    const auto target = device.Target();
+    constexpr std::uint32_t waveSize = 64;
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast<std::uint32_t>(Triangle.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    const auto pixelCode = Program();
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionFixedPoint);
+    pixel.targetOutputMode.at(0) = Float32AbgrExport;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::vector<std::uint32_t> pixelUserData(8, 0u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+
+    VkPipelineColorBlendAttachmentState blend{};
+    blend.blendEnable = VK_FALSE;
+    blend.colorWriteMask = RedGreen;
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.colors = {color};
+    state.color = color;
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = {{0, 0}, {Width, Height}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blends = {blend};
+    state.blend = blend;
+    state.blendConstants = {};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+}
+
+void Seed(Block& block, const std::array<float, 4>& texel) {
+    for (std::size_t offset = 0; offset < TargetBytes; offset += sizeof(texel)) std::memcpy(block.data + offset, texel.data(), sizeof(texel));
+}
+
+void Check(const Block& block, const std::array<float, 4>& seed, const std::string& what) {
+    for (std::uint32_t y = 0; y < Height; ++y) {
+        for (std::uint32_t x = 0; x < Width; ++x) {
+            std::array<float, 4> texel{};
+            std::memcpy(texel.data(), block.data + static_cast<std::size_t>(y) * RowBytes + static_cast<std::size_t>(x) * 16u, sizeof(texel));
+            const std::array<float, 4> expected{static_cast<float>(x), static_cast<float>(y), seed[2], seed[3]};
+            Require(texel == expected, what + ": pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") holds (" + std::to_string(texel[0]) + ", " + std::to_string(texel[1]) + ", " + std::to_string(texel[2]) + ", " + std::to_string(texel[3]) + ")");
+        }
+    }
+}
+
+void ExpectRecorded(const AgcDriver::Graphics::Context& context, const Block& block, const std::string& what) {
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(recorder != nullptr && !recorder->Idle(), what + ": the draw into a linear target in registered, write-watched memory without a host import waited for the GPU instead of being recorded");
+    Require(StorageTexture::FindPending(block.Address(), TargetBytes) != nullptr, what + ": the drawn linear target has no pending resident image");
+    Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) != nullptr, what + ": the draw made no host import of the linear target");
+    Require(block.Watched(), what + ": the linear target's host import took its memory out of the write watch");
+}
+
+void Created(AgcDriver::VulkanDevice& device) {
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    constexpr std::array<float, 4> seed{100.0f, 200.0f, 300.0f, 400.0f};
+    Block block(SettledBytes, 0);
+    Seed(block, seed);
+    Require(block.Watched(), "import created: the color target is not write-watched");
+    Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) == nullptr, "import created: the color target has a host import before the draw");
+    Draw(device, block);
+    ExpectRecorded(context, block, "import created");
+    device.WaitIdle();
+    Require(block.Watched(), "import created: the color target left the write watch after its store");
+    Check(block, seed, "import created, after the device wait");
+}
+
+bool Reused(AgcDriver::VulkanDevice& device) {
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    constexpr std::array<float, 4> first{100.0f, 200.0f, 300.0f, 400.0f};
+    constexpr std::array<float, 4> second{500.0f, 600.0f, 700.0f, 800.0f};
+    std::uint64_t address = 0;
+    {
+        Block block(SettledBytes, 0);
+        address = block.Address();
+        Seed(block, first);
+        Draw(device, block);
+        ExpectRecorded(context, block, "reuse, first block");
+        device.WaitIdle();
+        Check(block, first, "reuse, first block");
+    }
+    Block block(SettledBytes, static_cast<std::uintptr_t>(address));
+    if (block.Address() != address) {
+        std::puts("linear target import reuse skipped: the second block was not mapped at the first one's address");
+        return false;
+    }
+    Require(block.Watched(), "reuse: the block allocated at the freed block's address is not write-watched");
+    Seed(block, second);
+    Require(block.Watched(), "reuse: the CPU's store took the second block out of the write watch");
+    Draw(device, block);
+    ExpectRecorded(context, block, "reuse, second block");
+    device.WaitIdle();
+    Check(block, second, "reuse, the draw into the second block did not start from the CPU's bytes");
+    return true;
+}
+
+#ifdef _WIN32
+AgcDriver::Graphics::HostImportRequesterCounts ColorTargetCounts() {
+    return AgcDriver::Graphics::HostImportCounters().requesters[static_cast<std::size_t>(AgcDriver::Graphics::ImportRequester::ColorTarget)];
+}
+
+void Direct(AgcDriver::VulkanDevice& device) {
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    constexpr std::array<float, 4> seed{1300.0f, 1400.0f, 1500.0f, 1600.0f};
+    Block block(ResidentBytes, 0, true);
+    Seed(block, seed);
+    Require(block.Watched(), "direct memory: the color target is not write-watched");
+    Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) == nullptr, "direct memory: the color target has a host import before the draw");
+    Draw(device, block);
+    ExpectRecorded(context, block, "direct memory");
+    device.WaitIdle();
+    Require(block.Watched(), "direct memory: the color target left the write watch after its store");
+    Check(block, seed, "direct memory, after the device wait");
+}
+
+void PrivateGate(AgcDriver::VulkanDevice& device) {
+    using AgcDriver::Graphics::ImportRequester;
+    using AgcDriver::Graphics::ImportWatch;
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    Block direct(ResidentBytes / 4, 0, true);
+    Block flexible(2 * BlockBytes, 0);
+    GuestArena::GuestArenaCommit_nid_postfix(flexible.data + BlockBytes, BlockBytes, PAGE_READONLY, BlockBytes);
+    const auto decided = AgcDriver::Graphics::PrepareImportWatch(context);
+    const auto before = ColorTargetCounts();
+    AgcDriver::Graphics::SetImportWatch(context, ImportWatch::Unwatch);
+    const auto* compared = AgcDriver::Graphics::HostImportFor(context, direct.Address(), TargetBytes, ImportRequester::ColorTarget);
+    AgcDriver::Graphics::SetImportWatch(context, ImportWatch::Watch);
+    const auto* readOnly = AgcDriver::Graphics::HostImportFor(context, flexible.Address(), TargetBytes, ImportRequester::ColorTarget);
+    const auto refused = ColorTargetCounts();
+    const auto* watched = AgcDriver::Graphics::HostImportFor(context, direct.Address(), TargetBytes, ImportRequester::ColorTarget);
+    const auto made = ColorTargetCounts();
+    AgcDriver::Graphics::SetImportWatch(context, decided);
+    Require(compared == nullptr, "private gate: a color target's import of direct memory under the settle floor was made where it would be compared");
+    Require(readOnly == nullptr && refused.made == before.made && refused.refused == before.refused && refused.failed == before.failed, "private gate: a color target's import was made of private memory committed read-only while registered writable, or a refused color target import was counted");
+    Require(watched != nullptr && made.made == before.made + 1 && made.failed == before.failed, "private gate: a color target's import of direct memory registered writable was not made where imports stay watched");
+}
+#endif
+
+void OutsideWatch(AgcDriver::VulkanDevice& device) {
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    constexpr std::array<float, 4> seed{1700.0f, 1800.0f, 1900.0f, 2000.0f};
+    Block block(SettledBytes, 0);
+    Seed(block, seed);
+    AgcDriver::GuestMemory::Unwatch(block.Address() + SettledBytes - BlockBytes, BlockBytes);
+    Require(AgcDriver::GuestMemory::Watched(block.Address(), TargetBytes) && !block.Watched(), "run outside the write watch: the target is not watched, or the rest of its block still is");
+    Draw(device, block);
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(recorder == nullptr || recorder->Idle(), "run outside the write watch: a linear target whose import run has a block outside the write watch was drawn into a resident image");
+    Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) == nullptr, "run outside the write watch: the draw made a host import over a block outside the write watch");
+    Require(AgcDriver::GuestMemory::Watched(block.Address(), TargetBytes), "run outside the write watch: the draw took the target out of the write watch");
+    device.WaitIdle();
+    Check(block, seed, "run outside the write watch, after the device wait");
+}
+
+void ReadOnlyRegistry(AgcDriver::VulkanDevice& device) {
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    constexpr std::array<float, 4> seed{2500.0f, 2600.0f, 2700.0f, 2800.0f};
+    Block block(SettledBytes, 0, false, 0, false);
+    Seed(block, seed);
+    Require(block.Watched(), "range registered read-only: the color target is not write-watched");
+    Draw(device, block);
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(recorder == nullptr || recorder->Idle(), "range registered read-only: a linear target in memory the guest registered read-only was drawn into a resident image");
+    Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) == nullptr, "range registered read-only: the draw made a host import of memory the guest registered read-only");
+    device.WaitIdle();
+    Check(block, seed, "range registered read-only, after the device wait");
+}
+
+#ifdef _WIN32
+void Rewatched(AgcDriver::VulkanDevice& device) {
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    constexpr std::array<float, 4> seed{2100.0f, 2200.0f, 2300.0f, 2400.0f};
+    Block block(SettledBytes, 0, false, BlockBytes);
+    Seed(block, seed);
+    auto* last = block.data + SettledBytes - BlockBytes;
+    AgcDriver::GuestMemory::Unwatch(block.Address() + SettledBytes - BlockBytes, BlockBytes);
+    Draw(device, block);
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(recorder == nullptr || recorder->Idle(), "run back in the write watch: the draw before the block came back was drawn into a resident image");
+    device.WaitIdle();
+    GuestArena::GuestArenaReset_nid_postfix(last, BlockBytes);
+    GuestArena::GuestArenaCommit_nid_postfix(last, BlockBytes, PAGE_READWRITE, BlockBytes);
+    Require(block.Watched(), "run back in the write watch: committing the block again did not bring it back into the write watch");
+    Draw(device, block);
+    ExpectRecorded(context, block, "run back in the write watch, a refused run whose block came back");
+    device.WaitIdle();
+    Check(block, seed, "run back in the write watch, after the device wait");
+}
+#endif
+
+void Compared(AgcDriver::VulkanDevice& device) {
+    using AgcDriver::Graphics::ImportWatch;
+    const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device);
+    constexpr std::array<float, 4> seed{900.0f, 1000.0f, 1100.0f, 1200.0f};
+    const auto decided = AgcDriver::Graphics::PrepareImportWatch(context);
+    AgcDriver::Graphics::SetImportWatch(context, ImportWatch::Unwatch);
+    Block block(BlockBytes, 0);
+    Seed(block, seed);
+    Require(block.Watched(), "compared import: the color target is not write-watched");
+    Draw(device, block);
+    const auto* recorder = AgcDriver::Graphics::Recorder::Active();
+    Require(recorder == nullptr || recorder->Idle(), "compared import: a linear target whose host import would be compared was drawn into a resident image");
+    Require(AgcDriver::Graphics::ExistingHostImportFor(context, block.Address(), TargetBytes) == nullptr, "compared import: the draw made a host import that leaves the target's memory compared");
+    Require(StorageTexture::FindPending(block.Address(), TargetBytes) == nullptr, "compared import: the drawn linear target has a pending resident image");
+    Require(block.Watched(), "compared import: the draw took the color target out of the write watch");
+    device.WaitIdle();
+    Check(block, seed, "compared import, after the device wait");
+    AgcDriver::Graphics::SetImportWatch(context, decided);
+}
+
+}
+
+int main(int argc, char** argv) {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        if (!AgcDriver::GuestMemory::WriteWatched()) {
+            std::puts("skipped, guest memory is not write-watched");
+            return VulkanTestSkipped;
+        }
+        const std::string_view only = argc > 1 ? argv[1] : "";
+#ifndef _WIN32
+        const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(*device);
+        const auto decided = AgcDriver::Graphics::PrepareImportWatch(context);
+        AgcDriver::Graphics::SetImportWatch(context, AgcDriver::Graphics::ImportWatch::Watch);
+        struct Restore {
+            const AgcDriver::Graphics::Context& context;
+            AgcDriver::Graphics::ImportWatch decided;
+            ~Restore() { AgcDriver::Graphics::SetImportWatch(context, decided); }
+        } restore{context, decided};
+#endif
+        Block probe(SettledBytes, 0);
+        if (AgcDriver::Graphics::HostImportFor(AgcDriver::VulkanDeviceTestAccess::ContextOf(*device), probe.Address(), probe.bytes) == nullptr) {
+            std::puts("skipped, host imports are unavailable");
+            return VulkanTestSkipped;
+        }
+        const bool watched = probe.Watched();
+#ifndef _WIN32
+        Require(watched, "a host import of write-watched memory left the write watch although imports are set to stay watched");
+#endif
+        if (!watched) std::puts("linear target import creation: a host import of write-watched memory does not stay watched on this device; only the compared cases run");
+        if (watched && (only.empty() || only == "--created")) Created(*device);
+        if (watched && (only.empty() || only == "--reused") && Reused(*device)) std::puts("linear target import reuse passed");
+#ifdef _WIN32
+        if (watched && (only.empty() || only == "--direct")) Direct(*device);
+        if (only.empty() || only == "--private-gate") PrivateGate(*device);
+#endif
+        if (watched && (only.empty() || only == "--outside-watch")) OutsideWatch(*device);
+        if (watched && (only.empty() || only == "--read-only-registry")) ReadOnlyRegistry(*device);
+#ifdef _WIN32
+        if (watched && (only.empty() || only == "--rewatched")) Rewatched(*device);
+#endif
+        if (only.empty() || only == "--compared") Compared(*device);
+        std::puts("linear target import creation tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

docs/dev/ENVIRONMENT.md

@@ -309,7 +309,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti
 | `APS5_NO_RECORDER_SYNC_DRAWS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` |  | gives the draws the recorder waits for their own batch instead of recording them with the wait |
 | `APS5_NO_RECORD_COPIED_DRAWS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` |  | waits for copied-write draws at once instead of listing their write-back for later |
 | `APS5_NO_RECT_INDIRECT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp` |  | sends an indirect rectangle-list draw down the CPU path instead of the GPU-side path |
-| `APS5_NO_RESIDENT_LINEAR` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` |  | drops the residency of a linear color target whose address already has a host import, so it is copied in and out again |
+| `APS5_NO_RESIDENT_LINEAR` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` |  | drops the residency of linear color targets, so a target whose address has a host import, or whose import a draw would make, is copied in and out again |
 | `APS5_NO_RESIDENT_PRESENT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` |  | always presents through guest memory instead of the resident image |
 | `APS5_NO_RESIDENT_TARGETS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` |  | copies every render target in and out again instead of keeping it resident |
 | `APS5_NO_RESOURCE_CACHE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` |  | disables the shared resource cache, so every dispatch builds its resources fresh |

docs/dev/debt/libSceAgcDriver.md

@@ -130,6 +130,7 @@
 - [Pending storage-image results](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver), except drawn resident linear targets (next entry), are stored when something reads their memory, not when the GPU work that wrote them completes. A title that waits for that work, frees the image and reuses its memory can have the new data overwritten by the late store (#1970). A title's own CPU reads are not seen either, so a title that reads the results after waiting for a label reads stale memory. Explicit-size file writes (`write`, `pwrite`, `writev`, `pwritev`, AIO and `fwrite`) synchronize their source ranges through the [host-read hook](../../../core/libs/prx/libc/src/GuestArena.cpp); other host readers are not covered. `APS5_STORE_AT_FLIP=1` stores every pending result at each flip and `APS5_STORE_AT_LABEL=1` before each label the GPU writes; both store every pending image, not only the ones the waited-for work wrote, and neither is on by default.
 - [Resident linear color targets](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver): a CPU store inside a drawn scissor rectangle, on a pixel no primitive wrote, made before the target's store is overwritten by it, where the console keeps it. Recorded draws' results are stored before the graphics queue's next flip, label, end-of-pipe interrupt, drain or device wait, as the union of their scissor rectangles; a draw that brings a 17th rectangle that does not merge stores the target at once. A target with an attachment proxy, several mips or layers, a 3D or DCC surface, or results of a dispatch, fill or copy is stored after each draw. One that a fill or copy writes after its draws is stored whole at the next of those points, and GPU results in its CPU-written 64 KiB blocks are dropped. A store the draw pipeline's committer makes (label, end-of-pipe interrupt, device wait) skips a target whose range a queued draw still reads (`DrawPipeline::HeldReads`).
 - [Direct writes over a drawn linear target](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver): a CPU store into a 64 KiB block of a drawn resident linear target that lands while the completion of a direct write recorded before the draws checks and stamps that block (`StampKeepingNewerResults`) drops the draws' results in that block, where the console keeps them: the stamp's own collect records the store per page only, so the store reads the block as written by the driver.
+- [Linear color targets](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) with no host import are drawn synchronously (copied in, drawn, waited for, copied back), where the console renders into memory in stream order, unless the draw can make an import that keeps them write-watched. It makes one for a target in registered, writable memory only when the whole import run is write-watched and the import stays watched: a separate alias of shared memory, a direct import of 1 MiB or more of private memory that settles on Windows (when the import probe keeps the watch), or one the probe keeps watched elsewhere. Smaller private ranges on Windows, runs with memory outside the write watch and devices whose imports are compared stay synchronous: a compared import makes every refresh of the resident image compare the whole target. A target over an existing import is resident, compared or not. `APS5_NO_RESIDENT_LINEAR=1` keeps every linear target synchronous.
 - [`APS5_SKIP_REFUSED_DRAWS`](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp) (libSceAgcDriver) drops a draw whose graphics stage the recompiler refuses instead of failing the queue, so the frame lacks that draw, and the stage is compiled and refused again at each such draw. Compute refusals still fail.
 - [Pending storage-image results over memory outside the write watch](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) leave to the CPU the 64 KiB blocks whose bytes differ from the ones saved at the upload, the fill, the image copy, the previous store or the moment their range left the watch (`Unwatch`, or a host import that does not stay watched or whose page walk fails), and the blocks the watch saw the CPU write before that moment. A store made on the GPU saves its bytes through a host-visible copy that becomes the saved bytes when its batch completes; a store or a refresh that needs them earlier waits for that batch. A unit without a write generation saves the guest bytes when results land in it outside the watch. One whose range leaves the watch while its results are pending saves its bytes through the unwatch hook, which also runs when a failed collect takes a range out of the watch. A range that leaves the watch without the hook running over it leaves its pending results without saved bytes; storing such results throws. A unit whose results were stored while its blocks were watched keeps only a generation; when its blocks leave the watch while nothing is pending, the hook saves nothing, and results that land in it afterwards have neither saved bytes nor a watch, so storing them throws too.
 - [Storage images over uncommitted guest memory](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) - a storage image whose guest range has no committed page when it is created throws in the `StorageTexture` constructor, which aborts the guest. Several titles reach this intermittently in runs of a few minutes, on AMD and NVIDIA alike; in two of the three logs, lines about storage images over released or rewritten memory (`memory no longer registered`, `guest memory changed while GPU results were pending`) come shortly before it. The log does not name the image's address. What the console does with a storage descriptor over memory that is not committed is unknown, and which allocation the guest recreates there was not found.