Merge pull request #11152 from thuanquang/perf/resident-page-bookkeeping

perf(libc): keep no per-page physical state for resident guest pages on Windows

core/libs/prx/libc/include/WindowsMappings.hpp

@@ -93,7 +93,7 @@ public:
                 for (auto mapped = views.lower_bound(cursor & ~(pageBytes - 1)); mapped != views.end() && mapped->first < stop; ++mapped) {
                     mapped->second.protection = protection;
                     mapped->second.armed = 0;
-                    invalidate(*mapped->second.page);
+                    invalidate(mapped->first, mapped->second);
                 }
                 DWORD previous;
                 if (!VirtualProtect(reinterpret_cast<void*>(cursor), stop - cursor, protection, &previous)) fail("protect guest memory");
@@ -119,8 +119,9 @@ public:
         HANDLE duplicate = nullptr;
         if (!DuplicateHandle(GetCurrentProcess(), section, GetCurrentProcess(), &duplicate, 0, FALSE, DUPLICATE_SAME_ACCESS)) fail("keep shared guest section");
         const auto owned = std::make_shared<Section>(duplicate);
+        const auto key = reinterpret_cast<std::uintptr_t>(section);
         const auto slotOf = [&](std::size_t done) -> const std::shared_ptr<Physical>& {
-            auto& slot = physical[std::make_pair(reinterpret_cast<std::uintptr_t>(section), offset + done)];
+            auto& slot = physical[std::make_pair(key, offset + done)];
             if (!slot) slot = std::make_shared<Physical>();
             return slot;
         };
@@ -134,9 +135,11 @@ public:
             return shared;
         };
         const auto single = [&](std::size_t done) {
-            const auto& slot = *slotOf(done);
-            const auto shared = slot.page.lock();
-            return resident && !slot.stored && (!shared || shared->aliases.empty());
+            if (!resident || residentRun(key, offset + done) != residentRuns.end()) return false;
+            const auto found = physical.find(std::make_pair(key, offset + done));
+            if (found == physical.end()) return true;
+            const auto shared = found->second->page.lock();
+            return !found->second->stored && (!shared || shared->aliases.empty());
         };
         const bool perPage = !resident && sectionBytes == SIZE_MAX;
         if (perPage) commitSection(section, offset, bytes);
@@ -151,18 +154,19 @@ public:
                     if (!createdPrivate->empty() && createdPrivate->back().first + createdPrivate->back().second == base) createdPrivate->back().second += runBytes;
                     else createdPrivate->emplace_back(base, runBytes);
                 }
-                residentRuns.emplace(base, Run{base + runBytes, {}, static_cast<std::uint32_t>(runBytes >> 12)});
+                const auto run = residentRuns.emplace(base, Run{base + runBytes, {}, static_cast<std::uint32_t>(runBytes >> 12), key, offset + done}).first;
+                residentIndex.emplace(std::make_pair(key, offset + done), run);
                 for (std::size_t at = 0; at < runBytes; at += pageBytes) {
-                    const auto slot = slotOf(done + at);
-                    const auto shared = pageOf(*slot);
-                    if (slot->saved) {
-                        std::memcpy(reinterpret_cast<void*>(base + at), slot->saved->bytes + slot->savedOffset, pageBytes);
-                        slot->saved.reset();
+                    bool compared = false;
+                    if (const auto found = physical.find(std::make_pair(key, offset + done + at)); found != physical.end()) {
+                        const auto& slot = *found->second;
+                        if (slot.saved) std::memcpy(reinterpret_cast<void*>(base + at), slot.saved->bytes + slot.savedOffset, pageBytes);
+                        compared = slot.comparedWhenShared;
+                        physical.erase(found);
                     }
-                    shared->aliases.push_back(base + at);
-                    views.emplace(base + at, View{shared, protection, {}, 0, owned, offset + done + at, 0, 0, true, slot});
-                    invalidate(*shared);
+                    views.emplace(base + at, View{nullptr, protection, {}, 0, compared, 0, owned, offset + done + at, 0, 0, true});
                 }
+                forgetClean(base, base + runBytes);
                 remember(freshRanges, base, base + runBytes);
                 DWORD previous;
                 if (protection != PAGE_READWRITE && !VirtualProtect(reinterpret_cast<void*>(base), runBytes, protection, &previous)) fail("protect resident guest pages");
@@ -170,19 +174,16 @@ public:
                 continue;
             }
             if (sectionBytes != SIZE_MAX) {
+                if (const auto run = residentRun(key, offset + done); run != residentRuns.end()) demote(run->first);
                 const auto slot = slotOf(done);
                 const auto shared = pageOf(*slot);
-                const auto aliases = shared->aliases;
-                for (const auto alias : aliases) {
-                    if (views.at(alias).resident) demote(alias);
-                    views.at(alias).aliasDemoted = false;
-                }
+                for (const auto alias : shared->aliases) views.at(alias).aliasDemoted = false;
                 const auto viewBytes = std::min(pageBytes, sectionBytes - done);
                 split(base, pageBytes);
                 mapShared(base, section, offset + done, protection, viewProtection, viewBytes);
                 slot->stored = true;
                 shared->aliases.push_back(base);
-                const auto created = views.emplace(base, View{shared, protection, {}, 0, owned, offset + done, 0, 0, false, slot, hostPages(viewBytes)}).first;
+                const auto created = views.emplace(base, View{shared, protection, {}, 0, false, 0, owned, offset + done, 0, 0, false, slot, hostPages(viewBytes)}).first;
                 if (shared->comparedWhenShared) created->second.unreported = allSubpages;
                 remember(freshRanges, base, base + pageBytes);
                 invalidate(*shared);
@@ -193,13 +194,10 @@ public:
             pages.reserve((bytes - done) / pageBytes);
             auto runBytes = std::size_t{0};
             do {
+                if (const auto run = residentRun(key, offset + done + runBytes); run != residentRuns.end()) demote(run->first);
                 const auto slot = slotOf(done + runBytes);
                 const auto shared = pageOf(*slot);
-                const auto aliases = shared->aliases;
-                for (const auto alias : aliases) {
-                    if (views.at(alias).resident) demote(alias);
-                    views.at(alias).aliasDemoted = false;
-                }
+                for (const auto alias : shared->aliases) views.at(alias).aliasDemoted = false;
                 pages.emplace_back(slot, shared);
                 runBytes += pageBytes;
             } while (done + runBytes < bytes && !single(done + runBytes));
@@ -214,7 +212,7 @@ public:
                 slot->stored = true;
                 shared->aliases.push_back(base + at);
                 invalidate(*shared);
-                const auto created = views.emplace(base + at, View{std::move(shared), protection, {}, 0, owned, offset + done + at, 0, 0, false, std::move(slot)}).first;
+                const auto created = views.emplace(base + at, View{std::move(shared), protection, {}, 0, false, 0, owned, offset + done + at, 0, 0, false, std::move(slot)}).first;
                 if (created->second.page->comparedWhenShared) created->second.unreported = allSubpages;
             }
             remember(freshRanges, base, base + runBytes);
@@ -228,13 +226,18 @@ public:
         for (auto it = views.lower_bound(address); it != views.end() && it->first < address + bytes; ++it) {
             it->second.protection = protection;
             it->second.armed = 0;
-            invalidate(*it->second.page);
+            invalidate(it->first, it->second);
         }
     }
 
     void Pin(std::uintptr_t address, std::size_t bytes) {
         std::lock_guard lock(mutex);
         for (auto it = views.lower_bound(address & ~(pageBytes - 1)); it != views.end() && it->first < address + bytes; ++it) {
+            if (it->second.resident) {
+                ++it->second.pins;
+                invalidate(it->first, it->second);
+                continue;
+            }
             auto& page = *it->second.page;
             ++page.pins;
             for (const auto alias : page.aliases) {
@@ -250,9 +253,9 @@ public:
     void Unpin(std::uintptr_t address, std::size_t bytes) {
         std::lock_guard lock(mutex);
         for (auto it = views.lower_bound(address & ~(pageBytes - 1)); it != views.end() && it->first < address + bytes; ++it) {
-            auto& page = *it->second.page;
-            if (page.pins != 0) --page.pins;
-            invalidate(page);
+            auto& pins = it->second.resident ? it->second.pins : it->second.page->pins;
+            if (pins != 0) --pins;
+            invalidate(it->first, it->second);
         }
     }
 
@@ -261,6 +264,10 @@ public:
         collectableRanges.clear();
         for (auto it = views.lower_bound(address & ~(pageBytes - 1)); it != views.end() && it->first < address + bytes; ++it) {
             if (residentOnly && !it->second.resident && !it->second.aliasDemoted) continue;
+            if (it->second.resident) {
+                it->second.comparedWhenShared = true;
+                continue;
+            }
             auto& page = *it->second.page;
             const bool marked = page.comparedWhenShared;
             page.comparedWhenShared = true;
@@ -334,8 +341,9 @@ public:
         const auto key = reinterpret_cast<std::uintptr_t>(section);
         const auto first = physical.lower_bound(std::make_pair(key, std::uint64_t{0}));
         const auto last = physical.lower_bound(std::make_pair(key + 1, std::uint64_t{0}));
-        const bool blank = std::none_of(first, last, [](const auto& entry) { return entry.second->stored || !entry.second->page.expired(); });
+        bool blank = std::none_of(first, last, [](const auto& entry) { return entry.second->stored || !entry.second->page.expired(); });
         physical.erase(first, last);
+        for (auto run = residentIndex.lower_bound(std::make_pair(key, std::uint64_t{0})); run != residentIndex.end() && run->first.first == key; run = residentIndex.erase(run)) blank = false;
         return blank;
     }
 
@@ -351,7 +359,7 @@ public:
             if (store) ++view.hostWrites;
             else ++view.hostImports;
             const auto written = subpagesOf(it->first, address, end);
-            if (store) invalidate(*view.page, written);
+            if (store) invalidate(it->first, view, written);
             else if (view.resident) forgetClean(it->first, it->first + pageBytes);
             const auto opened = static_cast<std::uint8_t>((store || view.resident) ? view.armed & written : 0);
             if (opened == 0) continue;
@@ -367,7 +375,7 @@ public:
         for (auto it = views.lower_bound(address & ~(pageBytes - 1)); it != views.end() && it->first < end; ++it) {
             if (store) --it->second.hostWrites;
             else --it->second.hostImports;
-            if (store) invalidate(*it->second.page, subpagesOf(it->first, address, end));
+            if (store) invalidate(it->first, it->second, subpagesOf(it->first, address, end));
             else if (it->second.resident) forgetClean(it->first, it->first + pageBytes);
         }
     }
@@ -714,6 +722,8 @@ private:
         DWORD protection;
         std::array<std::uint64_t, subpages> seen;
         std::uint8_t armed;
+        bool comparedWhenShared = false;
+        std::uint32_t pins = 0;
         std::shared_ptr<Section> section;
         std::uint64_t offset;
         std::uint32_t hostWrites;
@@ -729,6 +739,8 @@ private:
         std::uintptr_t end;
         std::array<std::uint64_t, 8> collected;
         std::uint32_t fresh;
+        std::uintptr_t key;
+        std::uint64_t offset;
     };
     struct Window {
         void* view = nullptr;
@@ -882,10 +894,39 @@ private:
         return true;
     }
 
+    std::map<std::uintptr_t, Run>::iterator residentRun(std::uintptr_t key, std::uint64_t offset) {
+        const auto next = residentIndex.upper_bound(std::make_pair(key, offset));
+        if (next == residentIndex.begin()) return residentRuns.end();
+        const auto& [first, run] = *std::prev(next);
+        return first.first == key && offset - first.second < run->second.end - run->first ? run : residentRuns.end();
+    }
+
+    std::map<std::pair<std::uintptr_t, std::uint64_t>, std::map<std::uintptr_t, Run>::iterator>::iterator indexEntry(std::map<std::uintptr_t, Run>::iterator run) {
+        const auto found = residentIndex.find(std::make_pair(run->second.key, run->second.offset));
+        return found != residentIndex.end() && found->second == run ? found : residentIndex.end();
+    }
+
     void demote(std::uintptr_t address, bool alias = false) {
         const auto run = std::prev(residentRuns.upper_bound(address));
         const auto start = run->first;
         const auto stop = run->second.end;
+        const auto entry = indexEntry(run);
+        const bool indexed = entry != residentIndex.end();
+        std::vector<std::pair<std::shared_ptr<SharedPage>, std::shared_ptr<Physical>>> created;
+        created.reserve((stop - start) / pageBytes);
+        std::map<std::pair<std::uintptr_t, std::uint64_t>, std::shared_ptr<Physical>> staged;
+        for (auto page = start; page < stop; page += pageBytes) {
+            const auto& view = views.at(page);
+            auto shared = std::make_shared<SharedPage>();
+            shared->aliases.push_back(page);
+            shared->pins = view.pins;
+            shared->comparedWhenShared = view.comparedWhenShared;
+            auto slot = std::make_shared<Physical>();
+            slot->page = shared;
+            slot->comparedWhenShared = view.comparedWhenShared;
+            if (indexed) staged.emplace(std::make_pair(run->second.key, view.offset), slot);
+            created.emplace_back(std::move(shared), std::move(slot));
+        }
         DWORD previous;
         if (!VirtualProtect(reinterpret_cast<void*>(start), stop - start, PAGE_READONLY, &previous)) fail("fence resident guest pages");
         const auto section = views.at(start).section;
@@ -894,10 +935,15 @@ private:
         std::memcpy(window.bytes, reinterpret_cast<const void*>(start), stop - start);
         close(window);
         if (!VirtualFree(reinterpret_cast<void*>(start), stop - start, MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) fail("release resident guest pages");
+        physical.merge(staged);
+        for (auto& [key, slot] : staged) physical.find(key)->second = std::move(slot);
         for (auto page = start; page < stop; page += pageBytes) {
             auto& view = views.at(page);
             split(page, pageBytes);
             mapShared(page, view.section->handle, view.offset, view.protection);
+            auto& [shared, slot] = created[(page - start) / pageBytes];
+            view.page = std::move(shared);
+            view.slot = std::move(slot);
             view.slot->stored = true;
             view.resident = false;
             view.aliasDemoted = alias;
@@ -910,6 +956,7 @@ private:
             }
             invalidate(*view.page);
         }
+        if (indexed) residentIndex.erase(entry);
         residentRuns.erase(run);
     }
 
@@ -925,20 +972,28 @@ private:
         const auto stop = run->second.end;
         DWORD previous;
         if (!VirtualProtect(reinterpret_cast<void*>(start), stop - start, PAGE_READONLY, &previous)) fail("open resident guest pages");
+        const auto entry = indexEntry(run);
+        const bool indexed = entry != residentIndex.end();
         std::shared_ptr<Saved> saved;
         for (auto page = start; page < stop; page += pageBytes) {
             const auto found = views.find(page);
-            auto& view = found->second;
-            if (!blank(page)) {
-                if (!saved) saved = savedMemory(stop - start);
-                saved->Commit(page - start);
-                std::memcpy(saved->bytes + (page - start), reinterpret_cast<const void*>(page), pageBytes);
-                view.slot->saved = saved;
-                view.slot->savedOffset = page - start;
+            const auto& view = found->second;
+            const bool written = indexed && !blank(page);
+            if (written || (indexed && view.comparedWhenShared)) {
+                auto& slot = physical[std::make_pair(run->second.key, view.offset)];
+                if (!slot) slot = std::make_shared<Physical>();
+                slot->comparedWhenShared = view.comparedWhenShared;
+                if (written) {
+                    if (!saved) saved = savedMemory(stop - start);
+                    saved->Commit(page - start);
+                    std::memcpy(saved->bytes + (page - start), reinterpret_cast<const void*>(page), pageBytes);
+                    slot->saved = saved;
+                    slot->savedOffset = page - start;
+                }
             }
-            std::erase(view.page->aliases, page);
             views.erase(found);
         }
+        if (indexed) residentIndex.erase(entry);
         forget(freshRanges, start, stop);
         residentRuns.erase(run);
     }
@@ -1004,6 +1059,16 @@ private:
         }
     }
 
+    void invalidate(std::uintptr_t base, const View& view, std::uint8_t mask = allSubpages) {
+        if (view.page) {
+            invalidate(*view.page, mask);
+            return;
+        }
+        for (std::size_t sub = 0; sub < subpages; ++sub) {
+            if ((mask & (1u << sub)) != 0) forgetClean(base + sub * subpageBytes, base + (sub + 1) * subpageBytes);
+        }
+    }
+
     static std::uint8_t subpagesOf(std::uintptr_t base, std::uintptr_t address, std::uintptr_t end) {
         const auto from = std::max(address, base);
         const auto to = std::min(end, base + pageBytes);
@@ -1120,7 +1185,7 @@ private:
             if (memory.Type == MEM_MAPPED) {
                 if (!unmap(GetCurrentProcess(), reinterpret_cast<void*>(cursor), MEM_PRESERVE_PLACEHOLDER)) fail("unmap shared guest page");
                 for (auto found = views.lower_bound(cursor); found != views.end() && found->first < allocationEnd;) {
-                    std::erase(found->second.page->aliases, found->first);
+                    if (found->second.page != nullptr) std::erase(found->second.page->aliases, found->first);
                     found = views.erase(found);
                 }
                 forget(freshRanges, cursor, allocationEnd);
@@ -1197,6 +1262,7 @@ private:
     }
     std::map<std::pair<std::uintptr_t, std::uint64_t>, std::shared_ptr<Physical>> physical;
     std::map<std::uintptr_t, Run> residentRuns;
+    std::map<std::pair<std::uintptr_t, std::uint64_t>, std::map<std::uintptr_t, Run>::iterator> residentIndex;
     std::vector<std::shared_ptr<Saved>> savedPool;
     std::size_t savedPoolBytes = 0;
     std::map<std::uintptr_t, std::size_t> splitAliases;

core/libs/tests/GuestMemory.cpp

@@ -2259,6 +2259,63 @@ static void CheckRemappedDirectMemoryIsChargedOnce() {
 #endif
 }
 
+static void CheckResidentMappingBookkeepingStaysSmall() {
+#ifdef _WIN32
+    constexpr std::size_t page = 0x4000;
+    constexpr std::size_t chunk = 0x200000;
+    constexpr std::size_t bytes = std::size_t{1} << 30;
+    constexpr std::size_t aliased = 0x10000;
+    constexpr std::int64_t mebibyte = 1 << 20;
+    const auto privateUsage = [] {
+        PROCESS_MEMORY_COUNTERS_EX counters{};
+        counters.cb = sizeof(counters);
+        Require(K32GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PROCESS_MEMORY_COUNTERS*>(&counters), sizeof(counters)) != 0);
+        return static_cast<std::int64_t>(counters.PrivateUsage);
+    };
+    std::int64_t warm = 0;
+    Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page, 0, 0, &warm) == 0);
+    void* warmMapping = nullptr;
+    Require(sceKernelMapDirectMemory(&warmMapping, page, 0x33, 0, warm, 0) == 0);
+    Require(sceKernelMunmap(reinterpret_cast<std::uint64_t>(warmMapping), page) == 0);
+    Require(sceKernelReleaseDirectMemory(warm, page) == 0);
+    const auto before = privateUsage();
+    std::int64_t phys = 0;
+    Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, bytes, chunk, 0, &phys) == 0);
+    const auto allocated = privateUsage();
+    void* mapped = nullptr;
+    Require(sceKernelMapDirectMemory(&mapped, bytes, 0x33, 0, phys, chunk) == 0);
+    const auto overhead = privateUsage() - allocated - static_cast<std::int64_t>(bytes);
+    std::printf("resident direct mapping bookkeeping: %.2f MiB per GiB mapped\n", static_cast<double>(overhead) / static_cast<double>(mebibyte) / static_cast<double>(bytes >> 30));
+    std::fflush(stdout);
+    RequireResidentMapping(mapped);
+    Require(overhead < 16 * mebibyte * static_cast<std::int64_t>(bytes >> 30));
+    auto* memory = static_cast<volatile unsigned char*>(mapped);
+    const auto at = chunk * 3 + page * 5;
+    for (std::size_t index = 0; index < aliased / page; ++index) memory[at + index * page + 5] = static_cast<unsigned char>(0x11 * (index + 1));
+    void* second = nullptr;
+    Require(sceKernelMapDirectMemory(&second, aliased, 3, 0, phys + static_cast<std::int64_t>(at), 0) == 0);
+    auto* through = static_cast<volatile unsigned char*>(second);
+    for (std::size_t index = 0; index < aliased / page; ++index) {
+        Require(through[index * page + 5] == 0x11 * (index + 1));
+        through[index * page + 6] = static_cast<unsigned char>(0x50 + index);
+        Require(memory[at + index * page + 6] == 0x50 + index);
+    }
+    memory[chunk * 5 + 7] = 0x77;
+    auto* alias = static_cast<volatile unsigned char*>(GuestArena::GuestArenaMapAlias_nid_postfix(reinterpret_cast<std::uintptr_t>(mapped) + chunk * 5, chunk));
+    Require(alias != nullptr && alias[7] == 0x77);
+    alias[chunk - 1] = 0x78;
+    Require(memory[chunk * 6 - 1] == 0x78);
+    GuestArena::GuestArenaUnmapAlias_nid_postfix(const_cast<unsigned char*>(alias));
+    Require(sceKernelMunmap(reinterpret_cast<std::uint64_t>(second), aliased) == 0);
+    Require(sceKernelMunmap(reinterpret_cast<std::uint64_t>(mapped), bytes) == 0);
+    Require(sceKernelReleaseDirectMemory(phys, bytes) == 0);
+    const auto left = privateUsage() - before;
+    std::printf("private usage after unmapping and releasing: %+.2f MiB\n", static_cast<double>(left) / static_cast<double>(mebibyte));
+    std::fflush(stdout);
+    Require(left < 4 * mebibyte);
+#endif
+}
+
 static void CheckSharedWriteTracking() {
 #ifdef _WIN32
     constexpr std::size_t page = 0x4000;
@@ -3460,6 +3517,7 @@ int main(int argc, char** argv) {
         return 0;
     }
     Require(argc == 1);
+    CheckResidentMappingBookkeepingStaysSmall();
     CheckAvailableDirectMemoryIsLargestFreeBlock();
     CheckMappedLeasesPreserveReadableRuns();
     CheckVirtualQuerySplitFlexibleRanges();

core/libs/tests/WindowsWriteWatch.cpp

@@ -316,6 +316,142 @@ void CheckPrivateCommitTails() {
     Require(VirtualFree(memory, Pages * Page, MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS) && VirtualFree(memory, 0, MEM_RELEASE), "cannot release the private commit tail placeholder");
 }
 
+std::set<std::size_t> CollectPages(unsigned char* base, Collection mode) {
+    const auto address = reinterpret_cast<std::uintptr_t>(base);
+    auto cursor = address;
+    const auto end = address + Bytes;
+    std::set<std::size_t> result;
+    while (cursor < end) {
+        std::array<void*, 32> pages{};
+        std::size_t available = pages.size();
+        Require(Mappings::Get().Collect(cursor, end - cursor, pages.data(), &available, true, mode), "resident collection failed");
+        for (std::size_t i = 0; i < available; ++i) result.insert((reinterpret_cast<std::uintptr_t>(pages[i]) - address) / Small);
+        if (available < pages.size()) break;
+        cursor = reinterpret_cast<std::uintptr_t>(pages[available - 1]) + Small;
+    }
+    return result;
+}
+
+void CheckPinnedResidentPageShared() {
+    auto& mappings = Mappings::Get();
+    HANDLE section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE | SEC_RESERVE, 0, Bytes, nullptr);
+    auto* first = static_cast<unsigned char*>(mappings.Reserve(nullptr, Bytes));
+    auto* second = static_cast<unsigned char*>(mappings.Reserve(nullptr, Bytes));
+    Require(section && first && second, "cannot allocate pinned resident test memory");
+    mappings.ForgetSection(section);
+    const auto address = reinterpret_cast<std::uintptr_t>(first);
+    mappings.Map(first, Bytes, section, 0, PAGE_READWRITE, true, true);
+    MEMORY_BASIC_INFORMATION info{};
+    Require(VirtualQuery(first, &info, sizeof(info)) == sizeof(info) && info.Type == MEM_PRIVATE, "the pinned resident test memory is not resident");
+    mappings.Pin(address, 16384);
+    first[5] = 9;
+    mappings.Map(second, Bytes, section, 0, PAGE_READWRITE, true, true);
+    Require(second[5] == 9, "the second mapping does not share the pinned resident page");
+    CollectPages(first, Collection::Fresh);
+    Require(VirtualQuery(first, &info, sizeof(info)) == sizeof(info) && info.Protect == PAGE_READWRITE, "a pinned page was protected once its run became shared");
+    Require(CollectPages(first, Collection::Armed) == Pages(0, 4), "a pinned page was trusted as clean once its run became shared");
+    Require(CollectPages(first, Collection::Armed) == Pages(0, 4), "a pinned page stopped being reported once its run became shared");
+    mappings.Unpin(address, 16384);
+    Require(CollectPages(first, Collection::Armed) == Pages(0, 4), "unpin of a shared page was not invalidated");
+    Require(CollectPages(first, Collection::Armed).empty(), "unpin of a shared page did not restore tracking");
+    mappings.Reset(first, Bytes);
+    mappings.Reset(second, Bytes);
+    mappings.ForgetSection(section);
+    VirtualFree(first, 0, MEM_RELEASE);
+    VirtualFree(second, 0, MEM_RELEASE);
+    CloseHandle(section);
+}
+
+void CheckLargeResidentRunCollects() {
+    constexpr std::size_t Large = 0x40000;
+    auto& mappings = Mappings::Get();
+    HANDLE section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE | SEC_RESERVE, 0, Large, nullptr);
+    auto* first = static_cast<unsigned char*>(mappings.Reserve(nullptr, Large));
+    Require(section && first, "cannot allocate large resident run test memory");
+    mappings.ForgetSection(section);
+    mappings.Map(first, Large, section, 0, PAGE_READWRITE, true, true);
+    MEMORY_BASIC_INFORMATION info{};
+    Require(VirtualQuery(first + Large - Small, &info, sizeof(info)) == sizeof(info) && info.Type == MEM_PRIVATE, "the large resident run test memory is not resident");
+    const auto address = reinterpret_cast<std::uintptr_t>(first);
+    const auto collect = [&](Collection mode) {
+        std::set<std::size_t> result;
+        auto cursor = address;
+        while (cursor < address + Large) {
+            std::array<void*, 32> pages{};
+            std::size_t available = pages.size();
+            Require(mappings.Collect(cursor, address + Large - cursor, pages.data(), &available, true, mode), "large resident run collection failed");
+            for (std::size_t i = 0; i < available; ++i) result.insert((reinterpret_cast<std::uintptr_t>(pages[i]) - address) / Small);
+            if (available < pages.size()) break;
+            cursor = reinterpret_cast<std::uintptr_t>(pages[available - 1]) + Small;
+        }
+        return result;
+    };
+    collect(Collection::All);
+    static_cast<volatile unsigned char*>(first)[3 * Small] = 1;
+    static_cast<volatile unsigned char*>(first)[Large - Small] = 2;
+    Require(collect(Collection::Fresh).empty(), "a fresh collect of a large resident run reported written pages");
+    Require(collect(Collection::Armed) == std::set<std::size_t>{3, Large / Small - 1}, "a large resident run did not report its written pages");
+    Require(collect(Collection::Armed).empty(), "a large resident run reported its written pages twice");
+    mappings.Reset(first, Large);
+    mappings.ForgetSection(section);
+    VirtualFree(first, 0, MEM_RELEASE);
+    CloseHandle(section);
+}
+
+void CheckForgottenSectionWithResidentRun() {
+    auto& mappings = Mappings::Get();
+    HANDLE section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE | SEC_RESERVE, 0, Bytes, nullptr);
+    auto* first = static_cast<unsigned char*>(mappings.Reserve(nullptr, Bytes));
+    Require(section && first, "cannot allocate forgotten resident section test memory");
+    mappings.ForgetSection(section);
+    mappings.Map(first, Bytes, section, 0, PAGE_READWRITE, true, true);
+    MEMORY_BASIC_INFORMATION info{};
+    Require(VirtualQuery(first, &info, sizeof(info)) == sizeof(info) && info.Type == MEM_PRIVATE, "the forgotten resident section test memory is not resident");
+    first[0] = 7;
+    Require(!mappings.ForgetSection(section), "a section with a live resident run was reported blank");
+    mappings.Reset(first, Bytes);
+    mappings.ForgetSection(section);
+    VirtualFree(first, 0, MEM_RELEASE);
+    CloseHandle(section);
+}
+
+void CheckReplacedResidentRunKeepsTheNewRun(bool partial) {
+    auto& mappings = Mappings::Get();
+    HANDLE section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE | SEC_RESERVE, 0, Bytes, nullptr);
+    auto* first = static_cast<unsigned char*>(mappings.Reserve(nullptr, Bytes));
+    auto* second = static_cast<unsigned char*>(mappings.Reserve(nullptr, Bytes));
+    auto* third = static_cast<unsigned char*>(mappings.Reserve(nullptr, Bytes));
+    Require(section && first && second && third, "cannot allocate replaced resident run test memory");
+    const auto resident = [](unsigned char* address) {
+        MEMORY_BASIC_INFORMATION info{};
+        return VirtualQuery(address, &info, sizeof(info)) == sizeof(info) && info.Type == MEM_PRIVATE;
+    };
+    mappings.ForgetSection(section);
+    mappings.Map(first, Bytes, section, 0, PAGE_READWRITE, true, true);
+    first[0] = 1;
+    mappings.ForgetSection(section);
+    mappings.Map(second, Bytes, section, 0, PAGE_READWRITE, true, true);
+    Require(resident(first) && resident(second), "the replaced resident run test memory is not resident");
+    second[0] = 2;
+    if (partial) {
+        mappings.Reset(first + 16384, 16384);
+    } else {
+        mappings.Reset(first, Bytes);
+    }
+    mappings.Map(third, Bytes, section, 0, PAGE_READWRITE, true, true);
+    Require(third[0] == 2, "a mapping after the replaced run was released did not see the run that replaced it");
+    third[1] = 3;
+    Require(second[1] == 3, "a mapping after the replaced run was released diverged from the run that replaced it");
+    if (partial) mappings.Reset(first, Bytes);
+    mappings.Reset(second, Bytes);
+    mappings.Reset(third, Bytes);
+    mappings.ForgetSection(section);
+    VirtualFree(first, 0, MEM_RELEASE);
+    VirtualFree(second, 0, MEM_RELEASE);
+    VirtualFree(third, 0, MEM_RELEASE);
+    CloseHandle(section);
+}
+
 void CheckForgottenSectionKeepsNoPages() {
     auto& mappings = Mappings::Get();
     HANDLE section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE | SEC_RESERVE, 0, Bytes, nullptr);
@@ -741,6 +877,11 @@ int main(int argc, char** argv) {
         CheckSharedMapQueriesPerMapping();
         CheckSharedRunIsOneView();
         CheckReleasedPrivateAllocation();
+        CheckPinnedResidentPageShared();
+        CheckForgottenSectionWithResidentRun();
+        CheckLargeResidentRunCollects();
+        CheckReplacedResidentRunKeepsTheNewRun(false);
+        CheckReplacedResidentRunKeepsTheNewRun(true);
         CheckComparedWhenShared();
         CheckResidentOnlyComparedWhenShared();
         CheckComparedWhenSharedRemapped();