From 3f130031d818ea4973905e03ae62a0b2b03de515 Mon Sep 17 00:00:00 2001 From: thuanquang Date: Sun, 25 Oct 2026 21:20:02 +0700 Subject: [PATCH] perf(libc): keep no per-page physical state for resident guest pages on Windows A resident direct memory mapping created a SharedPage, a Physical, a physical map node and an aliases buffer for every 16 KiB page, about 18 MiB of heap per GiB mapped, although a page mapped once holds no state there. Resident pages now keep only their View, with pins and the compare-when-shared mark in it; an index from (section, offset) to the resident run finds a live resident mapping of a physical page, and the shared state is created when a run is demoted, or when it is evicted with data or a compare mark. ForgetSection drops the index entries of the section's runs. guest_memory on Windows 11 26200 (Ryzen 5 5600X): private memory per GiB mapped at map time drops from 30.9 MiB to 13.4 MiB. Co-Authored-By: Claude --- .../libs/prx/libc/include/WindowsMappings.hpp | 150 +++++++++++++----- core/libs/tests/GuestMemory.cpp | 58 +++++++ core/libs/tests/WindowsWriteWatch.cpp | 141 ++++++++++++++++ 3 files changed, 307 insertions(+), 42 deletions(-) diff --git a/core/libs/prx/libc/include/WindowsMappings.hpp b/core/libs/prx/libc/include/WindowsMappings.hpp index 4af1c81d98..78e6586eb3 100644 --- a/core/libs/prx/libc/include/WindowsMappings.hpp +++ b/core/libs/prx/libc/include/WindowsMappings.hpp @@ -94,7 +94,7 @@ public: if (mapped != views.end()) { mapped->second.protection = protection; mapped->second.armed = 0; - invalidate(*mapped->second.page); + invalidate(mapped->first, mapped->second); } DWORD previous; if (!VirtualProtect(reinterpret_cast(cursor), stop - cursor, protection, &previous)) fail("protect guest memory"); @@ -120,8 +120,9 @@ public: HANDLE duplicate = nullptr; if (!DuplicateHandle(GetCurrentProcess(), section, GetCurrentProcess(), &duplicate, 0, FALSE, DUPLICATE_SAME_ACCESS)) fail("keep shared guest section"); const auto owned = std::make_shared
(duplicate); + const auto key = reinterpret_cast(section); const auto slotOf = [&](std::size_t done) -> const std::shared_ptr& { - auto& slot = physical[std::make_pair(reinterpret_cast(section), offset + done)]; + auto& slot = physical[std::make_pair(key, offset + done)]; if (!slot) slot = std::make_shared(); return slot; }; @@ -135,9 +136,11 @@ public: return shared; }; const auto single = [&](std::size_t done) { - const auto& slot = *slotOf(done); - const auto shared = slot.page.lock(); - return resident && !slot.stored && (!shared || shared->aliases.empty()); + if (!resident || residentRun(key, offset + done) != residentRuns.end()) return false; + const auto found = physical.find(std::make_pair(key, offset + done)); + if (found == physical.end()) return true; + const auto shared = found->second->page.lock(); + return !found->second->stored && (!shared || shared->aliases.empty()); }; const bool perPage = !resident && sectionBytes == SIZE_MAX; if (perPage) commitSection(section, offset, bytes); @@ -152,18 +155,19 @@ public: if (!createdPrivate->empty() && createdPrivate->back().first + createdPrivate->back().second == base) createdPrivate->back().second += runBytes; else createdPrivate->emplace_back(base, runBytes); } - residentRuns.emplace(base, Run{base + runBytes, {}, static_cast(runBytes >> 12)}); + const auto run = residentRuns.emplace(base, Run{base + runBytes, {}, static_cast(runBytes >> 12), key, offset + done}).first; + residentIndex.emplace(std::make_pair(key, offset + done), run); for (std::size_t at = 0; at < runBytes; at += pageBytes) { - const auto slot = slotOf(done + at); - const auto shared = pageOf(*slot); - if (slot->saved) { - std::memcpy(reinterpret_cast(base + at), slot->saved->bytes + slot->savedOffset, pageBytes); - slot->saved.reset(); + bool compared = false; + if (const auto found = physical.find(std::make_pair(key, offset + done + at)); found != physical.end()) { + const auto& slot = *found->second; + if (slot.saved) std::memcpy(reinterpret_cast(base + at), slot.saved->bytes + slot.savedOffset, pageBytes); + compared = slot.comparedWhenShared; + physical.erase(found); } - shared->aliases.push_back(base + at); - views.emplace(base + at, View{shared, protection, {}, 0, owned, offset + done + at, 0, 0, true, slot}); - invalidate(*shared); + views.emplace(base + at, View{nullptr, protection, {}, 0, compared, 0, owned, offset + done + at, 0, 0, true}); } + forgetClean(base, base + runBytes); remember(freshRanges, base, base + runBytes); DWORD previous; if (protection != PAGE_READWRITE && !VirtualProtect(reinterpret_cast(base), runBytes, protection, &previous)) fail("protect resident guest pages"); @@ -171,19 +175,16 @@ public: continue; } if (sectionBytes != SIZE_MAX) { + if (const auto run = residentRun(key, offset + done); run != residentRuns.end()) demote(run->first); const auto slot = slotOf(done); const auto shared = pageOf(*slot); - const auto aliases = shared->aliases; - for (const auto alias : aliases) { - if (views.at(alias).resident) demote(alias); - views.at(alias).aliasDemoted = false; - } + for (const auto alias : shared->aliases) views.at(alias).aliasDemoted = false; const auto viewBytes = std::min(pageBytes, sectionBytes - done); split(base, pageBytes); mapShared(base, section, offset + done, protection, viewProtection, viewBytes); slot->stored = true; shared->aliases.push_back(base); - const auto created = views.emplace(base, View{shared, protection, {}, 0, owned, offset + done, 0, 0, false, slot, hostPages(viewBytes)}).first; + const auto created = views.emplace(base, View{shared, protection, {}, 0, false, 0, owned, offset + done, 0, 0, false, slot, hostPages(viewBytes)}).first; if (shared->comparedWhenShared) created->second.unreported = allSubpages; remember(freshRanges, base, base + pageBytes); invalidate(*shared); @@ -194,13 +195,10 @@ public: pages.reserve((bytes - done) / pageBytes); auto runBytes = std::size_t{0}; do { + if (const auto run = residentRun(key, offset + done + runBytes); run != residentRuns.end()) demote(run->first); const auto slot = slotOf(done + runBytes); const auto shared = pageOf(*slot); - const auto aliases = shared->aliases; - for (const auto alias : aliases) { - if (views.at(alias).resident) demote(alias); - views.at(alias).aliasDemoted = false; - } + for (const auto alias : shared->aliases) views.at(alias).aliasDemoted = false; pages.emplace_back(slot, shared); runBytes += pageBytes; } while (done + runBytes < bytes && !single(done + runBytes)); @@ -215,7 +213,7 @@ public: slot->stored = true; shared->aliases.push_back(base + at); invalidate(*shared); - const auto created = views.emplace(base + at, View{std::move(shared), protection, {}, 0, owned, offset + done + at, 0, 0, false, std::move(slot)}).first; + const auto created = views.emplace(base + at, View{std::move(shared), protection, {}, 0, false, 0, owned, offset + done + at, 0, 0, false, std::move(slot)}).first; if (created->second.page->comparedWhenShared) created->second.unreported = allSubpages; } remember(freshRanges, base, base + runBytes); @@ -229,13 +227,18 @@ public: for (auto it = views.lower_bound(address); it != views.end() && it->first < address + bytes; ++it) { it->second.protection = protection; it->second.armed = 0; - invalidate(*it->second.page); + invalidate(it->first, it->second); } } void Pin(std::uintptr_t address, std::size_t bytes) { std::lock_guard lock(mutex); for (auto it = views.lower_bound(address & ~(pageBytes - 1)); it != views.end() && it->first < address + bytes; ++it) { + if (it->second.resident) { + ++it->second.pins; + invalidate(it->first, it->second); + continue; + } auto& page = *it->second.page; ++page.pins; for (const auto alias : page.aliases) { @@ -251,9 +254,9 @@ public: void Unpin(std::uintptr_t address, std::size_t bytes) { std::lock_guard lock(mutex); for (auto it = views.lower_bound(address & ~(pageBytes - 1)); it != views.end() && it->first < address + bytes; ++it) { - auto& page = *it->second.page; - if (page.pins != 0) --page.pins; - invalidate(page); + auto& pins = it->second.resident ? it->second.pins : it->second.page->pins; + if (pins != 0) --pins; + invalidate(it->first, it->second); } } @@ -262,6 +265,10 @@ public: collectableRanges.clear(); for (auto it = views.lower_bound(address & ~(pageBytes - 1)); it != views.end() && it->first < address + bytes; ++it) { if (residentOnly && !it->second.resident && !it->second.aliasDemoted) continue; + if (it->second.resident) { + it->second.comparedWhenShared = true; + continue; + } auto& page = *it->second.page; const bool marked = page.comparedWhenShared; page.comparedWhenShared = true; @@ -335,8 +342,9 @@ public: const auto key = reinterpret_cast(section); const auto first = physical.lower_bound(std::make_pair(key, std::uint64_t{0})); const auto last = physical.lower_bound(std::make_pair(key + 1, std::uint64_t{0})); - const bool blank = std::none_of(first, last, [](const auto& entry) { return entry.second->stored || !entry.second->page.expired(); }); + bool blank = std::none_of(first, last, [](const auto& entry) { return entry.second->stored || !entry.second->page.expired(); }); physical.erase(first, last); + for (auto run = residentIndex.lower_bound(std::make_pair(key, std::uint64_t{0})); run != residentIndex.end() && run->first.first == key; run = residentIndex.erase(run)) blank = false; return blank; } @@ -352,7 +360,7 @@ public: if (store) ++view.hostWrites; else ++view.hostImports; const auto written = subpagesOf(it->first, address, end); - if (store) invalidate(*view.page, written); + if (store) invalidate(it->first, view, written); else if (view.resident) forgetClean(it->first, it->first + pageBytes); const auto opened = static_cast((store || view.resident) ? view.armed & written : 0); if (opened == 0) continue; @@ -368,7 +376,7 @@ public: for (auto it = views.lower_bound(address & ~(pageBytes - 1)); it != views.end() && it->first < end; ++it) { if (store) --it->second.hostWrites; else --it->second.hostImports; - if (store) invalidate(*it->second.page, subpagesOf(it->first, address, end)); + if (store) invalidate(it->first, it->second, subpagesOf(it->first, address, end)); else if (it->second.resident) forgetClean(it->first, it->first + pageBytes); } } @@ -715,6 +723,8 @@ private: DWORD protection; std::array seen; std::uint8_t armed; + bool comparedWhenShared = false; + std::uint32_t pins = 0; std::shared_ptr
section; std::uint64_t offset; std::uint32_t hostWrites; @@ -730,6 +740,8 @@ private: std::uintptr_t end; std::array collected; std::uint32_t fresh; + std::uintptr_t key; + std::uint64_t offset; }; struct Window { void* view = nullptr; @@ -883,10 +895,39 @@ private: return true; } + std::map::iterator residentRun(std::uintptr_t key, std::uint64_t offset) { + const auto next = residentIndex.upper_bound(std::make_pair(key, offset)); + if (next == residentIndex.begin()) return residentRuns.end(); + const auto& [first, run] = *std::prev(next); + return first.first == key && offset - first.second < run->second.end - run->first ? run : residentRuns.end(); + } + + std::map, std::map::iterator>::iterator indexEntry(std::map::iterator run) { + const auto found = residentIndex.find(std::make_pair(run->second.key, run->second.offset)); + return found != residentIndex.end() && found->second == run ? found : residentIndex.end(); + } + void demote(std::uintptr_t address, bool alias = false) { const auto run = std::prev(residentRuns.upper_bound(address)); const auto start = run->first; const auto stop = run->second.end; + const auto entry = indexEntry(run); + const bool indexed = entry != residentIndex.end(); + std::vector, std::shared_ptr>> created; + created.reserve((stop - start) / pageBytes); + std::map, std::shared_ptr> staged; + for (auto page = start; page < stop; page += pageBytes) { + const auto& view = views.at(page); + auto shared = std::make_shared(); + shared->aliases.push_back(page); + shared->pins = view.pins; + shared->comparedWhenShared = view.comparedWhenShared; + auto slot = std::make_shared(); + slot->page = shared; + slot->comparedWhenShared = view.comparedWhenShared; + if (indexed) staged.emplace(std::make_pair(run->second.key, view.offset), slot); + created.emplace_back(std::move(shared), std::move(slot)); + } DWORD previous; if (!VirtualProtect(reinterpret_cast(start), stop - start, PAGE_READONLY, &previous)) fail("fence resident guest pages"); const auto section = views.at(start).section; @@ -895,10 +936,15 @@ private: std::memcpy(window.bytes, reinterpret_cast(start), stop - start); close(window); if (!VirtualFree(reinterpret_cast(start), stop - start, MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) fail("release resident guest pages"); + physical.merge(staged); + for (auto& [key, slot] : staged) physical.find(key)->second = std::move(slot); for (auto page = start; page < stop; page += pageBytes) { auto& view = views.at(page); split(page, pageBytes); mapShared(page, view.section->handle, view.offset, view.protection); + auto& [shared, slot] = created[(page - start) / pageBytes]; + view.page = std::move(shared); + view.slot = std::move(slot); view.slot->stored = true; view.resident = false; view.aliasDemoted = alias; @@ -911,6 +957,7 @@ private: } invalidate(*view.page); } + if (indexed) residentIndex.erase(entry); residentRuns.erase(run); } @@ -926,20 +973,28 @@ private: const auto stop = run->second.end; DWORD previous; if (!VirtualProtect(reinterpret_cast(start), stop - start, PAGE_READONLY, &previous)) fail("open resident guest pages"); + const auto entry = indexEntry(run); + const bool indexed = entry != residentIndex.end(); std::shared_ptr saved; for (auto page = start; page < stop; page += pageBytes) { const auto found = views.find(page); - auto& view = found->second; - if (!blank(page)) { - if (!saved) saved = savedMemory(stop - start); - saved->Commit(page - start); - std::memcpy(saved->bytes + (page - start), reinterpret_cast(page), pageBytes); - view.slot->saved = saved; - view.slot->savedOffset = page - start; + const auto& view = found->second; + const bool written = indexed && !blank(page); + if (written || (indexed && view.comparedWhenShared)) { + auto& slot = physical[std::make_pair(run->second.key, view.offset)]; + if (!slot) slot = std::make_shared(); + slot->comparedWhenShared = view.comparedWhenShared; + if (written) { + if (!saved) saved = savedMemory(stop - start); + saved->Commit(page - start); + std::memcpy(saved->bytes + (page - start), reinterpret_cast(page), pageBytes); + slot->saved = saved; + slot->savedOffset = page - start; + } } - std::erase(view.page->aliases, page); views.erase(found); } + if (indexed) residentIndex.erase(entry); forget(freshRanges, start, stop); residentRuns.erase(run); } @@ -1005,6 +1060,16 @@ private: } } + void invalidate(std::uintptr_t base, const View& view, std::uint8_t mask = allSubpages) { + if (view.page) { + invalidate(*view.page, mask); + return; + } + for (std::size_t sub = 0; sub < subpages; ++sub) { + if ((mask & (1u << sub)) != 0) forgetClean(base + sub * subpageBytes, base + (sub + 1) * subpageBytes); + } + } + static std::uint8_t subpagesOf(std::uintptr_t base, std::uintptr_t address, std::uintptr_t end) { const auto from = std::max(address, base); const auto to = std::min(end, base + pageBytes); @@ -1121,7 +1186,7 @@ private: if (memory.Type == MEM_MAPPED) { if (!unmap(GetCurrentProcess(), reinterpret_cast(cursor), MEM_PRESERVE_PLACEHOLDER)) fail("unmap shared guest page"); for (auto found = views.lower_bound(cursor); found != views.end() && found->first < allocationEnd;) { - std::erase(found->second.page->aliases, found->first); + if (found->second.page != nullptr) std::erase(found->second.page->aliases, found->first); found = views.erase(found); } forget(freshRanges, cursor, allocationEnd); @@ -1198,6 +1263,7 @@ private: } std::map, std::shared_ptr> physical; std::map residentRuns; + std::map, std::map::iterator> residentIndex; std::vector> savedPool; std::size_t savedPoolBytes = 0; std::map splitAliases; diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp index df4e2041c9..66b8a2c536 100644 --- a/core/libs/tests/GuestMemory.cpp +++ b/core/libs/tests/GuestMemory.cpp @@ -2170,6 +2170,63 @@ static void CheckRemappedDirectMemoryIsChargedOnce() { #endif } +static void CheckResidentMappingBookkeepingStaysSmall() { +#ifdef _WIN32 + constexpr std::size_t page = 0x4000; + constexpr std::size_t chunk = 0x200000; + constexpr std::size_t bytes = std::size_t{1} << 30; + constexpr std::size_t aliased = 0x10000; + constexpr std::int64_t mebibyte = 1 << 20; + const auto privateUsage = [] { + PROCESS_MEMORY_COUNTERS_EX counters{}; + counters.cb = sizeof(counters); + Require(K32GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast(&counters), sizeof(counters)) != 0); + return static_cast(counters.PrivateUsage); + }; + std::int64_t warm = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page, 0, 0, &warm) == 0); + void* warmMapping = nullptr; + Require(sceKernelMapDirectMemory(&warmMapping, page, 0x33, 0, warm, 0) == 0); + Require(sceKernelMunmap(reinterpret_cast(warmMapping), page) == 0); + Require(sceKernelReleaseDirectMemory(warm, page) == 0); + const auto before = privateUsage(); + std::int64_t phys = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, bytes, chunk, 0, &phys) == 0); + const auto allocated = privateUsage(); + void* mapped = nullptr; + Require(sceKernelMapDirectMemory(&mapped, bytes, 0x33, 0, phys, chunk) == 0); + const auto overhead = privateUsage() - allocated - static_cast(bytes); + std::printf("resident direct mapping bookkeeping: %.2f MiB per GiB mapped\n", static_cast(overhead) / static_cast(mebibyte) / static_cast(bytes >> 30)); + std::fflush(stdout); + RequireResidentMapping(mapped); + Require(overhead < 16 * mebibyte * static_cast(bytes >> 30)); + auto* memory = static_cast(mapped); + const auto at = chunk * 3 + page * 5; + for (std::size_t index = 0; index < aliased / page; ++index) memory[at + index * page + 5] = static_cast(0x11 * (index + 1)); + void* second = nullptr; + Require(sceKernelMapDirectMemory(&second, aliased, 3, 0, phys + static_cast(at), 0) == 0); + auto* through = static_cast(second); + for (std::size_t index = 0; index < aliased / page; ++index) { + Require(through[index * page + 5] == 0x11 * (index + 1)); + through[index * page + 6] = static_cast(0x50 + index); + Require(memory[at + index * page + 6] == 0x50 + index); + } + memory[chunk * 5 + 7] = 0x77; + auto* alias = static_cast(GuestArena::GuestArenaMapAlias_nid_postfix(reinterpret_cast(mapped) + chunk * 5, chunk)); + Require(alias != nullptr && alias[7] == 0x77); + alias[chunk - 1] = 0x78; + Require(memory[chunk * 6 - 1] == 0x78); + GuestArena::GuestArenaUnmapAlias_nid_postfix(const_cast(alias)); + Require(sceKernelMunmap(reinterpret_cast(second), aliased) == 0); + Require(sceKernelMunmap(reinterpret_cast(mapped), bytes) == 0); + Require(sceKernelReleaseDirectMemory(phys, bytes) == 0); + const auto left = privateUsage() - before; + std::printf("private usage after unmapping and releasing: %+.2f MiB\n", static_cast(left) / static_cast(mebibyte)); + std::fflush(stdout); + Require(left < 4 * mebibyte); +#endif +} + static void CheckSharedWriteTracking() { #ifdef _WIN32 constexpr std::size_t page = 0x4000; @@ -3316,6 +3373,7 @@ int main(int argc, char** argv) { return 0; } Require(argc == 1); + CheckResidentMappingBookkeepingStaysSmall(); CheckAvailableDirectMemoryIsLargestFreeBlock(); CheckVirtualQuerySplitFlexibleRanges(); CheckVirtualQueryForNonReadableGuestRange(); diff --git a/core/libs/tests/WindowsWriteWatch.cpp b/core/libs/tests/WindowsWriteWatch.cpp index b3eecc2a92..f5c88e2acb 100644 --- a/core/libs/tests/WindowsWriteWatch.cpp +++ b/core/libs/tests/WindowsWriteWatch.cpp @@ -316,6 +316,142 @@ void CheckPrivateCommitTails() { Require(VirtualFree(memory, Pages * Page, MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS) && VirtualFree(memory, 0, MEM_RELEASE), "cannot release the private commit tail placeholder"); } +std::set CollectPages(unsigned char* base, Collection mode) { + const auto address = reinterpret_cast(base); + auto cursor = address; + const auto end = address + Bytes; + std::set result; + while (cursor < end) { + std::array pages{}; + std::size_t available = pages.size(); + Require(Mappings::Get().Collect(cursor, end - cursor, pages.data(), &available, true, mode), "resident collection failed"); + for (std::size_t i = 0; i < available; ++i) result.insert((reinterpret_cast(pages[i]) - address) / Small); + if (available < pages.size()) break; + cursor = reinterpret_cast(pages[available - 1]) + Small; + } + return result; +} + +void CheckPinnedResidentPageShared() { + auto& mappings = Mappings::Get(); + HANDLE section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE | SEC_RESERVE, 0, Bytes, nullptr); + auto* first = static_cast(mappings.Reserve(nullptr, Bytes)); + auto* second = static_cast(mappings.Reserve(nullptr, Bytes)); + Require(section && first && second, "cannot allocate pinned resident test memory"); + mappings.ForgetSection(section); + const auto address = reinterpret_cast(first); + mappings.Map(first, Bytes, section, 0, PAGE_READWRITE, true, true); + MEMORY_BASIC_INFORMATION info{}; + Require(VirtualQuery(first, &info, sizeof(info)) == sizeof(info) && info.Type == MEM_PRIVATE, "the pinned resident test memory is not resident"); + mappings.Pin(address, 16384); + first[5] = 9; + mappings.Map(second, Bytes, section, 0, PAGE_READWRITE, true, true); + Require(second[5] == 9, "the second mapping does not share the pinned resident page"); + CollectPages(first, Collection::Fresh); + Require(VirtualQuery(first, &info, sizeof(info)) == sizeof(info) && info.Protect == PAGE_READWRITE, "a pinned page was protected once its run became shared"); + Require(CollectPages(first, Collection::Armed) == Pages(0, 4), "a pinned page was trusted as clean once its run became shared"); + Require(CollectPages(first, Collection::Armed) == Pages(0, 4), "a pinned page stopped being reported once its run became shared"); + mappings.Unpin(address, 16384); + Require(CollectPages(first, Collection::Armed) == Pages(0, 4), "unpin of a shared page was not invalidated"); + Require(CollectPages(first, Collection::Armed).empty(), "unpin of a shared page did not restore tracking"); + mappings.Reset(first, Bytes); + mappings.Reset(second, Bytes); + mappings.ForgetSection(section); + VirtualFree(first, 0, MEM_RELEASE); + VirtualFree(second, 0, MEM_RELEASE); + CloseHandle(section); +} + +void CheckLargeResidentRunCollects() { + constexpr std::size_t Large = 0x40000; + auto& mappings = Mappings::Get(); + HANDLE section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE | SEC_RESERVE, 0, Large, nullptr); + auto* first = static_cast(mappings.Reserve(nullptr, Large)); + Require(section && first, "cannot allocate large resident run test memory"); + mappings.ForgetSection(section); + mappings.Map(first, Large, section, 0, PAGE_READWRITE, true, true); + MEMORY_BASIC_INFORMATION info{}; + Require(VirtualQuery(first + Large - Small, &info, sizeof(info)) == sizeof(info) && info.Type == MEM_PRIVATE, "the large resident run test memory is not resident"); + const auto address = reinterpret_cast(first); + const auto collect = [&](Collection mode) { + std::set result; + auto cursor = address; + while (cursor < address + Large) { + std::array pages{}; + std::size_t available = pages.size(); + Require(mappings.Collect(cursor, address + Large - cursor, pages.data(), &available, true, mode), "large resident run collection failed"); + for (std::size_t i = 0; i < available; ++i) result.insert((reinterpret_cast(pages[i]) - address) / Small); + if (available < pages.size()) break; + cursor = reinterpret_cast(pages[available - 1]) + Small; + } + return result; + }; + collect(Collection::All); + static_cast(first)[3 * Small] = 1; + static_cast(first)[Large - Small] = 2; + Require(collect(Collection::Fresh).empty(), "a fresh collect of a large resident run reported written pages"); + Require(collect(Collection::Armed) == std::set{3, Large / Small - 1}, "a large resident run did not report its written pages"); + Require(collect(Collection::Armed).empty(), "a large resident run reported its written pages twice"); + mappings.Reset(first, Large); + mappings.ForgetSection(section); + VirtualFree(first, 0, MEM_RELEASE); + CloseHandle(section); +} + +void CheckForgottenSectionWithResidentRun() { + auto& mappings = Mappings::Get(); + HANDLE section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE | SEC_RESERVE, 0, Bytes, nullptr); + auto* first = static_cast(mappings.Reserve(nullptr, Bytes)); + Require(section && first, "cannot allocate forgotten resident section test memory"); + mappings.ForgetSection(section); + mappings.Map(first, Bytes, section, 0, PAGE_READWRITE, true, true); + MEMORY_BASIC_INFORMATION info{}; + Require(VirtualQuery(first, &info, sizeof(info)) == sizeof(info) && info.Type == MEM_PRIVATE, "the forgotten resident section test memory is not resident"); + first[0] = 7; + Require(!mappings.ForgetSection(section), "a section with a live resident run was reported blank"); + mappings.Reset(first, Bytes); + mappings.ForgetSection(section); + VirtualFree(first, 0, MEM_RELEASE); + CloseHandle(section); +} + +void CheckReplacedResidentRunKeepsTheNewRun(bool partial) { + auto& mappings = Mappings::Get(); + HANDLE section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE | SEC_RESERVE, 0, Bytes, nullptr); + auto* first = static_cast(mappings.Reserve(nullptr, Bytes)); + auto* second = static_cast(mappings.Reserve(nullptr, Bytes)); + auto* third = static_cast(mappings.Reserve(nullptr, Bytes)); + Require(section && first && second && third, "cannot allocate replaced resident run test memory"); + const auto resident = [](unsigned char* address) { + MEMORY_BASIC_INFORMATION info{}; + return VirtualQuery(address, &info, sizeof(info)) == sizeof(info) && info.Type == MEM_PRIVATE; + }; + mappings.ForgetSection(section); + mappings.Map(first, Bytes, section, 0, PAGE_READWRITE, true, true); + first[0] = 1; + mappings.ForgetSection(section); + mappings.Map(second, Bytes, section, 0, PAGE_READWRITE, true, true); + Require(resident(first) && resident(second), "the replaced resident run test memory is not resident"); + second[0] = 2; + if (partial) { + mappings.Reset(first + 16384, 16384); + } else { + mappings.Reset(first, Bytes); + } + mappings.Map(third, Bytes, section, 0, PAGE_READWRITE, true, true); + Require(third[0] == 2, "a mapping after the replaced run was released did not see the run that replaced it"); + third[1] = 3; + Require(second[1] == 3, "a mapping after the replaced run was released diverged from the run that replaced it"); + if (partial) mappings.Reset(first, Bytes); + mappings.Reset(second, Bytes); + mappings.Reset(third, Bytes); + mappings.ForgetSection(section); + VirtualFree(first, 0, MEM_RELEASE); + VirtualFree(second, 0, MEM_RELEASE); + VirtualFree(third, 0, MEM_RELEASE); + CloseHandle(section); +} + void CheckForgottenSectionKeepsNoPages() { auto& mappings = Mappings::Get(); HANDLE section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE | SEC_RESERVE, 0, Bytes, nullptr); @@ -699,6 +835,11 @@ int main(int argc, char** argv) { CheckSharedMapQueriesPerMapping(); CheckSharedRunIsOneView(); CheckReleasedPrivateAllocation(); + CheckPinnedResidentPageShared(); + CheckForgottenSectionWithResidentRun(); + CheckLargeResidentRunCollects(); + CheckReplacedResidentRunKeepsTheNewRun(false); + CheckReplacedResidentRunKeepsTheNewRun(true); CheckComparedWhenShared(); CheckResidentOnlyComparedWhenShared(); CheckComparedWhenSharedRemapped();