Merge pull request #12227 from klosixgaming-arch/perf/evict-skip-unwritten-pages

perf(libc): skip never-written pages when evicting a resident direct-memory run

core/libs/prx/libc/include/WindowsMappings.hpp

@@ -155,6 +155,7 @@ public:
                     else createdPrivate->emplace_back(base, runBytes);
                 }
                 const auto run = residentRuns.emplace(base, Run{base + runBytes, {}, static_cast<std::uint32_t>(runBytes >> 12), key, offset + done}).first;
+                run->second.watched = watchResident;
                 residentIndex.emplace(std::make_pair(key, offset + done), run);
                 for (std::size_t at = 0; at < runBytes; at += pageBytes) {
                     bool compared = false;
@@ -360,7 +361,10 @@ public:
             else ++view.hostImports;
             const auto written = subpagesOf(it->first, address, end);
             if (store) invalidate(it->first, view, written);
-            else if (view.resident) forgetClean(it->first, it->first + pageBytes);
+            else if (view.resident) {
+                forgetClean(it->first, it->first + pageBytes);
+                markImported(it->first);
+            }
             const auto opened = static_cast<std::uint8_t>((store || view.resident) ? view.armed & written : 0);
             if (opened == 0) continue;
             protectSubpages(it->first, opened, view.protection, "open shared memory to a host write", view.viewBytes);
@@ -380,6 +384,22 @@ public:
         }
     }
 
+    static std::vector<bool> TouchedPages(std::uintptr_t start, std::size_t pages, void* const* written, std::size_t count) {
+        std::vector<bool> touched(pages);
+        for (std::size_t index = 0; index < count; ++index) {
+            const auto page = (reinterpret_cast<std::uintptr_t>(written[index]) - start) / pageBytes;
+            if (page < pages) touched[page] = true;
+        }
+        return touched;
+    }
+
+    void markImported(std::uintptr_t page) {
+        const auto next = residentRuns.upper_bound(page);
+        if (next == residentRuns.begin()) return;
+        const auto run = std::prev(next);
+        if (page < run->second.end) run->second.imported = true;
+    }
+
     void* MapAlias(std::uintptr_t address, std::size_t bytes) {
         std::lock_guard lock(mutex);
         const auto refuse = [&](const char* reason) {
@@ -741,6 +761,9 @@ private:
         std::uint32_t fresh;
         std::uintptr_t key;
         std::uint64_t offset;
+        bool watched = false;
+        bool reset = false;
+        bool imported = false;
     };
     struct Window {
         void* view = nullptr;
@@ -753,6 +776,7 @@ private:
     }
 
     void collectResident(std::uintptr_t start, Run& run, std::uintptr_t from, std::uintptr_t stop, void** pages, std::size_t* count, bool clear, Collection collection) {
+        if (clear) run.reset = true;
         const auto first = from & ~std::uintptr_t{4095};
         if (run.fresh == 0 && collection != Collection::Fresh) {
             ULONG_PTR reported = (stop - first + 4095) / 4096;
@@ -974,11 +998,23 @@ private:
         if (!VirtualProtect(reinterpret_cast<void*>(start), stop - start, PAGE_READONLY, &previous)) fail("open resident guest pages");
         const auto entry = indexEntry(run);
         const bool indexed = entry != residentIndex.end();
+        std::vector<void*> dirty;
+        std::vector<bool> touched;
+        bool filtered = false;
+        if (indexed && run->second.watched && !run->second.reset && !run->second.imported) {
+            dirty.resize((stop - start) / subpageBytes);
+            ULONG_PTR reported = dirty.size();
+            DWORD granularity = 0;
+            if (GetWriteWatch(0, reinterpret_cast<void*>(start), stop - start, dirty.data(), &reported, &granularity) == 0) {
+                touched = TouchedPages(start, (stop - start) / pageBytes, dirty.data(), reported);
+                filtered = true;
+            }
+        }
         std::shared_ptr<Saved> saved;
         for (auto page = start; page < stop; page += pageBytes) {
             const auto found = views.find(page);
             const auto& view = found->second;
-            const bool written = indexed && !blank(page);
+            const bool written = indexed && (!filtered || touched[(page - start) / pageBytes]) && !blank(page);
             if (written || (indexed && view.comparedWhenShared)) {
                 auto& slot = physical[std::make_pair(run->second.key, view.offset)];
                 if (!slot) slot = std::make_shared<Physical>();

core/libs/tests/WindowsMappingsRelease.cpp

@@ -1,5 +1,8 @@
 #include "prx/libc/include/WindowsMappings.hpp"
 #include <cstdio>
+#include <initializer_list>
+#include <utility>
+#include <vector>
 #include <stdexcept>
 #include <source_location>
 
@@ -69,10 +72,88 @@ static void CheckValidRelease() {
     CloseHandle(section);
 }
 
+static void CheckTouchedPagesAnyOrder() {
+    constexpr std::size_t page = 0x4000;
+    void* const ascending[] = {reinterpret_cast<void*>(page + 4096), reinterpret_cast<void*>(3 * page), reinterpret_cast<void*>(7 * page + 8192)};
+    void* const scrambled[] = {ascending[2], ascending[0], ascending[1]};
+    for (void* const* written : {ascending, scrambled}) {
+        const auto touched = GuestArena::WindowsMappings::TouchedPages(0, 8, written, 3);
+        for (std::size_t index = 0; index < 8; ++index) Require(touched[index] == (index == 1 || index == 3 || index == 7));
+    }
+}
+
+static void CheckEvictKeepsWrittenPages() {
+    constexpr std::size_t bytes = 2 * 1024 * 1024;
+    constexpr std::size_t step = 65536;
+    auto& mappings = GuestArena::WindowsMappings::Get();
+    auto* reserved = static_cast<unsigned char*>(mappings.Reserve(nullptr, 2 * bytes));
+    Require(reserved != nullptr);
+    const auto start = (reinterpret_cast<std::uintptr_t>(reserved) + bytes - 1) & ~static_cast<std::uintptr_t>(bytes - 1);
+    auto* base = reinterpret_cast<unsigned char*>(start);
+    const auto section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0, bytes, nullptr);
+    Require(section != nullptr);
+    const auto residentRun = [&] {
+        std::uintptr_t first = 0;
+        std::uintptr_t last = 0;
+        return mappings.ResidentSpan(start, 1, &first, &last) && first == start && last == start + bytes;
+    };
+    const auto checkOnly = [&](std::initializer_list<std::pair<std::size_t, unsigned char>> written) {
+        for (std::size_t at = 0; at < bytes; at += step) {
+            unsigned char expected = 0;
+            for (const auto& [offset, value] : written) {
+                if (offset / step * step == at) expected = value;
+            }
+            Require(base[at + 7] == expected);
+        }
+    };
+    const auto cycle = [&] {
+        mappings.Reset(base, bytes);
+        mappings.Map(base, bytes, section, 0, PAGE_READWRITE, true, true);
+        Require(residentRun());
+    };
+
+    mappings.Map(base, bytes, section, 0, PAGE_READWRITE, true, true);
+    Require(residentRun());
+    base[2 * step + 7] = 21;
+    base[bytes - step + 7] = 22;
+    cycle();
+    checkOnly({{2 * step + 7, 21}, {bytes - step + 7, 22}});
+
+    base[5 * step + 7] = 23;
+    std::vector<void*> pages(bytes / 4096);
+    std::size_t count = pages.size();
+    Require(mappings.Collect(start, bytes, pages.data(), &count, true));
+    base[9 * step + 7] = 24;
+    cycle();
+    checkOnly({{2 * step + 7, 21}, {bytes - step + 7, 22}, {5 * step + 7, 23}, {9 * step + 7, 24}});
+
+    count = pages.size();
+    Require(mappings.Collect(start, bytes, pages.data(), &count, true));
+    cycle();
+    checkOnly({{2 * step + 7, 21}, {bytes - step + 7, 22}, {5 * step + 7, 23}, {9 * step + 7, 24}});
+
+    Require(mappings.BeginHostWrite(start, bytes, false));
+    base[12 * step + 7] = 25;
+    Require(ResetWriteWatch(base + 12 * step, step) == 0);
+    mappings.EndHostWrite(start, bytes, false);
+    cycle();
+    checkOnly({{2 * step + 7, 21}, {bytes - step + 7, 22}, {5 * step + 7, 23}, {9 * step + 7, 24}, {12 * step + 7, 25}});
+
+    mappings.Reset(base, bytes);
+    Require(VirtualFree(reserved, 2 * bytes, MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS));
+    Require(VirtualFree(reserved, 0, MEM_RELEASE));
+    mappings.ForgetSection(section);
+    CloseHandle(section);
+}
+
 int main(int argc, char** argv) {
     try {
         if (argc == 2) CheckMetadataMismatch(static_cast<unsigned>(argv[1][0] - '0'));
-        else CheckValidRelease();
+        else {
+            CheckValidRelease();
+            CheckTouchedPagesAnyOrder();
+            CheckEvictKeepsWrittenPages();
+        }
     } catch (const std::exception& failure) {
         std::fprintf(stderr, "%s\n", failure.what());
         return 1;