From d3d5e2f262f0302d077fe183f333bc4fe60ffd0f Mon Sep 17 00:00:00 2001 From: thuanquang Date: Sat, 24 Oct 2026 10:33:30 +0700 Subject: [PATCH] perf(agc): hash the pages of an import window on several threads ImportWatched hashes every watched page of an import run before the import, and checks every page the import reports written against those hashes after it. On Windows the first touch of untouched write-watched memory faults each page in, so the first pass over a fresh 64 MiB run costs tens of milliseconds on the thread that draws. Both passes now split their pages over up to four threads, at least 1024 pages per thread. The checks after the import still run after the walk has reset the write watch, so a CPU store during the window is caught as before. Co-Authored-By: Claude --- .../Execution/src/GuestMemory.cpp | 91 +++++++++++++++---- .../libSceAgcDriver/tests/WriteTracking.cpp | 37 ++++++++ 2 files changed, 111 insertions(+), 17 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index 8648a7fa14..398ea6fe18 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -24,7 +24,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -853,6 +856,8 @@ namespace { constexpr std::size_t WriteBlockBytes = 65536; constexpr std::size_t WritePageBytes = 4096; constexpr std::size_t WritePagesPerBlock = WriteBlockBytes / WritePageBytes; +constexpr std::size_t ImportHashPagesPerThread = 1024; +constexpr unsigned ImportHashThreads = 4; enum class StampKind : std::uint8_t { Cpu, Driver, ImportWindow, Fresh }; @@ -877,6 +882,43 @@ std::uint64_t HashPage(const std::byte* page) { return hash ^ (hash >> 32); } +template +void SplitPages(std::size_t pages, const Range& range) { + static const unsigned hostThreads = std::max(1u, std::thread::hardware_concurrency() / 2); + const auto workers = std::min({ImportHashThreads, hostThreads, pages / ImportHashPagesPerThread}); + if (workers < 2) { + range(std::size_t{0}, pages); + return; + } + std::vector threads; + threads.reserve(workers - 1); + for (std::size_t worker = 1; worker < workers; ++worker) { + const auto from = pages * worker / workers; + const auto to = pages * (worker + 1) / workers; + try { + threads.emplace_back(range, from, to); + } catch (const std::system_error&) { + range(from, to); + } + } + range(std::size_t{0}, pages / workers); +} + +std::vector PagesOf(std::span> spans) { + std::vector pages; + for (const auto& [begin, end] : spans) { + for (auto page = begin; page < end; page += WritePageBytes) pages.push_back(page); + } + return pages; +} + +void HashPages(std::uint64_t first, std::span> spans, std::vector& hashes) { + const auto pages = PagesOf(spans); + SplitPages(pages.size(), [&](std::size_t from, std::size_t to) { + for (auto index = from; index < to; ++index) hashes[static_cast((pages[index] - first) / WritePageBytes)] = HashPage(reinterpret_cast(pages[index])); + }); +} + struct WindowCopy { std::uint64_t first; std::span hashes; @@ -1400,27 +1442,45 @@ struct StampRuns { StampKind kind; std::size_t stamped = 0; const WindowCopy* window = nullptr; + std::vector> windowRuns; }; +void noteWrittenRun(StampRuns& runs, std::uintptr_t begin, std::uintptr_t end, std::uint32_t generation) { + if (runs.kind != StampKind::Driver) runs.tracker.noteCpuStore(begin, end, generation); + else runs.tracker.noteWalkStore(begin, end, generation); +} + void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) { auto& runs = *static_cast(context); auto& tracker = runs.tracker; - const auto kind = runs.kind; if (end <= begin) return; ++runs.stamped; - const auto generation = tracker.generation.load(std::memory_order_relaxed); - if (kind != StampKind::Driver) tracker.noteCpuStore(begin, end, generation); - else tracker.noteWalkStore(begin, end, generation); - if (runs.window != nullptr) { - for (auto page = begin; page < end; page += WritePageBytes) tracker.stamp(tracker.blockOf(page), generation, runs.window->Holds(page) ? StampKind::ImportWindow : kind); - } else { - for (auto block = tracker.blockOf(begin); block <= tracker.blockOf(end - 1); ++block) tracker.stamp(block, generation, kind); + if (runs.window != nullptr) runs.windowRuns.emplace_back(begin, end); + else { + const auto generation = tracker.generation.load(std::memory_order_relaxed); + noteWrittenRun(runs, begin, end, generation); + for (auto block = tracker.blockOf(begin); block <= tracker.blockOf(end - 1); ++block) tracker.stamp(block, generation, runs.kind); } #ifndef _WIN32 collectDirtyRuns.fetch_add(1, std::memory_order_relaxed); #endif } +void stampWindowRuns(StampRuns& runs) { + const auto pages = PagesOf(runs.windowRuns); + std::vector held(pages.size(), 0); + SplitPages(pages.size(), [&](std::size_t from, std::size_t to) { + for (auto index = from; index < to; ++index) held[index] = runs.window->Holds(static_cast(pages[index])); + }); + auto& tracker = runs.tracker; + const auto generation = tracker.generation.load(std::memory_order_relaxed); + std::size_t index = 0; + for (const auto& [begin, end] : runs.windowRuns) { + noteWrittenRun(runs, static_cast(begin), static_cast(end), generation); + for (auto page = begin; page < end; page += WritePageBytes) tracker.stamp(tracker.blockOf(page), generation, held[index++] != 0 ? StampKind::ImportWindow : runs.kind); + } +} + #ifdef _WIN32 std::atomic freshCollectHook{nullptr}; @@ -1454,16 +1514,17 @@ bool collectWindowsWrites(StampRuns& runs, std::uint64_t first, std::uint64_t st bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, StampKind kind, bool armedOnly = false, const WindowCopy* window = nullptr) { ++tracker.generation; - StampRuns runs{tracker, kind, 0, window}; + StampRuns runs{tracker, kind, 0, window, {}}; #ifdef _WIN32 - return collectWindowsWrites(runs, first, stop, armedOnly); + const bool complete = collectWindowsWrites(runs, first, stop, armedOnly); #else const auto dirtyRuns = collectDirtyRuns.load(std::memory_order_relaxed); const auto collect = armedOnly ? &GuestWriteWatch::GuestWriteWatchCollectArmed_nid_postfix : &GuestWriteWatch::GuestWriteWatchCollect_nid_postfix; const bool complete = collect(static_cast(first), static_cast(stop - first), &stampWrittenRun, &runs); if (collectDirtyRuns.load(std::memory_order_relaxed) != dirtyRuns) collectDirty.fetch_add(1, std::memory_order_relaxed); - return complete; #endif + if (window != nullptr) stampWindowRuns(runs); + return complete; } std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoized) { @@ -1782,12 +1843,8 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function } } std::vector hashes(static_cast((stop - first) / page), 0); - for (const auto& [begin, end] : spans) { - CheckRange(reinterpret_cast(begin), static_cast(end - begin), 1, false); - for (auto cursor = begin; cursor < end; cursor += page) { - hashes[static_cast((cursor - first) / page)] = HashPage(reinterpret_cast(cursor)); - } - } + for (const auto& [begin, end] : spans) CheckRange(reinterpret_cast(begin), static_cast(end - begin), 1, false); + HashPages(first, spans, hashes); std::erase_if(spans, [&](const auto& span) { return !walkWrites(tracker, span.first, span.second, StampKind::Cpu, true); }); if (!runImport()) return false; const WindowCopy window{first, hashes}; diff --git a/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp b/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp index 267c945d64..25a91de385 100644 --- a/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp @@ -15,6 +15,7 @@ #ifdef __GLIBC__ #include #endif +#include #include #include #include @@ -1240,6 +1241,41 @@ void CheckImportWindowOverUnwatchedBlock() { Require(after != 0 && !UnchangedSince(base, Block, after), "a store after an import window over an unwatched block was missed"); } +void CheckImportWindowAcrossThreads() { + constexpr std::size_t blocks = 264; + constexpr std::size_t gap = 130; + constexpr std::size_t pageBytes = 4096; + void* memory = AllocateWatched(blocks * Block); + const auto base = reinterpret_cast(memory); + auto* bytes = static_cast(memory); + auto* plain = static_cast(memory); + std::memset(plain, 0x11, blocks * Block); + for (std::size_t page = 0; page < blocks * Block / pageBytes; ++page) { + const std::uint64_t mark = (page + 1) * 0x9E3779B97F4A7C15ull; + std::memcpy(plain + page * pageBytes, &mark, sizeof(mark)); + } + Unwatch(base + gap * Block, Block); + const auto synced = CollectWritesUncached(base, gap * Block); + const auto syncedAfter = CollectWritesUncached(base + (gap + 1) * Block, (blocks - gap - 1) * Block); + Require(synced != 0 && syncedAfter != 0, "the blocks of a large import window are not collected"); + constexpr std::array written{3, 70, 140, 250}; + Require(ImportWatched(base, blocks * Block, [&] { + for (std::size_t page = 0; page < blocks * Block / pageBytes; ++page) { + if (page * pageBytes / Block != gap) bytes[page * pageBytes + 0x100] = 0x11; + } + for (const auto block : written) bytes[block * Block + 0x3100] = 0x22; + return true; + }), "the large import window import failed"); + Require(Watched(base, gap * Block) && Watched(base + (gap + 1) * Block, (blocks - gap - 1) * Block) && !Watched(base + gap * Block, Block), "a large import window changed which blocks are watched"); + const auto since = [&](std::size_t block) { return block < gap ? synced : syncedAfter; }; + for (std::size_t block = 0; block < blocks; ++block) { + if (block == gap) continue; + const bool stored = std::find(written.begin(), written.end(), block) != written.end(); + if (stored) Require(WrittenSince(base + block * Block, Block, since(block)), "a store of new bytes within a large import window was missed in block " + std::to_string(block)); + else Require(UnchangedSince(base + block * Block, Block, since(block)), "a page a large import window resolved without changing its bytes reads as written in block " + std::to_string(block)); + } +} + void CheckImportWindowFootprint() { #ifdef __GLIBC__ constexpr std::size_t bytes = std::size_t{64} << 20; @@ -1562,6 +1598,7 @@ int main() { CheckImportWindowZeroPages(); CheckImportWindowOtherThread(); CheckImportWindowOverUnwatchedBlock(); + CheckImportWindowAcrossThreads(); CheckStoreAcrossImports(); CheckImportWindowFootprint(); CheckUnwatchedImport();