diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index 8648a7fa14..398ea6fe18 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -24,7 +24,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -853,6 +856,8 @@ namespace { constexpr std::size_t WriteBlockBytes = 65536; constexpr std::size_t WritePageBytes = 4096; constexpr std::size_t WritePagesPerBlock = WriteBlockBytes / WritePageBytes; +constexpr std::size_t ImportHashPagesPerThread = 1024; +constexpr unsigned ImportHashThreads = 4; enum class StampKind : std::uint8_t { Cpu, Driver, ImportWindow, Fresh }; @@ -877,6 +882,43 @@ std::uint64_t HashPage(const std::byte* page) { return hash ^ (hash >> 32); } +template +void SplitPages(std::size_t pages, const Range& range) { + static const unsigned hostThreads = std::max(1u, std::thread::hardware_concurrency() / 2); + const auto workers = std::min({ImportHashThreads, hostThreads, pages / ImportHashPagesPerThread}); + if (workers < 2) { + range(std::size_t{0}, pages); + return; + } + std::vector threads; + threads.reserve(workers - 1); + for (std::size_t worker = 1; worker < workers; ++worker) { + const auto from = pages * worker / workers; + const auto to = pages * (worker + 1) / workers; + try { + threads.emplace_back(range, from, to); + } catch (const std::system_error&) { + range(from, to); + } + } + range(std::size_t{0}, pages / workers); +} + +std::vector PagesOf(std::span> spans) { + std::vector pages; + for (const auto& [begin, end] : spans) { + for (auto page = begin; page < end; page += WritePageBytes) pages.push_back(page); + } + return pages; +} + +void HashPages(std::uint64_t first, std::span> spans, std::vector& hashes) { + const auto pages = PagesOf(spans); + SplitPages(pages.size(), [&](std::size_t from, std::size_t to) { + for (auto index = from; index < to; ++index) hashes[static_cast((pages[index] - first) / WritePageBytes)] = HashPage(reinterpret_cast(pages[index])); + }); +} + struct WindowCopy { std::uint64_t first; std::span hashes; @@ -1400,27 +1442,45 @@ struct StampRuns { StampKind kind; std::size_t stamped = 0; const WindowCopy* window = nullptr; + std::vector> windowRuns; }; +void noteWrittenRun(StampRuns& runs, std::uintptr_t begin, std::uintptr_t end, std::uint32_t generation) { + if (runs.kind != StampKind::Driver) runs.tracker.noteCpuStore(begin, end, generation); + else runs.tracker.noteWalkStore(begin, end, generation); +} + void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) { auto& runs = *static_cast(context); auto& tracker = runs.tracker; - const auto kind = runs.kind; if (end <= begin) return; ++runs.stamped; - const auto generation = tracker.generation.load(std::memory_order_relaxed); - if (kind != StampKind::Driver) tracker.noteCpuStore(begin, end, generation); - else tracker.noteWalkStore(begin, end, generation); - if (runs.window != nullptr) { - for (auto page = begin; page < end; page += WritePageBytes) tracker.stamp(tracker.blockOf(page), generation, runs.window->Holds(page) ? StampKind::ImportWindow : kind); - } else { - for (auto block = tracker.blockOf(begin); block <= tracker.blockOf(end - 1); ++block) tracker.stamp(block, generation, kind); + if (runs.window != nullptr) runs.windowRuns.emplace_back(begin, end); + else { + const auto generation = tracker.generation.load(std::memory_order_relaxed); + noteWrittenRun(runs, begin, end, generation); + for (auto block = tracker.blockOf(begin); block <= tracker.blockOf(end - 1); ++block) tracker.stamp(block, generation, runs.kind); } #ifndef _WIN32 collectDirtyRuns.fetch_add(1, std::memory_order_relaxed); #endif } +void stampWindowRuns(StampRuns& runs) { + const auto pages = PagesOf(runs.windowRuns); + std::vector held(pages.size(), 0); + SplitPages(pages.size(), [&](std::size_t from, std::size_t to) { + for (auto index = from; index < to; ++index) held[index] = runs.window->Holds(static_cast(pages[index])); + }); + auto& tracker = runs.tracker; + const auto generation = tracker.generation.load(std::memory_order_relaxed); + std::size_t index = 0; + for (const auto& [begin, end] : runs.windowRuns) { + noteWrittenRun(runs, static_cast(begin), static_cast(end), generation); + for (auto page = begin; page < end; page += WritePageBytes) tracker.stamp(tracker.blockOf(page), generation, held[index++] != 0 ? StampKind::ImportWindow : runs.kind); + } +} + #ifdef _WIN32 std::atomic freshCollectHook{nullptr}; @@ -1454,16 +1514,17 @@ bool collectWindowsWrites(StampRuns& runs, std::uint64_t first, std::uint64_t st bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, StampKind kind, bool armedOnly = false, const WindowCopy* window = nullptr) { ++tracker.generation; - StampRuns runs{tracker, kind, 0, window}; + StampRuns runs{tracker, kind, 0, window, {}}; #ifdef _WIN32 - return collectWindowsWrites(runs, first, stop, armedOnly); + const bool complete = collectWindowsWrites(runs, first, stop, armedOnly); #else const auto dirtyRuns = collectDirtyRuns.load(std::memory_order_relaxed); const auto collect = armedOnly ? &GuestWriteWatch::GuestWriteWatchCollectArmed_nid_postfix : &GuestWriteWatch::GuestWriteWatchCollect_nid_postfix; const bool complete = collect(static_cast(first), static_cast(stop - first), &stampWrittenRun, &runs); if (collectDirtyRuns.load(std::memory_order_relaxed) != dirtyRuns) collectDirty.fetch_add(1, std::memory_order_relaxed); - return complete; #endif + if (window != nullptr) stampWindowRuns(runs); + return complete; } std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoized) { @@ -1782,12 +1843,8 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function } } std::vector hashes(static_cast((stop - first) / page), 0); - for (const auto& [begin, end] : spans) { - CheckRange(reinterpret_cast(begin), static_cast(end - begin), 1, false); - for (auto cursor = begin; cursor < end; cursor += page) { - hashes[static_cast((cursor - first) / page)] = HashPage(reinterpret_cast(cursor)); - } - } + for (const auto& [begin, end] : spans) CheckRange(reinterpret_cast(begin), static_cast(end - begin), 1, false); + HashPages(first, spans, hashes); std::erase_if(spans, [&](const auto& span) { return !walkWrites(tracker, span.first, span.second, StampKind::Cpu, true); }); if (!runImport()) return false; const WindowCopy window{first, hashes}; diff --git a/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp b/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp index 267c945d64..25a91de385 100644 --- a/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp @@ -15,6 +15,7 @@ #ifdef __GLIBC__ #include #endif +#include #include #include #include @@ -1240,6 +1241,41 @@ void CheckImportWindowOverUnwatchedBlock() { Require(after != 0 && !UnchangedSince(base, Block, after), "a store after an import window over an unwatched block was missed"); } +void CheckImportWindowAcrossThreads() { + constexpr std::size_t blocks = 264; + constexpr std::size_t gap = 130; + constexpr std::size_t pageBytes = 4096; + void* memory = AllocateWatched(blocks * Block); + const auto base = reinterpret_cast(memory); + auto* bytes = static_cast(memory); + auto* plain = static_cast(memory); + std::memset(plain, 0x11, blocks * Block); + for (std::size_t page = 0; page < blocks * Block / pageBytes; ++page) { + const std::uint64_t mark = (page + 1) * 0x9E3779B97F4A7C15ull; + std::memcpy(plain + page * pageBytes, &mark, sizeof(mark)); + } + Unwatch(base + gap * Block, Block); + const auto synced = CollectWritesUncached(base, gap * Block); + const auto syncedAfter = CollectWritesUncached(base + (gap + 1) * Block, (blocks - gap - 1) * Block); + Require(synced != 0 && syncedAfter != 0, "the blocks of a large import window are not collected"); + constexpr std::array written{3, 70, 140, 250}; + Require(ImportWatched(base, blocks * Block, [&] { + for (std::size_t page = 0; page < blocks * Block / pageBytes; ++page) { + if (page * pageBytes / Block != gap) bytes[page * pageBytes + 0x100] = 0x11; + } + for (const auto block : written) bytes[block * Block + 0x3100] = 0x22; + return true; + }), "the large import window import failed"); + Require(Watched(base, gap * Block) && Watched(base + (gap + 1) * Block, (blocks - gap - 1) * Block) && !Watched(base + gap * Block, Block), "a large import window changed which blocks are watched"); + const auto since = [&](std::size_t block) { return block < gap ? synced : syncedAfter; }; + for (std::size_t block = 0; block < blocks; ++block) { + if (block == gap) continue; + const bool stored = std::find(written.begin(), written.end(), block) != written.end(); + if (stored) Require(WrittenSince(base + block * Block, Block, since(block)), "a store of new bytes within a large import window was missed in block " + std::to_string(block)); + else Require(UnchangedSince(base + block * Block, Block, since(block)), "a page a large import window resolved without changing its bytes reads as written in block " + std::to_string(block)); + } +} + void CheckImportWindowFootprint() { #ifdef __GLIBC__ constexpr std::size_t bytes = std::size_t{64} << 20; @@ -1562,6 +1598,7 @@ int main() { CheckImportWindowZeroPages(); CheckImportWindowOtherThread(); CheckImportWindowOverUnwatchedBlock(); + CheckImportWindowAcrossThreads(); CheckStoreAcrossImports(); CheckImportWindowFootprint(); CheckUnwatchedImport();