From 8d462f22467cca6d83ce5088192a58992503652b Mon Sep 17 00:00:00 2001 From: lucannez64 <58551020+lucannez64@users.noreply.github.com> Date: Sun, 18 Oct 2026 14:16:31 +0200 Subject: [PATCH 1/2] perf(indium): rebase lazy wakeups and batch tuning onto current polling Keep the merged per-device completion budget, bounded parsing, idle blocking and timeout accounting. Scope the registration/blocking handshake to each device, restrict live batch windows to the timer resolution, and test registration interleavings, spin-disabled polling, shutdown, opt-out and independent devices with actual patched methods. Co-authored-by: SypherrDev --- docs/SETUP.md | 3 + docs/TROUBLESHOOTING.md | 5 + patches/metal--deps--indium.patch | 147 +++++++++++++++++++++++++----- tests/fixtures/indium-wakeup.cpp | 146 +++++++++++++++++++++++++++++ tests/test_indium_wakeup.py | 63 +++++++++++++ tests/test_nvidia_runtime.py | 3 + 6 files changed, 342 insertions(+), 25 deletions(-) create mode 100644 tests/fixtures/indium-wakeup.cpp create mode 100644 tests/test_indium_wakeup.py diff --git a/docs/SETUP.md b/docs/SETUP.md index d427c62..1d0cbd0 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -135,6 +135,9 @@ Indium also contains experimental hand-off changes. They are **off by default**: | `/tmp/metal-gpu` | diagnostic: selects the Vulkan device used for Metal by index or case-insensitive name substring instead of the loader's first device. The OpenGL context that presents does not follow it; see [hybrid laptops](TROUBLESHOOTING.md#hybrid-graphics-laptops-amdintel-igpu--nvidia). `METAL_INDIUM_GPU` is the environment equivalent; the file wins | | `/tmp/metal-all-devices` | a `VkDevice` is created for every supported Vulkan device, as before 2026-10-16. Without it only the selected device gets one; on a machine with lavapipe installed this also drops lavapipe's idle rasterizer threads | | `/tmp/metal-no-drawable-vram` | drawables of a layer with `framebufferOnly` off (League sets it off) go back to host-visible system RAM, as before 2026-10-16. Without the file they are allocated in GPU memory like `framebufferOnly` drawables; nothing maps drawable memory from the CPU. On the RX 6600 this gave about 20 FPS more in a practice-tool fight and the same FPS when idle, not yet measured in an ARAM. `/tmp/metal-device.log` shows `drawable-vram=on` and `storage=vram` | +| `/tmp/metal-no-lazy-wakeup` | every completion registration signals the event loop's wake-up semaphore from the CPU again, as before 2026-10-18. The handshake is per device. Without the file a registration signals only while the poller has announced a blocking wait (a spinning poller re-reads the registrations on its next pass, and its spin ends when one arrives). On the NVIDIA proprietary driver each `vkSignalSemaphore` is an RM control ioctl that waits on a driver-wide lock, about 0.3 ms each and ~25 per League frame, and it was the largest kernel cost of the submit worker. Re-read every 256 registrations; the wait log counts `wakeupSignalled` and `wakeupSkipped` | +| `/tmp/metal-submit-batch-us` | the submit batch window in microseconds (500–100000, default 500); re-read every 256 checks, so it can be changed during a match. A presenting commit and every CPU wait still flush the batch at once. On the NVIDIA proprietary driver every `vkQueueSubmit2` is an RM control ioctl that waits on a driver-wide lock (about 0.4 ms each in a League match); with the default window a frame took about 8.5 submits, with `5000` about 2.5 (RTX 4090: 89 → 113 FPS, same match) | +| `/tmp/metal-submit-batch-max` | command buffers per batch (1–256, default 16), re-read like the window; `64` lets one batch hold a whole frame | | `/tmp/metal-debug-waits` | diagnostic: counts how often and how long threads block at the hand-off waits and in `nextDrawable`, every 5 s in `/tmp/metal-trace.log`; the first check writes a `waits enabled` line | Create these files from inside the guest, for example `darling shell touch /tmp/metal-debug-waits`, or while Darling is shut down. The guest `/tmp` is `~/.darling/private/tmp` seen through Darling's overlay mount. A file created on the host in that directory while Darling runs may stay invisible to guest processes that already looked for it. The same applies to removing a file. The older `/tmp/metal-no-present-dispatch-wait`, `/tmp/metal-no-deferred-blit` and `/tmp/metal-no-skip-cpu-devices` files are ignored. diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 0cc5bcf..d995a32 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -202,6 +202,11 @@ Compare matches on the same map: Bilgewater runs a few percent below Hexgate. On Hyprland, also check that direct scanout is off (`hyprctl getoption render:direct_scanout` prints `int: 0`). The first match with it off ran about 16 FPS faster than the previous one on the same build; see the next section. +On the NVIDIA proprietary driver, check the submit worker's kernel time: a thread of the game process spending most of its time in the kernel (`/proc//task//stat` system time, or `perf trace -s -p `) points at driver calls. Each `vkQueueSubmit2` and each CPU-side `vkSignalSemaphore` is an RM control ioctl (command `0xc020462a`) that queues on a driver-wide lock. Before 2026-10-18 every command buffer signalled the event loop's wake-up semaphore, ~25 times per frame; with lazy wake-ups (default) and a 5 ms batch window (`/tmp/metal-submit-batch-us` = `5000`, `/tmp/metal-submit-batch-max` = `64`) an RTX 4090 ran at 206–552 FPS against 91–147 FPS with `/tmp/metal-no-lazy-wakeup`, in alternating 20 s phases of one match (the later phases with wake-ups on were capped at 240 FPS) (see [setup](SETUP.md#metal-and-indium-overlays)). + +The NVIDIA performance figures above were reported on an earlier runtime base; +they have not been remeasured after integration with current `main`. + ## Hybrid graphics laptops (AMD/Intel iGPU + NVIDIA) Metal renders with Vulkan, and each finished frame is handed to an OpenGL (EGL) context that presents it. The hand-off uses opaque file descriptors, which only work when Vulkan and OpenGL run on the same GPU. Indium takes the first Vulkan device the loader lists, and on a hybrid laptop both that device and the EGL context are normally the integrated GPU, so the discrete GPU stays idle. A report in [issue #3](https://github.com/lucannez64/darling-lol/issues/3) (Ryzen 9 5900HS, RTX 3060 Laptop) measured 60-70 FPS in ARAM on the integrated GPU; on the NVIDIA GPU the game showed a black screen. **Running on an NVIDIA GPU is not validated.** diff --git a/patches/metal--deps--indium.patch b/patches/metal--deps--indium.patch index 9ae8d9b..72665e9 100644 --- a/patches/metal--deps--indium.patch +++ b/patches/metal--deps--indium.patch @@ -781,7 +781,7 @@ index 8546daa..7e4993c 100644 void updateBindings(); diff --git a/private-include/indium/device.private.hpp b/private-include/indium/device.private.hpp -index 58f1794..1f4bd5a 100644 +index 58f1794..3e7ee47 100644 --- a/private-include/indium/device.private.hpp +++ b/private-include/indium/device.private.hpp @@ -13,6 +13,8 @@ @@ -793,7 +793,7 @@ index 58f1794..1f4bd5a 100644 namespace Indium { class PrivateDevice; -@@ -25,8 +27,11 @@ namespace Indium { +@@ -25,8 +27,14 @@ namespace Indium { class PrivateDevice: public Device, public std::enable_shared_from_this { private: @@ -802,10 +802,13 @@ index 58f1794..1f4bd5a 100644 std::mutex _eventLoopMutex; std::mutex _pollingMutex; + uint64_t _completionSpinNanoseconds = 0; ++ // Sequentially consistent handshake: either re-snapshot or signal. ++ std::atomic _eventLoopRegistrations {0}; ++ std::atomic _eventLoopBlockedPollers {0}; // these are stored separately (rather than in a single structure) // so that we can easily copy them individually in the event loop polling function. -@@ -59,6 +64,8 @@ namespace Indium { +@@ -59,6 +67,8 @@ namespace Indium { } virtual std::string name() const override; @@ -814,7 +817,7 @@ index 58f1794..1f4bd5a 100644 virtual std::shared_ptr newCommandQueue() override; virtual std::shared_ptr newRenderPipelineState(const RenderPipelineDescriptor& descriptor) override; virtual std::shared_ptr newComputePipelineState(const ComputePipelineDescriptor& descriptor, PipelineOption options, std::shared_ptr reflection) override; -@@ -73,6 +80,12 @@ namespace Indium { +@@ -73,6 +83,12 @@ namespace Indium { virtual void pollEvents(uint64_t timeoutNanoseconds) override; virtual void wakeupEventLoop() override; @@ -827,7 +830,7 @@ index 58f1794..1f4bd5a 100644 void waitForSemaphore(VkSemaphore semaphore, uint64_t targetValue, std::function callback); /** -@@ -86,6 +99,11 @@ namespace Indium { +@@ -86,6 +102,11 @@ namespace Indium { * Similar to getTimelineSemaphore(), but wraps it in a shared pointer * that automatically releases the semaphore when all references to it die out. */ @@ -839,7 +842,7 @@ index 58f1794..1f4bd5a 100644 std::shared_ptr getWrappedTimelineSemaphore(); BinarySemaphore getBinarySemaphore(bool exportable = false); -@@ -111,5 +129,34 @@ namespace Indium { +@@ -111,5 +132,34 @@ namespace Indium { INDIUM_PROPERTY(VkPhysicalDeviceMemoryProperties, m, M,emoryProperties); INDIUM_PROPERTY_READONLY(Feature, f, F,eatures); @@ -1934,7 +1937,7 @@ index 23fd2d3..1a48b81 100644 + return _gpuAddress; }; diff --git a/src/indium/command-buffer.cpp b/src/indium/command-buffer.cpp -index 43f4a84..aaa1acb 100644 +index 43f4a84..f7640c5 100644 --- a/src/indium/command-buffer.cpp +++ b/src/indium/command-buffer.cpp @@ -1,3 +1,8 @@ @@ -2271,7 +2274,7 @@ index 43f4a84..aaa1acb 100644 auto encoder = std::make_shared(shared_from_this(), descriptor); { std::scoped_lock lock(_mutex); -@@ -91,10 +354,239 @@ std::shared_ptr Indium::PrivateCommandBuffer::com +@@ -91,10 +354,269 @@ std::shared_ptr Indium::PrivateCommandBuffer::com return computeCommandEncoder(ComputePassDescriptor { {}, dispatchType }); }; @@ -2393,6 +2396,35 @@ index 43f4a84..aaa1acb 100644 + } +} + ++// darling-lol: tunable batch window and size. On the NVIDIA proprietary driver every vkQueueSubmit2 is an RM control ++// ioctl that queues on a driver-wide lock (~0.4 ms each in a League match, ~8.5 submits per frame with the 500 us ++// window), so fewer, larger batches matter there. Guest files, re-read every 256 checks: /tmp/metal-submit-batch-us ++// (window in microseconds, 500-100000, default 500) and /tmp/metal-submit-batch-max (command buffers per batch, ++// 1-256, default 16). A presenting commit and every CPU wait still flush the batch at once. ++long readTuning(const char* path, long fallback, long low, long high) { ++ FILE* file = fopen(path, "r"); ++ if (!file) return fallback; ++ long value = fallback; ++ if (fscanf(file, "%ld", &value) != 1) value = fallback; ++ fclose(file); ++ return std::clamp(value, low, high); ++} ++ ++struct SubmitBatchTuning { ++ std::atomic windowMicroseconds {500}; ++ std::atomic maxItems {16}; ++ std::atomic calls {0}; ++}; ++ ++SubmitBatchTuning& submitBatchTuning() { ++ static auto* tuning = new SubmitBatchTuning; ++ if ((tuning->calls.fetch_add(1, std::memory_order_relaxed) & 255) == 0) { ++ tuning->windowMicroseconds.store(readTuning("/tmp/metal-submit-batch-us", 500, 500, 100000), std::memory_order_relaxed); ++ tuning->maxItems.store(readTuning("/tmp/metal-submit-batch-max", 16, 1, 256), std::memory_order_relaxed); ++ } ++ return *tuning; ++} ++ +void startSubmitBatchTimer(const std::shared_ptr& device) { + auto& state = submitBatchState(); + if (state.timerStarted) return; @@ -2407,8 +2439,9 @@ index 43f4a84..aaa1acb 100644 + * merging with latency; the old 2ms added visible input lag. */ + submitBatchWake().wait_for(lock, std::chrono::microseconds(500)); + } ++ const auto window = std::chrono::microseconds(submitBatchTuning().windowMicroseconds.load(std::memory_order_relaxed)); + std::unique_lock queueLock(device->graphicsQueueMutex()); -+ if (!state.items.empty() && std::chrono::steady_clock::now() - state.since >= std::chrono::microseconds(500)) flushSubmitBatchLocked(); ++ if (!state.items.empty() && std::chrono::steady_clock::now() - state.since >= window) flushSubmitBatchLocked(); + } + }).detach(); +} @@ -2511,7 +2544,7 @@ index 43f4a84..aaa1acb 100644 for (const auto& encoder: _commandEncoders) { if (auto renderEncoder = std::dynamic_pointer_cast(encoder)) { -@@ -110,16 +602,28 @@ void Indium::PrivateCommandBuffer::commit() { +@@ -110,16 +632,28 @@ void Indium::PrivateCommandBuffer::commit() { } } @@ -2541,7 +2574,7 @@ index 43f4a84..aaa1acb 100644 // for the event loop auto timelineSemaphore = _privateDevice->getWrappedTimelineSemaphore(); -@@ -138,14 +642,49 @@ void Indium::PrivateCommandBuffer::commit() { +@@ -138,14 +672,49 @@ void Indium::PrivateCommandBuffer::commit() { readWriteTextures.insert(readWriteTextures.end(), readWrite.begin(), readWrite.end()); } @@ -2594,7 +2627,7 @@ index 43f4a84..aaa1acb 100644 } std::vector signalInfos; -@@ -202,11 +741,13 @@ void Indium::PrivateCommandBuffer::commit() { +@@ -202,11 +771,13 @@ void Indium::PrivateCommandBuffer::commit() { auto privateTexture = std::dynamic_pointer_cast(texture); const auto& presentSema = presentationSemaphores[i]; @@ -2613,7 +2646,7 @@ index 43f4a84..aaa1acb 100644 handleTextureSemaphores(privateTexture); } -@@ -215,7 +756,15 @@ void Indium::PrivateCommandBuffer::commit() { +@@ -215,7 +786,15 @@ void Indium::PrivateCommandBuffer::commit() { // but it also leaves the resources for this command buffer tied up, essentially becoming a memory leak. // i've observed this in the cube example, and it happens more than once (because the example display semaphore is exhausted and never signaled). // UPDATE: upon further testing, it seems that this only occurs when the view is off-screen/hidden. weird. @@ -2630,7 +2663,7 @@ index 43f4a84..aaa1acb 100644 { std::unique_lock lock(self->_mutex); self->_completed = true; -@@ -234,8 +783,47 @@ void Indium::PrivateCommandBuffer::commit() { +@@ -234,8 +813,47 @@ void Indium::PrivateCommandBuffer::commit() { // the handlers may have captured a reference to us (from their surrounding scope), so clear out handlers so those references go away self->_scheduledHandlers.clear(); self->_completedHandlers.clear(); @@ -2668,7 +2701,7 @@ index 43f4a84..aaa1acb 100644 + } + batch.items.push_back(std::move(item)); + startSubmitBatchTimer(_privateDevice); -+ if (batch.items.size() >= 16) flushSubmitBatchLocked(); ++ if (batch.items.size() >= (size_t) submitBatchTuning().maxItems.load(std::memory_order_relaxed)) flushSubmitBatchLocked(); + return; + } + } @@ -2678,7 +2711,7 @@ index 43f4a84..aaa1acb 100644 commandBufferInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO; commandBufferInfo.commandBuffer = _commandBuffer; -@@ -252,23 +840,41 @@ void Indium::PrivateCommandBuffer::commit() { +@@ -252,23 +870,41 @@ void Indium::PrivateCommandBuffer::commit() { // the Device constructor tries to choose command queues that support as many operations as possible, but it's possible // that a particular device only supports certain operations on certain queues (e.g. maybe it only supports transfer operations // on an exclusive queue that doesn't support graphics or compute). @@ -2725,7 +2758,7 @@ index 43f4a84..aaa1acb 100644 }; void Indium::PrivateCommandBuffer::presentDrawable(std::shared_ptr drawable) { -@@ -307,9 +913,12 @@ void Indium::PrivateCommandBuffer::addCompletedHandler(std::functioncommandBuffer(), VK_PIPELINE_BIND_POINT_COMPUTE, _pso->layout(), 0, descriptorSets.size(), descriptorSets.data(), 0, nullptr); }; diff --git a/src/indium/device.cpp b/src/indium/device.cpp -index 322e186..4da420a 100644 +index 322e186..f712c7a 100644 --- a/src/indium/device.cpp +++ b/src/indium/device.cpp @@ -1,4 +1,9 @@ @@ -3161,11 +3194,11 @@ index 322e186..4da420a 100644 + } + creationOrder.push_back(i); + } - } ++ } + + for (size_t index: creationOrder) { + globalDeviceList.push_back(std::make_shared(candidates[index].device)); -+ } + } + + appendDeviceLog("created VkDevice for " + std::to_string(globalDeviceList.size()) + " of " + + std::to_string(candidates.size()) + " supported device(s)" @@ -3291,7 +3324,32 @@ index 322e186..4da420a 100644 funcInfo.bindings.insert(funcInfo.bindings.end(), info.bindings.begin(), info.bindings.end()); funcInfo.embeddedSamplers.insert(funcInfo.embeddedSamplers.end(), info.embeddedSamplers.begin(), info.embeddedSamplers.end()); } -@@ -396,6 +722,26 @@ void Indium::PrivateDevice::pollEvents(uint64_t timeoutNanoseconds) { +@@ -382,6 +708,19 @@ std::shared_ptr Indium::createSystemDefaultDevice() { + return globalDeviceList.empty() ? nullptr : globalDeviceList.front(); + }; + ++namespace { ++bool lazyWakeupEnabled() { ++ static std::atomic enabled {true}; ++ static std::atomic calls {0}; ++ if ((calls.fetch_add(1, std::memory_order_relaxed) & 255) == 0) ++ enabled.store(access("/tmp/metal-no-lazy-wakeup", F_OK) != 0, std::memory_order_relaxed); ++ return enabled.load(std::memory_order_relaxed); ++} ++void noteWakeup(const char* site) { ++ if (Indium::Waits::enabled()) Indium::Waits::record(site, 0); ++} ++} ++ + void Indium::PrivateDevice::pollEvents(uint64_t timeoutNanoseconds) { + // this is held for the entire duration of the poll so we can + // ensure we're the only one polling, which allows us to avoid +@@ -393,9 +732,35 @@ void Indium::PrivateDevice::pollEvents(uint64_t timeoutNanoseconds) { + const std::vector semaphores = _eventLoopSemaphores; + const std::vector values = _eventLoopWaitValues; + const std::vector> callbacks = _eventLoopCallbacks; ++ // All registrations covered by this snapshot, under the same mutex. ++ const uint64_t snapshotRegistrations = _eventLoopRegistrations.load(std::memory_order_seq_cst); lock.unlock(); @@ -3308,6 +3366,10 @@ index 322e186..4da420a 100644 + anyReady = DynamicVK::vkGetSemaphoreCounterValue(_device, semaphores[i], &count) == VK_SUCCESS && count >= values[i]; + } + if (anyReady) break; ++ if (_eventLoopRegistrations.load(std::memory_order_relaxed) != snapshotRegistrations) { ++ anyReady = true; // Re-snapshot registrations instead of waiting on stale handles. ++ break; ++ } + __builtin_ia32_pause(); + } while (std::chrono::steady_clock::now() < deadline); + const uint64_t elapsed = std::chrono::duration_cast( @@ -3318,8 +3380,43 @@ index 322e186..4da420a 100644 VkSemaphoreWaitInfo info {}; info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO; info.semaphoreCount = semaphores.size(); -@@ -480,9 +826,18 @@ void Indium::PrivateDevice::waitForSemaphore(VkSemaphore semaphore, uint64_t tar - wakeupEventLoop(); +@@ -403,7 +768,19 @@ void Indium::PrivateDevice::pollEvents(uint64_t timeoutNanoseconds) { + info.pValues = values.data(); + info.flags = VK_SEMAPHORE_WAIT_ANY_BIT; + +- auto result = DynamicVK::vkWaitSemaphores(_device, &info, timeoutNanoseconds); ++ // A registrant increments its counter before inspecting blocked pollers. ++ // The poller announces its wait before rechecking registrations. With both ++ // operations seq_cst, either this wait becomes nonblocking or the registrant ++ // signals the wake-up semaphore. The counters belong to this device only. ++ const bool announcedBlock = timeoutNanoseconds > 0; ++ if (announcedBlock) { ++ _eventLoopBlockedPollers.fetch_add(1, std::memory_order_seq_cst); ++ if (_eventLoopRegistrations.load(std::memory_order_seq_cst) != snapshotRegistrations) ++ timeoutNanoseconds = 0; ++ } ++ auto result = DynamicVK::vkWaitSemaphores(_device, &info, timeoutNanoseconds); ++ if (announcedBlock) ++ _eventLoopBlockedPollers.fetch_sub(1, std::memory_order_seq_cst); + + if (result == VK_TIMEOUT) { + // timed out with no semaphores ready +@@ -474,15 +851,30 @@ void Indium::PrivateDevice::waitForSemaphore(VkSemaphore semaphore, uint64_t tar + _eventLoopSemaphores.push_back(semaphore); + _eventLoopWaitValues.push_back(targetValue); + _eventLoopCallbacks.push_back(callback); ++ _eventLoopRegistrations.fetch_add(1, std::memory_order_seq_cst); + } + +- // now wakeup the event loop so it can start waiting on this new semaphore +- wakeupEventLoop(); ++ // A spinning poller re-snapshots registrations without a driver signal. ++ if (!lazyWakeupEnabled() || _eventLoopBlockedPollers.load(std::memory_order_seq_cst) > 0) { ++ noteWakeup("wakeupSignalled"); ++ wakeupEventLoop(); ++ } else { ++ noteWakeup("wakeupSkipped"); ++ } }; -// TODO: create a semaphore pool to avoid constantly creating and destroying semaphores @@ -3338,7 +3435,7 @@ index 322e186..4da420a 100644 VkSemaphoreTypeCreateInfo typeInfo {}; typeInfo.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO; typeInfo.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE; -@@ -505,7 +860,14 @@ Indium::TimelineSemaphore Indium::PrivateDevice::getTimelineSemaphore() { +@@ -505,7 +897,14 @@ Indium::TimelineSemaphore Indium::PrivateDevice::getTimelineSemaphore() { }; void Indium::PrivateDevice::putTimelineSemaphore(const TimelineSemaphore& semaphore) { @@ -3354,7 +3451,7 @@ index 322e186..4da420a 100644 }; std::shared_ptr Indium::PrivateDevice::getWrappedTimelineSemaphore() { -@@ -551,3 +913,160 @@ std::shared_ptr Indium::PrivateDevice::getWrappedBinary +@@ -551,3 +950,160 @@ std::shared_ptr Indium::PrivateDevice::getWrappedBinary delete ptr; }); }; diff --git a/tests/fixtures/indium-wakeup.cpp b/tests/fixtures/indium-wakeup.cpp new file mode 100644 index 0000000..1d89100 --- /dev/null +++ b/tests/fixtures/indium-wakeup.cpp @@ -0,0 +1,146 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using VkSemaphore = int; +struct VkSemaphoreWaitInfo { int sType, flags; size_t semaphoreCount; const VkSemaphore* pSemaphores; const uint64_t* pValues; }; +struct VkSemaphoreSignalInfo { int sType; VkSemaphore semaphore; uint64_t value; }; +constexpr int VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO=1, VK_SEMAPHORE_WAIT_ANY_BIT=1; +constexpr int VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO=2, VK_SUCCESS=0, VK_TIMEOUT=1; +static std::array,512> counters {}; +static std::atomic signalCalls {0}; +static std::atomic lastTimeout {0}; +static std::mutex kernelMutex; +static std::condition_variable kernelWake; +static std::function onWait, onCounter; +static bool lazy = true; +static bool lazyWakeupEnabled() { return lazy; } +static void noteWakeup(const char*) {} +struct TestMutex { + std::mutex mutex; + std::function afterUnlock; + void lock() { mutex.lock(); } + void unlock() { + auto hook=std::move(afterUnlock); + afterUnlock=nullptr; + mutex.unlock(); + if(hook) hook(); + } +}; +namespace DynamicVK { +int vkGetSemaphoreCounterValue(int, int semaphore, uint64_t* value) { + auto hook=std::move(onCounter); onCounter=nullptr; + if(hook) hook(); + *value=counters[semaphore].load(); return VK_SUCCESS; +} +int vkSignalSemaphore(int, const VkSemaphoreSignalInfo* info) { + { std::lock_guard lock(kernelMutex); counters[info->semaphore]=info->value; ++signalCalls; } + kernelWake.notify_all(); return VK_SUCCESS; +} +int vkWaitSemaphores(int, const VkSemaphoreWaitInfo* info, uint64_t timeout) { + lastTimeout=timeout; + auto hook=std::move(onWait); onWait=nullptr; + if(hook) hook(); + auto ready=[&]{ + for(size_t i=0;isemaphoreCount;++i) + if(counters[info->pSemaphores[i]].load()>=info->pValues[i]) return true; + return false; + }; + if(ready()) return VK_SUCCESS; + if(!timeout) return VK_TIMEOUT; + std::unique_lock lock(kernelMutex); + // A missed wakeup fails in a bounded interval instead of hanging the suite. + assert(kernelWake.wait_for(lock,std::chrono::seconds(2),ready)); + return VK_SUCCESS; +} +} +namespace Indium { +class PrivateDevice { +public: + std::mutex _pollingMutex; + TestMutex _eventLoopMutex; + int _device=0; + uint64_t _completionSpinNanoseconds=0; + std::atomic _eventLoopRegistrations {0}; + std::atomic _eventLoopBlockedPollers {0}; + std::vector _eventLoopSemaphores={1}; + std::vector _eventLoopWaitValues={1}; + std::vector> _eventLoopCallbacks={nullptr}; + void pollEvents(uint64_t); + void waitForSemaphore(VkSemaphore,uint64_t,std::function); + void wakeupEventLoop(); +}; +} +/* ACTUAL_PATCHED_METHODS */ +static void reset() { + for(auto& value:counters) value=0; + signalCalls=0; lastTimeout=0; lazy=true; onWait=nullptr; onCounter=nullptr; +} +int main() { + // Registration before snapshot is included without a host-side wakeup. + reset(); { + Indium::PrivateDevice d; int done=0; counters[2]=1; + d.waitForSemaphore(2,1,[&]{++done;}); d.pollEvents(UINT64_MAX); + assert(done==1 && signalCalls==0 && d._eventLoopBlockedPollers==0); + } + // Registration after snapshot but before announcing a blocking wait. + reset(); { + Indium::PrivateDevice d; int done=0; + d._eventLoopMutex.afterUnlock=[&]{ counters[2]=1; d.waitForSemaphore(2,1,[&]{++done;}); }; + d.pollEvents(UINT64_MAX); + assert(lastTimeout==0 && signalCalls==0 && done==0 && d._eventLoopBlockedPollers==0); + d.pollEvents(UINT64_MAX); assert(done==1); + } + // Registration after announcing a blocking wait must signal that wait. + reset(); { + Indium::PrivateDevice d; int done=0; + onWait=[&]{ assert(d._eventLoopBlockedPollers==1); counters[2]=1; d.waitForSemaphore(2,1,[&]{++done;}); }; + d.pollEvents(UINT64_MAX); + assert(signalCalls==1 && d._eventLoopBlockedPollers==0 && done==0); + d.pollEvents(UINT64_MAX); assert(done==1); + } + // A registration during the spin ends it and causes a fresh snapshot. + reset(); { + Indium::PrivateDevice d; int done=0; d._completionSpinNanoseconds=1000000; + d.waitForSemaphore(2,1,nullptr); + onCounter=[&]{ counters[3]=1; d.waitForSemaphore(3,1,[&]{++done;}); }; + d.pollEvents(UINT64_MAX); assert(lastTimeout==0 && signalCalls==0); + d.pollEvents(UINT64_MAX); assert(done==1); + } + // Zero-timeout polls never announce a blocking wait. + reset(); { + Indium::PrivateDevice d; + onWait=[&]{ assert(d._eventLoopBlockedPollers==0); d.waitForSemaphore(2,1,nullptr); }; + d.pollEvents(0); assert(lastTimeout==0 && signalCalls==0); + } + // Opt-out restores unconditional registration wakeups; shutdown always wakes. + reset(); { + Indium::PrivateDevice d; lazy=false; d.waitForSemaphore(2,1,nullptr); + assert(signalCalls==1); lazy=true; d.wakeupEventLoop(); assert(signalCalls==2); + } + // A blocked poller on another device cannot change this device's policy. + reset(); { + Indium::PrivateDevice a,b; b._eventLoopBlockedPollers=1; + a.waitForSemaphore(2,1,nullptr); assert(signalCalls==0 && b._eventLoopRegistrations==0); + } + // Exercise real producer/poller concurrency with spinning enabled and disabled. + for(uint64_t spin:{0ULL,1000000ULL}) { + reset(); Indium::PrivateDevice d; d._completionSpinNanoseconds=spin; + std::atomic done {0}; constexpr int count=200; + std::thread poller([&]{ while(done +#include +#include +#include +static const char* fixturePath; +static FILE* test_fopen(const char*, const char* mode) { return ::fopen(fixturePath,mode); } +#define fopen test_fopen +''' + tuning + r''' +#undef fopen +static void write(const char* text) { + FILE* f=fopen(fixturePath,"w"); assert(f); fputs(text,f); fclose(f); +} +int main(int argc,char** argv) { + assert(argc==2); fixturePath=argv[1]; + write("100"); + auto& initial=submitBatchTuning(); + assert(initial.windowMicroseconds==500 && initial.maxItems==100); + write("200000"); + for(int i=0;i<256;++i) submitBatchTuning(); + assert(initial.windowMicroseconds==100000 && initial.maxItems==256); + write("invalid"); + for(int i=0;i<256;++i) submitBatchTuning(); + assert(initial.windowMicroseconds==500 && initial.maxItems==16); +} +''' + self.run_cpp(fixture, config=True) + + def run_cpp(self, text, config=False): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) + (path / 'check.cpp').write_text(text) + compile = subprocess.run(['c++', '-std=c++17', '-pthread', str(path/'check.cpp'), + '-o', str(path/'check')], capture_output=True, text=True) + self.assertEqual(compile.returncode, 0, compile.stderr) + result = subprocess.run([str(path/'check'), *([str(path/'tuning')] if config else [])], + capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) diff --git a/tests/test_nvidia_runtime.py b/tests/test_nvidia_runtime.py index aaa37dd..85dc97d 100644 --- a/tests/test_nvidia_runtime.py +++ b/tests/test_nvidia_runtime.py @@ -126,6 +126,7 @@ int main() { poll = sources.function(source, 'Indium::PrivateDevice::pollEvents') harness = ''' #include +#include #include #include #include @@ -159,6 +160,8 @@ public: std::mutex _pollingMutex, _eventLoopMutex; int _device=0; uint64_t _completionSpinNanoseconds=1000000; + std::atomic _eventLoopRegistrations {0}; + std::atomic _eventLoopBlockedPollers {0}; std::vector _eventLoopSemaphores={1}; std::vector _eventLoopWaitValues={1}; std::vector> _eventLoopCallbacks={nullptr}; From 9c2ceeafd0d071e804d4647da90883a08be65f1f Mon Sep 17 00:00:00 2001 From: lucannez64 <58551020+lucannez64@users.noreply.github.com> Date: Sun, 18 Oct 2026 14:27:41 +0200 Subject: [PATCH 2/2] fix(indium): preserve device field offsets and clarify tuning defaults --- docs/SETUP.md | 2 +- patches/metal--deps--indium.patch | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/SETUP.md b/docs/SETUP.md index 1d0cbd0..3a246f9 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -124,7 +124,7 @@ sh build/runtime/tools/install-riot-metal-pair.sh The installer copies `build/riot-Metal` and `build/riot-libindium.dylib` into the guest `RuntimeFrameworks/Metal.framework` and `RuntimeLibraries`. It backs up the pair it replaces in `build/metal-pair-backups`; a repeated install keeps the older backup. `sh build/runtime/tools/install-riot-metal-pair.sh --rollback` restores that backup. `tools/install-riot-gpu-runtime.sh` still builds and installs the whole GPU set, including these two. Activation needs the namespace restart below. -Indium also contains experimental hand-off changes. They are **off by default**: without the files below, libindium behaves like the build measured before them, and two matches with them on had lower FPS. Each one is a guest file, read once per game process when the game starts. +The hand-off experiments (`metal-present-dispatch-wait`, `metal-deferred-blit` and `metal-worker-lease`) are **off by default**; two matches with them enabled had lower FPS. The table also lists diagnostics and tuning controls with their own defaults. Lazy event-loop wake-ups are **on by default** and `metal-no-lazy-wakeup` disables them. Files are read once per game process at startup unless an entry specifies a different refresh interval. | Guest file | Effect when present | |---|---| diff --git a/patches/metal--deps--indium.patch b/patches/metal--deps--indium.patch index 72665e9..f3c149f 100644 --- a/patches/metal--deps--indium.patch +++ b/patches/metal--deps--indium.patch @@ -781,7 +781,7 @@ index 8546daa..7e4993c 100644 void updateBindings(); diff --git a/private-include/indium/device.private.hpp b/private-include/indium/device.private.hpp -index 58f1794..3e7ee47 100644 +index 58f1794..75266fd 100644 --- a/private-include/indium/device.private.hpp +++ b/private-include/indium/device.private.hpp @@ -13,6 +13,8 @@ @@ -793,7 +793,7 @@ index 58f1794..3e7ee47 100644 namespace Indium { class PrivateDevice; -@@ -25,8 +27,14 @@ namespace Indium { +@@ -25,8 +27,11 @@ namespace Indium { class PrivateDevice: public Device, public std::enable_shared_from_this { private: @@ -802,13 +802,10 @@ index 58f1794..3e7ee47 100644 std::mutex _eventLoopMutex; std::mutex _pollingMutex; + uint64_t _completionSpinNanoseconds = 0; -+ // Sequentially consistent handshake: either re-snapshot or signal. -+ std::atomic _eventLoopRegistrations {0}; -+ std::atomic _eventLoopBlockedPollers {0}; // these are stored separately (rather than in a single structure) // so that we can easily copy them individually in the event loop polling function. -@@ -59,6 +67,8 @@ namespace Indium { +@@ -59,6 +64,8 @@ namespace Indium { } virtual std::string name() const override; @@ -817,7 +814,7 @@ index 58f1794..3e7ee47 100644 virtual std::shared_ptr newCommandQueue() override; virtual std::shared_ptr newRenderPipelineState(const RenderPipelineDescriptor& descriptor) override; virtual std::shared_ptr newComputePipelineState(const ComputePipelineDescriptor& descriptor, PipelineOption options, std::shared_ptr reflection) override; -@@ -73,6 +83,12 @@ namespace Indium { +@@ -73,6 +80,12 @@ namespace Indium { virtual void pollEvents(uint64_t timeoutNanoseconds) override; virtual void wakeupEventLoop() override; @@ -830,7 +827,7 @@ index 58f1794..3e7ee47 100644 void waitForSemaphore(VkSemaphore semaphore, uint64_t targetValue, std::function callback); /** -@@ -86,6 +102,11 @@ namespace Indium { +@@ -86,6 +99,11 @@ namespace Indium { * Similar to getTimelineSemaphore(), but wraps it in a shared pointer * that automatically releases the semaphore when all references to it die out. */ @@ -842,7 +839,7 @@ index 58f1794..3e7ee47 100644 std::shared_ptr getWrappedTimelineSemaphore(); BinarySemaphore getBinarySemaphore(bool exportable = false); -@@ -111,5 +132,34 @@ namespace Indium { +@@ -111,5 +129,37 @@ namespace Indium { INDIUM_PROPERTY(VkPhysicalDeviceMemoryProperties, m, M,emoryProperties); INDIUM_PROPERTY_READONLY(Feature, f, F,eatures); @@ -875,6 +872,9 @@ index 58f1794..3e7ee47 100644 + std::vector> _availableTimelineSemaphores; + std::mutex _renderPassMutex; + std::unordered_map _renderPasses; ++ // Sequentially consistent handshake: either re-snapshot or signal. ++ std::atomic _eventLoopRegistrations {0}; ++ std::atomic _eventLoopBlockedPollers {0}; }; }; diff --git a/private-include/indium/dynamic-vk.hpp b/private-include/indium/dynamic-vk.hpp