Mirror-bot/darling-lol
1
0 Fork 0
mirror of https://github.com/lucannez64/darling-lol.git synced 2026-10-27 19:54:38 +00:00
Code Issues Projects Releases Activity
Files
41da6d9e0adfd0af8162fd68ff1df15839f4bc9e
darling-lol/patches/metal--deps--indium.patch
T

8932 lines
443 KiB
Diff
Raw Blame History

diff --git a/include/indium/device.hpp b/include/indium/device.hpp
index 1572e13..5ebcb5b 100644
--- a/include/indium/device.hpp
+++ b/include/indium/device.hpp
@@ -51,6 +51,8 @@ namespace Indium {
*/
virtual void pollEvents(uint64_t timeoutNanoseconds) = 0;
virtual void wakeupEventLoop() = 0;
+ virtual bool supportsDepth24Stencil8PixelFormat() = 0;
+
};
std::shared_ptr<Device> createSystemDefaultDevice();
diff --git a/include/indium/library.hpp b/include/indium/library.hpp
index 70f2b28..2c2e579 100644
--- a/include/indium/library.hpp
+++ b/include/indium/library.hpp
@@ -1,6 +1,8 @@
#pragma once
#include <indium/base.hpp>
+#include <indium/types.hpp>
+#include <iridium/iridium.hpp>
#include <memory>
#include <string>
@@ -17,6 +19,11 @@ namespace Indium {
virtual std::shared_ptr<Device> device() = 0;
INDIUM_PROPERTY_READONLY(std::string, n, N,ame);
+
+ public:
+ // Shader argument metadata, used to build Metal pipeline reflection
+ virtual const std::vector<Iridium::ArgumentInfo>& argumentInfo() const = 0;
+ virtual FunctionType functionType() const = 0;
};
class Library {
diff --git a/include/indium/render-command-encoder.hpp b/include/indium/render-command-encoder.hpp
index d17993c..0458265 100644
--- a/include/indium/render-command-encoder.hpp
+++ b/include/indium/render-command-encoder.hpp
@@ -19,7 +19,16 @@ namespace Indium {
public:
virtual ~RenderCommandEncoder() = 0;
- virtual void setRenderPipelineState(std::shared_ptr<RenderPipelineState> renderPipelineState) = 0;
+ // (darling-lol:) The resource parameters are passed by const reference rather than by
+ // value: a by-value std::shared_ptr parameter makes the *caller* copy the pointer (an
+ // atomic increment plus the temporary's increment/decrement pair) and hands the callee a
+ // private address to copy out of. Every implementation here copies the pointer once into
+ // the table or op record that has to keep the resource alive anyway, so the reference
+ // removes one atomic round trip and one store-to-load chain per forwarded bind. These
+ // signatures are part of the Metal<->Indium pair (both are built together and installed
+ // together, see patches/metal.patch and patches/metal--deps--indium.patch), and no other
+ // runtime module calls a render encoder.
+ virtual void setRenderPipelineState(const std::shared_ptr<RenderPipelineState>& renderPipelineState) = 0;
virtual void setFrontFacingWinding(Winding frontFaceWinding) = 0;
virtual void setCullMode(CullMode cullMode) = 0;
virtual void setDepthBias(float depthBias, float slopeScale, float clamp) = 0;
@@ -31,7 +40,7 @@ namespace Indium {
virtual void setScissorRects(const ScissorRect* scissorRects, size_t count) = 0;
virtual void setScissorRects(const std::vector<ScissorRect>& scissorRects) = 0;
virtual void setBlendColor(float red, float green, float blue, float alpha) = 0;
- virtual void setDepthStencilState(std::shared_ptr<DepthStencilState> state) = 0;
+ virtual void setDepthStencilState(const std::shared_ptr<DepthStencilState>& state) = 0;
virtual void setTriangleFillMode(TriangleFillMode triangleFillMode) = 0;
virtual void setStencilReferenceValue(uint32_t value) = 0;
virtual void setStencilReferenceValue(uint32_t front, uint32_t back) = 0;
@@ -41,35 +50,37 @@ namespace Indium {
virtual void drawPrimitives(PrimitiveType primitiveType, size_t vertexStart, size_t vertexCount, size_t instanceCount) = 0;
virtual void drawPrimitives(PrimitiveType primitiveType, size_t vertexStart, size_t vertexCount) = 0;
- virtual void drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, std::shared_ptr<Buffer> indexBuffer, size_t indexBufferOffset, size_t instanceCount, int64_t baseVertex, size_t baseInstance) = 0;
- virtual void drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, std::shared_ptr<Buffer> indexBuffer, size_t indexBufferOffset, size_t instanceCount) = 0;
- virtual void drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, std::shared_ptr<Buffer> indexBuffer, size_t indexBufferOffset) = 0;
+ virtual void drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, const std::shared_ptr<Buffer>& indexBuffer, size_t indexBufferOffset, size_t instanceCount, int64_t baseVertex, size_t baseInstance) = 0;
+ virtual void drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, const std::shared_ptr<Buffer>& indexBuffer, size_t indexBufferOffset, size_t instanceCount) = 0;
+ virtual void drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, const std::shared_ptr<Buffer>& indexBuffer, size_t indexBufferOffset) = 0;
virtual void setVertexBytes(const void* bytes, size_t length, size_t index) = 0;
- virtual void setVertexBuffer(std::shared_ptr<Buffer> buffer, size_t offset, size_t index) = 0;
+ virtual void setVertexBuffer(const std::shared_ptr<Buffer>& buffer, size_t offset, size_t index) = 0;
virtual void setVertexBuffers(const std::vector<std::shared_ptr<Buffer>>& buffers, const std::vector<size_t>& offsets, Range<size_t> range) = 0;
virtual void setVertexBufferOffset(size_t offset, size_t index) = 0;
- virtual void setVertexSamplerState(std::shared_ptr<SamplerState> state, size_t index) = 0;
- virtual void setVertexSamplerState(std::shared_ptr<SamplerState> state, float lodMinClamp, float lodMaxClamp, size_t index) = 0;
+ virtual void setVertexSamplerState(const std::shared_ptr<SamplerState>& state, size_t index) = 0;
+ virtual void setVertexSamplerState(const std::shared_ptr<SamplerState>& state, float lodMinClamp, float lodMaxClamp, size_t index) = 0;
virtual void setVertexSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& states, Range<size_t> range) = 0;
virtual void setVertexSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& states, const std::vector<float>& lodMinClamps, const std::vector<float>& lodMaxClamps, Range<size_t> range) = 0;
- virtual void setVertexTexture(std::shared_ptr<Texture> texture, size_t index) = 0;
+ virtual void setVertexTexture(const std::shared_ptr<Texture>& texture, size_t index) = 0;
virtual void setVertexTextures(const std::vector<std::shared_ptr<Texture>>& textures, Range<size_t> range) = 0;
virtual void setFragmentBytes(const void* bytes, size_t length, size_t index) = 0;
- virtual void setFragmentBuffer(std::shared_ptr<Buffer> buffer, size_t offset, size_t index) = 0;
+ virtual void setFragmentBuffer(const std::shared_ptr<Buffer>& buffer, size_t offset, size_t index) = 0;
virtual void setFragmentBuffers(const std::vector<std::shared_ptr<Buffer>>& buffers, const std::vector<size_t>& offsets, Range<size_t> range) = 0;
virtual void setFragmentBufferOffset(size_t offset, size_t index) = 0;
- virtual void setFragmentSamplerState(std::shared_ptr<SamplerState> state, size_t index) = 0;
- virtual void setFragmentSamplerState(std::shared_ptr<SamplerState> state, float lodMinClamp, float lodMaxClamp, size_t index) = 0;
+ virtual void setFragmentSamplerState(const std::shared_ptr<SamplerState>& state, size_t index) = 0;
+ virtual void setFragmentSamplerState(const std::shared_ptr<SamplerState>& state, float lodMinClamp, float lodMaxClamp, size_t index) = 0;
virtual void setFragmentSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& states, Range<size_t> range) = 0;
virtual void setFragmentSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& states, const std::vector<float>& lodMinClamps, const std::vector<float>& lodMaxClamps, Range<size_t> range) = 0;
- virtual void setFragmentTexture(std::shared_ptr<Texture> texture, size_t index) = 0;
+ virtual void setFragmentTexture(const std::shared_ptr<Texture>& texture, size_t index) = 0;
virtual void setFragmentTextures(std::vector<std::shared_ptr<Texture>>& textures, Range<size_t> range) = 0;
virtual void useResource(std::shared_ptr<Resource> resource, ResourceUsage usage, RenderStages stages) = 0;
virtual void useResources(const std::vector<std::shared_ptr<Resource>>& resources, ResourceUsage usage, RenderStages stages) = 0;
virtual void useResource(std::shared_ptr<Resource> resource, ResourceUsage usage) = 0;
virtual void useResources(const std::vector<std::shared_ptr<Resource>>& resources, ResourceUsage usage) = 0;
+ virtual void memoryBarrier(size_t scope, RenderStages after, RenderStages before) = 0;
+
};
};
diff --git a/include/iridium/iridium.hpp b/include/iridium/iridium.hpp
index 900d03a..ae5eff1 100644
--- a/include/iridium/iridium.hpp
+++ b/include/iridium/iridium.hpp
@@ -1,6 +1,7 @@
#pragma once
#include <cstddef>
+#include <cstdint>
#include <vector>
#include <unordered_map>
@@ -90,9 +91,25 @@ namespace Iridium {
float lodMax;
};
+
+ struct StructMemberInfo {
+ size_t offset;
+ std::string typeName;
+ std::string name;
+ };
+ struct ArgumentInfo {
+ BindingType type;
+ size_t index = 0;
+ std::string name;
+ std::string typeName;
+ size_t size = 0;
+ size_t alignment = 0;
+ std::vector<StructMemberInfo> members;
+ };
struct FunctionInfo {
FunctionType type;
std::vector<BindingInfo> bindings;
+ std::vector<ArgumentInfo> arguments;
std::vector<EmbeddedSampler> embeddedSamplers;
};
diff --git a/private-include/indium/blit-command-encoder.private.hpp b/private-include/indium/blit-command-encoder.private.hpp
index d6e71b2..ae31d39 100644
--- a/private-include/indium/blit-command-encoder.private.hpp
+++ b/private-include/indium/blit-command-encoder.private.hpp
@@ -19,21 +19,28 @@ namespace Indium {
std::shared_ptr<PrivateDevice> _privateDevice;
BlitPassDescriptor _descriptor;
- void copy(std::shared_ptr<Texture> source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, std::shared_ptr<Texture> destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin, size_t sliceCount, size_t levelCount, Size size);
+ // destination textures that need presentation synchronization (drawables)
+ std::vector<std::shared_ptr<Texture>> _readWriteTextures;
+
+ void noteDestination(const std::shared_ptr<Texture>& destination);
+
+ void copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin, size_t sliceCount, size_t levelCount, Size size);
public:
PrivateBlitCommandEncoder(std::shared_ptr<PrivateCommandBuffer> commandBuffer);
PrivateBlitCommandEncoder(std::shared_ptr<PrivateCommandBuffer> commandBuffer, const BlitPassDescriptor& descriptor);
~PrivateBlitCommandEncoder();
- virtual void copy(std::shared_ptr<Buffer> source, size_t sourceOffset, std::shared_ptr<Buffer> destination, size_t destinationOffset, size_t size) override;
- virtual void copy(std::shared_ptr<Buffer> source, size_t sourceOffset, size_t sourceBytesPerRow, size_t sourceBytesPerImage, Size sourceSize, std::shared_ptr<Texture> destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin, BlitOption options = BlitOption::None) override;
- virtual void copy(std::shared_ptr<Texture> source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, std::shared_ptr<Buffer> destination, size_t destinationOffset, size_t destinationBytesPerRow, size_t destinationBytesPerImage, BlitOption options = BlitOption::None) override;
- virtual void copy(std::shared_ptr<Texture> source, std::shared_ptr<Texture> destination) override;
- virtual void copy(std::shared_ptr<Texture> source, size_t sourceSlice, size_t sourceLevel, std::shared_ptr<Texture> destination, size_t destinationSlice, size_t destinationLevel, size_t sliceCount, size_t levelCount) override;
- virtual void copy(std::shared_ptr<Texture> source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, std::shared_ptr<Texture> destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin) override;
- virtual void fillBuffer(std::shared_ptr<Buffer> buffer, Range<size_t> range, uint8_t value) override;
- virtual void generateMipmapsForTexture(std::shared_ptr<Texture> texture) override;
+ const std::vector<std::shared_ptr<Texture>>& readWriteTextures() const { return _readWriteTextures; };
+
+ virtual void copy(const std::shared_ptr<Buffer>& source, size_t sourceOffset, const std::shared_ptr<Buffer>& destination, size_t destinationOffset, size_t size) override;
+ virtual void copy(const std::shared_ptr<Buffer>& source, size_t sourceOffset, size_t sourceBytesPerRow, size_t sourceBytesPerImage, Size sourceSize, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin, BlitOption options = BlitOption::None) override;
+ virtual void copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, const std::shared_ptr<Buffer>& destination, size_t destinationOffset, size_t destinationBytesPerRow, size_t destinationBytesPerImage, BlitOption options = BlitOption::None) override;
+ virtual void copy(const std::shared_ptr<Texture>& source, const std::shared_ptr<Texture>& destination) override;
+ virtual void copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, size_t sliceCount, size_t levelCount) override;
+ virtual void copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin) override;
+ virtual void fillBuffer(const std::shared_ptr<Buffer>& buffer, Range<size_t> range, uint8_t value) override;
+ virtual void generateMipmapsForTexture(const std::shared_ptr<Texture>& texture) override;
virtual void endEncoding() override;
};
diff --git a/private-include/indium/buffer.private.hpp b/private-include/indium/buffer.private.hpp
index a348b2b..9ad8975 100644
--- a/private-include/indium/buffer.private.hpp
+++ b/private-include/indium/buffer.private.hpp
@@ -29,5 +29,10 @@ namespace Indium {
INDIUM_PROPERTY(VkBuffer, b, B,uffer) = VK_NULL_HANDLE;
INDIUM_PROPERTY(VkDeviceMemory, m, M,emory) = VK_NULL_HANDLE;
+
+ private:
+ // appended last: the device address of a buffer never changes, and it was asked from the driver for every buffer
+ // binding of every draw
+ uint64_t _gpuAddress = 0;
};
};
diff --git a/private-include/indium/command-buffer.private.hpp b/private-include/indium/command-buffer.private.hpp
index b7ffcce..5b1e84c 100644
--- a/private-include/indium/command-buffer.private.hpp
+++ b/private-include/indium/command-buffer.private.hpp
@@ -2,16 +2,22 @@
#include <vulkan/vulkan.h>
+#include <indium/blit-command-encoder.hpp>
#include <indium/command-buffer.hpp>
#include <indium/command-encoder.hpp>
+#include <indium/compute-command-encoder.hpp>
#include <vector>
#include <mutex>
#include <condition_variable>
+#include <atomic>
namespace Indium {
class PrivateCommandQueue;
class PrivateDevice;
+ class PrivateRenderCommandEncoder;
+ class PrivateBlitCommandEncoder;
+ class PrivateComputeCommandEncoder;
class PrivateCommandBuffer: public CommandBuffer, public std::enable_shared_from_this<PrivateCommandBuffer> {
private:
@@ -25,7 +31,6 @@ namespace Indium {
bool _committed = false;
std::condition_variable _completedCondvar;
bool _completed = false;
-
public:
PrivateCommandBuffer(std::shared_ptr<PrivateCommandQueue> commandQueue);
~PrivateCommandBuffer();
@@ -46,6 +51,25 @@ namespace Indium {
virtual std::shared_ptr<CommandQueue> commandQueue() override;
virtual std::shared_ptr<Device> device() override;
+ // creates the real render encoder immediately (what renderCommandEncoder() did before deferred encoding existed)
+ std::shared_ptr<PrivateRenderCommandEncoder> renderCommandEncoderNow(const RenderPassDescriptor& descriptor);
+ // same for blit/compute: the real encoder is created on the worker that replays the deferred op list
+ std::shared_ptr<PrivateBlitCommandEncoder> blitCommandEncoderNow();
+ std::shared_ptr<PrivateBlitCommandEncoder> blitCommandEncoderNow(const BlitPassDescriptor& descriptor);
+ std::shared_ptr<PrivateComputeCommandEncoder> computeCommandEncoderNow(const ComputePassDescriptor& descriptor);
+ // leases the Vulkan command buffer (pool, allocation, begin) on the calling thread if that has not happened yet;
+ // only used with /tmp/metal-worker-lease (by default the constructor leases it, as in round 3)
+ void ensureRecordingStarted();
+ void leaseRecording();
+ void deferredBegin();
+ void deferredEnd();
+ // blocks until every deferred encoder of this command buffer has been replayed
+ void waitDeferred();
+ // false while the commit has not started. A deferred replay may only be posted while the Vulkan command buffer
+ // is still open; a facade dropped after the commit (a Metal contract violation) then logs instead of recording
+ // into a closed command buffer.
+ bool replayAllowed() const { return !_committedAtomic.load(std::memory_order_acquire); }
+
void addScheduledHandlerLocked(std::function<void(std::shared_ptr<CommandBuffer>)> handler);
void addCompletedHandlerLocked(std::function<void(std::shared_ptr<CommandBuffer>)> handler);
@@ -53,5 +77,15 @@ namespace Indium {
INDIUM_PROPERTY_READONLY_OBJECT(PrivateDevice, p, P,rivateDevice);
INDIUM_PROPERTY(VkCommandBuffer, c, C,ommandBuffer) = VK_NULL_HANDLE;
+ private:
+ VkCommandPool _recordingPool = VK_NULL_HANDLE;
+ // Appended LAST on purpose: QuartzCore and Metal are compiled against the earlier layout of this class and read
+ // commandBuffer() and friends by offset, so no existing member may move.
+ // deferred (asynchronously replayed) render encoders that have ended but are not applied to _commandBuffer yet
+ unsigned _deferredPending = 0;
+ std::mutex _deferredMutex;
+ std::condition_variable _deferredCondvar;
+ // set at the start of the commit body (see replayAllowed()); appended last, like the members above
+ std::atomic<bool> _committedAtomic {false};
};
};
diff --git a/private-include/indium/command-encoder.private.hpp b/private-include/indium/command-encoder.private.hpp
index c3b1b93..3dab2ce 100644
--- a/private-include/indium/command-encoder.private.hpp
+++ b/private-include/indium/command-encoder.private.hpp
@@ -1,6 +1,7 @@
#pragma once
#include <vector>
+#include <algorithm>
#include <memory>
#include <optional>
#include <array>
@@ -17,117 +18,295 @@
#include <iridium/iridium.hpp>
+#include <cstring>
+#include <cstdio>
+#include <atomic>
+
namespace Indium {
+ bool bindingReuseEnabled();
+ // Metal's tracked resources carry writes from one encoder into the next.
+ // Queue submission order alone supplies execution order, not Vulkan memory
+ // visibility for attachment, shader, and transfer accesses.
+ inline void finishTrackedEncoder(VkCommandBuffer commandBuffer) {
+ VkMemoryBarrier barrier {};
+ barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
+ barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
+ barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
+ DynamicVK::vkCmdPipelineBarrier(commandBuffer,
+ VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
+ 0, 1, &barrier, 0, nullptr, 0, nullptr);
+ }
+ // First few times the app leaves a bound-by-the-shader resource unset: tells us which slots to look at.
+ inline void logUnboundResource(const char* kind, size_t binding, size_t index, size_t available) {
+ static std::atomic<unsigned> logged {0};
+ if (logged.fetch_add(1) >= 30) {
+ return;
+ }
+ if (FILE* file = fopen("/tmp/metal-trace.log", "a")) {
+ fprintf(file, "unbound-%s: binding=%zu index=%zu (have %zu)\n", kind, binding, index, available);
+ fclose(file);
+ }
+ }
+
+ // Descriptor writes are consumed synchronously by vkUpdateDescriptorSets.
+ // Keep common shader layouts on the stack; retain a sized heap fallback
+ // for shaders exceeding the inline capacity. Never move storage after
+ // pointers to descriptor infos have been published.
+ template<class T, size_t capacity = 64> class DescriptorScratch {
+ std::array<T, capacity> _inline;
+ std::vector<T> _large;
+ T* _data;
+ size_t _size = 0;
+ public:
+ explicit DescriptorScratch(size_t maximum) {
+ if (maximum > capacity) _large.resize(maximum);
+ _data = maximum > capacity ? _large.data() : _inline.data();
+ }
+ T& emplace_back() { T& value = _data[_size++]; value = T{}; return value; }
+ void push_back(const T& value) { _data[_size++] = value; }
+ void pop_back() { --_size; }
+ T* data() { return _data; }
+ size_t size() const { return _size; }
+ };
+
+ // Per-encoder binding tables are indexed by Metal slot and start empty in every render pass. Growing them one slot at
+ // a time reallocated (and moved) them for each new highest index, through the application's own operator new, which
+ // League replaces with an allocator that takes a mutex. Grow straight to a size that covers the usual slots.
+ template<class Vector> inline void growToIndex(Vector& vector, size_t index) {
+ if (vector.size() > index) return;
+ if (vector.capacity() <= index) vector.reserve(index < 16 ? 16 : index + 1);
+ vector.resize(index + 1);
+ }
+
+ template<class Base, class Private> struct ResolvedResource {
+ Base* object = nullptr;
+ Private* value = nullptr;
+ bool depthFeedback = false;
+ bool refresh(const std::shared_ptr<Base>& owner) {
+ if (object == owner.get()) return false;
+ object = owner.get();
+ value = dynamic_cast<Private*>(object);
+ return true;
+ }
+ };
+ struct ResolvedFunctionResources {
+ std::vector<ResolvedResource<Buffer, PrivateBuffer>> buffers;
+ std::vector<ResolvedResource<Texture, PrivateTexture>> textures;
+ std::vector<ResolvedResource<SamplerState, PrivateSamplerState>> samplers;
+ };
+
+ template<class T> inline bool sameResourceOwner(const std::shared_ptr<T>& a, const std::shared_ptr<T>& b) {
+ return a.get() == b.get() && !a.owner_before(b) && !b.owner_before(a);
+ }
struct FunctionResources {
std::vector<std::pair<std::shared_ptr<Buffer>, size_t>> buffers;
std::vector<std::shared_ptr<Texture>> textures;
std::vector<std::shared_ptr<SamplerState>> samplers;
+ uint64_t revision = 0;
- void setBytes(std::shared_ptr<Device> device, const void* bytes, size_t length, size_t index) {
- // TODO: we can make this "Private" instead
- auto buf = device->newBuffer(bytes, length, ResourceOptions::StorageModeShared);
+ // Small inline constants (setBytes) are bump-allocated from shared chunks: a dedicated buffer per call meant one
+ // GPU memory allocation (and a kernel address-space update) per draw. Copies of this struct share the chunk, which
+ // keeps it alive for as long as the command buffer needs it; allocations never overlap.
+ std::shared_ptr<Buffer> byteChunk;
+ size_t byteChunkUsed = 0;
+ // bumped only by texture and sampler changes: most draws change buffers or offsets and nothing else
+ uint64_t imageRevision = 0;
- if (buffers.size() <= index) {
- buffers.resize(index + 1);
+ void setBytes(std::shared_ptr<Device> device, const void* bytes, size_t length, size_t index) {
+ constexpr size_t chunkSize = 256 * 1024;
+ constexpr size_t alignment = 256;
+
+ // setBytes storage is private and immutable until GPU retirement.
+ // Repeated identical constants can keep their existing address.
+ if (bindingReuseEnabled() && byteChunk && index < buffers.size() && buffers[index].first.get() == byteChunk.get() &&
+ length <= 4096 && buffers[index].second <= byteChunkUsed &&
+ length <= byteChunkUsed - buffers[index].second &&
+ (!length || !memcmp(static_cast<const char*>(byteChunk->contents()) + buffers[index].second, bytes, length)))
+ return;
+
+ std::shared_ptr<Buffer> buf;
+ size_t offset = 0;
+
+ if (length > chunkSize / 8) {
+ // TODO: we can make this "Private" instead
+ buf = device->newBuffer(bytes, length, ResourceOptions::StorageModeShared);
+ } else {
+ offset = (byteChunkUsed + alignment - 1) & ~(alignment - 1);
+ if (!byteChunk || offset + length > chunkSize) {
+ byteChunk = std::dynamic_pointer_cast<PrivateDevice>(device)->acquireScratchChunk();
+ offset = 0;
+ }
+ memcpy(static_cast<char*>(byteChunk->contents()) + offset, bytes, length);
+ byteChunkUsed = offset + length;
+ buf = byteChunk;
}
- buffers[index] = std::make_pair(buf, 0);
+ growToIndex(buffers, index);
+
+ buffers[index] = std::make_pair(std::move(buf), offset);
+ ++revision;
};
- void setBuffer(std::shared_ptr<Buffer> buffer, size_t offset, size_t index) {
- if (buffers.size() <= index) {
- buffers.resize(index + 1);
- }
+ void setBuffer(const std::shared_ptr<Buffer>& buffer, size_t offset, size_t index) {
+ growToIndex(buffers, index);
- buffers[index] = std::make_pair(buffer, offset);
+ const bool ownerChanged = !sameResourceOwner(buffers[index].first, buffer);
+ if (ownerChanged || buffers[index].second != offset) ++revision;
+ if (ownerChanged) buffers[index].first = buffer;
+ buffers[index].second = offset;
};
void setBufferOffset(size_t offset, size_t index) {
+ if (index >= buffers.size()) {
+ // Metal requires the buffer to be bound first; this used to write past the end of the vector
+ logUnboundResource("buffer-offset", index, index, buffers.size());
+ return;
+ }
+ if (buffers[index].second != offset) ++revision;
buffers[index].second = offset;
};
- void setSamplerState(std::shared_ptr<SamplerState> state, std::optional<std::pair<float, float>> lodClamps, size_t index) {
- if (samplers.size() <= index) {
- samplers.resize(index + 1);
- }
+ void setSamplerState(const std::shared_ptr<SamplerState>& state, std::optional<std::pair<float, float>> lodClamps, size_t index) {
+ growToIndex(samplers, index);
if (lodClamps) {
auto privateState = std::dynamic_pointer_cast<PrivateSamplerState>(state);
samplers[index] = privateState->cloneWithClamps(lodClamps->first, lodClamps->second);
+ ++revision; ++imageRevision;
} else {
- samplers[index] = state;
+ if (!sameResourceOwner(samplers[index], state)) { samplers[index] = state; ++revision; ++imageRevision; }
}
};
- void setTexture(std::shared_ptr<Texture> texture, size_t index) {
- if (textures.size() <= index) {
- textures.resize(index + 1);
- }
+ void setTexture(const std::shared_ptr<Texture>& texture, size_t index) {
+ growToIndex(textures, index);
- textures[index] = texture;
+ if (!sameResourceOwner(textures[index], texture)) { textures[index] = texture; ++revision; ++imageRevision; }
};
};
- template<size_t setCount>
- std::array<VkDescriptorSet, setCount> createDescriptorSets(std::array<VkDescriptorSetLayout, setCount>& setLayouts, VkDescriptorPool pool, std::shared_ptr<PrivateDevice> privateDevice, const std::array<std::reference_wrapper<const FunctionResources>, setCount>& funcResources, const std::array<std::reference_wrapper<const FunctionInfo>, setCount>& functionInfos, std::vector<std::shared_ptr<Buffer>>& keepAliveBuffers) {
- std::array<VkDescriptorSet, setCount> descriptorSets {};
-
- VkDescriptorSetAllocateInfo setAllocateInfo {};
- setAllocateInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO;
- setAllocateInfo.descriptorPool = pool;
- setAllocateInfo.descriptorSetCount = descriptorSets.size();
- setAllocateInfo.pSetLayouts = setLayouts.data();
-
- if (DynamicVK::vkAllocateDescriptorSets(privateDevice->device(), &setAllocateInfo, descriptorSets.data()) != VK_SUCCESS) {
- // TODO
- abort();
- }
+ static constexpr std::array<VkDescriptorPoolSize, 5> poolSizes {
+ VkDescriptorPoolSize { VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 512 },
+ VkDescriptorPoolSize { VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC, 512 },
+ VkDescriptorPoolSize { VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 512 },
+ VkDescriptorPoolSize { VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 512 },
+ VkDescriptorPoolSize { VK_DESCRIPTOR_TYPE_SAMPLER, 512 },
+ };
- for (size_t i = 0; i < descriptorSets.size(); ++i) {
- std::vector<VkWriteDescriptorSet> writeDescSet;
- std::forward_list<VkDescriptorBufferInfo> bufInfos;
- std::forward_list<VkDescriptorImageInfo> imageInfos;
+ static inline VkDescriptorPool newDescriptorPool(std::shared_ptr<PrivateDevice> device) {
+ return device->acquireDescriptorPool();
+ }
- const FunctionResources& functionResources = funcResources[i];
- const FunctionInfo& funcInfo = functionInfos[i];
+ // Bump allocator for the small per-draw tables of buffer addresses (one dedicated buffer per table meant one GPU memory
+ // allocation per stage per draw).
+ struct AddressPool {
+ std::shared_ptr<Buffer> chunk;
+ size_t used = 0;
+ };
- if (functionResources.buffers.size() > 0) {
- std::vector<uint64_t> addresses;
+ struct BufferAddressTable {
+ VkBuffer buffer = VK_NULL_HANDLE;
+ uint32_t offset = 0;
+ VkDeviceSize range = 0;
+ };
+ inline BufferAddressTable makeBufferAddressTable(const FunctionResources& functionResources, const FunctionInfo& functionInfo,
+ const ResolvedFunctionResources* resolved, const std::shared_ptr<PrivateDevice>& privateDevice,
+ std::vector<std::shared_ptr<Buffer>>& keepAliveBuffers, AddressPool& addressPool,
+ const BufferAddressTable* previous = nullptr) {
+ DescriptorScratch<uint64_t> addresses(functionInfo.bindings.size());
// find the right buffer for each binding (using the binding index)
- for (size_t j = 0; j < funcInfo.bindings.size(); ++j) {
- auto& bindingInfo = funcInfo.bindings[j];
+ for (size_t j = 0; j < functionInfo.bindings.size(); ++j) {
+ auto& bindingInfo = functionInfo.bindings[j];
if (bindingInfo.type != Iridium::BindingType::Buffer) {
continue;
}
- if (bindingInfo.index >= functionResources.buffers.size()) {
- addresses.push_back(0);
+ PrivateBuffer* privateBuf = nullptr;
+ if (resolved && bindingInfo.index < resolved->buffers.size())
+ privateBuf = resolved->buffers[bindingInfo.index].value;
+ else if (bindingInfo.index < functionResources.buffers.size())
+ privateBuf = dynamic_cast<PrivateBuffer*>(functionResources.buffers[bindingInfo.index].first.get());
+ if (!privateBuf) {
+ // address 0 here was dereferenced by the shader: GPU page fault at 0x0, ring timeout, GPU reset
+ logUnboundResource("buffer", j, bindingInfo.index, functionResources.buffers.size());
+ addresses.push_back(privateDevice->fallbackBufferAddress());
continue;
}
-
- auto privateBuf = std::dynamic_pointer_cast<PrivateBuffer>(functionResources.buffers[bindingInfo.index].first);
addresses.push_back(privateBuf->gpuAddress() + functionResources.buffers[bindingInfo.index].second);
}
- auto addressBuffer = privateDevice->newBuffer(addresses.data(), functionResources.buffers.size() * 8, ResourceOptions::StorageModeShared);
- auto privateAddrBuf = std::dynamic_pointer_cast<PrivateBuffer>(addressBuffer);
+ constexpr size_t chunkSize = 256 * 1024;
+ constexpr size_t alignment = 256;
+ const size_t tableSize = (addresses.size() * sizeof(uint64_t) + 15) & ~size_t(15);
+ // Reuse immutable table bytes when setters changed unused slots
+ // or selected an equivalent GPU address. Do not retain storage
+ // from another chunk: its lifetime belongs to an earlier table.
+ if (bindingReuseEnabled() && previous && addressPool.chunk && previous->range == tableSize &&
+ previous->offset <= chunkSize && tableSize <= chunkSize - previous->offset &&
+ previous->buffer == static_cast<PrivateBuffer*>(addressPool.chunk.get())->buffer() &&
+ !memcmp(static_cast<const char*>(addressPool.chunk->contents()) + previous->offset,
+ addresses.data(), addresses.size() * sizeof(uint64_t)))
+ return *previous;
+ size_t tableOffset = (addressPool.used + alignment - 1) & ~(alignment - 1);
+ if (!addressPool.chunk || tableOffset + tableSize > chunkSize) {
+ addressPool.chunk = privateDevice->acquireScratchChunk();
+ // we need to keep the chunk alive until the operation is completed
+ keepAliveBuffers.push_back(addressPool.chunk);
+ tableOffset = 0;
+ }
+ memcpy(static_cast<char*>(addressPool.chunk->contents()) + tableOffset, addresses.data(), addresses.size() * sizeof(uint64_t));
+ addressPool.used = tableOffset + tableSize;
+ // plain pointer: a shared_ptr cast here was an atomic reference count round trip per stage per draw
+ auto* privateAddressBuffer = static_cast<PrivateBuffer*>(addressPool.chunk.get());
- // we need to keep this buffer alive until the operation is completed
- keepAliveBuffers.push_back(addressBuffer);
+ return {privateAddressBuffer->buffer(), static_cast<uint32_t>(tableOffset), tableSize};
+ }
- auto& info = bufInfos.emplace_front();
- info.buffer = privateAddrBuf->buffer();
- info.offset = 0;
- info.range = VK_WHOLE_SIZE;
+ template<size_t setCount>
+ std::array<VkDescriptorSet, setCount> createDescriptorSets(std::array<VkDescriptorSetLayout, setCount>& setLayouts, VkDescriptorPool& pool, std::vector<VkDescriptorPool>& retiredPools, std::shared_ptr<PrivateDevice> privateDevice, const std::array<std::reference_wrapper<const FunctionResources>, setCount>& funcResources, const std::array<std::reference_wrapper<const FunctionInfo>, setCount>& functionInfos, std::vector<std::shared_ptr<Buffer>>& keepAliveBuffers, AddressPool& addressPool, const std::array<const ResolvedFunctionResources*, setCount>& resolved = {}, const std::array<const BufferAddressTable*, setCount>& dynamicTables = {}) {
+ std::array<VkDescriptorSet, setCount> descriptorSets {};
+
+ VkDescriptorSetAllocateInfo setAllocateInfo {};
+ setAllocateInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO;
+ setAllocateInfo.descriptorPool = pool;
+ setAllocateInfo.descriptorSetCount = descriptorSets.size();
+ setAllocateInfo.pSetLayouts = setLayouts.data();
+
+ auto result = DynamicVK::vkAllocateDescriptorSets(privateDevice->device(), &setAllocateInfo, descriptorSets.data());
+ if (result == VK_ERROR_OUT_OF_POOL_MEMORY || result == VK_ERROR_FRAGMENTED_POOL) {
+ // Keep previous sets alive until this encoder's GPU work completes.
+ auto next = newDescriptorPool(privateDevice);
+ retiredPools.push_back(pool);
+ pool = next;
+ setAllocateInfo.descriptorPool = pool;
+ result = DynamicVK::vkAllocateDescriptorSets(privateDevice->device(), &setAllocateInfo, descriptorSets.data());
+ }
+ if (result != VK_SUCCESS) throw std::runtime_error("Descriptor set allocation failed: " + std::to_string(result));
+
+
+ for (size_t i = 0; i < descriptorSets.size(); ++i) {
+ const FunctionResources& functionResources = funcResources[i];
+ const FunctionInfo& funcInfo = functionInfos[i];
+ DescriptorScratch<VkWriteDescriptorSet> writeDescSet(funcInfo.bindings.size() + 1);
+ VkDescriptorBufferInfo bufferInfo {};
+ DescriptorScratch<VkDescriptorImageInfo> imageInfos(funcInfo.bindings.size());
+
+ if (std::any_of(funcInfo.bindings.begin(), funcInfo.bindings.end(), [](const auto& binding) { return binding.type == Iridium::BindingType::Buffer; })) {
+ const auto table = dynamicTables[i] ? *dynamicTables[i] :
+ makeBufferAddressTable(functionResources, funcInfo, resolved[i], privateDevice, keepAliveBuffers, addressPool);
+ auto& info = bufferInfo;
+ info.buffer = table.buffer;
+ info.offset = dynamicTables[i] ? 0 : table.offset;
+ info.range = table.range;
auto& descSet = writeDescSet.emplace_back();
descSet.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
descSet.dstSet = descriptorSets[i];
descSet.dstBinding = 0;
descSet.dstArrayElement = 0;
- descSet.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER;
+ descSet.descriptorType = dynamicTables[i] ? VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC : VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER;
descSet.descriptorCount = 1;
descSet.pBufferInfo = &info;
}
@@ -136,17 +315,27 @@ namespace Indium {
auto& bindingInfo = funcInfo.bindings[j];
if (bindingInfo.type == Iridium::BindingType::Texture) {
- if (bindingInfo.index >= functionResources.textures.size()) {
- continue;
+ PrivateTexture* privateTexture = nullptr;
+ if (resolved[i] && bindingInfo.index < resolved[i]->textures.size())
+ privateTexture = resolved[i]->textures[bindingInfo.index].value;
+ else if (bindingInfo.index < functionResources.textures.size())
+ privateTexture = dynamic_cast<PrivateTexture*>(functionResources.textures[bindingInfo.index].get());
+
+ auto& info = imageInfos.emplace_back();
+ if (privateTexture) {
+ info.imageView = privateTexture->sampledImageView();
+ info.imageLayout = privateTexture->imageLayout();
+ } else {
+ // leaving the descriptor unwritten lets the shader read whatever the set memory held
+ logUnboundResource("texture", j, bindingInfo.index, functionResources.textures.size());
+ if (!privateDevice->nullDescriptorEnabled()) {
+ imageInfos.pop_back();
+ continue;
+ }
+ info.imageView = VK_NULL_HANDLE;
+ info.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
}
- auto texture = functionResources.textures[bindingInfo.index];
- auto privateTexture = std::dynamic_pointer_cast<PrivateTexture>(texture);
-
- auto& info = imageInfos.emplace_front();
- info.imageView = privateTexture->imageView();
- info.imageLayout = privateTexture->imageLayout();
-
auto& descSet = writeDescSet.emplace_back();
descSet.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
descSet.dstSet = descriptorSets[i];
@@ -156,19 +345,21 @@ namespace Indium {
descSet.descriptorCount = 1;
descSet.pImageInfo = &info;
} else if (bindingInfo.type == Iridium::BindingType::Sampler) {
- bool embeddedSampler = false;
-
- if (bindingInfo.index == SIZE_MAX) {
- // this binding uses an embedded sampler
- embeddedSampler = true;
- } else if (bindingInfo.index >= functionResources.samplers.size()) {
- continue;
+ PrivateSamplerState* privateSampler = nullptr;
+ if (bindingInfo.index == SIZE_MAX) {
+ privateSampler = dynamic_cast<PrivateSamplerState*>(funcInfo.embeddedSamplerStates[bindingInfo.embeddedSamplerIndex].get());
+ } else if (resolved[i] && bindingInfo.index < resolved[i]->samplers.size()) {
+ privateSampler = resolved[i]->samplers[bindingInfo.index].value;
+ } else if (bindingInfo.index < functionResources.samplers.size()) {
+ privateSampler = dynamic_cast<PrivateSamplerState*>(functionResources.samplers[bindingInfo.index].get());
+ }
+ if (!privateSampler) {
+ logUnboundResource("sampler", j, bindingInfo.index, functionResources.samplers.size());
+ privateSampler = dynamic_cast<PrivateSamplerState*>(privateDevice->fallbackSampler().get());
+ if (!privateSampler) continue;
}
- auto sampler = embeddedSampler ? funcInfo.embeddedSamplerStates[bindingInfo.embeddedSamplerIndex] : functionResources.samplers[bindingInfo.index];
- auto privateSampler = std::dynamic_pointer_cast<PrivateSamplerState>(sampler);
-
- auto& info = imageInfos.emplace_front();
+ auto& info = imageInfos.emplace_back();
info.sampler = privateSampler->sampler();
auto& descSet = writeDescSet.emplace_back();
@@ -188,10 +379,4 @@ namespace Indium {
return descriptorSets;
};
- static constexpr std::array<VkDescriptorPoolSize, 4> poolSizes {
- VkDescriptorPoolSize { VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 512 },
- VkDescriptorPoolSize { VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 512 },
- VkDescriptorPoolSize { VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 512 },
- VkDescriptorPoolSize { VK_DESCRIPTOR_TYPE_SAMPLER, 512 },
- };
};
diff --git a/private-include/indium/command-queue.private.hpp b/private-include/indium/command-queue.private.hpp
index edfc54e..6d58cf6 100644
--- a/private-include/indium/command-queue.private.hpp
+++ b/private-include/indium/command-queue.private.hpp
@@ -3,6 +3,8 @@
#include <indium/command-queue.hpp>
#include <vulkan/vulkan.h>
+#include <mutex>
+#include <vector>
namespace Indium {
class PrivateDevice;
@@ -22,5 +24,12 @@ namespace Indium {
INDIUM_PROPERTY_READONLY_OBJECT(PrivateDevice, p, P,rivateDevice);
INDIUM_PROPERTY(VkCommandPool, c, C,ommandPool) = VK_NULL_HANDLE;
+ private:
+ std::mutex _poolMutex;
+ struct RecordingPool { VkCommandPool pool; VkCommandBuffer buffer; };
+ std::vector<RecordingPool> _availablePools;
+ public:
+ VkCommandPool acquireCommandPool(VkCommandBuffer& buffer);
+ void releaseCommandPool(VkCommandPool pool, VkCommandBuffer buffer);
};
};
diff --git a/private-include/indium/compute-command-encoder.private.hpp b/private-include/indium/compute-command-encoder.private.hpp
index 8546daa..7e4993c 100644
--- a/private-include/indium/compute-command-encoder.private.hpp
+++ b/private-include/indium/compute-command-encoder.private.hpp
@@ -15,12 +15,14 @@ namespace Indium {
FunctionResources _functionResources;
std::shared_ptr<PrivateComputePipelineState> _pso;
VkDescriptorPool _pool = VK_NULL_HANDLE;
+ std::vector<VkDescriptorPool> _retiredPools;
// XXX: not sure if we actually need to store pipelines until the command buffer finishes executing,
// but let's do it just in case.
std::vector<VkPipeline> _savedPipelines;
std::vector<FunctionResources> _savedFunctionResources;
std::vector<std::shared_ptr<Buffer>> _keepAliveBuffers;
+ AddressPool _addressPool;
void updateBindings();
diff --git a/private-include/indium/device.private.hpp b/private-include/indium/device.private.hpp
index 58f1794..75266fd 100644
--- a/private-include/indium/device.private.hpp
+++ b/private-include/indium/device.private.hpp
@@ -13,6 +13,8 @@
#include <functional>
#include <atomic>
#include <optional>
+#include <unordered_map>
+#include <indium/render-pass-key.private.hpp>
namespace Indium {
class PrivateDevice;
@@ -25,8 +27,11 @@ namespace Indium {
class PrivateDevice: public Device, public std::enable_shared_from_this<PrivateDevice> {
private:
+ std::mutex _scratchMutex;
+ std::vector<std::shared_ptr<Buffer>> _scratchFree;
std::mutex _eventLoopMutex;
std::mutex _pollingMutex;
+ uint64_t _completionSpinNanoseconds = 0;
// these are stored separately (rather than in a single structure)
// so that we can easily copy them individually in the event loop polling function.
@@ -59,6 +64,8 @@ namespace Indium {
}
virtual std::string name() const override;
+ virtual bool supportsDepth24Stencil8PixelFormat() override;
+
virtual std::shared_ptr<CommandQueue> newCommandQueue() override;
virtual std::shared_ptr<RenderPipelineState> newRenderPipelineState(const RenderPipelineDescriptor& descriptor) override;
virtual std::shared_ptr<ComputePipelineState> newComputePipelineState(const ComputePipelineDescriptor& descriptor, PipelineOption options, std::shared_ptr<ComputePipelineReflection> reflection) override;
@@ -73,6 +80,12 @@ namespace Indium {
virtual void pollEvents(uint64_t timeoutNanoseconds) override;
virtual void wakeupEventLoop() override;
+ // device address of a zero-filled 64 KiB buffer; never 0
+ uint64_t fallbackBufferAddress();
+ std::shared_ptr<SamplerState> fallbackSampler();
+ // VK_EXT_robustness2 nullDescriptor was enabled: null image views / vertex buffers are legal
+ bool nullDescriptorEnabled() const { return _nullDescriptor; }
+
void waitForSemaphore(VkSemaphore semaphore, uint64_t targetValue, std::function<void()> callback);
/**
@@ -86,6 +99,11 @@ namespace Indium {
* Similar to getTimelineSemaphore(), but wraps it in a shared pointer
* that automatically releases the semaphore when all references to it die out.
*/
+ // Fixed-size scratch buffers for per-draw constants and address tables. They go back to a free list when the last
+ // reference is dropped (i.e. when the command buffers that used them are done), instead of being freed and re-allocated.
+ static constexpr size_t scratchChunkSize = 256 * 1024;
+ std::shared_ptr<Buffer> acquireScratchChunk();
+
std::shared_ptr<TimelineSemaphore> getWrappedTimelineSemaphore();
BinarySemaphore getBinarySemaphore(bool exportable = false);
@@ -111,5 +129,37 @@ namespace Indium {
INDIUM_PROPERTY(VkPhysicalDeviceMemoryProperties, m, M,emoryProperties);
INDIUM_PROPERTY_READONLY(Feature, f, F,eatures);
+
+ private:
+ // These stay LAST: QuartzCore and Metal are compiled against this layout (they read _device and friends at fixed
+ // offsets), so adding members earlier in the class breaks the installed frameworks.
+ // Stand-ins for resources the app never bound (see createDescriptorSets): shaders read buffers through raw device
+ // addresses, so an unbound index used to become address 0 and faulted the GPU there.
+ std::mutex _fallbackMutex;
+ std::shared_ptr<Buffer> _fallbackBuffer;
+ std::shared_ptr<SamplerState> _fallbackSampler;
+ bool _nullDescriptor = false;
+ std::mutex _graphicsQueueMutex;
+ std::mutex _oneshotCommandPoolMutex;
+ public:
+ std::mutex& graphicsQueueMutex() { return _graphicsQueueMutex; }
+ std::mutex& oneshotCommandPoolMutex() { return _oneshotCommandPoolMutex; }
+ VkDescriptorPool acquireDescriptorPool();
+ void releaseDescriptorPool(VkDescriptorPool pool);
+ // Cached handles belong to the device and remain alive through all GPU uses.
+ std::pair<VkRenderPass, bool> acquireRenderPass(const VkRenderPassCreateInfo& info);
+ // Memory type index for a buffer or image of the given storage mode, or SIZE_MAX. Non-virtual and without
+ // state in the class, so the layout above is unchanged.
+ size_t findMemoryType(uint32_t memoryTypeBits, StorageMode storageMode, bool image);
+ private:
+ std::mutex _descriptorPoolMutex;
+ std::vector<VkDescriptorPool> _availableDescriptorPools;
+ std::mutex _timelineSemaphorePoolMutex;
+ std::vector<std::pair<VkSemaphore, uint64_t>> _availableTimelineSemaphores;
+ std::mutex _renderPassMutex;
+ std::unordered_map<RenderPassKey, VkRenderPass, RenderPassKey::Hash> _renderPasses;
+ // Sequentially consistent handshake: either re-snapshot or signal.
+ std::atomic<uint64_t> _eventLoopRegistrations {0};
+ std::atomic<unsigned> _eventLoopBlockedPollers {0};
};
};
diff --git a/private-include/indium/dynamic-vk.hpp b/private-include/indium/dynamic-vk.hpp
index 4957eca..fb2a315 100644
--- a/private-include/indium/dynamic-vk.hpp
+++ b/private-include/indium/dynamic-vk.hpp
@@ -6,6 +6,9 @@
#include <indium/instance.private.hpp>
#include <mutex>
+#include <atomic>
+#include <stdexcept>
+#include <string>
namespace Indium {
namespace DynamicVK {
@@ -14,6 +17,8 @@ namespace Indium {
extern PFN_vkEnumerateInstanceLayerProperties vkEnumerateInstanceLayerProperties;
bool init();
+ void beginDiagnosticCapture(VkInstance instance);
+ void endDiagnosticCapture(bool presented);
void finit();
/**
@@ -24,7 +29,7 @@ namespace Indium {
bool eagerlyResolveRequired();
struct DynamicFunctionBase {
- void* pointer;
+ std::atomic<void*> pointer {nullptr};
std::once_flag _resolveFlag;
const char* name;
@@ -33,10 +38,11 @@ namespace Indium {
{};
bool resolve() {
+ if (pointer.load(std::memory_order_acquire)) return true;
std::call_once(_resolveFlag, [&]() {
- pointer = reinterpret_cast<void*>(vkGetInstanceProcAddr(globalInstance, name));
+ pointer.store(reinterpret_cast<void*>(vkGetInstanceProcAddr(globalInstance, name)), std::memory_order_release);
});
- return !!pointer;
+ return pointer.load(std::memory_order_acquire) != nullptr;
};
bool isAvailable() {
@@ -58,16 +64,20 @@ namespace Indium {
{};
Ret operator ()(Arg... arg) {
- if (!resolve()) {
- throw std::runtime_error(std::string("Failed to resolve ") + name);
- }
- Ret (*func)(Arg...) = reinterpret_cast<decltype(func)>(pointer);
+ void* address = pointer.load(std::memory_order_acquire);
+ if (!address) {
+ if (!resolve()) throw std::runtime_error(std::string("Failed to resolve ") + name);
+ address = pointer.load(std::memory_order_acquire);
+ }
+ Ret (*func)(Arg...) = reinterpret_cast<decltype(func)>(address);
return func(std::forward<Arg>(arg)...);
};
};
#define INDIUM_DYNAMICVK_FUNCTION_FOREACH(_macro) \
_macro(vkAcquireNextImageKHR) \
+ _macro(vkResetCommandPool) \
+ _macro(vkResetDescriptorPool) \
_macro(vkAllocateCommandBuffers) \
_macro(vkAllocateDescriptorSets) \
_macro(vkAllocateMemory) \
@@ -88,6 +98,10 @@ namespace Indium {
_macro(vkCmdDraw) \
_macro(vkCmdDrawIndexed) \
_macro(vkCmdEndRenderPass) \
+ _macro(vkCmdResetQueryPool) \
+ _macro(vkCmdWriteTimestamp) \
+ _macro(vkCreateQueryPool) \
+ _macro(vkGetQueryPoolResults) \
_macro(vkCmdFillBuffer) \
_macro(vkCmdPipelineBarrier) \
_macro(vkCmdSetBlendConstants) \
@@ -157,6 +171,7 @@ namespace Indium {
_macro(vkGetImageMemoryRequirements) \
_macro(vkGetPhysicalDeviceFeatures2) \
_macro(vkGetPhysicalDeviceMemoryProperties) \
+ _macro(vkGetPhysicalDeviceFormatProperties) \
_macro(vkGetPhysicalDeviceProperties) \
_macro(vkGetPhysicalDeviceProperties2) \
_macro(vkGetPhysicalDeviceQueueFamilyProperties) \
diff --git a/private-include/indium/library.private.hpp b/private-include/indium/library.private.hpp
index 8752aa2..faf50b2 100644
--- a/private-include/indium/library.private.hpp
+++ b/private-include/indium/library.private.hpp
@@ -26,6 +26,7 @@ namespace Indium {
std::vector<Iridium::BindingInfo> bindings;
std::vector<Iridium::EmbeddedSampler> embeddedSamplers;
std::vector<std::shared_ptr<SamplerState>> embeddedSamplerStates;
+ std::vector<Iridium::ArgumentInfo> arguments;
};
class PrivateFunction: public Function {
@@ -33,6 +34,8 @@ namespace Indium {
PrivateFunction(std::shared_ptr<PrivateLibrary> library, const std::string& name, const FunctionInfo& functionInfo);
virtual std::shared_ptr<Device> device() override;
+ virtual const std::vector<Iridium::ArgumentInfo>& argumentInfo() const override;
+ virtual FunctionType functionType() const override { return _functionInfo.functionType; }
INDIUM_PROPERTY_READONLY_OBJECT(PrivateDevice, p, P,rivateDevice);
diff --git a/private-include/indium/pipeline.private.hpp b/private-include/indium/pipeline.private.hpp
index a43f532..6f2d7dd 100644
--- a/private-include/indium/pipeline.private.hpp
+++ b/private-include/indium/pipeline.private.hpp
@@ -33,7 +33,7 @@ namespace Indium {
}
};
- void processFunction(std::shared_ptr<PrivateFunction> function, size_t layoutIndex) {
+ void processFunction(std::shared_ptr<PrivateFunction> function, size_t layoutIndex, bool dynamicBuffers = false) {
size_t index = 0;
std::vector<VkDescriptorSetLayoutBinding> bindings;
@@ -61,7 +61,7 @@ namespace Indium {
// we need a UBO to store the buffer addresses
auto& binding = bindings.emplace_back();
binding.binding = 0;
- binding.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER;
+ binding.descriptorType = dynamicBuffers ? VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC : VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER;
binding.descriptorCount = 1;
binding.stageFlags = functionTypeToVkShaderStageFlags(function->functionInfo().functionType);
}
diff --git a/private-include/indium/render-command-encoder.private.hpp b/private-include/indium/render-command-encoder.private.hpp
index a41b951..2f670e9 100644
--- a/private-include/indium/render-command-encoder.private.hpp
+++ b/private-include/indium/render-command-encoder.private.hpp
@@ -5,12 +5,15 @@
#include <indium/sampler.private.hpp>
#include <indium/device.hpp>
#include <indium/command-encoder.private.hpp>
+#include <indium/resource-owner-cache.private.hpp>
#include <vulkan/vulkan.h>
#include <array>
#include <variant>
#include <vector>
+#include <unordered_set>
+#include <unordered_map>
namespace Indium {
class PrivateCommandBuffer;
@@ -29,19 +32,41 @@ namespace Indium {
VkFramebuffer _framebuffer = VK_NULL_HANDLE;
VkRenderPass _renderPass = VK_NULL_HANDLE;
VkDescriptorPool _pool = VK_NULL_HANDLE;
+ std::vector<VkDescriptorPool> _retiredPools;
std::vector<FunctionResources> _savedFunctionResources;
std::array<FunctionResources, 2> _functionResources {};
std::vector<std::shared_ptr<Buffer>> _keepAliveBuffers;
-
- void updateBindings();
+ AddressPool _addressPool;
+ std::array<VkDescriptorSetLayout, 2> _boundLayouts {};
+ std::array<VkDescriptorSet, 2> _boundDescriptorSets {};
+ std::array<FunctionResources, 2> _boundResources {};
+ std::vector<std::shared_ptr<PrivateRenderPipelineState>> _keepAlivePipelineStates;
+
+ // A draw that samples the depth texture it has attached (soft-particle style depth fade) must not also write
+ // depth: that is a feedback loop, and writing makes later draws fail their depth test under the effect (the
+ // ward preview hid the lower half of the champion). Kept last: other frameworks read this class by layout.
+ std::shared_ptr<PrivateTexture> _depthRoot;
+ bool _depthWriteRequested = false;
+ bool _depthWriteSuppressed = false;
+ // Unique owners keep every resource referenced by earlier draws alive.
+ std::unordered_set<const void*> _retainedResourcePointers;
+ std::vector<std::shared_ptr<void>> _retainedResourceOwners;
+
+ void updateBindings(VkCommandBuffer commandBuffer);
public:
PrivateRenderCommandEncoder(std::shared_ptr<PrivateCommandBuffer> commandBuffer, const RenderPassDescriptor& descriptor);
~PrivateRenderCommandEncoder();
-
- virtual void setRenderPipelineState(std::shared_ptr<RenderPipelineState> renderPipelineState) override;
+ bool hasHeavyRetirement() const {
+ return _resolvedBufferOwners.size() + _resolvedTextureOwners.size() +
+ _pipelineOwners.size() + _retainedResourceOwners.size() >= 64 ||
+ !_retiredPools.empty();
+ }
+
+ virtual void memoryBarrier(size_t scope, RenderStages after, RenderStages before) override;
+ virtual void setRenderPipelineState(const std::shared_ptr<RenderPipelineState>& renderPipelineState) override;
virtual void setFrontFacingWinding(Winding frontFaceWinding) override;
virtual void setCullMode(CullMode cullMode) override;
virtual void setDepthBias(float depthBias, float slopeScale, float clamp) override;
@@ -53,7 +78,7 @@ namespace Indium {
virtual void setScissorRects(const ScissorRect* scissorRects, size_t count) override;
virtual void setScissorRects(const std::vector<ScissorRect>& scissorRects) override;
virtual void setBlendColor(float red, float green, float blue, float alpha) override;
- virtual void setDepthStencilState(std::shared_ptr<DepthStencilState> state) override;
+ virtual void setDepthStencilState(const std::shared_ptr<DepthStencilState>& state) override;
virtual void setTriangleFillMode(TriangleFillMode triangleFillMode) override;
virtual void setStencilReferenceValue(uint32_t value) override;
virtual void setStencilReferenceValue(uint32_t front, uint32_t back) override;
@@ -63,30 +88,30 @@ namespace Indium {
virtual void drawPrimitives(PrimitiveType primitiveType, size_t vertexStart, size_t vertexCount, size_t instanceCount) override;
virtual void drawPrimitives(PrimitiveType primitiveType, size_t vertexStart, size_t vertexCount) override;
- virtual void drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, std::shared_ptr<Buffer> indexBuffer, size_t indexBufferOffset, size_t instanceCount, int64_t baseVertex, size_t baseInstance) override;
- virtual void drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, std::shared_ptr<Buffer> indexBuffer, size_t indexBufferOffset, size_t instanceCount) override;
- virtual void drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, std::shared_ptr<Buffer> indexBuffer, size_t indexBufferOffset) override;
+ virtual void drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, const std::shared_ptr<Buffer>& indexBuffer, size_t indexBufferOffset, size_t instanceCount, int64_t baseVertex, size_t baseInstance) override;
+ virtual void drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, const std::shared_ptr<Buffer>& indexBuffer, size_t indexBufferOffset, size_t instanceCount) override;
+ virtual void drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, const std::shared_ptr<Buffer>& indexBuffer, size_t indexBufferOffset) override;
virtual void setVertexBytes(const void* bytes, size_t length, size_t index) override;
- virtual void setVertexBuffer(std::shared_ptr<Buffer> buffer, size_t offset, size_t index) override;
+ virtual void setVertexBuffer(const std::shared_ptr<Buffer>& buffer, size_t offset, size_t index) override;
virtual void setVertexBuffers(const std::vector<std::shared_ptr<Buffer>>& buffers, const std::vector<size_t>& offsets, Range<size_t> range) override;
virtual void setVertexBufferOffset(size_t offset, size_t index) override;
- virtual void setVertexSamplerState(std::shared_ptr<SamplerState> state, size_t index) override;
- virtual void setVertexSamplerState(std::shared_ptr<SamplerState> state, float lodMinClamp, float lodMaxClamp, size_t index) override;
+ virtual void setVertexSamplerState(const std::shared_ptr<SamplerState>& state, size_t index) override;
+ virtual void setVertexSamplerState(const std::shared_ptr<SamplerState>& state, float lodMinClamp, float lodMaxClamp, size_t index) override;
virtual void setVertexSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& states, Range<size_t> range) override;
virtual void setVertexSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& states, const std::vector<float>& lodMinClamps, const std::vector<float>& lodMaxClamps, Range<size_t> range) override;
- virtual void setVertexTexture(std::shared_ptr<Texture> texture, size_t index) override;
+ virtual void setVertexTexture(const std::shared_ptr<Texture>& texture, size_t index) override;
virtual void setVertexTextures(const std::vector<std::shared_ptr<Texture>>& textures, Range<size_t> range) override;
virtual void setFragmentBytes(const void* bytes, size_t length, size_t index) override;
- virtual void setFragmentBuffer(std::shared_ptr<Buffer> buffer, size_t offset, size_t index) override;
+ virtual void setFragmentBuffer(const std::shared_ptr<Buffer>& buffer, size_t offset, size_t index) override;
virtual void setFragmentBuffers(const std::vector<std::shared_ptr<Buffer>>& buffers, const std::vector<size_t>& offsets, Range<size_t> range) override;
virtual void setFragmentBufferOffset(size_t offset, size_t index) override;
- virtual void setFragmentSamplerState(std::shared_ptr<SamplerState> state, size_t index) override;
- virtual void setFragmentSamplerState(std::shared_ptr<SamplerState> state, float lodMinClamp, float lodMaxClamp, size_t index) override;
+ virtual void setFragmentSamplerState(const std::shared_ptr<SamplerState>& state, size_t index) override;
+ virtual void setFragmentSamplerState(const std::shared_ptr<SamplerState>& state, float lodMinClamp, float lodMaxClamp, size_t index) override;
virtual void setFragmentSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& states, Range<size_t> range) override;
virtual void setFragmentSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& states, const std::vector<float>& lodMinClamps, const std::vector<float>& lodMaxClamps, Range<size_t> range) override;
- virtual void setFragmentTexture(std::shared_ptr<Texture> texture, size_t index) override;
+ virtual void setFragmentTexture(const std::shared_ptr<Texture>& texture, size_t index) override;
virtual void setFragmentTextures(std::vector<std::shared_ptr<Texture>>& textures, Range<size_t> range) override;
virtual void useResource(std::shared_ptr<Resource> resource, ResourceUsage usage, RenderStages stages) override;
@@ -95,8 +120,89 @@ namespace Indium {
virtual void useResources(const std::vector<std::shared_ptr<Resource>>& resources, ResourceUsage usage) override;
virtual void endEncoding() override;
+ // begins the Vulkan render pass on the first draw when the pass has no clear (see render-command-encoder.cpp)
+ void beginPassIfNeeded();
INDIUM_PROPERTY_OBJECT_VECTOR(Texture, r, R,eadOnlyTextures);
INDIUM_PROPERTY_OBJECT_VECTOR(Texture, r, R,eadWriteTextures);
+ private:
+ // Appended after existing properties: preserve every earlier field offset.
+ std::array<ResolvedFunctionResources, 2> _resolvedResources;
+ struct BindingKeys {
+ std::vector<std::pair<Buffer*, size_t>> buffers;
+ std::vector<Texture*> textures;
+ std::vector<SamplerState*> samplers;
+ };
+ std::array<BindingKeys, 2> _boundResourceKeys;
+ std::array<uint64_t, 2> _boundResourceRevisions {};
+ std::array<const FunctionInfo*, 2> _boundFunctionInfos {};
+ uint64_t _descriptorStageUpdates = 0;
+ uint64_t _descriptorStageReuses = 0;
+ std::array<BufferAddressTable, 2> _bufferAddressTables {};
+ std::array<uint32_t, 2> _lastDrawDynamicOffsets {};
+ VkPipeline _lastDrawPipeline = VK_NULL_HANDLE;
+ VkPrimitiveTopology _lastDrawTopology = VK_PRIMITIVE_TOPOLOGY_MAX_ENUM;
+ VkPipelineLayout _lastDescriptorPipelineLayout = VK_NULL_HANDLE;
+ std::array<VkDescriptorSet, 2> _lastDrawDescriptorSets {};
+ std::vector<VkBuffer> _lastVertexBuffers;
+ std::vector<VkDeviceSize> _lastVertexOffsets;
+ ResolvedResource<Buffer, PrivateBuffer> _lastIndexResource;
+ VkBuffer _lastIndexBuffer = VK_NULL_HANDLE;
+ VkDeviceSize _lastIndexOffset = 0;
+ VkIndexType _lastIndexType = VK_INDEX_TYPE_MAX_ENUM;
+
+ uint64_t _lastVertexResourceRevision = UINT64_MAX;
+ const PrivateRenderPipelineState* _lastVertexBindingPSO = nullptr;
+ ResourceOwnerCache<Buffer, PrivateBuffer> _resolvedBufferOwners;
+ void resolveBuffer(size_t stage, size_t index) {
+ auto& entries = _resolvedResources[stage].buffers;
+ growToIndex(entries, index);
+ auto& entry = entries[index];
+ const auto& owner = _functionResources[stage].buffers[index].first;
+ if (entry.object == owner.get()) return;
+ entry.object = owner.get();
+ if (!owner) { entry.value = nullptr; return; }
+ auto [found, inserted] = _resolvedBufferOwners.acquire(owner);
+ entry.value = found->value;
+ }
+ ResourceOwnerCache<RenderPipelineState, PrivateRenderPipelineState> _pipelineOwners;
+ ResourceOwnerCache<Texture, PrivateTexture> _resolvedTextureOwners;
+ void resolveTexture(size_t stage, size_t index) {
+ auto& entries = _resolvedResources[stage].textures;
+ growToIndex(entries, index);
+ auto& entry = entries[index];
+ const auto& owner = _functionResources[stage].textures[index];
+ if (entry.object == owner.get()) return;
+ entry.object = owner.get();
+ if (!owner) { entry.value = nullptr; entry.depthFeedback = false; return; }
+ auto [found, inserted] = _resolvedTextureOwners.acquire(owner);
+ auto& cached = *found;
+ if (inserted) {
+ auto* root = cached.value;
+ while (root) {
+ if (root == _depthRoot.get()) { cached.depthFeedback = true; break; }
+ root = dynamic_cast<PrivateTexture*>(root->parentTexture().get());
+ }
+ }
+ entry.value = cached.value;
+ entry.depthFeedback = cached.depthFeedback;
+ }
+ bool _renderPassDeviceOwned = false;
+ void resolveSampler(size_t stage, size_t index) {
+ auto& entries = _resolvedResources[stage].samplers;
+ growToIndex(entries, index);
+ const auto& owner = _functionResources[stage].samplers[index];
+ if (entries[index].refresh(owner) && owner && _retainedResourcePointers.insert(owner.get()).second)
+ _retainedResourceOwners.push_back(owner);
+ }
+
+ // appended last on purpose: lazily begun render pass
+ VkRenderPassBeginInfo _lazyBeginInfo {};
+ bool _passBegun = true;
+
+ // appended last: buffers-only fast path of updateBindings
+ std::array<uint64_t, 2> _boundImageRevisions {};
+ std::array<bool, 2> _boundKeysStale {};
+ std::array<bool, 2> _boundStageHasBuffers {};
};
};
diff --git a/private-include/indium/texture.private.hpp b/private-include/indium/texture.private.hpp
index ccc27bb..e40e40c 100644
--- a/private-include/indium/texture.private.hpp
+++ b/private-include/indium/texture.private.hpp
@@ -88,6 +88,7 @@ namespace Indium {
virtual void precommit(std::shared_ptr<Indium::PrivateCommandBuffer> cmdbuf);
virtual bool needsExportablePresentationSemaphore() const;
+ virtual VkImageView sampledImageView() { return imageView(); }
};
class TextureView: public PrivateTexture {
@@ -102,6 +103,7 @@ namespace Indium {
Range<size_t> _levels;
Range<size_t> _layers;
TextureSwizzleChannels _swizzle;
+ VkImageView _sampledImageView = VK_NULL_HANDLE;
public:
TextureView(std::shared_ptr<PrivateTexture> original, PixelFormat pixelFormat, TextureType textureType, VkImageAspectFlags imageAspect, TextureSwizzleChannels swizzle, const Range<size_t>& levels, const Range<size_t>& layers);
@@ -125,6 +127,7 @@ namespace Indium {
virtual size_t parentRelativeSlice() const override;
virtual VkImageView imageView() override;
+ virtual VkImageView sampledImageView() override { return _sampledImageView ? _sampledImageView : imageView(); }
virtual VkImage image() override;
virtual std::shared_ptr<Device> device() override;
virtual VkImageLayout imageLayout() override;
@@ -150,6 +153,7 @@ namespace Indium {
VkImageView _imageView;
VkDeviceMemory _memory;
StorageMode _storageMode;
+ VkImageView _sampledImageView = VK_NULL_HANDLE;
public:
ConcreteTexture(std::shared_ptr<PrivateDevice> device, const TextureDescriptor& descriptor);
@@ -169,6 +173,7 @@ namespace Indium {
virtual TextureSwizzleChannels swizzle() const override;
virtual VkImageView imageView() override;
+ virtual VkImageView sampledImageView() override { return _sampledImageView ? _sampledImageView : imageView(); }
virtual VkImage image() override;
virtual VkImageLayout imageLayout() override;
virtual size_t vulkanArrayLength() const override;
diff --git a/private-include/indium/types.private.hpp b/private-include/indium/types.private.hpp
index d49b5f4..45bbd42 100644
--- a/private-include/indium/types.private.hpp
+++ b/private-include/indium/types.private.hpp
@@ -5,6 +5,7 @@
#include <vulkan/vulkan.h>
#include <memory>
+#include <stdexcept>
#include <vulkan/vulkan_core.h>
namespace Indium {
@@ -187,7 +188,7 @@ namespace Indium {
static constexpr size_t pixelFormatToByteCount(PixelFormat pixelFormat) {
switch (pixelFormat) {
case PixelFormat::Invalid: return 0;
- case PixelFormat::A8Unorm: return 0;
+ case PixelFormat::A8Unorm: return 1;
case PixelFormat::R8Unorm: return 1;
case PixelFormat::R8Unorm_sRGB: return 1;
case PixelFormat::R8Snorm: return 1;
@@ -244,6 +245,127 @@ namespace Indium {
}
};
+
+ struct PixelTransferBlock { size_t width, height, bytes; };
+
+ // Buffer pitches count bytes per block row; Vulkan copy pitches count texels.
+ static constexpr PixelTransferBlock pixelFormatTransferBlock(PixelFormat format) {
+ switch (format) {
+ case PixelFormat::BC1_RGBA:
+ case PixelFormat::BC1_RGBA_sRGB:
+ case PixelFormat::BC4_RUnorm:
+ case PixelFormat::BC4_RSnorm:
+ case PixelFormat::EAC_R11Unorm:
+ case PixelFormat::EAC_R11Snorm:
+ case PixelFormat::ETC2_RGB8:
+ case PixelFormat::ETC2_RGB8_sRGB:
+ case PixelFormat::ETC2_RGB8A1:
+ case PixelFormat::ETC2_RGB8A1_sRGB:
+ return { 4, 4, 8 };
+ case PixelFormat::BC2_RGBA:
+ case PixelFormat::BC2_RGBA_sRGB:
+ case PixelFormat::BC3_RGBA:
+ case PixelFormat::BC3_RGBA_sRGB:
+ case PixelFormat::BC5_RGUnorm:
+ case PixelFormat::BC5_RGSnorm:
+ case PixelFormat::BC6H_RGBFloat:
+ case PixelFormat::BC6H_RGBUfloat:
+ case PixelFormat::BC7_RGBAUnorm:
+ case PixelFormat::BC7_RGBAUnorm_sRGB:
+ case PixelFormat::EAC_RG11Unorm:
+ case PixelFormat::EAC_RG11Snorm:
+ case PixelFormat::EAC_RGBA8:
+ case PixelFormat::EAC_RGBA8_sRGB:
+ return { 4, 4, 16 };
+ case PixelFormat::GBGR422:
+ case PixelFormat::BGRG422:
+ return { 2, 1, 4 };
+ case PixelFormat::Depth16Unorm:
+ return { 1, 1, 2 };
+ case PixelFormat::Depth32Float:
+ return { 1, 1, 4 };
+ case PixelFormat::Stencil8:
+ case PixelFormat::X32_Stencil8:
+ case PixelFormat::X24_Stencil8:
+ return { 1, 1, 1 };
+ case PixelFormat::ASTC_4x4_sRGB:
+ case PixelFormat::ASTC_4x4_LDR:
+ case PixelFormat::ASTC_4x4_HDR:
+ return { 4, 4, 16 };
+ case PixelFormat::ASTC_5x4_sRGB:
+ case PixelFormat::ASTC_5x4_LDR:
+ case PixelFormat::ASTC_5x4_HDR:
+ return { 5, 4, 16 };
+ case PixelFormat::ASTC_5x5_sRGB:
+ case PixelFormat::ASTC_5x5_LDR:
+ case PixelFormat::ASTC_5x5_HDR:
+ return { 5, 5, 16 };
+ case PixelFormat::ASTC_6x5_sRGB:
+ case PixelFormat::ASTC_6x5_LDR:
+ case PixelFormat::ASTC_6x5_HDR:
+ return { 6, 5, 16 };
+ case PixelFormat::ASTC_6x6_sRGB:
+ case PixelFormat::ASTC_6x6_LDR:
+ case PixelFormat::ASTC_6x6_HDR:
+ return { 6, 6, 16 };
+ case PixelFormat::ASTC_8x5_sRGB:
+ case PixelFormat::ASTC_8x5_LDR:
+ case PixelFormat::ASTC_8x5_HDR:
+ return { 8, 5, 16 };
+ case PixelFormat::ASTC_8x6_sRGB:
+ case PixelFormat::ASTC_8x6_LDR:
+ case PixelFormat::ASTC_8x6_HDR:
+ return { 8, 6, 16 };
+ case PixelFormat::ASTC_8x8_sRGB:
+ case PixelFormat::ASTC_8x8_LDR:
+ case PixelFormat::ASTC_8x8_HDR:
+ return { 8, 8, 16 };
+ case PixelFormat::ASTC_10x5_sRGB:
+ case PixelFormat::ASTC_10x5_LDR:
+ case PixelFormat::ASTC_10x5_HDR:
+ return { 10, 5, 16 };
+ case PixelFormat::ASTC_10x6_sRGB:
+ case PixelFormat::ASTC_10x6_LDR:
+ case PixelFormat::ASTC_10x6_HDR:
+ return { 10, 6, 16 };
+ case PixelFormat::ASTC_10x8_sRGB:
+ case PixelFormat::ASTC_10x8_LDR:
+ case PixelFormat::ASTC_10x8_HDR:
+ return { 10, 8, 16 };
+ case PixelFormat::ASTC_10x10_sRGB:
+ case PixelFormat::ASTC_10x10_LDR:
+ case PixelFormat::ASTC_10x10_HDR:
+ return { 10, 10, 16 };
+ case PixelFormat::ASTC_12x10_sRGB:
+ case PixelFormat::ASTC_12x10_LDR:
+ case PixelFormat::ASTC_12x10_HDR:
+ return { 12, 10, 16 };
+ case PixelFormat::ASTC_12x12_sRGB:
+ case PixelFormat::ASTC_12x12_LDR:
+ case PixelFormat::ASTC_12x12_HDR:
+ return { 12, 12, 16 };
+ default: return { 1, 1, pixelFormatToByteCount(format) };
+ }
+ }
+
+ struct BufferImagePitch { uint32_t rowLength, imageHeight; size_t rowBytes, imageBytes; };
+
+ static inline BufferImagePitch bufferImagePitch(PixelFormat format, size_t width, size_t height,
+ size_t bytesPerRow, size_t bytesPerImage) {
+ auto block = pixelFormatTransferBlock(format);
+ if (!block.bytes) throw std::runtime_error("Unsupported pixel format for buffer/image transfer");
+ size_t tightRow = ((width + block.width - 1) / block.width) * block.bytes;
+ size_t rows = (height + block.height - 1) / block.height;
+ size_t row = bytesPerRow ? bytesPerRow : tightRow;
+ if (!row || row < tightRow || row % block.bytes ||
+ (bytesPerImage && (bytesPerImage % row || bytesPerImage / row < rows))) {
+ throw std::runtime_error("Invalid buffer/image transfer pitch");
+ }
+ return { static_cast<uint32_t>(bytesPerRow ? row / block.bytes * block.width : 0),
+ static_cast<uint32_t>(bytesPerImage ? bytesPerImage / row * block.height : 0),
+ row, bytesPerImage ? bytesPerImage : row * rows };
+ }
+
static constexpr bool pixelFormatIsCompressed(PixelFormat pixelFormat) {
switch (pixelFormat) {
case PixelFormat::R8Unorm:
@@ -716,8 +838,8 @@ namespace Indium {
case VertexFormat::UInt2: return VK_FORMAT_R32G32_UINT;
case VertexFormat::UInt3: return VK_FORMAT_R32G32B32_UINT;
case VertexFormat::UInt4: return VK_FORMAT_R32G32B32A32_UINT;
- case VertexFormat::Int1010102Normalized: return VK_FORMAT_A2R10G10B10_SINT_PACK32;
- case VertexFormat::UInt1010102Normalized: return VK_FORMAT_A2R10G10B10_UINT_PACK32;
+ case VertexFormat::Int1010102Normalized: return VK_FORMAT_A2B10G10R10_SNORM_PACK32;
+ case VertexFormat::UInt1010102Normalized: return VK_FORMAT_A2B10G10R10_UNORM_PACK32;
case VertexFormat::UChar4Normalized_BGRA: return VK_FORMAT_B8G8R8A8_UNORM;
case VertexFormat::UChar: return VK_FORMAT_R8_UINT;
case VertexFormat::Char: return VK_FORMAT_R8_SINT;
diff --git a/private-include/iridium/dynamic-llvm.hpp b/private-include/iridium/dynamic-llvm.hpp
index b3756ef..ceb41ce 100644
--- a/private-include/iridium/dynamic-llvm.hpp
+++ b/private-include/iridium/dynamic-llvm.hpp
@@ -2,10 +2,14 @@
#include <llvm-c/BitReader.h>
#include <llvm-c/Core.h>
+#include <llvm-c/Transforms/PassBuilder.h>
+#include <llvm-c/Error.h>
#include <dlfcn.h>
#include <mutex>
+#include <string>
+#include <stdexcept>
namespace Iridium {
namespace DynamicLLVM {
@@ -58,6 +62,9 @@ namespace Iridium {
};
#define IRIDIUM_DYNAMICLLVM_FUNCTION_FOREACH(_macro) \
+ _macro(LLVMGlobalGetValueType) \
+ _macro(LLVMGetGEPSourceElementType) \
+ _macro(LLVMIsAPoisonValue) \
_macro(LLVMBasicBlockAsValue) \
_macro(LLVMConstIntGetSExtValue) \
_macro(LLVMConstIntGetZExtValue) \
@@ -68,6 +75,15 @@ namespace Iridium {
_macro(LLVMCreateMemoryBufferWithMemoryRange) \
_macro(LLVMDisposeMemoryBuffer) \
_macro(LLVMDisposeMessage) \
+ _macro(LLVMRunPasses) \
+ _macro(LLVMCreatePassBuilderOptions) \
+ _macro(LLVMDisposePassBuilderOptions) \
+ _macro(LLVMGetErrorMessage) \
+ _macro(LLVMDisposeErrorMessage) \
+ _macro(LLVMGetTarget) \
+ _macro(LLVMSetTarget) \
+ _macro(LLVMIsNull) \
+ _macro(LLVMPrintValueToString) \
_macro(LLVMDisposeModule) \
_macro(LLVMGetAlignment) \
_macro(LLVMGetArrayLength) \
@@ -77,6 +93,11 @@ namespace Iridium {
_macro(LLVMGetElementAsConstant) \
_macro(LLVMGetElementType) \
_macro(LLVMGetFCmpPredicate) \
+ _macro(LLVMGetICmpPredicate) \
+ _macro(LLVMInt32Type) \
+ _macro(LLVMGetFirstUse) \
+ _macro(LLVMGetNextUse) \
+ _macro(LLVMGetUser) \
_macro(LLVMGetFirstBasicBlock) \
_macro(LLVMGetFirstInstruction) \
_macro(LLVMGetFirstNamedMetadata) \
@@ -84,6 +105,7 @@ namespace Iridium {
_macro(LLVMGetIncomingValue) \
_macro(LLVMGetIndices) \
_macro(LLVMGetInitializer) \
+ _macro(LLVMGetAllocatedType) \
_macro(LLVMGetInstructionOpcode) \
_macro(LLVMGetIntTypeWidth) \
_macro(LLVMGetLastInstruction) \
@@ -107,6 +129,7 @@ namespace Iridium {
_macro(LLVMGetPointerAddressSpace) \
_macro(LLVMGetReturnType) \
_macro(LLVMGetSuccessor) \
+ _macro(LLVMGetNumSuccessors) \
_macro(LLVMGetTypeKind) \
_macro(LLVMGetUndefMaskElem) \
_macro(LLVMGetValueKind) \
diff --git a/private-include/iridium/spirv.hpp b/private-include/iridium/spirv.hpp
index 95cd3de..8fa6b1a 100644
--- a/private-include/iridium/spirv.hpp
+++ b/private-include/iridium/spirv.hpp
@@ -68,6 +68,7 @@ namespace Iridium {
Input = 1,
Uniform = 2,
Output = 3,
+ Private = 6,
Function = 7,
StorageBuffer = 12,
PhysicalStorageBuffer = 5349,
@@ -82,6 +83,7 @@ namespace Iridium {
Int64 = 11,
Int16 = 22,
Int8 = 39,
+ ImageQuery = 50,
PhysicalStorageBufferAddresses = 5347,
};
@@ -255,6 +257,10 @@ namespace Iridium {
ImageFormat imageFormat = ImageFormat::Unknown;
ResultID imageRealSampleTypeID = ResultIDInvalid;
+ // Types of Private/Function memory must not carry the explicit layout (Offset/ArrayStride)
+ // that buffer types need, so those are separate types. This IS part of the type's identity.
+ bool layoutFree = false;
+
// this does NOT need to be considered for equality
ResultID arrayLengthID = ResultIDInvalid;
@@ -372,6 +378,7 @@ struct std::hash<Iridium::SPIRV::Type> {
result = ((result << 1) ^ std::hash<size_t>()(type.scalarWidth)) >> 1;
result = ((result << 1) ^ std::hash<bool>()(type.integerIsSigned)) >> 1;
result = ((result << 1) ^ std::hash<size_t>()(type.entryCount)) >> 1;
+ result = ((result << 1) ^ std::hash<bool>()(type.layoutFree)) >> 1;
for (const auto& member: type.structureMembers) {
result = ((result << 1) ^ std::hash<Iridium::SPIRV::ResultID>()(member.id)) >> 1;
for (const auto& decoration: member.decorations) {
@@ -412,6 +419,7 @@ namespace Iridium {
struct GlobalVariable {
StorageClass storageClass;
ResultID typeID;
+ ResultID initializer = ResultIDInvalid;
};
struct FunctionVariable {
@@ -485,7 +493,7 @@ namespace Iridium {
IMul = 132,
FMul = 133,
UDiv = 134,
- SDiv = 133,
+ SDiv = 135,
FDiv = 136,
UMod = 137,
SRem = 138,
@@ -618,12 +626,14 @@ namespace Iridium {
std::unordered_map<ResultID, DynamicByteWriter> _functionWriters;
PrivateFunctionInfo* _currentFunctionInfo = nullptr;
DynamicByteWriter* _currentFunctionWriter = nullptr;
+ std::vector<std::pair<size_t, uintptr_t>> _phiFixups;
std::unordered_set<Capability> _requiredCapabilities;
uint8_t _versionMajor = 1;
uint8_t _versionMinor = 0;
uint16_t _generatorID = 0;
uint16_t _generatorVersion = 0;
std::unordered_map<std::pair<uintmax_t, ResultID>, ResultID> _constantScalars;
+ std::unordered_map<std::pair<uintmax_t, ResultID>, ResultID> _constantBools;
std::unordered_map<ResultID, ResultID> _undefinedValues;
std::unordered_map<ResultID, ResultID> _nullValues;
std::unordered_map<ResultID, ResultID> _resultIDTypes;
@@ -673,7 +683,8 @@ namespace Iridium {
ResultID lookupResultID(uintptr_t association) const;
void addEntryPoint(const EntryPoint& entryPoint);
- ResultID addGlobalVariable(ResultID typeID, StorageClass storageClass);
+ ResultID addGlobalVariable(ResultID typeID, StorageClass storageClass, ResultID initializer = ResultIDInvalid);
+ ResultID addFunctionVariable(ResultID typeID);
void addDecoration(ResultID resultID, const Decoration& decoration);
FunctionInfo declareFunction(ResultID functionType);
@@ -686,6 +697,7 @@ namespace Iridium {
ResultID declareConstantComposite(ResultID typeID, std::vector<ResultID> elements);
ResultID declareUndefinedValue(ResultID typeID);
+ ResultID declareConstantBool(bool value);
ResultID declareNullValue(ResultID typeID);
ResultID insertLabel(ResultID id = ResultIDInvalid);
@@ -717,9 +729,13 @@ namespace Iridium {
ResultID encodeFOrdGreaterThan(ResultID operand1, ResultID operand2);
void encodeBranch(ResultID targetLabel);
void encodeBranchConditional(ResultID condition, ResultID trueLabel, ResultID falseLabel);
- ResultID encodePhi(ResultID resultTypeID, std::vector<std::pair<ResultID, ResultID>> variablesAndBlocks);
+ ResultID encodePhi(ResultID resultTypeID, std::vector<std::pair<ResultID, ResultID>> variablesAndBlocks, ResultID result = ResultIDInvalid, std::vector<uintptr_t> forwardAssociations = {});
void encodeSelectionMerge(ResultID mergeBlock);
ResultID encodeSqrt(ResultID resultTypeID, ResultID target);
+ // Generic emitters for instructions without a dedicated helper. The operands are result IDs.
+ ResultID encodeRawOp(uint16_t opcode, ResultID resultTypeID, const std::vector<ResultID>& operands);
+ void encodeRawOpNoResult(uint16_t opcode, const std::vector<ResultID>& operands);
+ ResultID encodeGLSLOp(GLSLOpcode opcode, ResultID resultTypeID, const std::vector<ResultID>& operands);
ResultID encodeTrunc(ResultID resultTypeID, ResultID operand);
void encodeDebugPrint(std::string format, std::vector<ResultID> arguments);
diff --git a/src/indium-kit/layer.cpp b/src/indium-kit/layer.cpp
index 8d00405..6a5fde8 100644
--- a/src/indium-kit/layer.cpp
+++ b/src/indium-kit/layer.cpp
@@ -105,6 +105,7 @@ void IndiumKit::PrivateDrawable::present() {
info.pImageIndices = &_swapchainImageIndex;
info.pResults = nullptr;
+ std::lock_guard<std::mutex> queueLock(privateDevice->graphicsQueueMutex());
if (DynamicVK::vkQueuePresentKHR(privateDevice->graphicsQueue(), &info) != VK_SUCCESS) {
// TODO
abort();
diff --git a/src/indium/blit-command-encoder.cpp b/src/indium/blit-command-encoder.cpp
index e58f0ba..616d177 100644
--- a/src/indium/blit-command-encoder.cpp
+++ b/src/indium/blit-command-encoder.cpp
@@ -3,6 +3,7 @@
#include <indium/texture.private.hpp>
#include <indium/buffer.private.hpp>
#include <indium/dynamic-vk.hpp>
+#include <indium/command-encoder.private.hpp>
Indium::BlitCommandEncoder::~BlitCommandEncoder() {};
@@ -17,14 +18,27 @@ Indium::PrivateBlitCommandEncoder::PrivateBlitCommandEncoder(std::shared_ptr<Pri
_descriptor(descriptor)
{};
+void Indium::PrivateBlitCommandEncoder::noteDestination(const std::shared_ptr<Texture>& destination) {
+ // Drawables are presented by another API, so writes to them must signal a presentation semaphore like render passes do
+ auto privateTexture = std::dynamic_pointer_cast<PrivateTexture>(destination);
+ if (privateTexture && privateTexture->needsExportablePresentationSemaphore()) {
+ for (const auto& known: _readWriteTextures) {
+ if (known == destination) {
+ return;
+ }
+ }
+ _readWriteTextures.push_back(destination);
+ }
+};
+
Indium::PrivateBlitCommandEncoder::~PrivateBlitCommandEncoder() {
// nothing for now
};
-void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Buffer> source, size_t sourceOffset, std::shared_ptr<Buffer> destination, size_t destinationOffset, size_t size) {
+void Indium::PrivateBlitCommandEncoder::copy(const std::shared_ptr<Buffer>& source, size_t sourceOffset, const std::shared_ptr<Buffer>& destination, size_t destinationOffset, size_t size) {
auto cmdbuf = _privateCommandBuffer.lock();
- auto privateSource = std::dynamic_pointer_cast<PrivateBuffer>(source);
- auto privateDest = std::dynamic_pointer_cast<PrivateBuffer>(destination);
+ auto* privateSource = dynamic_cast<PrivateBuffer*>(source.get());
+ auto* privateDest = dynamic_cast<PrivateBuffer*>(destination.get());
VkBufferCopy info {};
@@ -37,18 +51,18 @@ void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Buffer> source, siz
DynamicVK::vkCmdCopyBuffer(cmdbuf->commandBuffer(), privateSource->buffer(), privateDest->buffer(), 1, &info);
};
-void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Buffer> source, size_t sourceOffset, size_t sourceBytesPerRow, size_t sourceBytesPerImage, Size sourceSize, std::shared_ptr<Texture> destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin, BlitOption options) {
+void Indium::PrivateBlitCommandEncoder::copy(const std::shared_ptr<Buffer>& source, size_t sourceOffset, size_t sourceBytesPerRow, size_t sourceBytesPerImage, Size sourceSize, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin, BlitOption options) {
+ noteDestination(destination);
auto cmdbuf = _privateCommandBuffer.lock();
- auto privateSource = std::dynamic_pointer_cast<PrivateBuffer>(source);
- auto privateDest = std::dynamic_pointer_cast<PrivateTexture>(destination);
+ auto* privateSource = dynamic_cast<PrivateBuffer*>(source.get());
+ auto* privateDest = dynamic_cast<PrivateTexture*>(destination.get());
if (options != BlitOption::None) {
throw std::runtime_error("TODO: support blit options");
}
auto aspect = pixelFormatToVkImageAspectFlags(privateDest->pixelFormat());
- // FIXME: handle compressed formats
- size_t bytesPerPixel = pixelFormatToByteCount(privateDest->pixelFormat());
+ auto pitch = bufferImagePitch(privateDest->pixelFormat(), sourceSize.width, sourceSize.height, sourceBytesPerRow, sourceBytesPerImage);
// first, transition the image to the optimal layout for transfer destinations
VkImageMemoryBarrier barrier {};
@@ -71,9 +85,9 @@ void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Buffer> source, siz
// now encode the copy
VkBufferImageCopy copyInfo {};
- copyInfo.bufferOffset = 0;
- copyInfo.bufferRowLength = sourceBytesPerRow / bytesPerPixel;
- copyInfo.bufferImageHeight = sourceBytesPerImage / sourceBytesPerRow;
+ copyInfo.bufferOffset = sourceOffset;
+ copyInfo.bufferRowLength = pitch.rowLength;
+ copyInfo.bufferImageHeight = pitch.imageHeight;
copyInfo.imageSubresource.aspectMask = aspect;
copyInfo.imageSubresource.mipLevel = destinationLevel;
copyInfo.imageSubresource.baseArrayLayer = destinationSlice;
@@ -95,18 +109,17 @@ void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Buffer> source, siz
DynamicVK::vkCmdPipelineBarrier(cmdbuf->commandBuffer(), VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
};
-void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Texture> source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, std::shared_ptr<Buffer> destination, size_t destinationOffset, size_t destinationBytesPerRow, size_t destinationBytesPerImage, BlitOption options) {
+void Indium::PrivateBlitCommandEncoder::copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, const std::shared_ptr<Buffer>& destination, size_t destinationOffset, size_t destinationBytesPerRow, size_t destinationBytesPerImage, BlitOption options) {
auto cmdbuf = _privateCommandBuffer.lock();
- auto privateSource = std::dynamic_pointer_cast<PrivateTexture>(source);
- auto privateDest = std::dynamic_pointer_cast<PrivateBuffer>(destination);
+ auto* privateSource = dynamic_cast<PrivateTexture*>(source.get());
+ auto* privateDest = dynamic_cast<PrivateBuffer*>(destination.get());
if (options != BlitOption::None) {
throw std::runtime_error("TODO: support blit options");
}
auto aspect = pixelFormatToVkImageAspectFlags(privateSource->pixelFormat());
- // FIXME: handle compressed formats
- size_t bytesPerPixel = pixelFormatToByteCount(privateSource->pixelFormat());
+ auto pitch = bufferImagePitch(privateSource->pixelFormat(), sourceSize.width, sourceSize.height, destinationBytesPerRow, destinationBytesPerImage);
// first, transition the image to the optimal layout for transfer sources
VkImageMemoryBarrier barrier {};
@@ -129,9 +142,9 @@ void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Texture> source, si
// now encode the copy
VkBufferImageCopy copyInfo {};
- copyInfo.bufferOffset = 0;
- copyInfo.bufferRowLength = destinationBytesPerRow / bytesPerPixel;
- copyInfo.bufferImageHeight = destinationBytesPerImage / destinationBytesPerRow;
+ copyInfo.bufferOffset = destinationOffset;
+ copyInfo.bufferRowLength = pitch.rowLength;
+ copyInfo.bufferImageHeight = pitch.imageHeight;
copyInfo.imageSubresource.aspectMask = aspect;
copyInfo.imageSubresource.mipLevel = sourceLevel;
copyInfo.imageSubresource.baseArrayLayer = sourceSlice;
@@ -153,10 +166,11 @@ void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Texture> source, si
DynamicVK::vkCmdPipelineBarrier(cmdbuf->commandBuffer(), VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
};
-void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Texture> source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, std::shared_ptr<Texture> destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin, size_t sliceCount, size_t levelCount, Size size) {
+void Indium::PrivateBlitCommandEncoder::copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin, size_t sliceCount, size_t levelCount, Size size) {
+ noteDestination(destination);
auto cmdbuf = _privateCommandBuffer.lock();
- auto privateSource = std::dynamic_pointer_cast<PrivateTexture>(source);
- auto privateDest = std::dynamic_pointer_cast<PrivateTexture>(destination);
+ auto* privateSource = dynamic_cast<PrivateTexture*>(source.get());
+ auto* privateDest = dynamic_cast<PrivateTexture*>(destination.get());
auto aspect = pixelFormatToVkImageAspectFlags(privateSource->pixelFormat());
@@ -231,10 +245,10 @@ void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Texture> source, si
DynamicVK::vkCmdPipelineBarrier(cmdbuf->commandBuffer(), VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, sizeof(barriers) / sizeof(*barriers), barriers);
};
-void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Texture> source, std::shared_ptr<Texture> destination) {
+void Indium::PrivateBlitCommandEncoder::copy(const std::shared_ptr<Texture>& source, const std::shared_ptr<Texture>& destination) {
auto cmdbuf = _privateCommandBuffer.lock();
- auto privateSource = std::dynamic_pointer_cast<PrivateTexture>(source);
- auto privateDest = std::dynamic_pointer_cast<PrivateTexture>(destination);
+ auto* privateSource = dynamic_cast<PrivateTexture*>(source.get());
+ auto* privateDest = dynamic_cast<PrivateTexture*>(destination.get());
size_t sourceMipLevel = SIZE_MAX;
size_t destMipLevel = SIZE_MAX;
@@ -282,18 +296,18 @@ void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Texture> source, st
copy(source, 0, sourceMipLevel, Origin { 0, 0, 0 }, destination, 0, destMipLevel, Origin { 0, 0, 0 }, sliceCount, mipCount, Size { source->width() / divisor, source->height() / divisor, source->depth() / divisor });
};
-void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Texture> source, size_t sourceSlice, size_t sourceLevel, std::shared_ptr<Texture> destination, size_t destinationSlice, size_t destinationLevel, size_t sliceCount, size_t levelCount) {
+void Indium::PrivateBlitCommandEncoder::copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, size_t sliceCount, size_t levelCount) {
size_t divisor = static_cast<size_t>(1) << sourceLevel;
copy(source, sourceSlice, sourceLevel, Origin { 0, 0, 0 }, destination, destinationSlice, destinationLevel, Origin { 0, 0, 0 }, sliceCount, levelCount, Size { source->width() / divisor, source->height() / divisor, source->depth() / divisor });
};
-void Indium::PrivateBlitCommandEncoder::copy(std::shared_ptr<Texture> source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, std::shared_ptr<Texture> destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin) {
+void Indium::PrivateBlitCommandEncoder::copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin) {
copy(source, sourceSlice, sourceLevel, sourceOrigin, destination, destinationSlice, destinationLevel, destinationOrigin, 1, 1, sourceSize);
};
-void Indium::PrivateBlitCommandEncoder::fillBuffer(std::shared_ptr<Buffer> buffer, Range<size_t> range, uint8_t value) {
+void Indium::PrivateBlitCommandEncoder::fillBuffer(const std::shared_ptr<Buffer>& buffer, Range<size_t> range, uint8_t value) {
auto cmdbuf = _privateCommandBuffer.lock();
- auto privateBuffer = std::dynamic_pointer_cast<PrivateBuffer>(buffer);
+ auto* privateBuffer = dynamic_cast<PrivateBuffer*>(buffer.get());
uint32_t value32 =
((uint32_t)value << 24) |
((uint32_t)value << 16) |
@@ -314,8 +328,9 @@ void Indium::PrivateBlitCommandEncoder::fillBuffer(std::shared_ptr<Buffer> buffe
DynamicVK::vkCmdFillBuffer(cmdbuf->commandBuffer(), privateBuffer->buffer(), range.start, range.length, value32);
};
-void Indium::PrivateBlitCommandEncoder::generateMipmapsForTexture(std::shared_ptr<Texture> texture) {
- auto privateTexture = std::dynamic_pointer_cast<PrivateTexture>(texture);
+void Indium::PrivateBlitCommandEncoder::generateMipmapsForTexture(const std::shared_ptr<Texture>& texture) {
+ noteDestination(texture);
+ auto* privateTexture = dynamic_cast<PrivateTexture*>(texture.get());
auto cmdbuf = _privateCommandBuffer.lock();
auto aspect = pixelFormatToVkImageAspectFlags(privateTexture->pixelFormat());
@@ -439,5 +454,5 @@ void Indium::PrivateBlitCommandEncoder::generateMipmapsForTexture(std::shared_pt
};
void Indium::PrivateBlitCommandEncoder::endEncoding() {
- // nothing for now
+ finishTrackedEncoder(_privateCommandBuffer.lock()->commandBuffer());
};
diff --git a/src/indium/buffer.cpp b/src/indium/buffer.cpp
index 23fd2d3..1a48b81 100644
--- a/src/indium/buffer.cpp
+++ b/src/indium/buffer.cpp
@@ -1,6 +1,8 @@
#include <indium/buffer.private.hpp>
#include <indium/device.private.hpp>
#include <indium/dynamic-vk.hpp>
+#include <indium/counters.private.hpp>
+#include <indium/memory-pool.private.hpp>
#include <cstring>
@@ -43,49 +45,17 @@ Indium::PrivateBuffer::PrivateBuffer(std::shared_ptr<PrivateDevice> device, size
DynamicVK::vkGetBufferMemoryRequirements(_privateDevice->device(), _buffer, &requirements);
- size_t targetIndex = SIZE_MAX;
-
- for (size_t i = 0; i < _privateDevice->memoryProperties().memoryTypeCount; ++i) {
- const auto& type = _privateDevice->memoryProperties().memoryTypes[i];
-
- if ((requirements.memoryTypeBits & (1 << i)) == 0) {
- continue;
- }
-
- if ((_storageMode == StorageMode::Managed || _storageMode == StorageMode::Shared) && (type.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0) {
- continue;
- }
-
- if (_storageMode == StorageMode::Shared && (type.propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) == 0) {
- continue;
- }
-
- // okay, this is good enough
- targetIndex = i;
- break;
- }
+ size_t targetIndex = _privateDevice->findMemoryType(requirements.memoryTypeBits, _storageMode, false);
if (targetIndex == SIZE_MAX) {
throw std::runtime_error("No suitable memory region found for buffer with requested storage mode");
}
- VkMemoryAllocateInfo allocateInfo {};
- allocateInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
- allocateInfo.allocationSize = requirements.size;
- allocateInfo.memoryTypeIndex = targetIndex;
+ Indium::count(length < 4096 ? "bufferAlloc<4K" : (length < 65536 ? "bufferAlloc<64K" : (length < 4194304 ? "bufferAlloc<4M" : "bufferAlloc>=4M")), length);
+ auto allocation = MemoryPool::allocate(_privateDevice->device(), reinterpret_cast<uint64_t>(_buffer), targetIndex, requirements.size, requirements.alignment, false, true);
+ _memory = allocation.memory;
- VkMemoryAllocateFlagsInfo allocateFlags {};
- allocateFlags.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO;
- allocateFlags.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT;
-
- allocateInfo.pNext = &allocateFlags;
-
- if (DynamicVK::vkAllocateMemory(_privateDevice->device(), &allocateInfo, nullptr, &_memory) != VK_SUCCESS) {
- // TODO
- abort();
- }
-
- DynamicVK::vkBindBufferMemory(_privateDevice->device(), _buffer, _memory, 0);
+ DynamicVK::vkBindBufferMemory(_privateDevice->device(), _buffer, _memory, allocation.offset);
};
Indium::PrivateBuffer::PrivateBuffer(std::shared_ptr<PrivateDevice> device, const void* pointer, size_t length, ResourceOptions options):
@@ -103,9 +73,11 @@ Indium::PrivateBuffer::PrivateBuffer(std::shared_ptr<PrivateDevice> device, cons
if (_storageMode == StorageMode::Managed) {
VkMappedMemoryRange range {};
range.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE;
+ MemoryPool::Allocation allocation;
+ MemoryPool::lookup(reinterpret_cast<uint64_t>(_buffer), allocation);
range.memory = _memory;
- range.size = VK_WHOLE_SIZE;
- range.offset = 0;
+ range.size = allocation.size;
+ range.offset = allocation.offset;
if (DynamicVK::vkFlushMappedMemoryRanges(_privateDevice->device(), 1, &range) != VK_SUCCESS) {
// TODO
abort();
@@ -114,12 +86,10 @@ Indium::PrivateBuffer::PrivateBuffer(std::shared_ptr<PrivateDevice> device, cons
};
Indium::PrivateBuffer::~PrivateBuffer() {
- if (_mapped) {
- DynamicVK::vkUnmapMemory(_privateDevice->device(), _memory);
- }
-
+ // the mapping belongs to the shared memory block
+ auto owner = reinterpret_cast<uint64_t>(_buffer);
DynamicVK::vkDestroyBuffer(_privateDevice->device(), _buffer, nullptr);
- DynamicVK::vkFreeMemory(_privateDevice->device(), _memory, nullptr);
+ MemoryPool::free(_privateDevice->device(), owner);
};
std::shared_ptr<Indium::Device> Indium::PrivateBuffer::device() {
@@ -136,7 +106,8 @@ void* Indium::PrivateBuffer::contents() {
}
if (!_mapped) {
- if (DynamicVK::vkMapMemory(_privateDevice->device(), _memory, 0, VK_WHOLE_SIZE, 0, &_mapped) != VK_SUCCESS) {
+ _mapped = MemoryPool::map(_privateDevice->device(), reinterpret_cast<uint64_t>(_buffer));
+ if (!_mapped) {
// TODO
abort();
}
@@ -148,9 +119,16 @@ void* Indium::PrivateBuffer::contents() {
void Indium::PrivateBuffer::didModifyRange(Range<size_t> range) {
VkMappedMemoryRange vulkanRange {};
vulkanRange.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE;
+ // flush whole 256-byte granules of this buffer's part of the block (granule >= nonCoherentAtomSize)
+ MemoryPool::Allocation allocation;
+ MemoryPool::lookup(reinterpret_cast<uint64_t>(_buffer), allocation);
+ VkDeviceSize first = range.start / 256 * 256;
+ VkDeviceSize last = (range.start + range.length + 255) / 256 * 256;
+ if (last > allocation.size) last = allocation.size;
+ if (first >= last) return;
vulkanRange.memory = _memory;
- vulkanRange.size = range.length;
- vulkanRange.offset = range.start;
+ vulkanRange.size = last - first;
+ vulkanRange.offset = allocation.offset + first;
if (DynamicVK::vkFlushMappedMemoryRanges(_privateDevice->device(), 1, &vulkanRange) != VK_SUCCESS) {
// TODO
abort();
@@ -158,8 +136,10 @@ void Indium::PrivateBuffer::didModifyRange(Range<size_t> range) {
};
uint64_t Indium::PrivateBuffer::gpuAddress() {
+ if (_gpuAddress) return _gpuAddress;
VkBufferDeviceAddressInfo info {};
info.sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO;
info.buffer = _buffer;
- return DynamicVK::vkGetBufferDeviceAddress(_privateDevice->device(), &info);
+ _gpuAddress = DynamicVK::vkGetBufferDeviceAddress(_privateDevice->device(), &info);
+ return _gpuAddress;
};
diff --git a/src/indium/command-buffer.cpp b/src/indium/command-buffer.cpp
index 43f4a84..f7640c5 100644
--- a/src/indium/command-buffer.cpp
+++ b/src/indium/command-buffer.cpp
@@ -1,3 +1,8 @@
+#include <indium/submit-thread.private.hpp>
+#include <indium/gpu-timing.private.hpp>
+#include <indium/submit-batch.private.hpp>
+#include <indium/deferred-render-encoder.private.hpp>
+#include <indium/deferred-blit-encoder.private.hpp>
#include <indium/command-buffer.private.hpp>
#include <indium/command-queue.private.hpp>
#include <indium/device.private.hpp>
@@ -8,7 +13,116 @@
#include <indium/compute-command-encoder.private.hpp>
#include <indium/dynamic-vk.hpp>
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <chrono>
+#include <cstdio>
+#include <map>
+#include <mutex>
+#include <string>
+#include <unistd.h>
+#include <indium/counters.private.hpp>
#include <condition_variable>
+#include <indium/retirement-queue.private.hpp>
+
+// Aggregated timings, printed every 5 s, when /tmp/metal-debug-timing exists in the guest at process start
+static bool indiumTimingEnabled() {
+ static const bool enabled = access("/tmp/metal-debug-timing", F_OK) == 0;
+ return enabled;
+}
+static double indiumNow() {
+ return std::chrono::duration<double>(std::chrono::steady_clock::now().time_since_epoch()).count();
+}
+static void indiumTiming(const char* what, double seconds) {
+ struct Stat { double sum = 0, max = 0; unsigned long count = 0; };
+ static std::mutex lock;
+ static std::map<std::string, Stat> stats;
+ static double last = 0;
+ std::lock_guard<std::mutex> guard(lock);
+ Stat& stat = stats[what];
+ stat.sum += seconds;
+ stat.count++;
+ if (seconds > stat.max) stat.max = seconds;
+ double now = indiumNow();
+ if (now - last >= 5) {
+ last = now;
+ if (FILE* log = fopen("/tmp/metal-trace.log", "a")) {
+ fprintf(log, "indium-timing@%.0f:", now);
+ for (auto& entry: stats) {
+ fprintf(log, " %s[n=%lu avg=%.1fms max=%.1fms]", entry.first.c_str(), entry.second.count, entry.second.sum / entry.second.count * 1000, entry.second.max * 1000);
+ }
+ fprintf(log, "\n");
+ fclose(log);
+ }
+ stats.clear();
+ }
+}
+
+// Opt-in, bounded capture: no per-frame file I/O, no draw logging.
+// Timestamps describe presenting command-buffer commits, not compositor scanout.
+namespace {
+struct FrameIntervalSample {
+ std::atomic<int64_t> commit{0}, submitted{0}, presented{0}, completed{0};
+};
+static int64_t frameIntervalNow() {
+ return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
+}
+static FrameIntervalSample* beginFrameIntervalSample(bool presenting) {
+ if (!presenting) return nullptr;
+ static const int64_t start = frameIntervalNow();
+ static std::atomic<bool> enabled {false};
+ static std::atomic<size_t> checks {0};
+ // Late activation permits profiling the existing match without another login.
+ // Only one file check per 128 presenting frames until activated; none after.
+ if (!enabled.load(std::memory_order_relaxed) &&
+ (checks.fetch_add(1, std::memory_order_relaxed) % 128) == 0) {
+ if (FILE* flag = fopen("/tmp/metal-frame-interval-capture", "r")) {
+ fclose(flag);
+ enabled.store(true, std::memory_order_relaxed);
+ }
+ }
+ if (!enabled.load(std::memory_order_relaxed)) return nullptr;
+ const int64_t now = frameIntervalNow();
+ static const bool testing = getenv("RIOT_FRAME_CAPTURE_TEST") != nullptr;
+ if (!testing && now - start < 90000000000LL) return nullptr; // discard reconnect/warmup
+ static std::array<FrameIntervalSample, 4096> samples;
+ static std::atomic<size_t> count{0};
+ static std::atomic<bool> dumped{false};
+ static const size_t limit = testing ? 8 : samples.size();
+ const size_t index = count.fetch_add(1, std::memory_order_relaxed);
+ if (index < limit) {
+ samples[index].commit.store(now, std::memory_order_relaxed);
+ return &samples[index];
+ }
+ if (dumped.load(std::memory_order_relaxed)) return nullptr;
+
+ if (dumped.exchange(true)) return nullptr;
+ if (FILE* log = fopen("/tmp/metal-frame-intervals.csv", "w")) {
+ fprintf(log, "commit_ns,submitted_ns,presented_ns,completed_ns\n");
+ for (size_t i=0;i<limit;++i) { const auto& sample=samples[i]; fprintf(log, "%lld,%lld,%lld,%lld\n", (long long)sample.commit.load(), (long long)sample.submitted.load(), (long long)sample.presented.load(), (long long)sample.completed.load()); }
+ fclose(log);
+ }
+ return nullptr;
+}
+}
+
+namespace {
+// Submission counters for A/B runs: one line per 1024 presented frames in guest /tmp/indium-submit-stats.log.
+std::atomic<uint64_t> statCommits {0}, statSubmits {0}, statPresents {0};
+void submitStatsPresented() {
+ const uint64_t presents = statPresents.fetch_add(1, std::memory_order_relaxed) + 1;
+ if (presents % 1024 != 0) return;
+ static uint64_t lastCommits = 0, lastSubmits = 0;
+ const uint64_t commits = statCommits.load(std::memory_order_relaxed), submits = statSubmits.load(std::memory_order_relaxed);
+ if (FILE* log = fopen("/tmp/indium-submit-stats.log", "a")) {
+ fprintf(log, "%lld batch=%d commits_per_frame=%.2f submits_per_frame=%.2f\n", (long long)frameIntervalNow(), (int)Indium::submitBatchEnabled(),
+ (commits - lastCommits) / 1024.0, (submits - lastSubmits) / 1024.0);
+ fclose(log);
+ }
+ lastCommits = commits; lastSubmits = submits;
+}
+}
Indium::CommandBuffer::~CommandBuffer() {};
@@ -19,18 +133,47 @@ std::shared_ptr<Indium::Device> Indium::PrivateCommandBuffer::device() {
return _privateDevice;
};
+namespace {
+// OFF by default (round-3 behaviour: the creating thread leases the Vulkan command buffer in the constructor). Guest
+// flag file /tmp/metal-worker-lease (checked once per process, so the constructor and every later recorder agree)
+// makes the lease lazy: the thread that first records leases it, which with deferred encoding is the submit worker.
+// Round 4 had this on unconditionally; that round regressed in matches, so it is an opt-in experiment now.
+bool workerLeaseEnabled() {
+ static const bool value = access("/tmp/metal-worker-lease", F_OK) == 0;
+ return value;
+}
+}
+
Indium::PrivateCommandBuffer::PrivateCommandBuffer(std::shared_ptr<PrivateCommandQueue> commandQueue):
_privateCommandQueue(commandQueue)
{
_privateDevice = _privateCommandQueue->privateDevice();
+ // Default: lease now, on the creating thread. With /tmp/metal-worker-lease the Vulkan command buffer (pool reset,
+ // allocation, begin) is leased lazily by the thread that first records into it (see ensureRecordingStarted()).
+ if (!workerLeaseEnabled()) leaseRecording();
+};
+
+// Only called with /tmp/metal-worker-lease (callers test workerLeaseEnabled() first; without the flag the
+// constructor has already leased).
+void Indium::PrivateCommandBuffer::ensureRecordingStarted() {
+ std::scoped_lock lock(_mutex);
+ if (_commandBuffer) return;
+ leaseRecording();
+};
+
+// The lease itself; runs in the constructor or under _mutex in ensureRecordingStarted().
+void Indium::PrivateCommandBuffer::leaseRecording() {
+ // A pool is exclusively leased through recording and GPU completion.
+ // Freeing another buffer from the same pool can otherwise race recording.
+ _recordingPool = _privateCommandQueue->acquireCommandPool(_commandBuffer);
VkCommandBufferAllocateInfo allocInfo {};
allocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
- allocInfo.commandPool = _privateCommandQueue->commandPool();
+ allocInfo.commandPool = _recordingPool;
allocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
allocInfo.commandBufferCount = 1;
- if (DynamicVK::vkAllocateCommandBuffers(_privateDevice->device(), &allocInfo, &_commandBuffer) != VK_SUCCESS) {
+ if (!_commandBuffer && DynamicVK::vkAllocateCommandBuffers(_privateDevice->device(), &allocInfo, &_commandBuffer) != VK_SUCCESS) {
// TODO: handle this in a more C++-friendly way
abort();
}
@@ -41,13 +184,64 @@ Indium::PrivateCommandBuffer::PrivateCommandBuffer(std::shared_ptr<PrivateComman
// TODO: same
abort();
}
+ GpuTiming::beginSpan(_privateDevice->device(), _commandBuffer);
};
+namespace {
+struct EncoderRetirement {
+ std::vector<std::shared_ptr<Indium::CommandEncoder>> encoders;
+ std::shared_ptr<Indium::PrivateCommandQueue> queue;
+ VkCommandPool pool;
+ VkCommandBuffer buffer;
+ void operator()() {
+ encoders.clear();
+ // a command buffer that never recorded (created and dropped without an encoder) leased no pool
+ if (pool) queue->releaseCommandPool(pool, buffer);
+ queue.reset();
+ }
+};
+static Indium::RetirementQueue<EncoderRetirement>& encoderRetirementQueue() {
+ static Indium::RetirementQueue<EncoderRetirement> queue;
+ return queue;
+}
+}
+namespace Indium {
+void finishCommandRetirement() { encoderRetirementQueue().shutdown(); }
+}
+namespace {
+// Live experiment (guest flag, re-read every 256 command buffers like the async-encode flag):
+// /tmp/metal-retire-tiny makes the tiny buffers take the worker path too, i.e. the encoder
+// destructors and the command-pool release run on the retirement thread instead of the glyph
+// thread that released the MTLCommandBuffer. Only the *cost* is in question here - the handoff is
+// the same one heavy buffers already use, the queue's contract is completed GPU work (and an
+// unsubmitted buffer holds no GPU work), and a full queue still falls back to the inline release -
+// so the flag is the measurement, and the default stays the measured-cheaper inline release.
+bool retireTinyBuffers() {
+ static std::atomic<unsigned> calls { 0 };
+ static std::atomic<bool> value { false };
+ if ((calls.fetch_add(1, std::memory_order_relaxed) & 255) == 0)
+ value.store(access("/tmp/metal-retire-tiny", F_OK) == 0, std::memory_order_relaxed);
+ return value.load(std::memory_order_relaxed);
+}
+}
Indium::PrivateCommandBuffer::~PrivateCommandBuffer() {
- DynamicVK::vkFreeCommandBuffers(_privateDevice->device(), _privateCommandQueue->commandPool(), 1, &_commandBuffer);
+ // Submitted buffers are held by the completion callback until GPU work and
+ // user completion handlers finish. Unsubmitted buffers have no GPU users.
+ // Tiny buffers cost less to release directly than to signal a Darwin
+ // condition variable and wake another worker through Darling's server.
+ bool heavy = _commandEncoders.size() >= 4 || retireTinyBuffers();
+ if (!heavy) {
+ for (const auto& encoder : _commandEncoders) {
+ auto* render = dynamic_cast<PrivateRenderCommandEncoder*>(encoder.get());
+ if (render && render->hasHeavyRetirement()) { heavy = true; break; }
+ }
+ }
+ EncoderRetirement job {std::move(_commandEncoders), _privateCommandQueue, _recordingPool, _commandBuffer};
+ if (!heavy || !encoderRetirementQueue().tryPush(std::move(job))) job();
};
-std::shared_ptr<Indium::RenderCommandEncoder> Indium::PrivateCommandBuffer::renderCommandEncoder(const RenderPassDescriptor& descriptor) {
+std::shared_ptr<Indium::PrivateRenderCommandEncoder> Indium::PrivateCommandBuffer::renderCommandEncoderNow(const RenderPassDescriptor& descriptor) {
+ if (workerLeaseEnabled()) ensureRecordingStarted();
auto encoder = std::make_shared<Indium::PrivateRenderCommandEncoder>(shared_from_this(), descriptor);
{
std::scoped_lock lock(_mutex);
@@ -56,7 +250,65 @@ std::shared_ptr<Indium::RenderCommandEncoder> Indium::PrivateCommandBuffer::rend
return encoder;
};
+std::shared_ptr<Indium::PrivateBlitCommandEncoder> Indium::PrivateCommandBuffer::blitCommandEncoderNow() {
+ return blitCommandEncoderNow(BlitPassDescriptor {});
+};
+
+// Creates the real blit encoder on whichever thread replays the deferred op list (see deferred-blit-encoder.private.hpp).
+std::shared_ptr<Indium::PrivateBlitCommandEncoder> Indium::PrivateCommandBuffer::blitCommandEncoderNow(const BlitPassDescriptor& descriptor) {
+ if (workerLeaseEnabled()) ensureRecordingStarted();
+ auto encoder = std::make_shared<Indium::PrivateBlitCommandEncoder>(shared_from_this(), descriptor);
+ {
+ std::scoped_lock lock(_mutex);
+ _commandEncoders.push_back(encoder);
+ }
+ return encoder;
+};
+
+std::shared_ptr<Indium::PrivateComputeCommandEncoder> Indium::PrivateCommandBuffer::computeCommandEncoderNow(const ComputePassDescriptor& descriptor) {
+ if (workerLeaseEnabled()) ensureRecordingStarted();
+ auto encoder = std::make_shared<Indium::PrivateComputeCommandEncoder>(shared_from_this(), descriptor);
+ {
+ std::scoped_lock lock(_mutex);
+ _commandEncoders.push_back(encoder);
+ }
+ return encoder;
+};
+
+void Indium::PrivateCommandBuffer::deferredBegin() {
+ std::lock_guard<std::mutex> lock(_deferredMutex);
+ ++_deferredPending;
+}
+
+void Indium::PrivateCommandBuffer::deferredEnd() {
+ {
+ std::lock_guard<std::mutex> lock(_deferredMutex);
+ --_deferredPending;
+ }
+ _deferredCondvar.notify_all();
+}
+
+void Indium::PrivateCommandBuffer::waitDeferred() {
+ std::unique_lock<std::mutex> lock(_deferredMutex);
+ Waits::blockOn("waitDeferred", _deferredCondvar, lock, [this] { return _deferredPending == 0; });
+}
+
+std::shared_ptr<Indium::RenderCommandEncoder> Indium::PrivateCommandBuffer::renderCommandEncoder(const RenderPassDescriptor& descriptor) {
+ if (AsyncEncode::enabled()) {
+ auto encoder = std::make_shared<AsyncEncode::DeferredRenderCommandEncoder>(shared_from_this(), descriptor);
+ std::scoped_lock lock(_mutex);
+ _commandEncoders.push_back(encoder);
+ return encoder;
+ }
+ waitDeferred(); // an earlier encoder of this buffer may still be waiting for the worker
+ return renderCommandEncoderNow(descriptor);
+};
+
std::shared_ptr<Indium::BlitCommandEncoder> Indium::PrivateCommandBuffer::blitCommandEncoder() {
+ // the opt-in flag is tested first: AsyncEncode::enabled() bumps a shared call counter (and re-reads its flag files
+ // every 256 calls), which the default path must not pay or perturb
+ if (AsyncEncode::deferBlitEnabled() && AsyncEncode::enabled()) return blitCommandEncoder(BlitPassDescriptor {});
+ waitDeferred(); // never record into the Vulkan command buffer while deferred render encoders are being replayed
auto encoder = std::make_shared<Indium::PrivateBlitCommandEncoder>(shared_from_this());
{
std::scoped_lock lock(_mutex);
@@ -66,6 +318,16 @@ std::shared_ptr<Indium::BlitCommandEncoder> Indium::PrivateCommandBuffer::blitCo
};
std::shared_ptr<Indium::BlitCommandEncoder> Indium::PrivateCommandBuffer::blitCommandEncoder(const BlitPassDescriptor& descriptor) {
+ if (AsyncEncode::deferBlitEnabled() && AsyncEncode::enabled()) {
+ // records on the caller, replays on the ordered worker: no wait, no vkCmd* on the game's thread
+ auto encoder = std::make_shared<AsyncEncode::DeferredBlitCommandEncoder>(shared_from_this(), descriptor);
+ {
+ std::scoped_lock lock(_mutex);
+ _commandEncoders.push_back(encoder);
+ }
+ return encoder;
+ }
+ waitDeferred(); // never record into the Vulkan command buffer while deferred render encoders are being replayed
auto encoder = std::make_shared<Indium::PrivateBlitCommandEncoder>(shared_from_this(), descriptor);
{
std::scoped_lock lock(_mutex);
@@ -79,6 +341,7 @@ std::shared_ptr<Indium::ComputeCommandEncoder> Indium::PrivateCommandBuffer::com
};
std::shared_ptr<Indium::ComputeCommandEncoder> Indium::PrivateCommandBuffer::computeCommandEncoder(const ComputePassDescriptor& descriptor) {
+ waitDeferred(); // never record into the Vulkan command buffer while deferred render encoders are being replayed
auto encoder = std::make_shared<Indium::PrivateComputeCommandEncoder>(shared_from_this(), descriptor);
{
std::scoped_lock lock(_mutex);
@@ -91,10 +354,269 @@ std::shared_ptr<Indium::ComputeCommandEncoder> Indium::PrivateCommandBuffer::com
return computeCommandEncoder(ComputePassDescriptor { {}, dispatchType });
};
+namespace {
+// Pending merged submission. Everything is guarded by the device's graphicsQueueMutex (held by every submitter).
+struct PendingSubmit {
+ VkCommandBuffer commandBuffer;
+ std::vector<VkSemaphoreSubmitInfo> waits;
+ std::vector<VkSemaphoreSubmitInfo> signals;
+ FrameIntervalSample* frameSample = nullptr;
+};
+
+struct SubmitBatchState {
+ std::vector<PendingSubmit> items;
+ std::map<VkSemaphore, uint64_t> signaled; // timeline value signalled inside the pending batch
+ std::shared_ptr<Indium::PrivateDevice> device;
+ VkCommandPool barrierPool = VK_NULL_HANDLE;
+ VkCommandBuffer barrierCommandBuffer = VK_NULL_HANDLE;
+ std::chrono::steady_clock::time_point since;
+ bool timerStarted = false;
+};
+
+SubmitBatchState& submitBatchState() {
+ static SubmitBatchState* state = new SubmitBatchState;
+ return *state;
+}
+
+std::condition_variable& submitBatchWake() {
+ static std::condition_variable* cv = new std::condition_variable;
+ return *cv;
+}
+
+VkCommandBuffer submitBatchBarrier(const std::shared_ptr<Indium::PrivateDevice>& device) {
+ auto& state = submitBatchState();
+ if (state.barrierCommandBuffer) return state.barrierCommandBuffer;
+ // Queue submission must not acquire the upload pool lock: uploaders take that lock before the queue lock.
+ // Keep the immutable batch barrier in its own process-lifetime pool, guarded by the queue mutex.
+ VkCommandPoolCreateInfo poolInfo {};
+ poolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
+ poolInfo.queueFamilyIndex = *device->graphicsQueueFamilyIndex();
+ if (Indium::DynamicVK::vkCreateCommandPool(device->device(), &poolInfo, nullptr, &state.barrierPool) != VK_SUCCESS) abort();
+ VkCommandBufferAllocateInfo allocInfo {};
+ allocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
+ allocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
+ allocInfo.commandPool = state.barrierPool;
+ allocInfo.commandBufferCount = 1;
+ VkCommandBuffer cb = VK_NULL_HANDLE;
+ if (Indium::DynamicVK::vkAllocateCommandBuffers(device->device(), &allocInfo, &cb) != VK_SUCCESS) abort();
+ VkCommandBufferBeginInfo begin {};
+ begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
+ begin.flags = VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT;
+ Indium::DynamicVK::vkBeginCommandBuffer(cb, &begin);
+ VkMemoryBarrier barrier {};
+ barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
+ barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
+ barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
+ Indium::DynamicVK::vkCmdPipelineBarrier(cb, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &barrier, 0, nullptr, 0, nullptr);
+ Indium::DynamicVK::vkEndCommandBuffer(cb);
+ state.barrierCommandBuffer = cb;
+ return cb;
+}
+
+// caller holds the graphics queue mutex
+void flushSubmitBatchLocked() {
+ auto& state = submitBatchState();
+ if (state.items.empty()) return;
+ auto device = state.device;
+ auto items = std::move(state.items);
+ state.items.clear();
+ state.signaled.clear();
+
+ // one batch: external waits (timeline waits deduplicated to the highest value), all signals (highest value per timeline)
+ std::map<VkSemaphore, VkSemaphoreSubmitInfo> waitTimeline, signalTimeline;
+ std::vector<VkSemaphoreSubmitInfo> waitBinary, signalBinary;
+ std::vector<VkCommandBufferSubmitInfo> commandBuffers;
+ const VkCommandBuffer barrier = items.size() > 1 ? submitBatchBarrier(device) : VK_NULL_HANDLE;
+ for (size_t i = 0; i < items.size(); ++i) {
+ if (i > 0) {
+ VkCommandBufferSubmitInfo barrierInfo {};
+ barrierInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO;
+ barrierInfo.commandBuffer = barrier;
+ commandBuffers.push_back(barrierInfo);
+ }
+ VkCommandBufferSubmitInfo info {};
+ info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO;
+ info.commandBuffer = items[i].commandBuffer;
+ commandBuffers.push_back(info);
+ for (const auto& w: items[i].waits) {
+ if (w.value == 0) { waitBinary.push_back(w); continue; }
+ auto it = waitTimeline.find(w.semaphore);
+ if (it == waitTimeline.end() || it->second.value < w.value) waitTimeline[w.semaphore] = w;
+ }
+ for (const auto& sg: items[i].signals) {
+ if (sg.value == 0) { signalBinary.push_back(sg); continue; }
+ auto it = signalTimeline.find(sg.semaphore);
+ if (it == signalTimeline.end() || it->second.value < sg.value) signalTimeline[sg.semaphore] = sg;
+ }
+ }
+ std::vector<VkSemaphoreSubmitInfo> waits = waitBinary, signals = signalBinary;
+ for (const auto& [sem, w]: waitTimeline) waits.push_back(w);
+ for (const auto& [sem, sg]: signalTimeline) signals.push_back(sg);
+
+ VkSubmitInfo2 info {};
+ info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2;
+ info.commandBufferInfoCount = commandBuffers.size();
+ info.pCommandBufferInfos = commandBuffers.data();
+ info.signalSemaphoreInfoCount = signals.size();
+ info.pSignalSemaphoreInfos = signals.data();
+ info.waitSemaphoreInfoCount = waits.size();
+ info.pWaitSemaphoreInfos = waits.data();
+ Indium::DynamicVK::beginDiagnosticCapture(Indium::globalInstance);
+ if (Indium::DynamicVK::vkQueueSubmit2(device->graphicsQueue(), 1, &info, VK_NULL_HANDLE) != VK_SUCCESS) abort();
+ statSubmits.fetch_add(1, std::memory_order_relaxed);
+ Indium::DynamicVK::endDiagnosticCapture(false);
+ Indium::count("batchedSubmits", 1);
+ Indium::count("batchedCommandBuffers", items.size());
+ for (auto& item: items) {
+ if (item.frameSample) item.frameSample->submitted.store(frameIntervalNow(), std::memory_order_release);
+ }
+}
+
+// darling-lol: tunable batch window and size. On the NVIDIA proprietary driver every vkQueueSubmit2 is an RM control
+// ioctl that queues on a driver-wide lock (~0.4 ms each in a League match, ~8.5 submits per frame with the 500 us
+// window), so fewer, larger batches matter there. Guest files, re-read every 256 checks: /tmp/metal-submit-batch-us
+// (window in microseconds, 500-100000, default 500) and /tmp/metal-submit-batch-max (command buffers per batch,
+// 1-256, default 16). A presenting commit and every CPU wait still flush the batch at once.
+long readTuning(const char* path, long fallback, long low, long high) {
+ FILE* file = fopen(path, "r");
+ if (!file) return fallback;
+ long value = fallback;
+ if (fscanf(file, "%ld", &value) != 1) value = fallback;
+ fclose(file);
+ return std::clamp(value, low, high);
+}
+
+struct SubmitBatchTuning {
+ std::atomic<long> windowMicroseconds {500};
+ std::atomic<long> maxItems {16};
+ std::atomic<unsigned> calls {0};
+};
+
+SubmitBatchTuning& submitBatchTuning() {
+ static auto* tuning = new SubmitBatchTuning;
+ if ((tuning->calls.fetch_add(1, std::memory_order_relaxed) & 255) == 0) {
+ tuning->windowMicroseconds.store(readTuning("/tmp/metal-submit-batch-us", 500, 500, 100000), std::memory_order_relaxed);
+ tuning->maxItems.store(readTuning("/tmp/metal-submit-batch-max", 16, 1, 256), std::memory_order_relaxed);
+ }
+ return *tuning;
+}
+
+void startSubmitBatchTimer(const std::shared_ptr<Indium::PrivateDevice>& device) {
+ auto& state = submitBatchState();
+ if (state.timerStarted) return;
+ state.timerStarted = true;
+ std::thread([device]() {
+ auto& state = submitBatchState();
+ std::mutex wakeMutex;
+ for (;;) {
+ {
+ std::unique_lock<std::mutex> lock(wakeMutex);
+ /* darling-lol perf: 500µs batch window balances submission
+ * merging with latency; the old 2ms added visible input lag. */
+ submitBatchWake().wait_for(lock, std::chrono::microseconds(500));
+ }
+ const auto window = std::chrono::microseconds(submitBatchTuning().windowMicroseconds.load(std::memory_order_relaxed));
+ std::unique_lock<std::mutex> queueLock(device->graphicsQueueMutex());
+ if (!state.items.empty() && std::chrono::steady_clock::now() - state.since >= window) flushSubmitBatchLocked();
+ }
+ }).detach();
+}
+}
+
+// ON by default. League commits 29-38 command buffers per frame (one per render pass); submitted one by one, each is an
+// amdgpu_cs ioctl. Merged they become 3-4 submissions per frame: main-thread kernel time 12.1% -> 7.1% of wall time in
+// a Baron fight (three interleaved 8 s windows each), fight 67.8 -> 70.9 fps, auto-attack 89 -> 96+, idle 98 -> 102
+// (2026-10-10, tools/batch-ab.sh). Live switch: guest file /tmp/metal-no-submit-batch turns it off, re-read every 512 calls.
+bool Indium::submitBatchEnabled() {
+ static std::atomic<bool> enabled {true};
+ static std::atomic<unsigned> calls {0};
+ if ((calls.fetch_add(1, std::memory_order_relaxed) & 511) == 0)
+ enabled.store(access("/tmp/metal-no-submit-batch", F_OK) != 0, std::memory_order_relaxed);
+ return enabled.load(std::memory_order_relaxed);
+}
+
+void Indium::flushPendingSubmits() {
+ auto& state = submitBatchState();
+ std::shared_ptr<Indium::PrivateDevice> device = state.device; // set under the queue mutex by the first batched commit
+ if (!device) return;
+ // runs on the game thread (waitUntilCompleted) and on the upload worker; the counter attributes blocks to their caller.
+ if (Waits::enabled() && !SubmitThread::onWorker()) {
+ Waits::blockOnLock("queueMutex", device->graphicsQueueMutex());
+ std::unique_lock<std::mutex> queueLock(device->graphicsQueueMutex(), std::adopt_lock);
+ flushSubmitBatchLocked();
+ return;
+ }
+ std::unique_lock<std::mutex> queueLock(device->graphicsQueueMutex());
+ flushSubmitBatchLocked();
+}
+
+namespace {
+std::atomic<bool> bindingReuse{true};
+void refreshBindingReuse() {
+ static std::atomic<unsigned> calls{0};
+ // A bounded diagnostic switch for same-process combat A/B. Do not perform
+ // filesystem lookups in the per-draw/per-constant hot paths.
+ if ((calls.fetch_add(1,std::memory_order_relaxed) & 63)==0)
+ bindingReuse.store(access("/tmp/metal-no-binding-reuse",F_OK)!=0,std::memory_order_relaxed);
+}
+// Frame-end drain waits for a snapshot of queued work. The opt-in guest flag file
+// /tmp/metal-present-dispatch-wait (checked once per process) makes the frame-end sync narrower: the caller waits only
+// until this commit's own task has run (its present dispatch, and by FIFO order every replay queued before it), not for
+// tasks queued after it. Round 4 had this on by default; that round regressed in matches, so it is opt-in now.
+bool presentDispatchWaitEnabled() {
+ static const bool value = access("/tmp/metal-present-dispatch-wait", F_OK) == 0;
+ return value;
+}
+}
+bool Indium::bindingReuseEnabled() {return bindingReuse.load(std::memory_order_relaxed);}
+
void Indium::PrivateCommandBuffer::commit() {
+ refreshBindingReuse();
+ // With deferred render encoding the encoders are applied on the ordered worker, so the whole commit (it reads what the
+ // encoders recorded, ends the Vulkan command buffer and reserves timeline values in submission order) must run there too.
+ // It also has to when earlier commits are still queued: submission order is commit order.
+ if (!SubmitThread::onWorker() && (AsyncEncode::enabled() || SubmitThread::pending())) {
+ auto self = shared_from_this();
+ // read on the calling thread: the worker's commit consumes the list
+ const bool presents = !_drawablesToPresent.empty();
+ // Frame-end sync: the present is queued to the main queue by the worker's commit. The game pumps its run loop
+ // right after this call, so let the worker catch up first (it ran in parallel during the frame, the wait is its
+ // remaining lag); otherwise the present misses this pump and is shown one frame late, sometimes two per pump.
+ // Guest flag file /tmp/metal-async-no-frame-sync disables it for A/B runs.
+ if (presents && presentDispatchWaitEnabled() && access("/tmp/metal-async-no-frame-sync", F_OK) != 0) {
+ // Opt-in narrow wait (/tmp/metal-present-dispatch-wait): this commit's task signals the token when it is
+ // done, i.e. when the present dispatch (and, because the worker is FIFO, every replay queued before it) has
+ // happened. Waiting for the token instead of the queue's idle state does not block on later tasks.
+ SubmitThread::Token token;
+ SubmitThread::post([self, &token]() {
+ struct Signal {
+ SubmitThread::Token* token;
+ ~Signal() { token->signal(); }
+ } signal { &token };
+ AsyncEncode::PoolScope pool;
+ self->commit();
+ });
+ token.wait("presentDispatchWait");
+ return;
+ }
+ // Post first, then wait for the work already queued; later producers cannot extend this wait.
+ SubmitThread::post([self]() {
+ AsyncEncode::PoolScope pool;
+ self->commit();
+ });
+ if (presents && access("/tmp/metal-async-no-frame-sync", F_OK) != 0) {
+ SubmitThread::drain("drain@commit");
+ }
+ return;
+ }
+ auto* frameSample = beginFrameIntervalSample(!_drawablesToPresent.empty());
+ // with /tmp/metal-worker-lease the commit is the last recorder of this command buffer, so it leases it when nothing
+ // did before (empty buffer); by default the constructor has leased it
+ if (workerLeaseEnabled()) ensureRecordingStarted();
std::unique_lock lock(_mutex);
_committed = true;
+ _committedAtomic.store(true, std::memory_order_release);
for (const auto& encoder: _commandEncoders) {
if (auto renderEncoder = std::dynamic_pointer_cast<PrivateRenderCommandEncoder>(encoder)) {
@@ -110,16 +632,28 @@ void Indium::PrivateCommandBuffer::commit() {
}
}
+ if (auto blitEncoder = std::dynamic_pointer_cast<PrivateBlitCommandEncoder>(encoder)) {
+ for (const auto& texture: blitEncoder->readWriteTextures()) {
+ if (auto privateTexture = std::dynamic_pointer_cast<PrivateTexture>(texture)) {
+ privateTexture->precommit(shared_from_this());
+ }
+ }
+ }
+
// TODO: same for other encoders
}
+ GpuTiming::endSpan(_commandBuffer);
if (DynamicVK::vkEndCommandBuffer(_commandBuffer) != VK_SUCCESS) {
// TODO
abort();
}
- // to keep ourselves alive until we're done
+ // Everything below talks to the Vulkan queue and runs on the ordered submit worker (see submit-thread.private.hpp);
+ // this thread only had to finish recording. The task keeps the command buffer alive until it is done.
+ lock.unlock();
auto self = shared_from_this();
+ auto submitTask = [this, self, frameSample]() {
// for the event loop
auto timelineSemaphore = _privateDevice->getWrappedTimelineSemaphore();
@@ -138,14 +672,49 @@ void Indium::PrivateCommandBuffer::commit() {
readWriteTextures.insert(readWriteTextures.end(), readWrite.begin(), readWrite.end());
}
- // TODO: implement this for other encoders (we need to synchronize those texture accesses as well)
+ if (auto blitEncoder = std::dynamic_pointer_cast<PrivateBlitCommandEncoder>(encoder)) {
+ // only the writes to drawables are tracked for now, so that blitting into a drawable is synchronized with its presentation
+ auto& readWrite = blitEncoder->readWriteTextures();
+ readWriteTextures.insert(readWriteTextures.end(), readWrite.begin(), readWrite.end());
+ }
+
+ // TODO: implement this for the other texture accesses of the blit and compute encoders (we need to synchronize those as well)
}
+ // Views share synchronization with their root texture. Deduplicate that owner,
+ // including read/write overlap, before locking or reserving timeline values.
+ auto rootTexture = [](std::shared_ptr<Texture> texture) {
+ while (auto parent = texture->parentTexture()) texture = parent;
+ return texture;
+ };
+ auto canonicalize = [&](std::vector<std::shared_ptr<Texture>>& textures) {
+ std::vector<std::shared_ptr<Texture>> unique;
+ for (const auto& texture : textures) {
+ auto root = rootTexture(texture);
+ if (std::find(unique.begin(), unique.end(), root) == unique.end()) unique.push_back(root);
+ }
+ textures = std::move(unique);
+ };
+ canonicalize(readWriteTextures);
+ canonicalize(readOnlyTextures);
+ readOnlyTextures.erase(std::remove_if(readOnlyTextures.begin(), readOnlyTextures.end(),
+ [&](const auto& texture) { return std::find(readWriteTextures.begin(), readWriteTextures.end(), texture) != readWriteTextures.end(); }), readOnlyTextures.end());
+
+ // Reserve texture timeline values in submission order. Locking only the
+ // Vulkan call would let a later waiter be submitted ahead of its signal.
+ std::unique_lock<std::mutex> queueLock(_privateDevice->graphicsQueueMutex());
+
for (const auto& texture: readWriteTextures) {
auto privateTexture = std::dynamic_pointer_cast<PrivateTexture>(texture);
- auto sema = _privateDevice->getWrappedBinarySemaphore(privateTexture->needsExportablePresentationSemaphore());
+ // Ordinary offscreen textures synchronize through their timeline semaphore.
+ // Their presentation binary semaphore has no consumer. Keep it for
+ // drawable subclasses (including the native non-exportable swapchain).
+ std::shared_ptr<BinarySemaphore> sema;
+ if (!std::dynamic_pointer_cast<ConcreteTexture>(privateTexture)) {
+ sema = _privateDevice->getWrappedBinarySemaphore(privateTexture->needsExportablePresentationSemaphore());
+ privateTexture->beginUpdatingPresentationSemaphore(sema);
+ }
presentationSemaphores.push_back(sema);
- privateTexture->beginUpdatingPresentationSemaphore(sema);
}
std::vector<VkSemaphoreSubmitInfo> signalInfos;
@@ -202,11 +771,13 @@ void Indium::PrivateCommandBuffer::commit() {
auto privateTexture = std::dynamic_pointer_cast<PrivateTexture>(texture);
const auto& presentSema = presentationSemaphores[i];
- VkSemaphoreSubmitInfo signalInfo {};
- signalInfo.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO;
- signalInfo.semaphore = presentSema->semaphore;
- signalInfo.stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
- signalInfos.push_back(signalInfo);
+ if (presentSema) {
+ VkSemaphoreSubmitInfo signalInfo {};
+ signalInfo.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO;
+ signalInfo.semaphore = presentSema->semaphore;
+ signalInfo.stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
+ signalInfos.push_back(signalInfo);
+ }
handleTextureSemaphores(privateTexture);
}
@@ -215,7 +786,15 @@ void Indium::PrivateCommandBuffer::commit() {
// but it also leaves the resources for this command buffer tied up, essentially becoming a memory leak.
// i've observed this in the cube example, and it happens more than once (because the example display semaphore is exhausted and never signaled).
// UPDATE: upon further testing, it seems that this only occurs when the view is off-screen/hidden. weird.
- _privateDevice->waitForSemaphore(timelineSemaphore->semaphore, timelineSemaphore->count, [self, timelineSemaphore, extraWaitSemaphores, presentationSemaphores]() {
+ const double submitTime = indiumTimingEnabled() ? indiumNow() : 0;
+ _privateDevice->waitForSemaphore(timelineSemaphore->semaphore, timelineSemaphore->count, [self, timelineSemaphore, extraWaitSemaphores, presentationSemaphores, submitTime, frameSample]() {
+ if (frameSample) frameSample->completed.store(frameIntervalNow(), std::memory_order_release);
+ double callbackStart = 0;
+ if (submitTime != 0) {
+ callbackStart = indiumNow();
+ indiumTiming("submitToCallback", callbackStart - submitTime);
+ }
+ GpuTiming::collect(self->_privateDevice->device(), self->_commandBuffer, self->_privateDevice->properties().limits.timestampPeriod);
{
std::unique_lock lock(self->_mutex);
self->_completed = true;
@@ -234,8 +813,47 @@ void Indium::PrivateCommandBuffer::commit() {
// the handlers may have captured a reference to us (from their surrounding scope), so clear out handlers so those references go away
self->_scheduledHandlers.clear();
self->_completedHandlers.clear();
+
+ if (submitTime != 0) {
+ indiumTiming("handlersDuration", indiumNow() - callbackStart);
+ }
});
+ statCommits.fetch_add(1, std::memory_order_relaxed);
+ if (!_drawablesToPresent.empty()) submitStatsPresented();
+ if (Indium::submitBatchEnabled() && _drawablesToPresent.empty()) {
+ bool batchable = true;
+ for (const auto& ps: presentationSemaphores) if (ps) { batchable = false; break; }
+ if (batchable) {
+ auto& batch = submitBatchState();
+ batch.device = _privateDevice;
+ PendingSubmit item;
+ item.commandBuffer = _commandBuffer;
+ item.frameSample = frameSample;
+ for (const auto& w: waitInfos) {
+ // a timeline value signalled earlier in this batch is guaranteed by the order (and the barrier) inside it
+ if (w.value != 0) {
+ auto it = batch.signaled.find(w.semaphore);
+ if (it != batch.signaled.end() && it->second >= w.value) continue;
+ }
+ item.waits.push_back(w);
+ }
+ item.signals = signalInfos;
+ if (batch.items.empty()) batch.since = std::chrono::steady_clock::now();
+ for (const auto& sg: signalInfos) {
+ if (sg.value == 0) continue;
+ auto& v = batch.signaled[sg.semaphore];
+ if (v < sg.value) v = sg.value;
+ }
+ batch.items.push_back(std::move(item));
+ startSubmitBatchTimer(_privateDevice);
+ if (batch.items.size() >= (size_t) submitBatchTuning().maxItems.load(std::memory_order_relaxed)) flushSubmitBatchLocked();
+ return;
+ }
+ }
+ // not batched: everything committed earlier goes first so the submission order is preserved
+ flushSubmitBatchLocked();
+
commandBufferInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO;
commandBufferInfo.commandBuffer = _commandBuffer;
@@ -252,23 +870,41 @@ void Indium::PrivateCommandBuffer::commit() {
// the Device constructor tries to choose command queues that support as many operations as possible, but it's possible
// that a particular device only supports certain operations on certain queues (e.g. maybe it only supports transfer operations
// on an exclusive queue that doesn't support graphics or compute).
- if (DynamicVK::vkQueueSubmit2(_privateDevice->graphicsQueue(), 1, &info, VK_NULL_HANDLE) != VK_SUCCESS) {
+ const double submitCallStart = submitTime != 0 ? indiumNow() : 0;
+ if (submitTime != 0) {
+ indiumTiming("waitSemaphoresPerSubmit", waitInfos.size() / 1000.0);
+ }
+ DynamicVK::beginDiagnosticCapture(Indium::globalInstance);
+ VkResult submitResult = DynamicVK::vkQueueSubmit2(_privateDevice->graphicsQueue(), 1, &info, VK_NULL_HANDLE);
+ statSubmits.fetch_add(1, std::memory_order_relaxed);
+ if (submitResult != VK_SUCCESS) {
// TODO
abort();
}
- lock.unlock();
+ if (frameSample) frameSample->submitted.store(frameIntervalNow(), std::memory_order_release);
+ if (submitTime != 0) {
+ indiumTiming("vkQueueSubmit2", indiumNow() - submitCallStart);
+ }
+
// now that the binary semaphore signals are pending, we can allow them to be used
- for (const auto& texture: readWriteTextures) {
- auto privateTexture = std::dynamic_pointer_cast<PrivateTexture>(texture);
- privateTexture->endUpdatingPresentationSemaphore();
+ for (size_t i = 0; i < readWriteTextures.size(); ++i) {
+ if (presentationSemaphores[i]) {
+ auto privateTexture = std::dynamic_pointer_cast<PrivateTexture>(readWriteTextures[i]);
+ privateTexture->endUpdatingPresentationSemaphore();
+ }
}
+ queueLock.unlock();
// we can now queue drawables for presentation and they'll be synchronized properly
for (const auto& drawable: _drawablesToPresent) {
drawable->present();
}
+ if (frameSample) frameSample->presented.store(frameIntervalNow(), std::memory_order_release);
+ DynamicVK::endDiagnosticCapture(!_drawablesToPresent.empty());
+ };
+ SubmitThread::run(std::move(submitTask));
};
void Indium::PrivateCommandBuffer::presentDrawable(std::shared_ptr<Drawable> drawable) {
@@ -307,9 +943,12 @@ void Indium::PrivateCommandBuffer::addCompletedHandler(std::function<void(std::s
};
void Indium::PrivateCommandBuffer::waitUntilCompleted() {
+ SubmitThread::drain("drain@waitUntilCompleted"); // a commit posted to the worker has not necessarily been submitted yet
+ Indium::flushPendingSubmits();
+ // diagnostic counter (guest /tmp/metal-debug-timing flag): the calling thread blocks until the GPU has finished this buffer
+ const auto waitStart = std::chrono::steady_clock::now();
std::unique_lock lock(_mutex);
- while (!_completed) {
- _completedCondvar.wait(lock);
- }
+ Waits::blockOn("waitUntilCompleted", _completedCondvar, lock, [this] { return _completed; });
+ Indium::count("waitUntilCompletedMicroseconds", std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - waitStart).count());
};
diff --git a/src/indium/command-queue.cpp b/src/indium/command-queue.cpp
index 5704602..d2cd020 100644
--- a/src/indium/command-queue.cpp
+++ b/src/indium/command-queue.cpp
@@ -22,6 +22,7 @@ Indium::PrivateCommandQueue::PrivateCommandQueue(std::shared_ptr<PrivateDevice>
// TODO: handle this in a more C++-friendly way
abort();
}
+ _availablePools.push_back({_commandPool, VK_NULL_HANDLE});
_supportsGraphics = !!_privateDevice->graphicsQueueFamilyIndex();
_supportsCompute = _supportsGraphics ? (_privateDevice->computeQueueFamilyIndex() ? *_privateDevice->computeQueueFamilyIndex() == *_privateDevice->graphicsQueueFamilyIndex() : false) : !!_privateDevice->computeQueueFamilyIndex();
@@ -29,11 +30,36 @@ Indium::PrivateCommandQueue::PrivateCommandQueue(std::shared_ptr<PrivateDevice>
};
Indium::PrivateCommandQueue::~PrivateCommandQueue() {
- if (_commandPool) {
- DynamicVK::vkDestroyCommandPool(_privateDevice->device(), _commandPool, nullptr);
+ for (auto pool: _availablePools) {
+ DynamicVK::vkDestroyCommandPool(_privateDevice->device(), pool.pool, nullptr);
}
};
+VkCommandPool Indium::PrivateCommandQueue::acquireCommandPool(VkCommandBuffer& buffer) {
+ std::lock_guard<std::mutex> lock(_poolMutex);
+ if (!_availablePools.empty()) {
+ auto pool = _availablePools.back();
+ _availablePools.pop_back();
+ // The previous owner retains this exclusive lease through GPU completion.
+ // Reset without RELEASE_RESOURCES so RADV can reuse its recording storage.
+ if (DynamicVK::vkResetCommandPool(_privateDevice->device(), pool.pool, 0) != VK_SUCCESS) abort();
+ buffer = pool.buffer;
+ return pool.pool;
+ }
+ VkCommandPoolCreateInfo info {};
+ info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
+ info.queueFamilyIndex = _privateDevice->graphicsQueueFamilyIndex() ? *_privateDevice->graphicsQueueFamilyIndex() : *_privateDevice->computeQueueFamilyIndex();
+ buffer = VK_NULL_HANDLE;
+ VkCommandPool pool = VK_NULL_HANDLE;
+ if (DynamicVK::vkCreateCommandPool(_privateDevice->device(), &info, nullptr, &pool) != VK_SUCCESS) abort();
+ return pool;
+}
+
+void Indium::PrivateCommandQueue::releaseCommandPool(VkCommandPool pool, VkCommandBuffer buffer) {
+ std::lock_guard<std::mutex> lock(_poolMutex);
+ _availablePools.push_back({pool, buffer});
+}
+
std::shared_ptr<Indium::CommandBuffer> Indium::PrivateCommandQueue::commandBuffer() {
return std::make_shared<PrivateCommandBuffer>(shared_from_this());
};
diff --git a/src/indium/compute-command-encoder.cpp b/src/indium/compute-command-encoder.cpp
index 5f62ad0..5957bec 100644
--- a/src/indium/compute-command-encoder.cpp
+++ b/src/indium/compute-command-encoder.cpp
@@ -9,27 +9,19 @@ Indium::PrivateComputeCommandEncoder::PrivateComputeCommandEncoder(std::shared_p
_privateDevice(std::dynamic_pointer_cast<PrivateDevice>(commandBuffer->device())),
_descriptor(descriptor)
{
- VkDescriptorPoolCreateInfo poolCreateInfo {};
- poolCreateInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO;
- poolCreateInfo.poolSizeCount = poolSizes.size();
- poolCreateInfo.pPoolSizes = poolSizes.data();
- poolCreateInfo.maxSets = 64;
-
- if (DynamicVK::vkCreateDescriptorPool(_privateDevice->device(), &poolCreateInfo, nullptr, &_pool) != VK_SUCCESS) {
- // TODO
- abort();
- }
+ _pool = newDescriptorPool(_privateDevice);
};
Indium::PrivateComputeCommandEncoder::~PrivateComputeCommandEncoder() {
for (auto& pipeline: _savedPipelines) {
DynamicVK::vkDestroyPipeline(_privateDevice->device(), pipeline, nullptr);
}
- DynamicVK::vkDestroyDescriptorPool(_privateDevice->device(), _pool, 0);
+ _privateDevice->releaseDescriptorPool(_pool);
+ for (auto pool : _retiredPools) _privateDevice->releaseDescriptorPool(pool);
};
void Indium::PrivateComputeCommandEncoder::endEncoding() {
- // nothing for now
+ finishTrackedEncoder(_privateCommandBuffer.lock()->commandBuffer());
};
void Indium::PrivateComputeCommandEncoder::setComputePipelineState(std::shared_ptr<ComputePipelineState> state) {
@@ -138,7 +130,7 @@ Indium::DispatchType Indium::PrivateComputeCommandEncoder::dispatchType() {
void Indium::PrivateComputeCommandEncoder::updateBindings() {
auto buf = _privateCommandBuffer.lock();
- std::array<VkDescriptorSet, 1> descriptorSets = createDescriptorSets(_pso->descriptorSetLayouts().layouts, _pool, _privateDevice, { _functionResources }, { _pso->functionInfo() }, _keepAliveBuffers);
+ std::array<VkDescriptorSet, 1> descriptorSets = createDescriptorSets(_pso->descriptorSetLayouts().layouts, _pool, _retiredPools, _privateDevice, { _functionResources }, { _pso->functionInfo() }, _keepAliveBuffers, _addressPool);
DynamicVK::vkCmdBindDescriptorSets(buf->commandBuffer(), VK_PIPELINE_BIND_POINT_COMPUTE, _pso->layout(), 0, descriptorSets.size(), descriptorSets.data(), 0, nullptr);
};
diff --git a/src/indium/device.cpp b/src/indium/device.cpp
index 322e186..f712c7a 100644
--- a/src/indium/device.cpp
+++ b/src/indium/device.cpp
@@ -1,4 +1,9 @@
+#include <chrono>
+#include <cstdio>
+#include <indium/completion-spin.private.hpp>
#include <indium/device.private.hpp>
+#include <indium/submit-thread.private.hpp>
+#include <indium/command-encoder.private.hpp>
#include <indium/instance.private.hpp>
#include <indium/command-queue.private.hpp>
#include <indium/render-pipeline.private.hpp>
@@ -12,39 +17,207 @@
#include <iridium/iridium.hpp>
+#include <algorithm>
#include <set>
#include <stdexcept>
#include <thread>
#include <unordered_set>
+#include <utility>
+#include <cctype>
+#include <cstdio>
+#include <cstdlib>
#include <cstring>
+#include <string>
+#include <unistd.h>
#include <vector>
std::vector<std::shared_ptr<Indium::PrivateDevice>> Indium::globalDeviceList;
+namespace {
+ // One line per event, appended to the guest's /tmp/metal-device.log (host:
+ // ~/.darling/private/tmp/metal-device.log). Low volume: a few lines while the
+ // device list is built, plus one when the presenting OpenGL context first
+ // imports a Vulkan drawable. It exists so a user can tell us which GPU
+ // actually rendered a match, which the game's own logs and FPS numbers do not
+ // show.
+ void appendDeviceLog(const std::string& line) {
+ if (FILE* log = fopen("/tmp/metal-device.log", "a")) {
+ fprintf(log, "[MetalDevice] pid=%d %s\n", getpid(), line.c_str());
+ fclose(log);
+ }
+ }
+
+ std::string hexValue(unsigned long long value, int width) {
+ char buffer[32];
+ snprintf(buffer, sizeof(buffer), "%0*llx", width, value);
+ return buffer;
+ }
+
+ std::string uuidToString(const uint8_t uuid[16]) {
+ static const char* digits = "0123456789abcdef";
+ std::string out;
+ out.reserve(32);
+ for (int i = 0; i < 16; ++i) {
+ out += digits[(uuid[i] >> 4) & 0xf];
+ out += digits[uuid[i] & 0xf];
+ }
+ return out;
+ }
+
+ const char* deviceTypeName(VkPhysicalDeviceType type) {
+ switch (type) {
+ case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return "integrated";
+ case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return "discrete";
+ case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return "virtual";
+ case VK_PHYSICAL_DEVICE_TYPE_CPU: return "cpu";
+ default: return "other";
+ }
+ }
+
+ std::string memoryFlagsString(VkMemoryPropertyFlags flags) {
+ static const std::pair<VkMemoryPropertyFlags, const char*> names[] = {
+ { VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, "DEVICE_LOCAL" },
+ { VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, "HOST_VISIBLE" },
+ { VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, "HOST_COHERENT" },
+ { VK_MEMORY_PROPERTY_HOST_CACHED_BIT, "HOST_CACHED" },
+ { VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT, "LAZILY_ALLOCATED" },
+ { VK_MEMORY_PROPERTY_PROTECTED_BIT, "PROTECTED" },
+ { VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD, "DEVICE_COHERENT_AMD" },
+ { VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD, "DEVICE_UNCACHED_AMD" },
+ };
+ std::string out = "0x" + hexValue(flags, 1) + "(";
+ bool first = true;
+ VkMemoryPropertyFlags rest = flags;
+ for (const auto& name: names) {
+ if ((flags & name.first) == 0) continue;
+ out += (first ? "" : "|") + std::string(name.second);
+ first = false;
+ rest &= ~name.first;
+ }
+ if (rest) out += (first ? "0x" : "|0x") + hexValue(rest, 1);
+ else if (first) out += "none";
+ return out + ")";
+ }
+
+ // The memory types and heaps of the device Metal renders on, logged once when it
+ // is selected, so the per-allocation lines below can be read without vulkaninfo.
+ void appendMemoryTable(const VkPhysicalDeviceMemoryProperties& memory) {
+ for (uint32_t i = 0; i < memory.memoryHeapCount; ++i) {
+ const auto& heap = memory.memoryHeaps[i];
+ appendDeviceLog("memory heap=" + std::to_string(i) + " size=" + std::to_string(heap.size >> 20) + "MiB"
+ + " deviceLocal=" + ((heap.flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) ? "yes" : "no"));
+ }
+ for (uint32_t i = 0; i < memory.memoryTypeCount; ++i) {
+ const auto& type = memory.memoryTypes[i];
+ appendDeviceLog("memory type=" + std::to_string(i) + " flags=" + memoryFlagsString(type.propertyFlags)
+ + " heap=" + std::to_string(type.heapIndex));
+ }
+ }
+
+ struct GpuSelector {
+ std::string value;
+ std::string source; // empty when the loader's default order should decide
+ };
+
+ // Explicit selection, in the style of the other opt-in guest files. The guest
+ // file /tmp/metal-gpu wins over the METAL_INDIUM_GPU environment variable: the
+ // file is scoped to the guest and does not depend on the environment reaching
+ // the game through the Riot/League client. The value is either the index of a
+ // supported device or a case-insensitive substring of its Vulkan name, so
+ // "nvidia", "amd" or "llvmpipe" all work.
+ GpuSelector readGpuSelector() {
+ if (FILE* file = fopen("/tmp/metal-gpu", "r")) {
+ char buffer[256] = {0};
+ char* got = fgets(buffer, sizeof(buffer), file);
+ fclose(file);
+ if (got) {
+ std::string value(buffer);
+ while (!value.empty() && (value.back() == '\n' || value.back() == '\r' || value.back() == ' ' || value.back() == '\t')) {
+ value.pop_back();
+ }
+ if (!value.empty()) return { value, "file:/tmp/metal-gpu" };
+ }
+ }
+ if (const char* env = getenv("METAL_INDIUM_GPU")) {
+ if (env[0]) return { std::string(env), "env:METAL_INDIUM_GPU" };
+ }
+ return {};
+ }
+
+ bool isAllDigits(const std::string& value) {
+ return !value.empty() && std::all_of(value.begin(), value.end(), [](unsigned char c) { return std::isdigit(c) != 0; });
+ }
+
+ bool nameContains(const std::string& name, const std::string& needle) {
+ return std::search(name.begin(), name.end(), needle.begin(), needle.end(), [](char a, char b) {
+ return std::tolower((unsigned char) a) == std::tolower((unsigned char) b);
+ }) != name.end();
+ }
+
+ struct DeviceCandidate {
+ VkPhysicalDevice device;
+ VkPhysicalDeviceProperties properties;
+ uint8_t deviceUUID[16];
+ uint8_t driverUUID[16];
+ };
+}
+
void Indium::initGlobalDeviceList() {
std::vector<VkPhysicalDevice> physicalDevices;
uint32_t count = 0;
auto result = DynamicVK::vkEnumeratePhysicalDevices(globalInstance, &count, nullptr);
if (result != VK_SUCCESS && result != VK_INCOMPLETE) {
- // TODO: maybe warn?
+ appendDeviceLog("vkEnumeratePhysicalDevices failed when counting devices; Metal will have no device");
return;
}
physicalDevices.resize(count);
result = DynamicVK::vkEnumeratePhysicalDevices(globalInstance, &count, physicalDevices.data());
if (result != VK_SUCCESS && result != VK_INCOMPLETE) {
- // TODO: maybe warn?
+ appendDeviceLog("vkEnumeratePhysicalDevices failed when listing devices; Metal will have no device");
return;
}
+ // The loader's order used to decide the default device (createSystemDefaultDevice
+ // returns the first entry). On a hybrid laptop that is usually the integrated
+ // GPU the display is wired to, not the discrete GPU. Record every device the
+ // loader lists, then choose explicitly.
+ GpuSelector selector = readGpuSelector();
+ const bool skipCPUDevices = access("/tmp/metal-skip-cpu-devices", F_OK) == 0;
+ const bool allDevices = access("/tmp/metal-all-devices", F_OK) == 0;
+ appendDeviceLog("selector=" + (selector.source.empty() ? std::string("none") : selector.source + " value=\"" + selector.value + "\"")
+ + " skipCPUDevices=" + (skipCPUDevices ? "1" : "0") + " allDevices=" + (allDevices ? "1" : "0")
+ + " physicalDeviceCount=" + std::to_string(physicalDevices.size()));
+
+ std::vector<DeviceCandidate> candidates;
+
for (auto&& device: physicalDevices) {
- VkPhysicalDeviceProperties props;
- DynamicVK::vkGetPhysicalDeviceProperties(device, &props);
+ VkPhysicalDeviceProperties properties;
+ DynamicVK::vkGetPhysicalDeviceProperties(device, &properties);
+
+ VkPhysicalDeviceIDProperties idProperties {};
+ idProperties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES;
+ VkPhysicalDeviceProperties2 properties2 {};
+ properties2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
+ properties2.pNext = &idProperties;
+ if (DynamicVK::vkGetPhysicalDeviceProperties2) {
+ DynamicVK::vkGetPhysicalDeviceProperties2(device, &properties2);
+ }
- if (VK_API_VERSION_VARIANT(props.apiVersion) != 0 || props.apiVersion < VK_API_VERSION_1_3) {
- // unsupported device
+ std::string line = "physical name=\"" + std::string(properties.deviceName) + "\""
+ + " type=" + deviceTypeName(properties.deviceType)
+ + " vendorID=0x" + hexValue(properties.vendorID, 4)
+ + " driverVersion=0x" + hexValue(properties.driverVersion, 8)
+ + " apiVersion=" + std::to_string(VK_API_VERSION_MAJOR(properties.apiVersion)) + "."
+ + std::to_string(VK_API_VERSION_MINOR(properties.apiVersion)) + "."
+ + std::to_string(VK_API_VERSION_PATCH(properties.apiVersion))
+ + " deviceUUID=" + uuidToString(idProperties.deviceUUID)
+ + " driverUUID=" + uuidToString(idProperties.driverUUID);
+
+ if (VK_API_VERSION_VARIANT(properties.apiVersion) != 0 || properties.apiVersion < VK_API_VERSION_1_3) {
+ appendDeviceLog(line + " supported=no reason=apiVersion");
continue;
}
@@ -67,12 +240,105 @@ void Indium::initGlobalDeviceList() {
DynamicVK::vkGetPhysicalDeviceFeatures2(device, &features);
if (!features12.timelineSemaphore) {
- // unsupported device
+ appendDeviceLog(line + " supported=no reason=timelineSemaphore");
continue;
}
- globalDeviceList.push_back(std::make_shared<PrivateDevice>(std::move(device)));
+ appendDeviceLog(line + " supported=yes");
+ DeviceCandidate candidate {};
+ candidate.device = device;
+ candidate.properties = properties;
+ memcpy(candidate.deviceUUID, idProperties.deviceUUID, 16);
+ memcpy(candidate.driverUUID, idProperties.driverUUID, 16);
+ candidates.push_back(candidate);
+ }
+
+ // Resolve the explicit selection against the supported list. An explicit choice
+ // is honoured even when it names a software (CPU) device, because the user asked
+ // for it; the skip-CPU rule only affects the automatic default.
+ long selectedIndex = -1;
+ std::string selectedReason;
+
+ if (!selector.source.empty()) {
+ if (isAllDigits(selector.value)) {
+ long index = strtol(selector.value.c_str(), nullptr, 10);
+ if (index >= 0 && index < (long) candidates.size()) {
+ selectedIndex = index;
+ selectedReason = selector.source + " index " + selector.value;
+ }
+ }
+ if (selectedIndex < 0) {
+ for (size_t i = 0; i < candidates.size(); ++i) {
+ if (nameContains(candidates[i].properties.deviceName, selector.value)) {
+ selectedIndex = (long) i;
+ selectedReason = selector.source + " name~=\"" + selector.value + "\"";
+ break;
+ }
+ }
+ }
+ if (selectedIndex < 0) {
+ appendDeviceLog("selector " + selector.source + " value=\"" + selector.value + "\" matched no supported device; falling back to the loader default");
+ }
+ }
+
+ const bool haveHardwareDevice = std::any_of(candidates.begin(), candidates.end(), [](const DeviceCandidate& candidate) {
+ return candidate.properties.deviceType != VK_PHYSICAL_DEVICE_TYPE_CPU;
+ });
+
+ if (selectedIndex < 0) {
+ for (size_t i = 0; i < candidates.size(); ++i) {
+ if (skipCPUDevices && haveHardwareDevice && candidates[i].properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_CPU) {
+ continue;
+ }
+ selectedIndex = (long) i;
+ selectedReason = skipCPUDevices ? "loader-first-hardware" : "loader-first";
+ break;
+ }
+ }
+
+ if (selectedIndex < 0) {
+ appendDeviceLog("no usable supported Vulkan device found; Metal will have no device");
+ return;
+ }
+
+ DeviceCandidate& selected = candidates[selectedIndex];
+ appendDeviceLog("selected name=\"" + std::string(selected.properties.deviceName) + "\""
+ + " type=" + deviceTypeName(selected.properties.deviceType)
+ + " vendorID=0x" + hexValue(selected.properties.vendorID, 4)
+ + " deviceUUID=" + uuidToString(selected.deviceUUID)
+ + " reason=" + (selectedReason.empty() ? std::string("loader-first") : selectedReason));
+
+ VkPhysicalDeviceMemoryProperties selectedMemory {};
+ DynamicVK::vkGetPhysicalDeviceMemoryProperties(selected.device, &selectedMemory);
+ appendMemoryTable(selectedMemory);
+
+ // A VkDevice is created for the selected device only: its driver is initialized
+ // and, on a hybrid laptop, the discrete GPU is not woken for a device the
+ // rendering GPU cannot share memory with anyway. The old behaviour -- a VkDevice
+ // for every supported device, in the loader's order -- stays reachable with the
+ // opt-in guest file /tmp/metal-all-devices. MTLCopyAllDevices only ever exposes
+ // the default device, so nothing in the client depends on the full list.
+ std::vector<size_t> creationOrder;
+ creationOrder.push_back((size_t) selectedIndex);
+ if (allDevices) {
+ for (size_t i = 0; i < candidates.size(); ++i) {
+ if (i == (size_t) selectedIndex) {
+ continue;
+ }
+ if (skipCPUDevices && haveHardwareDevice && candidates[i].properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_CPU) {
+ continue;
+ }
+ creationOrder.push_back(i);
+ }
+ }
+
+ for (size_t index: creationOrder) {
+ globalDeviceList.push_back(std::make_shared<PrivateDevice>(candidates[index].device));
}
+
+ appendDeviceLog("created VkDevice for " + std::to_string(globalDeviceList.size()) + " of "
+ + std::to_string(candidates.size()) + " supported device(s)"
+ + (allDevices ? " (allDevices)" : " (selected only)"));
};
void Indium::finitGlobalDeviceList() {
@@ -83,8 +349,16 @@ Indium::Device::~Device() {};
Indium::PrivateDevice::PrivateDevice(VkPhysicalDevice physicalDevice):
_physicalDevice(physicalDevice)
-{
+{ Indium::SubmitThread::ensureStarted();
+
DynamicVK::vkGetPhysicalDeviceProperties(_physicalDevice, &_properties);
+ _completionSpinNanoseconds = completionSpinBudget(_properties.vendorID, nullptr);
+ if (FILE* file = fopen("/tmp/metal-wait-spin-us", "r")) {
+ char value[64] = {};
+ if (fscanf(file, "%63s", value) == 1)
+ _completionSpinNanoseconds = completionSpinBudget(_properties.vendorID, value);
+ fclose(file);
+ }
std::vector<VkQueueFamilyProperties> queueFamilies;
uint32_t count = 0;
@@ -233,6 +507,36 @@ Indium::PrivateDevice::PrivateDevice(VkPhysicalDevice physicalDevice):
}
}
+ // An unbound vertex-buffer slot is passed to vkCmdBindVertexBuffers as VK_NULL_HANDLE (see
+ // PrivateRenderCommandEncoder::updateBindings). Without nullDescriptor that is invalid and the GPU fetches from
+ // address 0: on RADV it faulted the gfx ring (page fault at 0x0) and reset the GPU. Enable only that feature, not the
+ // costlier robustBufferAccess2/robustImageAccess2.
+ bool robustness2Enabled = false;
+ VkPhysicalDeviceRobustness2FeaturesEXT robustness2 {};
+ robustness2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT;
+ {
+ bool hasRobustness2 = false;
+ for (const auto& prop: extProps) {
+ if (strcmp(prop.extensionName, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME) == 0) {
+ hasRobustness2 = true;
+ }
+ }
+ if (hasRobustness2) {
+ VkPhysicalDeviceFeatures2 query {};
+ query.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;
+ query.pNext = &robustness2;
+ DynamicVK::vkGetPhysicalDeviceFeatures2(_physicalDevice, &query);
+ if (robustness2.nullDescriptor) {
+ robustness2.robustBufferAccess2 = VK_FALSE;
+ robustness2.robustImageAccess2 = VK_FALSE;
+ robustness2.pNext = features13.pNext;
+ features13.pNext = &robustness2;
+ extensions.push_back(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME);
+ robustness2Enabled = true;
+ }
+ }
+ }
+
VkDeviceCreateInfo deviceCreateInfo {};
deviceCreateInfo.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO;
deviceCreateInfo.pQueueCreateInfos = queueCreateInfos.data();
@@ -246,6 +550,7 @@ Indium::PrivateDevice::PrivateDevice(VkPhysicalDevice physicalDevice):
}
_features = indiumFeatures;
+ _nullDescriptor = robustness2.nullDescriptor == VK_TRUE && robustness2Enabled;
for (const auto& index: queueFamilyIndices) {
VkQueue queue;
@@ -297,6 +602,9 @@ Indium::PrivateDevice::PrivateDevice(VkPhysicalDevice physicalDevice):
};
Indium::PrivateDevice::~PrivateDevice() {
+ for (const auto& entry : _renderPasses) DynamicVK::vkDestroyRenderPass(_device, entry.second, nullptr);
+ for (const auto& entry : _availableTimelineSemaphores) DynamicVK::vkDestroySemaphore(_device, entry.first, nullptr);
+ for (auto pool : _availableDescriptorPools) DynamicVK::vkDestroyDescriptorPool(_device, pool, nullptr);
if (_oneshotCommandPool) {
DynamicVK::vkDestroyCommandPool(_device, _oneshotCommandPool, nullptr);
}
@@ -308,6 +616,23 @@ std::string Indium::PrivateDevice::name() const {
return _properties.deviceName;
};
+std::pair<VkRenderPass, bool> Indium::PrivateDevice::acquireRenderPass(const VkRenderPassCreateInfo& info) {
+ auto key = renderPassKey(info);
+ std::unique_lock<std::mutex> lock(_renderPassMutex, std::defer_lock);
+ if (key) {
+ lock.lock();
+ auto found = _renderPasses.find(*key);
+ if (found != _renderPasses.end()) return {found->second, true};
+ // No eviction: submitted encoders and pipelines may still refer to a
+ // cached handle. Unusual descriptions beyond the bound remain private.
+ if (_renderPasses.size() >= 256) { key.reset(); lock.unlock(); }
+ }
+ VkRenderPass pass = VK_NULL_HANDLE;
+ if (DynamicVK::vkCreateRenderPass(_device, &info, nullptr, &pass) != VK_SUCCESS) abort();
+ if (key) _renderPasses.emplace(*key, pass);
+ return {pass, key.has_value()};
+}
+
std::shared_ptr<Indium::CommandQueue> Indium::PrivateDevice::newCommandQueue() {
return std::make_shared<PrivateCommandQueue>(shared_from_this());
};
@@ -317,9 +642,9 @@ std::shared_ptr<Indium::RenderPipelineState> Indium::PrivateDevice::newRenderPip
};
std::shared_ptr<Indium::ComputePipelineState> Indium::PrivateDevice::newComputePipelineState(const ComputePipelineDescriptor& descriptor, PipelineOption options, std::shared_ptr<ComputePipelineReflection> reflection) {
- if (options != PipelineOption::None) {
- throw std::runtime_error("TODO: support compute pipeline options");
- }
+ // Reflection is retained on the function; these flags do not change GPU code.
+ if ((static_cast<size_t>(options) & ~size_t(3)) != 0)
+ throw std::invalid_argument("Unsupported compute pipeline option");
return std::make_shared<PrivateComputePipelineState>(shared_from_this(), descriptor);
};
@@ -357,6 +682,7 @@ std::shared_ptr<Indium::Library> Indium::PrivateDevice::newLibrary(const void* d
break;
}
+ funcInfo.arguments = info.arguments;
funcInfo.bindings.insert(funcInfo.bindings.end(), info.bindings.begin(), info.bindings.end());
funcInfo.embeddedSamplers.insert(funcInfo.embeddedSamplers.end(), info.embeddedSamplers.begin(), info.embeddedSamplers.end());
}
@@ -382,6 +708,19 @@ std::shared_ptr<Indium::Device> Indium::createSystemDefaultDevice() {
return globalDeviceList.empty() ? nullptr : globalDeviceList.front();
};
+namespace {
+bool lazyWakeupEnabled() {
+ static std::atomic<bool> enabled {true};
+ static std::atomic<unsigned> calls {0};
+ if ((calls.fetch_add(1, std::memory_order_relaxed) & 255) == 0)
+ enabled.store(access("/tmp/metal-no-lazy-wakeup", F_OK) != 0, std::memory_order_relaxed);
+ return enabled.load(std::memory_order_relaxed);
+}
+void noteWakeup(const char* site) {
+ if (Indium::Waits::enabled()) Indium::Waits::record(site, 0);
+}
+}
+
void Indium::PrivateDevice::pollEvents(uint64_t timeoutNanoseconds) {
// this is held for the entire duration of the poll so we can
// ensure we're the only one polling, which allows us to avoid
@@ -393,9 +732,35 @@ void Indium::PrivateDevice::pollEvents(uint64_t timeoutNanoseconds) {
const std::vector<VkSemaphore> semaphores = _eventLoopSemaphores;
const std::vector<uint64_t> values = _eventLoopWaitValues;
const std::vector<std::function<void()>> callbacks = _eventLoopCallbacks;
+ // All registrations covered by this snapshot, under the same mutex.
+ const uint64_t snapshotRegistrations = _eventLoopRegistrations.load(std::memory_order_seq_cst);
lock.unlock();
+ // NVIDIA's blocking timeline wait can wake 1-2 ms after completion. Poll
+ // briefly while work is pending; an idle device should block immediately.
+ if (_completionSpinNanoseconds && timeoutNanoseconds && semaphores.size() > 1) {
+ const auto started = std::chrono::steady_clock::now();
+ const auto deadline = started + std::chrono::nanoseconds(
+ std::min(_completionSpinNanoseconds, timeoutNanoseconds));
+ bool anyReady = false;
+ do {
+ for (size_t i = 0; i < semaphores.size() && !anyReady; ++i) {
+ uint64_t count = 0;
+ anyReady = DynamicVK::vkGetSemaphoreCounterValue(_device, semaphores[i], &count) == VK_SUCCESS && count >= values[i];
+ }
+ if (anyReady) break;
+ if (_eventLoopRegistrations.load(std::memory_order_relaxed) != snapshotRegistrations) {
+ anyReady = true; // Re-snapshot registrations instead of waiting on stale handles.
+ break;
+ }
+ __builtin_ia32_pause();
+ } while (std::chrono::steady_clock::now() < deadline);
+ const uint64_t elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>(
+ std::chrono::steady_clock::now() - started).count();
+ timeoutNanoseconds = anyReady ? 0 : timeoutNanoseconds - std::min(timeoutNanoseconds, elapsed);
+ }
+
VkSemaphoreWaitInfo info {};
info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO;
info.semaphoreCount = semaphores.size();
@@ -403,7 +768,19 @@ void Indium::PrivateDevice::pollEvents(uint64_t timeoutNanoseconds) {
info.pValues = values.data();
info.flags = VK_SEMAPHORE_WAIT_ANY_BIT;
- auto result = DynamicVK::vkWaitSemaphores(_device, &info, timeoutNanoseconds);
+ // A registrant increments its counter before inspecting blocked pollers.
+ // The poller announces its wait before rechecking registrations. With both
+ // operations seq_cst, either this wait becomes nonblocking or the registrant
+ // signals the wake-up semaphore. The counters belong to this device only.
+ const bool announcedBlock = timeoutNanoseconds > 0;
+ if (announcedBlock) {
+ _eventLoopBlockedPollers.fetch_add(1, std::memory_order_seq_cst);
+ if (_eventLoopRegistrations.load(std::memory_order_seq_cst) != snapshotRegistrations)
+ timeoutNanoseconds = 0;
+ }
+ auto result = DynamicVK::vkWaitSemaphores(_device, &info, timeoutNanoseconds);
+ if (announcedBlock)
+ _eventLoopBlockedPollers.fetch_sub(1, std::memory_order_seq_cst);
if (result == VK_TIMEOUT) {
// timed out with no semaphores ready
@@ -474,15 +851,30 @@ void Indium::PrivateDevice::waitForSemaphore(VkSemaphore semaphore, uint64_t tar
_eventLoopSemaphores.push_back(semaphore);
_eventLoopWaitValues.push_back(targetValue);
_eventLoopCallbacks.push_back(callback);
+ _eventLoopRegistrations.fetch_add(1, std::memory_order_seq_cst);
}
- // now wakeup the event loop so it can start waiting on this new semaphore
- wakeupEventLoop();
+ // A spinning poller re-snapshots registrations without a driver signal.
+ if (!lazyWakeupEnabled() || _eventLoopBlockedPollers.load(std::memory_order_seq_cst) > 0) {
+ noteWakeup("wakeupSignalled");
+ wakeupEventLoop();
+ } else {
+ noteWakeup("wakeupSkipped");
+ }
};
-// TODO: create a semaphore pool to avoid constantly creating and destroying semaphores
+// Non-exported timeline handles are exclusively leased until the last owner
+// (including GPU-completion callbacks) releases them. Preserve monotonic values.
Indium::TimelineSemaphore Indium::PrivateDevice::getTimelineSemaphore() {
+ {
+ std::lock_guard<std::mutex> lock(_timelineSemaphorePoolMutex);
+ if (!_availableTimelineSemaphores.empty()) {
+ const auto entry = _availableTimelineSemaphores.back();
+ _availableTimelineSemaphores.pop_back();
+ return TimelineSemaphore { shared_from_this(), entry.first, entry.second };
+ }
+ }
VkSemaphoreTypeCreateInfo typeInfo {};
typeInfo.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO;
typeInfo.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE;
@@ -505,7 +897,14 @@ Indium::TimelineSemaphore Indium::PrivateDevice::getTimelineSemaphore() {
};
void Indium::PrivateDevice::putTimelineSemaphore(const TimelineSemaphore& semaphore) {
- DynamicVK::vkDestroySemaphore(_device, semaphore.semaphore, nullptr);
+ {
+ std::lock_guard<std::mutex> lock(_timelineSemaphorePoolMutex);
+ if (_availableTimelineSemaphores.size() < 256 && semaphore.count != UINT64_MAX) {
+ _availableTimelineSemaphores.emplace_back(semaphore.semaphore, semaphore.count);
+ return;
+ }
+ }
+ DynamicVK::vkDestroySemaphore(_device, semaphore.semaphore, nullptr);
};
std::shared_ptr<Indium::TimelineSemaphore> Indium::PrivateDevice::getWrappedTimelineSemaphore() {
@@ -551,3 +950,160 @@ std::shared_ptr<Indium::BinarySemaphore> Indium::PrivateDevice::getWrappedBinary
delete ptr;
});
};
+
+bool Indium::PrivateDevice::supportsDepth24Stencil8PixelFormat() {
+ VkFormatProperties properties {};
+ DynamicVK::vkGetPhysicalDeviceFormatProperties(_physicalDevice, VK_FORMAT_D24_UNORM_S8_UINT, &properties);
+ return (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT) != 0;
+}
+
+uint64_t Indium::PrivateDevice::fallbackBufferAddress() {
+ std::scoped_lock lock(_fallbackMutex);
+ if (!_fallbackBuffer) {
+ std::vector<char> zeros(64 * 1024, 0);
+ _fallbackBuffer = newBuffer(zeros.data(), zeros.size(), ResourceOptions::StorageModeShared);
+ }
+ return _fallbackBuffer->gpuAddress();
+};
+
+std::shared_ptr<Indium::SamplerState> Indium::PrivateDevice::fallbackSampler() {
+ {
+ std::scoped_lock lock(_fallbackMutex);
+ if (_fallbackSampler) {
+ return _fallbackSampler;
+ }
+ }
+ auto sampler = newSamplerState(SamplerDescriptor {});
+ std::scoped_lock lock(_fallbackMutex);
+ if (!_fallbackSampler) {
+ _fallbackSampler = sampler;
+ }
+ return _fallbackSampler;
+};
+
+std::shared_ptr<Indium::Buffer> Indium::PrivateDevice::acquireScratchChunk() {
+ std::shared_ptr<Buffer> base;
+
+ {
+ std::scoped_lock lock(_scratchMutex);
+ if (!_scratchFree.empty()) {
+ base = std::move(_scratchFree.back());
+ _scratchFree.pop_back();
+ }
+ }
+
+ if (!base) {
+ base = newBuffer(scratchChunkSize, ResourceOptions::StorageModeShared);
+ }
+
+ // the returned handle gives the chunk back instead of destroying it
+ auto self = shared_from_this();
+ return std::shared_ptr<Buffer>(base.get(), [self, base](Buffer*) {
+ std::scoped_lock lock(self->_scratchMutex);
+ self->_scratchFree.push_back(base);
+ });
+};
+
+VkDescriptorPool Indium::PrivateDevice::acquireDescriptorPool() {
+ VkDescriptorPool pool = VK_NULL_HANDLE;
+ {
+ std::lock_guard<std::mutex> lock(_descriptorPoolMutex);
+ if (!_availableDescriptorPools.empty()) {
+ pool = _availableDescriptorPools.back();
+ _availableDescriptorPools.pop_back();
+ }
+ }
+ // Returned only after its encoder and GPU completion owner are released.
+ // Each acquisition has exclusive ownership; reset retains driver storage.
+ if (pool) {
+ if (DynamicVK::vkResetDescriptorPool(_device, pool, 0) != VK_SUCCESS) abort();
+ return pool;
+ }
+ VkDescriptorPoolCreateInfo info {};
+ info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO;
+ info.poolSizeCount = poolSizes.size();
+ info.pPoolSizes = poolSizes.data();
+ info.maxSets = 64;
+ if (DynamicVK::vkCreateDescriptorPool(_device, &info, nullptr, &pool) != VK_SUCCESS) abort();
+ return pool;
+}
+void Indium::PrivateDevice::releaseDescriptorPool(VkDescriptorPool pool) {
+ if (!pool) return;
+ {
+ std::lock_guard<std::mutex> lock(_descriptorPoolMutex);
+ if (_availableDescriptorPools.size() < 128) {
+ _availableDescriptorPools.push_back(pool);
+ return;
+ }
+ }
+ DynamicVK::vkDestroyDescriptorPool(_device, pool, nullptr);
+}
+
+// The Vulkan spec orders memory types so that a type whose flags are a subset of
+// another's comes first. A flag-less type (system RAM the GPU reads over PCIe,
+// which NVIDIA exposes) therefore precedes every DEVICE_LOCAL type, and the first
+// type allowed by memoryTypeBits put resources only the GPU touches in system RAM
+// there. Private and Memoryless resources now take the first allowed DEVICE_LOCAL
+// type and fall back to that first fit. Shared and Managed keep the first fit
+// that is host visible (and coherent for Shared): moving them to DEVICE_LOCAL and
+// HOST_VISIBLE BAR memory would make their CPU reads uncached. On RADV type 0 is
+// DEVICE_LOCAL, so nothing changes there.
+size_t Indium::PrivateDevice::findMemoryType(uint32_t memoryTypeBits, StorageMode storageMode, bool image) {
+ const auto& memory = _memoryProperties;
+ const bool gpuOnly = storageMode == StorageMode::Private || storageMode == StorageMode::Memoryless;
+ size_t firstFit = SIZE_MAX;
+ size_t deviceLocal = SIZE_MAX;
+
+ for (size_t i = 0; i < memory.memoryTypeCount; ++i) {
+ const auto& type = memory.memoryTypes[i];
+
+ if ((memoryTypeBits & (1u << i)) == 0) {
+ continue;
+ }
+
+ if ((storageMode == StorageMode::Managed || storageMode == StorageMode::Shared) && (type.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0) {
+ continue;
+ }
+
+ if (storageMode == StorageMode::Shared && (type.propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) == 0) {
+ continue;
+ }
+
+ if (firstFit == SIZE_MAX) {
+ firstFit = i;
+ }
+
+ if (!gpuOnly || (type.propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0) {
+ deviceLocal = i;
+ break;
+ }
+ }
+
+ const size_t chosen = deviceLocal != SIZE_MAX ? deviceLocal : firstFit;
+ if (chosen == SIZE_MAX) {
+ return chosen;
+ }
+
+ // Logged the first time each (kind, storage mode, type) occurs in the process;
+ // afterwards this is one relaxed load. 2 kinds x 4 modes x 32 types = 256 bits.
+ static std::atomic<uint64_t> logged[4] {};
+ const size_t bit = ((image ? 4u : 0u) + (static_cast<size_t>(storageMode) & 3)) * 32 + chosen;
+ const uint64_t mask = uint64_t(1) << (bit & 63);
+ if ((logged[bit >> 6].load(std::memory_order_relaxed) & mask) == 0 && (logged[bit >> 6].fetch_or(mask) & mask) == 0) {
+ static const char* modeNames[] = { "shared", "managed", "private", "memoryless" };
+ auto describe = [&](size_t index) {
+ const auto& type = memory.memoryTypes[index];
+ return std::to_string(index) + " flags=" + memoryFlagsString(type.propertyFlags)
+ + " heap=" + std::to_string(type.heapIndex)
+ + " heapDeviceLocal=" + ((memory.memoryHeaps[type.heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) ? "yes" : "no");
+ };
+ appendDeviceLog(std::string("memory alloc kind=") + (image ? "texture" : "buffer")
+ + " storage=" + modeNames[static_cast<size_t>(storageMode) & 3]
+ + " typeBits=0x" + hexValue(memoryTypeBits, 8)
+ + " type=" + describe(chosen)
+ + " firstFit=" + describe(firstFit)
+ + (chosen != firstFit ? " changed=yes" : " changed=no"));
+ }
+
+ return chosen;
+};
diff --git a/src/indium/dynamic-vk.cpp b/src/indium/dynamic-vk.cpp
index a4677c3..69ef23f 100644
--- a/src/indium/dynamic-vk.cpp
+++ b/src/indium/dynamic-vk.cpp
@@ -1,4 +1,5 @@
#include <indium/dynamic-vk.hpp>
+#include <cstdlib>
#ifdef DARLING
#include <elfcalls.h>
@@ -6,6 +7,10 @@
#endif
static void* libraryHandle = NULL;
+static void (*captureBegin)(void*) = nullptr;
+static void (*captureEnd)(int) = nullptr;
+void Indium::DynamicVK::beginDiagnosticCapture(VkInstance instance) { if (captureBegin) captureBegin(instance); }
+void Indium::DynamicVK::endDiagnosticCapture(bool presented) { if (captureEnd) captureEnd(presented); }
PFN_vkGetInstanceProcAddr Indium::DynamicVK::vkGetInstanceProcAddr = NULL;
PFN_vkCreateInstance Indium::DynamicVK::vkCreateInstance = NULL;
PFN_vkEnumerateInstanceLayerProperties Indium::DynamicVK::vkEnumerateInstanceLayerProperties = NULL;
@@ -32,6 +37,16 @@ static Indium::DynamicVK::DynamicFunctionBase* const eagerlyResolvedFunctions[]
#endif
bool Indium::DynamicVK::init() {
+#ifdef DARLING
+ const char* capture = getenv("RCLOL_RENDERDOC_CAPTURE");
+ if (capture && capture[0] == '1') {
+ void* bridge = _elfcalls->dlopen("@PROJECT_ROOT@/build/renderdoc/capture-bridge.so");
+ if (bridge) {
+ captureBegin = reinterpret_cast<void(*)(void*)>(DLSYM(bridge, "riot_capture_begin"));
+ captureEnd = reinterpret_cast<void(*)(int)>(DLSYM(bridge, "riot_capture_end"));
+ }
+ }
+#endif
#ifdef DARLING
libraryHandle = _elfcalls->dlopen("libvulkan.so.1");
#else
diff --git a/src/indium/indium.cpp b/src/indium/indium.cpp
index e7c06fe..6e1f4b1 100644
--- a/src/indium/indium.cpp
+++ b/src/indium/indium.cpp
@@ -10,15 +10,26 @@
#include <string_view>
#include <iostream>
+#include <cstdio>
+#include <mutex>
+#include <unistd.h>
VkInstance Indium::globalInstance = VK_NULL_HANDLE;
static VkDebugUtilsMessengerEXT debugMessenger = VK_NULL_HANDLE;
static volatile int noOptOut;
+static FILE* validationLog = nullptr;
+static std::mutex validationLogMutex;
static VKAPI_ATTR VkBool32 VKAPI_CALL debugCallback(VkDebugUtilsMessageSeverityFlagBitsEXT severity, VkDebugUtilsMessageTypeFlagsEXT type, const VkDebugUtilsMessengerCallbackDataEXT* callbackData, void* userData) {
- std::cerr << "Validation message: " << callbackData->pMessage << std::endl;
+ if (validationLog) {
+ std::lock_guard<std::mutex> lock(validationLogMutex);
+ fprintf(validationLog, "pid=%d severity=%u id=%s: %s\n", getpid(), unsigned(severity), callbackData->pMessageIdName ? callbackData->pMessageIdName : "", callbackData->pMessage);
+ fflush(validationLog);
+ } else {
+ std::cerr << "Validation message: " << callbackData->pMessage << std::endl;
+ }
if ((severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) != 0) {
// convenient breakpoint
@@ -43,6 +54,11 @@ void Indium::init(const char** additionalExtensions, size_t additionalExtensionC
VK_KHR_SURFACE_EXTENSION_NAME,
};
+ if (access("/tmp/metal-validation", F_OK) == 0) {
+ enableValidation = true;
+ validationLog = fopen("/tmp/metal-validation.log", "a");
+ }
+
uint32_t count;
std::vector<VkLayerProperties> layerProps;
@@ -97,13 +113,13 @@ void Indium::init(const char** additionalExtensions, size_t additionalExtensionC
VkValidationFeaturesEXT valFeat {};
std::vector<VkValidationFeatureEnableEXT> valFeatures {
- VK_VALIDATION_FEATURE_ENABLE_DEBUG_PRINTF_EXT,
+ VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
};
valFeat.sType = VK_STRUCTURE_TYPE_VALIDATION_FEATURES_EXT;
valFeat.enabledValidationFeatureCount = valFeatures.size();
valFeat.pEnabledValidationFeatures = valFeatures.data();
- if (enableValidation) {
+ if (enableValidation && access("/tmp/metal-validation-sync", F_OK) == 0) {
createInfo.pNext = &valFeat;
}
@@ -133,7 +149,10 @@ void Indium::init(const char** additionalExtensions, size_t additionalExtensionC
initGlobalDeviceList();
};
+namespace Indium { void finishCommandRetirement(); }
+
void Indium::finit() {
+ finishCommandRetirement();
finitGlobalDeviceList();
if (debugMessenger && DynamicVK::vkDestroyDebugUtilsMessengerEXT) {
@@ -146,4 +165,5 @@ void Indium::finit() {
DynamicVK::finit();
Iridium::finit();
+ if (validationLog) { fclose(validationLog); validationLog = nullptr; }
};
diff --git a/src/indium/library.cpp b/src/indium/library.cpp
index d56a213..66a6019 100644
--- a/src/indium/library.cpp
+++ b/src/indium/library.cpp
@@ -1,7 +1,13 @@
+#include <indium/gpu-timing.private.hpp>
+#include <indium/outline-chain.private.hpp>
#include <indium/library.private.hpp>
#include <indium/device.private.hpp>
#include <indium/sampler.hpp>
#include <indium/dynamic-vk.hpp>
+#include <indium/counters.private.hpp>
+#include <cstdio>
+#include <sys/stat.h>
+#include <unistd.h>
Indium::Function::~Function() {};
Indium::Library::~Library() {};
@@ -26,6 +32,8 @@ Indium::PrivateLibrary::PrivateLibrary(std::shared_ptr<PrivateDevice> device, co
_privateDevice(device),
_functionInfos(functionInfos)
{
+ GpuTiming::noteLibrary(this, data, dataLength);
+ OutlineChain::noteLibrary(this, data, dataLength);
VkShaderModuleCreateInfo createInfo {};
createInfo.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO;
createInfo.codeSize = dataLength;
@@ -36,6 +44,20 @@ Indium::PrivateLibrary::PrivateLibrary(std::shared_ptr<PrivateDevice> device, co
abort();
}
+ // Diagnostic: with /tmp/metal-debug-spirv present, keep the first translated modules for offline inspection
+ if (Indium::spirvDebugEnabled()) {
+ static unsigned dumped = 0;
+ if (dumped++ < 8192) {
+ mkdir("/tmp/spv", 0777);
+ char path[96];
+ snprintf(path, sizeof(path), "/tmp/spv/lib-%p.spv", (void*) this);
+ if (FILE* file = fopen(path, "wb")) {
+ fwrite(data, 1, dataLength, file);
+ fclose(file);
+ }
+ }
+ }
+
// create sampler states for embedded samplers
for (auto& [name, funcInfo]: _functionInfos) {
for (const auto& embeddedSampler: funcInfo.embeddedSamplers) {
@@ -146,3 +168,5 @@ Indium::PrivateLibrary::~PrivateLibrary() {
std::shared_ptr<Indium::Function> Indium::PrivateLibrary::newFunction(const std::string& name) {
return std::make_shared<PrivateFunction>(shared_from_this(), name, _functionInfos[name]);
};
+
+const std::vector<Iridium::ArgumentInfo>& Indium::PrivateFunction::argumentInfo() const { return _functionInfo.arguments; }
diff --git a/src/indium/render-command-encoder.cpp b/src/indium/render-command-encoder.cpp
index b27ffe5..dd7434d 100644
--- a/src/indium/render-command-encoder.cpp
+++ b/src/indium/render-command-encoder.cpp
@@ -1,3 +1,8 @@
+#include <atomic>
+#include <indium/outline-chain.private.hpp>
+#include <cstdarg>
+#include <unistd.h>
+#include <cstdio>
#include <indium/render-command-encoder.private.hpp>
#include <indium/command-buffer.private.hpp>
#include <indium/render-pass.hpp>
@@ -12,35 +17,120 @@
#include <indium/depth-stencil.private.hpp>
#include <indium/command-encoder.private.hpp>
#include <indium/dynamic-vk.hpp>
+#include <indium/counters.private.hpp>
+#include <indium/gpu-timing.private.hpp>
#include <vulkan/vulkan_core.h>
+#include <time.h>
+namespace {
+// diagnostic (guest flags /tmp/metal-gpu-timing and, to start logging, /tmp/metal-seqlog): viewport, scissor, draw sizes and
+// small constant uploads, interleaved with the pass log in /tmp/indium-gpu-seq.log
+static bool seqLogActive() {
+ static unsigned calls = 0;
+ static bool active = false;
+ if ((calls++ & 255) == 0) active = access("/tmp/metal-seqlog", F_OK) == 0;
+ return active;
+}
+static void seqLog(const char* fmt, ...) __attribute__((format(printf, 1, 2)));
+static void seqLog(const char* fmt, ...) {
+ static FILE* file = fopen("/tmp/indium-gpu-seq.log", "a");
+ if (!file || !seqLogActive()) return;
+ va_list args;
+ va_start(args, fmt);
+ vfprintf(file, fmt, args);
+ va_end(args);
+ fflush(file);
+}
+static void seqLogBytes(const char* stage, const void* bytes, size_t length, size_t index) {
+ if (length > 96 || !bytes) return;
+ char buf[512];
+ int n = snprintf(buf, sizeof(buf), " %s%zu len=%zu f=", stage, index, length);
+ const float* f = static_cast<const float*>(bytes);
+ for (size_t i = 0; i + 4 <= length && n < 480; i += 4) n += snprintf(buf + n, sizeof(buf) - n, "%.3g ", f[i / 4]);
+ seqLog("%s\n", buf);
+}
+}
+
+
+
+namespace {
+ // Diagnostic (guest /tmp flag files, re-read twice a second): skip every draw of render passes by attachment size,
+ // to measure what each pass costs on the GPU. Not for play.
+ struct DrawSkip { const void* encoder = nullptr; bool skip = false; };
+ thread_local DrawSkip drawSkip;
+
+ // Diagnostic flag files, refreshed every 2048 passes (a clock_gettime per pass was a syscall in the render hot path).
+ bool shouldSkipPass(size_t width, size_t height, bool drawableFormat) {
+ static unsigned counter = 0;
+ static bool f1180, f512, f1024, f1080, f242, fui, fscene;
+ if ((counter++ & 2047) == 0) {
+ f1180 = access("/tmp/metal-skip-1180", F_OK) == 0;
+ f512 = access("/tmp/metal-skip-512", F_OK) == 0;
+ f1024 = access("/tmp/metal-skip-1024", F_OK) == 0;
+ f1080 = access("/tmp/metal-skip-1080", F_OK) == 0;
+ f242 = access("/tmp/metal-skip-242", F_OK) == 0;
+ fui = access("/tmp/metal-skip-ui", F_OK) == 0;
+ fscene = access("/tmp/metal-skip-scene", F_OK) == 0;
+ }
+ if (width == 1180 && height == 1184) return f1180;
+ if (width == 512 && height == 512) return f512;
+ if (width == 1024 && height == 1024) return f1024;
+ if (width == 1920 && height == 1080) return f1080 || (drawableFormat ? fui : fscene);
+ if (width == 242 && height == 242) return f242;
+ return false;
+ }
+}
Indium::RenderCommandEncoder::~RenderCommandEncoder() {};
-Indium::PrivateRenderCommandEncoder::PrivateRenderCommandEncoder(std::shared_ptr<PrivateCommandBuffer> commandBuffer, const RenderPassDescriptor& descriptor):
+Indium::PrivateRenderCommandEncoder::PrivateRenderCommandEncoder(std::shared_ptr<PrivateCommandBuffer> commandBuffer, const RenderPassDescriptor& inputDescriptor):
_privateCommandBuffer(commandBuffer),
- _descriptor(descriptor),
+ _descriptor(inputDescriptor),
_privateDevice(commandBuffer->privateDevice())
{
+ auto selectSubresource = [](RenderPassAttachmentDescriptor& attachment) {
+ if (!attachment.texture) return;
+ auto texture = std::dynamic_pointer_cast<PrivateTexture>(attachment.texture);
+ if (texture->textureType() != TextureType::e2D && texture->textureType() != TextureType::e2DArray) return;
+ if (!attachment.level && !attachment.slice && texture->mipmapLevelCount() == 1 && texture->arrayLength() == 1) return;
+ if (attachment.level >= texture->mipmapLevelCount() || attachment.slice >= texture->arrayLength())
+ throw std::invalid_argument("Render attachment subresource out of range");
+ attachment.texture = texture->newTextureView(texture->pixelFormat(), TextureType::e2D,
+ Range<size_t>{attachment.level, 1}, Range<size_t>{attachment.slice, 1});
+ attachment.level = 0;
+ attachment.slice = 0;
+ };
+ for (auto& color : _descriptor.colorAttachments) selectSubresource(color);
+ if (_descriptor.depthAttachment) selectSubresource(*_descriptor.depthAttachment);
+ if (_descriptor.stencilAttachment) selectSubresource(*_descriptor.stencilAttachment);
+ const auto& descriptor = _descriptor;
auto buf = _privateCommandBuffer.lock();
auto vkDevice = _privateDevice->device();
auto vkCmdBuf = buf->commandBuffer();
- VkDescriptorPoolCreateInfo poolCreateInfo {};
- poolCreateInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO;
- poolCreateInfo.poolSizeCount = poolSizes.size();
- poolCreateInfo.pPoolSizes = poolSizes.data();
- poolCreateInfo.maxSets = 64; // i guess
-
- if (DynamicVK::vkCreateDescriptorPool(vkDevice, &poolCreateInfo, nullptr, &_pool) != VK_SUCCESS) {
- // TODO
- abort();
+ std::shared_ptr<Texture> firstTexture;
+ for (const auto& color: descriptor.colorAttachments) {
+ if (color.texture) { firstTexture = color.texture; break; }
}
-
- auto firstTexture = descriptor.colorAttachments.front().texture;
+ if (!firstTexture) firstTexture = descriptor.depthAttachment ? descriptor.depthAttachment->texture :
+ (descriptor.stencilAttachment ? descriptor.stencilAttachment->texture : nullptr);
+ if (!firstTexture) throw std::invalid_argument("Render pass requires an attachment");
+ drawSkip.encoder = this;
+ drawSkip.skip = shouldSkipPass(firstTexture->width(), firstTexture->height(), firstTexture->pixelFormat() == PixelFormat::RGBA8Unorm);
+ // Depth and stencil may be given as different texture objects (e.g. a view of a combined format) that share one image
+ if (descriptor.depthAttachment && descriptor.stencilAttachment &&
+ descriptor.depthAttachment->texture != descriptor.stencilAttachment->texture) {
+ auto depthImage = std::dynamic_pointer_cast<PrivateTexture>(descriptor.depthAttachment->texture);
+ auto stencilImage = std::dynamic_pointer_cast<PrivateTexture>(descriptor.stencilAttachment->texture);
+ if (!depthImage || !stencilImage || depthImage->image() != stencilImage->image())
+ throw std::invalid_argument("Separate depth and stencil textures are not supported");
+ }
+ auto depthStencilTexture = descriptor.depthAttachment ? descriptor.depthAttachment->texture :
+ (descriptor.stencilAttachment ? descriptor.stencilAttachment->texture : nullptr);
std::vector<VkClearValue> clearValues;
for (const auto& color: descriptor.colorAttachments) {
+ if (!color.texture) continue;
auto clearColor = color.clearColor;
VkClearValue clearValue {};
clearValue.color.float32[0] = clearColor.red;
@@ -65,36 +155,59 @@ Indium::PrivateRenderCommandEncoder::PrivateRenderCommandEncoder(std::shared_ptr
std::vector<VkSubpassDependency> dependencies;
for (const auto& color: descriptor.colorAttachments) {
+ if (!color.texture) continue;
auto privateTexture = std::dynamic_pointer_cast<PrivateTexture>(color.texture);
VkAttachmentDescription desc {};
desc.format = pixelFormatToVkFormat(color.texture->pixelFormat());
desc.samples = VK_SAMPLE_COUNT_1_BIT; // TODO: support multisampling
- desc.loadOp = loadActionToVkAttachmentLoadOp(color.loadAction, true);
- desc.storeOp = storeActionToVkAttachmentStoreOp(color.storeAction, true);
+ // Desktop Metal leaves an attachment's memory alone for DontCare, and the game depends on it. In Vulkan,
+ // DONT_CARE plus an UNDEFINED initial layout lets the driver throw the contents away (RADV re-initialises the
+ // image's compression metadata), so only an explicit Clear may discard; everything else is preserved.
+ const bool colorClear = color.loadAction == LoadAction::Clear;
+ desc.loadOp = colorClear ? VK_ATTACHMENT_LOAD_OP_CLEAR : VK_ATTACHMENT_LOAD_OP_LOAD;
+ // always STORE: free on this kind of GPU, and DontCare/Unknown/resolve actions must not discard either
+ desc.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
desc.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
desc.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
- desc.initialLayout = (color.loadAction == LoadAction::Load) ? privateTexture->imageLayout() : VK_IMAGE_LAYOUT_UNDEFINED;
+ desc.initialLayout = colorClear ? VK_IMAGE_LAYOUT_UNDEFINED : privateTexture->imageLayout();
desc.finalLayout = privateTexture->imageLayout();
renderPassAttachments.push_back(desc);
}
- if (descriptor.depthAttachment) {
- auto privateTexture = std::dynamic_pointer_cast<PrivateTexture>(descriptor.depthAttachment->texture);
- VkAttachmentDescription desc {};
- desc.format = pixelFormatToVkFormat(privateTexture->pixelFormat());
- desc.samples = VK_SAMPLE_COUNT_1_BIT;
- desc.loadOp = loadActionToVkAttachmentLoadOp(descriptor.depthAttachment->loadAction, false);
- desc.storeOp = storeActionToVkAttachmentStoreOp(descriptor.depthAttachment->storeAction, false);
- desc.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
- desc.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
- desc.initialLayout = (descriptor.depthAttachment->loadAction == LoadAction::Load) ? privateTexture->imageLayout() : VK_IMAGE_LAYOUT_UNDEFINED;
- desc.finalLayout = privateTexture->imageLayout();
- renderPassAttachments.push_back(desc);
- }
- if (descriptor.stencilAttachment) {
- throw std::runtime_error("TODO: support stencil attachments");
- }
+ if (depthStencilTexture) {
+ auto texture = std::dynamic_pointer_cast<PrivateTexture>(depthStencilTexture);
+ VkAttachmentDescription desc {};
+ desc.format = pixelFormatToVkFormat(texture->pixelFormat());
+ desc.samples = VK_SAMPLE_COUNT_1_BIT;
+ // Same rule as for colour (see above): the game binds the scene depth buffer to a run of effect passes with
+ // DontCare/DontCare and then resumes the scene with Load. With UNDEFINED/DONT_CARE the depth buffer came back
+ // as garbage in tiles, which is what punched black blocks into everything drawn afterwards. An aspect the
+ // pass does not attach is preserved too. Default for depth and stencil is Clear.
+ auto isClear = [](const auto& attachment) {
+ return attachment && (attachment->loadAction == LoadAction::Clear || attachment->loadAction == LoadAction::Default);
+ };
+ const bool depthClear = isClear(descriptor.depthAttachment);
+ const bool stencilClear = isClear(descriptor.stencilAttachment);
+ desc.loadOp = depthClear ? VK_ATTACHMENT_LOAD_OP_CLEAR : VK_ATTACHMENT_LOAD_OP_LOAD;
+ desc.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
+ desc.stencilLoadOp = stencilClear ? VK_ATTACHMENT_LOAD_OP_CLEAR : VK_ATTACHMENT_LOAD_OP_LOAD;
+ desc.stencilStoreOp = VK_ATTACHMENT_STORE_OP_STORE;
+ desc.initialLayout = (depthClear && stencilClear) ? VK_IMAGE_LAYOUT_UNDEFINED : texture->imageLayout();
+ desc.finalLayout = texture->imageLayout();
+ renderPassAttachments.push_back(desc);
+ _readWriteTextures.push_back(depthStencilTexture);
+ _depthRoot = texture;
+ while (auto parent = std::dynamic_pointer_cast<PrivateTexture>(_depthRoot->parentTexture())) _depthRoot = parent;
+ }
+ // League uses fragment-to-fragment barriers for texture/render-target scope.
+ VkSubpassDependency dependency {};
+ dependency.srcSubpass = dependency.dstSubpass = 0;
+ dependency.srcStageMask = dependency.dstStageMask = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
+ dependency.srcAccessMask = dependency.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT |
+ VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
+ dependency.dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT;
+ dependencies.push_back(dependency);
std::vector<VkAttachmentReference> colorAttachments;
VkAttachmentReference depthStencilAttachment {};
@@ -102,13 +215,15 @@ Indium::PrivateRenderCommandEncoder::PrivateRenderCommandEncoder(std::shared_ptr
size_t index = 0;
for (const auto& color: descriptor.colorAttachments) {
VkAttachmentReference ref {};
- ref.attachment = index;
- ref.layout = VK_IMAGE_LAYOUT_GENERAL;
+ ref.attachment = color.texture ? index : VK_ATTACHMENT_UNUSED;
+ // textures that rest outside GENERAL (see ConcreteTexture::imageLayout) are written as colour attachments
+ ref.layout = (color.texture && std::dynamic_pointer_cast<PrivateTexture>(color.texture)->imageLayout() != VK_IMAGE_LAYOUT_GENERAL) ?
+ VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_GENERAL;
colorAttachments.push_back(ref);
- ++index;
+ if (color.texture) ++index;
}
- if (descriptor.depthAttachment) {
+ if (depthStencilTexture) {
depthStencilAttachment.attachment = index;
depthStencilAttachment.layout = VK_IMAGE_LAYOUT_GENERAL;
++index;
@@ -129,19 +244,19 @@ Indium::PrivateRenderCommandEncoder::PrivateRenderCommandEncoder(std::shared_ptr
renderPassInfo.dependencyCount = dependencies.size();
renderPassInfo.pDependencies = dependencies.data();
- if (DynamicVK::vkCreateRenderPass(vkDevice, &renderPassInfo, nullptr, &_renderPass) != VK_SUCCESS) {
- // TODO
- abort();
- }
+ auto renderPassLease = _privateDevice->acquireRenderPass(renderPassInfo);
+ _renderPass = renderPassLease.first;
+ _renderPassDeviceOwned = renderPassLease.second;
std::vector<VkImageView> framebufferAttachments;
for (const auto& colorAttachment: descriptor.colorAttachments) {
+ if (!colorAttachment.texture) continue;
framebufferAttachments.push_back(std::dynamic_pointer_cast<PrivateTexture>(colorAttachment.texture)->imageView());
}
- if (descriptor.depthAttachment) {
- framebufferAttachments.push_back(std::dynamic_pointer_cast<PrivateTexture>(descriptor.depthAttachment->texture)->imageView());
+ if (depthStencilTexture) {
+ framebufferAttachments.push_back(std::dynamic_pointer_cast<PrivateTexture>(depthStencilTexture)->imageView());
}
VkFramebufferCreateInfo framebufferInfo {};
@@ -165,7 +280,51 @@ Indium::PrivateRenderCommandEncoder::PrivateRenderCommandEncoder(std::shared_ptr
renderPassBeginInfo.renderArea.extent.height = firstTexture->height();
renderPassBeginInfo.clearValueCount = clearValues.size();
renderPassBeginInfo.pClearValues = clearValues.data();
- DynamicVK::vkCmdBeginRenderPass(vkCmdBuf, &renderPassBeginInfo, VK_SUBPASS_CONTENTS_INLINE);
+ if (GpuTiming::enabled()) {
+ std::string key = std::to_string(firstTexture->width()) + "x" + std::to_string(firstTexture->height());
+ int n = 0;
+ for (const auto& color: descriptor.colorAttachments) {
+ if (!color.texture) continue;
+ key += " c" + std::to_string(n++) + "=" + std::to_string((int) color.texture->pixelFormat()) + "/" + std::to_string((int) color.loadAction) + "/" + std::to_string((int) color.storeAction);
+ }
+ if (depthStencilTexture) {
+ key += " d=" + std::to_string((int) depthStencilTexture->pixelFormat());
+ if (descriptor.depthAttachment) key += "/" + std::to_string((int) descriptor.depthAttachment->loadAction) + "/" + std::to_string((int) descriptor.depthAttachment->storeAction);
+ }
+ GpuTiming::noteTarget(key);
+ if (seqLogActive()) {
+ std::string line = key;
+ for (const auto& color: descriptor.colorAttachments) {
+ char tmp[40];
+ if (color.texture) { snprintf(tmp, sizeof(tmp), " ct=%p", (void*) color.texture.get()); line += tmp; }
+ }
+ if (depthStencilTexture) { char tmp[40]; snprintf(tmp, sizeof(tmp), " dt=%p", (void*) depthStencilTexture.get()); line += tmp; }
+ seqLog("%s\n", line.c_str());
+ }
+ }
+ {
+ // A pass without a clear that never draws (the game's skipped outline passes, empty passes) does nothing in Metal
+ // either, but a begun-and-ended Vulkan render pass costs a pipeline flush. Begin it on the first draw instead.
+ // Live switch: guest file /tmp/metal-no-lazy-pass turns it off (re-read every 256 passes).
+ static std::atomic<bool> lazySetting {true};
+ static std::atomic<unsigned> lazyChecks {0};
+ if ((lazyChecks.fetch_add(1, std::memory_order_relaxed) & 255) == 0)
+ lazySetting.store(access("/tmp/metal-no-lazy-pass", F_OK) != 0, std::memory_order_relaxed);
+ const bool lazyAllowed = lazySetting.load(std::memory_order_relaxed);
+ bool hasClear = false;
+ for (const auto& attachment: renderPassAttachments) {
+ if (attachment.loadOp == VK_ATTACHMENT_LOAD_OP_CLEAR || attachment.stencilLoadOp == VK_ATTACHMENT_LOAD_OP_CLEAR) hasClear = true;
+ }
+ if (lazyAllowed && !hasClear) {
+ _lazyBeginInfo = renderPassBeginInfo;
+ _lazyBeginInfo.clearValueCount = 0;
+ _lazyBeginInfo.pClearValues = nullptr;
+ _passBegun = false;
+ } else {
+ GpuTiming::beginPass(vkDevice, vkCmdBuf, firstTexture->width(), firstTexture->height());
+ DynamicVK::vkCmdBeginRenderPass(vkCmdBuf, &renderPassBeginInfo, VK_SUBPASS_CONTENTS_INLINE);
+ }
+ }
// set default values
@@ -190,16 +349,21 @@ Indium::PrivateRenderCommandEncoder::~PrivateRenderCommandEncoder() {
if (_framebuffer) {
DynamicVK::vkDestroyFramebuffer(_privateDevice->device(), _framebuffer, nullptr);
}
- if (_renderPass) {
+ if (_renderPass && !_renderPassDeviceOwned) {
DynamicVK::vkDestroyRenderPass(_privateDevice->device(), _renderPass, nullptr);
}
- DynamicVK::vkDestroyDescriptorPool(_privateDevice->device(), _pool, 0);
+ if (_pool) _privateDevice->releaseDescriptorPool(_pool);
+ for (auto pool : _retiredPools) _privateDevice->releaseDescriptorPool(pool);
};
-void Indium::PrivateRenderCommandEncoder::setRenderPipelineState(std::shared_ptr<RenderPipelineState> renderPipelineState) {
- auto buf = _privateCommandBuffer.lock();
- _privatePSO = std::dynamic_pointer_cast<PrivateRenderPipelineState>(renderPipelineState);
- _privatePSO->recreatePipeline(_renderPass, false);
+void Indium::PrivateRenderCommandEncoder::setRenderPipelineState(const std::shared_ptr<RenderPipelineState>& renderPipelineState) {
+ if (_privatePSO == renderPipelineState) return;
+ // Keep each pipeline once, including all layouts and immutable samplers
+ // referenced by earlier draws, until this encoder retires.
+ auto [entry, inserted] = _pipelineOwners.acquire(renderPipelineState);
+ if (!entry || !entry->value) throw std::invalid_argument("Invalid render pipeline state");
+ _privatePSO = std::static_pointer_cast<PrivateRenderPipelineState>(entry->owner);
+ _privatePSO->recreatePipeline(_renderPass, false);
};
void Indium::PrivateRenderCommandEncoder::setFrontFacingWinding(Winding frontFaceWinding) {
@@ -225,6 +389,7 @@ void Indium::PrivateRenderCommandEncoder::setDepthClipMode(DepthClipMode depthCl
}
};
+
void Indium::PrivateRenderCommandEncoder::setViewport(const Viewport& viewport) {
setViewports(&viewport, 1);
};
@@ -239,7 +404,7 @@ void Indium::PrivateRenderCommandEncoder::setViewports(const Viewport* viewports
// so we use a Vulkan 1.1 feature here and just flip the Y-axis of the viewport
// (with a corresponding change in the origin).
vkViewport.x = viewport.originX;
- vkViewport.y = viewport.height - viewport.originY;
+ vkViewport.y = viewport.height + viewport.originY;
vkViewport.width = viewport.width;
vkViewport.height = -viewport.height;
// the documentation for setViewport and setViewports states that znear and zfar must be between
@@ -251,6 +416,7 @@ void Indium::PrivateRenderCommandEncoder::setViewports(const Viewport* viewports
vkViewport.maxDepth = std::clamp(viewport.zfar, 0., 1.);
tmp.push_back(vkViewport);
}
+ if (GpuTiming::enabled() && !tmp.empty()) seqLog(" VP %.0f %.0f %.0f %.0f\n", tmp[0].x, tmp[0].y, tmp[0].width, tmp[0].height);
DynamicVK::vkCmdSetViewportWithCount(buf->commandBuffer(), tmp.size(), tmp.data());
};
@@ -274,6 +440,7 @@ void Indium::PrivateRenderCommandEncoder::setScissorRects(const ScissorRect* sci
vkRect.extent.height = scissorRect.height;
tmp.push_back(vkRect);
}
+ if (GpuTiming::enabled() && !tmp.empty()) seqLog(" SC %d %d %u %u\n", tmp[0].offset.x, tmp[0].offset.y, tmp[0].extent.width, tmp[0].extent.height);
DynamicVK::vkCmdSetScissorWithCount(buf->commandBuffer(), tmp.size(), tmp.data());
};
@@ -287,43 +454,217 @@ void Indium::PrivateRenderCommandEncoder::setBlendColor(float red, float green,
DynamicVK::vkCmdSetBlendConstants(buf->commandBuffer(), tmp);
};
-void Indium::PrivateRenderCommandEncoder::updateBindings() {
- // TODO: better descriptor set resource management
-
- auto buf = _privateCommandBuffer.lock();
-
- std::array<VkDescriptorSet, 2> descriptorSets = createDescriptorSets(_privatePSO->descriptorSetLayouts().layouts, _pool, _privateDevice, { _functionResources[0], _functionResources[1] }, { _privatePSO->vertexFunctionInfo(), _privatePSO->fragmentFunctionInfo() }, _keepAliveBuffers);
+void Indium::PrivateRenderCommandEncoder::updateBindings(VkCommandBuffer commandBuffer) {
+
+ const std::array<std::reference_wrapper<const FunctionInfo>, 2> infos {
+ _privatePSO->vertexFunctionInfo(), _privatePSO->fragmentFunctionInfo()
+ };
+ for (size_t stage = 0; stage < 2; ++stage) {
+ const auto layout = _privatePSO->descriptorSetLayouts().layouts[stage];
+ const auto& current = _functionResources[stage];
+ const auto& previous = _boundResourceKeys[stage];
+ bool unchanged = _boundDescriptorSets[stage] && _boundLayouts[stage] == layout;
+ // No setter changed the stage, and the shader binding plan is identical.
+ // Checking the plan as well as the Vulkan layout matters: compatible
+ // layouts can still refer to different Metal resource indices.
+ if (unchanged && _boundFunctionInfos[stage] == &infos[stage].get() &&
+ _boundResourceRevisions[stage] == current.revision) {
+ ++_descriptorStageReuses;
+ continue;
+ }
+ // Buffers-only change (the usual draw: a new uniform offset, nothing else). Same layout, same binding plan and
+ // no texture or sampler setter since the set was bound: only the address table is rebuilt, without comparing
+ // every binding. The per-binding keys are not maintained here, so the general path below must not trust them.
+ if (unchanged && _boundFunctionInfos[stage] == &infos[stage].get() &&
+ _boundImageRevisions[stage] == current.imageRevision && _boundStageHasBuffers[stage]) {
+ const auto previousTable = _bufferAddressTables[stage];
+ auto& table = _bufferAddressTables[stage];
+ table = makeBufferAddressTable(current, infos[stage], &_resolvedResources[stage], _privateDevice, _keepAliveBuffers, _addressPool, &previousTable);
+ if (previousTable.buffer != table.buffer || previousTable.range != table.range) {
+ if (!_pool) _pool = newDescriptorPool(_privateDevice);
+ std::array<VkDescriptorSetLayout, 1> layouts {layout};
+ _boundDescriptorSets[stage] = createDescriptorSets<1>(layouts, _pool, _retiredPools, _privateDevice,
+ {std::cref(current)}, {infos[stage]}, _keepAliveBuffers, _addressPool, {&_resolvedResources[stage]}, {&table})[0];
+ }
+ _boundKeysStale[stage] = true;
+ _boundResourceRevisions[stage] = current.revision;
+ ++_descriptorStageUpdates;
+ continue;
+ }
+ bool imageBindingsUnchanged = unchanged;
+ if (_boundKeysStale[stage]) unchanged = false; // buffer keys were skipped by the fast path
+ for (const auto& binding : infos[stage].get().bindings) {
+ const auto index = binding.index;
+ if (binding.type == Iridium::BindingType::Texture)
+ imageBindingsUnchanged &= (index < current.textures.size() ? current.textures[index].get() : nullptr) ==
+ (index < previous.textures.size() ? previous.textures[index] : nullptr);
+ else if (binding.type == Iridium::BindingType::Sampler && index != SIZE_MAX)
+ imageBindingsUnchanged &= (index < current.samplers.size() ? current.samplers[index].get() : nullptr) ==
+ (index < previous.samplers.size() ? previous.samplers[index] : nullptr);
+ }
+
+ for (const auto& binding: infos[stage].get().bindings) {
+ if (!unchanged) break;
+ const auto index = binding.index;
+ if (binding.type == Iridium::BindingType::Buffer) {
+ const auto* a = index < current.buffers.size() ? &current.buffers[index] : nullptr;
+ const auto* b = index < previous.buffers.size() ? &previous.buffers[index] : nullptr;
+ unchanged = (a ? a->first.get() : nullptr) == (b ? b->first : nullptr) &&
+ (a ? a->second : 0) == (b ? b->second : 0);
+ } else if (binding.type == Iridium::BindingType::Texture) {
+ unchanged = (index < current.textures.size() ? current.textures[index].get() : nullptr) ==
+ (index < previous.textures.size() ? previous.textures[index] : nullptr);
+ } else if (binding.type == Iridium::BindingType::Sampler && index != SIZE_MAX) {
+ unchanged = (index < current.samplers.size() ? current.samplers[index].get() : nullptr) ==
+ (index < previous.samplers.size() ? previous.samplers[index] : nullptr);
+ }
+ }
+ if (!unchanged) {
+ const bool hasBuffers = std::any_of(infos[stage].get().bindings.begin(), infos[stage].get().bindings.end(),
+ [](const auto& binding) { return binding.type == Iridium::BindingType::Buffer; });
+ _boundStageHasBuffers[stage] = hasBuffers;
+ _boundKeysStale[stage] = false;
+ const auto previousTable = _bufferAddressTables[stage];
+ auto& table = _bufferAddressTables[stage];
+ table = hasBuffers ? makeBufferAddressTable(current, infos[stage], &_resolvedResources[stage], _privateDevice, _keepAliveBuffers, _addressPool, &previousTable) : BufferAddressTable{};
+ // Address values occupy immutable storage. Only the dynamic offset
+ // changes while images, samplers, layout, and backing chunk match.
+ if (!imageBindingsUnchanged || previousTable.buffer != table.buffer || previousTable.range != table.range) {
+ // Clear-only passes never allocate descriptors. Lease a pool
+ // only when a draw first needs a descriptor set.
+ if (!_pool) _pool = newDescriptorPool(_privateDevice);
+ std::array<VkDescriptorSetLayout, 1> layouts {layout};
+ _boundDescriptorSets[stage] = createDescriptorSets<1>(layouts, _pool, _retiredPools, _privateDevice,
+ {std::cref(current)}, {infos[stage]}, _keepAliveBuffers, _addressPool, {&_resolvedResources[stage]}, {&table})[0];
+ _boundLayouts[stage] = layout;
+ }
+ auto& keys = _boundResourceKeys[stage];
+ for (const auto& binding : infos[stage].get().bindings) {
+ const auto index = binding.index;
+ if (binding.type == Iridium::BindingType::Buffer) {
+ growToIndex(keys.buffers, index);
+ keys.buffers[index] = index < current.buffers.size() ?
+ std::make_pair(current.buffers[index].first.get(), current.buffers[index].second) : std::make_pair((Buffer*)nullptr, size_t(0));
+ } else if (binding.type == Iridium::BindingType::Texture) {
+ growToIndex(keys.textures, index);
+ keys.textures[index] = index < current.textures.size() ? current.textures[index].get() : nullptr;
+ } else if (binding.type == Iridium::BindingType::Sampler && index != SIZE_MAX) {
+ growToIndex(keys.samplers, index);
+ keys.samplers[index] = index < current.samplers.size() ? current.samplers[index].get() : nullptr;
+ }
+ }
+ // Buffer and texture resolution caches already retain these owners.
+ // Samplers are retained when their binding changes in resolveSampler().
+ ++_descriptorStageUpdates;
+ } else {
+ ++_descriptorStageReuses;
+ }
+ _boundResourceRevisions[stage] = current.revision;
+ _boundFunctionInfos[stage] = &infos[stage].get();
+ _boundImageRevisions[stage] = current.imageRevision;
+ }
+ const auto& descriptorSets = _boundDescriptorSets;
+
+ if (_depthRoot) {
+ bool feedback = false;
+ for (const auto& texture : _resolvedResources[1].textures) {
+ if (texture.depthFeedback) { feedback = true; break; }
+ }
+ if (feedback && _depthWriteRequested && !_depthWriteSuppressed) {
+ DynamicVK::vkCmdSetDepthWriteEnable(commandBuffer, VK_FALSE);
+ _depthWriteSuppressed = true;
+ static unsigned logged = 0;
+ if (logged < 5 && ++logged) {
+ if (FILE* file = fopen("/tmp/metal-trace.log", "a")) {
+ fprintf(file, "depth-feedback: suppressed depth write on a draw that samples its own depth attachment\n");
+ fclose(file);
+ }
+ }
+ } else if (!feedback && _depthWriteSuppressed) {
+ DynamicVK::vkCmdSetDepthWriteEnable(commandBuffer, _depthWriteRequested ? VK_TRUE : VK_FALSE);
+ _depthWriteSuppressed = false;
+ }
+ }
- DynamicVK::vkCmdBindDescriptorSets(buf->commandBuffer(), VK_PIPELINE_BIND_POINT_GRAPHICS, _privatePSO->pipelineLayout(), 0, descriptorSets.size(), descriptorSets.data(), 0, nullptr);
+ const auto pipelineLayout = _privatePSO->pipelineLayout();
+ std::array<uint32_t, 2> dynamicOffsets {};
+ uint32_t dynamicOffsetCount = 0;
+ for (const auto& table : _bufferAddressTables)
+ if (table.buffer) dynamicOffsets[dynamicOffsetCount++] = table.offset;
+ if (_lastDescriptorPipelineLayout != pipelineLayout || _lastDrawDescriptorSets != descriptorSets || _lastDrawDynamicOffsets != dynamicOffsets) {
+ DynamicVK::vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, descriptorSets.size(), descriptorSets.data(), dynamicOffsetCount, dynamicOffsets.data());
+ _lastDrawDynamicOffsets = dynamicOffsets;
+ _lastDescriptorPipelineLayout = pipelineLayout;
+ _lastDrawDescriptorSets = descriptorSets;
+ }
const auto& vertexInputBindings = _privatePSO->vertexInputBindings();
- if (vertexInputBindings.size() > 0) {
- std::vector<VkBuffer> buffers;
- std::vector<VkDeviceSize> offsets;
-
- buffers.resize(vertexInputBindings.size());
- offsets.resize(vertexInputBindings.size());
+ if (!vertexInputBindings.empty() &&
+ (_lastVertexResourceRevision != _functionResources[0].revision || _lastVertexBindingPSO != _privatePSO.get())) {
+ auto& buffers = _lastVertexBuffers;
+ auto& offsets = _lastVertexOffsets;
+ bool vertexBindingsChanged = buffers.size() < vertexInputBindings.size();
+ buffers.resize(vertexInputBindings.size());
+ offsets.resize(vertexInputBindings.size());
for (size_t vulkanIndex = 0; vulkanIndex < vertexInputBindings.size(); ++vulkanIndex) {
const auto& metalIndex = vertexInputBindings[vulkanIndex];
- if (metalIndex >= _functionResources[0].buffers.size()) {
- // technically, this requires the `nullDescriptor` feature, but we should never run into this case anyways.
+ PrivateBuffer* privateBuffer = nullptr;
+ size_t offset = 0;
+ if (metalIndex < _functionResources[0].buffers.size()) {
+ const auto& [buffer, bufferOffset] = _functionResources[0].buffers[metalIndex];
+ privateBuffer = metalIndex < _resolvedResources[0].buffers.size() ?
+ _resolvedResources[0].buffers[metalIndex].value : dynamic_cast<PrivateBuffer*>(buffer.get());
+ offset = bufferOffset;
+ }
+ const auto previousBuffer = buffers[vulkanIndex];
+ const auto previousOffset = offsets[vulkanIndex];
+ if (!privateBuffer) {
+ // the pipeline fetches vertices from a Metal buffer index the app never bound. Device creation enables
+ // nullDescriptor so VK_NULL_HANDLE is legal here (reads return zero); before that this faulted the GPU at address 0.
+ static unsigned unboundLogged = 0;
+ if (unboundLogged < 20) {
+ ++unboundLogged;
+ if (FILE* file = fopen("/tmp/metal-trace.log", "a")) {
+ fprintf(file, "unbound-vertex-buffer: vk%zu <- mtl%u (resources.buffers.size=%zu)\n", vulkanIndex, (unsigned) metalIndex, _functionResources[0].buffers.size());
+ fclose(file);
+ }
+ }
buffers[vulkanIndex] = VK_NULL_HANDLE;
offsets[vulkanIndex] = 0;
} else {
- auto [buffer, offset] = _functionResources[0].buffers[metalIndex];
- auto privateBuffer = std::dynamic_pointer_cast<PrivateBuffer>(buffer);
buffers[vulkanIndex] = privateBuffer->buffer();
offsets[vulkanIndex] = offset;
+ // The unique resource owner above retains this attribute buffer.
}
+ vertexBindingsChanged |= previousBuffer != buffers[vulkanIndex] || previousOffset != offsets[vulkanIndex];
}
- DynamicVK::vkCmdBindVertexBuffers(buf->commandBuffer(), 0, vertexInputBindings.size(), buffers.data(), offsets.data());
+ static unsigned bindingsLogged = 0;
+ if (bindingsLogged < 40 && Indium::spirvDebugEnabled()) {
+ ++bindingsLogged;
+ if (FILE* file = fopen("/tmp/metal-trace.log", "a")) {
+ fprintf(file, "vertex-bindings (resources.buffers.size=%zu):", _functionResources[0].buffers.size());
+ for (size_t i = 0; i < vertexInputBindings.size(); ++i) {
+ fprintf(file, " [vk%zu<-mtl%u %p+%llu]", i, (unsigned) vertexInputBindings[i], (void*) buffers[i], (unsigned long long) offsets[i]);
+ }
+ fprintf(file, "\n");
+ fclose(file);
+ }
+ }
+ if (vertexBindingsChanged)
+ DynamicVK::vkCmdBindVertexBuffers(commandBuffer, 0, vertexInputBindings.size(), buffers.data(), offsets.data());
+ _lastVertexResourceRevision = _functionResources[0].revision;
+ _lastVertexBindingPSO = _privatePSO.get();
}
};
void Indium::PrivateRenderCommandEncoder::drawPrimitives(PrimitiveType primitiveType, size_t vertexStart, size_t vertexCount, size_t instanceCount, size_t baseInstance) {
+ if (drawSkip.encoder == this && drawSkip.skip) return;
+ if (OutlineChain::shouldSkipDraw(_privatePSO.get(), vertexCount)) return;
+ beginPassIfNeeded();
+ GpuTiming::countDraw();
auto buf = _privateCommandBuffer.lock();
// bind the pipeline with the right topology class for this primitive
@@ -343,20 +684,26 @@ void Indium::PrivateRenderCommandEncoder::drawPrimitives(PrimitiveType primitive
default:
throw BadEnumValue();
}
- DynamicVK::vkCmdBindPipeline(buf->commandBuffer(), VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
-
- DynamicVK::vkCmdSetPrimitiveTopology(buf->commandBuffer(), primitiveTypeToVkPrimitiveTopology(primitiveType));
+ if (_lastDrawPipeline != pipeline) {
+ DynamicVK::vkCmdBindPipeline(buf->commandBuffer(), VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
+ _lastDrawPipeline = pipeline;
+ }
+ const auto topology = primitiveTypeToVkPrimitiveTopology(primitiveType);
+ if (_lastDrawTopology != topology) {
+ DynamicVK::vkCmdSetPrimitiveTopology(buf->commandBuffer(), topology);
+ _lastDrawTopology = topology;
+ }
// TODO: avoid re-binding descriptors on every draw.
- updateBindings();
+ updateBindings(buf->commandBuffer());
+ if (GpuTiming::enabled()) seqLog(" DRAW %p v=%zu i=%zu\n", (void*) _privatePSO.get(), vertexCount, instanceCount);
DynamicVK::vkCmdDraw(buf->commandBuffer(), vertexCount, instanceCount, vertexStart, baseInstance);
+ GpuTiming::afterDraw(buf->commandBuffer(), (uint64_t)(uintptr_t) _privatePSO.get());
// when we emit a draw call, we save the current function resources so that they stay alive until the command buffer is done.
// TODO: do this more efficiently by essentially doing COW: after a draw call, we only save additional references to resources
// if someone tries to overwrite them.
- _savedFunctionResources.push_back(_functionResources[0]);
- _savedFunctionResources.push_back(_functionResources[1]);
};
void Indium::PrivateRenderCommandEncoder::drawPrimitives(PrimitiveType primitiveType, size_t vertexStart, size_t vertexCount, size_t instanceCount) {
@@ -368,22 +715,42 @@ void Indium::PrivateRenderCommandEncoder::drawPrimitives(PrimitiveType primitive
};
void Indium::PrivateRenderCommandEncoder::setVertexBytes(const void* bytes, size_t length, size_t index) {
+ if (GpuTiming::enabled()) seqLogBytes("VB", bytes, length, index);
auto cmdBuf = _privateCommandBuffer.lock();
_functionResources[0].setBytes(cmdBuf->device(), bytes, length, index);
+ resolveBuffer(0, index);
};
+void Indium::PrivateRenderCommandEncoder::beginPassIfNeeded() {
+ if (_passBegun) return;
+ _passBegun = true;
+ auto buf = _privateCommandBuffer.lock();
+ GpuTiming::beginPass(_privateDevice->device(), buf->commandBuffer(), _lazyBeginInfo.renderArea.extent.width, _lazyBeginInfo.renderArea.extent.height);
+ DynamicVK::vkCmdBeginRenderPass(buf->commandBuffer(), &_lazyBeginInfo, VK_SUBPASS_CONTENTS_INLINE);
+}
+
void Indium::PrivateRenderCommandEncoder::endEncoding() {
auto buf = _privateCommandBuffer.lock();
- DynamicVK::vkCmdEndRenderPass(buf->commandBuffer());
+ if (_passBegun) {
+ DynamicVK::vkCmdEndRenderPass(buf->commandBuffer());
+ GpuTiming::endPass(buf->commandBuffer());
+ }
+ finishTrackedEncoder(buf->commandBuffer());
+ auto updates = _descriptorStageUpdates;
+ auto reuses = _descriptorStageReuses;
+ if (updates) Indium::count("descriptorStageUpdate", updates);
+ if (reuses) Indium::count("descriptorStageReuse", reuses);
};
-void Indium::PrivateRenderCommandEncoder::setVertexBuffer(std::shared_ptr<Buffer> buffer, size_t offset, size_t index) {
+void Indium::PrivateRenderCommandEncoder::setVertexBuffer(const std::shared_ptr<Buffer>& buffer, size_t offset, size_t index) {
_functionResources[0].setBuffer(buffer, offset, index);
+ resolveBuffer(0, index);
};
void Indium::PrivateRenderCommandEncoder::setVertexBuffers(const std::vector<std::shared_ptr<Buffer>>& buffers, const std::vector<size_t>& offsets, Range<size_t> range) {
for (size_t i = 0; i < range.length; ++i) {
- setVertexBuffer(buffers[i], offsets[i], range.start + i);
+ _functionResources[0].setBuffer(buffers[i], offsets[i], range.start + i);
+ resolveBuffer(0, range.start + i);
}
};
@@ -391,12 +758,14 @@ void Indium::PrivateRenderCommandEncoder::setVertexBufferOffset(size_t offset, s
_functionResources[0].setBufferOffset(offset, index);
};
-void Indium::PrivateRenderCommandEncoder::setVertexSamplerState(std::shared_ptr<SamplerState> state, size_t index) {
+void Indium::PrivateRenderCommandEncoder::setVertexSamplerState(const std::shared_ptr<SamplerState>& state, size_t index) {
_functionResources[0].setSamplerState(state, std::nullopt, index);
+ resolveSampler(0, index);
};
-void Indium::PrivateRenderCommandEncoder::setVertexSamplerState(std::shared_ptr<SamplerState> state, float lodMinClamp, float lodMaxClamp, size_t index) {
+void Indium::PrivateRenderCommandEncoder::setVertexSamplerState(const std::shared_ptr<SamplerState>& state, float lodMinClamp, float lodMaxClamp, size_t index) {
_functionResources[0].setSamplerState(state, std::make_pair(lodMinClamp, lodMaxClamp), index);
+ resolveSampler(0, index);
};
void Indium::PrivateRenderCommandEncoder::setVertexSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& states, Range<size_t> range) {
@@ -411,28 +780,34 @@ void Indium::PrivateRenderCommandEncoder::setVertexSamplerStates(const std::vect
}
};
-void Indium::PrivateRenderCommandEncoder::setVertexTexture(std::shared_ptr<Texture> texture, size_t index) {
+void Indium::PrivateRenderCommandEncoder::setVertexTexture(const std::shared_ptr<Texture>& texture, size_t index) {
_functionResources[0].setTexture(texture, index);
+ resolveTexture(0, index);
};
void Indium::PrivateRenderCommandEncoder::setVertexTextures(const std::vector<std::shared_ptr<Texture>>& textures, Range<size_t> range) {
for (size_t i = 0; i < range.length; ++i) {
- setVertexTexture(textures[i], range.start + i);
+ _functionResources[0].setTexture(textures[i], range.start + i);
+ resolveTexture(0, range.start + i);
}
};
void Indium::PrivateRenderCommandEncoder::setFragmentBytes(const void* bytes, size_t length, size_t index) {
+ if (GpuTiming::enabled()) seqLogBytes("FB", bytes, length, index);
auto cmdBuf = _privateCommandBuffer.lock();
_functionResources[1].setBytes(cmdBuf->device(), bytes, length, index);
+ resolveBuffer(1, index);
};
-void Indium::PrivateRenderCommandEncoder::setFragmentBuffer(std::shared_ptr<Buffer> buffer, size_t offset, size_t index) {
+void Indium::PrivateRenderCommandEncoder::setFragmentBuffer(const std::shared_ptr<Buffer>& buffer, size_t offset, size_t index) {
_functionResources[1].setBuffer(buffer, offset, index);
+ resolveBuffer(1, index);
};
void Indium::PrivateRenderCommandEncoder::setFragmentBuffers(const std::vector<std::shared_ptr<Buffer>>& buffers, const std::vector<size_t>& offsets, Range<size_t> range) {
for (size_t i = 0; i < range.length; ++i) {
- setFragmentBuffer(buffers[i], offsets[i], range.start + i);
+ _functionResources[1].setBuffer(buffers[i], offsets[i], range.start + i);
+ resolveBuffer(1, range.start + i);
}
};
@@ -440,12 +815,14 @@ void Indium::PrivateRenderCommandEncoder::setFragmentBufferOffset(size_t offset,
_functionResources[1].setBufferOffset(offset, index);
};
-void Indium::PrivateRenderCommandEncoder::setFragmentSamplerState(std::shared_ptr<SamplerState> state, size_t index) {
+void Indium::PrivateRenderCommandEncoder::setFragmentSamplerState(const std::shared_ptr<SamplerState>& state, size_t index) {
_functionResources[1].setSamplerState(state, std::nullopt, index);
+ resolveSampler(1, index);
};
-void Indium::PrivateRenderCommandEncoder::setFragmentSamplerState(std::shared_ptr<SamplerState> state, float lodMinClamp, float lodMaxClamp, size_t index) {
+void Indium::PrivateRenderCommandEncoder::setFragmentSamplerState(const std::shared_ptr<SamplerState>& state, float lodMinClamp, float lodMaxClamp, size_t index) {
_functionResources[1].setSamplerState(state, std::make_pair(lodMinClamp, lodMaxClamp), index);
+ resolveSampler(1, index);
};
void Indium::PrivateRenderCommandEncoder::setFragmentSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& states, Range<size_t> range) {
@@ -460,17 +837,22 @@ void Indium::PrivateRenderCommandEncoder::setFragmentSamplerStates(const std::ve
}
};
-void Indium::PrivateRenderCommandEncoder::setFragmentTexture(std::shared_ptr<Texture> texture, size_t index) {
+void Indium::PrivateRenderCommandEncoder::setFragmentTexture(const std::shared_ptr<Texture>& texture, size_t index) {
_functionResources[1].setTexture(texture, index);
+ resolveTexture(1, index);
};
void Indium::PrivateRenderCommandEncoder::setFragmentTextures(std::vector<std::shared_ptr<Texture>>& textures, Range<size_t> range) {
for (size_t i = 0; i < range.length; ++i) {
- setFragmentTexture(textures[i], range.start + i);
+ _functionResources[1].setTexture(textures[i], range.start + i);
+ resolveTexture(1, range.start + i);
}
};
-void Indium::PrivateRenderCommandEncoder::drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, std::shared_ptr<Buffer> indexBuffer, size_t indexBufferOffset, size_t instanceCount, int64_t baseVertex, size_t baseInstance) {
+void Indium::PrivateRenderCommandEncoder::drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, const std::shared_ptr<Buffer>& indexBuffer, size_t indexBufferOffset, size_t instanceCount, int64_t baseVertex, size_t baseInstance) {
+ if (drawSkip.encoder == this && drawSkip.skip) return;
+ beginPassIfNeeded();
+ GpuTiming::countDraw();
auto buf = _privateCommandBuffer.lock();
// bind the pipeline with the right topology class for this primitive
@@ -490,39 +872,62 @@ void Indium::PrivateRenderCommandEncoder::drawIndexedPrimitives(PrimitiveType pr
default:
throw BadEnumValue();
}
- DynamicVK::vkCmdBindPipeline(buf->commandBuffer(), VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
-
- DynamicVK::vkCmdSetPrimitiveTopology(buf->commandBuffer(), primitiveTypeToVkPrimitiveTopology(primitiveType));
+ if (_lastDrawPipeline != pipeline) {
+ DynamicVK::vkCmdBindPipeline(buf->commandBuffer(), VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
+ _lastDrawPipeline = pipeline;
+ }
+ const auto topology = primitiveTypeToVkPrimitiveTopology(primitiveType);
+ if (_lastDrawTopology != topology) {
+ DynamicVK::vkCmdSetPrimitiveTopology(buf->commandBuffer(), topology);
+ _lastDrawTopology = topology;
+ }
// TODO: avoid re-binding descriptors on every draw.
- updateBindings();
-
- // we need to keep this buffer alive until we complete the render
- _keepAliveBuffers.push_back(indexBuffer);
-
- auto privateIndexBuffer = std::dynamic_pointer_cast<PrivateBuffer>(indexBuffer);
-
- DynamicVK::vkCmdBindIndexBuffer(buf->commandBuffer(), privateIndexBuffer->buffer(), indexBufferOffset, indexTypeToVkIndexType(indexType));
+ updateBindings(buf->commandBuffer());
+
+ // Earlier draws retain the owner until GPU completion; this also makes
+ // cached resource identities immune to allocator address reuse.
+ if (_lastIndexResource.object != indexBuffer.get()) {
+ auto [owner, inserted] = _resolvedBufferOwners.acquire(indexBuffer);
+ _lastIndexResource.object = indexBuffer.get();
+ _lastIndexResource.value = owner ? owner->value : nullptr;
+ }
+ const auto indexHandle = _lastIndexResource.value->buffer();
+ const auto vkIndexType = indexTypeToVkIndexType(indexType);
+ if (_lastIndexBuffer != indexHandle || _lastIndexOffset != indexBufferOffset || _lastIndexType != vkIndexType) {
+ DynamicVK::vkCmdBindIndexBuffer(buf->commandBuffer(), indexHandle, indexBufferOffset, vkIndexType);
+ _lastIndexBuffer = indexHandle;
+ _lastIndexOffset = indexBufferOffset;
+ _lastIndexType = vkIndexType;
+ }
+ if (GpuTiming::enabled()) seqLog(" DRAWI %p n=%zu i=%zu\n", (void*) _privatePSO.get(), indexCount, instanceCount);
DynamicVK::vkCmdDrawIndexed(buf->commandBuffer(), indexCount, instanceCount, 0, baseVertex, baseInstance);
+ GpuTiming::afterDraw(buf->commandBuffer(), (uint64_t)(uintptr_t) _privatePSO.get());
// see drawPrimitives() to know why we do this
- _savedFunctionResources.push_back(_functionResources[0]);
- _savedFunctionResources.push_back(_functionResources[1]);
};
-void Indium::PrivateRenderCommandEncoder::drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, std::shared_ptr<Buffer> indexBuffer, size_t indexBufferOffset, size_t instanceCount) {
+void Indium::PrivateRenderCommandEncoder::drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, const std::shared_ptr<Buffer>& indexBuffer, size_t indexBufferOffset, size_t instanceCount) {
drawIndexedPrimitives(primitiveType, indexCount, indexType, indexBuffer, indexBufferOffset, instanceCount, 0, 0);
};
-void Indium::PrivateRenderCommandEncoder::drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, std::shared_ptr<Buffer> indexBuffer, size_t indexBufferOffset) {
+void Indium::PrivateRenderCommandEncoder::drawIndexedPrimitives(PrimitiveType primitiveType, size_t indexCount, IndexType indexType, const std::shared_ptr<Buffer>& indexBuffer, size_t indexBufferOffset) {
drawIndexedPrimitives(primitiveType, indexCount, indexType, indexBuffer, indexBufferOffset, 1);
};
-void Indium::PrivateRenderCommandEncoder::setDepthStencilState(std::shared_ptr<DepthStencilState> state) {
+void Indium::PrivateRenderCommandEncoder::setDepthStencilState(const std::shared_ptr<DepthStencilState>& state) {
auto buf = _privateCommandBuffer.lock();
auto privateState = std::dynamic_pointer_cast<PrivateDepthStencilState>(state);
- auto& desc = privateState->descriptor();
-
+ if (!privateState) {
+ DynamicVK::vkCmdSetDepthTestEnable(buf->commandBuffer(), VK_FALSE);
+ DynamicVK::vkCmdSetDepthWriteEnable(buf->commandBuffer(), VK_FALSE);
+ DynamicVK::vkCmdSetStencilTestEnable(buf->commandBuffer(), VK_FALSE);
+ return;
+ }
+ auto& desc = privateState->descriptor();
+
+ _depthWriteRequested = desc.depthWriteEnabled;
+ _depthWriteSuppressed = false;
DynamicVK::vkCmdSetDepthWriteEnable(buf->commandBuffer(), desc.depthWriteEnabled ? VK_TRUE : VK_FALSE);
DynamicVK::vkCmdSetDepthCompareOp(buf->commandBuffer(), compareFunctionToVkCompareOp(desc.depthCompareFunction));
DynamicVK::vkCmdSetDepthTestEnable(buf->commandBuffer(), VK_TRUE);
@@ -694,3 +1099,18 @@ void Indium::PrivateRenderCommandEncoder::useResources(const std::vector<std::sh
// TODO: check what Metal does in this case. this is just an educated guess
useResources(resources, usage, RenderStages::Vertex | RenderStages::Fragment | RenderStages::Tile | RenderStages::Object | RenderStages::Mesh);
};
+
+void Indium::PrivateRenderCommandEncoder::memoryBarrier(size_t scope, RenderStages after, RenderStages before) {
+ if (after != RenderStages::Fragment || before != RenderStages::Fragment)
+ throw std::invalid_argument("Only fragment-to-fragment render barriers are supported");
+ if (!scope || (scope & ~size_t(7))) throw std::invalid_argument("Invalid render barrier scope");
+ auto buf = _privateCommandBuffer.lock();
+ VkMemoryBarrier barrier {};
+ barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
+ if (scope & 3) barrier.srcAccessMask |= VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
+ if (scope & 4) barrier.srcAccessMask |= VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
+ barrier.dstAccessMask = barrier.srcAccessMask;
+ VkPipelineStageFlags stages = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
+ if (scope & 4) stages |= VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
+ DynamicVK::vkCmdPipelineBarrier(buf->commandBuffer(), stages, stages, VK_DEPENDENCY_BY_REGION_BIT, 1, &barrier, 0, nullptr, 0, nullptr);
+}
diff --git a/src/indium/render-pipeline.cpp b/src/indium/render-pipeline.cpp
index 1b6bb3b..97a1116 100644
--- a/src/indium/render-pipeline.cpp
+++ b/src/indium/render-pipeline.cpp
@@ -1,8 +1,13 @@
+#include <indium/outline-chain.private.hpp>
+#include <indium/gpu-timing.private.hpp>
+#include <unistd.h>
+#include <cstdio>
#include <indium/render-pipeline.private.hpp>
#include <indium/device.private.hpp>
#include <indium/library.private.hpp>
#include <indium/types.private.hpp>
#include <indium/dynamic-vk.hpp>
+#include <indium/counters.private.hpp>
#include <stdexcept>
@@ -22,9 +27,33 @@ Indium::PrivateRenderPipelineState::PrivateRenderPipelineState(std::shared_ptr<P
_vertexFunction = std::dynamic_pointer_cast<PrivateFunction>(descriptor.vertexFunction);
_fragmentFunction = std::dynamic_pointer_cast<PrivateFunction>(descriptor.fragmentFunction);
-
- _descriptorSetLayouts.processFunction(_vertexFunction, 0);
- _descriptorSetLayouts.processFunction(_fragmentFunction, 1);
+ GpuTiming::notePipeline(this, _vertexFunction ? GpuTiming::libraryId(_vertexFunction->library().get()) : std::string("-"), _fragmentFunction ? GpuTiming::libraryId(_fragmentFunction->library().get()) : std::string("-"));
+ OutlineChain::notePipeline(this, _fragmentFunction ? (const void*) _fragmentFunction->library().get() : nullptr);
+
+ if (Indium::spirvDebugEnabled()) {
+ static unsigned created = 0;
+ if (created++ < 8192) {
+ if (FILE* file = fopen("/tmp/metal-trace.log", "a")) {
+ fprintf(file, "indium-pso %p vs-lib=%p fs-lib=%p layouts:", (void*) this,
+ _vertexFunction ? (void*) _vertexFunction->library().get() : nullptr, _fragmentFunction ? (void*) _fragmentFunction->library().get() : nullptr);
+ if (_vertexDescriptor) {
+ for (const auto& [index, layout]: _vertexDescriptor->layouts) {
+ fprintf(file, " [%zu stride=%zu step=%d]", (size_t) index, (size_t) layout.stride, (int) layout.stepFunction);
+ }
+ fprintf(file, " attrs:");
+ for (const auto& [index, attr]: _vertexDescriptor->attributes) {
+ fprintf(file, " [%zu fmt=%d off=%zu buf=%zu]", (size_t) index, (int) attr.format, (size_t) attr.offset, (size_t) attr.bufferIndex);
+ }
+ } else {
+ fprintf(file, " none");
+ }
+ fprintf(file, "\n");
+ fclose(file);
+ }
+ }
+ }
+ _descriptorSetLayouts.processFunction(_vertexFunction, 0, true);
+ _descriptorSetLayouts.processFunction(_fragmentFunction, 1, true);
};
Indium::PrivateRenderPipelineState::~PrivateRenderPipelineState() {
@@ -67,6 +96,14 @@ void Indium::PrivateRenderPipelineState::recreatePipeline(VkRenderPass compatibl
shaderStage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
shaderStage.stage = VK_SHADER_STAGE_VERTEX_BIT;
+ static unsigned libsLogged = 0;
+ if (libsLogged < 8192 && Indium::spirvDebugEnabled()) {
+ ++libsLogged;
+ if (FILE* file = fopen("/tmp/metal-trace.log", "a")) {
+ fprintf(file, "pipeline-libs vs=%p fs=%p\n", (void*) _vertexFunction->library().get(), _fragmentFunction ? (void*) _fragmentFunction->library().get() : nullptr);
+ fclose(file);
+ }
+ }
shaderStage.module = _vertexFunction->library()->shaderModule();
shaderStage.pName = vertexFunctionName.c_str();
stages.push_back(shaderStage);
@@ -168,6 +205,7 @@ void Indium::PrivateRenderPipelineState::recreatePipeline(VkRenderPass compatibl
tmp.dstAlphaBlendFactor = blendFactorToVkBlendFactor(colorInfo.destinationAlphaBlendFactor);
tmp.alphaBlendOp = blendOperationToVkBlendOp(colorInfo.alphaBlendOperation);
tmp.colorWriteMask = colorWriteMaskToVkColorComponentFlags(colorInfo.writeMask);
+ if (colorInfo.pixelFormat == PixelFormat::Invalid) tmp.colorWriteMask = 0;
colorBlendStates.push_back(tmp);
}
VkPipelineColorBlendStateCreateInfo colorBlendState {};
diff --git a/src/indium/sampler.cpp b/src/indium/sampler.cpp
index 71392df..d82e5f0 100644
--- a/src/indium/sampler.cpp
+++ b/src/indium/sampler.cpp
@@ -19,10 +19,14 @@ Indium::PrivateSamplerState::PrivateSamplerState(std::shared_ptr<PrivateDevice>
info.mipLodBias = 0; // not sure where to get this from
info.anisotropyEnable = (descriptor.maxAnisotropy > 1) ? VK_TRUE : VK_FALSE;
info.maxAnisotropy = descriptor.maxAnisotropy;
- info.compareEnable = descriptor.normalizedCoordinates ? VK_TRUE : VK_FALSE; // FIXME: this assumes that the implementation defers to whatever the shader state specifies
+ // Metal only uses the compare function for comparison sampling (shadow maps); a sampler left at Never is used for
+ // ordinary reads, which are invalid (and in practice return nothing useful) with a Vulkan comparison sampler.
+ info.compareEnable = (descriptor.normalizedCoordinates && descriptor.compareFunction != CompareFunction::Never) ? VK_TRUE : VK_FALSE;
info.compareOp = compareFunctionToVkCompareOp(descriptor.compareFunction);
- info.minLod = descriptor.normalizedCoordinates ? descriptor.lodMinClamp : 0;
- info.maxLod = descriptor.normalizedCoordinates ? descriptor.lodMaxClamp : 0;
+ // A sampler without a mip filter reads the base level only, whatever the texture has.
+ const bool mipmapped = descriptor.mipFilter != SamplerMipFilter::NotMipmapped;
+ info.minLod = (descriptor.normalizedCoordinates && mipmapped) ? descriptor.lodMinClamp : 0;
+ info.maxLod = (descriptor.normalizedCoordinates && mipmapped) ? descriptor.lodMaxClamp : 0;
info.borderColor = samplerBorderColorToVkBorderColor(descriptor.borderColor);
info.unnormalizedCoordinates = descriptor.normalizedCoordinates ? VK_FALSE : VK_TRUE;
diff --git a/src/indium/texture.cpp b/src/indium/texture.cpp
index 1e3e971..cbdd7ff 100644
--- a/src/indium/texture.cpp
+++ b/src/indium/texture.cpp
@@ -1,8 +1,90 @@
+#include <indium/submit-batch.private.hpp>
+#include <indium/submit-thread.private.hpp>
+#include <indium/deferred-render-encoder.private.hpp>
+#include <unistd.h>
#include <indium/texture.private.hpp>
+#include <indium/memory-pool.private.hpp>
#include <indium/device.private.hpp>
#include <indium/types.private.hpp>
#include <indium/buffer.private.hpp>
#include <indium/dynamic-vk.hpp>
+#include <indium/counters.private.hpp>
+#include <chrono>
+#include <atomic>
+#include <deque>
+#include <memory>
+
+// Texture creation (layout transition) and replaceRegion (staging copy) each submit a one-shot command buffer. They used to
+// wait for its fence, and that fence only signals once the GPU has also finished everything submitted before it: in a League
+// fight the main thread spent 5.7% of its time in that wait (DRM_IOCTL_SYNCOBJ_WAIT, sampled 2026-10-10). The work is ordered
+// by the queue, so later command buffers see its result without anyone waiting; only the one-shot buffer, its fence and the
+// staging buffer have to outlive it. They are parked here and released once their fence has signalled.
+// MEASURED SLOWER and therefore OFF by default: three interleaved fight pairs gave 81.6/78.1/78.4 fps without the wait
+// against 84.9/81.9/81.7 with it (2026-10-10, GPU 95-97% busy, so the wait was not costing throughput).
+// Guest file /tmp/metal-async-uploads enables it (re-read every 64 one-shot submissions).
+namespace {
+ struct PendingOneshot {
+ VkFence fence;
+ VkCommandBuffer commandBuffer;
+ VkImage image;
+ std::shared_ptr<void> keepAlive;
+ };
+
+ // guarded by the device's oneshotCommandPoolMutex
+ std::deque<PendingOneshot>& pendingOneshots() {
+ static auto* list = new std::deque<PendingOneshot>;
+ return *list;
+ }
+
+ bool syncUploads() {
+ static std::atomic<bool> sync {true};
+ static std::atomic<unsigned> calls {0};
+ if ((calls.fetch_add(1, std::memory_order_relaxed) & 63) == 0)
+ sync.store(access("/tmp/metal-async-uploads", F_OK) != 0, std::memory_order_relaxed);
+ return sync.load(std::memory_order_relaxed);
+ }
+
+ void releaseOneshot(const std::shared_ptr<Indium::PrivateDevice>& device, PendingOneshot& entry) {
+ Indium::DynamicVK::vkDestroyFence(device->device(), entry.fence, nullptr);
+ Indium::DynamicVK::vkFreeCommandBuffers(device->device(), device->oneshotCommandPool(), 1, &entry.commandBuffer);
+ }
+
+ // caller holds the pool mutex. Releases finished entries in submission order; with `through` set, waits for and releases
+ // everything up to the last entry that uses that image (the image is about to be destroyed).
+ void retireOneshots(const std::shared_ptr<Indium::PrivateDevice>& device, VkImage through = VK_NULL_HANDLE) {
+ auto& list = pendingOneshots();
+ size_t mustWait = 0;
+ if (through != VK_NULL_HANDLE) {
+ for (size_t i = 0; i < list.size(); ++i) if (list[i].image == through) mustWait = i + 1;
+ }
+ if (list.size() > 512 && mustWait < list.size() - 512) mustWait = list.size() - 512; // bounded backlog
+ while (!list.empty()) {
+ auto& front = list.front();
+ if (mustWait > 0) {
+ Indium::Waits::Block block("oneshotFence");
+ if (Indium::DynamicVK::vkWaitForFences(device->device(), 1, &front.fence, VK_TRUE, 1ull * 1000 * 1000 * 1000) != VK_SUCCESS) abort();
+ --mustWait;
+ } else if (Indium::DynamicVK::vkWaitForFences(device->device(), 1, &front.fence, VK_TRUE, 0) != VK_SUCCESS) { // zero timeout: a status poll
+ break;
+ }
+ releaseOneshot(device, front);
+ list.pop_front();
+ }
+ }
+
+ // caller holds the pool mutex and has submitted `commandBuffer` with `fence`
+ void finishOneshot(const std::shared_ptr<Indium::PrivateDevice>& device, VkFence fence, VkCommandBuffer commandBuffer, VkImage image, std::shared_ptr<void> keepAlive) {
+ PendingOneshot entry {fence, commandBuffer, image, std::move(keepAlive)};
+ if (syncUploads()) {
+ // this is a GPU wait on the uploading thread; counted under /tmp/metal-debug-waits like the hand-off waits
+ Indium::Waits::Block block("oneshotFence");
+ if (Indium::DynamicVK::vkWaitForFences(device->device(), 1, &fence, VK_TRUE, 1ull * 1000 * 1000 * 1000) != VK_SUCCESS) abort();
+ releaseOneshot(device, entry);
+ return;
+ }
+ pendingOneshots().push_back(std::move(entry));
+ }
+}
#include <cstring>
@@ -37,8 +119,8 @@ std::shared_ptr<Indium::Texture> Indium::PrivateTexture::newTextureView(PixelFor
return newTextureView(
pixelFormat,
textureType(),
- Range<size_t> { parentRelativeLevel(), mipmapLevelCount() },
- Range<size_t> { parentRelativeSlice(), arrayLength() }
+ Range<size_t> { 0, mipmapLevelCount() },
+ Range<size_t> { 0, arrayLength() }
);
};
@@ -78,12 +160,16 @@ Indium::TextureView::TextureView(std::shared_ptr<PrivateTexture> original, Pixel
_layers(layers),
_swizzle(swizzle)
{
- auto parentLevelStart = _original->parentRelativeLevel();
+ size_t parentLevelStart = 0;
+ size_t parentLayerStart = 0;
+ for (std::shared_ptr<Texture> parent = _original; parent; parent = parent->parentTexture()) {
+ parentLevelStart += parent->parentRelativeLevel();
+ parentLayerStart += parent->parentRelativeSlice();
+ }
auto parentLevelEnd = parentLevelStart + _original->mipmapLevelCount() - 1;
auto levelStart = std::min(parentLevelStart + _levels.start, parentLevelEnd);
auto levelEnd = std::min(levelStart + _levels.length - 1, parentLevelEnd);
- auto parentLayerStart = _original->parentRelativeSlice();
auto parentLayerEnd = parentLayerStart + _original->arrayLength() - 1;
auto layerStart = std::min(parentLayerStart + _layers.start, parentLayerEnd);
auto layerEnd = std::min(layerStart + _layers.length - 1, parentLayerEnd);
@@ -109,9 +195,16 @@ Indium::TextureView::TextureView(std::shared_ptr<PrivateTexture> original, Pixel
// TODO
abort();
}
+ if ((info.subresourceRange.aspectMask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
+ // Combined depth/stencil is legal for attachments, but descriptors require one aspect.
+ info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT;
+ if (DynamicVK::vkCreateImageView(_device->device(), &info, nullptr, &_sampledImageView) != VK_SUCCESS) abort();
+ }
+
};
Indium::TextureView::~TextureView() {
+ if (_sampledImageView) DynamicVK::vkDestroyImageView(_device->device(), _sampledImageView, nullptr);
DynamicVK::vkDestroyImageView(std::dynamic_pointer_cast<PrivateDevice>(_original->device())->device(), _imageView, nullptr);
};
@@ -124,15 +217,15 @@ Indium::TextureType Indium::TextureView::textureType() const {
};
size_t Indium::TextureView::width() const {
- return _original->width();
+ return std::max(size_t(1), _original->width() >> _levels.start);
};
size_t Indium::TextureView::height() const {
- return _original->height();
+ return std::max(size_t(1), _original->height() >> _levels.start);
};
size_t Indium::TextureView::depth() const {
- return _original->depth();
+ return std::max(size_t(1), _original->depth() >> _levels.start);
};
size_t Indium::TextureView::mipmapLevelCount() const {
@@ -326,13 +419,42 @@ Indium::ConcreteTexture::ConcreteTexture(std::shared_ptr<PrivateDevice> device,
info.tiling = (descriptor.allowGPUOptimizedContents || !canBeLinear) ? VK_IMAGE_TILING_OPTIMAL : VK_IMAGE_TILING_LINEAR;
- // we don't know ahead of time how the image is going to be used, so specify everything we support
- info.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
- if (isColor) {
+ // Optional usages must be supported by the format and selected tiling.
+ // Compressed/sRGB formats commonly cannot be storage images or attachments.
+ VkFormatProperties formatProperties {};
+ DynamicVK::vkGetPhysicalDeviceFormatProperties(_device->physicalDevice(), info.format, &formatProperties);
+ if (info.tiling == VK_IMAGE_TILING_LINEAR && !formatProperties.linearTilingFeatures) {
+ info.tiling = VK_IMAGE_TILING_OPTIMAL;
+ }
+ VkFormatFeatureFlags formatFeatures = info.tiling == VK_IMAGE_TILING_LINEAR ? formatProperties.linearTilingFeatures : formatProperties.optimalTilingFeatures;
+ info.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
+ // RADV cannot keep DCC (colour compression) on GFX9 for an image that may be a storage image, so every render target
+ // that carried STORAGE_BIT cost full uncompressed bandwidth. A render target that Metal does not declare
+ // ShaderWrite does not need it. Guest flag file /tmp/metal-keep-storage restores the old behaviour for A/B runs.
+ static const bool keepStorage = access("/tmp/metal-keep-storage", F_OK) == 0;
+ const size_t metalUsage = static_cast<size_t>(_descriptor.usage);
+ const bool renderTargetOnly = (metalUsage & static_cast<size_t>(TextureUsage::RenderTarget)) != 0 &&
+ (metalUsage & static_cast<size_t>(TextureUsage::ShaderWrite)) == 0;
+ if ((formatFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && (keepStorage || !renderTargetOnly)) {
+ info.usage |= VK_IMAGE_USAGE_STORAGE_BIT;
+ }
+ if (isColor && (formatFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)) {
info.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
- } else if (isDepthStencil) {
+ } else if (isDepthStencil && (formatFeatures & VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT)) {
info.usage |= VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
}
+ {
+ static const bool logTextures = access("/tmp/metal-gpu-timing", F_OK) == 0;
+ if (logTextures) {
+ if (FILE* f = fopen("/tmp/indium-textures.log", "a")) {
+ fprintf(f, "%ux%ux%u mips=%u fmt=%u usage=%zu opt=%d tiling=%s storage=%d storageMode=%u\n", (unsigned) _descriptor.width,
+ (unsigned) _descriptor.height, (unsigned) _descriptor.depth, (unsigned) _descriptor.mipmapLevelCount, (unsigned) _descriptor.pixelFormat,
+ metalUsage, (int) descriptor.allowGPUOptimizedContents, info.tiling == VK_IMAGE_TILING_LINEAR ? "LINEAR" : "OPTIMAL",
+ (info.usage & VK_IMAGE_USAGE_STORAGE_BIT) ? 1 : 0, (unsigned) _storageMode);
+ fclose(f);
+ }
+ }
+ }
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; // TODO: check if this should be "concurrent" instead. i think we're good, though.
info.queueFamilyIndexCount = 0;
info.pQueueFamilyIndices = nullptr;
@@ -347,45 +469,24 @@ Indium::ConcreteTexture::ConcreteTexture(std::shared_ptr<PrivateDevice> device,
DynamicVK::vkGetImageMemoryRequirements(_device->device(), _image, &reqs);
- size_t targetIndex = SIZE_MAX;
-
- for (size_t i = 0; i < _device->memoryProperties().memoryTypeCount; ++i) {
- const auto& type = _device->memoryProperties().memoryTypes[i];
-
- if ((reqs.memoryTypeBits & (1 << i)) == 0) {
- continue;
- }
-
- if ((_storageMode == StorageMode::Managed || _storageMode == StorageMode::Shared) && (type.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0) {
- continue;
- }
-
- if (_storageMode == StorageMode::Shared && (type.propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) == 0) {
- continue;
- }
-
- // okay, this is good enough
- targetIndex = i;
- break;
- }
+ size_t targetIndex = _device->findMemoryType(reqs.memoryTypeBits, _storageMode, true);
if (targetIndex == SIZE_MAX) {
throw std::runtime_error("No suitable memory region found for image with requested storage mode");
}
- VkMemoryAllocateInfo allocateInfo {};
- allocateInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
- allocateInfo.allocationSize = reqs.size;
- allocateInfo.memoryTypeIndex = targetIndex;
+ Indium::count("textureAlloc", reqs.size);
+ auto allocation = MemoryPool::allocate(_device->device(), reinterpret_cast<uint64_t>(_image), targetIndex, reqs.size, reqs.alignment, true, false);
+ _memory = allocation.memory;
- if (DynamicVK::vkAllocateMemory(_device->device(), &allocateInfo, nullptr, &_memory) != VK_SUCCESS) {
- // TODO
- abort();
- }
-
- DynamicVK::vkBindImageMemory(_device->device(), _image, _memory, 0);
+ DynamicVK::vkBindImageMemory(_device->device(), _image, _memory, allocation.offset);
// transition the image into the general layout
+ // Drain before taking the upload pool lock: a queued commit may need the pool to construct its batch barrier.
+ SubmitThread::drain();
+ flushPendingSubmits();
+ std::unique_lock<std::mutex> poolLock(_device->oneshotCommandPoolMutex());
+ retireOneshots(_device);
VkCommandBufferAllocateInfo cmdBufAllocInfo {};
cmdBufAllocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
cmdBufAllocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
@@ -409,7 +510,7 @@ Indium::ConcreteTexture::ConcreteTexture(std::shared_ptr<PrivateDevice> device,
barrier.srcAccessMask = VK_ACCESS_NONE;
barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
- barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL;
+ barrier.newLayout = imageLayout();
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.image = _image;
@@ -438,15 +539,13 @@ Indium::ConcreteTexture::ConcreteTexture(std::shared_ptr<PrivateDevice> device,
submitInfo.commandBufferCount = 1;
submitInfo.pCommandBuffers = &cmdBuf;
- DynamicVK::vkQueueSubmit(_device->graphicsQueue(), 1, &submitInfo, theFence);
- if (DynamicVK::vkWaitForFences(_device->device(), 1, &theFence, VK_TRUE, /* 1s */ 1ull * 1000 * 1000 * 1000) != VK_SUCCESS) {
- // TODO
- abort();
+ {
+ flushPendingSubmits(); // same for commits merged into a pending batch (takes the queue mutex itself)
+ std::lock_guard<std::mutex> queueLock(_device->graphicsQueueMutex());
+ if (DynamicVK::vkQueueSubmit(_device->graphicsQueue(), 1, &submitInfo, theFence) != VK_SUCCESS) abort();
}
-
- DynamicVK::vkDestroyFence(_device->device(), theFence, nullptr);
-
- DynamicVK::vkFreeCommandBuffers(_device->device(), _device->oneshotCommandPool(), 1, &cmdBuf);
+ finishOneshot(_device, theFence, cmdBuf, _image, nullptr);
+ poolLock.unlock();
VkImageViewCreateInfo info2 {};
info2.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
@@ -463,12 +562,24 @@ Indium::ConcreteTexture::ConcreteTexture(std::shared_ptr<PrivateDevice> device,
// TODO
abort();
}
+ if ((info2.subresourceRange.aspectMask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
+ // Combined depth/stencil is legal for attachments, but descriptors require one aspect.
+ info2.subresourceRange.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT;
+ if (DynamicVK::vkCreateImageView(_device->device(), &info2, nullptr, &_sampledImageView) != VK_SUCCESS) abort();
+ }
+
};
Indium::ConcreteTexture::~ConcreteTexture() {
+ {
+ std::unique_lock<std::mutex> poolLock(_device->oneshotCommandPoolMutex());
+ retireOneshots(_device, _image);
+ }
+ if (_sampledImageView) DynamicVK::vkDestroyImageView(_device->device(), _sampledImageView, nullptr);
DynamicVK::vkDestroyImageView(_device->device(), _imageView, nullptr);
+ auto owner = reinterpret_cast<uint64_t>(_image);
DynamicVK::vkDestroyImage(_device->device(), _image, nullptr);
- DynamicVK::vkFreeMemory(_device->device(), _memory, nullptr);
+ MemoryPool::free(_device->device(), owner);
};
Indium::TextureType Indium::ConcreteTexture::textureType() const {
@@ -537,7 +648,20 @@ VkImage Indium::ConcreteTexture::image() {
return _image;
};
+// RADV (GFX9) keeps colour compression (DCC) only outside VK_IMAGE_LAYOUT_GENERAL. A plain colour render target that is
+// also sampled (Metal usage RenderTarget|ShaderRead) rests in SHADER_READ_ONLY_OPTIMAL and is a COLOR_ATTACHMENT_OPTIMAL
+// attachment; everything else keeps GENERAL. Guest flag file /tmp/metal-no-opt-layouts disables it (checked once).
VkImageLayout Indium::ConcreteTexture::imageLayout() {
+ static const bool optLayouts = access("/tmp/metal-no-opt-layouts", F_OK) != 0;
+ if (optLayouts) {
+ const size_t usage = static_cast<size_t>(_descriptor.usage);
+ const size_t renderTarget = static_cast<size_t>(TextureUsage::RenderTarget);
+ const size_t shaderWrite = static_cast<size_t>(TextureUsage::ShaderWrite);
+ if ((usage & renderTarget) && !(usage & shaderWrite) && _descriptor.textureType == TextureType::e2D &&
+ _descriptor.mipmapLevelCount == 1 && _descriptor.arrayLength == 1 &&
+ (pixelFormatToVkImageAspectFlags(_descriptor.pixelFormat) & VK_IMAGE_ASPECT_COLOR_BIT))
+ return VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+ }
return VK_IMAGE_LAYOUT_GENERAL;
};
@@ -546,6 +670,16 @@ void Indium::ConcreteTexture::replaceRegion(Indium::Region region, size_t mipmap
};
void Indium::ConcreteTexture::replaceRegion(Indium::Region region, size_t mipmapLevel, size_t slice, const void* bytes, size_t bytesPerRow, size_t bytesPerImage) {
+ // diagnostic counters (only active with the guest /tmp/metal-debug-timing flag): how often CPU uploads happen and how long the
+ // synchronous staging copy + fence wait blocks the calling thread
+ const auto uploadStart = std::chrono::steady_clock::now();
+ struct UploadTimer {
+ std::chrono::steady_clock::time_point start; size_t bytes;
+ ~UploadTimer() {
+ Indium::count("replaceRegionCalls", bytes);
+ Indium::count("replaceRegionMicroseconds", std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - start).count());
+ }
+ } uploadTimer {uploadStart, bytesPerImage ? bytesPerImage : bytesPerRow * region.size.height};
if (_storageMode != StorageMode::Managed && _storageMode != StorageMode::Shared) {
throw std::runtime_error("Invalid usage of replaceRegion on non-managed and non-shared texture");
}
@@ -556,99 +690,109 @@ void Indium::ConcreteTexture::replaceRegion(Indium::Region region, size_t mipmap
auto aspect = pixelFormatToVkImageAspectFlags(_descriptor.pixelFormat);
- // create a temporary buffer copy
- // FIXME: this doesn't work for PVRTC textures
- // actually, FIXME: this doesn't handle a few cases, particularly with array texture types (since i haven't seen those yet at all)
- auto byteSize = bytesPerImage == 0 ? (region.size.height * bytesPerRow) : (region.size.depth * bytesPerImage);
- auto tmpBuf = std::dynamic_pointer_cast<PrivateBuffer>(_device->newBuffer(bytes, byteSize, ResourceOptions::StorageModeShared));
-
- // FIXME: handle compressed formats
- size_t bytesPerPixel = pixelFormatToByteCount(_descriptor.pixelFormat);
-
- VkCommandBufferAllocateInfo cmdBufAllocInfo {};
- cmdBufAllocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
- cmdBufAllocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
- cmdBufAllocInfo.commandPool = _device->oneshotCommandPool();
- cmdBufAllocInfo.commandBufferCount = 1;
-
- VkCommandBuffer cmdBuf;
- if (DynamicVK::vkAllocateCommandBuffers(_device->device(), &cmdBufAllocInfo, &cmdBuf) != VK_SUCCESS) {
- // TODO
- abort();
- }
-
- VkCommandBufferBeginInfo cmdBufBeginInfo {};
- cmdBufBeginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
- cmdBufBeginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
-
- DynamicVK::vkBeginCommandBuffer(cmdBuf, &cmdBufBeginInfo);
-
- // first, transition the image to the optimal layout for transfer destinations
- VkImageMemoryBarrier barrier {};
- barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
- barrier.srcAccessMask = VK_ACCESS_NONE;
- barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
- barrier.oldLayout = VK_IMAGE_LAYOUT_GENERAL;
- barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
- barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
- barrier.image = _image;
- barrier.subresourceRange.aspectMask = aspect;
- barrier.subresourceRange.baseMipLevel = mipmapLevel;
- barrier.subresourceRange.levelCount = 1;
- barrier.subresourceRange.baseArrayLayer = slice;
- barrier.subresourceRange.layerCount = 1;
-
- DynamicVK::vkCmdPipelineBarrier(cmdBuf, VK_PIPELINE_STAGE_NONE, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
-
- // now encode the copy
- VkBufferImageCopy copyInfo {};
- copyInfo.bufferOffset = 0;
- copyInfo.bufferRowLength = bytesPerRow / bytesPerPixel;
- copyInfo.bufferImageHeight = bytesPerImage / bytesPerRow;
- copyInfo.imageSubresource.aspectMask = aspect;
- copyInfo.imageSubresource.mipLevel = mipmapLevel;
- copyInfo.imageSubresource.baseArrayLayer = slice;
- copyInfo.imageSubresource.layerCount = 1;
- copyInfo.imageOffset.x = region.origin.x;
- copyInfo.imageOffset.y = region.origin.y;
- copyInfo.imageOffset.z = region.origin.z;
- copyInfo.imageExtent.width = region.size.width;
- copyInfo.imageExtent.height = region.size.height;
- copyInfo.imageExtent.depth = region.size.depth;
- DynamicVK::vkCmdCopyBufferToImage(cmdBuf, tmpBuf->buffer(), _image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &copyInfo);
-
- // finally, transition the image back to the general layout
- barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
- barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL;
- barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
- barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
- DynamicVK::vkCmdPipelineBarrier(cmdBuf, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
-
- DynamicVK::vkEndCommandBuffer(cmdBuf);
-
- VkFenceCreateInfo fenceCreateInfo {};
- fenceCreateInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-
- VkFence theFence = VK_NULL_HANDLE;
+ auto pitch = bufferImagePitch(_descriptor.pixelFormat, region.size.width, region.size.height, bytesPerRow, bytesPerImage);
+ auto block = pixelFormatTransferBlock(_descriptor.pixelFormat);
+ size_t rows = (region.size.height + block.height - 1) / block.height;
+ size_t tightRow = ((region.size.width + block.width - 1) / block.width) * block.bytes;
+ size_t byteSize = (region.size.depth - 1) * pitch.imageBytes + (rows - 1) * pitch.rowBytes + tightRow;
+ auto tmpBuf = std::dynamic_pointer_cast<PrivateBuffer>(_device->newBuffer(bytes, byteSize, ResourceOptions::StorageModeShared));
+
+ // With deferred encoding on, the upload is queued behind the pending replays on the same worker (FIFO keeps the order)
+ // instead of draining it from this thread in the middle of a frame.
+ auto upload = [=, self = shared_from_this()]() {
+ // Drain before taking the upload pool lock: a queued commit may need the pool to construct its batch barrier.
+ SubmitThread::drain();
+ flushPendingSubmits();
+ std::unique_lock<std::mutex> poolLock(_device->oneshotCommandPoolMutex());
+ retireOneshots(_device);
+ VkCommandBufferAllocateInfo cmdBufAllocInfo {};
+ cmdBufAllocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
+ cmdBufAllocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
+ cmdBufAllocInfo.commandPool = _device->oneshotCommandPool();
+ cmdBufAllocInfo.commandBufferCount = 1;
+
+ VkCommandBuffer cmdBuf;
+ if (DynamicVK::vkAllocateCommandBuffers(_device->device(), &cmdBufAllocInfo, &cmdBuf) != VK_SUCCESS) {
+ // TODO
+ abort();
+ }
- if (DynamicVK::vkCreateFence(_device->device(), &fenceCreateInfo, nullptr, &theFence) != VK_SUCCESS) {
- // TODO
- abort();
- }
+ VkCommandBufferBeginInfo cmdBufBeginInfo {};
+ cmdBufBeginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
+ cmdBufBeginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
+
+ DynamicVK::vkBeginCommandBuffer(cmdBuf, &cmdBufBeginInfo);
+
+ // first, transition the image to the optimal layout for transfer destinations
+ VkImageMemoryBarrier barrier {};
+ barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
+ barrier.srcAccessMask = VK_ACCESS_NONE;
+ barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
+ barrier.oldLayout = imageLayout();
+ barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+ barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+ barrier.image = _image;
+ barrier.subresourceRange.aspectMask = aspect;
+ barrier.subresourceRange.baseMipLevel = mipmapLevel;
+ barrier.subresourceRange.levelCount = 1;
+ barrier.subresourceRange.baseArrayLayer = slice;
+ barrier.subresourceRange.layerCount = 1;
+
+ DynamicVK::vkCmdPipelineBarrier(cmdBuf, VK_PIPELINE_STAGE_NONE, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
+
+ // now encode the copy
+ VkBufferImageCopy copyInfo {};
+ copyInfo.bufferOffset = 0;
+ copyInfo.bufferRowLength = pitch.rowLength;
+ copyInfo.bufferImageHeight = pitch.imageHeight;
+ copyInfo.imageSubresource.aspectMask = aspect;
+ copyInfo.imageSubresource.mipLevel = mipmapLevel;
+ copyInfo.imageSubresource.baseArrayLayer = slice;
+ copyInfo.imageSubresource.layerCount = 1;
+ copyInfo.imageOffset.x = region.origin.x;
+ copyInfo.imageOffset.y = region.origin.y;
+ copyInfo.imageOffset.z = region.origin.z;
+ copyInfo.imageExtent.width = region.size.width;
+ copyInfo.imageExtent.height = region.size.height;
+ copyInfo.imageExtent.depth = region.size.depth;
+ DynamicVK::vkCmdCopyBufferToImage(cmdBuf, tmpBuf->buffer(), _image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &copyInfo);
+
+ // finally, transition the image back to the general layout
+ barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+ barrier.newLayout = imageLayout();
+ barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
+ barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
+ DynamicVK::vkCmdPipelineBarrier(cmdBuf, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
+
+ DynamicVK::vkEndCommandBuffer(cmdBuf);
+
+ VkFenceCreateInfo fenceCreateInfo {};
+ fenceCreateInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+
+ VkFence theFence = VK_NULL_HANDLE;
+
+ if (DynamicVK::vkCreateFence(_device->device(), &fenceCreateInfo, nullptr, &theFence) != VK_SUCCESS) {
+ // TODO
+ abort();
+ }
- VkSubmitInfo submitInfo {};
- submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
- submitInfo.commandBufferCount = 1;
- submitInfo.pCommandBuffers = &cmdBuf;
+ VkSubmitInfo submitInfo {};
+ submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
+ submitInfo.commandBufferCount = 1;
+ submitInfo.pCommandBuffers = &cmdBuf;
- DynamicVK::vkQueueSubmit(_device->graphicsQueue(), 1, &submitInfo, theFence);
- if (DynamicVK::vkWaitForFences(_device->device(), 1, &theFence, VK_TRUE, /* 1s */ 1ull * 1000 * 1000 * 1000) != VK_SUCCESS) {
- // TODO
- abort();
+ {
+ flushPendingSubmits(); // same for commits merged into a pending batch (takes the queue mutex itself)
+ std::lock_guard<std::mutex> queueLock(_device->graphicsQueueMutex());
+ if (DynamicVK::vkQueueSubmit(_device->graphicsQueue(), 1, &submitInfo, theFence) != VK_SUCCESS) abort();
+ }
+ finishOneshot(_device, theFence, cmdBuf, _image, tmpBuf);
+ poolLock.unlock();
+ };
+ if (AsyncEncode::enabled() && !SubmitThread::onWorker()) {
+ SubmitThread::post(upload);
+ } else {
+ upload();
}
-
- DynamicVK::vkDestroyFence(_device->device(), theFence, nullptr);
-
- DynamicVK::vkFreeCommandBuffers(_device->device(), _device->oneshotCommandPool(), 1, &cmdBuf);
};
diff --git a/src/iridium/air.cpp b/src/iridium/air.cpp
index 21c501f..9f6663b 100644
--- a/src/iridium/air.cpp
+++ b/src/iridium/air.cpp
@@ -4,9 +4,46 @@
#include <iridium/dynamic-llvm.hpp>
#include <llvm-c/BitReader.h>
+#include <set>
+#include <algorithm>
+#include <functional>
+#include <llvm/Config/llvm-config.h>
+
+// LLVM 23 split the C API's LLVMBr opcode into LLVMUncondBr and LLVMCondBr.
+#if LLVM_VERSION_MAJOR >= 23
+#define IRIDIUM_BR_CASES case LLVMUncondBr: case LLVMCondBr
+#define IRIDIUM_IS_BR(op) ((op) == LLVMUncondBr || (op) == LLVMCondBr)
+#else
+#define IRIDIUM_BR_CASES case LLVMBr
+#define IRIDIUM_IS_BR(op) ((op) == LLVMBr)
+#endif
namespace DynamicLLVM = Iridium::DynamicLLVM;
+// AIR links varyings by their user location, not by their order in each stage.
+// Optimized fragment functions may omit entries that the vertex stage still writes.
+static uint32_t airInterfaceLocation(const std::vector<LLVMValueRef>& metadata, uint32_t fallback) {
+ for (size_t i = 0; i < metadata.size(); ++i) {
+ if (!DynamicLLVM::LLVMIsAMDString(metadata[i])) continue;
+ unsigned length = 0;
+ const char* bytes = DynamicLLVM::LLVMGetMDString(metadata[i], &length);
+ std::string_view value(bytes, length);
+ constexpr std::string_view prefix = "user(locn";
+ if (value.size() > prefix.size() + 1 && value.substr(0, prefix.size()) == prefix && value.back() == ')') {
+ uint32_t location = 0;
+ for (char digit : value.substr(prefix.size(), value.size() - prefix.size() - 1)) {
+ if (digit < '0' || digit > '9') throw std::runtime_error("Invalid AIR varying location");
+ location = location * 10 + static_cast<uint32_t>(digit - '0');
+ }
+ return location;
+ }
+ if (value == "air.render_target" && i + 1 < metadata.size())
+ return static_cast<uint32_t>(DynamicLLVM::LLVMConstIntGetZExtValue(metadata[i + 1]));
+ }
+ return fallback;
+}
+
+
// special thanks to https://github.com/YuAo/MetalLibraryArchive for information on the library format
// we use LLVM's C API because it's much more stable than the C++ API and it's good enough for our needs
@@ -29,6 +66,16 @@ Iridium::AIR::Function::Function(Type type, const std::string& name, const void*
}
_module = LLVMSupport::Module(moduleRef, DynamicLLVM::LLVMDisposeModule);
+ // Normalize local aggregate copies into SSA, then give nested selections and loops
+ // distinct merges before lowering them into Vulkan's structured control flow.
+ std::string triple=DynamicLLVM::LLVMGetTarget(moduleRef);
+ DynamicLLVM::LLVMSetTarget(moduleRef,"spirv64-unknown-unknown");
+ auto options=DynamicLLVM::LLVMCreatePassBuilderOptions();
+ auto error=DynamicLLVM::LLVMRunPasses(moduleRef,"function(sroa,mem2reg,loop-unroll,sroa,mem2reg,simplifycfg,structurizecfg)",nullptr,options);
+ DynamicLLVM::LLVMDisposePassBuilderOptions(options);
+ DynamicLLVM::LLVMSetTarget(moduleRef,triple.c_str());
+ if(error){auto message=DynamicLLVM::LLVMGetErrorMessage(error);std::string detail=message?message:"unknown LLVM error";DynamicLLVM::LLVMDisposeErrorMessage(message);throw std::runtime_error("Cannot normalize AIR control flow: "+detail);}
+
_function = DynamicLLVM::LLVMGetNamedFunction(_module.get(), _name.c_str());
@@ -41,7 +88,86 @@ Iridium::AIR::Function::Function(Type type, const std::string& name, const void*
// TEST
#include <iostream>
-class ImpossibleResultID: public std::exception {};
+static thread_local std::string translationContext;
+static thread_local LLVMValueRef translationInstruction=nullptr;
+static std::string describeLLVMValue(LLVMValueRef value) {
+ char* raw = DynamicLLVM::LLVMPrintValueToString(value);
+ std::string result = raw ? raw : "unknown value";
+ if (raw) DynamicLLVM::LLVMDisposeMessage(raw);
+ return result;
+}
+class ImpossibleResultID: public std::runtime_error {
+public: explicit ImpossibleResultID(const std::string& detail = "Unsupported AIR type"): std::runtime_error(detail) {}
+};
+
+// While set, struct and array types are created without explicit layout, as Private/Function memory needs.
+static thread_local bool g_layoutFreeTypes = false;
+
+class LayoutFreeScope {
+public:
+ explicit LayoutFreeScope(bool enabled): _previous(g_layoutFreeTypes) { g_layoutFreeTypes = enabled; }
+ ~LayoutFreeScope() { g_layoutFreeTypes = _previous; }
+private:
+ bool _previous;
+};
+
+// A pointer bitcast retains its intended element type in consuming loads/GEPs.
+static LLVMTypeRef pointerElementType(LLVMValueRef value, unsigned depth = 0) {
+ if (depth > 16) throw std::runtime_error("AIR pointer-use chain exceeds limit");
+ for (auto use = DynamicLLVM::LLVMGetFirstUse(value); use; use = DynamicLLVM::LLVMGetNextUse(use)) {
+ auto user = DynamicLLVM::LLVMGetUser(use);
+ if (DynamicLLVM::LLVMGetValueKind(user) != LLVMInstructionValueKind) continue;
+ auto opcode = DynamicLLVM::LLVMGetInstructionOpcode(user);
+ if (opcode == LLVMGetElementPtr && DynamicLLVM::LLVMGetOperand(user, 0) == value)
+ return DynamicLLVM::LLVMGetGEPSourceElementType(user);
+ if (opcode == LLVMLoad && DynamicLLVM::LLVMGetOperand(user, 0) == value)
+ return DynamicLLVM::LLVMTypeOf(user);
+ if ((opcode == LLVMBitCast || opcode == LLVMAddrSpaceCast) && DynamicLLVM::LLVMGetOperand(user, 0) == value)
+ return pointerElementType(user, depth + 1);
+ // the pointer stored through carries the stored value's type
+ if (opcode == LLVMStore && DynamicLLVM::LLVMGetOperand(user, 1) == value)
+ return DynamicLLVM::LLVMTypeOf(DynamicLLVM::LLVMGetOperand(user, 0));
+ // a pointer that is merged with others is typed by how the merged pointer is used
+ if (opcode == LLVMPHI || opcode == LLVMSelect) {
+ try {
+ return pointerElementType(user, depth + 1);
+ } catch (const std::runtime_error&) {
+ // try the remaining uses
+ }
+ }
+ }
+ // Nothing constrains the pointee (an argument the shader never reads, for example), so any
+ // 32-bit type describes it as well as another.
+ return DynamicLLVM::LLVMInt32Type();
+}
+// Modern LLVM removes pointee types from pointers. Loads and GEP retain them.
+static LLVMTypeRef bufferElementType(LLVMValueRef function, size_t parameterIndex) {
+ auto parameter = DynamicLLVM::LLVMGetParam(function, parameterIndex);
+ for (auto block = DynamicLLVM::LLVMGetFirstBasicBlock(function); block; block = DynamicLLVM::LLVMGetNextBasicBlock(block)) {
+ for (auto inst = DynamicLLVM::LLVMGetFirstInstruction(block); inst; inst = DynamicLLVM::LLVMGetNextInstruction(inst)) {
+ auto opcode = DynamicLLVM::LLVMGetInstructionOpcode(inst);
+ if (opcode == LLVMGetElementPtr && DynamicLLVM::LLVMGetOperand(inst, 0) == parameter)
+ return DynamicLLVM::LLVMGetGEPSourceElementType(inst);
+ if (opcode == LLVMLoad && DynamicLLVM::LLVMGetOperand(inst, 0) == parameter)
+ return DynamicLLVM::LLVMTypeOf(inst);
+ }
+ }
+ return pointerElementType(parameter);
+}
+static LLVMTypeRef gepElementType(LLVMValueRef inst) {
+ auto type = DynamicLLVM::LLVMGetGEPSourceElementType(inst);
+ for (unsigned i = 2; i < DynamicLLVM::LLVMGetNumOperands(inst); ++i) {
+ switch (DynamicLLVM::LLVMGetTypeKind(type)) {
+ case LLVMStructTypeKind:
+ type = DynamicLLVM::LLVMStructGetTypeAtIndex(type, DynamicLLVM::LLVMConstIntGetZExtValue(DynamicLLVM::LLVMGetOperand(inst, i)));
+ break;
+ case LLVMArrayTypeKind: case LLVMVectorTypeKind:
+ type = DynamicLLVM::LLVMGetElementType(type); break;
+ default: throw std::runtime_error("Unsupported AIR GEP element type");
+ }
+ }
+ return type;
+}
static Iridium::SPIRV::ResultID llvmTypeToSPIRVType(Iridium::SPIRV::Builder& builder, LLVMTypeRef llvmType) {
using namespace Iridium::SPIRV;
@@ -68,6 +194,10 @@ static Iridium::SPIRV::ResultID llvmTypeToSPIRVType(Iridium::SPIRV::Builder& bui
return builder.declareType(Type(Type::FloatTag {}, 128));
case LLVMIntegerTypeKind:
+ if (DynamicLLVM::LLVMGetIntTypeWidth(llvmType) == 1) {
+ // AIR uses i1 for booleans, which SPIR-V has no integer type for
+ return builder.declareType(Type(Type::BooleanTag {}));
+ }
return builder.declareType(Type(Type::IntegerTag {}, DynamicLLVM::LLVMGetIntTypeWidth(llvmType), /* TODO: how to determine this? */ true));
case LLVMFunctionTypeKind: {
@@ -117,28 +247,35 @@ static Iridium::SPIRV::ResultID llvmTypeToSPIRVType(Iridium::SPIRV::Builder& bui
}
// align the final offset to determine the structure size
offset = (offset + (structAlignment - 1)) & ~(structAlignment - 1);
- return builder.declareType(Type(Type::StructureTag {}, members, offset, structAlignment));
+ Type structType(Type::StructureTag {}, members, offset, structAlignment);
+ structType.layoutFree = g_layoutFreeTypes;
+ return builder.declareType(structType);
} break;
case LLVMPointerTypeKind: {
StorageClass storageClass = StorageClass::Output;
// TODO: somehow determine the appropriate storage class
//auto addrSpace = DynamicLLVM::LLVMGetPointerAddressSpace(llvmType);
- return builder.declareType(Type(Type::PointerTag {}, storageClass, llvmTypeToSPIRVType(builder, DynamicLLVM::LLVMGetElementType(llvmType)), 8));
+ throw std::runtime_error("Opaque pointer requires instruction-specific element type: " + (translationInstruction?describeLLVMValue(translationInstruction):translationContext));
} break;
case LLVMVectorTypeKind: {
auto type = llvmTypeToSPIRVType(builder, DynamicLLVM::LLVMGetElementType(llvmType));
auto typeInst = *builder.reverseLookupType(type);
auto elmCount = DynamicLLVM::LLVMGetVectorSize(llvmType);
- return builder.declareType(Type(Type::VectorTag {}, elmCount, type, typeInst.size * elmCount, ((elmCount == 3 || elmCount == 4) ? 4 : 2) * typeInst.alignment));
+ // AIR's data layout allocates three-component vectors like four-component vectors.
+ // This padding belongs to the vector even inside a packed struct or array.
+ const auto allocatedCount = elmCount == 3 ? 4 : elmCount;
+ return builder.declareType(Type(Type::VectorTag {}, elmCount, type, typeInst.size * allocatedCount, ((elmCount == 3 || elmCount == 4) ? 4 : 2) * typeInst.alignment));
} break;
case LLVMArrayTypeKind: {
auto type = llvmTypeToSPIRVType(builder, DynamicLLVM::LLVMGetElementType(llvmType));
auto typeInst = *builder.reverseLookupType(type);
auto elmCount = DynamicLLVM::LLVMGetArrayLength(llvmType);
- return builder.declareType(Type(Type::ArrayTag {}, type, elmCount, typeInst.size * elmCount, typeInst.alignment));
+ Type arrayType(Type::ArrayTag {}, type, elmCount, typeInst.size * elmCount, typeInst.alignment);
+ arrayType.layoutFree = g_layoutFreeTypes;
+ return builder.declareType(arrayType);
} break;
default:
@@ -162,14 +299,60 @@ static Iridium::SPIRV::ResultID llvmValueToResultID(Iridium::SPIRV::Builder& bui
{
ResultID maybe = builder.lookupResultID(reinterpret_cast<uintptr_t>(llvmValue));
if (maybe != ResultIDInvalid) {
+ if (kind == LLVMGlobalVariableValueKind) builder.referenceGlobalVariable(maybe);
return maybe;
}
}
+ // LLVM 23 represents vector splats (e.g. `<3 x float> zeroinitializer`) as vector-typed ConstantInt/ConstantFP values
+ // rather than ConstantAggregateZero/ConstantDataVector. Declare the scalar and splat it into a composite.
+ if ((kind == LLVMConstantIntValueKind || kind == LLVMConstantFPValueKind) && typeKind == LLVMVectorTypeKind) {
+ auto elementType = DynamicLLVM::LLVMGetElementType(type);
+ auto elementKind = DynamicLLVM::LLVMGetTypeKind(elementType);
+ ResultID element = ResultIDInvalid;
+ if (kind == LLVMConstantIntValueKind) {
+ auto val = DynamicLLVM::LLVMConstIntGetZExtValue(llvmValue);
+ auto width = DynamicLLVM::LLVMGetIntTypeWidth(elementType);
+ element = width == 1 ? builder.declareConstantBool(val != 0) : ((width <= 32) ? builder.declareConstantScalar<int32_t>(val) : builder.declareConstantScalar<int64_t>(val));
+ } else {
+ LLVMBool losesInfo = false;
+ auto val = DynamicLLVM::LLVMConstRealGetDouble(llvmValue, &losesInfo);
+ switch (elementKind) {
+ case LLVMFloatTypeKind: element = builder.declareConstantScalar<float>(val); break;
+ case LLVMDoubleTypeKind: element = builder.declareConstantScalar<double>(val); break;
+ case LLVMHalfTypeKind: {
+ Iridium::Float16 tmp = val;
+ element = builder.declareConstantScalar<const Iridium::Float16*>(&tmp);
+ } break;
+ default:
+ throw ImpossibleResultID("Unsupported LLVM value: " + describeLLVMValue(llvmValue));
+ }
+ }
+ std::vector<ResultID> vals(DynamicLLVM::LLVMGetVectorSize(type), element);
+ return builder.declareConstantComposite(llvmTypeToSPIRVType(builder, type), vals);
+ }
+
switch (kind) {
+ case LLVMGlobalVariableValueKind: {
+ auto initializer = DynamicLLVM::LLVMGetInitializer(llvmValue);
+ if (!initializer) throw ImpossibleResultID("External AIR globals are unsupported");
+ LayoutFreeScope layoutFree(true);
+ auto valueType = llvmTypeToSPIRVType(builder, DynamicLLVM::LLVMGlobalGetValueType(llvmValue));
+ auto pointerType = builder.declareType(Type(Type::PointerTag{}, StorageClass::Private, valueType, 8));
+ auto initialValue = llvmValueToResultID(builder, initializer);
+ auto result = builder.addGlobalVariable(pointerType, StorageClass::Private, initialValue);
+ builder.setResultType(result, pointerType);
+ builder.associateExistingResultID(result, reinterpret_cast<uintptr_t>(llvmValue));
+ builder.referenceGlobalVariable(result);
+ return result;
+ }
+
case LLVMConstantIntValueKind: {
auto val = DynamicLLVM::LLVMConstIntGetZExtValue(llvmValue);
auto width = DynamicLLVM::LLVMGetIntTypeWidth(type );
+ if (width == 1) {
+ return builder.declareConstantBool(val != 0);
+ }
// TODO: how to determine if it's signed or not?
return (width <= 32) ? builder.declareConstantScalar<int32_t>(val) : builder.declareConstantScalar<int64_t>(val);
} break;
@@ -191,13 +374,15 @@ static Iridium::SPIRV::ResultID llvmValueToResultID(Iridium::SPIRV::Builder& bui
} break;
default:
- throw ImpossibleResultID();
+ throw ImpossibleResultID("Unsupported LLVM value: " + describeLLVMValue(llvmValue));
}
} break;
- case LLVMConstantVectorValueKind: {
+ case LLVMConstantStructValueKind:
+ case LLVMConstantArrayValueKind:
+ case LLVMConstantVectorValueKind: {
std::vector<ResultID> vals;
- auto count = DynamicLLVM::LLVMGetVectorSize(type);
+ auto count = DynamicLLVM::LLVMGetTypeKind(type)==LLVMStructTypeKind ? DynamicLLVM::LLVMCountStructElementTypes(type) : (DynamicLLVM::LLVMGetTypeKind(type)==LLVMArrayTypeKind ? DynamicLLVM::LLVMGetArrayLength(type) : DynamicLLVM::LLVMGetVectorSize(type));
for (size_t i = 0; i < count; ++i) {
auto constant = DynamicLLVM::LLVMGetOperand(llvmValue, i);
@@ -207,10 +392,11 @@ static Iridium::SPIRV::ResultID llvmValueToResultID(Iridium::SPIRV::Builder& bui
return builder.declareConstantComposite(llvmTypeToSPIRVType(builder, type), vals);
} break;
- case LLVMConstantDataVectorValueKind: {
+ case LLVMConstantDataArrayValueKind:
+ case LLVMConstantDataVectorValueKind: {
std::vector<ResultID> vals;
- auto count = DynamicLLVM::LLVMGetVectorSize(type);
+ auto count = typeKind == LLVMArrayTypeKind ? DynamicLLVM::LLVMGetArrayLength(type) : DynamicLLVM::LLVMGetVectorSize(type);
for (size_t i = 0; i < count; ++i) {
auto constant = DynamicLLVM::LLVMGetElementAsConstant(llvmValue, i);
@@ -220,7 +406,8 @@ static Iridium::SPIRV::ResultID llvmValueToResultID(Iridium::SPIRV::Builder& bui
return builder.declareConstantComposite(llvmTypeToSPIRVType(builder, type), vals);
} break;
- case LLVMUndefValueValueKind: {
+ case LLVMPoisonValueValueKind:
+ case LLVMUndefValueValueKind: {
return builder.declareUndefinedValue(llvmTypeToSPIRVType(builder, type));
} break;
@@ -231,6 +418,21 @@ static Iridium::SPIRV::ResultID llvmValueToResultID(Iridium::SPIRV::Builder& bui
case LLVMConstantExprValueKind: {
switch (DynamicLLVM::LLVMGetConstOpcode(llvmValue)) {
+ case LLVMGetElementPtr: {
+ auto base = llvmValueToResultID(builder, DynamicLLVM::LLVMGetOperand(llvmValue,0));
+ auto baseType = *builder.reverseLookupType(builder.lookupResultType(base));
+ auto firstIndex = DynamicLLVM::LLVMGetOperand(llvmValue,1);
+ if (DynamicLLVM::LLVMGetValueKind(firstIndex) != LLVMConstantIntValueKind || DynamicLLVM::LLVMConstIntGetZExtValue(firstIndex) != 0)
+ throw ImpossibleResultID("Constant AIR GEP requires a zero initial index");
+ std::vector<ResultID> indices;
+ for (unsigned i=2;i<DynamicLLVM::LLVMGetNumOperands(llvmValue);++i) indices.push_back(llvmValueToResultID(builder,DynamicLLVM::LLVMGetOperand(llvmValue,i)));
+ auto element = llvmTypeToSPIRVType(builder,gepElementType(llvmValue));
+ auto pointer = builder.declareType(Type(Type::PointerTag{},baseType.pointerStorageClass,element,8));
+ auto result = indices.empty() ? base : builder.encodeAccessChain(pointer,base,indices);
+ builder.setResultType(result,pointer);
+ return result;
+ }
+
case LLVMBitCast: {
auto target = DynamicLLVM::LLVMGetOperand(llvmValue, 0);
auto targetID = llvmValueToResultID(builder, target);
@@ -254,13 +456,315 @@ static Iridium::SPIRV::ResultID llvmValueToResultID(Iridium::SPIRV::Builder& bui
} break;
default:
- throw ImpossibleResultID();
+ throw ImpossibleResultID("Unsupported LLVM value: " + describeLLVMValue(llvmValue));
}
} break;
default:
- throw ImpossibleResultID();
+ throw ImpossibleResultID("Unsupported LLVM value: " + describeLLVMValue(llvmValue));
+ }
+};
+
+
+static std::vector<std::string_view> splitAirName(std::string_view name) {
+ std::vector<std::string_view> parts;
+ for (size_t start = 0; start <= name.size();) {
+ auto dot = name.find('.', start);
+ if (dot == std::string_view::npos) {
+ dot = name.size();
+ }
+ parts.push_back(name.substr(start, dot - start));
+ start = dot + 1;
+ }
+ return parts;
+};
+
+// "f32", "f16", "v3f32", ...
+static bool airTypeTokenIsFloat(std::string_view token) {
+ if (token.empty()) {
+ return false;
}
+ if (token[0] == 'v') {
+ token.remove_prefix(1);
+ while (!token.empty() && token[0] >= '0' && token[0] <= '9') {
+ token.remove_prefix(1);
+ }
+ }
+ return !token.empty() && token[0] == 'f';
+};
+
+// SPIR-V core opcodes the translation below needs that the Opcode enum does not name
+namespace SPIRVOp {
+ constexpr uint16_t ConvertFToU = 109;
+ constexpr uint16_t ConvertFToS = 110;
+ constexpr uint16_t ConvertSToF = 111;
+ constexpr uint16_t ConvertUToF = 112;
+ constexpr uint16_t UConvert = 113;
+ constexpr uint16_t SConvert = 114;
+ constexpr uint16_t FConvert = 115;
+ constexpr uint16_t Bitcast = 124;
+ constexpr uint16_t FRem = 140;
+ constexpr uint16_t Any = 154;
+ constexpr uint16_t All = 155;
+ constexpr uint16_t Select = 169;
+ constexpr uint16_t IEqual = 170;
+ constexpr uint16_t INotEqual = 171;
+ constexpr uint16_t UGreaterThan = 172;
+ constexpr uint16_t SGreaterThan = 173;
+ constexpr uint16_t UGreaterThanEqual = 174;
+ constexpr uint16_t SGreaterThanEqual = 175;
+ constexpr uint16_t ULessThan = 176;
+ constexpr uint16_t SLessThan = 177;
+ constexpr uint16_t ULessThanEqual = 178;
+ constexpr uint16_t SLessThanEqual = 179;
+ constexpr uint16_t FOrdEqual = 180;
+ constexpr uint16_t FUnordEqual = 181;
+ constexpr uint16_t FOrdNotEqual = 182;
+ constexpr uint16_t FUnordNotEqual = 183;
+ constexpr uint16_t FOrdLessThan = 184;
+ constexpr uint16_t FUnordLessThan = 185;
+ constexpr uint16_t FOrdGreaterThan = 186;
+ constexpr uint16_t FUnordGreaterThan = 187;
+ constexpr uint16_t FOrdLessThanEqual = 188;
+ constexpr uint16_t FUnordLessThanEqual = 189;
+ constexpr uint16_t FOrdGreaterThanEqual = 190;
+ constexpr uint16_t FUnordGreaterThanEqual = 191;
+ constexpr uint16_t Ordered = 162;
+ constexpr uint16_t Unordered = 163;
+ constexpr uint16_t DPdx = 207;
+ constexpr uint16_t DPdy = 208;
+ constexpr uint16_t Fwidth = 209;
+ constexpr uint16_t Kill = 252;
+};
+
+static bool translateAirTexture(Iridium::SPIRV::Builder& builder, LLVMValueRef inst, std::string_view name, Iridium::SPIRV::ResultID resultType, Iridium::FunctionType stage, Iridium::SPIRV::ResultID& result) {
+ using namespace Iridium::SPIRV;
+ bool compare = name.find("air.sample_compare_depth_") == 0;
+ bool sample = name.find("air.sample_texture_") == 0 || compare;
+ bool read = name.find("air.read_texture_") == 0;
+ bool write = name.find("air.write_texture_") == 0;
+ bool width = name == "air.get_width_texture_2d", height = name == "air.get_height_texture_2d";
+ if (!sample && !read && !write && !width && !height) return false;
+ auto operand = [&](unsigned i) { return llvmValueToResultID(builder, DynamicLLVM::LLVMGetOperand(inst, i)); };
+ auto texturePointer = operand(0);
+ auto pointerType = *builder.reverseLookupType(builder.lookupResultType(texturePointer));
+ auto imageTypeID = pointerType.targetType;
+ auto imageType = *builder.reverseLookupType(imageTypeID);
+ auto image = builder.encodeLoad(imageTypeID, texturePointer);
+ unsigned dims = imageType.imageDimensionality == Dim::e1D ? 1 : (imageType.imageDimensionality == Dim::e2D ? 2 : 3);
+ bool array = imageType.imageIsArrayed;
+ auto llvmResult = DynamicLLVM::LLVMTypeOf(inst);
+ bool wrapped = DynamicLLVM::LLVMGetTypeKind(llvmResult) == LLVMStructTypeKind;
+ auto valueTypeID = wrapped ? llvmTypeToSPIRVType(builder, DynamicLLVM::LLVMStructGetTypeAtIndex(llvmResult,0)) : resultType;
+ auto finish = [&](ResultID value) {
+ if (!wrapped) { result=value; return; }
+ auto partial=builder.encodeCompositeInsert(resultType,value,builder.declareUndefinedValue(resultType),{0});
+ result=builder.encodeCompositeInsert(resultType,builder.declareConstantScalar<int8_t>(0),partial,{1});
+ };
+ if (width || height) {
+ builder.requireCapability(Capability::ImageQuery);
+ auto component=builder.declareType(Type(Type::IntegerTag{},32,true));
+ auto sizeType=builder.declareType(Type(Type::VectorTag{},dims+(array?1:0),component,4*(dims+(array?1:0)),4));
+ auto sizes=builder.encodeRawOp(103,sizeType,{image,operand(1)}); // OpImageQuerySizeLod
+ result=builder.encodeCompositeExtract(resultType,sizes,{height?1u:0u});return true;
+ }
+ auto coordinates = operand(compare?3:(sample?2:1));
+ auto coordinateType = *builder.reverseLookupType(builder.lookupResultType(coordinates));
+ if (array) {
+ std::vector<ResultID> components;
+ for(unsigned i=0;i<dims;++i) components.push_back(builder.encodeCompositeExtract(coordinateType.targetType,coordinates,{i}));
+ auto layer=operand(sample?3:2);
+ if(sample) layer=builder.encodeRawOp(111,coordinateType.targetType,{layer}); // OpConvertSToF
+ components.push_back(layer);
+ auto composite=builder.declareType(Type(Type::VectorTag{},dims+1,coordinateType.targetType,(dims+1)*4,4));
+ coordinates=builder.encodeRawOp(80,composite,components);
+ }
+ auto vectorType=builder.declareType(Type(Type::VectorTag{},4,imageType.targetType,16,16));
+ ResultID value;
+ if(sample) {
+ auto samplerPointer=operand(1);
+ auto samplerType=*builder.reverseLookupType(builder.lookupResultType(samplerPointer));
+ auto sampler=builder.encodeLoad(samplerType.targetType,samplerPointer);
+ auto sampledType=builder.declareType(Type(Type::SampledImageTag{},imageTypeID));
+ auto sampled=builder.encodeSampledImage(sampledType,image,sampler);
+ unsigned extra=array?1:0;
+ bool cube=imageType.imageDimensionality==Dim::eCube;
+ unsigned flagIndex=compare?(cube?5:7):(cube?3+extra:5+extra);
+ auto lodFlag=DynamicLLVM::LLVMGetOperand(inst,flagIndex);
+ if(DynamicLLVM::LLVMGetValueKind(lodFlag)!=LLVMConstantIntValueKind)
+ throw std::runtime_error("Dynamic AIR sample LOD mode: " + describeLLVMValue(inst));
+ bool explicitLod=DynamicLLVM::LLVMConstIntGetZExtValue(lodFlag)!=0 || stage!=Iridium::FunctionType::Fragment;
+ std::vector<ResultID> args{sampled,coordinates};
+ if(compare)args.push_back(operand(4));
+ auto offset=cube?nullptr:DynamicLLVM::LLVMGetOperand(inst,flagIndex-1);
+ bool hasOffset=offset && !DynamicLLVM::LLVMIsNull(offset);
+ unsigned mask=(explicitLod?2:0)|(hasOffset?8:0);
+ if(mask){args.push_back(mask);if(explicitLod)args.push_back(operand(flagIndex+1));if(hasOffset)args.push_back(llvmValueToResultID(builder,offset));}
+ value=builder.encodeRawOp(compare?(explicitLod?90:89):(explicitLod?88:87),compare?valueTypeID:vectorType,args);
+ if(compare){finish(value);return true;}
+ } else if(read) {
+ std::vector<ResultID> args{image,coordinates};
+ if(imageType.imageIsSampledIndication==1){args.push_back(2);args.push_back(operand(array?3:2));}
+ value=builder.encodeRawOp(imageType.imageIsSampledIndication==1?95:98,vectorType,args);
+ } else {
+ builder.encodeRawOpNoResult(99,{image,coordinates,operand(array?3:2)});result=ResultIDInvalid;return true;
+ }
+ if(vectorType!=valueTypeID) {
+ auto target=*builder.reverseLookupType(valueTypeID);
+ auto component=*builder.reverseLookupType(target.targetType);
+ value=component.backingType==Type::BackingType::FloatingPoint && component.scalarWidth==16 ? builder.encodeFConvert(valueTypeID,value) : builder.encodeBitcast(valueTypeID,value);
+ }
+ finish(value);return true;
+}
+
+// LLVM integers are signless; SPIR-V conversion instructions require explicit signedness.
+static Iridium::SPIRV::ResultID integerTypeWithSign(Iridium::SPIRV::Builder& builder,Iridium::SPIRV::ResultID type,bool sign){
+ using namespace Iridium::SPIRV;auto value=*builder.reverseLookupType(type);
+ if(value.backingType==Type::BackingType::Vector){auto component=integerTypeWithSign(builder,value.targetType,sign);return builder.declareType(Type(Type::VectorTag{},value.entryCount,component,value.size,value.alignment));}
+ return builder.declareType(Type(Type::IntegerTag{},value.scalarWidth,sign));
+}
+// Translates the AIR math, conversion, derivative and discard intrinsics. Returns false when `name`
+// is not one of them. `terminatesBlock` is set for instructions that end the SPIR-V block (discard).
+static bool translateAirIntrinsic(Iridium::SPIRV::Builder& builder, LLVMValueRef inst, std::string_view name, Iridium::SPIRV::ResultID type, Iridium::SPIRV::ResultID& resID, bool& terminatesBlock) {
+ using namespace Iridium::SPIRV;
+
+ auto operand = [&](unsigned index) {
+ return llvmValueToResultID(builder, DynamicLLVM::LLVMGetOperand(inst, index));
+ };
+
+ if (name == "air.discard_fragment") {
+ builder.encodeRawOpNoResult(SPIRVOp::Kill, {});
+ terminatesBlock = true;
+ resID = ResultIDInvalid;
+ return true;
+ }
+
+ auto parts = splitAirName(name);
+ if (parts.size() < 3 || parts[0] != "air") {
+ return false;
+ }
+
+ std::string_view fn = parts[1];
+ if (fn.substr(0, 5) == "fast_") {
+ fn.remove_prefix(5);
+ } else if (fn.substr(0, 8) == "precise_") {
+ fn.remove_prefix(8);
+ }
+
+ // floating-point functions: air.<fn>.<type>
+ if (parts.size() == 3 && airTypeTokenIsFloat(parts[2])) {
+ struct MathOp {
+ std::string_view name;
+ GLSLOpcode opcode;
+ unsigned arity;
+ };
+ static const MathOp mathOps[] = {
+ { "sqrt", GLSLOpcode::Sqrt, 1 }, { "rsqrt", GLSLOpcode::InverseSqrt, 1 }, { "pow", GLSLOpcode::Pow, 2 },
+ { "powr", GLSLOpcode::Pow, 2 }, { "exp", GLSLOpcode::Exp, 1 }, { "exp2", GLSLOpcode::Exp2, 1 },
+ { "log", GLSLOpcode::Log, 1 }, { "log2", GLSLOpcode::Log2, 1 }, { "sin", GLSLOpcode::Sin, 1 },
+ { "cos", GLSLOpcode::Cos, 1 }, { "tan", GLSLOpcode::Tan, 1 }, { "asin", GLSLOpcode::Asin, 1 },
+ { "acos", GLSLOpcode::Acos, 1 }, { "atan", GLSLOpcode::Atan, 1 }, { "atan2", GLSLOpcode::Atan2, 2 },
+ { "sinh", GLSLOpcode::Sinh, 1 }, { "cosh", GLSLOpcode::Cosh, 1 }, { "tanh", GLSLOpcode::Tanh, 1 },
+ { "fabs", GLSLOpcode::FAbs, 1 }, { "floor", GLSLOpcode::Floor, 1 }, { "ceil", GLSLOpcode::Ceil, 1 },
+ { "fract", GLSLOpcode::Fract, 1 }, { "rint", GLSLOpcode::RoundEven, 1 }, { "round", GLSLOpcode::Round, 1 },
+ { "trunc", GLSLOpcode::Trunc, 1 }, { "sign", GLSLOpcode::FSign, 1 }, { "fmin", GLSLOpcode::FMin, 2 },
+ { "fmax", GLSLOpcode::FMax, 2 }, { "fma", GLSLOpcode::Fma, 3 }, { "mix", GLSLOpcode::FMix, 3 },
+ { "clamp", GLSLOpcode::FClamp, 3 }, { "step", GLSLOpcode::Step, 2 }, { "smoothstep", GLSLOpcode::SmoothStep, 3 },
+ { "normalize", GLSLOpcode::Normalize, 1 }, { "length", GLSLOpcode::Length, 1 }, { "distance", GLSLOpcode::Distance, 2 },
+ { "cross", GLSLOpcode::Cross, 2 }, { "reflect", GLSLOpcode::Reflect, 2 },
+ };
+
+ for (auto& op: mathOps) {
+ if (op.name == fn) {
+ std::vector<ResultID> arguments;
+ for (unsigned i = 0; i < op.arity; ++i) {
+ arguments.push_back(operand(i));
+ }
+ resID = builder.encodeGLSLOp(op.opcode, type, arguments);
+ return true;
+ }
+ }
+
+ if (fn == "fmod") {
+ resID = builder.encodeRawOp(SPIRVOp::FRem, type, { operand(0), operand(1) });
+ return true;
+ }
+ if (fn == "dot") {
+ resID = builder.encodeArithBinop(Opcode::Dot, type, operand(0), operand(1));
+ return true;
+ }
+ if (fn == "dfdx" || fn == "dfdy" || fn == "fwidth") {
+ resID = builder.encodeRawOp(fn == "dfdx" ? SPIRVOp::DPdx : (fn == "dfdy" ? SPIRVOp::DPdy : SPIRVOp::Fwidth), type, { operand(0) });
+ return true;
+ }
+ return false;
+ }
+
+ // boolean vector reductions: air.all.v2i1, air.any.v4i1
+ if (parts.size() == 3 && (fn == "all" || fn == "any")) {
+ resID = builder.encodeRawOp(fn == "all" ? SPIRVOp::All : SPIRVOp::Any, type, { operand(0) });
+ return true;
+ }
+
+ // integer functions: air.<fn>.<s|u>.<type>
+ if (parts.size() == 4 && (parts[2] == "s" || parts[2] == "u")) {
+ bool isSigned = parts[2] == "s";
+
+ if (fn == "min" || fn == "max") {
+ auto opcode = fn == "min" ? (isSigned ? GLSLOpcode::SMin : GLSLOpcode::UMin) : (isSigned ? GLSLOpcode::SMax : GLSLOpcode::UMax);
+ resID = builder.encodeGLSLOp(opcode, type, { operand(0), operand(1) });
+ return true;
+ }
+ if (fn == "clamp") {
+ resID = builder.encodeGLSLOp(isSigned ? GLSLOpcode::SClamp : GLSLOpcode::UClamp, type, { operand(0), operand(1), operand(2) });
+ return true;
+ }
+ if (fn == "abs") {
+ resID = builder.encodeGLSLOp(GLSLOpcode::SAbs, type, { operand(0) });
+ return true;
+ }
+ return false;
+ }
+
+ // conversions: air.convert.<dst kind>.<dst type>.<src kind>.<src type>
+ if (parts.size() == 6 && fn == "convert") {
+ auto dstKind = parts[2];
+ auto dstType = parts[3];
+ auto srcKind = parts[4];
+ auto srcType = parts[5];
+ auto argument = operand(0);
+
+ if (srcType == "i1") {
+ // bool to number: 1 or 0
+ auto one = dstKind == "f" ? builder.declareConstantScalar<float>(1) : builder.declareConstantScalar<int32_t>(1);
+ auto zero = dstKind == "f" ? builder.declareConstantScalar<float>(0) : builder.declareConstantScalar<int32_t>(0);
+ resID = builder.encodeRawOp(SPIRVOp::Select, type, { argument, one, zero });
+ return true;
+ }
+
+ if (dstKind == "f" && srcKind == "s") {
+ resID = builder.encodeRawOp(SPIRVOp::ConvertSToF, type, { argument });
+ } else if (dstKind == "f" && srcKind == "u") {
+ auto argType=llvmTypeToSPIRVType(builder,DynamicLLVM::LLVMTypeOf(DynamicLLVM::LLVMGetOperand(inst,0)));
+ auto unsignedType=integerTypeWithSign(builder,argType,false);
+ resID=builder.encodeRawOp(SPIRVOp::ConvertUToF,type,{builder.encodeBitcast(unsignedType,argument)});
+ } else if (dstKind == "s" && srcKind == "f") {
+ resID = builder.encodeRawOp(SPIRVOp::ConvertFToS, type, { argument });
+ } else if (dstKind == "u" && srcKind == "f") {
+ auto unsignedType=integerTypeWithSign(builder,type,false);
+ resID=builder.encodeBitcast(type,builder.encodeRawOp(SPIRVOp::ConvertFToU,unsignedType,{argument}));
+ } else if (dstKind == "f" && srcKind == "f") {
+ // same type: nothing to convert
+ resID = (dstType == srcType) ? argument : builder.encodeRawOp(SPIRVOp::FConvert, type, { argument });
+ } else if ((dstKind == "s" || dstKind == "u") && (srcKind == "s" || srcKind == "u")) {
+ resID = (dstType == srcType) ? argument : builder.encodeRawOp(dstKind == "s" ? SPIRVOp::SConvert : SPIRVOp::UConvert, type, { argument });
+ } else {
+ return false;
+ }
+ return true;
+ }
+
+ return false;
};
void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& outputInfo) {
@@ -312,7 +816,7 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
});
// declare the GLSL PerVertex structure
- auto perVertexStructType = builder.declareType(SPIRV::Type(SPIRV::Type::StructureTag {}, {
+ SPIRV::Type perVertexType(SPIRV::Type::StructureTag {}, {
SPIRV::Type::Member {
vec4Type,
0,
@@ -321,13 +825,21 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
}
},
// TODO: declare the rest of the structure
- }, floatTypeInst.size * 4, floatTypeInst.alignment));
+ }, floatTypeInst.size * 4, floatTypeInst.alignment);
+ perVertexType.layoutFree = true;
+ auto perVertexStructType = builder.declareType(perVertexType);
auto perVertexStructPtrType = builder.declareType(SPIRV::Type(SPIRV::Type::PointerTag {}, SPIRV::StorageClass::Output, perVertexStructType, 8));
perVertexVar = builder.addGlobalVariable(perVertexStructPtrType, SPIRV::StorageClass::Output);
builder.addDecoration(perVertexStructType, SPIRV::Decoration { SPIRV::DecorationType::Block });
builder.referenceGlobalVariable(perVertexVar);
+ // Indium also creates a point-topology pipeline; Vulkan requires PointSize to be written.
+ auto pointSizePtr = builder.declareType(SPIRV::Type(SPIRV::Type::PointerTag {}, SPIRV::StorageClass::Output, floatType, 8));
+ auto pointSizeVar = builder.addGlobalVariable(pointSizePtr, SPIRV::StorageClass::Output);
+ builder.addDecoration(pointSizeVar, SPIRV::Decoration { SPIRV::DecorationType::Builtin, { static_cast<uint32_t>(SPIRV::BuiltinID::PointSize) } });
+ builder.referenceGlobalVariable(pointSizeVar);
+ builder.encodeStore(pointSizeVar, builder.declareConstantScalar<float>(1.0f));
// declare the GLSL VertexIndex input variable
vertexIndexVar = builder.addGlobalVariable(ptrIntType, SPIRV::StorageClass::Input);
@@ -425,7 +937,7 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
// if the function returns a structure, we need to separate the components
// into separate output variables.
- auto llfuncType = DynamicLLVM::LLVMGetElementType(DynamicLLVM::LLVMTypeOf(_function));
+ auto llfuncType = DynamicLLVM::LLVMGlobalGetValueType(_function);
auto funcRetType = DynamicLLVM::LLVMGetReturnType(llfuncType);
std::vector<LLVMTypeRef> funcParamTypes(DynamicLLVM::LLVMCountParamTypes(llfuncType));
DynamicLLVM::LLVMGetParamTypes(llfuncType, funcParamTypes.data());
@@ -460,7 +972,7 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
auto type = llvmTypeToSPIRVType(builder, DynamicLLVM::LLVMStructGetTypeAtIndex(funcRetType, i));
auto ptrType = builder.declareType(SPIRV::Type(SPIRV::Type::PointerTag {}, SPIRV::StorageClass::Output, type, 8));
auto var = builder.addGlobalVariable(ptrType, SPIRV::StorageClass::Output);
- builder.addDecoration(var, SPIRV::Decoration { SPIRV::DecorationType::Location, { location } });
+ builder.addDecoration(var, SPIRV::Decoration { SPIRV::DecorationType::Location, { airInterfaceLocation(returnValueMemberInfo, location) } });
++location;
_outputValueIDs.push_back(var);
builder.referenceGlobalVariable(var);
@@ -472,13 +984,60 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
auto type = llvmTypeToSPIRVType(builder, funcRetType);
auto ptrType = builder.declareType(SPIRV::Type(SPIRV::Type::PointerTag {}, SPIRV::StorageClass::Output, type, 8));
auto var = builder.addGlobalVariable(ptrType, SPIRV::StorageClass::Output);
- builder.addDecoration(var, SPIRV::Decoration { SPIRV::DecorationType::Location, { 0 } });
+ uint32_t outputLocation = 0;
+ if (!returnValueOperands.empty()) {
+ std::vector<LLVMValueRef> info(DynamicLLVM::LLVMGetMDNodeNumOperands(returnValueOperands[0]));
+ DynamicLLVM::LLVMGetMDNodeOperands(returnValueOperands[0], info.data());
+ outputLocation = airInterfaceLocation(info, 0);
+ }
+ builder.addDecoration(var, SPIRV::Decoration { SPIRV::DecorationType::Location, { outputLocation } });
_outputValueIDs.push_back(var);
builder.referenceGlobalVariable(var);
}
std::vector<LLVMValueRef> parameterOperands(DynamicLLVM::LLVMGetMDNodeNumOperands(rootInfoOperands[2]));
DynamicLLVM::LLVMGetMDNodeOperands(rootInfoOperands[2], parameterOperands.data());
+ // Preserve AIR reflection metadata before translating resource bindings.
+ for (auto parameter : parameterOperands) {
+ std::vector<LLVMValueRef> info(DynamicLLVM::LLVMGetMDNodeNumOperands(parameter));
+ DynamicLLVM::LLVMGetMDNodeOperands(parameter, info.data());
+ if (info.size() < 2) continue;
+ auto kind = llvmMDStringToStringView(info[1]);
+ ArgumentInfo arg;
+ if (kind == "air.buffer") arg.type = BindingType::Buffer;
+ else if (kind == "air.texture") arg.type = BindingType::Texture;
+ else if (kind == "air.sampler") arg.type = BindingType::Sampler;
+ else continue;
+ for (size_t i = 2; i + 1 < info.size(); ++i) {
+ if (!DynamicLLVM::LLVMIsAMDString(info[i])) continue;
+ auto key = llvmMDStringToStringView(info[i]);
+ if (key == "air.location_index") arg.index = DynamicLLVM::LLVMConstIntGetZExtValue(info[i+1]);
+ else if (key == "air.arg_name") arg.name = llvmMDStringToStringView(info[i+1]);
+ else if (key == "air.arg_type_name") arg.typeName = llvmMDStringToStringView(info[i+1]);
+ else if (key == "air.arg_type_size") arg.size = DynamicLLVM::LLVMConstIntGetZExtValue(info[i+1]);
+ else if (key == "air.arg_type_align_size") arg.alignment = DynamicLLVM::LLVMConstIntGetZExtValue(info[i+1]);
+ else if (key == "air.struct_type_info") {
+ std::vector<LLVMValueRef> members(DynamicLLVM::LLVMGetMDNodeNumOperands(info[i+1]));
+ DynamicLLVM::LLVMGetMDNodeOperands(info[i+1], members.data());
+ // AIR stores each direct member as offset, size, array length, type, name. A member of
+ // struct type is preceded by an "air.struct_type_info" key and the node describing it.
+ // Reflection is informational, so anything unrecognized stops the walk instead of failing.
+ for (size_t j = 0; j < members.size();) {
+ if (DynamicLLVM::LLVMIsAMDString(members[j])) {
+ j += 2;
+ continue;
+ }
+ if (j + 4 >= members.size() || !DynamicLLVM::LLVMIsAMDString(members[j+3]) || !DynamicLLVM::LLVMIsAMDString(members[j+4]))
+ break;
+ arg.members.push_back({static_cast<size_t>(DynamicLLVM::LLVMConstIntGetZExtValue(members[j])),
+ std::string(llvmMDStringToStringView(members[j+3])), std::string(llvmMDStringToStringView(members[j+4]))});
+ j += 5;
+ }
+ }
+ }
+ funcInfo.arguments.push_back(std::move(arg));
+ }
+
//
// analyze parameters and mark special values (like the vertex ID)
@@ -519,7 +1078,9 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
auto kind = llvmMDStringToStringView(parameterInfo[1]);
if (kind == "air.buffer") {
- auto type = llvmTypeToSPIRVType(builder, DynamicLLVM::LLVMGetElementType(funcParamTypes[i]));
+ translationInstruction=nullptr;
+ translationContext = "buffer parameter " + std::to_string(i) + " of " + _name;
+ auto type = llvmTypeToSPIRVType(builder, bufferElementType(_function, i));
auto addrPtrTypeInst = SPIRV::Type(SPIRV::Type::PointerTag {}, SPIRV::StorageClass::PhysicalStorageBuffer, type, 8);
auto addrPtrType = builder.declareType(addrPtrTypeInst);
bufferMembers.push_back(SPIRV::Type::Member { addrPtrType, 8 * bufferIndex, {} });
@@ -575,7 +1136,21 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
_parameterIDs.push_back(load);
builder.associateExistingResultID(load, llparamVal);
builder.setResultType(load, intType);
- } else if (kind == "air.buffer") {
+ } else if (kind == "air.instance_id") {
+ auto var=builder.addGlobalVariable(ptrIntType,SPIRV::StorageClass::Input);
+ builder.addDecoration(var,SPIRV::Decoration{SPIRV::DecorationType::Builtin,{43}});builder.referenceGlobalVariable(var);
+ auto load=builder.encodeLoad(intType,var);_parameterIDs.push_back(load);builder.associateExistingResultID(load,llparamVal);builder.setResultType(load,intType);
+ } else if (kind == "air.front_facing") {
+ auto boolType = builder.declareType(SPIRV::Type(SPIRV::Type::BooleanTag{}));
+ auto boolPtrType = builder.declareType(SPIRV::Type(SPIRV::Type::PointerTag{}, SPIRV::StorageClass::Input, boolType, 8));
+ auto var = builder.addGlobalVariable(boolPtrType, SPIRV::StorageClass::Input);
+ builder.addDecoration(var, SPIRV::Decoration{SPIRV::DecorationType::Builtin, {17}}); // FrontFacing
+ builder.referenceGlobalVariable(var);
+ auto load = builder.encodeLoad(boolType, var);
+ _parameterIDs.push_back(load);
+ builder.associateExistingResultID(load, llparamVal);
+ builder.setResultType(load, boolType);
+ } else if (kind == "air.buffer") {
// find the location index info
size_t infoIdx = 0;
for (; infoIdx < parameterInfo.size(); ++infoIdx) {
@@ -620,7 +1195,7 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
auto var = builder.addGlobalVariable(ptrType, SPIRV::StorageClass::Input);
auto load = builder.encodeLoad(type, var);
- builder.addDecoration(var, SPIRV::Decoration { SPIRV::DecorationType::Location, { paramLocation } });
+ builder.addDecoration(var, SPIRV::Decoration { SPIRV::DecorationType::Location, { airInterfaceLocation(parameterInfo, paramLocation) } });
++paramLocation;
builder.referenceGlobalVariable(var);
@@ -631,9 +1206,18 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
builder.setResultType(load, type);
} else if (kind == "air.vertex_input") {
auto type = llvmTypeToSPIRVType(builder, funcParamTypes[i]);
- auto ptrType = builder.declareType(SPIRV::Type(SPIRV::Type::PointerTag {}, SPIRV::StorageClass::Input, type, 8));
+ auto inputType = type;
+ // LLVM integers have no sign; AIR metadata preserves the interface's unsigned type.
+ for (size_t j = 0; j + 1 < parameterInfo.size(); ++j) {
+ if (llvmMDStringToStringView(parameterInfo[j]) != "air.arg_type_name") continue;
+ auto name = llvmMDStringToStringView(parameterInfo[j + 1]);
+ if (name.substr(0, 4) == "uint" || name.substr(0, 6) == "ushort" || name.substr(0, 5) == "uchar" || name.substr(0, 5) == "ulong")
+ inputType = integerTypeWithSign(builder, type, false);
+ }
+ auto ptrType = builder.declareType(SPIRV::Type(SPIRV::Type::PointerTag {}, SPIRV::StorageClass::Input, inputType, 8));
auto var = builder.addGlobalVariable(ptrType, SPIRV::StorageClass::Input);
- auto load = builder.encodeLoad(type, var);
+ auto load = builder.encodeLoad(inputType, var);
+ if (inputType != type) load = builder.encodeBitcast(type, load);
// find the location index info
size_t infoIdx = 0;
@@ -761,14 +1345,19 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
realSampleType = fakeSampleType;
}
- // TODO: same here (find a better way...)
- if (textureClassName == "texture2d") {
- dimensionality = SPIRV::Dim::e2D;
- } else if (textureClassName == "texturecube") {
- dimensionality = SPIRV::Dim::eCube;
- }
-
- auto imageType = builder.declareType(SPIRV::Type(SPIRV::Type::ImageTag {}, fakeSampleType, realSampleType, dimensionality, 2, false, false, accessType == TextureAccessType::Sample ? 1 : 2, SPIRV::ImageFormat::Unknown));
+ if (sampleTypeName == "uint" || sampleTypeName == "int") {
+ fakeSampleType = builder.declareType(SPIRV::Type(SPIRV::Type::IntegerTag {},32,sampleTypeName=="int"));
+ realSampleType = fakeSampleType;
+ }
+ if (fakeSampleType == SPIRV::ResultIDInvalid) throw std::runtime_error("Unsupported AIR texture sample type");
+ bool arrayed = textureClassName.find("_array") != std::string_view::npos;
+ if ((textureClassName.find("texture2d") == 0 || textureClassName.find("depth2d") == 0)) dimensionality = SPIRV::Dim::e2D;
+ else if ((textureClassName.find("texturecube") == 0 || textureClassName.find("depthcube") == 0)) dimensionality = SPIRV::Dim::eCube;
+ else if (textureClassName == "texture3d") dimensionality = SPIRV::Dim::e3D;
+ else if (textureClassName == "texture1d") dimensionality = SPIRV::Dim::e1D;
+ else throw std::runtime_error("Unsupported AIR texture dimension: " + std::string(textureClassName));
+
+ auto imageType = builder.declareType(SPIRV::Type(SPIRV::Type::ImageTag {}, fakeSampleType, realSampleType, dimensionality, textureClassName.find("depth")==0?1:0, arrayed, false, accessType == TextureAccessType::Sample ? 1 : 2, SPIRV::ImageFormat::Unknown));
auto imagePtrType = builder.declareType(SPIRV::Type(SPIRV::Type::PointerTag {}, SPIRV::StorageClass::UniformConstant, imageType, 8));
auto var = builder.addGlobalVariable(imagePtrType, SPIRV::StorageClass::UniformConstant);
//auto load = builder.encodeLoad(imageType, var);
@@ -959,68 +1548,98 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
enum class CFGType {
None = 0,
Selection,
+ Loop,
};
struct CFGInfo {
CFGType type = CFGType::None;
SPIRV::ResultID selectionMergeBlock = SPIRV::ResultIDInvalid;
+ SPIRV::ResultID continueBlock = SPIRV::ResultIDInvalid;
};
std::unordered_map<SPIRV::ResultID, CFGInfo> cfgInfos;
- for (LLVMBasicBlockRef bb = DynamicLLVM::LLVMGetFirstBasicBlock(_function); bb != nullptr; bb = DynamicLLVM::LLVMGetNextBasicBlock(bb)) {
- auto lastInst = DynamicLLVM::LLVMGetLastInstruction(bb);
- auto lastInstOpcode = DynamicLLVM::LLVMGetInstructionOpcode(lastInst);
-
- CFGInfo cfgInfo {};
-
- if (lastInstOpcode == LLVMBr && DynamicLLVM::LLVMIsConditional(lastInst)) {
- // check if this is a basic conditional/selection (i.e. an `if`)
- auto firstBlock = DynamicLLVM::LLVMGetSuccessor(lastInst, 0);
- auto secondBlock = DynamicLLVM::LLVMGetSuccessor(lastInst, 1);
-
- auto lastInstFirstBlock = DynamicLLVM::LLVMGetLastInstruction(firstBlock);
- auto lastInstSecondBlock = DynamicLLVM::LLVMGetLastInstruction(secondBlock);
- auto lastInstOpcodeFirstBlock = DynamicLLVM::LLVMGetInstructionOpcode(lastInstFirstBlock);
- auto lastInstOpcodeSecondBlock = DynamicLLVM::LLVMGetInstructionOpcode(lastInstSecondBlock);
-
- SPIRV::ResultID mergeBlock = SPIRV::ResultIDInvalid;
-
- // check if the first block is the special block
- if (lastInstOpcodeFirstBlock == LLVMBr && !DynamicLLVM::LLVMIsConditional(lastInstFirstBlock)) {
- // make sure the target block is the same
- auto block = DynamicLLVM::LLVMGetSuccessor(lastInstFirstBlock, 0);
-
- if (block == secondBlock) {
- // perfect, the second block is the merge block then
- mergeBlock = builder.lookupResultID(reinterpret_cast<uintptr_t>(secondBlock));
- }
- } else if (lastInstOpcodeSecondBlock == LLVMBr && !DynamicLLVM::LLVMIsConditional(lastInstSecondBlock)) {
- // make sure the target block is the same
- auto block = DynamicLLVM::LLVMGetSuccessor(lastInstSecondBlock, 0);
-
- if (block == firstBlock) {
- // perfect, the first block is the merge block then
- mergeBlock = builder.lookupResultID(reinterpret_cast<uintptr_t>(firstBlock));
- }
- }
-
- if (mergeBlock != SPIRV::ResultIDInvalid) {
- cfgInfo.type = CFGType::Selection;
- cfgInfo.selectionMergeBlock = mergeBlock;
- }
- }
-
- if (cfgInfo.type != CFGType::None) {
- cfgInfos[builder.lookupResultID(reinterpret_cast<uintptr_t>(bb))] = cfgInfo;
- }
- }
-
+ std::vector<LLVMBasicBlockRef> blocks;
+ for(auto bb=DynamicLLVM::LLVMGetFirstBasicBlock(_function);bb;bb=DynamicLLVM::LLVMGetNextBasicBlock(bb))blocks.push_back(bb);
+ std::set<LLVMBasicBlockRef> all(blocks.begin(),blocks.end());
+ std::unordered_map<LLVMBasicBlockRef,std::set<LLVMBasicBlockRef>> postdom;
+ for(auto bb:blocks){auto term=DynamicLLVM::LLVMGetLastInstruction(bb);postdom[bb]=DynamicLLVM::LLVMGetNumSuccessors(term)?all:std::set<LLVMBasicBlockRef>{bb};}
+ bool changed=true;
+ while(changed){changed=false;for(auto bb:blocks){
+ auto term=DynamicLLVM::LLVMGetLastInstruction(bb);auto count=DynamicLLVM::LLVMGetNumSuccessors(term);
+ if(!count)continue;
+ auto next=postdom[DynamicLLVM::LLVMGetSuccessor(term,0)];
+ for(unsigned i=1;i<count;++i){std::set<LLVMBasicBlockRef> common;auto& other=postdom[DynamicLLVM::LLVMGetSuccessor(term,i)];std::set_intersection(next.begin(),next.end(),other.begin(),other.end(),std::inserter(common,common.begin()));next=std::move(common);}
+ next.insert(bb);if(next!=postdom[bb]){postdom[bb]=std::move(next);changed=true;}
+ }}
+ for(auto bb:blocks){auto term=DynamicLLVM::LLVMGetLastInstruction(bb);
+ if(!IRIDIUM_IS_BR(DynamicLLVM::LLVMGetInstructionOpcode(term)) || !DynamicLLVM::LLVMIsConditional(term))continue;
+ LLVMBasicBlockRef merge=nullptr;
+ for(auto candidate:postdom[bb])if(candidate!=bb && (!merge || postdom[candidate].size()>postdom[merge].size()))merge=candidate;
+ if(merge)cfgInfos[builder.lookupResultID(reinterpret_cast<uintptr_t>(bb))]={CFGType::Selection,builder.lookupResultID(reinterpret_cast<uintptr_t>(merge))};
+ }
+
+ // Identify natural loop back edges from dominators; their headers need OpLoopMerge.
+ std::unordered_map<LLVMBasicBlockRef,SPIRV::ResultID> splitContinues;
+ std::unordered_map<LLVMBasicBlockRef,std::pair<SPIRV::ResultID,SPIRV::ResultID>> splitLoopSelections;
+ std::unordered_map<LLVMBasicBlockRef,std::set<LLVMBasicBlockRef>> predecessors,dom;
+ for(auto bb:blocks){auto term=DynamicLLVM::LLVMGetLastInstruction(bb);for(unsigned i=0;i<DynamicLLVM::LLVMGetNumSuccessors(term);++i)predecessors[DynamicLLVM::LLVMGetSuccessor(term,i)].insert(bb);dom[bb]=bb==blocks.front()?std::set<LLVMBasicBlockRef>{bb}:all;}
+ changed=true;while(changed){changed=false;for(auto bb:blocks){if(bb==blocks.front())continue;auto next=all;for(auto pred:predecessors[bb]){std::set<LLVMBasicBlockRef> common;std::set_intersection(next.begin(),next.end(),dom[pred].begin(),dom[pred].end(),std::inserter(common,common.begin()));next=std::move(common);}next.insert(bb);if(next!=dom[bb]){dom[bb]=std::move(next);changed=true;}}}
+ for(auto latch:blocks){auto term=DynamicLLVM::LLVMGetLastInstruction(latch);for(unsigned i=0;i<DynamicLLVM::LLVMGetNumSuccessors(term);++i){
+ auto header=DynamicLLVM::LLVMGetSuccessor(term,i);if(!dom[latch].count(header))continue;
+ std::set<LLVMBasicBlockRef> loop{header,latch};std::vector<LLVMBasicBlockRef> pending;if(latch!=header)pending.push_back(latch);
+ while(!pending.empty()){auto block=pending.back();pending.pop_back();for(auto pred:predecessors[block])if(loop.insert(pred).second && pred!=header)pending.push_back(pred);}
+ LLVMBasicBlockRef merge=nullptr;for(auto candidate:postdom[header])if(!loop.count(candidate) && (!merge || postdom[candidate].size()>postdom[merge].size()))merge=candidate;
+ if(!merge)throw std::runtime_error("AIR loop has no supported merge block");
+ auto headerID=builder.lookupResultID(reinterpret_cast<uintptr_t>(header));auto latchID=builder.lookupResultID(reinterpret_cast<uintptr_t>(latch));
+ auto existing=cfgInfos.find(headerID);if(existing!=cfgInfos.end() && existing->second.type==CFGType::Loop && existing->second.continueBlock!=latchID)throw std::runtime_error("AIR loop has multiple continue blocks");
+ auto continueID=builder.reserveResultID();splitContinues[latch]=continueID;
+ cfgInfos[headerID]={CFGType::Loop,builder.lookupResultID(reinterpret_cast<uintptr_t>(merge)),continueID};
+ // Within a loop, a path that breaks leaves the enclosing construct. Ignore that
+ // path when selecting the merge for an inner selection, and stop at the continue.
+ std::unordered_map<LLVMBasicBlockRef,std::set<LLVMBasicBlockRef>> local;
+ for(auto block:loop)local[block]=block==latch?std::set<LLVMBasicBlockRef>{block}:loop;
+ bool localChanged=true;while(localChanged){localChanged=false;for(auto block:loop){if(block==latch)continue;
+ auto term=DynamicLLVM::LLVMGetLastInstruction(block);auto next=loop;
+ for(unsigned j=0;j<DynamicLLVM::LLVMGetNumSuccessors(term);++j){auto target=DynamicLLVM::LLVMGetSuccessor(term,j);if(!loop.count(target) || target==header)continue;
+ std::set<LLVMBasicBlockRef> common;std::set_intersection(next.begin(),next.end(),local[target].begin(),local[target].end(),std::inserter(common,common.begin()));next=std::move(common);}
+ next.insert(block);if(next!=local[block]){local[block]=std::move(next);localChanged=true;}
+ }}
+ auto headerTerm=DynamicLLVM::LLVMGetLastInstruction(header);
+ if(IRIDIUM_IS_BR(DynamicLLVM::LLVMGetInstructionOpcode(headerTerm)) && DynamicLLVM::LLVMIsConditional(headerTerm) && loop.count(DynamicLLVM::LLVMGetSuccessor(headerTerm,0)) && loop.count(DynamicLLVM::LLVMGetSuccessor(headerTerm,1))){
+ LLVMBasicBlockRef innerMerge=nullptr;for(auto candidate:local[header])if(candidate!=header && local[candidate]!=loop && (!innerMerge || local[candidate].size()>local[innerMerge].size()))innerMerge=candidate;
+ if(!innerMerge)throw std::runtime_error("AIR loop header selection has no merge");
+ auto selectionBlock=builder.reserveResultID();splitLoopSelections[header]={selectionBlock,builder.lookupResultID(reinterpret_cast<uintptr_t>(innerMerge))};
+ }
+ for(auto block:loop){auto id=builder.lookupResultID(reinterpret_cast<uintptr_t>(block));auto found=cfgInfos.find(id);if(found==cfgInfos.end() || found->second.type!=CFGType::Selection)continue;
+ if(block==latch){cfgInfos.erase(found);continue;}
+ LLVMBasicBlockRef innerMerge=nullptr;for(auto candidate:local[block])if(candidate!=block && candidate!=header && local[candidate]!=loop && (!innerMerge || local[candidate].size()>local[innerMerge].size()))innerMerge=candidate;
+ if(innerMerge)found->second.selectionMergeBlock=builder.lookupResultID(reinterpret_cast<uintptr_t>(innerMerge));
+ }
+
+ }}
+ // A selection that exits an enclosing construct can use an unreachable merge.
+ // Distinct constructs cannot name the same merge block in structured SPIR-V.
+ std::vector<SPIRV::ResultID> unreachableMerges;
+ std::set<SPIRV::ResultID> claimedMerges;
+ auto headers=blocks;std::sort(headers.begin(),headers.end(),[&](auto a,auto b){return dom[a].size()<dom[b].size();});
+ for(auto bb:headers){auto found=cfgInfos.find(builder.lookupResultID(reinterpret_cast<uintptr_t>(bb)));if(found==cfgInfos.end())continue;
+ auto& cfg=found->second;if(!claimedMerges.insert(cfg.selectionMergeBlock).second){auto id=builder.reserveResultID();cfg.selectionMergeBlock=id;unreachableMerges.push_back(id);}
+ }
+ // SPIR-V requires non-PHI definitions to precede their uses. AIR block order need not do so.
+ std::set<LLVMBasicBlockRef> visited;std::vector<LLVMBasicBlockRef> ordered;
+ std::function<void(LLVMBasicBlockRef)> visit=[&](auto bb){if(!visited.insert(bb).second)return;auto term=DynamicLLVM::LLVMGetLastInstruction(bb);for(unsigned i=0;i<DynamicLLVM::LLVMGetNumSuccessors(term);++i)visit(DynamicLLVM::LLVMGetSuccessor(term,i));ordered.push_back(bb);};
+ visit(blocks.front());std::reverse(ordered.begin(),ordered.end());
+ for(auto bb:blocks)if(!visited.count(bb))ordered.push_back(bb);
+ blocks=std::move(ordered);
+ // PHI outputs are visible across back edges and blocks emitted later in the AIR order.
+ for(auto bb:blocks)for(auto inst=DynamicLLVM::LLVMGetFirstInstruction(bb);inst;inst=DynamicLLVM::LLVMGetNextInstruction(inst))
+ if(DynamicLLVM::LLVMGetInstructionOpcode(inst)==LLVMPHI){auto id=builder.associateResultID(reinterpret_cast<uintptr_t>(inst));builder.setResultType(id,llvmTypeToSPIRVType(builder,DynamicLLVM::LLVMTypeOf(inst)));}
//
// translate instructions
//
isFirst = true;
- for (LLVMBasicBlockRef bb = DynamicLLVM::LLVMGetFirstBasicBlock(_function); bb != nullptr; bb = DynamicLLVM::LLVMGetNextBasicBlock(bb)) {
+ for (auto bb : blocks) {
if (isFirst) {
isFirst = false;
@@ -1030,16 +1649,54 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
builder.insertLabel(builder.lookupResultID(reinterpret_cast<uintptr_t>(bb)));
}
+ bool blockTerminated = false;
+
for (LLVMValueRef inst = DynamicLLVM::LLVMGetFirstInstruction(bb); inst != nullptr; inst = DynamicLLVM::LLVMGetNextInstruction(inst)) {
+ translationInstruction = inst;
+ // OpKill ends its block; whatever follows it in the AIR block (its branch) is dead
+ if (blockTerminated) {
+ break;
+ }
+
auto opcode = DynamicLLVM::LLVMGetInstructionOpcode(inst);
switch (opcode) {
+ case LLVMSExt: case LLVMTrunc: case LLVMFPToUI: case LLVMFPToSI:
+ case LLVMUIToFP: case LLVMSIToFP: case LLVMFPExt: {
+ auto source=DynamicLLVM::LLVMGetOperand(inst,0);auto arg=llvmValueToResultID(builder,source);
+ auto type=llvmTypeToSPIRVType(builder,DynamicLLVM::LLVMTypeOf(inst));
+ uint16_t op=opcode==LLVMSExt?114:(opcode==LLVMTrunc?113:(opcode==LLVMFPToUI?109:(opcode==LLVMFPToSI?110:(opcode==LLVMUIToFP?112:(opcode==LLVMSIToFP?111:115)))));
+ auto resultType=opcode==LLVMFPToUI?integerTypeWithSign(builder,type,false):type;
+ if(opcode==LLVMUIToFP){auto unsignedType=integerTypeWithSign(builder,llvmTypeToSPIRVType(builder,DynamicLLVM::LLVMTypeOf(source)),false);arg=builder.encodeBitcast(unsignedType,arg);}
+ auto result=builder.encodeRawOp(op,resultType,{arg});if(resultType!=type)result=builder.encodeBitcast(type,result);
+ builder.associateExistingResultID(result,reinterpret_cast<uintptr_t>(inst));builder.setResultType(result,type);
+ } break;
+ case LLVMFNeg: {
+ auto type=llvmTypeToSPIRVType(builder,DynamicLLVM::LLVMTypeOf(inst));auto result=builder.encodeRawOp(127,type,{llvmValueToResultID(builder,DynamicLLVM::LLVMGetOperand(inst,0))});
+ builder.associateExistingResultID(result,reinterpret_cast<uintptr_t>(inst));builder.setResultType(result,type);
+ } break;
+ case LLVMUnreachable:builder.encodeRawOpNoResult(255,{});blockTerminated=true;break;
case LLVMZExt: {
auto source = DynamicLLVM::LLVMGetOperand(inst, 0);
auto targetType = DynamicLLVM::LLVMTypeOf(inst);
auto type = llvmTypeToSPIRVType(builder, targetType);
- auto resID = builder.encodeUConvert(type, llvmValueToResultID(builder, source));
+ auto argument = llvmValueToResultID(builder, source);
+ // AIR i1 is a SPIR-V boolean, not an integer accepted by OpUConvert.
+ // Keep the declared destination type so later PHIs see the same signedness.
+ auto sourceType = DynamicLLVM::LLVMTypeOf(source);
+ auto constant = [&](int value) {
+ switch (DynamicLLVM::LLVMGetIntTypeWidth(targetType)) {
+ case 8: return builder.declareConstantScalar<int8_t>(value);
+ case 16: return builder.declareConstantScalar<int16_t>(value);
+ case 32: return builder.declareConstantScalar<int32_t>(value);
+ case 64: return builder.declareConstantScalar<int64_t>(value);
+ default: throw std::runtime_error("Unsupported AIR boolean extension width");
+ }
+ };
+ auto resID = DynamicLLVM::LLVMGetTypeKind(sourceType) == LLVMIntegerTypeKind && DynamicLLVM::LLVMGetIntTypeWidth(sourceType) == 1
+ ? builder.encodeRawOp(169, type, {argument, constant(1), constant(0)})
+ : builder.encodeBitcast(type, builder.encodeUConvert(type, argument));
builder.associateExistingResultID(resID, reinterpret_cast<uintptr_t>(inst));
builder.setResultType(resID, type);
@@ -1048,6 +1705,18 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
// TODO: look into using physical addressing instead of logical addressing, which should simplify the translation process
// (since i *think* that's the same addressing mode used by Metal code)
+ case LLVMAlloca: {
+ auto allocated = DynamicLLVM::LLVMGetAllocatedType(inst);
+ auto count = DynamicLLVM::LLVMGetOperand(inst,0);
+ if (DynamicLLVM::LLVMGetValueKind(count) != LLVMConstantIntValueKind || DynamicLLVM::LLVMConstIntGetZExtValue(count) != 1)
+ throw std::runtime_error("Dynamic/multiple-element AIR allocation is unsupported");
+ LayoutFreeScope layoutFree(true);
+ auto valueType=llvmTypeToSPIRVType(builder,allocated);
+ auto pointerType=builder.declareType(SPIRV::Type(SPIRV::Type::PointerTag{},SPIRV::StorageClass::Function,valueType,8));
+ auto result=builder.addFunctionVariable(pointerType);
+ builder.associateExistingResultID(result,reinterpret_cast<uintptr_t>(inst));
+ } break;
+
case LLVMGetElementPtr: {
auto base = DynamicLLVM::LLVMGetOperand(inst, 0);
auto targetType = DynamicLLVM::LLVMTypeOf(inst);
@@ -1060,17 +1729,32 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
indices.push_back(llvmValueToResultID(builder, llindex));
}
- auto tmp = llvmTypeToSPIRVType(builder, targetType);
auto tmp2 = llvmValueToResultID(builder, base);
+ auto baseStorageClass = builder.reverseLookupType(builder.lookupResultType(tmp2))->pointerStorageClass;
+ LayoutFreeScope layoutFree(baseStorageClass == SPIRV::StorageClass::Private || baseStorageClass == SPIRV::StorageClass::Function);
+ auto tmp = builder.declareType(SPIRV::Type(SPIRV::Type::PointerTag {}, SPIRV::StorageClass::PhysicalStorageBuffer, llvmTypeToSPIRVType(builder, gepElementType(inst)), 8));
// ensure the resulting pointer storage class is the same as the input pointer storage class
auto type = *builder.reverseLookupType(tmp);
auto origTypeID = builder.lookupResultType(tmp2);
auto origType = *builder.reverseLookupType(origTypeID);
+ auto sourceType=llvmTypeToSPIRVType(builder,DynamicLLVM::LLVMGetGEPSourceElementType(inst));
+ if(origType.targetType!=sourceType){origType.targetType=sourceType;origTypeID=builder.declareType(origType);tmp2=builder.encodeBitcast(origTypeID,tmp2);builder.setResultType(tmp2,origTypeID);}
type.pointerStorageClass = origType.pointerStorageClass;
auto resultType = builder.declareType(type);
auto origTypeTarget = *builder.reverseLookupType(origType.targetType);
+ if (origType.pointerStorageClass != SPIRV::StorageClass::PhysicalStorageBuffer) {
+ auto firstIndex = DynamicLLVM::LLVMGetOperand(inst,1);
+ if (DynamicLLVM::LLVMGetValueKind(firstIndex) != LLVMConstantIntValueKind || DynamicLLVM::LLVMConstIntGetZExtValue(firstIndex) != 0)
+ throw std::runtime_error("Logical AIR GEP requires a zero initial index");
+ indices.erase(indices.begin());
+ auto result=indices.empty() ? tmp2 : builder.encodeAccessChain(resultType,tmp2,indices);
+ builder.associateExistingResultID(result,reinterpret_cast<uintptr_t>(inst));
+ builder.setResultType(result,resultType);
+ break;
+ }
+
// strangely enough, SPIR-V provides no instruction that can do an initial pointer offset like LLVM's GEP does.
// there's OpPtrAccessChain, which is tantalizingly named, but unfortunately that instruction doesn't work as expected
// either. i've tried using the raw index (i.e. in element units) and the multiplied index (i.e. in bytes), but no dice.
@@ -1112,6 +1796,9 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
op = access;
}
+ // Opaque LLVM pointers may be used with more than one load type.
+ auto currentPointer=*builder.reverseLookupType(builder.lookupResultType(op));
+ if(currentPointer.targetType!=type){auto pointer=builder.declareType(SPIRV::Type(SPIRV::Type::PointerTag{},currentPointer.pointerStorageClass,type,8));op=builder.encodeBitcast(pointer,op);builder.setResultType(op,pointer);}
auto resID = builder.encodeLoad(type, op, alignment);
builder.associateExistingResultID(resID, reinterpret_cast<uintptr_t>(inst));
@@ -1149,50 +1836,8 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
auto arg = DynamicLLVM::LLVMGetOperand(inst, 0);
resID = builder.encodeConvertUToF(type, llvmValueToResultID(builder, arg));
- } else if (name == "air.sample_texture_2d.v4f16" || name == "air.sample_texture_2d.v4f32" || name == "air.sample_texture_cube.v4f16") {
- auto textureArg = DynamicLLVM::LLVMGetOperand(inst, 0);
- auto samplerArg = DynamicLLVM::LLVMGetOperand(inst, 1);
- auto textureCoordArg = DynamicLLVM::LLVMGetOperand(inst, 2);
- auto someBooleanArgTODO = DynamicLLVM::LLVMGetOperand(inst, 3);
- auto offsetArg = DynamicLLVM::LLVMGetOperand(inst, 4);
- auto someOtherBooleanArgTODO = DynamicLLVM::LLVMGetOperand(inst, 5);
- auto someFloatArgTODO = DynamicLLVM::LLVMGetOperand(inst, 6);
- auto someOtherFloatArgTODO = DynamicLLVM::LLVMGetOperand(inst, 6);
- auto someI32ArgTODO = DynamicLLVM::LLVMGetOperand(inst, 7);
-
- auto textureArgID = llvmValueToResultID(builder, textureArg);
- auto samplerArgID = llvmValueToResultID(builder, samplerArg);
-
- auto texturePtrType = builder.lookupResultType(textureArgID);
- auto textureType = builder.reverseLookupType(texturePtrType)->targetType;
- auto textureLoad = builder.encodeLoad(textureType, textureArgID);
-
- auto samplerType = builder.declareType(SPIRV::Type(SPIRV::Type::SamplerTag {}));
- auto samplerPtrType = builder.declareType(SPIRV::Type(SPIRV::Type::PointerTag {}, SPIRV::StorageClass::UniformConstant, samplerType, 8));
-#if 1
- auto samplerLoad = builder.encodeLoad(samplerType, samplerArgID);
-#else
- auto samplerArgCast = builder.encodeBitcast(samplerPtrType, samplerArgID);
- auto samplerLoad = builder.encodeLoad(samplerType, samplerArgCast);
-#endif
-
- auto sampledImageType = builder.declareType(SPIRV::Type(SPIRV::Type::SampledImageTag {}, textureType));
- auto sampledImage = builder.encodeSampledImage(sampledImageType, textureLoad, samplerLoad);
-
- // WORKAROUND: as explained before, we use 32-bit floats in place of 16-bit floats for image sampling.
- auto textureSampleComponentType = builder.declareType(SPIRV::Type(SPIRV::Type::FloatTag {}, 32));
- auto textureSampleType = builder.declareType(SPIRV::Type(SPIRV::Type::VectorTag {}, 4, textureSampleComponentType, 16, 8));
- auto sampled = builder.encodeImageSampleImplicitLod(textureSampleType, sampledImage, llvmValueToResultID(builder, textureCoordArg));
-
- if (name == "air.sample_texture_2d.v4f16" || name == "air.sample_texture_cube.v4f16") {
- auto convertedComponentType = builder.declareType(SPIRV::Type(SPIRV::Type::FloatTag {}, 16));
- auto convertedType = builder.declareType(SPIRV::Type(SPIRV::Type::VectorTag {}, 4, convertedComponentType, 8, 8));
- sampled = builder.encodeFConvert(convertedType, sampled);
- }
-
- auto partialResult = builder.encodeCompositeInsert(type, sampled, builder.declareUndefinedValue(type), { 0 });
- resID = builder.encodeCompositeInsert(type, builder.declareConstantScalar<int8_t>(0), partialResult, { 1 });
- } else if (name == "air.convert.f.v4f32.f.v4f16" || name == "air.convert.f.v4f16.f.v4f32") {
+ } else if (translateAirTexture(builder, inst, name, type, funcInfo.type, resID)) {
+ } else if (name == "air.convert.f.v4f32.f.v4f16" || name == "air.convert.f.v4f16.f.v4f32") {
auto arg = DynamicLLVM::LLVMGetOperand(inst, 0);
resID = builder.encodeFConvert(type, llvmValueToResultID(builder, arg));
@@ -1223,14 +1868,33 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
auto operand2 = DynamicLLVM::LLVMGetOperand(inst, 1);
resID = builder.encodeFMax(type, llvmValueToResultID(builder, operand1), llvmValueToResultID(builder, operand2));
+ } else if (translateAirIntrinsic(builder, inst, name, type, resID, blockTerminated)) {
+ // handled
} else {
- throw std::runtime_error(std::string("TODO: support actual function calls (name = ") + name.data() + ")");
+ throw std::runtime_error(std::string("TODO: support actual function calls (name = ") + std::string(name) + ")");
}
- builder.associateExistingResultID(resID, reinterpret_cast<uintptr_t>(inst));
- builder.setResultType(resID, type);
+ if (resID != SPIRV::ResultIDInvalid) {
+ builder.associateExistingResultID(resID, reinterpret_cast<uintptr_t>(inst));
+ builder.setResultType(resID, type);
+ }
} break;
+ case LLVMAdd: case LLVMSub: case LLVMMul: case LLVMUDiv: case LLVMSDiv:
+ case LLVMURem: case LLVMSRem: case LLVMShl: case LLVMLShr: case LLVMAShr:
+ case LLVMAnd: case LLVMOr: case LLVMXor: {
+ auto lhs=llvmValueToResultID(builder,DynamicLLVM::LLVMGetOperand(inst,0));
+ auto rhs=llvmValueToResultID(builder,DynamicLLVM::LLVMGetOperand(inst,1));
+ auto type=llvmTypeToSPIRVType(builder,DynamicLLVM::LLVMTypeOf(inst));
+ uint16_t op=0;
+ switch(opcode){case LLVMAdd:op=128;break;case LLVMSub:op=130;break;case LLVMMul:op=132;break;
+ case LLVMUDiv:op=134;break;case LLVMSDiv:op=135;break;case LLVMURem:op=137;break;case LLVMSRem:op=138;break;
+ case LLVMShl:op=196;break;case LLVMLShr:op=194;break;case LLVMAShr:op=195;break;
+ case LLVMAnd:op=199;break;case LLVMOr:op=197;break;case LLVMXor:op=198;break;default:break;}
+ auto t=*builder.reverseLookupType(type);
+ if(t.backingType==SPIRV::Type::BackingType::Boolean){op=opcode==LLVMAnd?167:(opcode==LLVMOr?166:165);}
+ auto result=builder.encodeRawOp(op,type,{lhs,rhs});builder.associateExistingResultID(result,reinterpret_cast<uintptr_t>(inst));
+ } break;
case LLVMFMul:
case LLVMFDiv:
case LLVMFAdd:
@@ -1365,28 +2029,81 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
auto op1 = DynamicLLVM::LLVMGetOperand(inst, 0);
auto op2 = DynamicLLVM::LLVMGetOperand(inst, 1);
auto predicate = DynamicLLVM::LLVMGetFCmpPredicate(inst);
+ auto type = llvmTypeToSPIRVType(builder, DynamicLLVM::LLVMTypeOf(inst));
- SPIRV::ResultID resID = SPIRV::ResultIDInvalid;
- auto type = builder.declareType(SPIRV::Type(SPIRV::Type::BooleanTag {}));
-
+ uint16_t spirvOpcode = 0;
switch (predicate) {
- case LLVMRealOGT: {
- resID = builder.encodeFOrdGreaterThan(llvmValueToResultID(builder, op1), llvmValueToResultID(builder, op2));
- } break;
+ case LLVMRealOEQ: spirvOpcode = SPIRVOp::FOrdEqual; break;
+ case LLVMRealOGT: spirvOpcode = SPIRVOp::FOrdGreaterThan; break;
+ case LLVMRealOGE: spirvOpcode = SPIRVOp::FOrdGreaterThanEqual; break;
+ case LLVMRealOLT: spirvOpcode = SPIRVOp::FOrdLessThan; break;
+ case LLVMRealOLE: spirvOpcode = SPIRVOp::FOrdLessThanEqual; break;
+ case LLVMRealONE: spirvOpcode = SPIRVOp::FOrdNotEqual; break;
+ case LLVMRealORD: spirvOpcode = SPIRVOp::Ordered; break;
+ case LLVMRealUNO: spirvOpcode = SPIRVOp::Unordered; break;
+ case LLVMRealUEQ: spirvOpcode = SPIRVOp::FUnordEqual; break;
+ case LLVMRealUGT: spirvOpcode = SPIRVOp::FUnordGreaterThan; break;
+ case LLVMRealUGE: spirvOpcode = SPIRVOp::FUnordGreaterThanEqual; break;
+ case LLVMRealULT: spirvOpcode = SPIRVOp::FUnordLessThan; break;
+ case LLVMRealULE: spirvOpcode = SPIRVOp::FUnordLessThanEqual; break;
+ case LLVMRealUNE: spirvOpcode = SPIRVOp::FUnordNotEqual; break;
+ default:
+ throw std::runtime_error("TODO: handle fcmp predicate: " + std::to_string(predicate));
+ }
+
+ auto resID = builder.encodeRawOp(spirvOpcode, type, { llvmValueToResultID(builder, op1), llvmValueToResultID(builder, op2) });
- case LLVMRealOLT: {
- resID = builder.encodeFOrdLessThan(llvmValueToResultID(builder, op1), llvmValueToResultID(builder, op2));
- } break;
+ builder.associateExistingResultID(resID, reinterpret_cast<uintptr_t>(inst));
+ builder.setResultType(resID, type);
+ } break;
+ case LLVMICmp: {
+ auto op1 = DynamicLLVM::LLVMGetOperand(inst, 0);
+ auto op2 = DynamicLLVM::LLVMGetOperand(inst, 1);
+ auto predicate = DynamicLLVM::LLVMGetICmpPredicate(inst);
+ auto type = llvmTypeToSPIRVType(builder, DynamicLLVM::LLVMTypeOf(inst));
+
+ uint16_t spirvOpcode = 0;
+ switch (predicate) {
+ case LLVMIntEQ: spirvOpcode = SPIRVOp::IEqual; break;
+ case LLVMIntNE: spirvOpcode = SPIRVOp::INotEqual; break;
+ case LLVMIntUGT: spirvOpcode = SPIRVOp::UGreaterThan; break;
+ case LLVMIntUGE: spirvOpcode = SPIRVOp::UGreaterThanEqual; break;
+ case LLVMIntULT: spirvOpcode = SPIRVOp::ULessThan; break;
+ case LLVMIntULE: spirvOpcode = SPIRVOp::ULessThanEqual; break;
+ case LLVMIntSGT: spirvOpcode = SPIRVOp::SGreaterThan; break;
+ case LLVMIntSGE: spirvOpcode = SPIRVOp::SGreaterThanEqual; break;
+ case LLVMIntSLT: spirvOpcode = SPIRVOp::SLessThan; break;
+ case LLVMIntSLE: spirvOpcode = SPIRVOp::SLessThanEqual; break;
default:
- throw std::runtime_error("TODO: handle fcmp predicate: " + std::to_string(predicate));
+ throw std::runtime_error("TODO: handle icmp predicate: " + std::to_string(predicate));
}
+ auto resID = builder.encodeRawOp(spirvOpcode, type, { llvmValueToResultID(builder, op1), llvmValueToResultID(builder, op2) });
+
+ builder.associateExistingResultID(resID, reinterpret_cast<uintptr_t>(inst));
+ builder.setResultType(resID, type);
+ } break;
+
+ case LLVMSelect: {
+ auto type = llvmTypeToSPIRVType(builder, DynamicLLVM::LLVMTypeOf(inst));
+ auto resID = builder.encodeRawOp(SPIRVOp::Select, type, {
+ llvmValueToResultID(builder, DynamicLLVM::LLVMGetOperand(inst, 0)),
+ llvmValueToResultID(builder, DynamicLLVM::LLVMGetOperand(inst, 1)),
+ llvmValueToResultID(builder, DynamicLLVM::LLVMGetOperand(inst, 2)),
+ });
+
builder.associateExistingResultID(resID, reinterpret_cast<uintptr_t>(inst));
builder.setResultType(resID, type);
} break;
- case LLVMBr: {
+ IRIDIUM_BR_CASES: {
+ auto loop=cfgInfos.find(builder.lookupResultID(reinterpret_cast<uintptr_t>(bb)));
+ if(loop!=cfgInfos.end() && loop->second.type==CFGType::Loop){
+ builder.encodeRawOpNoResult(246,{loop->second.selectionMergeBlock,loop->second.continueBlock,0});
+ auto selection=splitLoopSelections.find(bb);if(selection!=splitLoopSelections.end()){builder.encodeBranch(selection->second.first);builder.insertLabel(selection->second.first);builder.encodeSelectionMerge(selection->second.second);}
+ }
+ auto split=splitContinues.find(bb);if(split!=splitContinues.end()){builder.encodeBranch(split->second);builder.insertLabel(split->second);}
if (DynamicLLVM::LLVMIsConditional(inst)) {
auto condition = DynamicLLVM::LLVMGetCondition(inst);
auto trueLabel = DynamicLLVM::LLVMBasicBlockAsValue(DynamicLLVM::LLVMGetSuccessor(inst, 0));
@@ -1422,25 +2139,38 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
auto type = llvmTypeToSPIRVType(builder, lltype);
std::vector<std::pair<SPIRV::ResultID, SPIRV::ResultID>> variablesAndBlocks;
+ std::vector<uintptr_t> forwardAssociations;
auto opCount = DynamicLLVM::LLVMCountIncoming(inst);
for (size_t i = 0; i < opCount; ++i) {
auto variable = DynamicLLVM::LLVMGetIncomingValue(inst, i);
auto label = DynamicLLVM::LLVMBasicBlockAsValue(DynamicLLVM::LLVMGetIncomingBlock(inst, i));
- variablesAndBlocks.push_back(std::make_pair(llvmValueToResultID(builder, variable), llvmValueToResultID(builder, label)));
+ auto id=builder.lookupResultID(reinterpret_cast<uintptr_t>(variable));
+ bool forward=id==SPIRV::ResultIDInvalid && DynamicLLVM::LLVMGetValueKind(variable)==LLVMInstructionValueKind;
+ auto split=splitContinues.find(DynamicLLVM::LLVMGetIncomingBlock(inst,i));
+ auto labelID=split==splitContinues.end()?llvmValueToResultID(builder,label):split->second;
+ auto selection=splitLoopSelections.find(DynamicLLVM::LLVMGetIncomingBlock(inst,i));if(selection!=splitLoopSelections.end())labelID=selection->second.first;
+ variablesAndBlocks.push_back({forward?SPIRV::ResultIDInvalid:llvmValueToResultID(builder,variable),labelID});
+ forwardAssociations.push_back(forward?reinterpret_cast<uintptr_t>(variable):0);
}
- auto resID = builder.encodePhi(type, variablesAndBlocks);
+ auto resID = builder.encodePhi(type, variablesAndBlocks, builder.lookupResultID(reinterpret_cast<uintptr_t>(inst)), forwardAssociations);
builder.associateExistingResultID(resID, reinterpret_cast<uintptr_t>(inst));
builder.setResultType(resID, type);
} break;
case LLVMBitCast: {
+ // Unused opaque-pointer casts can infer a different pointee type.
+ // They have no effect and logical-pointer OpBitcast is illegal in SPIR-V.
+ if (!DynamicLLVM::LLVMGetFirstUse(inst)) break;
auto arg = DynamicLLVM::LLVMGetOperand(inst, 0);
auto lltype = DynamicLLVM::LLVMTypeOf(inst);
- auto type = llvmTypeToSPIRVType(builder, lltype);
auto argID = llvmValueToResultID(builder, arg);
+ auto sourceType = *builder.reverseLookupType(builder.lookupResultType(argID));
+ auto type = DynamicLLVM::LLVMGetTypeKind(lltype) == LLVMPointerTypeKind ?
+ builder.declareType(SPIRV::Type(SPIRV::Type::PointerTag {}, sourceType.pointerStorageClass,
+ llvmTypeToSPIRVType(builder, pointerElementType(inst)), 8)) : llvmTypeToSPIRVType(builder, lltype);
// ensure the resulting pointer storage class is the same as the input pointer storage class
auto typeInst = *builder.reverseLookupType(type);
@@ -1474,6 +2204,7 @@ void Iridium::AIR::Function::analyze(SPIRV::Builder& builder, OutputInfo& output
}
}
+ for(auto label:unreachableMerges){builder.insertLabel(label);builder.encodeRawOpNoResult(255,{});}
builder.endFunction();
};
diff --git a/src/iridium/spirv.cpp b/src/iridium/spirv.cpp
index 5528342..4c085e6 100644
--- a/src/iridium/spirv.cpp
+++ b/src/iridium/spirv.cpp
@@ -1,3 +1,4 @@
+#include <algorithm>
#include <iridium/spirv.hpp>
#include <optional>
#include <functional>
@@ -19,6 +20,7 @@ bool Iridium::SPIRV::Type::operator==(const Type& other) const {
backingType != other.backingType ||
scalarWidth != other.scalarWidth ||
integerIsSigned != other.integerIsSigned ||
+ layoutFree != other.layoutFree ||
entryCount != other.entryCount ||
structureMembers.size() != other.structureMembers.size() ||
pointerStorageClass != other.pointerStorageClass ||
@@ -224,12 +226,19 @@ void Iridium::SPIRV::Builder::addEntryPoint(const EntryPoint& entryPoint) {
_entryPoints.push_back(entryPoint);
};
-Iridium::SPIRV::ResultID Iridium::SPIRV::Builder::addGlobalVariable(ResultID typeID, StorageClass storageClass) {
+Iridium::SPIRV::ResultID Iridium::SPIRV::Builder::addGlobalVariable(ResultID typeID, StorageClass storageClass, ResultID initializer) {
auto resultID = reserveResultID();
- _globalVariables[resultID] = GlobalVariable { storageClass, typeID };
+ _globalVariables[resultID] = GlobalVariable { storageClass, typeID, initializer };
return resultID;
};
+Iridium::SPIRV::ResultID Iridium::SPIRV::Builder::addFunctionVariable(ResultID typeID) {
+ auto id = reserveResultID();
+ _currentFunctionInfo->variables[id] = FunctionVariable { typeID, ResultIDInvalid };
+ setResultType(id, typeID);
+ return id;
+};
+
void Iridium::SPIRV::Builder::addDecoration(ResultID resultID, const Decoration& decoration) {
if (_decorations.find(resultID) == _decorations.end()) {
_decorations[resultID] = std::unordered_set<Decoration> { decoration };
@@ -281,6 +290,9 @@ Iridium::SPIRV::ResultID Iridium::SPIRV::Builder::beginFunction(ResultID id) {
};
void Iridium::SPIRV::Builder::endFunction() {
+ auto position=_currentFunctionWriter->offset();
+ for(auto fixup:_phiFixups){auto id=lookupResultID(fixup.second);if(id==ResultIDInvalid)throw std::runtime_error("Unresolved forward PHI input");_currentFunctionWriter->seek(fixup.first);_currentFunctionWriter->writeIntegerLE<uint32_t>(id);}
+ _phiFixups.clear();_currentFunctionWriter->seek(position);
_currentFunctionWriter = nullptr;
};
@@ -425,7 +437,8 @@ Iridium::SPIRV::ResultID Iridium::SPIRV::Builder::insertLabel(ResultID id) {
};
void Iridium::SPIRV::Builder::referenceGlobalVariable(ResultID id) {
- _currentFunctionInfo->referencedGlobalVariables.push_back(id);
+ auto& refs = _currentFunctionInfo->referencedGlobalVariables;
+ if (std::find(refs.begin(), refs.end(), id) == refs.end()) refs.push_back(id);
};
Iridium::SPIRV::ResultID Iridium::SPIRV::Builder::encodeUConvert(ResultID typeID, ResultID operand) {
@@ -719,12 +732,15 @@ void Iridium::SPIRV::Builder::encodeBranchConditional(ResultID condition, Result
endInstruction(std::move(tmp));
};
-Iridium::SPIRV::ResultID Iridium::SPIRV::Builder::encodePhi(ResultID resultTypeID, std::vector<std::pair<ResultID, ResultID>> variablesAndBlocks) {
- auto result = reserveResultID();
+Iridium::SPIRV::ResultID Iridium::SPIRV::Builder::encodePhi(ResultID resultTypeID, std::vector<std::pair<ResultID, ResultID>> variablesAndBlocks, ResultID result, std::vector<uintptr_t> forwardAssociations) {
+ if(result==ResultIDInvalid)result = reserveResultID();
auto tmp = beginInstruction(Opcode::Phi, *_currentFunctionWriter);
_currentFunctionWriter->writeIntegerLE<uint32_t>(resultTypeID);
_currentFunctionWriter->writeIntegerLE<uint32_t>(result);
- for (const auto& [variable, block]: variablesAndBlocks) {
+ size_t index=0;
+ for (const auto& [variable, block]: variablesAndBlocks) {
+ if(index<forwardAssociations.size() && forwardAssociations[index])_phiFixups.push_back({_currentFunctionWriter->offset(),forwardAssociations[index]});
+ ++index;
_currentFunctionWriter->writeIntegerLE<uint32_t>(variable);
_currentFunctionWriter->writeIntegerLE<uint32_t>(block);
}
@@ -910,6 +926,7 @@ void* Iridium::SPIRV::Builder::finalize(size_t& outputSize) {
}
endInstruction(std::move(tmp));
}
+ if (type.layoutFree) continue;
tmp = beginInstruction(Opcode::MemberDecorate, _writer);
_writer.writeIntegerLE<uint32_t>(id);
_writer.writeIntegerLE<uint32_t>(i);
@@ -934,7 +951,7 @@ void* Iridium::SPIRV::Builder::finalize(size_t& outputSize) {
// now emit automatic type decorations
for (const auto& [type, id]: _typeIDs) {
- if (type.backingType == Type::BackingType::RuntimeArray || type.backingType == Type::BackingType::Array) {
+ if (!type.layoutFree && (type.backingType == Type::BackingType::RuntimeArray || type.backingType == Type::BackingType::Array)) {
auto elmTypeInst = _reverseTypeIDs[type.targetType];
tmp = beginInstruction(Opcode::Decorate, _writer);
_writer.writeIntegerLE<uint32_t>(id);
@@ -1120,7 +1137,7 @@ void* Iridium::SPIRV::Builder::finalize(size_t& outputSize) {
_writer.writeIntegerLE<uint32_t>(var.typeID);
_writer.writeIntegerLE<uint32_t>(id);
_writer.writeIntegerLE<uint32_t>(static_cast<uint32_t>(var.storageClass));
- // TODO: support initializers
+ if (var.initializer != ResultIDInvalid) _writer.writeIntegerLE<uint32_t>(var.initializer);
endInstruction(std::move(tmp));
}
@@ -1199,3 +1216,54 @@ void* Iridium::SPIRV::Builder::finalize(size_t& outputSize) {
outputSize = _writer.size();
return _writer.extractBuffer();
};
+
+Iridium::SPIRV::ResultID Iridium::SPIRV::Builder::declareConstantBool(bool value) {
+ auto typeID = declareType(Type(Type::BooleanTag {}));
+ auto key = std::make_pair(static_cast<uintmax_t>(value ? 1 : 0), typeID);
+ auto it = _constantBools.find(key);
+
+ if (it != _constantBools.end()) {
+ return it->second;
+ }
+
+ auto id = reserveResultID();
+
+ _constantBools.emplace(key, id);
+
+ auto tmp = beginInstruction(value ? Opcode::ConstantTrue : Opcode::ConstantFalse, _constants);
+ _constants.writeIntegerLE<uint32_t>(typeID);
+ _constants.writeIntegerLE<uint32_t>(id);
+ endInstruction(std::move(tmp));
+
+ return id;
+};
+
+Iridium::SPIRV::ResultID Iridium::SPIRV::Builder::encodeRawOp(uint16_t opcode, ResultID resultTypeID, const std::vector<ResultID>& operands) {
+ auto result = reserveResultID();
+ auto tmp = beginInstruction(opcode, *_currentFunctionWriter);
+ _currentFunctionWriter->writeIntegerLE<uint32_t>(resultTypeID);
+ _currentFunctionWriter->writeIntegerLE<uint32_t>(result);
+ for (auto operand: operands) {
+ _currentFunctionWriter->writeIntegerLE<uint32_t>(operand);
+ }
+ endInstruction(std::move(tmp));
+ return result;
+};
+
+void Iridium::SPIRV::Builder::encodeRawOpNoResult(uint16_t opcode, const std::vector<ResultID>& operands) {
+ auto tmp = beginInstruction(opcode, *_currentFunctionWriter);
+ for (auto operand: operands) {
+ _currentFunctionWriter->writeIntegerLE<uint32_t>(operand);
+ }
+ endInstruction(std::move(tmp));
+};
+
+Iridium::SPIRV::ResultID Iridium::SPIRV::Builder::encodeGLSLOp(GLSLOpcode opcode, ResultID resultTypeID, const std::vector<ResultID>& operands) {
+ auto result = reserveResultID();
+ auto tmp = beginGLSLInstruction(opcode, *_currentFunctionWriter, resultTypeID, result);
+ for (auto operand: operands) {
+ _currentFunctionWriter->writeIntegerLE<uint32_t>(operand);
+ }
+ endInstruction(std::move(tmp));
+ return result;
+};
diff --git a/private-include/indium/counters.private.hpp b/private-include/indium/counters.private.hpp
new file mode 100644
--- /dev/null
+++ b/private-include/indium/counters.private.hpp
@@ -0,0 +1,44 @@
+#pragma once
+
+#include <cstdio>
+#include <map>
+#include <mutex>
+#include <string>
+#include <time.h>
+#include <unistd.h>
+
+namespace Indium {
+ // Diagnostic SPIR-V/binding dumps: checked once per process (an access() per draw is a syscall plus path translation).
+ inline bool spirvDebugEnabled() {
+ static const bool enabled = access("/tmp/metal-debug-spirv", F_OK) == 0;
+ return enabled;
+ }
+
+ // Diagnostic event counters, written to the guest's /tmp/metal-trace.log every 5 s when /tmp/metal-debug-timing exists.
+ inline void count(const char* what, double amount = 1) {
+ static const bool enabled = access("/tmp/metal-debug-timing", F_OK) == 0;
+ if (!enabled) {
+ return;
+ }
+ static std::mutex lock;
+ static std::map<std::string, std::pair<unsigned long, double>> counts;
+ static time_t last = 0;
+ std::lock_guard<std::mutex> guard(lock);
+ auto& entry = counts[what];
+ entry.first++;
+ entry.second += amount;
+ time_t now = time(nullptr);
+ if (now - last >= 5) {
+ last = now;
+ if (FILE* log = fopen("/tmp/metal-trace.log", "a")) {
+ fprintf(log, "indium-counts@%ld:", (long) now);
+ for (auto& item: counts) {
+ fprintf(log, " %s[n=%lu sum=%.0f]", item.first.c_str(), item.second.first, item.second.second);
+ }
+ fprintf(log, "\n");
+ fclose(log);
+ }
+ counts.clear();
+ }
+ }
+};
diff --git a/private-include/indium/deferred-render-encoder.private.hpp b/private-include/indium/deferred-render-encoder.private.hpp
new file mode 100644
--- /dev/null
+++ b/private-include/indium/deferred-render-encoder.private.hpp
@@ -0,0 +1,372 @@
+#pragma once
+
+// Deferred render-command encoding.
+//
+// League's main thread spends 2-4.5 ms of every frame inside the Metal-to-Vulkan translation (PrivateRenderCommandEncoder:
+// binding bookkeeping, descriptor sets, vkCmd* calls). Nothing about that work has to happen on the game's thread: Metal
+// semantics only require that the encoder calls are applied in order before the command buffer is committed.
+//
+// This facade implements the public RenderCommandEncoder interface by recording every call into a compact op list
+// (copying anything the caller may reuse: setBytes data, viewport/scissor arrays) and, at endEncoding(), hands the list to
+// the ordered worker of submit-thread.private.hpp, which creates the real PrivateRenderCommandEncoder, replays the ops into
+// it and ends it. The same worker later runs the command buffer's commit(), so encoders are always fully replayed before
+// the commit looks at them. Everything that touches the same Vulkan command buffer from another thread (blit/compute
+// encoders, a direct render encoder) first waits for the command buffer's pending replays (waitDeferred()).
+//
+// Live switch (guest file, re-read every 256 encoders): /tmp/metal-async-encode present = on. Off by default while it is
+// validated. Header-only: the build script compiles only the .cpp files of the CMake target.
+
+#include <indium/render-command-encoder.hpp>
+#include <indium/render-command-encoder.private.hpp>
+#include <indium/command-buffer.private.hpp>
+#include <indium/render-pass.hpp>
+#include <indium/submit-thread.private.hpp>
+
+#include <atomic>
+#include <cstdint>
+#include <cstdio>
+#include <dlfcn.h>
+#include <memory>
+#include <mutex>
+#include <vector>
+#include <unistd.h>
+#include <cstring>
+#include <cstdlib>
+
+namespace Indium {
+ namespace AsyncEncode {
+ inline bool enabled() {
+ static std::atomic<bool> value {false};
+ static std::atomic<unsigned> calls {0};
+ // On by default in the League game process only (fight A/B 2026-10-12: 60.4 -> 72.4 fps, p99 28.7 -> 22.9 ms
+ // with the frame-end sync in PrivateCommandBuffer::commit). Guest flag files, re-read every 256 calls:
+ // /tmp/metal-no-async-encode turns it off there, /tmp/metal-async-encode turns it on in any process.
+ static const bool leagueGame = [] { const char* name = getprogname(); return name && strcmp(name, "LeagueofLegends") == 0; }();
+ if ((calls.fetch_add(1, std::memory_order_relaxed) & 255) == 0)
+ value.store(access("/tmp/metal-async-encode", F_OK) == 0 ||
+ (leagueGame && access("/tmp/metal-no-async-encode", F_OK) != 0), std::memory_order_relaxed);
+ return value.load(std::memory_order_relaxed);
+ }
+
+ // Objective-C autorelease pool for the worker thread (libobjc is not a link dependency of Indium)
+ struct PoolScope {
+ void* token = nullptr;
+ void (*pop)(void*) = nullptr;
+ PoolScope() {
+ static void* (*push)(void) = reinterpret_cast<void* (*)(void)>(dlsym(RTLD_DEFAULT, "objc_autoreleasePoolPush"));
+ static void (*popFn)(void*) = reinterpret_cast<void (*)(void*)>(dlsym(RTLD_DEFAULT, "objc_autoreleasePoolPop"));
+ if (push && popFn) { token = push(); pop = popFn; }
+ }
+ ~PoolScope() { if (pop) pop(token); }
+ };
+
+ enum class Kind: uint8_t {
+ SetPipeline, FrontFacing, CullMode, DepthBias, DepthClip, Viewports, Scissors, BlendColor, DepthStencil, FillMode,
+ StencilRef, StencilRefFB, VisibilityMode,
+ DrawPrim, DrawIdx,
+ VBytes, VBuffer, VBufferOffset, VSampler, VSamplerClamp, VTexture,
+ FBytes, FBuffer, FBufferOffset, FSampler, FSamplerClamp, FTexture,
+ UseResource, UseResourceStages, MemoryBarrier,
+ };
+
+ struct Op {
+ Kind kind;
+ uint8_t pad8 = 0;
+ uint32_t e1 = 0, e2 = 0;
+ size_t n[5] = {0, 0, 0, 0, 0};
+ float f[4] = {0, 0, 0, 0};
+ std::shared_ptr<void> p;
+ explicit Op(Kind k) : kind(k) {}
+ };
+
+ // Recording buffers are recycled: growing them through the allocator on every encoder is what this is avoiding.
+ struct Buffers {
+ std::vector<Op> ops;
+ std::vector<uint8_t> bytes;
+ std::vector<Viewport> viewports;
+ std::vector<ScissorRect> scissors;
+ void clear() { ops.clear(); bytes.clear(); viewports.clear(); scissors.clear(); }
+ };
+
+ struct BufferPool {
+ std::mutex mutex;
+ std::vector<std::unique_ptr<Buffers>> free;
+ };
+
+ inline BufferPool& bufferPool() {
+ // The detached worker may return buffers during process teardown.
+ static auto* pool = new BufferPool;
+ return *pool;
+ }
+
+ inline std::unique_ptr<Buffers> acquireBuffers() {
+ auto& pool = bufferPool();
+ {
+ std::lock_guard<std::mutex> lock(pool.mutex);
+ if (!pool.free.empty()) { auto b = std::move(pool.free.back()); pool.free.pop_back(); return b; }
+ }
+ auto b = std::make_unique<Buffers>();
+ b->ops.reserve(512);
+ b->bytes.reserve(8192);
+ return b;
+ }
+
+ inline void releaseBuffers(std::unique_ptr<Buffers> b) {
+ b->clear();
+ if (b->ops.capacity() > 65536 || b->bytes.capacity() > 1048576) return;
+ auto& pool = bufferPool();
+ std::lock_guard<std::mutex> lock(pool.mutex);
+ if (pool.free.size() < 16) pool.free.push_back(std::move(b));
+ }
+
+ // One render pass being recorded. The recording is handed to the worker in chunks while the caller is still
+ // encoding (see DeferredRenderCommandEncoder::add), so `real` and `failed` are only touched by the worker.
+ struct State {
+ std::shared_ptr<PrivateCommandBuffer> commandBuffer;
+ RenderPassDescriptor descriptor;
+ std::shared_ptr<PrivateRenderCommandEncoder> real;
+ bool failed = false;
+ State(std::shared_ptr<PrivateCommandBuffer> cb, const RenderPassDescriptor& d) : commandBuffer(std::move(cb)), descriptor(d) {}
+ };
+
+ // Apply one chunk of recorded calls; `last` also ends the real encoder.
+ inline void replay(State& st, const Buffers& r, bool last) {
+ PoolScope pool;
+ if (!st.real && !st.failed) {
+ try {
+ st.real = st.commandBuffer->renderCommandEncoderNow(st.descriptor);
+ } catch (const std::exception& e) {
+ st.failed = true;
+ if (FILE* f = fopen("/tmp/metal-trace.log", "a")) { fprintf(f, "async-encode: encoder creation failed: %s\n", e.what()); fclose(f); }
+ }
+ }
+ if (!st.real) return;
+ PrivateRenderCommandEncoder* real = st.real.get();
+ for (const Op& op : r.ops) {
+ try {
+ switch (op.kind) {
+ case Kind::SetPipeline: real->setRenderPipelineState(std::static_pointer_cast<RenderPipelineState>(op.p)); break;
+ case Kind::FrontFacing: real->setFrontFacingWinding(static_cast<Winding>(op.e1)); break;
+ case Kind::CullMode: real->setCullMode(static_cast<CullMode>(op.e1)); break;
+ case Kind::DepthBias: real->setDepthBias(op.f[0], op.f[1], op.f[2]); break;
+ case Kind::DepthClip: real->setDepthClipMode(static_cast<DepthClipMode>(op.e1)); break;
+ case Kind::Viewports: real->setViewports(r.viewports.data() + op.n[0], op.n[1]); break;
+ case Kind::Scissors: real->setScissorRects(r.scissors.data() + op.n[0], op.n[1]); break;
+ case Kind::BlendColor: real->setBlendColor(op.f[0], op.f[1], op.f[2], op.f[3]); break;
+ case Kind::DepthStencil: real->setDepthStencilState(std::static_pointer_cast<DepthStencilState>(op.p)); break;
+ case Kind::FillMode: real->setTriangleFillMode(static_cast<TriangleFillMode>(op.e1)); break;
+ case Kind::StencilRef: real->setStencilReferenceValue(op.e1); break;
+ case Kind::StencilRefFB: real->setStencilReferenceValue(op.e1, op.e2); break;
+ case Kind::VisibilityMode: real->setVisibilityResultMode(static_cast<VisibilityResultMode>(op.e1), op.n[0]); break;
+ case Kind::DrawPrim: real->drawPrimitives(static_cast<PrimitiveType>(op.e1), op.n[0], op.n[1], op.n[2], op.n[3]); break;
+ case Kind::DrawIdx:
+ real->drawIndexedPrimitives(static_cast<PrimitiveType>(op.e1), op.n[0], static_cast<IndexType>(op.e2),
+ std::static_pointer_cast<Buffer>(op.p), op.n[1], op.n[2], static_cast<int64_t>(op.n[3]), op.n[4]);
+ break;
+ case Kind::VBytes: real->setVertexBytes(r.bytes.data() + op.n[0], op.n[1], op.n[2]); break;
+ case Kind::VBuffer: real->setVertexBuffer(std::static_pointer_cast<Buffer>(op.p), op.n[0], op.n[1]); break;
+ case Kind::VBufferOffset: real->setVertexBufferOffset(op.n[0], op.n[1]); break;
+ case Kind::VSampler: real->setVertexSamplerState(std::static_pointer_cast<SamplerState>(op.p), op.n[0]); break;
+ case Kind::VSamplerClamp: real->setVertexSamplerState(std::static_pointer_cast<SamplerState>(op.p), op.f[0], op.f[1], op.n[0]); break;
+ case Kind::VTexture: real->setVertexTexture(std::static_pointer_cast<Texture>(op.p), op.n[0]); break;
+ case Kind::FBytes: real->setFragmentBytes(r.bytes.data() + op.n[0], op.n[1], op.n[2]); break;
+ case Kind::FBuffer: real->setFragmentBuffer(std::static_pointer_cast<Buffer>(op.p), op.n[0], op.n[1]); break;
+ case Kind::FBufferOffset: real->setFragmentBufferOffset(op.n[0], op.n[1]); break;
+ case Kind::FSampler: real->setFragmentSamplerState(std::static_pointer_cast<SamplerState>(op.p), op.n[0]); break;
+ case Kind::FSamplerClamp: real->setFragmentSamplerState(std::static_pointer_cast<SamplerState>(op.p), op.f[0], op.f[1], op.n[0]); break;
+ case Kind::FTexture: real->setFragmentTexture(std::static_pointer_cast<Texture>(op.p), op.n[0]); break;
+ case Kind::UseResource: real->useResource(std::static_pointer_cast<Resource>(op.p), static_cast<ResourceUsage>(op.e1)); break;
+ case Kind::UseResourceStages: real->useResource(std::static_pointer_cast<Resource>(op.p), static_cast<ResourceUsage>(op.e1), static_cast<RenderStages>(op.n[0])); break;
+ case Kind::MemoryBarrier: real->memoryBarrier(op.n[0], static_cast<RenderStages>(op.n[1]), static_cast<RenderStages>(op.n[2])); break;
+ }
+ } catch (const std::exception& e) {
+ if (FILE* f = fopen("/tmp/metal-trace.log", "a")) { fprintf(f, "async-encode: op %d failed: %s\n", (int) op.kind, e.what()); fclose(f); }
+ }
+ }
+ if (last) {
+ try { real->endEncoding(); } catch (...) {}
+ st.real.reset();
+ }
+ }
+
+ class DeferredRenderCommandEncoder final: public RenderCommandEncoder {
+ // calls per chunk handed to the worker while the pass is still being recorded
+ static constexpr size_t chunkOps = 128;
+
+ std::shared_ptr<State> _state;
+ std::unique_ptr<Buffers> _rec;
+ Buffers* _r;
+ bool _begun = false;
+ bool _ended = false;
+
+ // The worker starts on a pass as soon as its first chunk is complete instead of after endEncoding: at the
+ // frame-end sync it used to have the whole last pass still to do while the caller waited (27% of the main
+ // thread in fights). A chunk owns the bytes/viewports/scissors its calls refer to, so any boundary is valid.
+ void flush(bool last) {
+ auto st = _state;
+ auto cb = st->commandBuffer;
+ if (!_begun) { _begun = true; cb->deferredBegin(); }
+ Buffers* chunk = _rec.release();
+ if (!cb->replayAllowed()) {
+ // an encoder dropped after the commit (a Metal contract violation): the Vulkan command buffer is
+ // closed, so drop the chunk instead of recording into it, and balance the deferred count
+ releaseBuffers(std::unique_ptr<Buffers>(chunk));
+ if (last) {
+ cb->deferredEnd();
+ st.reset();
+ }
+ if (!last) { _rec = acquireBuffers(); _r = _rec.get(); }
+ else _r = nullptr;
+ return;
+ }
+ SubmitThread::post([st, cb, chunk, last]() mutable {
+ std::unique_ptr<Buffers> owned(chunk);
+ replay(*st, *owned, last);
+ releaseBuffers(std::move(owned));
+ if (last) {
+ st.reset();
+ cb->deferredEnd();
+ }
+ });
+ if (!last) { _rec = acquireBuffers(); _r = _rec.get(); }
+ else _r = nullptr;
+ }
+
+ Op& add(Kind k) {
+ // before recording the next call: the previous ones are complete (their fields are filled in after add())
+ if (_r->ops.size() >= chunkOps) flush(false);
+ _r->ops.emplace_back(k);
+ return _r->ops.back();
+ }
+ // The op keeps the resource alive until the worker replays it; taking the reference
+ // straight from the caller's shared_ptr (the setters now take it by const reference)
+ // costs exactly one increment and no temporary, where the by-value form left an
+ // increment/decrement pair plus a store-to-load chain per forwarded bind.
+ template<class T>
+ void addRef(Kind k, const std::shared_ptr<T>& ref, size_t a, size_t b) { Op& o = add(k); o.p = ref; o.n[0] = a; o.n[1] = b; }
+
+ public:
+ DeferredRenderCommandEncoder(std::shared_ptr<PrivateCommandBuffer> cb, const RenderPassDescriptor& d):
+ _state(std::make_shared<State>(std::move(cb), d)), _rec(acquireBuffers()), _r(_rec.get()) {}
+
+ ~DeferredRenderCommandEncoder() override { endEncoding(); }
+
+ void endEncoding() override {
+ if (_ended) return;
+ _ended = true;
+ flush(true);
+ _state.reset();
+ }
+
+ void setRenderPipelineState(const std::shared_ptr<RenderPipelineState>& s) override { addRef(Kind::SetPipeline, s, 0, 0); }
+ void setFrontFacingWinding(Winding w) override { add(Kind::FrontFacing).e1 = static_cast<uint32_t>(w); }
+ void setCullMode(CullMode m) override { add(Kind::CullMode).e1 = static_cast<uint32_t>(m); }
+ void setDepthBias(float bias, float slope, float clamp) override { Op& o = add(Kind::DepthBias); o.f[0] = bias; o.f[1] = slope; o.f[2] = clamp; }
+ void setDepthClipMode(DepthClipMode m) override { add(Kind::DepthClip).e1 = static_cast<uint32_t>(m); }
+ void setViewport(const Viewport& v) override { setViewports(&v, 1); }
+ void setViewports(const Viewport* v, size_t count) override {
+ Op& o = add(Kind::Viewports); o.n[0] = _r->viewports.size(); o.n[1] = count;
+ _r->viewports.insert(_r->viewports.end(), v, v + count);
+ }
+ void setViewports(const std::vector<Viewport>& v) override { setViewports(v.data(), v.size()); }
+ void setScissorRect(const ScissorRect& s) override { setScissorRects(&s, 1); }
+ void setScissorRects(const ScissorRect* s, size_t count) override {
+ Op& o = add(Kind::Scissors); o.n[0] = _r->scissors.size(); o.n[1] = count;
+ _r->scissors.insert(_r->scissors.end(), s, s + count);
+ }
+ void setScissorRects(const std::vector<ScissorRect>& s) override { setScissorRects(s.data(), s.size()); }
+ void setBlendColor(float r, float g, float b, float a) override { Op& o = add(Kind::BlendColor); o.f[0] = r; o.f[1] = g; o.f[2] = b; o.f[3] = a; }
+ void setDepthStencilState(const std::shared_ptr<DepthStencilState>& s) override { addRef(Kind::DepthStencil, s, 0, 0); }
+ void setTriangleFillMode(TriangleFillMode m) override { add(Kind::FillMode).e1 = static_cast<uint32_t>(m); }
+ void setStencilReferenceValue(uint32_t v) override { add(Kind::StencilRef).e1 = v; }
+ void setStencilReferenceValue(uint32_t front, uint32_t back) override { Op& o = add(Kind::StencilRefFB); o.e1 = front; o.e2 = back; }
+ void setVisibilityResultMode(VisibilityResultMode m, size_t offset) override { Op& o = add(Kind::VisibilityMode); o.e1 = static_cast<uint32_t>(m); o.n[0] = offset; }
+
+ void drawPrimitives(PrimitiveType t, size_t start, size_t count, size_t instances, size_t baseInstance) override {
+ Op& o = add(Kind::DrawPrim); o.e1 = static_cast<uint32_t>(t); o.n[0] = start; o.n[1] = count; o.n[2] = instances; o.n[3] = baseInstance;
+ }
+ void drawPrimitives(PrimitiveType t, size_t start, size_t count, size_t instances) override { drawPrimitives(t, start, count, instances, 0); }
+ void drawPrimitives(PrimitiveType t, size_t start, size_t count) override { drawPrimitives(t, start, count, 1, 0); }
+
+ void drawIndexedPrimitives(PrimitiveType t, size_t indexCount, IndexType it, const std::shared_ptr<Buffer>& indexBuffer, size_t indexBufferOffset,
+ size_t instanceCount, int64_t baseVertex, size_t baseInstance) override {
+ Op& o = add(Kind::DrawIdx);
+ o.e1 = static_cast<uint32_t>(t); o.e2 = static_cast<uint32_t>(it); o.p = indexBuffer;
+ o.n[0] = indexCount; o.n[1] = indexBufferOffset; o.n[2] = instanceCount; o.n[3] = static_cast<size_t>(baseVertex); o.n[4] = baseInstance;
+ }
+ void drawIndexedPrimitives(PrimitiveType t, size_t indexCount, IndexType it, const std::shared_ptr<Buffer>& b, size_t off, size_t instances) override {
+ drawIndexedPrimitives(t, indexCount, it, b, off, instances, 0, 0);
+ }
+ void drawIndexedPrimitives(PrimitiveType t, size_t indexCount, IndexType it, const std::shared_ptr<Buffer>& b, size_t off) override {
+ drawIndexedPrimitives(t, indexCount, it, b, off, 1, 0, 0);
+ }
+
+ void setVertexBytes(const void* bytes, size_t length, size_t index) override {
+ Op& o = add(Kind::VBytes); o.n[0] = _r->bytes.size(); o.n[1] = length; o.n[2] = index;
+ const auto* p = static_cast<const uint8_t*>(bytes);
+ _r->bytes.insert(_r->bytes.end(), p, p + length);
+ }
+ void setVertexBuffer(const std::shared_ptr<Buffer>& b, size_t offset, size_t index) override { addRef(Kind::VBuffer, b, offset, index); }
+ void setVertexBuffers(const std::vector<std::shared_ptr<Buffer>>& b, const std::vector<size_t>& o, Range<size_t> r) override {
+ for (size_t i = 0; i < r.length; ++i) addRef(Kind::VBuffer, b[i], o[i], r.start + i);
+ }
+ void setVertexBufferOffset(size_t offset, size_t index) override { Op& o = add(Kind::VBufferOffset); o.n[0] = offset; o.n[1] = index; }
+ void setVertexSamplerState(const std::shared_ptr<SamplerState>& s, size_t index) override { addRef(Kind::VSampler, s, index, 0); }
+ void setVertexSamplerState(const std::shared_ptr<SamplerState>& s, float lo, float hi, size_t index) override {
+ Op& o = add(Kind::VSamplerClamp); o.p = s; o.f[0] = lo; o.f[1] = hi; o.n[0] = index;
+ }
+ void setVertexSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& s, Range<size_t> r) override {
+ for (size_t i = 0; i < r.length; ++i) setVertexSamplerState(s[i], r.start + i);
+ }
+ void setVertexSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& s, const std::vector<float>& lo, const std::vector<float>& hi, Range<size_t> r) override {
+ for (size_t i = 0; i < r.length; ++i) setVertexSamplerState(s[i], lo[i], hi[i], r.start + i);
+ }
+ void setVertexTexture(const std::shared_ptr<Texture>& t, size_t index) override { addRef(Kind::VTexture, t, index, 0); }
+ void setVertexTextures(const std::vector<std::shared_ptr<Texture>>& t, Range<size_t> r) override {
+ for (size_t i = 0; i < r.length; ++i) addRef(Kind::VTexture, t[i], r.start + i, 0);
+ }
+
+ void setFragmentBytes(const void* bytes, size_t length, size_t index) override {
+ Op& o = add(Kind::FBytes); o.n[0] = _r->bytes.size(); o.n[1] = length; o.n[2] = index;
+ const auto* p = static_cast<const uint8_t*>(bytes);
+ _r->bytes.insert(_r->bytes.end(), p, p + length);
+ }
+ void setFragmentBuffer(const std::shared_ptr<Buffer>& b, size_t offset, size_t index) override { addRef(Kind::FBuffer, b, offset, index); }
+ void setFragmentBuffers(const std::vector<std::shared_ptr<Buffer>>& b, const std::vector<size_t>& o, Range<size_t> r) override {
+ for (size_t i = 0; i < r.length; ++i) addRef(Kind::FBuffer, b[i], o[i], r.start + i);
+ }
+ void setFragmentBufferOffset(size_t offset, size_t index) override { Op& o = add(Kind::FBufferOffset); o.n[0] = offset; o.n[1] = index; }
+ void setFragmentSamplerState(const std::shared_ptr<SamplerState>& s, size_t index) override { addRef(Kind::FSampler, s, index, 0); }
+ void setFragmentSamplerState(const std::shared_ptr<SamplerState>& s, float lo, float hi, size_t index) override {
+ Op& o = add(Kind::FSamplerClamp); o.p = s; o.f[0] = lo; o.f[1] = hi; o.n[0] = index;
+ }
+ void setFragmentSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& s, Range<size_t> r) override {
+ for (size_t i = 0; i < r.length; ++i) setFragmentSamplerState(s[i], r.start + i);
+ }
+ void setFragmentSamplerStates(const std::vector<std::shared_ptr<SamplerState>>& s, const std::vector<float>& lo, const std::vector<float>& hi, Range<size_t> r) override {
+ for (size_t i = 0; i < r.length; ++i) setFragmentSamplerState(s[i], lo[i], hi[i], r.start + i);
+ }
+ void setFragmentTexture(const std::shared_ptr<Texture>& t, size_t index) override { addRef(Kind::FTexture, t, index, 0); }
+ void setFragmentTextures(std::vector<std::shared_ptr<Texture>>& t, Range<size_t> r) override {
+ for (size_t i = 0; i < r.length; ++i) addRef(Kind::FTexture, t[i], r.start + i, 0);
+ }
+
+ void useResource(std::shared_ptr<Resource> res, ResourceUsage usage, RenderStages stages) override {
+ Op& o = add(Kind::UseResourceStages); o.p = std::move(res); o.e1 = static_cast<uint32_t>(usage); o.n[0] = static_cast<size_t>(stages);
+ }
+ void useResources(const std::vector<std::shared_ptr<Resource>>& res, ResourceUsage usage, RenderStages stages) override {
+ // one op per resource: the real encoder batches barriers per call, which only affects how they are grouped
+ for (const auto& r : res) useResource(r, usage, stages);
+ }
+ void useResource(std::shared_ptr<Resource> res, ResourceUsage usage) override {
+ Op& o = add(Kind::UseResource); o.p = std::move(res); o.e1 = static_cast<uint32_t>(usage);
+ }
+ void useResources(const std::vector<std::shared_ptr<Resource>>& res, ResourceUsage usage) override {
+ for (const auto& r : res) useResource(r, usage);
+ }
+ void memoryBarrier(size_t scope, RenderStages after, RenderStages before) override {
+ Op& o = add(Kind::MemoryBarrier); o.n[0] = scope; o.n[1] = static_cast<size_t>(after); o.n[2] = static_cast<size_t>(before);
+ }
+ };
+ }
+}
diff --git a/private-include/indium/gpu-timing.private.hpp b/private-include/indium/gpu-timing.private.hpp
new file mode 100644
--- /dev/null
+++ b/private-include/indium/gpu-timing.private.hpp
@@ -0,0 +1,259 @@
+#pragma once
+
+// Diagnostic GPU profiler built on Vulkan timestamp queries (guest flag files, checked once at start):
+// /tmp/metal-gpu-timing per render pass: start/end on the GPU timeline -> /tmp/indium-gpu-timing.log
+// line per command buffer: "cb <WxH>:<draws>:<ms>:<start_ns> ..."
+// /tmp/metal-gpu-timing-draws additionally a timestamp after every draw; GPU time is attributed to the pipeline
+// state (vertex+fragment function names in /tmp/indium-gpu-pso.log) and summed per
+// (pso, target size) into /tmp/indium-gpu-draws.log (rewritten every 5 s)
+// Header-only, function-local statics: no class gains a member (QuartzCore/Metal read Indium objects at fixed offsets).
+
+#include <indium/dynamic-vk.hpp>
+
+#include <chrono>
+#include <cstdint>
+#include <cstdio>
+#include <map>
+#include <mutex>
+#include <string>
+#include <tuple>
+#include <unordered_map>
+#include <vector>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+namespace Indium {
+ namespace GpuTiming {
+ struct Pass {
+ uint32_t first;
+ uint32_t width;
+ uint32_t height;
+ uint32_t draws;
+ std::vector<uint64_t> drawPso; // pso id per draw timestamp, in order
+ };
+
+ struct State {
+ static constexpr uint32_t Capacity = 1u << 20; // 8 MB: ~1700 passes of in-flight history, no wrap hazard
+ static constexpr uint32_t DrawSlots = 600;
+ std::mutex mutex;
+ VkQueryPool pool = VK_NULL_HANDLE;
+ uint32_t next = 0;
+ std::unordered_map<VkCommandBuffer, std::vector<Pass>> passes;
+ std::map<std::tuple<uint64_t, uint32_t, uint32_t>, std::pair<double, uint64_t>> perPso; // ms, draws
+ std::map<std::tuple<uint64_t, uint32_t, uint32_t>, std::pair<double, uint64_t>> perPsoSingle; // passes with one draw
+ std::chrono::steady_clock::time_point lastDump = std::chrono::steady_clock::now();
+ FILE* log = nullptr;
+ bool enabled = access("/tmp/metal-gpu-timing", F_OK) == 0;
+ bool drawLevel = access("/tmp/metal-gpu-timing-draws", F_OK) == 0;
+ };
+
+ inline State& state() {
+ static State* instance = new State;
+ return *instance;
+ }
+
+ inline bool enabled() { return state().enabled; }
+
+ inline thread_local uint32_t currentDraws = 0;
+
+ inline void countDraw() {
+ if (state().enabled) ++currentDraws;
+ }
+
+ // SPIR-V library identity: FNV-1a over the module, and (draw-level mode) the module saved as <hash>.spv
+ inline std::unordered_map<const void*, std::string>& libraryIds() {
+ static auto* ids = new std::unordered_map<const void*, std::string>;
+ return *ids;
+ }
+
+ inline void noteLibrary(const void* library, const char* data, size_t length) {
+ auto& s = state();
+ if (!s.enabled || !s.drawLevel) return;
+ uint64_t h = 1469598103934665603ull;
+ for (size_t i = 0; i < length; ++i) { h ^= (unsigned char) data[i]; h *= 1099511628211ull; }
+ char id[48];
+ snprintf(id, sizeof(id), "%08llx:%zu", (unsigned long long) (h & 0xffffffffull), length);
+ {
+ std::lock_guard<std::mutex> lock(s.mutex);
+ libraryIds()[library] = id;
+ }
+ char path[96];
+ snprintf(path, sizeof(path), "/tmp/indium-shaders/%08llx.spv", (unsigned long long) (h & 0xffffffffull));
+ mkdir("/tmp/indium-shaders", 0755);
+ if (access(path, F_OK) != 0) {
+ if (FILE* f = fopen(path, "wb")) { fwrite(data, 1, length, f); fclose(f); }
+ }
+ }
+
+ inline std::string libraryId(const void* library) {
+ auto& s = state();
+ std::lock_guard<std::mutex> lock(s.mutex);
+ auto it = libraryIds().find(library);
+ return it == libraryIds().end() ? std::string("-") : it->second;
+ }
+
+ inline void notePipeline(const void* pso, const std::string& vertex, const std::string& fragment) {
+ auto& s = state();
+ if (!s.enabled || !s.drawLevel) return;
+ if (FILE* f = fopen("/tmp/indium-gpu-pso.log", "a")) {
+ fprintf(f, "%llx %s %s\n", (unsigned long long) (uintptr_t) pso, vertex.c_str(), fragment.c_str());
+ fclose(f);
+ }
+ }
+
+ // before vkCmdBeginRenderPass (outside any render pass)
+ inline void beginPass(VkDevice device, VkCommandBuffer cb, uint32_t width, uint32_t height) {
+ auto& s = state();
+ if (!s.enabled) return;
+ std::lock_guard<std::mutex> lock(s.mutex);
+ if (s.pool == VK_NULL_HANDLE) {
+ VkQueryPoolCreateInfo info {};
+ info.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO;
+ info.queryType = VK_QUERY_TYPE_TIMESTAMP;
+ info.queryCount = State::Capacity;
+ if (DynamicVK::vkCreateQueryPool(device, &info, nullptr, &s.pool) != VK_SUCCESS) {
+ s.enabled = false;
+ return;
+ }
+ }
+ uint32_t block = 2 + (s.drawLevel ? State::DrawSlots : 0);
+ if (s.next + block > State::Capacity) s.next = 0;
+ uint32_t first = s.next;
+ s.next += block;
+ DynamicVK::vkCmdResetQueryPool(cb, s.pool, first, block);
+ DynamicVK::vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, s.pool, first);
+ s.passes[cb].push_back(Pass { first, width, height, 0, {} });
+ currentDraws = 0;
+ }
+
+ // render target configuration of every pass: "<WxH> c<n>=<format>/<load>/<store>... d=<format>/<load>/<store>" -> count,
+ // rewritten to /tmp/indium-gpu-targets.log every 5 s (pixel format/action numbers are Indium enum values)
+ inline void noteTarget(const std::string& key) {
+ auto& s = state();
+ if (!s.enabled) return;
+ static auto* counts = new std::map<std::string, uint64_t>;
+ static auto last = std::chrono::steady_clock::now();
+ std::lock_guard<std::mutex> lock(s.mutex);
+ ++(*counts)[key];
+ auto now = std::chrono::steady_clock::now();
+ if (now - last < std::chrono::seconds(5)) return;
+ last = now;
+ if (FILE* f = fopen("/tmp/indium-gpu-targets.log", "w")) {
+ for (const auto& [k, v] : *counts) fprintf(f, "%llu %s\n", (unsigned long long) v, k.c_str());
+ fclose(f);
+ }
+ }
+
+ // whole command buffer: logged like a pass with size 3x3 (first pass of the command buffer, end written at commit)
+ inline void beginSpan(VkDevice device, VkCommandBuffer cb) { beginPass(device, cb, 3, 3); }
+
+ inline void endSpan(VkCommandBuffer cb) {
+ auto& s = state();
+ if (!s.enabled) return;
+ std::lock_guard<std::mutex> lock(s.mutex);
+ auto it = s.passes.find(cb);
+ if (it == s.passes.end() || it->second.empty() || it->second.front().width != 3) return;
+ DynamicVK::vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, s.pool, it->second.front().first + 1);
+ }
+
+ // after each vkCmdDraw / vkCmdDrawIndexed
+ inline void afterDraw(VkCommandBuffer cb, uint64_t pso) {
+ auto& s = state();
+ if (!s.enabled || !s.drawLevel) return;
+ std::lock_guard<std::mutex> lock(s.mutex);
+ auto it = s.passes.find(cb);
+ if (it == s.passes.end() || it->second.empty()) return;
+ auto& pass = it->second.back();
+ if (pass.drawPso.size() >= State::DrawSlots - 1) return;
+ DynamicVK::vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, s.pool, pass.first + 2 + (uint32_t) pass.drawPso.size());
+ pass.drawPso.push_back(pso);
+ }
+
+ // after vkCmdEndRenderPass
+ inline void endPass(VkCommandBuffer cb) {
+ auto& s = state();
+ if (!s.enabled) return;
+ std::lock_guard<std::mutex> lock(s.mutex);
+ auto it = s.passes.find(cb);
+ if (it == s.passes.end() || it->second.empty()) return;
+ auto& pass = it->second.back();
+ pass.draws = currentDraws;
+ DynamicVK::vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, s.pool, pass.first + 1);
+ }
+
+ inline void dumpDraws(State& s) {
+ if (!s.drawLevel) return;
+ auto now = std::chrono::steady_clock::now();
+ if (now - s.lastDump < std::chrono::seconds(5)) return;
+ s.lastDump = now;
+ if (FILE* f = fopen("/tmp/indium-gpu-draws.log", "w")) {
+ for (const auto& [key, v] : s.perPso)
+ fprintf(f, "%llx %ux%u %llu %.4f\n", (unsigned long long) std::get<0>(key), std::get<1>(key), std::get<2>(key),
+ (unsigned long long) v.second, v.first);
+ fclose(f);
+ }
+ if (FILE* f = fopen("/tmp/indium-gpu-single.log", "w")) {
+ for (const auto& [key, v] : s.perPsoSingle)
+ fprintf(f, "%llx %ux%u %llu %.4f\n", (unsigned long long) std::get<0>(key), std::get<1>(key), std::get<2>(key),
+ (unsigned long long) v.second, v.first);
+ fclose(f);
+ }
+ }
+
+ // when the command buffer has completed on the GPU
+ inline void collect(VkDevice device, VkCommandBuffer cb, double periodNs) {
+ auto& s = state();
+ if (!s.enabled) return;
+ std::vector<Pass> passes;
+ {
+ std::lock_guard<std::mutex> lock(s.mutex);
+ auto it = s.passes.find(cb);
+ if (it == s.passes.end()) return;
+ passes = std::move(it->second);
+ s.passes.erase(it);
+ }
+ if (passes.empty()) return;
+ std::string line;
+ char buf[96];
+ std::vector<uint64_t> t;
+ std::vector<std::tuple<uint64_t, uint32_t, uint32_t, double>> samples; // pso, w, h, ms
+ std::vector<std::tuple<uint64_t, uint32_t, uint32_t, double>> singles;
+ for (const auto& pass : passes) {
+ uint32_t n = 2 + (uint32_t) pass.drawPso.size();
+ t.assign(n, 0);
+ // the mutex is NOT held while waiting for the GPU: the render thread takes it for every pass and draw
+ if (DynamicVK::vkGetQueryPoolResults(device, s.pool, pass.first, n, n * sizeof(uint64_t), t.data(), sizeof(uint64_t),
+ VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT) != VK_SUCCESS) continue;
+ snprintf(buf, sizeof(buf), " %ux%u:%u:%.3f:%.0f", pass.width, pass.height, pass.draws,
+ (double) (t[1] - t[0]) * periodNs / 1e6, (double) t[0] * periodNs);
+ line += buf;
+ uint64_t previous = t[0];
+ for (size_t i = 0; i < pass.drawPso.size(); ++i) {
+ uint64_t now = t[2 + i];
+ double ms = now > previous ? (double) (now - previous) * periodNs / 1e6 : 0;
+ previous = now > previous ? now : previous;
+ samples.emplace_back(pass.drawPso[i], pass.width, pass.height, ms);
+ if (pass.drawPso.size() == 1) singles.emplace_back(pass.drawPso[i], pass.width, pass.height, (double) (t[1] - t[0]) * periodNs / 1e6);
+ }
+ }
+ std::lock_guard<std::mutex> lock(s.mutex);
+ for (const auto& [pso, w, h, ms] : samples) {
+ auto& entry = s.perPso[std::make_tuple(pso, w, h)];
+ entry.first += ms;
+ entry.second += 1;
+ }
+ for (const auto& [pso, w, h, ms] : singles) {
+ auto& entry = s.perPsoSingle[std::make_tuple(pso, w, h)];
+ entry.first += ms;
+ entry.second += 1;
+ }
+ if (!s.log) s.log = fopen("/tmp/indium-gpu-timing.log", "a");
+ if (s.log) {
+ fprintf(s.log, "cb%s\n", line.c_str());
+ fflush(s.log);
+ }
+ dumpDraws(s);
+ }
+ }
+}
diff --git a/private-include/indium/memory-pool.private.hpp b/private-include/indium/memory-pool.private.hpp
new file mode 100644
--- /dev/null
+++ b/private-include/indium/memory-pool.private.hpp
@@ -0,0 +1,233 @@
+#pragma once
+
+// Suballocation of device memory for buffers and textures.
+//
+// Every buffer and texture used to own a VkDeviceMemory. RADV hands the kernel the complete list of a device's memory
+// objects with every submission (required once buffer device addresses are in use), so vkQueueSubmit2 cost grew with the
+// number of live resources: about 1.1 ms per frame in League, 77% of it in amdgpu_cs_ioctl walking that list.
+// Resources up to MaxPooledSize now share 32 MB blocks; larger ones keep a dedicated allocation.
+//
+// Header-only on purpose (function-local statics, one instance per libindium) and keyed by the Vulkan handle of the
+// owning buffer/image, so no class gains a member: QuartzCore and Metal read some Indium objects at fixed offsets.
+// Create guest /tmp/metal-no-pool before the process starts to get one allocation per resource again.
+
+#include <indium/dynamic-vk.hpp>
+
+#include <chrono>
+#include <cstdint>
+#include <cstdlib>
+#include <deque>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+#include <vector>
+#include <unistd.h>
+
+namespace Indium {
+ namespace MemoryPool {
+ struct Allocation {
+ VkDeviceMemory memory = VK_NULL_HANDLE;
+ VkDeviceSize offset = 0;
+ VkDeviceSize size = 0;
+ };
+
+ namespace Detail {
+ static constexpr VkDeviceSize BlockSize = 32ull << 20;
+ static constexpr VkDeviceSize MaxPooledSize = 4ull << 20;
+ static constexpr VkDeviceSize Granule = 256;
+ // a freed range is not handed out again before this long: work already submitted to the GPU may still read it
+ static constexpr std::chrono::milliseconds Quarantine {1000};
+
+ struct Block {
+ VkDeviceMemory memory = VK_NULL_HANDLE;
+ VkDeviceSize size = 0;
+ uint32_t key = 0;
+ bool dedicated = false;
+ void* mapped = nullptr;
+ size_t live = 0;
+ std::map<VkDeviceSize, VkDeviceSize> free; // offset -> size
+ };
+
+ struct Entry {
+ std::shared_ptr<Block> block;
+ VkDeviceSize offset;
+ VkDeviceSize size;
+ };
+
+ struct Pending {
+ std::chrono::steady_clock::time_point when;
+ std::shared_ptr<Block> block;
+ VkDeviceSize offset;
+ VkDeviceSize size;
+ };
+
+ struct State {
+ std::mutex mutex;
+ std::unordered_map<uint64_t, Entry> entries;
+ std::unordered_map<uint32_t, std::vector<std::shared_ptr<Block>>> blocks;
+ std::deque<Pending> pending;
+ bool disabled = access("/tmp/metal-no-pool", F_OK) == 0;
+ };
+
+ inline State& state() {
+ static State* instance = new State; // never destroyed: resources are released during process teardown
+ return *instance;
+ }
+
+ inline VkDeviceSize alignUp(VkDeviceSize value, VkDeviceSize alignment) {
+ return (value + alignment - 1) / alignment * alignment;
+ }
+
+ inline VkDeviceMemory allocateMemory(VkDevice device, uint32_t typeIndex, VkDeviceSize size, bool deviceAddress) {
+ VkMemoryAllocateInfo info {};
+ info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
+ info.allocationSize = size;
+ info.memoryTypeIndex = typeIndex;
+ VkMemoryAllocateFlagsInfo flags {};
+ flags.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO;
+ flags.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT;
+ if (deviceAddress) info.pNext = &flags;
+ VkDeviceMemory memory = VK_NULL_HANDLE;
+ if (DynamicVK::vkAllocateMemory(device, &info, nullptr, &memory) != VK_SUCCESS) abort();
+ return memory;
+ }
+
+ inline void releaseRange(State& s, VkDevice device, const std::shared_ptr<Block>& block, VkDeviceSize offset, VkDeviceSize size) {
+ auto next = block->free.lower_bound(offset);
+ if (next != block->free.begin()) {
+ auto previous = std::prev(next);
+ if (previous->first + previous->second == offset) {
+ offset = previous->first;
+ size += previous->second;
+ block->free.erase(previous);
+ }
+ }
+ if (next != block->free.end() && offset + size == next->first) {
+ size += next->second;
+ block->free.erase(next);
+ }
+ block->free[offset] = size;
+ if (block->live != 0 || size != block->size) return;
+ // the block is empty: keep one spare per kind, give the others back
+ auto& list = s.blocks[block->key];
+ size_t empty = 0;
+ for (const auto& other : list) {
+ if (other->live == 0 && other->free.size() == 1 && other->free.begin()->second == other->size) ++empty;
+ }
+ if (empty <= 1) return;
+ for (auto it = list.begin(); it != list.end(); ++it) {
+ if (*it == block) { list.erase(it); break; }
+ }
+ DynamicVK::vkFreeMemory(device, block->memory, nullptr);
+ block->memory = VK_NULL_HANDLE;
+ }
+
+ inline void drain(State& s, VkDevice device) {
+ auto now = std::chrono::steady_clock::now();
+ while (!s.pending.empty() && now - s.pending.front().when >= Quarantine) {
+ Pending item = std::move(s.pending.front());
+ s.pending.pop_front();
+ if (item.block->dedicated) {
+ DynamicVK::vkFreeMemory(device, item.block->memory, nullptr);
+ } else {
+ --item.block->live;
+ releaseRange(s, device, item.block, item.offset, item.size);
+ }
+ }
+ }
+ }
+
+ // `owner` is the buffer or image handle the memory will be bound to
+ inline Allocation allocate(VkDevice device, uint64_t owner, uint32_t typeIndex, VkDeviceSize size, VkDeviceSize alignment, bool image, bool deviceAddress) {
+ using namespace Detail;
+ auto& s = state();
+ std::lock_guard<std::mutex> lock(s.mutex);
+ drain(s, device);
+ Entry entry;
+ if (s.disabled || size > MaxPooledSize) {
+ entry.block = std::make_shared<Block>();
+ entry.block->dedicated = true;
+ entry.block->size = size;
+ entry.block->memory = allocateMemory(device, typeIndex, size, deviceAddress);
+ entry.offset = 0;
+ entry.size = size;
+ } else {
+ if (alignment < Granule) alignment = Granule;
+ size = alignUp(size, Granule);
+ const uint32_t key = (typeIndex << 2) | (image ? 2u : 0u) | (deviceAddress ? 1u : 0u);
+ auto& list = s.blocks[key];
+ for (int attempt = 0; attempt < 2 && !entry.block; ++attempt) {
+ for (auto& block : list) {
+ for (auto it = block->free.begin(); it != block->free.end(); ++it) {
+ VkDeviceSize start = alignUp(it->first, alignment);
+ VkDeviceSize end = it->first + it->second;
+ if (start + size > end) continue;
+ VkDeviceSize rangeStart = it->first;
+ block->free.erase(it);
+ if (start > rangeStart) block->free[rangeStart] = start - rangeStart;
+ if (start + size < end) block->free[start + size] = end - (start + size);
+ entry.block = block;
+ entry.offset = start;
+ entry.size = size;
+ ++block->live;
+ break;
+ }
+ if (entry.block) break;
+ }
+ if (!entry.block && attempt == 0) {
+ auto block = std::make_shared<Block>();
+ block->key = key;
+ block->size = BlockSize;
+ block->memory = allocateMemory(device, typeIndex, BlockSize, deviceAddress);
+ block->free[0] = BlockSize;
+ list.push_back(block);
+ }
+ }
+ if (!entry.block) abort();
+ }
+ s.entries[owner] = entry;
+ return Allocation { entry.block->memory, entry.offset, entry.size };
+ }
+
+ inline bool lookup(uint64_t owner, Allocation& result) {
+ auto& s = Detail::state();
+ std::lock_guard<std::mutex> lock(s.mutex);
+ auto it = s.entries.find(owner);
+ if (it == s.entries.end()) return false;
+ result = Allocation { it->second.block->memory, it->second.offset, it->second.size };
+ return true;
+ }
+
+ // host pointer to the start of the owner's memory (the memory type must be host-visible)
+ inline void* map(VkDevice device, uint64_t owner) {
+ auto& s = Detail::state();
+ std::lock_guard<std::mutex> lock(s.mutex);
+ auto it = s.entries.find(owner);
+ if (it == s.entries.end()) return nullptr;
+ auto& block = it->second.block;
+ if (!block->mapped) {
+ if (DynamicVK::vkMapMemory(device, block->memory, 0, VK_WHOLE_SIZE, 0, &block->mapped) != VK_SUCCESS) abort();
+ }
+ return static_cast<char*>(block->mapped) + it->second.offset;
+ }
+
+ // call after the owning buffer/image has been destroyed
+ inline void free(VkDevice device, uint64_t owner) {
+ using namespace Detail;
+ auto& s = state();
+ std::lock_guard<std::mutex> lock(s.mutex);
+ auto it = s.entries.find(owner);
+ if (it == s.entries.end()) return;
+ Entry entry = std::move(it->second);
+ s.entries.erase(it);
+ if (entry.block->dedicated) {
+ // nothing can reuse a dedicated allocation, and the kernel keeps it alive for work in flight
+ DynamicVK::vkFreeMemory(device, entry.block->memory, nullptr);
+ } else {
+ s.pending.push_back(Pending { std::chrono::steady_clock::now(), entry.block, entry.offset, entry.size });
+ }
+ drain(s, device);
+ }
+ }
+}
diff --git a/private-include/indium/outline-chain.private.hpp b/private-include/indium/outline-chain.private.hpp
new file mode 100644
--- /dev/null
+++ b/private-include/indium/outline-chain.private.hpp
@@ -0,0 +1,124 @@
+#pragma once
+
+// League's hover/target outline is a chain of full-screen 1080p passes that ping-pong between two textures: a mask pass,
+// then N passes with one fragment shader (SPIR-V hash 87819980) and M passes with a second (f61b5318), then a composite.
+// Every pass reads the previous result, so a pass can be dropped without breaking the data flow: the next pass simply reads
+// a result with fewer iterations applied. This keeps only the first 2*K passes of the first phase of each chain.
+//
+// Live settings (guest files, re-read every 2048 draws; create them from inside the guest):
+// /tmp/metal-outline-pairs K1: passes of the first phase kept = 2*K1 (default 0; 16 keeps everything)
+// /tmp/metal-outline-pairs2 K2: passes of the second phase kept = 2*K2 (default 16 = all of them)
+// Header-only with function-local statics (no class gains a member).
+
+#include <atomic>
+#include <cstdint>
+#include <mutex>
+#include <unordered_map>
+#include <unistd.h>
+#include <cstdio>
+
+namespace Indium {
+ namespace OutlineChain {
+ struct State {
+ std::mutex mutex;
+ std::unordered_map<const void*, uint32_t> libraryHash;
+ std::unordered_map<const void*, uint8_t> pipelineKind; // 1 = first phase shader, 2 = second phase shader
+ };
+
+ inline State& state() {
+ static State* instance = new State;
+ return *instance;
+ }
+
+ inline void noteLibrary(const void* library, const char* data, size_t length) {
+ uint64_t h = 1469598103934665603ull;
+ for (size_t i = 0; i < length; ++i) { h ^= (unsigned char) data[i]; h *= 1099511628211ull; }
+ auto& s = state();
+ std::lock_guard<std::mutex> lock(s.mutex);
+ s.libraryHash[library] = (uint32_t) (h & 0xffffffffull);
+ }
+
+ inline void notePipeline(const void* pipeline, const void* fragmentLibrary) {
+ if (!fragmentLibrary) return;
+ auto& s = state();
+ std::lock_guard<std::mutex> lock(s.mutex);
+ auto it = s.libraryHash.find(fragmentLibrary);
+ if (it == s.libraryHash.end()) return;
+ if (it->second == 0x87819980u) s.pipelineKind[pipeline] = 1;
+ else if (it->second == 0xf61b5318u) s.pipelineKind[pipeline] = 2;
+ }
+
+ inline void readSettings(int& k1, int& k2) {
+ static std::atomic<int> v1 {0}, v2 {16};
+ static std::atomic<unsigned> calls {0};
+ if ((calls.fetch_add(1) & 2047) == 0) { // each read is two fopen()s through Darling's path translation
+ auto read = [](const char* path, int fallback) {
+ int out = fallback;
+ if (FILE* f = fopen(path, "r")) {
+ int v = -1;
+ if (fscanf(f, "%d", &v) == 1 && v >= 0 && v <= 16) out = v;
+ fclose(f);
+ }
+ return out;
+ };
+ v1.store(read("/tmp/metal-outline-pairs", 0));
+ v2.store(read("/tmp/metal-outline-pairs2", 16));
+ }
+ k1 = v1.load();
+ k2 = v2.load();
+ }
+
+ inline void report(int k, uint8_t kind, bool skipped, size_t vertexCount) {
+ static std::atomic<unsigned long> draws {0}, kind1 {0}, kind1v4 {0}, kind2 {0}, skippedDraws {0};
+ static std::atomic<unsigned> tick {0};
+ ++draws;
+ if (kind == 1) ++kind1;
+ if (kind == 1 && vertexCount == 4) ++kind1v4;
+ if (kind == 2) ++kind2;
+ if (skipped) ++skippedDraws;
+ if ((tick.fetch_add(1) & 1023) == 0) {
+ auto& s = state();
+ size_t kinds = 0, libs = 0;
+ { std::lock_guard<std::mutex> lock(s.mutex); kinds = s.pipelineKind.size(); libs = s.libraryHash.size(); }
+ char path[64];
+ snprintf(path, sizeof(path), "/tmp/indium-outline-%d.log", (int) getpid());
+ if (FILE* f = fopen(path, "w")) {
+ fprintf(f, "k=%d draws=%lu kind1=%lu kind1_v4=%lu kind2=%lu skipped=%lu pipelineKinds=%zu libraries=%zu\n", k,
+ draws.load(), kind1.load(), kind1v4.load(), kind2.load(), skippedDraws.load(), kinds, libs);
+ fclose(f);
+ }
+ }
+ }
+
+ // called for every drawPrimitives; true: skip this draw
+ inline bool shouldSkipDraw(const void* pipeline, size_t vertexCount) {
+ static thread_local unsigned run1 = 0, run2 = 0;
+ int k1 = 0, k2 = 16;
+ readSettings(k1, k2);
+ uint8_t kind = 0;
+ {
+ auto& s = state();
+ std::lock_guard<std::mutex> lock(s.mutex);
+ auto it = s.pipelineKind.find(pipeline);
+ if (it != s.pipelineKind.end()) kind = it->second;
+ }
+ if (vertexCount != 4 || kind == 0) {
+ // any other draw ends the chain
+ run1 = run2 = 0;
+ return false;
+ }
+ bool skip;
+ if (kind == 1) {
+ // a new chain starts with its first-phase passes
+ if (run2 != 0) { run1 = run2 = 0; }
+ skip = run1 >= (unsigned) (2 * k1);
+ ++run1;
+ } else {
+ skip = run2 >= (unsigned) (2 * k2);
+ ++run2;
+ }
+ // report(k1, kind, skip, vertexCount); // diagnostic: rewrote a guest log file every 1024 draws
+ return skip;
+ }
+ }
+}
diff --git a/private-include/indium/render-pass-key.private.hpp b/private-include/indium/render-pass-key.private.hpp
new file mode 100644
--- /dev/null
+++ b/private-include/indium/render-pass-key.private.hpp
@@ -0,0 +1,56 @@
+#pragma once
+#include <vulkan/vulkan.h>
+#include <array>
+#include <algorithm>
+#include <optional>
+
+namespace Indium {
+// Exact descriptions, rather than Vulkan compatibility, determine reuse: load
+// actions and layouts affect preservation of the game's attachment contents.
+struct RenderPassKey {
+ std::array<uint32_t, 256> words {};
+ size_t size = 0;
+ bool operator==(const RenderPassKey& other) const {
+ return size == other.size && std::equal(words.begin(), words.begin() + size, other.words.begin());
+ }
+ struct Hash {
+ size_t operator()(const RenderPassKey& key) const {
+ size_t result = 14695981039346656037ull;
+ for (size_t i = 0; i < key.size; ++i) result = (result ^ key.words[i]) * 1099511628211ull;
+ return result;
+ }
+ };
+};
+inline std::optional<RenderPassKey> renderPassKey(const VkRenderPassCreateInfo& info) {
+ // The encoder emits one subpass without input/resolve/preserve attachments.
+ // Unsupported structures retain the original uncached Vulkan path.
+ if (info.pNext || info.subpassCount != 1 || info.attachmentCount > 16 || info.dependencyCount > 2)
+ return std::nullopt;
+ const auto& subpass = info.pSubpasses[0];
+ if (subpass.inputAttachmentCount || subpass.pResolveAttachments || subpass.preserveAttachmentCount || subpass.colorAttachmentCount > 16)
+ return std::nullopt;
+ RenderPassKey key;
+ auto add = [&](uint32_t word) { key.words[key.size++] = word; };
+ add(info.flags); add(info.attachmentCount);
+ for (uint32_t i = 0; i < info.attachmentCount; ++i) {
+ const auto& a = info.pAttachments[i];
+ add(a.flags); add(a.format); add(a.samples); add(a.loadOp); add(a.storeOp);
+ add(a.stencilLoadOp); add(a.stencilStoreOp); add(a.initialLayout); add(a.finalLayout);
+ }
+ add(subpass.flags); add(subpass.pipelineBindPoint); add(subpass.colorAttachmentCount);
+ for (uint32_t i = 0; i < subpass.colorAttachmentCount; ++i) {
+ add(subpass.pColorAttachments[i].attachment); add(subpass.pColorAttachments[i].layout);
+ }
+ add(subpass.pDepthStencilAttachment != nullptr);
+ if (subpass.pDepthStencilAttachment) {
+ add(subpass.pDepthStencilAttachment->attachment); add(subpass.pDepthStencilAttachment->layout);
+ }
+ add(info.dependencyCount);
+ for (uint32_t i = 0; i < info.dependencyCount; ++i) {
+ const auto& d = info.pDependencies[i];
+ add(d.srcSubpass); add(d.dstSubpass); add(d.srcStageMask); add(d.dstStageMask);
+ add(d.srcAccessMask); add(d.dstAccessMask); add(d.dependencyFlags);
+ }
+ return key;
+}
+}
diff --git a/private-include/indium/resource-owner-cache.private.hpp b/private-include/indium/resource-owner-cache.private.hpp
new file mode 100644
--- /dev/null
+++ b/private-include/indium/resource-owner-cache.private.hpp
@@ -0,0 +1,62 @@
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <utility>
+#include <vector>
+
+namespace Indium {
+ // Encoder-local, insert-only cache. Contiguous owners avoid allocating and
+ // freeing a hash node for every resource. Owners are never erased while
+ // recorded GPU commands can reference them, so pointer identities stay valid.
+ template<class Base, class Private> class ResourceOwnerCache {
+ public:
+ struct Entry {
+ std::shared_ptr<Base> owner;
+ Private* value = nullptr;
+ bool depthFeedback = false;
+ };
+ private:
+ std::vector<Entry> _owners;
+ std::vector<size_t> _buckets; // owner index + 1; zero is an empty bucket
+
+ static size_t hash(Base* object) {
+ uint64_t value = reinterpret_cast<uintptr_t>(object);
+ value ^= value >> 30;
+ value *= UINT64_C(0xbf58476d1ce4e5b9);
+ value ^= value >> 27;
+ return static_cast<size_t>(value);
+ }
+ void grow() {
+ std::vector<size_t> buckets(_buckets.empty() ? 16 : _buckets.size() * 2, 0);
+ for (size_t i = 0; i < _owners.size(); ++i) {
+ size_t slot = hash(_owners[i].owner.get()) & (buckets.size() - 1);
+ while (buckets[slot]) slot = (slot + 1) & (buckets.size() - 1);
+ buckets[slot] = i + 1;
+ }
+ _buckets.swap(buckets);
+ }
+ public:
+ size_t size() const { return _owners.size(); }
+ // Returned pointers are valid until the next insertion, like vector
+ // elements. Callers copy the resolved pointer into their binding slot.
+ std::pair<Entry*, bool> acquire(const std::shared_ptr<Base>& owner) {
+ if (!owner) return {nullptr, false};
+ if (_buckets.empty()) grow();
+ size_t slot = hash(owner.get()) & (_buckets.size() - 1);
+ while (_buckets[slot]) {
+ auto& entry = _owners[_buckets[slot] - 1];
+ if (entry.owner.get() == owner.get()) return {&entry, false};
+ slot = (slot + 1) & (_buckets.size() - 1);
+ }
+ if ((_owners.size() + 1) * 2 > _buckets.size()) {
+ grow();
+ slot = hash(owner.get()) & (_buckets.size() - 1);
+ while (_buckets[slot]) slot = (slot + 1) & (_buckets.size() - 1);
+ }
+ _owners.push_back(Entry{owner, dynamic_cast<Private*>(owner.get()), false});
+ _buckets[slot] = _owners.size();
+ return {&_owners.back(), true};
+ }
+ };
+}
diff --git a/private-include/indium/retirement-queue.private.hpp b/private-include/indium/retirement-queue.private.hpp
new file mode 100644
--- /dev/null
+++ b/private-include/indium/retirement-queue.private.hpp
@@ -0,0 +1,63 @@
+#pragma once
+
+#include <array>
+#include <condition_variable>
+#include <mutex>
+#include <optional>
+#include <thread>
+#include <utility>
+
+namespace Indium {
+ // Completed GPU work only. A full queue falls back to synchronous cleanup,
+ // bounding retained resources even if the worker cannot keep up.
+ template<class Job, size_t capacity = 32> class RetirementQueue {
+ std::array<std::optional<Job>, capacity> _jobs;
+ size_t _head = 0, _count = 0;
+ bool _stopping = false;
+ std::mutex _mutex;
+ std::condition_variable _ready;
+ std::thread _worker;
+
+ void run() {
+ for (;;) {
+ std::optional<Job> job;
+ {
+ std::unique_lock<std::mutex> lock(_mutex);
+ _ready.wait(lock, [&] { return _count || _stopping; });
+ if (!_count && _stopping) return;
+ job = std::move(_jobs[_head]);
+ _jobs[_head].reset();
+ _head = (_head + 1) % capacity;
+ --_count;
+ }
+ // Resource destructors and driver calls never hold the queue lock.
+ (*job)();
+ }
+ }
+ public:
+ RetirementQueue() : _worker([this] { run(); }) {}
+ RetirementQueue(const RetirementQueue&) = delete;
+ ~RetirementQueue() { shutdown(); }
+
+ bool tryPush(Job&& job) {
+ {
+ std::lock_guard<std::mutex> lock(_mutex);
+ if (_stopping || _count == capacity) return false;
+ _jobs[(_head + _count) % capacity].emplace(std::move(job));
+ ++_count;
+ }
+ _ready.notify_one();
+ return true;
+ }
+ // Called before Vulkan/device teardown, while all driver entry points
+ // remain valid. Drain queued jobs before joining the worker.
+ void shutdown() {
+ {
+ std::lock_guard<std::mutex> lock(_mutex);
+ _stopping = true;
+ }
+ _ready.notify_one();
+ if (_worker.joinable()) _worker.join();
+ }
+ };
+}
diff --git a/private-include/indium/submission-batch.private.hpp b/private-include/indium/submission-batch.private.hpp
new file mode 100644
--- /dev/null
+++ b/private-include/indium/submission-batch.private.hpp
@@ -0,0 +1,54 @@
+#pragma once
+#include <vulkan/vulkan.h>
+#include <array>
+#include <cassert>
+#include <vector>
+
+namespace Indium {
+ // Protected by the device graphics-queue mutex. Each packet preserves the
+ // original submit boundary and its semaphore waits/signals. Only the host
+ // driver call is combined; commands are not reordered or merged.
+ class SubmissionBatch {
+ static constexpr size_t capacity = 128;
+ struct Packet {
+ VkSubmitFlags flags = 0;
+ VkCommandBufferSubmitInfo command {};
+ std::vector<VkSemaphoreSubmitInfo> waits, signals;
+ };
+ std::array<Packet, capacity> _packets;
+ size_t _count = 0;
+ public:
+ bool empty() const { return _count == 0; }
+ bool full() const { return _count == capacity; }
+ void append(const VkSubmitInfo2& info) {
+ assert(!full() && !info.pNext && info.commandBufferInfoCount == 1);
+ auto& packet = _packets[_count++];
+ packet.flags = info.flags;
+ packet.command = info.pCommandBufferInfos[0];
+ packet.waits.clear(); packet.signals.clear();
+ if (info.waitSemaphoreInfoCount)
+ packet.waits.assign(info.pWaitSemaphoreInfos, info.pWaitSemaphoreInfos + info.waitSemaphoreInfoCount);
+ if (info.signalSemaphoreInfoCount)
+ packet.signals.assign(info.pSignalSemaphoreInfos, info.pSignalSemaphoreInfos + info.signalSemaphoreInfoCount);
+ }
+ template<class Submit> VkResult flush(Submit&& submit) {
+ if (empty()) return VK_SUCCESS;
+ std::array<VkSubmitInfo2, capacity> infos {};
+ for (size_t i = 0; i < _count; ++i) {
+ const auto& packet = _packets[i];
+ auto& info = infos[i];
+ info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2;
+ info.flags = packet.flags;
+ info.commandBufferInfoCount = 1;
+ info.pCommandBufferInfos = &packet.command;
+ info.waitSemaphoreInfoCount = packet.waits.size();
+ info.pWaitSemaphoreInfos = packet.waits.data();
+ info.signalSemaphoreInfoCount = packet.signals.size();
+ info.pSignalSemaphoreInfos = packet.signals.data();
+ }
+ VkResult result = submit(static_cast<uint32_t>(_count), infos.data());
+ if (result == VK_SUCCESS) _count = 0;
+ return result;
+ }
+ };
+}
diff --git a/private-include/indium/submit-batch.private.hpp b/private-include/indium/submit-batch.private.hpp
new file mode 100644
--- /dev/null
+++ b/private-include/indium/submit-batch.private.hpp
@@ -0,0 +1,12 @@
+#pragma once
+
+// Merging consecutive command buffer submissions into one vkQueueSubmit2 batch (see command-buffer.cpp).
+// League commits ~30 small command buffers per frame; each used to cost its own queue submit (an amdgpu CS ioctl plus
+// timeline-semaphore ioctls). Opt-in: guest flag file /tmp/metal-submit-batch (checked once).
+// A pending batch is flushed before anything that waits for or depends on submitted work (presenting commit, waitUntilCompleted,
+// texture uploads) and by a 2 ms timer.
+
+namespace Indium {
+ bool submitBatchEnabled();
+ void flushPendingSubmits();
+}
diff --git a/private-include/indium/submit-thread.private.hpp b/private-include/indium/submit-thread.private.hpp
new file mode 100644
--- /dev/null
+++ b/private-include/indium/submit-thread.private.hpp
@@ -0,0 +1,175 @@
+#pragma once
+
+// Ordered queue-submission worker.
+//
+// PrivateCommandBuffer::commit() used to do all its Vulkan queue work (semaphore reservation, vkQueueSubmit2, drawable
+// presentation) on the game's main thread: about 1.9 ms per frame in League's Baron fight, 77% of it inside the kernel's
+// amdgpu CS ioctl. The caller thread now only finishes recording; one FIFO worker performs the queue work, so the
+// submission order equals the commit order (timeline values are reserved inside the same task, in that order).
+// Every other path that touches the graphics queue (texture uploads) calls drain() first so it cannot overtake a
+// pending commit. Opt-in: guest flag file /tmp/metal-submit-thread (checked once) enables it; the A/B showed no gain, so submission stays synchronous.
+// Header-only with function-local statics: no class gains a member.
+
+#include <indium/waits.private.hpp>
+
+#include <condition_variable>
+#include <deque>
+#include <cstdint>
+#include <limits>
+#include <functional>
+#include <mutex>
+#include <thread>
+#include <unistd.h>
+
+namespace Indium {
+ namespace SubmitThread {
+ struct State {
+ std::mutex mutex;
+ std::condition_variable wake;
+ std::condition_variable idle;
+ std::deque<std::function<void()>> queue;
+ size_t active = 0;
+ uint64_t posted = 0, completed = 0;
+ size_t drainWaiters = 0;
+ uint64_t drainTarget = std::numeric_limits<uint64_t>::max();
+ bool started = false;
+ const bool enabled = access("/tmp/metal-submit-thread", F_OK) == 0;
+ };
+
+ // One-shot completion token for a queued task: the worker signals it at the end of the task, so a caller that
+ // only has to wait for *its own* commit (the present dispatch it just queued) does not wait for the tasks that
+ // were queued after it. The token must outlive the task, which the waiting caller guarantees.
+ struct Token {
+ std::mutex mutex;
+ std::condition_variable condvar;
+ bool done = false;
+ void signal() {
+ {
+ std::lock_guard<std::mutex> lock(mutex);
+ done = true;
+ // The waiter may destroy this stack token as soon as it sees done.
+ // Finish notifying while the token's mutex still protects its lifetime.
+ condvar.notify_all();
+ }
+ }
+ // blocks until the task carrying this token has finished (counting the block under `site`)
+ void wait(const char* site) {
+ std::unique_lock<std::mutex> lock(mutex);
+ Waits::blockOn(site, condvar, lock, [this] { return done; });
+ }
+ };
+
+ inline State& state() {
+ static State* instance = new State;
+ return *instance;
+ }
+
+ inline bool& workerThread() {
+ static thread_local bool value = false;
+ return value;
+ }
+
+ inline bool onWorker() { return workerThread(); }
+
+ inline void loop() {
+ workerThread() = true;
+ auto& s = state();
+ for (;;) {
+ std::function<void()> task;
+ {
+ std::unique_lock<std::mutex> lock(s.mutex);
+ s.wake.wait(lock, [&] { return !s.queue.empty(); });
+ task = std::move(s.queue.front());
+ s.queue.pop_front();
+ ++s.active;
+ }
+ task();
+ {
+ std::lock_guard<std::mutex> lock(s.mutex);
+ --s.active;
+ ++s.completed;
+ if (s.drainWaiters && s.completed >= s.drainTarget) s.idle.notify_all();
+ }
+ }
+ }
+
+ // run `task` after every task queued before it; synchronously when disabled or already on the worker
+ inline void run(std::function<void()> task) {
+ auto& s = state();
+ if (!s.enabled || onWorker()) {
+ task();
+ return;
+ }
+ bool needsWake;
+ {
+ std::lock_guard<std::mutex> lock(s.mutex);
+ if (!s.started) {
+ s.started = true;
+ std::thread thread(loop);
+ thread.detach();
+ }
+ // An active worker checks the queue before sleeping again. Only
+ // the idle/empty transition needs a wake; test under the same
+ // mutex as the worker's wait to avoid losing that transition.
+ needsWake = s.active == 0 && s.queue.empty();
+ s.queue.push_back(std::move(task));
+ ++s.posted;
+ }
+ if (needsWake) s.wake.notify_one();
+ }
+
+ // start the worker now (called when the device is created): a thread created lazily later would inherit the
+ // affinity of whichever thread happened to create it (the game's main thread is pinned to its own core)
+ inline void ensureStarted() {
+ auto& s = state();
+ std::lock_guard<std::mutex> lock(s.mutex);
+ if (!s.started) {
+ s.started = true;
+ std::thread thread(loop);
+ thread.detach();
+ }
+ }
+
+ // like run(), but always on the worker (deferred render encoding uses the same FIFO, regardless of the opt-in flag)
+ inline void post(std::function<void()> task) {
+ auto& s = state();
+ bool needsWake;
+ {
+ std::lock_guard<std::mutex> lock(s.mutex);
+ if (!s.started) {
+ s.started = true;
+ std::thread thread(loop);
+ thread.detach();
+ }
+ // An active worker checks the queue before sleeping again. Only
+ // the idle/empty transition needs a wake; test under the same
+ // mutex as the worker's wait to avoid losing that transition.
+ needsWake = s.active == 0 && s.queue.empty();
+ s.queue.push_back(std::move(task));
+ ++s.posted;
+ }
+ if (needsWake) s.wake.notify_one();
+ }
+
+ // true while the worker has queued or running tasks
+ inline bool pending() {
+ auto& s = state();
+ std::lock_guard<std::mutex> lock(s.mutex);
+ return !s.queue.empty() || s.active != 0;
+ }
+
+ // block until every task queued so far has finished (no-op on the worker itself)
+ inline void drain(const char* site = "drain") {
+ auto& s = state();
+ if (onWorker()) return;
+ std::unique_lock<std::mutex> lock(s.mutex);
+ const uint64_t target = s.posted;
+ if (s.completed >= target) return;
+ ++s.drainWaiters;
+ if (target < s.drainTarget) s.drainTarget = target;
+ // FIFO completion is sufficient: later producers must not extend this wait.
+ Waits::blockOn(site, s.idle, lock, [&] { return s.completed >= target; });
+ if (--s.drainWaiters == 0) s.drainTarget = std::numeric_limits<uint64_t>::max();
+ }
+ }
+}
diff --git a/include/indium/blit-command-encoder.hpp b/include/indium/blit-command-encoder.hpp
index b000d25..98baa9b 100644
--- a/include/indium/blit-command-encoder.hpp
+++ b/include/indium/blit-command-encoder.hpp
@@ -28,13 +28,13 @@ namespace Indium {
public:
virtual ~BlitCommandEncoder() = 0;
- virtual void copy(std::shared_ptr<Buffer> source, size_t sourceOffset, std::shared_ptr<Buffer> destination, size_t destinationOffset, size_t size) = 0;
- virtual void copy(std::shared_ptr<Buffer> source, size_t sourceOffset, size_t sourceBytesPerRow, size_t sourceBytesPerImage, Size sourceSize, std::shared_ptr<Texture> destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin, BlitOption options = BlitOption::None) = 0;
- virtual void copy(std::shared_ptr<Texture> source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, std::shared_ptr<Buffer> destination, size_t destinationOffset, size_t destinationBytesPerRow, size_t destinationBytesPerImage, BlitOption options = BlitOption::None) = 0;
- virtual void copy(std::shared_ptr<Texture> source, std::shared_ptr<Texture> destination) = 0;
- virtual void copy(std::shared_ptr<Texture> source, size_t sourceSlice, size_t sourceLevel, std::shared_ptr<Texture> destination, size_t destinationSlice, size_t destinationLevel, size_t sliceCount, size_t levelCount) = 0;
- virtual void copy(std::shared_ptr<Texture> source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, std::shared_ptr<Texture> destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin) = 0;
- virtual void fillBuffer(std::shared_ptr<Buffer> buffer, Range<size_t> range, uint8_t value) = 0;
- virtual void generateMipmapsForTexture(std::shared_ptr<Texture> texture) = 0;
+ virtual void copy(const std::shared_ptr<Buffer>& source, size_t sourceOffset, const std::shared_ptr<Buffer>& destination, size_t destinationOffset, size_t size) = 0;
+ virtual void copy(const std::shared_ptr<Buffer>& source, size_t sourceOffset, size_t sourceBytesPerRow, size_t sourceBytesPerImage, Size sourceSize, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin, BlitOption options = BlitOption::None) = 0;
+ virtual void copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, const std::shared_ptr<Buffer>& destination, size_t destinationOffset, size_t destinationBytesPerRow, size_t destinationBytesPerImage, BlitOption options = BlitOption::None) = 0;
+ virtual void copy(const std::shared_ptr<Texture>& source, const std::shared_ptr<Texture>& destination) = 0;
+ virtual void copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, size_t sliceCount, size_t levelCount) = 0;
+ virtual void copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin) = 0;
+ virtual void fillBuffer(const std::shared_ptr<Buffer>& buffer, Range<size_t> range, uint8_t value) = 0;
+ virtual void generateMipmapsForTexture(const std::shared_ptr<Texture>& texture) = 0;
};
};
diff --git a/private-include/indium/deferred-blit-encoder.private.hpp b/private-include/indium/deferred-blit-encoder.private.hpp
new file mode 100644
--- /dev/null
+++ b/private-include/indium/deferred-blit-encoder.private.hpp
@@ -0,0 +1,239 @@
+#pragma once
+
+// Deferred blit-command encoding.
+//
+// A blit encoder used to record its vkCmd* calls (and their barriers) directly into the command buffer, on the game's
+// thread, after waiting for every pending render replay of that command buffer (PrivateCommandBuffer::waitDeferred()).
+// Both the wait and the driver calls are on the frame's critical path, and nothing about them has to happen there:
+// Metal only requires that the calls are applied in order before the command buffer is committed.
+//
+// This facade implements the same public BlitCommandEncoder interface by recording the calls into a compact op list
+// (holding a reference to every buffer and texture they mention) and, at endEncoding(), posting the list to the ordered
+// worker of submit-thread.private.hpp -- the same FIFO the deferred render chunks and the commit itself run on. The
+// real PrivateBlitCommandEncoder is created, fed and ended there, in command-buffer order, before the commit that
+// closes the Vulkan command buffer. The deferred count keeps waitDeferred() correct for anything that still records
+// directly (compute encoders, and every encoder while deferred render encoding is off).
+//
+// Only used while deferred render encoding is on: with it off, render encoders record into the Vulkan command buffer
+// on the calling thread, so a blit replay has to stay on that thread too (the direct path keeps its waitDeferred()).
+// OFF by default (round-3 behaviour: blit encoders record directly on the calling thread after waitDeferred()). Guest
+// flag file /tmp/metal-deferred-blit (checked once per process) turns the facade on. Round 4 had it on by default; that
+// round regressed in matches, so it is an opt-in experiment now.
+
+#include <indium/blit-command-encoder.private.hpp>
+#include <indium/buffer.hpp>
+#include <indium/command-buffer.private.hpp>
+#include <indium/submit-thread.private.hpp>
+#include <indium/texture.hpp>
+
+#include <atomic>
+#include <cstdio>
+#include <memory>
+#include <stdexcept>
+#include <unistd.h>
+#include <vector>
+
+namespace Indium {
+ namespace AsyncEncode {
+ inline bool deferBlitEnabled() {
+ static const bool value = access("/tmp/metal-deferred-blit", F_OK) == 0;
+ return value;
+ }
+
+ // Blits are short lists of large operations; one chunk per encoder (posted at endEncoding) is enough, and it
+ // keeps the whole pass on one worker task, i.e. in one command-buffer order.
+ enum class BlitKind: uint8_t {
+ BufferToBuffer, BufferToTexture, TextureToBuffer, TextureToTexture, TextureToTextureLevels, TextureToTextureRegion, Mipmaps, FillBuffer,
+ };
+
+ struct BlitOp {
+ BlitKind kind = BlitKind::BufferToBuffer;
+ uint8_t value = 0;
+ // per-kind arguments, in the order of the call (offsets, slices, levels, row/image strides)
+ size_t n[8] = {0, 0, 0, 0, 0, 0, 0, 0};
+ Origin origin {};
+ Origin destinationOrigin {};
+ Size size {};
+ Range<size_t> range {0, 0};
+ std::shared_ptr<void> source;
+ std::shared_ptr<void> destination;
+ };
+
+ // One blit pass being recorded. `real` and `failed` are only touched by the worker.
+ struct BlitState {
+ std::shared_ptr<PrivateCommandBuffer> commandBuffer;
+ BlitPassDescriptor descriptor;
+ std::shared_ptr<PrivateBlitCommandEncoder> real;
+ bool failed = false;
+ BlitState(std::shared_ptr<PrivateCommandBuffer> cb, const BlitPassDescriptor& d): commandBuffer(std::move(cb)), descriptor(d) {}
+ };
+
+ inline void blitLog(const char* message) {
+ if (FILE* f = fopen("/tmp/metal-trace.log", "a")) {
+ fprintf(f, "async-blit: %s\n", message);
+ fclose(f);
+ }
+ }
+
+ // Apply the recorded calls and end the real encoder. Runs on the worker.
+ inline void replayBlit(BlitState& st, std::vector<BlitOp>& ops) {
+ PoolScope pool;
+ if (!st.real && !st.failed) {
+ try {
+ st.real = st.commandBuffer->blitCommandEncoderNow(st.descriptor);
+ } catch (const std::exception& e) {
+ st.failed = true;
+ if (FILE* f = fopen("/tmp/metal-trace.log", "a")) { fprintf(f, "async-blit: encoder creation failed: %s\n", e.what()); fclose(f); }
+ }
+ }
+ if (st.real) {
+ PrivateBlitCommandEncoder* real = st.real.get();
+ for (BlitOp& op: ops) {
+ try {
+ switch (op.kind) {
+ case BlitKind::BufferToBuffer:
+ real->copy(std::static_pointer_cast<Buffer>(op.source), op.n[0], std::static_pointer_cast<Buffer>(op.destination), op.n[1], op.n[2]);
+ break;
+ case BlitKind::BufferToTexture:
+ real->copy(std::static_pointer_cast<Buffer>(op.source), op.n[0], op.n[1], op.n[2], op.size,
+ std::static_pointer_cast<Texture>(op.destination), op.n[3], op.n[4], op.origin);
+ break;
+ case BlitKind::TextureToBuffer:
+ real->copy(std::static_pointer_cast<Texture>(op.source), op.n[0], op.n[1], op.origin, op.size,
+ std::static_pointer_cast<Buffer>(op.destination), op.n[2], op.n[3], op.n[4]);
+ break;
+ case BlitKind::TextureToTexture:
+ real->copy(std::static_pointer_cast<Texture>(op.source), std::static_pointer_cast<Texture>(op.destination));
+ break;
+ case BlitKind::TextureToTextureLevels:
+ real->copy(std::static_pointer_cast<Texture>(op.source), op.n[0], op.n[1],
+ std::static_pointer_cast<Texture>(op.destination), op.n[2], op.n[3], op.n[4], op.n[5]);
+ break;
+ case BlitKind::TextureToTextureRegion:
+ real->copy(std::static_pointer_cast<Texture>(op.source), op.n[0], op.n[1], op.origin, op.size,
+ std::static_pointer_cast<Texture>(op.destination), op.n[2], op.n[3], op.destinationOrigin);
+ break;
+ case BlitKind::Mipmaps:
+ real->generateMipmapsForTexture(std::static_pointer_cast<Texture>(op.source));
+ break;
+ case BlitKind::FillBuffer:
+ real->fillBuffer(std::static_pointer_cast<Buffer>(op.source), op.range, op.value);
+ break;
+ }
+ } catch (const std::exception& e) {
+ if (FILE* f = fopen("/tmp/metal-trace.log", "a")) { fprintf(f, "async-blit: op %d failed: %s\n", (int) op.kind, e.what()); fclose(f); }
+ }
+ }
+ try { real->endEncoding(); } catch (...) {}
+ st.real.reset();
+ }
+ }
+
+ class DeferredBlitCommandEncoder final: public BlitCommandEncoder {
+ std::shared_ptr<BlitState> _state;
+ std::vector<BlitOp> _ops;
+ bool _begun = false;
+ bool _ended = false;
+
+ // Hand everything recorded so far to the worker. The ops own the resources they reference, so the game may
+ // release them (and us) as soon as this returns.
+ void flush() {
+ auto st = _state;
+ auto cb = st->commandBuffer;
+ if (!_begun) { _begun = true; cb->deferredBegin(); }
+ auto ops = std::make_shared<std::vector<BlitOp>>(std::move(_ops));
+ _ops.clear();
+ if (!cb->replayAllowed()) {
+ // an encoder dropped after the commit: the Vulkan command buffer is closed, so balance the deferred
+ // count and drop the ops instead of recording into it
+ blitLog("encoder dropped after commit");
+ cb->deferredEnd();
+ return;
+ }
+ SubmitThread::post([st, cb, ops]() mutable {
+ replayBlit(*st, *ops);
+ st.reset();
+ cb->deferredEnd();
+ });
+ }
+
+ BlitOp& add(BlitKind kind) {
+ _ops.emplace_back();
+ BlitOp& op = _ops.back();
+ op.kind = kind;
+ return op;
+ }
+
+ public:
+ DeferredBlitCommandEncoder(std::shared_ptr<PrivateCommandBuffer> cb, const BlitPassDescriptor& d):
+ _state(std::make_shared<BlitState>(std::move(cb), d)) {}
+ explicit DeferredBlitCommandEncoder(std::shared_ptr<PrivateCommandBuffer> cb):
+ DeferredBlitCommandEncoder(std::move(cb), BlitPassDescriptor {}) {}
+
+ ~DeferredBlitCommandEncoder() override { endEncoding(); }
+
+ void endEncoding() override {
+ if (_ended) return;
+ _ended = true;
+ flush();
+ _state.reset();
+ }
+
+ // The real encoder rejects blit options; doing it here keeps the exception on the calling thread.
+ static void rejectOptions(BlitOption options) {
+ if (options != BlitOption::None) throw std::runtime_error("TODO: support blit options");
+ }
+
+ void copy(const std::shared_ptr<Buffer>& source, size_t sourceOffset, const std::shared_ptr<Buffer>& destination, size_t destinationOffset, size_t size) override {
+ BlitOp& op = add(BlitKind::BufferToBuffer);
+ op.source = source; op.destination = destination;
+ op.n[0] = sourceOffset; op.n[1] = destinationOffset; op.n[2] = size;
+ }
+ void copy(const std::shared_ptr<Buffer>& source, size_t sourceOffset, size_t sourceBytesPerRow, size_t sourceBytesPerImage, Size sourceSize, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin, BlitOption options = BlitOption::None) override {
+ rejectOptions(options);
+ BlitOp& op = add(BlitKind::BufferToTexture);
+ op.source = source; op.destination = destination;
+ op.n[0] = sourceOffset; op.n[1] = sourceBytesPerRow; op.n[2] = sourceBytesPerImage;
+ op.n[3] = destinationSlice; op.n[4] = destinationLevel;
+ op.origin = destinationOrigin; op.size = sourceSize;
+ }
+ void copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, const std::shared_ptr<Buffer>& destination, size_t destinationOffset, size_t destinationBytesPerRow, size_t destinationBytesPerImage, BlitOption options = BlitOption::None) override {
+ rejectOptions(options);
+ BlitOp& op = add(BlitKind::TextureToBuffer);
+ op.source = source; op.destination = destination;
+ op.n[0] = sourceSlice; op.n[1] = sourceLevel;
+ op.n[2] = destinationOffset; op.n[3] = destinationBytesPerRow; op.n[4] = destinationBytesPerImage;
+ op.origin = sourceOrigin; op.size = sourceSize;
+ }
+ void copy(const std::shared_ptr<Texture>& source, const std::shared_ptr<Texture>& destination) override {
+ BlitOp& op = add(BlitKind::TextureToTexture);
+ op.source = source; op.destination = destination;
+ }
+ void copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, size_t sliceCount, size_t levelCount) override {
+ BlitOp& op = add(BlitKind::TextureToTextureLevels);
+ op.source = source; op.destination = destination;
+ op.n[0] = sourceSlice; op.n[1] = sourceLevel; op.n[2] = destinationSlice; op.n[3] = destinationLevel;
+ op.n[4] = sliceCount; op.n[5] = levelCount;
+ }
+ void copy(const std::shared_ptr<Texture>& source, size_t sourceSlice, size_t sourceLevel, Origin sourceOrigin, Size sourceSize, const std::shared_ptr<Texture>& destination, size_t destinationSlice, size_t destinationLevel, Origin destinationOrigin) override {
+ BlitOp& op = add(BlitKind::TextureToTextureRegion);
+ op.source = source; op.destination = destination;
+ op.n[0] = sourceSlice; op.n[1] = sourceLevel; op.n[2] = destinationSlice; op.n[3] = destinationLevel;
+ op.origin = sourceOrigin; op.destinationOrigin = destinationOrigin; op.size = sourceSize;
+ }
+ void fillBuffer(const std::shared_ptr<Buffer>& buffer, Range<size_t> range, uint8_t value) override {
+ // the real encoder rejects misaligned ranges; keep the exception on the calling thread
+ if ((range.start & 3) != 0) throw std::runtime_error("Misaligned fillBuffer offset");
+ if ((range.length & 3) != 0) throw std::runtime_error("Misaligned fillBuffer length");
+ BlitOp& op = add(BlitKind::FillBuffer);
+ op.source = buffer;
+ op.range = range;
+ op.value = value;
+ }
+ void generateMipmapsForTexture(const std::shared_ptr<Texture>& texture) override {
+ BlitOp& op = add(BlitKind::Mipmaps);
+ op.source = texture;
+ }
+ };
+ }
+}
diff --git a/private-include/indium/waits.private.hpp b/private-include/indium/waits.private.hpp
new file mode 100644
--- /dev/null
+++ b/private-include/indium/waits.private.hpp
@@ -0,0 +1,145 @@
+#pragma once
+
+// Wait-site counters.
+//
+// The main thread's frame budget is spent partly on blocking in our own hand-offs (the submit worker's drain, the
+// deferred-encoder sync, texture uploads) and in the compositor's drawable hand-out. A sample profile cannot attribute
+// that time (the blocked PC is inside a futex helper), so every wait site counts how often the calling thread *actually
+// blocked* and how long, and appends a summary to the guest's /tmp/metal-trace.log every 5 s.
+//
+// Everything here is always compiled in; the counters are only active when /tmp/metal-debug-waits exists when the
+// process starts (checked once). Disabled, each site costs one predictable branch: a call site must establish that it
+// is about to block (a false predicate under the lock, a free-drawable check) before it constructs a Block, and that
+// check is the only work done when the flag is absent.
+//
+// Header-only: the counter state is per dylib (each library writes its own summary), so no class gains a member. The
+// functions are declared hidden: an inline function's static locals are otherwise exported as weak definitions (even
+// under -fvisibility-inlines-hidden), and dyld coalesces those across images, so libindium and QuartzCore would share
+// one copy -- harmless only while both images were built from the same version of this header. Hidden keeps the state
+// in each image, as documented.
+//
+// Flag file pitfall (why a match can show no waits lines although the file "exists"): the guest /tmp is
+// ~/.darling/private/tmp *inside Darling's overlay mount*, i.e. the host path is the overlay's upper directory. A file
+// created there from the host while the Darling namespace is running is not guaranteed to be visible to guest
+// processes: once a guest process has looked the name up and got ENOENT, the overlay caches the negative entry and
+// keeps answering ENOENT. Create flag files from inside the guest (`darling shell touch /tmp/metal-debug-waits`) or
+// while Darling is shut down. When the counters are on, the first check writes a "waits enabled" line, so its absence
+// from /tmp/metal-trace.log means the flag was not seen.
+
+#include <chrono>
+#include <condition_variable>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <ctime>
+#include <map>
+#include <mutex>
+#include <string>
+#include <dlfcn.h>
+#include <time.h>
+#include <unistd.h>
+
+#define INDIUM_WAITS_LOCAL __attribute__((visibility("hidden")))
+
+namespace Indium {
+ namespace Waits {
+ // The summaries append to the guest's /tmp/metal-trace.log like the other diagnostics; DARLING_WAITS_LOG
+ // overrides the path so the host-side test does not append to the runtime's log. Only read when the counters
+ // are on (i.e. from the dump path).
+ INDIUM_WAITS_LOCAL inline const char* logPath() {
+ static const char* path = []() -> const char* {
+ const char* override = getenv("DARLING_WAITS_LOG");
+ return override && *override ? override : "/tmp/metal-trace.log";
+ }();
+ return path;
+ }
+
+ // Guest flag file, read once per process (per image) at the first wait site that is reached. When it is on, that
+ // first check appends one "waits enabled" line naming the image, so a match log proves the path is live even if
+ // no site ever blocks.
+ INDIUM_WAITS_LOCAL inline bool readEnabled() {
+ if (access("/tmp/metal-debug-waits", F_OK) != 0) return false;
+ Dl_info info {};
+ const char* image = dladdr(reinterpret_cast<void*>(&readEnabled), &info) && info.dli_fname ? info.dli_fname : "?";
+ if (FILE* log = fopen(logPath(), "a")) {
+ fprintf(log, "waits enabled@%ld: pid=%d image=%s\n", (long) time(nullptr), (int) getpid(), image);
+ fclose(log);
+ }
+ return true;
+ }
+
+ INDIUM_WAITS_LOCAL inline bool enabled() {
+ static const bool value = readEnabled();
+ return value;
+ }
+
+ // Monotonic nanoseconds. steady_clock is what the rest of Indium uses (libc++ maps it to CLOCK_UPTIME_RAW, i.e.
+ // mach_absolute_time, on Darwin); clock_gettime(CLOCK_MONOTONIC) is wall-clock-derived in Darwin's libc
+ // (gettimeofday minus the boot time, with a kern.boottime sysctl fallback), so it is neither monotonic nor cheap
+ // under Darling.
+ INDIUM_WAITS_LOCAL inline uint64_t now() {
+ return static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(
+ std::chrono::steady_clock::now().time_since_epoch()).count());
+ }
+
+ INDIUM_WAITS_LOCAL inline void record(const char* site, uint64_t nanoseconds) {
+ struct Stat { uint64_t blocks = 0, sum = 0, max = 0; };
+ static std::mutex lock;
+ static std::map<std::string, Stat> stats;
+ static time_t last = 0;
+ std::lock_guard<std::mutex> guard(lock);
+ Stat& stat = stats[site];
+ ++stat.blocks;
+ stat.sum += nanoseconds;
+ if (nanoseconds > stat.max) stat.max = nanoseconds;
+ time_t now = time(nullptr);
+ if (now - last >= 5) {
+ last = now;
+ if (FILE* log = fopen(logPath(), "a")) {
+ fprintf(log, "waits@%ld:", (long) now);
+ for (auto& entry: stats) {
+ fprintf(log, " %s[n=%llu sum=%.1fus max=%.1fus]", entry.first.c_str(),
+ (unsigned long long) entry.second.blocks, entry.second.sum / 1000.0, entry.second.max / 1000.0);
+ }
+ fprintf(log, "\n");
+ fclose(log);
+ }
+ stats.clear();
+ }
+ }
+
+ // One blocking episode of `site`. The constructor checks the flag and the destructor records (a no-op when the
+ // counter is off). Use it around a wait that is known to block; blockOn()/blockOnLock() do that check for the
+ // sites where the caller cannot know in advance.
+ struct INDIUM_WAITS_LOCAL Block {
+ const char* site;
+ uint64_t start = 0;
+ explicit Block(const char* site_): site(site_) { if (enabled()) start = now(); }
+ ~Block() { if (start != 0) record(site, now() - start); }
+ };
+
+ // Wait for `predicate` on `condvar`, counting the block only when the predicate is already false at the call
+ // (i.e. the wait really has to sleep). The predicate is re-checked by the wait itself, so both paths are safe.
+ template<class Predicate>
+ INDIUM_WAITS_LOCAL inline void blockOn(const char* site, std::condition_variable& condvar, std::unique_lock<std::mutex>& lock, Predicate predicate) {
+ if (!enabled() || predicate()) {
+ condvar.wait(lock, predicate);
+ return;
+ }
+ Block block(site);
+ condvar.wait(lock, predicate);
+ }
+
+ // Lock `mutex`, counting the block only when it is contended at the time of the call. try_lock only runs when
+ // the flag is on, so the disabled cost is the same single branch.
+ INDIUM_WAITS_LOCAL inline void blockOnLock(const char* site, std::mutex& mutex) {
+ if (!enabled()) {
+ mutex.lock();
+ return;
+ }
+ if (mutex.try_lock()) return;
+ Block block(site);
+ mutex.lock();
+ }
+ }
+}
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8d95863..1af3167 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2,6 +2,41 @@ project(indium)
cmake_minimum_required(VERSION 3.20)
find_package(Vulkan REQUIRED)
+
+# Iridium compiles against these LLVM headers and loads the shared libLLVM of the same major at run time, so the LLVM
+# found here needs an llvm-config that reports a shared library. find_package also looks in <prefix>/llvm/lib/cmake/llvm
+# for every PATH entry, so with /opt/rocm/bin on PATH it picks ROCm's private LLVM ahead of the system one, and a cached
+# LLVM_DIR can name an LLVM that was removed since. Unless LLVM_DIR names a usable LLVM, use the llvm-config on PATH.
+function(indium_llvm_config_works tool result)
+ set(${result} FALSE PARENT_SCOPE)
+ execute_process(COMMAND "${tool}" --link-shared --libnames
+ OUTPUT_VARIABLE libname RESULT_VARIABLE status ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+ if (status EQUAL 0 AND libname)
+ set(${result} TRUE PARENT_SCOPE)
+ endif()
+endfunction()
+
+set(indium_llvm_usable FALSE)
+if (LLVM_DIR)
+ # <prefix>/lib/cmake/llvm or <prefix>/lib64/cmake/llvm
+ get_filename_component(indium_llvm_prefix "${LLVM_DIR}/../../.." ABSOLUTE)
+ indium_llvm_config_works("${indium_llvm_prefix}/bin/llvm-config" indium_llvm_usable)
+endif()
+if (NOT indium_llvm_usable)
+ find_program(INDIUM_PATH_LLVM_CONFIG llvm-config)
+ if (INDIUM_PATH_LLVM_CONFIG)
+ indium_llvm_config_works("${INDIUM_PATH_LLVM_CONFIG}" indium_path_llvm_usable)
+ execute_process(COMMAND "${INDIUM_PATH_LLVM_CONFIG}" --cmakedir
+ OUTPUT_VARIABLE indium_path_llvm_dir RESULT_VARIABLE indium_status ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+ if (indium_path_llvm_usable AND indium_status EQUAL 0 AND EXISTS "${indium_path_llvm_dir}")
+ if (LLVM_DIR)
+ message(STATUS "LLVM_DIR=${LLVM_DIR} has no llvm-config with a shared library; using ${indium_path_llvm_dir}")
+ endif()
+ set(LLVM_DIR "${indium_path_llvm_dir}" CACHE PATH "The directory containing a CMake configuration file for LLVM." FORCE)
+ endif()
+ endif()
+endif()
+
find_package(LLVM REQUIRED CONFIG)
if (DARLING)
@@ -158,11 +193,35 @@ if (NOT DARLING)
)
endif()
-execute_process(
- COMMAND llvm-config --link-shared --libnames
- OUTPUT_VARIABLE HOST_LLVM_LIBNAME
- OUTPUT_STRIP_TRAILING_WHITESPACE
-)
+# Use an llvm-config of the same major as the headers found above. A stale cached LLVM_DIR can point at
+# a removed LLVM whose llvm-config exists but whose library is gone, so fall back to the one on PATH.
+set(HOST_LLVM_LIBNAME "")
+foreach(candidate IN ITEMS "${LLVM_TOOLS_BINARY_DIR}/llvm-config" llvm-config)
+ execute_process(
+ COMMAND "${candidate}" --version
+ OUTPUT_VARIABLE candidate_version
+ RESULT_VARIABLE candidate_status
+ ERROR_QUIET
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ )
+ if (candidate_status EQUAL 0 AND candidate_version MATCHES "^${LLVM_VERSION_MAJOR}\\.")
+ execute_process(
+ COMMAND "${candidate}" --link-shared --libnames
+ OUTPUT_VARIABLE HOST_LLVM_LIBNAME
+ RESULT_VARIABLE candidate_status
+ ERROR_QUIET
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ )
+ if (candidate_status EQUAL 0 AND HOST_LLVM_LIBNAME)
+ break()
+ endif()
+ set(HOST_LLVM_LIBNAME "")
+ endif()
+endforeach()
+if (NOT HOST_LLVM_LIBNAME)
+ message(FATAL_ERROR "No working llvm-config for LLVM ${LLVM_VERSION} (LLVM_DIR=${LLVM_DIR}). "
+ "Install an LLVM with a shared libLLVM, or pass -DLLVM_DIR=<its lib/cmake/llvm directory>.")
+endif()
target_compile_definitions(iridium PRIVATE
"HOST_LLVM_LIBNAME=\"${HOST_LLVM_LIBNAME}\""
diff --git a/src/iridium/dynamic-llvm.cpp b/src/iridium/dynamic-llvm.cpp
index 1b5876e..03af560 100644
--- a/src/iridium/dynamic-llvm.cpp
+++ b/src/iridium/dynamic-llvm.cpp
@@ -1,4 +1,7 @@
#include <iridium/dynamic-llvm.hpp>
+#include <llvm/Config/llvm-config.h>
+#include <stdexcept>
+#include <string>
#ifdef DARLING
#include <elfcalls.h>
@@ -19,7 +22,20 @@ bool Iridium::DynamicLLVM::init() {
libraryHandle = dlopen(HOST_LLVM_LIBNAME, RTLD_LAZY);
#endif
- return !!libraryHandle;
+ if (!libraryHandle) return false;
+ // LLVM's C API enum values changed in LLVM 23. A different runtime major
+ // misclassifies constants and branch instructions even when symbols resolve.
+ using GetVersion = void (*)(unsigned*, unsigned*, unsigned*);
+ auto getVersion = reinterpret_cast<GetVersion>(resolveSymbol("LLVMGetVersion"));
+ unsigned major = 0, minor = 0, patch = 0;
+ if (getVersion) getVersion(&major, &minor, &patch);
+ if (major != LLVM_VERSION_MAJOR) {
+ finit();
+ throw std::runtime_error("Iridium LLVM version mismatch: built with LLVM " +
+ std::to_string(LLVM_VERSION_MAJOR) + ", loaded LLVM " + std::to_string(major) +
+ " from " HOST_LLVM_LIBNAME "; rebuild and reinstall the runtime with matching LLVM headers and library");
+ }
+ return true;
};
void Iridium::DynamicLLVM::finit() {
diff --git a/private-include/indium/completion-spin.private.hpp b/private-include/indium/completion-spin.private.hpp
new file mode 100644
index 0000000..962fdad
--- /dev/null
+++ b/private-include/indium/completion-spin.private.hpp
@@ -0,0 +1,18 @@
+#pragma once
+#include <cstdint>
+
+namespace Indium {
+// A malformed override uses the vendor default. Saturation bounds both CPU use
+// and the microseconds-to-nanoseconds conversion, even for arbitrarily long input.
+inline uint64_t completionSpinBudget(uint32_t vendorID, const char* overrideValue) {
+ const uint64_t fallback = vendorID == 0x10de ? 2000000 : 0;
+ if (!overrideValue || !*overrideValue) return fallback;
+ uint64_t microseconds = 0;
+ for (const char* p = overrideValue; *p; ++p) {
+ if (*p < '0' || *p > '9') return fallback;
+ microseconds = microseconds * 10 + (*p - '0');
+ if (microseconds > 10000) microseconds = 10000;
+ }
+ return microseconds * 1000;
+}
+}
Reference in new issue View Git Blame Copy Permalink