diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 57ab097a4d..195fb08679 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -2445,7 +2445,7 @@ void ShaderResources::buildComplete() { for (auto element = first; element < last; ++element) { const auto index = binding.imageAllocations[element]; if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) images.push_back({samplers[index]->Handle(), VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED}); - else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) images.push_back({VK_NULL_HANDLE, storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL}); + else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) images.push_back({VK_NULL_HANDLE, storageAtomic64[index] ? storageTextures[index]->Atomic64View(storageMips[index], storageFirstLayer[index]) : storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL}); else images.push_back({VK_NULL_HANDLE, textureFirstLayer[index] ? textures[index]->FirstLayerView() : textures[index]->View(), textures[index]->Layout()}); } if (recorded) runs.emplace_back(write.dstArrayElement, write.descriptorCount); @@ -5226,7 +5226,7 @@ void ShaderResources::resolveImageTables() { storageFirstLayer.push_back(firstLayer); storageWritten.push_back(true); storageAtomic.push_back(element < binding.imageAtomic.size() && binding.imageAtomic[element]); - storageAtomic64.push_back(false); + storageAtomic64.push_back(element < binding.imageAtomic64.size() && binding.imageAtomic64[element]); describedRanges.push_back({"storage", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast(resource.tileMode), resource.dccAddress, true}); bindings[storageBinding].imageAllocations[absolute] = storageTextures.size() - 1u; continue; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 91adec15aa..83f380d3c9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -3367,6 +3367,19 @@ void verifyImageTableRegistration() { const auto* atomicTable = TableBinding(*atomic.result, DescriptorRole::ImageStorageTable); require((atomicFault & Abi::PoisonFlag) == 0u && atomicTable != nullptr && atomicTable->count != 0u && atomicTable->imageAtomic.size() == atomicTable->count && std::all_of(atomicTable->imageWritten.begin(), atomicTable->imageWritten.end(), [](bool written) { return written; }) && std::all_of(atomicTable->imageAtomic.begin(), atomicTable->imageAtomic.end(), [](bool atomic) { return atomic; }), "image table registration: the image_atomic_add table was not bound as atomic storage elements"); require(TableBinding(*atomic.result, DescriptorRole::ImageTable) == nullptr || TableBinding(*atomic.result, DescriptorRole::ImageTable)->count == 0u, "image table registration: an atomic-only table bound sampled elements"); + auto atomic64Code = storeCode; + atomic64Code[8] = 0xf0440308u; + guest.heap[0] = TableT(texture.bytes.data(), TableFormat32_32UInt, TableType2D); + static constexpr std::array int64ImageCapabilities{29u, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, spv::CapabilityInt64ImageEXT}; + auto atomic64Request = TableRequest(atomic64Code, userData); + atomic64Request.target.supportedCapabilities = int64ImageCapabilities; + const auto atomic64 = CompileTable(atomic64Request); + const auto& atomic64Plan = *atomic64.capture->plan; + require(atomic64Plan.info.images.size() == 1u && atomic64Plan.info.images[0].table == 0u && atomic64Plan.info.images[0].tableOperation == TableOperation::Supported, "image table registration: a table T# feeding image_atomic64_add was not a supported table operation"); + const auto& atomic64Map = TableMap(*atomic64.result); + const auto atomic64Fault = atomic64Map[Abi::TableHeader(0) + Abi::TableFault]; + const auto* atomic64Table = TableBinding(*atomic64.result, DescriptorRole::ImageStorageTable); + require((atomic64Fault & Abi::PoisonFlag) == 0u && atomic64Table != nullptr && atomic64Table->count != 0u && atomic64Table->imageAtomic64.size() == atomic64Table->count && std::all_of(atomic64Table->imageAtomic64.begin(), atomic64Table->imageAtomic64.end(), [](bool atomic) { return atomic; }), "image table registration: the image_atomic64_add table was not bound as 64-bit atomic storage elements"); const std::array, 2> unsupportedStores{{{0xf0240f08u, "image_store_mip"}, {0xf0280f08u, "image_store_pck"}}}; for (const auto& [instruction, name] : unsupportedStores) { auto code = storeCode; @@ -3797,7 +3810,7 @@ void verifySelectedRecordDirectOperations() { require(word0 == 0xf0440308u || (TableBinding(*compiled.result, DescriptorRole::ImageTable) == nullptr && compiled.result->imageTableFaults == 0u && TableCovered(compiled.regions, &heap[2], sizeof(heap[2]))), (what + " through a T# whose record the CPU evaluates was not bound from its record").c_str()); const auto wholeCode = WithImageOperation(TableWholeCode, word0, false); const auto whole = GetResourcePlan(TableRequest(wholeCode, userData)); - require(whole->info.images.size() == 1u && whole->info.images[0].table == 0u && whole->info.images[0].tableOperation == (word0 == 0xf0200f08u || word0 == 0xf0440108u ? TableOperation::Supported : TableOperation::Unsupported), (what + " through a T# keyed per lane left the image table or has the wrong table operation").c_str()); + require(whole->info.images.size() == 1u && whole->info.images[0].table == 0u && whole->info.images[0].tableOperation == (word0 == 0xf0200f08u || word0 == 0xf0440108u || word0 == 0xf0440308u ? TableOperation::Supported : TableOperation::Unsupported), (what + " through a T# keyed per lane left the image table or has the wrong table operation").c_str()); } const auto mixedCode = WithImageOperation(TableSrtKeyCode, 0xf0200f08u, true); const auto mixed = GetResourcePlan(TableRequest(mixedCode, userData)); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp index d5a89a1586..5ee8abca21 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp @@ -47,6 +47,7 @@ constexpr std::uint32_t MaxGroups = 8; constexpr std::uint32_t Format32UInt = 20; constexpr std::uint32_t Format32SInt = 21; constexpr std::uint32_t Format32Float = 22; +constexpr std::uint32_t Format32_32UInt = 62; constexpr std::uint32_t Format11_11_10UInt = 34; constexpr std::uint32_t Format8_8Srgb = 129; constexpr std::uint32_t Type2D = 9; @@ -56,6 +57,7 @@ constexpr std::uint32_t DepthSwizzleMode = 0x18; alignas(256) std::array FloatTexels{}; alignas(256) std::array OtherFloatTexels{}; alignas(256) std::array UintTexels{}; +alignas(256) std::array Uint64Texels{}; alignas(256) std::array SintTexels{}; alignas(4096) std::array DepthStorage{}; alignas(256) std::array Table{}; @@ -208,6 +210,12 @@ alignas(256) constexpr std::array AtomicAddLaneTableCode{ 0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e040280u, 0x7e080301u, 0xf0448108u, 0x00090401u, 0xbf810000u, }; +alignas(256) constexpr std::array AtomicAdd64LaneTableCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0x8f108202u, 0xf4200444u, 0x20000000u, 0x9312b011u, + 0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e040280u, 0x7e080301u, 0x7e0a02ffu, 0x00000000u, 0xf0448308u, + 0x00090401u, 0xbf810000u, +}; + alignas(256) constexpr std::array AtomicSwapLaneTableCode{ 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0x8f108202u, 0xf4200444u, 0x20000000u, 0x9312b011u, 0xf42c0802u, 0x24000000u, 0x360200bfu, 0x7e040280u, 0x7e080301u, 0xf03c8108u, 0x00090401u, 0xbf810000u, @@ -1168,6 +1176,32 @@ void RunAtomicTests(AgcDriver::VulkanDevice& device) { for (std::uint32_t x = 0; x < Width; ++x) Require(UintTexels[x] == (x ^ 1u) && std::bit_cast(FloatTexels[x]) == 0x3f800000u + x, "image table atomic: an add through a null entry reached an image:\n" + null.log); } +void RunAtomic64Tests(AgcDriver::VulkanDevice& device) { + if (!TargetHasCapability(device.Target(), spv::CapabilityInt64ImageEXT)) { + std::puts("skipped the 64-bit table atomic rows, the device has no shaderImageInt64Atomics"); + return; + } + struct Registration { + void* data; + Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, true); } + ~Registration() { GuestAllocations::Mutation().Remove(data); } + } registered(Uint64Texels.data(), sizeof(Uint64Texels)); + SetClassRecords(); + for (std::uint32_t x = 0; x < Width; ++x) Uint64Texels[x] = 0x5000000000000000ull + x; + SetRecord(9, PointClamp, Image(Uint64Texels.data(), Format32_32UInt)); + Keys = {}; + Keys[0] = 9; + const auto added = Run(device, AtomicAdd64LaneTableCode, 1, Lanes, 64); + AgcDriver::Graphics::StorageTexture::FlushAllPending("image table atomic64 test"); + Require(!Faulted(added), "image table atomic64: an add through a table faulted:\n" + added.log); + Require(added.imageTableCount != 0u, "image table atomic64: the add bound no storage table elements"); + for (std::uint32_t x = 0; x < Width; ++x) Require(Uint64Texels[x] == 0x5000000000000000ull + x + x, "image table atomic64: texel " + std::to_string(x) + " is wrong"); + Keys[0] = 6; + const auto null = Run(device, AtomicAdd64LaneTableCode, 1, Lanes, 64); + AgcDriver::Graphics::StorageTexture::FlushAllPending("image table atomic64 test"); + for (std::uint32_t x = 0; x < Width; ++x) Require(Uint64Texels[x] == 0x5000000000000000ull + x + x, "image table atomic64: an add through a null entry reached an image:\n" + null.log); +} + void RunSelectedStoreTests(AgcDriver::VulkanDevice& device) { alignas(256) static std::array storeTexels{}; struct Registration { @@ -1561,6 +1595,7 @@ int main() { RunModeTests(*device); RunStoreTests(*device); RunAtomicTests(*device); + RunAtomic64Tests(*device); RunSelectedStoreTests(*device); RunPointerTests(*device); RunPointerRecordTests(*device); diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp index ae3696883c..ad1a0f01b5 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp @@ -21,6 +21,7 @@ struct ImageTableElement { DescriptorBindingKind kind = DescriptorBindingKind::Buffers; std::uint32_t samplers = 0; bool atomic = false; + bool atomic64 = false; }; struct ImageTableSamplerElement { diff --git a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp index f11023d4b4..f9a4d4e0b9 100644 --- a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp +++ b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp @@ -616,6 +616,7 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con physical.imageDepthCompare.push_back(element.depthCompare); physical.imageWritten.push_back(ImageBindingResourceClass(element.kind) == ImageResourceClass::Storage); physical.imageAtomic.push_back(element.atomic); + physical.imageAtomic64.push_back(element.atomic64); std::uint32_t samplers = 0; for (std::uint32_t sampler = 0; sampler < entry.tableSamplerMasks.size(); ++sampler) { if ((element.samplers & (1u << sampler)) != 0u) samplers |= entry.tableSamplerMasks[sampler]; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 921b9bb653..89ee86f9d6 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -1716,17 +1716,17 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i auto element = words.dwords; if (mode.conversionFormat != IrBufferFormat::Invalid || mode.depthBits) element[3] = (element[3] & ~0xfffu) | ShaderImageIdentitySwizzle; const auto kind = DescriptorBindingForImage(mode); - const auto hash = hashWords(element, (static_cast(image.atomic) << 24u) ^ (static_cast(mode.dimension) << 16u) ^ (static_cast(mode.depthCompare) << 8u) ^ static_cast(kind)); + const auto hash = hashWords(element, (static_cast(image.atomic64) << 25u) ^ (static_cast(image.atomic) << 24u) ^ (static_cast(mode.dimension) << 16u) ^ (static_cast(mode.depthCompare) << 8u) ^ static_cast(kind)); auto index = elementIndex.Find(hash, [&](std::uint32_t known) { const auto& entry = out.elements[known]; - return entry.words == element && entry.dimension == mode.dimension && entry.depthCompare == mode.depthCompare && entry.kind == kind && entry.atomic == image.atomic; + return entry.words == element && entry.dimension == mode.dimension && entry.depthCompare == mode.depthCompare && entry.kind == kind && entry.atomic == image.atomic && entry.atomic64 == image.atomic64; }); if (!index.has_value() && out.elements.size() >= Abi::ImageElementBudget) { record[Abi::ImageRecordFlags] = poison(resource, PoisonReason::Budget, &words); } else { if (!index.has_value()) { index = static_cast(out.elements.size()); - out.elements.push_back({element, mode.dimension, mode.depthCompare, kind, 0u, image.atomic}); + out.elements.push_back({element, mode.dimension, mode.depthCompare, kind, 0u, image.atomic, image.atomic64}); elementIndex.Insert(hash, *index); } out.elements[*index].samplers |= directSamplers[resource]; diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index ee6873ef41..d0d0482387 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -1775,7 +1775,7 @@ private: continue; } image.table = next++; - if (image.atomic64 || image.packed || image.byElements != 0u) { + if (image.packed || image.byElements != 0u) { image.tableOperation = TableOperation::Unsupported; } } diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 1e2974aba2..6c372caccf 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -54,7 +54,7 @@ - An [image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) T# of an MSAA or FMASK image, or of a color format under a depth compare, faults when selected: tables have no multisampled descriptors and no emulated color compare. - Direct [FMASK image loads](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) of 16 samples, CMASK compression (bar the 8-fragment native owner path), packed accesses and other layouts throw. Uncompressed FMASK of up to 8 samples reads a single-mip 2D or 2D-array 8-, 16- or 32-bit `UINT` snapshot with `Z_64KB_X` addressing; selectors may name X, zero or one. `Fmask32_S8_F8` returns the stored word; others expand to 4 bits per sample, an index at or past the fragment count reading 8 (unknown), samples past the count 0. Layouts follow PAL's [expanded values](https://github.com/GPUOpen-Drivers/pal/blob/c5e800072a32f68b6ccc4422936d96167c6e0728/src/core/hw/gfxip/gfx9/gfx9MaskRam.cpp#L5005) and [FMASK surfaces](https://github.com/GPUOpen-Drivers/pal/blob/c5e800072a32f68b6ccc4422936d96167c6e0728/src/core/imported/addrlib/src/core/addrlib2.cpp#L877). Console results, unused nibbles included, are unmeasured; compressed color sampling is unsupported. - A [DIM 2D read of a 2D MSAA T#](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) returns sample 0, which is not measured on the console's gfx1013, at an implicit LOD above 0, at an explicit one for texels over 16 bits except formats 22 and 56 in SW_64KB_R_X, or for other formats and swizzle modes; 64- and 128-bit SW_64KB_Z_X texels are not covered: gfx1031 does not address their samples. Such a read is an `image_load` without a mip, `image_sample`, `image_sample_lz` or `image_sample_l` of a direct sampled image; other accesses (gathers, bias, gradients, offsets, LOD clamps, depth compares, `image_load_mip`, `image_get_resinfo`, `image_get_lod`, PCK, BY2/BY4) throw, as does a depth-bits T# (16_UINT, 32_UINT or 32_SINT in a Z swizzle mode). On gfx1031 (RX 6700 XT, HIP), such reads return sample 0 at 2x, 4x and 8x for 8- to 128-bit texels in SW_64KB_R_X and 8- to 32-bit ones in SW_64KB_Z_X (formats 1, 5, 7, 11, 13, 14, 18, 22, 29, 56, 60, 64, 71, 77, 133). -- `image_store_mip`, `image_store_pck`, PCK2/PCK4 and MIMG BY2/BY4 through an [image table](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) T# fault (`unsupported operation`) when an active lane executes them; the store is dropped and the load reads zeros. 32-bit image atomics through a table lower to `OpAtomic*` on the table `AtomicView` (`Device` scope, relaxed, `AcquireRelease`/`UniformMemory` barrier, as the direct path) instead of faulting; 64-bit atomics still fault (no `Atomic64View` on the table path). +- `image_store_mip`, `image_store_pck`, PCK2/PCK4 and MIMG BY2/BY4 through an [image table](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) T# fault (`unsupported operation`) when an active lane executes them; the store is dropped and the load reads zeros. 32-bit and 64-bit image atomics through a table lower to `OpAtomic*` on the table `AtomicView`/`Atomic64View` (`Device` scope, relaxed, `AcquireRelease`/`UniformMemory` barrier as the direct path) instead of faulting; the host needs `VK_FORMAT_FEATURE_STORAGE_IMAGE_ATOMIC_BIT` for the view format. Packed formats and BY element counts still fault. - An [image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) S# with a min or max reduction or FORCE_UNNORMALIZED faults when selected, as does an integer, converted or depth-bits table T# paired with a direct min or max S# that filters between texels. - Uint and Sint [image table](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp) entries and direct T#s under a table S# sample through a point-filtered copy of the S#: Vulkan leaves linear filtering of integer formats undefined; the console is assumed to point-sample. - Comparison sampling of a color texture through an [image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) S# throws at the first dispatch or draw that binds it.