diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 6bf99b3a18..dac4d2e473 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1742,6 +1742,8 @@ target_include_directories(agc_driver_buffer_descriptor_runtime_tests PRIVATE $< target_include_directories(agc_driver_shutdown_threads_tests PRIVATE $) +target_include_directories(agc_driver_image_converted_unorm_tests PRIVATE $) + target_include_directories(agc_driver_prepared_publication_tests PRIVATE $) target_include_directories(agc_driver_prepared_graphics_tests PRIVATE $) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp index 81ebec7f75..b304c65624 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp @@ -158,6 +158,7 @@ struct PreparedShaders : PreparedShaderState { std::exception_ptr failure; std::atomic demanded{false}; bool pending = false; + std::vector convertedSamples; }; struct RegisteredShaderState { @@ -217,6 +218,7 @@ ShaderRecompiler::RectListShaders DrawRectangle(const ShaderSnapshot& front, con std::shared_ptr SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request); ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request); +std::shared_ptr CaptureInvocation(ShaderMemory& memory, const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request, ShaderRecompiler::PreparedShaderInvocation& invocation); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp index a13acf5c8b..ae76ddf5bb 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp @@ -13,8 +13,8 @@ void Driver::verifyDataHit(const ShaderSnapshot& snapshot, std::size_t codeOffse std::abort(); }; auto verifyMemory = std::make_shared(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter()); - const auto invocation = InvocationFor(snapshot, codeOffset, request); - const auto capture = verifyMemory->Capture(invocation); + auto invocation = InvocationFor(snapshot, codeOffset, request); + const auto capture = CaptureInvocation(*verifyMemory, snapshot, codeOffset, request, invocation); const auto regions = verifyMemory->Regions(); request.context.memory = regions; const auto result = invocation.Materialize(*capture); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index 4b67aa557c..565d951689 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -331,14 +331,14 @@ void Driver::dispatch(QueueState& queue, std::span packet, static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr; try { - const auto invocation = InvocationFor(snapshot, codeOffset, request); + auto invocation = InvocationFor(snapshot, codeOffset, request); timing.Mark("prepared_invocation"); const auto waitedBefore = traceCapSync() ? Graphics::Recorder::ThreadWaitedMs() : 0.0; forgetAtCapture = GuestMemory::ForgetSerial(); capture = [&] { const SampledReadScope sampling(evidenceReads); - return shaderMemory->Capture(invocation); + return CaptureInvocation(*shaderMemory, snapshot, codeOffset, request, invocation); }(); captured = shaderMemory->Regions(); request.context.memory = captured; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp index 616d63c690..c761433da8 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp @@ -27,7 +27,7 @@ std::shared_ptr Driver::materializeDraw ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, graphics.stages.mesh, graphics.stages.tessellation, {drawParameters.indexAddress, drawParameters.indexCount, drawParameters.indexSize, drawParameters.instanceCount}} }; const auto waitedBefore = traceCapSync() || profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; - const auto invocation = InvocationFor(*program.snapshot, program.codeOffset, request); + auto invocation = InvocationFor(*program.snapshot, program.codeOffset, request); timing.Mark("prepared_invocation"); auto& stageCapture = stageCaptures[i]; stageCapture.forgetSerial = GuestMemory::ForgetSerial(); @@ -49,7 +49,7 @@ std::shared_ptr Driver::materializeDraw const auto capture = [&] { const SampledReadScope sampling(evidenceReads); try { - return shaderMemory.Capture(invocation); + return CaptureInvocation(shaderMemory, *program.snapshot, program.codeOffset, request, invocation); } catch (const std::exception& error) { throw named(error); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index dcfaf75142..58293e3852 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -9,6 +9,7 @@ #include "prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp" #include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp" #include "CompiledVariant.hpp" +#include "Optimization/ResourceMaterializer.hpp" #include "Optimization/ResourceProgram.hpp" #include #include @@ -540,6 +541,10 @@ void PublishRegisteredShader(std::shared_ptr& registry, const st namespace { void ReportPreparedAtUse(const ShaderSnapshot& snapshot, const ShaderRecompiler::RecompileRequest& request) { + if (request.context.convertedSamples) { + APS5_LOG_OUT_DEBUG("Shader 0x%llx samples a converted image; preparing its converted-sample variant at use", static_cast(request.shader.codeAddress)); + return; + } if (snapshot.codeAddress == NullPixelProgramAddress() && request.shader.stage == ShaderRecompiler::ShaderStage::Fragment && request.context.waveSize == 32u) { APS5_LOG_ERR("The null pixel program has no wave%u artifact for this draw; preparing it at draw", request.context.waveSize); return; @@ -716,12 +721,13 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s source.shader.code = linked.code; source.context.linkedCallFallbacks = linked.fallbacks; } + std::unique_lock lock(snapshot.prepared->mutex); + AwaitRegisteredPreparation(*snapshot.prepared, lock); + if (std::ranges::find(snapshot.prepared->convertedSamples, codeOffset) != snapshot.prepared->convertedSamples.end()) source.context.convertedSamples = true; auto invocationRequest = source; struct PreparedKeyStorage {}; auto& key = HostThreadLocal, PreparedKeyStorage>(); ShaderRecompiler::BuildPreparedShaderKey(source, key); - std::unique_lock lock(snapshot.prepared->mutex); - AwaitRegisteredPreparation(*snapshot.prepared, lock); for (const auto& entry : snapshot.prepared->entries) { if (entry.codeOffset != codeOffset) continue; invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*entry.handle); @@ -737,6 +743,21 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s snapshot.prepared->entries.push_back({codeOffset, std::move(handle)}); return std::move(*invocation); } + +std::shared_ptr CaptureInvocation(ShaderMemory& memory, const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request, ShaderRecompiler::PreparedShaderInvocation& invocation) { + try { + return memory.Capture(invocation); + } catch (const ShaderRecompiler::ConvertedSampleRequired&) { + if (invocation.Request().context.convertedSamples) throw; + } + { + std::lock_guard lock(snapshot.prepared->mutex); + auto& offsets = snapshot.prepared->convertedSamples; + if (std::ranges::find(offsets, codeOffset) == offsets.end()) offsets.push_back(codeOffset); + } + invocation = InvocationFor(snapshot, codeOffset, request); + return memory.Capture(invocation); +} namespace { std::optional DecodeFloatMode(const Registers& registers, std::uint8_t type) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp index 252f9eb8b9..94de1a8025 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp @@ -2,6 +2,8 @@ #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libc/include/GuestAllocations.hpp" +#include "Optimization/ResourceMaterializer.hpp" +#include "Optimization/ResourceProgram.hpp" #include "Recompiler.hpp" #include "VulkanTestDevice.hpp" #ifdef _WIN32 @@ -48,6 +50,15 @@ constexpr std::size_t BlockBytes = 65536; constexpr std::size_t RoundTripOffset = 16384; constexpr std::size_t StoreOffset = 32768; constexpr std::array PointSampler{0x00000092u, 0x00fff000u, 0u, 0u}; +constexpr std::array BilinearSampler{0x00000092u, 0x00fff000u, 0x00500000u, 0u}; +constexpr std::array WrapBilinearSampler{0x00000090u, 0x00fff000u, 0x00500000u, 0u}; +constexpr std::array WrapPointSampler{0x00000090u, 0x00fff000u, 0u, 0u}; +constexpr std::array BorderSampler{0x000000b6u, 0x00fff000u, 0u, 0u}; +constexpr std::uint32_t SampleStep = 0x3a200000u; +constexpr std::uint32_t OffGridStep = 0x3a206000u; +constexpr std::uint32_t SampleStart = 0x39c00000u; +constexpr std::uint32_t WrappedStart = 0xbac00000u; +constexpr double SampleTolerance = 1.0e-5; alignas(256) std::array Output{}; alignas(256) std::array Input{}; @@ -73,6 +84,13 @@ constexpr std::array SamplerCode(std::uint32_t word0) { }; } +constexpr std::array SampleAtCode(std::uint32_t step, std::uint32_t start) { + return { + 0x34060084u, 0x7e280d00u, 0x102828ffu, step, 0x062828ffu, start, 0x7e2a0280u, 0xf09c0f08u, + 0x00610a14u, 0xbf8c3f70u, 0xe0781000u, 0x80000a03u, 0xbf810000u, + }; +} + alignas(256) constexpr std::array RoundTrip{ 0x7e080218u, 0x343c0885u, 0x4a3c3d00u, 0x7e3e0280u, 0xf0001708u, 0x00010a1eu, 0xbf8c3f70u, 0xf0201708u, 0x00040a1eu, 0xbf810000u, @@ -88,6 +106,10 @@ alignas(256) constexpr auto StoreD16 = StoreCode(0x7u, D16); alignas(256) constexpr auto SampleLz = SamplerCode(0xf09c0f08u); alignas(256) constexpr auto Gather4Lz = SamplerCode(0xf11c0108u); alignas(256) constexpr auto GetLod = SamplerCode(0xf1800308u); +alignas(256) constexpr auto SampleAt = SampleAtCode(SampleStep, SampleStart); +alignas(256) constexpr auto SampleWrapped = SampleAtCode(SampleStep, WrappedStart); +alignas(256) constexpr auto SampleOffGrid = SampleAtCode(OffGridStep, SampleStart); +alignas(256) constexpr auto SampleWrappedOffGrid = SampleAtCode(OffGridStep, WrappedStart); constexpr std::array StoreValues{ 0x00000000u, 0x3f800000u, 0x3f000000u, 0x3eaaaaabu, 0x80000000u, 0xbf800000u, @@ -333,6 +355,83 @@ void CheckFloatSample(AgcDriver::VulkanDevice& device, const std::uint8_t* texel } } +double ChannelValue(std::uint32_t texel, std::uint32_t swizzle, std::uint32_t component) { + const auto selector = (swizzle >> (component * 3u)) & 7u; + if (selector == 0u) return 0.0; + if (selector == 1u) return 1.0; + return static_cast(Field(texel, selector - 4u)) / (selector == 6u ? 1023.0 : 2047.0); +} + +void CheckSample(AgcDriver::VulkanDevice& device, const std::uint8_t* texels, std::span code, std::uint32_t step, std::uint32_t start, std::uint32_t swizzle, const std::array& sampler, bool linear, bool wrap, const std::string& what) { + Run(device, code, TextureDescriptor(texels, UnormFormat, swizzle, LoadWidth), sampler, 1); + const auto texelAt = [&](std::int64_t index) { + const auto width = static_cast(LoadWidth); + const auto addressed = wrap ? ((index % width) + width) % width : std::clamp(index, 0, width - 1); + std::uint32_t texel = 0; + std::memcpy(&texel, texels + addressed * 4u, 4u); + return texel; + }; + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto x = (static_cast(tid) * static_cast(std::bit_cast(step)) + static_cast(std::bit_cast(start))) * LoadWidth; + for (std::uint32_t component = 0; component < 4u; ++component) { + double expected = 0.0; + if (linear) { + const auto centre = x - 0.5; + const auto first = static_cast(std::floor(centre)); + const auto weight = std::nearbyint((centre - std::floor(centre)) * 256.0) / 256.0; + const auto left = ChannelValue(texelAt(first), swizzle, component); + const auto right = ChannelValue(texelAt(first + 1), swizzle, component); + expected = left + (right - left) * weight; + } else { + expected = ChannelValue(texelAt(static_cast(std::floor(x))), swizzle, component); + } + const auto actual = static_cast(std::bit_cast(Output[tid * 4u + component])); + Require(std::fabs(actual - expected) <= SampleTolerance * std::max(1.0, std::fabs(expected)), what + ": thread " + std::to_string(tid) + " component " + std::to_string(component) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected)); + } + } +} + +void CheckConvertedPreparation(AgcDriver::VulkanDevice& device, const std::uint8_t* texels) { + std::vector userData(24, 0u); + const auto texture = TextureDescriptor(texels, UnormFormat, SwizzleXYZ1, LoadWidth); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + std::copy(PointSampler.begin(), PointSampler.end(), userData.begin() + 12); + const std::span code = SampleAt; + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {true, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + auto converted = request; + converted.context.convertedSamples = true; + const auto convertedMode = [](const ShaderRecompiler::ImageResource& mode) { return mode.conversionFormat == ShaderRecompiler::IrBufferFormat::Format11_11_10UNorm; }; + const auto plain = ShaderRecompiler::GetResourcePlan(request); + Require(plain->info.images.size() == 1u && std::ranges::none_of(plain->info.runtimeImageModes.at(0), convertedMode), "a sampled image is prepared with converted-sample modes before a converted descriptor was seen"); + const auto prepared = ShaderRecompiler::GetResourcePlan(converted); + Require(prepared->info.images.size() == 1u && std::ranges::count_if(prepared->info.runtimeImageModes.at(0), convertedMode) == 1, "a sampled image is not prepared with the converted unorm sample mode once a converted descriptor was seen"); + const auto handle = ShaderRecompiler::PrepareShader(request); + Require(!ShaderRecompiler::PreparedShaderInvocation::TryCreate(converted, handle).has_value(), "an artifact prepared without converted-sample modes matches a request that asks for them"); + const auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(request, handle); + Require(invocation.has_value(), "the prepared artifact does not match its own request"); + ShaderRecompiler::SrtRuntime runtime; + runtime.userData = userData; + runtime.shaderBase = request.shader.codeAddress; + std::string reason; + try { + static_cast(invocation->Capture(runtime)); + } catch (const ShaderRecompiler::ConvertedSampleRequired& error) { + reason = error.what(); + } + Require(!reason.empty(), "capturing a converted descriptor through an artifact prepared without converted-sample modes does not ask for them"); + const auto convertedHandle = ShaderRecompiler::PrepareShader(converted); + const auto convertedInvocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(converted, convertedHandle); + Require(convertedInvocation.has_value(), "the artifact prepared with converted-sample modes does not match its request"); + static_cast(convertedInvocation->Capture(runtime)); +} + void RequireRefused(AgcDriver::VulkanDevice& device, std::span code, const std::uint8_t* texels, std::uint32_t swizzle, const std::array& extra, const std::string& reason, const std::string& what, std::uint32_t format = UnormFormat) { std::string refusal; try { @@ -368,7 +467,14 @@ int main() { CheckFloatSample(*device, texels + FloatOffset, {0u, 0u, 0x7fc00000u, 0x3f800000u}, "image_sample_lz of R10_G11_B11_FLOAT results still on the GPU"); CheckFloatLoad(*device, texels + FloatOffset); CheckFloatSample(*device, texels + FloatOffset, {0x7f800000u, 0x7f800000u, 0x387c0000u, 0x3f800000u}, "image_sample_lz of R10_G11_B11_FLOAT"); - RequireRefused(*device, SampleLz, texels, SwizzleXYZ1, PointSampler, "sampling, gathering or querying LOD of a converted unorm image", "image_sample_lz"); + CheckSample(*device, texels, SampleAt, SampleStep, SampleStart, SwizzleXYZ1, PointSampler, false, false, "point image_sample_lz X Y Z 1"); + CheckSample(*device, texels, SampleAt, SampleStep, SampleStart, SwizzleZYX1, BilinearSampler, true, false, "bilinear image_sample_lz Z Y X 1"); + CheckSample(*device, texels, SampleWrapped, SampleStep, WrappedStart, SwizzleX011, WrapPointSampler, false, true, "wrapped point image_sample_lz X 0 1 1"); + CheckSample(*device, texels, SampleWrapped, SampleStep, WrappedStart, SwizzleXYZ1, WrapBilinearSampler, true, true, "wrapped bilinear image_sample_lz X Y Z 1"); + CheckSample(*device, texels, SampleOffGrid, OffGridStep, SampleStart, SwizzleZYX1, BilinearSampler, true, false, "off-grid bilinear image_sample_lz Z Y X 1"); + CheckSample(*device, texels, SampleWrappedOffGrid, OffGridStep, WrappedStart, SwizzleXYZ1, WrapBilinearSampler, true, true, "off-grid wrapped bilinear image_sample_lz X Y Z 1"); + CheckConvertedPreparation(*device, texels); + RequireRefused(*device, SampleLz, texels, SwizzleXYZ1, BorderSampler, "implemented only with wrap or clamp-to-edge addressing", "image_sample_lz with clamp-to-border addressing"); RequireRefused(*device, Gather4Lz, texels, SwizzleXYZ1, PointSampler, "sampling, gathering or querying LOD of a converted unorm image", "image_gather4_lz"); RequireRefused(*device, GetLod, texels, SwizzleXYZ1, PointSampler, "sampling, gathering or querying LOD of a converted unorm image", "image_get_lod"); RequireRefused(*device, LoadD16, texels, SwizzleXYZ1, {}, "converted unorm image with 16-bit data", "image_load d16"); diff --git a/core/shader/recompiler/CacheKey.hpp b/core/shader/recompiler/CacheKey.hpp index f501252286..c3a4160bca 100644 --- a/core/shader/recompiler/CacheKey.hpp +++ b/core/shader/recompiler/CacheKey.hpp @@ -52,6 +52,7 @@ public: append(key, request.context.pixel); append(key, request.context.vertex); append(key, request.context.floatMode); + append(key, request.context.convertedSamples); appendFallbacks(key, request); appendMesh(key, request); appendTessellation(key, request); @@ -85,6 +86,7 @@ private: append(key, request.context.pixel); append(key, request.context.vertex); append(key, request.context.floatMode); + append(key, request.context.convertedSamples); appendFallbacks(key, request); appendMesh(key, request); appendTessellation(key, request); diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp index 4750cccb14..5fe5a3aa87 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp @@ -33,6 +33,8 @@ struct BufferResource { enum class ImageMipMode { None, DynamicStorage }; namespace EmulatedCompare { +inline constexpr std::uint32_t ConvertedSampleUnsupported = 1u << 27u; +inline constexpr std::uint32_t ConvertedSampleRequiresSingleLevel = 1u << 28u; inline constexpr std::uint32_t NativeOffsetUnsupported = 1u << 29u; inline constexpr std::uint32_t Unsupported = 1u << 31u; inline constexpr std::uint32_t RequiresSingleLevel = 1u << 30u; @@ -94,6 +96,7 @@ struct ImageResource { bool flatLineCompatible = true; bool flatVolumeLineCompatible = true; bool foldTexelOffsetsOnly = false; + bool convertedSamples = false; std::uint32_t byElements = 0; std::uint32_t byComponents = 0; IrBufferFormat packedFormat = IrBufferFormat::Invalid; diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp index 2a33e8a6d8..8b787835b9 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -47,6 +48,11 @@ struct RuntimeImageModeMatch { std::string detail; }; +class ConvertedSampleRequired : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + class ResourceMaterializer { public: static std::uint32_t EmulatedCompareState(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint32_t index); diff --git a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp index 79e73779e2..1dd71e0f65 100644 --- a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp +++ b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp @@ -301,7 +301,8 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l } std::vector compareStates(info.images.size()); for (std::uint32_t index = 0; index < info.images.size(); ++index) { - if (!info.images[index].depthCompare) continue; + const bool convertedSample = !info.images[index].srgbDecodeCompatible && index < info.runtimeImageModes.size() && std::ranges::any_of(info.runtimeImageModes.at(index), [](const ImageResource& mode) { return mode.conversionFormat == IrBufferFormat::Format11_11_10UNorm; }); + if (!info.images[index].depthCompare && !convertedSample) continue; const auto compare = compareStates[index] = ResourceMaterializer::EmulatedCompareState(info, snapshot, index); const auto first = PipelineSpecialization::CompareBase + index * PipelineSpecialization::CompareWords; const std::array values{EmulatedCompare::Function(compare), (compare & EmulatedCompare::Linear) != 0u, EmulatedCompare::AddressX(compare), EmulatedCompare::AddressY(compare), EmulatedCompare::Reference(compare), (compare & EmulatedCompare::BorderWhite) != 0u}; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index ca0eef25c8..c5f59e3afb 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -228,7 +228,13 @@ ImageDecode inspectImageDescriptor(const DescriptorValue& descriptor, const Imag decoded.conversionFormat = RemapTextureFormat(format) != format && !nativeRepacked ? format : IrBufferFormat::Invalid; if (format == IrBufferFormat::Format11_11_10UNorm || (format == IrBufferFormat::Format10_11_11Float && !nativeRepacked)) { const bool floating = format == IrBufferFormat::Format10_11_11Float; - if (!base.srgbDecodeCompatible) return rejectImage(PoisonReason::Conversion, floating ? "samples or gathers a converted float image, or queries its LOD, which is not implemented" : "sampling, gathering or querying LOD of a converted unorm image is not implemented"); + if (!base.srgbDecodeCompatible) { + if (floating) return rejectImage(PoisonReason::Conversion, "samples or gathers a converted float image, or queries its LOD, which is not implemented"); + if (storage || base.depthCompare || (base.emulatedCompare & EmulatedCompare::ConvertedSampleUnsupported) != 0u) return rejectImage(PoisonReason::Conversion, "sampling, gathering or querying LOD of a converted unorm image is not implemented"); + if (base.table != NoTable) return rejectImage(PoisonReason::Conversion, "samples a converted unorm image selected through an image table, which is not implemented"); + if (decoded.cube || (decoded.dimension != RdnaImageDimension::Dim2D && decoded.dimension != RdnaImageDimension::Dim2DArray)) return rejectImage(PoisonReason::Conversion, "samples a converted unorm image that is not a 2D or 2D array view, which is not implemented"); + if ((base.emulatedCompare & EmulatedCompare::ConvertedSampleRequiresSingleLevel) != 0u && ((descriptor.dwords[3] >> 12u) & 0xfu) != ((descriptor.dwords[3] >> 16u) & 0xfu)) return rejectImage(PoisonReason::Conversion, "samples a converted unorm image with several mip levels at a LOD that is not 0, which is not implemented"); + } if (!base.depthBitsCompatible) return rejectImage(PoisonReason::Conversion, floating ? "reads or writes a converted float image with 16-bit data, which is not implemented" : "reads or writes a converted unorm image with 16-bit data, which is not implemented"); for (std::uint32_t component = 0; component < 4u; ++component) { if (((descriptorImageSwizzle(descriptor) >> (component * 3u)) & 7u) == 7u) return rejectImage(PoisonReason::Conversion, "selects a channel the converted image format does not have"); @@ -1124,10 +1130,45 @@ std::uint32_t colorCompareReference(IrBufferFormat format) { } } +bool convertedSampleCompatible(const ImageResource& image) { + return image.resourceClass != ImageResourceClass::Storage && !image.depthCompare && !image.atomic && !image.packed && image.table == NoTable && image.depthBitsCompatible && (image.emulatedCompare & EmulatedCompare::ConvertedSampleUnsupported) == 0u && (image.dimension == RdnaImageDimension::Dim2D || image.dimension == RdnaImageDimension::Dim2DArray); +} + +std::uint32_t convertedSampleState(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint32_t index) { + const auto& image = info.images[index]; + if (image.srgbDecodeCompatible || image.depthCompare || image.resourceClass == ImageResourceClass::Storage || index >= snapshot.images.size()) return 0u; + const auto& descriptor = snapshot.images[index]; + if (descriptor.dwordCount != 8u || nullImageDescriptor(image, descriptor)) return 0u; + const auto format = rawImageFormat(descriptor); + if (format != IrBufferFormat::Format11_11_10UNorm) return 0u; + std::optional samplerState; + for (const auto& pair : info.sampledPairs) { + if (pair.image != index) continue; + if (pair.sampler >= snapshot.samplers.size() || snapshot.samplers[pair.sampler].dwordCount != 4u) throw std::runtime_error("sampling of a converted image has no sampler descriptor"); + const auto& words = snapshot.samplers[pair.sampler].dwords; + if (((words[0] >> 29u) & 0x3u) != 0u) throw std::runtime_error("sampling of a converted image through a min or max reduction sampler is not implemented"); + if (((words[0] >> 15u) & 0x1u) != 0u) throw std::runtime_error("sampling of a converted image does not implement unnormalized coordinates"); + const auto magFilter = (words[2] >> 20u) & 0x3u; + const auto minFilter = (words[2] >> 22u) & 0x3u; + if (magFilter != minFilter || magFilter > 1u) throw std::runtime_error("sampling of a converted image is implemented only with equal point or bilinear minification and magnification filters"); + const auto addressMode = [](std::uint32_t clamp) { + if (clamp == 0u) return EmulatedCompare::AddressWrap; + if (clamp == 2u) return EmulatedCompare::AddressEdge; + throw std::runtime_error("sampling of a converted image is implemented only with wrap or clamp-to-edge addressing"); + }; + const auto state = EmulatedCompare::Enabled | (magFilter == 1u ? EmulatedCompare::Linear : 0u) | (addressMode(words[0] & 0x7u) << EmulatedCompare::ClampXShift) | (addressMode((words[0] >> 3u) & 0x7u) << EmulatedCompare::ClampYShift); + if (samplerState.has_value() && *samplerState != state) throw std::runtime_error("sampling of a converted image through samplers that disagree is not implemented"); + samplerState = state; + } + if (!samplerState.has_value()) throw std::runtime_error("sampling of a converted image has no paired sampler"); + return *samplerState; +} + std::uint32_t emulatedCompareState(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint32_t index) { const auto& image = info.images[index]; const auto& descriptor = snapshot.images[index]; - if (!image.depthCompare || descriptor.dwordCount != 8u || nullImageDescriptor(image, descriptor)) return 0u; + if (!image.depthCompare) return convertedSampleState(info, snapshot, index); + if (descriptor.dwordCount != 8u || nullImageDescriptor(image, descriptor)) return 0u; const auto format = rawImageFormat(descriptor); if (format == IrBufferFormat::Format32Float || format == IrBufferFormat::Format16UNorm || IsDepthBitsTexture(descriptor.dwords[1], descriptor.dwords[3])) return 0u; if (image.table != NoTable || (image.emulatedCompare & EmulatedCompare::Unsupported) != 0u) throw std::runtime_error("unsupported color comparison image instructions"); @@ -1206,7 +1247,11 @@ void materializeTables(const IrResourcePlan& plan, ResourceSnapshot& snapshot) { for (std::uint32_t i = 0u; i < plan.info.images.size(); ++i) { const auto& image = plan.info.images[i]; if (image.table != NoTable) continue; - static_cast(ResourceMaterializer::RuntimeImageMode(image, snapshot.images.at(i), plan.info.runtimeImageModes.at(i))); + const auto& descriptor = snapshot.images.at(i); + if (!image.convertedSamples && !image.srgbDecodeCompatible && convertedSampleCompatible(image) && descriptor.dwordCount == 8u && !nullImageDescriptor(image, descriptor)) { + if (rawImageFormat(descriptor) == IrBufferFormat::Format11_11_10UNorm) throw ConvertedSampleRequired("samples a converted image through a shader prepared without converted-sample modes"); + } + static_cast(ResourceMaterializer::RuntimeImageMode(image, descriptor, plan.info.runtimeImageModes.at(i))); } } @@ -1553,10 +1598,8 @@ std::vector ResourceMaterializer::RuntimeImageModes(const ImageRe append(IrTextureNumericClass::Uint, IrBufferFormat::Invalid, IrBufferFormat::Invalid, false, false); if (!storage) append(IrTextureNumericClass::Sint, IrBufferFormat::Invalid, IrBufferFormat::Invalid, false, false); append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UInt, IrBufferFormat::Invalid, false, false); - if (image.srgbDecodeCompatible && image.depthBitsCompatible) { - append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UNorm, IrBufferFormat::Invalid, false, false); - append(IrTextureNumericClass::Uint, IrBufferFormat::Format10_11_11Float, IrBufferFormat::Invalid, false, false); - } + if ((image.srgbDecodeCompatible || (image.convertedSamples && convertedSampleCompatible(image))) && image.depthBitsCompatible) append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UNorm, IrBufferFormat::Invalid, false, false); + if (image.srgbDecodeCompatible && image.depthBitsCompatible) append(IrTextureNumericClass::Uint, IrBufferFormat::Format10_11_11Float, IrBufferFormat::Invalid, false, false); if (!storage) { if (image.depthBitsCompatible) { append(IrTextureNumericClass::Float, IrBufferFormat::Invalid, IrBufferFormat::Invalid, true, false); diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index 73b74dad2a..1cecfa9918 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -1644,6 +1644,9 @@ private: constexpr auto unsupported = RdnaImageSampleFlagLod | RdnaImageSampleFlagDerivative; if (op == IrOpcode::ImageGatherRaw || (memory.imageSampleFlags & unsupported) != 0u) image.emulatedCompare |= EmulatedCompare::Unsupported; if ((memory.imageSampleFlags & RdnaImageSampleFlagLevelZero) == 0u) image.emulatedCompare |= EmulatedCompare::RequiresSingleLevel; + } else if (ImageOpcodeInfoOf(op).needsSampler) { + if (op != IrOpcode::ImageSampleRaw) image.emulatedCompare |= EmulatedCompare::ConvertedSampleUnsupported; + if ((memory.imageSampleFlags & RdnaImageSampleFlagLevelZero) == 0u) image.emulatedCompare |= EmulatedCompare::ConvertedSampleRequiresSingleLevel; } } diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index d08bc8a148..b38254cbf6 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -299,6 +299,9 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request, ShaderPreparat deadCodeEliminator.Eliminate(program); dumpIr("resources"); program.Resources().srgbDecodeFormats = request.target.srgbDecodeFormats; + if (request.context.convertedSamples) { + for (auto& image : program.Resources().info.images) image.convertedSamples = true; + } return program; } @@ -1110,7 +1113,7 @@ RecompileResult materializeVariant(SourceEntry& source, const RecompileRequest& return result; } -RecompileResult RecompileImpl(const RecompileRequest& request) { +RecompileResult recompileOnce(const RecompileRequest& request) { static_cast(RequestInputInfo(request)); RequestMemoryView memory(request.context.memory); const auto runtime = memory.MakeRuntime(request.context.userData, request.shader.codeAddress); @@ -1128,6 +1131,17 @@ RecompileResult RecompileImpl(const RecompileRequest& request) { return materializeVariant(*source, request, snapshot); } +RecompileResult RecompileImpl(const RecompileRequest& request) { + try { + return recompileOnce(request); + } catch (const ConvertedSampleRequired&) { + if (request.context.convertedSamples) throw; + } + auto converted = request; + converted.context.convertedSamples = true; + return recompileOnce(converted); +} + // APS5_NO_RESULT_MEMO=1: every Recompile(request, capture) materializes its own result as before. bool ResultMemo() { static const bool resultMemo = std::getenv("APS5_NO_RESULT_MEMO") == nullptr; diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 38de551273..312c4eeebf 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -222,6 +222,7 @@ struct GuestContext { std::span memory; std::optional floatMode; std::span linkedCallFallbacks; + bool convertedSamples = false; }; struct MeshTargetLimits { diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 3c9e663e94..4f79261c51 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -140,7 +140,7 @@ constexpr std::size_t fieldCount() { static_assert(fieldCount() == 23, "ShaderInfo changed: update the info encoder"); static_assert(fieldCount() == 12, "BufferResource changed: update the info encoder"); -static_assert(fieldCount() == 38, "ImageResource changed: update the info encoder"); +static_assert(fieldCount() == 39, "ImageResource changed: update the info encoder"); static_assert(fieldCount() == 8, "SamplerResource changed: update the info encoder"); static_assert(fieldCount() == 3, "SampledResourcePair changed: update the info encoder"); static_assert(fieldCount() == 5, "StageInput changed: update the info encoder"); @@ -708,6 +708,7 @@ void encodeShaderInfo(Writer& writer, const ShaderInfo& info) { out.Value(image.flatLineCompatible); out.Value(image.flatVolumeLineCompatible); out.Value(image.foldTexelOffsetsOnly); + out.Value(image.convertedSamples); out.Value(image.byElements); out.Value(image.byComponents); out.Value(image.packedFormat); @@ -810,6 +811,7 @@ void decodeShaderInfo(Reader& reader, ShaderInfo& info) { in.Value(image.flatLineCompatible); in.Value(image.flatVolumeLineCompatible); in.Value(image.foldTexelOffsetsOnly); + in.Value(image.convertedSamples); in.Value(image.byElements); in.Value(image.byComponents); in.Value(image.packedFormat); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index 9d60e3f054..e68784d5d1 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -1081,7 +1081,8 @@ SampleSetup MakeSampleSetup(SpirvValueEmitContext& ctx, const ImageEmitAccess& a if (dref && access.image.conversionFormat != IrBufferFormat::Invalid) { ctx.Fail(access.inst, "uses depth comparison with a packed integer image"); } - if (ImageConversionFormat(access.image).type == SpirvFormatComponentType::Unorm) { + const bool filtered = !dref && access.inst.Opcode() == IrOpcode::ImageSampleRaw; + if (!filtered && ImageConversionFormat(access.image).type == SpirvFormatComponentType::Unorm) { ctx.Fail(access.inst, "samples or gathers a converted unorm image, which needs filtering in the shader and is not implemented"); } if (ImageConversionFormat(access.image).type == SpirvFormatComponentType::Float) { @@ -1243,7 +1244,7 @@ void EmitGatherOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, con ctx.Define(access.inst, ResultVector(ctx, access, UnpackImageGather(ctx, access, gathered), resultNumericClass, false, true)); } -void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, const SampleSetup& setup) { +void EmitEmulatedSample(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, const SampleSetup& setup, bool converted) { auto& state = ctx.state; const auto& mem = access.mem; const auto& image = access.image; @@ -1254,9 +1255,9 @@ void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess return EmitAndConstant(state, Binary(state, spv::OpShiftRightLogical, TypeU32(state), compare, ConstantU32(state, fields.at(word).first)), fields.at(word).second); }; const auto equal = [&](std::uint32_t value, std::uint32_t literal) { return Binary(state, spv::OpIEqual, TypeBool(state), value, ConstantU32(state, literal)); }; - if (HasFlag(mem, RdnaImageSampleFlagDerivative) || HasFlag(mem, RdnaImageSampleFlagLod)) ctx.Fail(access.inst, "compares against a color texture with gradients or an explicit LOD, which is not implemented"); + if (!converted && (HasFlag(mem, RdnaImageSampleFlagDerivative) || HasFlag(mem, RdnaImageSampleFlagLod))) ctx.Fail(access.inst, "compares against a color texture with gradients or an explicit LOD, which is not implemented"); const bool arrayed = image.dimension == RdnaImageDimension::Dim2DArray; - if (image.dimension != RdnaImageDimension::Dim2D && !arrayed) ctx.Fail(access.inst, "compares against a color texture that is not a 2D or 2D array view, which is not implemented"); + if (image.dimension != RdnaImageDimension::Dim2D && !arrayed) ctx.Fail(access.inst, converted ? "samples a converted image that is not a 2D or 2D array view, which is not implemented" : "compares against a color texture that is not a 2D or 2D array view, which is not implemented"); const auto f32 = TypeF32(state); const auto i32 = TypeI32(state); const auto u32 = TypeU32(state); @@ -1285,9 +1286,11 @@ void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess const auto rounded = Unary(state, spv::OpConvertFToS, i32, ext(f32, GLSLstd450Floor, {Binary(state, spv::OpFAdd, f32, extract(f32, setup.coord, 2u), f32Constant(0.5f))})); layer = Unary(state, spv::OpBitcast, u32, ext(i32, GLSLstd450SClamp, {rounded, ConstantI32(state, 0), Binary(state, spv::OpISub, i32, layers, ConstantI32(state, 1))})); } - auto reference = DrefValueF32(ctx, access, setup.layout); - reference = Select(state, f32, equal(parameter(4u), EmulatedCompare::ReferenceUnorm), ext(f32, GLSLstd450FClamp, {reference, f32Constant(0.0f), f32Constant(1.0f)}), reference); - reference = Select(state, f32, equal(parameter(4u), EmulatedCompare::ReferenceSnorm), ext(f32, GLSLstd450FClamp, {reference, f32Constant(-1.0f), f32Constant(1.0f)}), reference); + auto reference = converted ? 0u : DrefValueF32(ctx, access, setup.layout); + if (!converted) { + reference = Select(state, f32, equal(parameter(4u), EmulatedCompare::ReferenceUnorm), ext(f32, GLSLstd450FClamp, {reference, f32Constant(0.0f), f32Constant(1.0f)}), reference); + reference = Select(state, f32, equal(parameter(4u), EmulatedCompare::ReferenceSnorm), ext(f32, GLSLstd450FClamp, {reference, f32Constant(-1.0f), f32Constant(1.0f)}), reference); + } const auto address = [&](std::uint32_t index, std::uint32_t extent, std::uint32_t mode) { const auto remainder = Binary(state, spv::OpSRem, i32, index, extent); const auto negative = Binary(state, spv::OpSLessThan, TypeBool(state), remainder, ConstantI32(state, 0)); @@ -1304,7 +1307,7 @@ void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess const auto one = f32Constant(1.0f); const auto zero = f32Constant(0.0f); const auto borderRed = Select(state, f32, equal(parameter(5u), 1u), one, zero); - const auto compareTexel = [&](std::uint32_t x, std::uint32_t y) { + const auto compareTexel = [&](std::uint32_t x, std::uint32_t y) -> std::array { const auto modeX = parameter(2u); const auto modeY = parameter(3u); const auto ux = Unary(state, spv::OpBitcast, u32, address(x, width, modeX)); @@ -1312,6 +1315,14 @@ void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess const auto coord = state.module.AllocateId(); if (arrayed) state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 3), coord, ux, uy, layer); else state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 2), coord, ux, uy); + if (converted) { + const auto raw = state.module.AllocateId(); + state.module.AddFunction(spv::OpImageFetch, ImageVectorType(state, IrTextureNumericClass::Uint, 4), raw, descriptor, coord, spv::ImageOperandsLodMask, ConstantU32(state, 0u)); + const auto unpacked = UnpackImageTexel(ctx, access, raw); + std::array channels{}; + for (std::uint32_t channel = 0; channel < 4u; ++channel) channels[channel] = Unary(state, spv::OpBitcast, f32, extract(u32, unpacked, channel)); + return channels; + } const auto texel = state.module.AllocateId(); state.module.AddFunction(spv::OpImageFetch, ImageVectorType(state, IrTextureNumericClass::Float, 4), texel, descriptor, coord, spv::ImageOperandsLodMask, ConstantU32(state, 0u)); auto red = extract(f32, texel, 3u); @@ -1325,7 +1336,7 @@ void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess result = Select(state, f32, equal(parameter(0u), index + 1u), compared, result); } result = Select(state, f32, equal(parameter(4u), EmulatedCompare::ReferenceZero), zero, result); - return result; + return {result, result, result, result}; }; const auto scaledU = Binary(state, spv::OpFMul, f32, extract(f32, setup.coord, 0u), Unary(state, spv::OpConvertSToF, f32, width)); const auto scaledV = Binary(state, spv::OpFMul, f32, extract(f32, setup.coord, 1u), Unary(state, spv::OpConvertSToF, f32, height)); @@ -1339,24 +1350,49 @@ void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess const auto texelIndex = [&](std::uint32_t coordinate, std::uint32_t offset) { return Binary(state, spv::OpIAdd, i32, Unary(state, spv::OpConvertFToS, i32, ext(f32, GLSLstd450Floor, {coordinate})), offset); }; - const auto result = EmitValueIfElse(state, equal(parameter(1u), 0u), f32, [&] { - return compareTexel(texelIndex(scaledU, offsetX), texelIndex(scaledV, offsetY)); + const auto filteredType = converted ? TypeF32Vector(state, 4) : f32; + const auto vector = [&](const std::array& channels) { + if (!converted) return channels[0]; + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, filteredType, value, channels[0], channels[1], channels[2], channels[3]); + return value; + }; + const auto mix = [&](std::uint32_t first, std::uint32_t second, std::uint32_t weight) { + return ext(filteredType, GLSLstd450FMix, {first, second, vector({weight, weight, weight, weight})}); + }; + const auto filtered = EmitValueIfElse(state, equal(parameter(1u), 0u), filteredType, [&] { + return vector(compareTexel(texelIndex(scaledU, offsetX), texelIndex(scaledV, offsetY))); }, [&] { const auto half = f32Constant(0.5f); const auto centreU = Binary(state, spv::OpFSub, f32, scaledU, half); const auto centreV = Binary(state, spv::OpFSub, f32, scaledV, half); const auto floorU = ext(f32, GLSLstd450Floor, {centreU}); const auto floorV = ext(f32, GLSLstd450Floor, {centreV}); - const auto weightU = Binary(state, spv::OpFSub, f32, centreU, floorU); - const auto weightV = Binary(state, spv::OpFSub, f32, centreV, floorV); + const auto weight = [&](std::uint32_t centre, std::uint32_t floored) { + const auto fraction = Binary(state, spv::OpFSub, f32, centre, floored); + if (!converted) return fraction; + const auto steps = ext(f32, GLSLstd450RoundEven, {Binary(state, spv::OpFMul, f32, fraction, f32Constant(256.0f))}); + return Binary(state, spv::OpFMul, f32, steps, f32Constant(1.0f / 256.0f)); + }; + const auto weightU = weight(centreU, floorU); + const auto weightV = weight(centreV, floorV); const auto x0 = Binary(state, spv::OpIAdd, i32, Unary(state, spv::OpConvertFToS, i32, floorU), offsetX); const auto y0 = Binary(state, spv::OpIAdd, i32, Unary(state, spv::OpConvertFToS, i32, floorV), offsetY); const auto x1 = Binary(state, spv::OpIAdd, i32, x0, ConstantI32(state, 1)); const auto y1 = Binary(state, spv::OpIAdd, i32, y0, ConstantI32(state, 1)); - const auto top = ext(f32, GLSLstd450FMix, {compareTexel(x0, y0), compareTexel(x1, y0), weightU}); - const auto bottom = ext(f32, GLSLstd450FMix, {compareTexel(x0, y1), compareTexel(x1, y1), weightU}); - return ext(f32, GLSLstd450FMix, {top, bottom, weightV}); + const auto topLeft = vector(compareTexel(x0, y0)); + const auto topRight = vector(compareTexel(x1, y0)); + const auto top = mix(topLeft, topRight, weightU); + const auto bottomLeft = vector(compareTexel(x0, y1)); + const auto bottomRight = vector(compareTexel(x1, y1)); + const auto bottom = mix(bottomLeft, bottomRight, weightU); + return mix(top, bottom, weightV); }); + if (converted) { + ctx.Define(access.inst, ResultVector(ctx, access, filtered, IrTextureNumericClass::Float, false, false)); + return; + } + const auto result = filtered; const auto channelResult = [&](std::uint32_t channel) { const auto selector = RuntimeImageSwizzle(state, mem.resource, channel); return Select(state, f32, equal(selector, 0u), zero, Select(state, f32, equal(selector, 1u), one, result)); @@ -1373,7 +1409,11 @@ void EmitSampleOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, con const auto& mem = access.mem; const auto& image = access.image; if (setup.dref && (image.emulatedCompare & EmulatedCompare::Enabled) != 0u) { - EmitEmulatedCompareSample(ctx, access, setup); + EmitEmulatedSample(ctx, access, setup, false); + return; + } + if (!setup.dref && image.conversionFormat == IrBufferFormat::Format11_11_10UNorm) { + EmitEmulatedSample(ctx, access, setup, true); return; } const bool explicitLod = ImageSampleExplicitLod(mem.imageSampleFlags, state.program.Resources().stage); diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 28cd206d18..2c4290c2b5 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -127,7 +127,7 @@ - [v_cvt_pkrtz_f16_f32 and v_pack_b32_f16 denormals](../../../core/shader/recompiler/Translation/src/ConvertInstructions.cpp) follow the `FLOAT_MODE` f16 denormal field: `v_cvt_pkrtz_f16_f32` flushes each denormal half result to its signed zero when bit 1 is clear (0 and 1), and `v_pack_b32_f16` flushes each denormal half input to its signed zero when bit 0 is clear (0 and 2). Measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) at IEEE=0 and IEEE=1 with denorm16 0..3 and f32 denormals kept, over 256 pairs of 16 f32 values around the f16 denormal range for `v_cvt_pkrtz_f16_f32` and 256 pairs of 16 halves (zeros, denormals, minimum normals, ±1, the largest finite value, infinities and quiet and signaling NaNs) for `v_pack_b32_f16`. Shaders without a known float mode keep the previous lowering. The PS5's gfx1013 is not measured. - [Comparison sampling of a color texture](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (`image_sample_c`/`image_sample_c_lz` on a texture that is not R32 float or R16 unorm) is emulated in the shader, since Vulkan compares only depth formats: the red channel is fetched, compared and, for bilinear filtering, blended (compare, then filter). Only 2D, 2D array and cube views, float/unorm/snorm formats, wrap, clamp-to-edge and clamp-to-border addressing (border red 0 for the black borders, 1 for opaque white) and point or bilinear filtering at one level are implemented, with texel offsets added to the texel indices; LOD biases and clamps are accepted, as they cannot select another level of a single-level sample. Half-border addressing, the border color table, gradients, gathers, other dimensions, anisotropy (except on the 8-bit formats below) and mip filtering throw. Under bilinear filtering, taps past a clamp-to-border edge compare the border value and blend with the texel taps, so results near the edge depend on the filter weights. AMD documentation does not say how a color format's reference is handled: it is clamped to [0, 1] for unorm and [-1, 1] for snorm, as Vulkan does for unorm depth, and not clamped for float. 8-bit sRGB and UNORM (`Format8Srgb`, `Format8_8Srgb`, `Format8_8_8_8Srgb`, `Format8UNorm`, `Format8_8UNorm`, `Format8_8_8_8UNorm`) return 0 for every compare function, measured with `image_sample_c_lz` on gfx1035 over references -0.5..1.5 and texel codes 0..255. A bilinear cube of those 8-bit formats without `DISABLE_CUBE_WRAP`, and an anisotropic S# on them, return 0 for the same reason; float, snorm and the other unorm bilinear cubes without that bit still throw. Bilinear weights are full float, not the hardware's fixed-point subtexel weights - [Negative color-texture comparison wrap indices](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (libSceAgcDriver) use explicit `OpSRem` normalization to implement modulo; the negative-offset point-wrap regression fails with direct `OpSMod` and passes with normalization on Mesa 22.3.6 llvmpipe (LLVM 15.0.6), but negative-index wrap has not been measured on PS5 hardware. -- [Converted unorm images](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (10_11_11_UNORM, read and written through an R32_UINT view): samples, gathers, LOD queries, 16-bit data and a DST_SEL of the missing W channel throw. +- [Converted unorm images](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (10_11_11_UNORM, read and written through an R32_UINT view) are sampled in the shader, the way color comparisons are emulated: `image_sample` of a 2D or 2D array view fetches the one or four texels at LOD 0, decodes them and blends them, for point or bilinear filtering with equal minification and magnification filters and wrap or clamp-to-edge addressing. A shader gets these sample modes only once a converted descriptor reaches one of its sampled images: that capture throws `ConvertedSampleRequired`, the driver records the program and prepares it again with `GuestContext::convertedSamples`, so such a shader is prepared twice per run and its first capture is repeated. The record lives with the registered program and is not kept across runs. Bilinear weights are rounded to the hardware's 8-bit grid (`roundEven(fraction * 256) / 256`, ties to even), as measured with `image_sample_lz` on RDNA2 (gfx1035) for 10_11_11 UNORM texels by adriadam10 in #5958; point sampling, decoding, swizzles and wrap and clamp-to-edge addressing matched the same measurement. A sample that is not `_lz` needs an image with one mip level; gathers, LOD queries, clamp-to-border and mirrored addressing, other filters, min/max reduction, unnormalized coordinates, images selected at run time, 16-bit data and a DST_SEL of the missing W channel throw. - [Converted unorm image loads](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) divide by 2^n - 1 with the buffer format path's `OpFDiv`, which Vulkan allows to be up to 2.5 ULP off, so some drivers are not bit-exact. - [64-bit LDS atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`ds_*_u64`, `ds_*_b64`, `ds_*_f64`) read and write the two dwords under a workgroup lock stored after the guest LDS, as LDS is a 32-bit array without 64-bit atomics. On RDNA2 (gfx1036) the LDS serialises a 32-bit atomic on a slot's high dword and a 64-bit atomic on that slot: 128 lanes of `ds_add_u32` and 128 lanes of `ds_add_u64` on one slot sum exactly. Only that pair was measured. Every 32-bit LDS atomic of a program that contains a 64-bit LDS atomic therefore takes the same lock, including a 32-bit `flat_atomic_*` whose address lands in the LDS aperture (a 64-bit one takes it already); the flat forms were not measured against a 64-bit atomic either (the other 32-bit LDS atomics, `ds_min/max`, `ds_cmpst`, the float forms, `ds_inc/dec`, `ds_mskor` and `ds_wrxchg`, are extrapolated from `ds_add_u32` and were not measured against a 64-bit atomic); a program without a 64-bit LDS atomic keeps its native 32-bit atomics. `ds_condxchg32_rtn_b64` is two `ds_mskor` halves, each under the lock but not atomic as a pair, so a 64-bit atomic between the two halves is not excluded. Sub-dword writes (`ds_write_b8`, `ds_write_b16` and their `_d16_hi` forms) and plain `ds_read_*`, `ds_write_*`, `ds_append_u32` and `ds_consume_u32` are not taken under the lock, so a plain access racing a 64-bit atomic on the same dwords can see or overwrite one half of it; whether the hardware orders them was not measured. The lock serialises the workgroup's LDS atomics of such a program; the throughput cost was not measured. `VK_KHR_workgroup_memory_explicit_layout` with `shaderSharedInt64Atomics` would make both widths native and is not used. A 64-bit atomic at an address that is 4-byte but not 8-byte aligned faults the wave on gfx1036, as on gfx1035. When the LDS lives in device memory (compute LDS beyond the device's shared-memory limit) and the device has `shaderBufferInt64Atomics`, no lock is used: a 64-bit atomic is a compare-exchange loop on a 64-bit view of the storage buffer and a 32-bit atomic is a native buffer atomic, a misaligned 64-bit atomic records the `Unaligned` shader fault with the LDS byte address instead of updating, so a mixed-width pair on one slot relies on the device ordering both widths on one buffer address; only `ds_add_u32` against `ds_add_u64` was measured, on gfx1036 (AMD Windows driver 32.0.21042.62) and an RTX 4070 SUPER. The lock is not used there because AMD's Windows driver compiles an acquire or release on a storage buffer, at workgroup scope or wider, to `s_barrier`, and those barriers inside the lock's divergent loop pair with the program's own `s_barrier`s. The fence after each LDS atomic compiles to `s_barrier` on that driver as well, so a wave that skips an LDS atomic that other waves of its workgroup execute misses a barrier they reach; its effect was not measured. On that path a 64-bit atomic at an address that is 4-byte but not 8-byte aligned is skipped and returns 0. - [LDS float atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`ds_min/max/cmpst_f32`, `ds_min/max/cmpst_f64` and their `_rtn` forms) follow the hardware measured on RDNA2 (gfx1036) over 14 x 14 operand sets (zeros, ones, infinities, denormals, quiet and signalling NaNs of both signs and with payloads) in IEEE 0 and 1 with every value 0-3 of the type's denormal field: both operands are flushed to a signed zero when bit 0 of the field is clear (0 and 2; 1 and 3 keep them; the f32 field for the f32 forms, the f16/f64 field for the f64 forms; a shader without a known float mode flushes f32 and keeps f64), then a signalling-NaN data operand is stored quieted, else a signalling-NaN old value is stored quieted, else a quiet-NaN old value is replaced by the data, else a quiet-NaN data operand loses, else the ordered compare with -0 below +0 decides, and the flushed value is what is written, also when the old value wins. `cmpst` compares the flushed bits (both zeros equal, NaN never equal) and leaves memory unchanged when they differ. The returned value is always the old value as it was in memory. `IEEE_MODE` has no effect. The same bit rule serves the buffer float atomics (`AtomicFloatBits`). Not measured: a denormal or signalling-NaN `data1` of `cmpst` (stored as given). `ds_add_f32` and `ds_add_rtn_f32` follow RDNA1 (gfx1010, RX 5700 XT), measured with the [hardware oracle](../HW_ORACLE.md) template run through the Windows driver's comgr and HIP runtime over 1574 operand pairs (signed zeros, denormals, the smallest normals, infinities, quiet and signalling NaNs with payloads, the largest normals, random pairs near the denormal range and pairs that nearly cancel) in all 16 combinations of the f32 denormal field and the f32 round mode, and at IEEE 1 and DX10 clamp 0: both operands are flushed to a signed zero when bit 0 of the field is clear and the result when bit 1 is clear, the sum is rounded to nearest even whatever the round mode, a NaN data operand is stored quieted, else a NaN old value is stored quieted, `+inf + -inf` stores 0xffc00000, and `IEEE_MODE` and `DX10_CLAMP` have no effect. The sum is the host's `OpFAdd` between those flushes, so where the field keeps f32 denormals the result also depends on the host preserving them, which the module does not request. thuanquang measured the same rule on RDNA2 (gfx1031, RX 6700 XT) in #5492, with the same operand classes and 18 modes in wave32 and wave64; the PS5's gfx1013 is not measured.