perf(shader): emit exact f32 add, subtract and multiply as fma on MoltenVK

The recompiler marks every guest f32 add, subtract and multiply
NoContraction, because the guest rounds each one separately.
SPIRV-Cross turns each of them into a call to a helper that Metal
compiles without optimization, which dominates fragment time on
MoltenVK, and its multiply helper, fma(a, b, +0.0), turns a -0.0
product into +0.0.

On MoltenVK they are now GLSL Fma whose second step is exact, which
gives the separately rounded result: a * b is fma(a, b, -0.0), a + b
is fma(1.0, a, b) and a - b is fma(-1.0, b, a). The sin, cos and log2
series and the offset image coordinates and gather texel scale use the
same forms. The choice is a SpirvTarget flag set for
VK_DRIVER_ID_MOLTENVK, part of the shader cache key and of the request
serialization (version 44); other drivers are unchanged.

agc_driver_f32_separate_rounding checks a multiply feeding an add or a
subtract on rows where fusing them changes the result. It,
agc_driver_f32_arithmetic_nan and agc_driver_f32_denormal_flush also
run the fused forms on devices that do not use them, so lavapipe checks
them against the same rows.

Co-authored-by: Ycaro Gabriel <[email protected]>

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -321,6 +321,7 @@ struct VulkanDevice::State {
     bool meshShader = false;
     bool fragmentShaderBarycentric = false;
     bool shaderTerminateInvocation = false;
+    bool fusedExactArithmetic = false;
     bool geometryShader = false;
     bool largePoints = false;
     bool sampleRateShading = false;
@@ -1220,6 +1221,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         driverId = driver.driverID;
         state->driverInfo = std::string(driver.driverName) + " " + driver.driverInfo;
         state->individuallyFreeDescriptorSets = driver.driverID == VK_DRIVER_ID_MESA_LLVMPIPE;
+        state->fusedExactArithmetic = driver.driverID == VK_DRIVER_ID_MOLTENVK;
         state->subgroupControlBarriers = SubgroupControlBarrierUsable(driver.driverID);
     }
     auto byteFeatures = QueryBdaByteFeatures(selected, state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2"), availableExtensions);
@@ -3739,6 +3741,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::Target() const {
     require(state->shaderProfile != nullptr, "shader device profile is unavailable");
     auto target = state->shaderProfile->Target();
     target.float32FlushToZero = state->float32FlushToZero;
+    target.fusedExactArithmetic = state->fusedExactArithmetic;
     target.float32DenormPreserve = state->float32DenormPreserve;
     return target;
 }

core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp

@@ -31,6 +31,7 @@
 #include "ShaderDiskCache.hpp"
 #include <spirv/unified1/spirv.hpp>
 #include <algorithm>
+#include <set>
 #include <array>
 #include <cstddef>
 #include <cstdlib>
@@ -4003,6 +4004,51 @@ void verifyLanesOutsideHostSubgroup() {
     require(pixel(permlanex16, 32u) && pixel(permlane16, 8u), "host lanes: a v_permlane16_b32 row that the host subgroup may hold did not build");
 }
 
+void verifyFusedExactArithmetic() {
+    using namespace ShaderRecompiler;
+    const auto pixel = [](bool fused) {
+        static const std::vector<std::uint32_t> code{0x7e000200u, 0x10020100u, 0x06040101u, 0x08060302u, 0x7e000303u, 0xf800180fu, 0u, 0xbf810000u};
+        ShaderPixelStageInfo info{};
+        info.inputAddr = PixelInputBit(PixelInput::PositionX);
+        info.posX = true;
+        info.targetOutputMode[0] = 9u;
+        info.targetExportMapping.fill(0xe4u);
+        RecompileRequest request{};
+        request.shader = {ShaderStage::Fragment, 0x30000u, code, 0, {}};
+        request.context.waveSize = 64;
+        request.context.pixel = info;
+        request.target.vulkanVersion = 0x00401000u;
+        request.target.spirvVersion = 0x00010300u;
+        request.target.subgroupSize = 64;
+        request.target.fusedExactArithmetic = fused;
+        request.layout.pushConstantSizeBytes = 128;
+        request.useCache = false;
+        return Recompile(request).spirv;
+    };
+    struct Counts {
+        std::uint32_t exactArithmetic = 0;
+        std::uint32_t fmas = 0;
+    };
+    const auto count = [](std::span<const std::uint32_t> spirv) {
+        std::set<std::uint32_t> exact;
+        std::vector<std::uint32_t> arithmetic;
+        Counts result;
+        for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) {
+            const auto op = spirv[cursor] & 0xffffu;
+            const auto words = spirv[cursor] >> 16u;
+            if (op == spv::OpDecorate && words == 3u && spirv[cursor + 2] == spv::DecorationNoContraction) exact.insert(spirv[cursor + 1]);
+            if (op == spv::OpFAdd || op == spv::OpFSub || op == spv::OpFMul) arithmetic.push_back(spirv[cursor + 2]);
+            if (op == spv::OpExtInst && words >= 5u && spirv[cursor + 4] == 50u) ++result.fmas;
+        }
+        result.exactArithmetic = static_cast<std::uint32_t>(std::count_if(arithmetic.begin(), arithmetic.end(), [&](std::uint32_t id) { return exact.contains(id); }));
+        return result;
+    };
+    const auto plain = count(pixel(false));
+    const auto fused = count(pixel(true));
+    require(plain.exactArithmetic >= 3u, "fused exact arithmetic: the f32 multiply, add and subtract were not exact operations without the flag");
+    require(fused.exactArithmetic == 0u && fused.fmas >= plain.fmas + 3u, "fused exact arithmetic: the f32 multiply, add and subtract did not become fused multiply-adds with the flag");
+}
+
 void verifyHalfWaveReduction() {
     using namespace ShaderRecompiler;
     static constexpr std::array<std::uint32_t, 5> capabilities{spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic, spv::CapabilityClipDistance};
@@ -4898,7 +4944,7 @@ void verifyPixelRequestSerialization() {
     minimal.context.pixel->layerExported = true;
     minimal.context.pixel->ditheredAlphaToCoverage = true;
     const auto encoded = serializer.Serialize(minimal);
-    require(requestPrefix(encoded, 8u) == "NVNQQSsAAAA=", "new requests did not use serialization version 43");
+    require(requestPrefix(encoded, 8u) == "NVNQQSwAAAA=", "new requests did not use serialization version 44");
     const std::array<std::uint32_t, 2> fallbacks{0x40u, 0x1ac4u};
     auto withFallbacks = minimal;
     withFallbacks.context.linkedCallFallbacks = fallbacks;
@@ -5016,7 +5062,7 @@ void verifyPixelRequestSerialization() {
     for (std::size_t bytes = 0; bytes < 45u; ++bytes) {
         expectFailure([&] { static_cast<void>(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated pixel mapping, packing, dual-source, alpha-to-coverage, stencil reference export flag, primitive class, layer export, dither or front-face flag, quad pixel mask or quad sample mask was accepted");
     }
-    for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQSwAAAA="}) {
+    for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQS0AAAA="}) {
         expectFailure([&] { static_cast<void>(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted");
     }
     static constexpr std::string_view version18Request{
@@ -5110,6 +5156,7 @@ void verifyVertexInfoVersion19() {
     version20Vertex.pop_back();
     version20Vertex.pop_back();
     version20Vertex.pop_back();
+    version20Vertex.pop_back();
     version20Vertex[4] = 20u;
     const auto replay20Vertex = serializer.Deserialize(encodeBase64(version20Vertex));
     require(replay20Vertex.request.context.vertex.has_value() && replay20Vertex.request.context.vertex->paClVsOutCntl == 0x89abcdefu, "a main version20 request lost PA_CL_VS_OUT_CNTL");
@@ -5121,6 +5168,7 @@ void verifyVertexInfoVersion19() {
     legacy.pop_back();
     legacy.pop_back();
     legacy.pop_back();
+    legacy.pop_back();
     legacy.erase(legacy.begin() + offset, legacy.begin() + offset + 4u);
     for (const auto version : {16u, 17u, 18u, 19u}) {
         legacy[4] = static_cast<std::uint8_t>(version);
@@ -5168,12 +5216,13 @@ void verifyVertexInfoVersion19() {
     orderedVersion34.pop_back();
     orderedVersion34.pop_back();
     orderedVersion34.pop_back();
+    orderedVersion34.pop_back();
     orderedVersion34.erase(orderedVersion34.begin() + static_cast<std::ptrdiff_t>(orderOffset - 1u), orderedVersion34.begin() + static_cast<std::ptrdiff_t>(orderOffset + sampleOrderBytes));
     orderedVersion34[4] = 34u;
     const auto orderedReplay34 = serializer.Deserialize(encodeBase64(orderedVersion34));
     require(orderedReplay34.request.context.pixel.has_value() && orderedReplay34.request.context.pixel->sampleOrder == std::array<std::uint32_t, 4>{} && serializer.Serialize(orderedReplay34.request) == encodedPixel, "a version 34 request read a sample order, or misread the fields after its pixel iteration fields");
     auto orderedVersion30 = orderedBytes;
-    orderedVersion30.resize(orderedVersion30.size() - 4u);
+    orderedVersion30.resize(orderedVersion30.size() - 5u);
     orderedVersion30.erase(orderedVersion30.begin() + static_cast<std::ptrdiff_t>(blendOffset + 4u), orderedVersion30.begin() + static_cast<std::ptrdiff_t>(orderOffset + sampleOrderBytes));
     orderedVersion30[4] = 30u;
     const auto orderedReplay30 = serializer.Deserialize(encodeBase64(orderedVersion30));
@@ -5182,6 +5231,7 @@ void verifyVertexInfoVersion19() {
     orderedSpanned.target.spannedWaves = true;
     auto orderedVersion32 = decodeBase64(serializer.Serialize(orderedSpanned));
     orderedVersion32.pop_back();
+    orderedVersion32.pop_back();
     require(orderedVersion32.size() > 2u && orderedVersion32.back() == 0u && orderedVersion32[orderedVersion32.size() - 2u] == 0u && orderedVersion32[orderedVersion32.size() - 3u] == 1u, "the spanned-waves flag is not the byte before the depth remap flag");
     orderedVersion32.pop_back();
     orderedVersion32.pop_back();
@@ -5206,6 +5256,7 @@ void verifyVertexInfoVersion19() {
     version30.pop_back();
     version30.pop_back();
     version30.pop_back();
+    version30.pop_back();
     version30.erase(version30.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u + sampleMaskBytes + 4u), version30.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes));
     version30[4] = 30u;
     const auto replay30 = serializer.Deserialize(encodeBase64(version30));
@@ -5214,6 +5265,7 @@ void verifyVertexInfoVersion19() {
     spanned.target.spannedWaves = true;
     auto version32 = decodeBase64(serializer.Serialize(spanned));
     version32.pop_back();
+    version32.pop_back();
     require(version32.size() > 2u && version32.back() == 0u && version32[version32.size() - 2u] == 0u && version32[version32.size() - 3u] == 1u, "the spanned-waves flag is not the byte before the depth remap flag");
     version32.pop_back();
     version32.pop_back();
@@ -5226,9 +5278,11 @@ void verifyVertexInfoVersion19() {
     version29.pop_back();
     version29.pop_back();
     version29.pop_back();
+    version29.pop_back();
     version29.erase(version29.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u + sampleMaskBytes), version29.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes));
     auto version36 = sampledBytes;
     version36.pop_back();
+    version36.pop_back();
     version36.erase(version36.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes - 1u), version36.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes));
     version36[4] = 36u;
     const auto replay36 = serializer.Deserialize(encodeBase64(version36));
@@ -5241,6 +5295,7 @@ void verifyVertexInfoVersion19() {
     version28.pop_back();
     version28.pop_back();
     version28.pop_back();
+    version28.pop_back();
     version28.erase(version28.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u), version28.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes));
     version28[4] = 28u;
     const auto replay28 = serializer.Deserialize(encodeBase64(version28));
@@ -5250,6 +5305,7 @@ void verifyVertexInfoVersion19() {
     version27.pop_back();
     version27.pop_back();
     version27.pop_back();
+    version27.pop_back();
     version27.erase(version27.begin() + static_cast<std::ptrdiff_t>(maskOffset), version27.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes));
     version27[4] = 27u;
     const auto replay27 = serializer.Deserialize(encodeBase64(version27));
@@ -5259,6 +5315,7 @@ void verifyVertexInfoVersion19() {
     legacyPixel.pop_back();
     legacyPixel.pop_back();
     legacyPixel.pop_back();
+    legacyPixel.pop_back();
     legacyPixel.erase(legacyPixel.begin() + static_cast<std::ptrdiff_t>(allBitsOffset), legacyPixel.begin() + static_cast<std::ptrdiff_t>(allBitsOffset + 2u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes));
     legacyPixel[4] = 26u;
     const auto replay26 = serializer.Deserialize(encodeBase64(legacyPixel));
@@ -5309,10 +5366,11 @@ void verifyDepthRemapRequests() {
     const auto plainBytes = decodeBase64(plain);
     const auto remappedBytes = decodeBase64(remapped);
     constexpr std::size_t pointTailBytes = 13u;
-    require(plainBytes.size() == remappedBytes.size() && plainBytes.size() > 7u + pointTailBytes && std::equal(plainBytes.begin(), plainBytes.end() - 7 - pointTailBytes, remappedBytes.begin()) && std::equal(plainBytes.end() - pointTailBytes, plainBytes.end(), remappedBytes.end() - pointTailBytes), "the depth remap flag and mode are not the 6 bytes before the f32 flush and denormal preserve flags and the point tail of a vertex request");
+    require(plainBytes.size() == remappedBytes.size() && plainBytes.size() > 8u + pointTailBytes && std::equal(plainBytes.begin(), plainBytes.end() - 8 - pointTailBytes, remappedBytes.begin()) && std::equal(plainBytes.end() - 1 - pointTailBytes, plainBytes.end(), remappedBytes.end() - 1 - pointTailBytes), "the depth remap flag and mode are not the 6 bytes before the f32 flush and denormal preserve flags, the point tail and the fused arithmetic flag of a vertex request");
     for (const std::uint8_t legacyVersion : {32u, 34u}) {
         auto legacyBytes = remappedBytes;
         legacyBytes.pop_back();
+        legacyBytes.pop_back();
         legacyBytes[4] = legacyVersion;
         legacyBytes.resize(legacyBytes.size() - 6u - pointTailBytes);
         const auto legacy = serializer.Deserialize(encodeBase64(legacyBytes));
@@ -5321,6 +5379,7 @@ void verifyDepthRemapRequests() {
     }
     auto version35 = remappedBytes;
     version35.pop_back();
+    version35.pop_back();
     version35[4] = 35u;
     version35.resize(version35.size() - 1u - pointTailBytes);
     const auto replay35 = serializer.Deserialize(encodeBase64(version35));
@@ -5368,8 +5427,8 @@ void verifyMergedHullConfiguration() {
     require(!serializer.Deserialize(separate).request.graphics->tessellation->mergedHull, "a separate hull configuration was read as merged");
     constexpr std::size_t pointTailBytes = 13u;
     auto legacy = decodeBase64(merged);
-    require(legacy.size() > 53u + pointTailBytes && legacy[4] == 43u, "a merged hull request did not use serialization version 43");
-    legacy.resize(legacy.size() - pointTailBytes);
+    require(legacy.size() > 54u + pointTailBytes && legacy[4] == 44u, "a merged hull request did not use serialization version 44");
+    legacy.resize(legacy.size() - 1u - pointTailBytes);
     legacy.pop_back();
     require(legacy[legacy.size() - 52u] == 1u, "the merged hull flag is not the last byte of the tessellation configuration");
     legacy.erase(legacy.end() - 52);
@@ -7764,6 +7823,7 @@ int main(int argc, char** argv) {
         verifyProgramCounterRelativeData();
         verifyLanesOutsideHostSubgroup();
         verifyHalfWaveReduction();
+        verifyFusedExactArithmetic();
         verifyNarrowVertexWave();
         verifySingleLaneVertexWave();
         verifyMeshConfiguration();

core/libs/prx/libSceAgcDriver/tests/execution/F32ArithmeticNan.cpp

@@ -23,6 +23,7 @@ constexpr std::uint32_t Inputs = 4;
 constexpr std::uint32_t Results = 16;
 alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
 alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+bool FusedExactArithmetic = false;
 
 alignas(256) constexpr std::array<std::uint32_t, 57> Code{
     0x34020084u, 0x34060086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x7e160280u, 0x7e180280u,
@@ -172,7 +173,7 @@ std::string Hex(std::uint32_t value) {
 }
 
 void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) {
-    Require(actual == expected, std::string("f32 arithmetic nan: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
+    Require(actual == expected, std::string("f32 arithmetic nan: ") + (FusedExactArithmetic ? "fused " : "") + "lane " + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
 }
 
 bool IsNan32(std::uint32_t value) {
@@ -202,6 +203,7 @@ void Run(AgcDriver::VulkanDevice& device, const std::optional<ShaderRecompiler::
     };
     request.useCache = false;
     request.context.floatMode = floatMode;
+    request.target.fusedExactArithmetic = FusedExactArithmetic;
     const auto result = ShaderRecompiler::Recompile(request);
     device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
     device.WaitIdle();
@@ -223,12 +225,16 @@ int main() {
     try {
         const auto device = OpenVulkanTestDevice();
         if (!device) return VulkanTestSkipped;
-        Run(*device, std::nullopt);
-        Check(ExpectedNoIeee, "no float mode");
-        Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false});
-        Check(ExpectedNoIeee, "IEEE=0");
-        Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, false});
-        Check(ExpectedIeee, "IEEE=1");
+        for (const bool fused : {false, true}) {
+            if (!fused && device->Target().fusedExactArithmetic) continue;
+            FusedExactArithmetic = fused;
+            Run(*device, std::nullopt);
+            Check(ExpectedNoIeee, "no float mode");
+            Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false});
+            Check(ExpectedNoIeee, "IEEE=0");
+            Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, false});
+            Check(ExpectedIeee, "IEEE=1");
+        }
         std::puts("f32 arithmetic nan tests passed");
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/execution/F32DenormalFlush.cpp

@@ -382,8 +382,10 @@ std::string Hex(std::uint32_t value) {
     return text;
 }
 
+bool FusedExactArithmetic = false;
+
 void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) {
-    Require(actual == expected, std::string("f32 denormal flush: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
+    Require(actual == expected, std::string("f32 denormal flush: ") + (FusedExactArithmetic ? "fused " : "") + "lane " + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected));
 }
 
 bool IsNan(std::uint32_t value) {
@@ -434,6 +436,7 @@ void Run(AgcDriver::VulkanDevice& device, const std::optional<ShaderRecompiler::
     request.useCache = false;
     request.context.floatMode = floatMode;
     request.target.float32FlushToZero = HardwareFlush;
+    request.target.fusedExactArithmetic = FusedExactArithmetic;
     const auto result = ShaderRecompiler::Recompile(request);
     device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
     device.WaitIdle();
@@ -451,7 +454,7 @@ void Check(const char* mode) {
             if (hostSplits && tid == TinyProductLane && (i == 4u || i == 5u)) continue;
             const bool floatResult = i < 12u || i >= 15u;
             if (floatResult && IsNan(Expected[tid][i])) {
-                Require(IsNan(out[i]), std::string("f32 denormal flush: lane ") + std::to_string(tid) + " " + mode + " " + Names[i] + " is " + Hex(out[i]) + ", expected a NaN");
+                Require(IsNan(out[i]), std::string("f32 denormal flush: ") + (FusedExactArithmetic ? "fused " : "") + "lane " + std::to_string(tid) + " " + mode + " " + Names[i] + " is " + Hex(out[i]) + ", expected a NaN");
                 continue;
             }
             Expect(tid, out[i], Expected[tid][i], (std::string(mode) + " " + Names[i]).c_str());
@@ -516,9 +519,12 @@ int main() {
             Require(!softwareDeclares && hardwareDeclares == declared, declared ? "f32 denormal flush: only the hardware-flush module declares DenormFlushToZero 32" : "f32 denormal flush: a module flushed natively declares DenormFlushToZero 32 without the device capability");
             Require(hardwareWords < software, "f32 denormal flush: the hardware-flush module is " + std::to_string(hardwareWords) + " words, the emulated one " + std::to_string(software));
         }
-        for (const bool flush : {false, true}) {
+        for (const std::uint32_t variant : {0u, 1u, 2u, 3u}) {
+            const bool flush = (variant & 1u) != 0u;
             if (flush && !hardware) continue;
+            if ((variant & 2u) == 0u && device->Target().fusedExactArithmetic) continue;
             HardwareFlush = flush;
+            FusedExactArithmetic = (variant & 2u) != 0u;
             Run(*device, std::nullopt);
             Check("no float mode");
             Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false});

core/libs/prx/libSceAgcDriver/tests/execution/F32SeparateRounding.cpp

@@ -0,0 +1,177 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 8;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+bool FusedExactArithmetic = false;
+
+alignas(256) constexpr std::array<std::uint32_t, 18> Code{
+    0x34020084u, 0x34060085u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x10140b04u, 0x06160d0au, 0x08180d0au,
+    0x0a1a0d0au, 0x061c1506u, 0x081e1506u, 0x06200d04u, 0x10220b10u, 0xe0781000u, 0x80010a03u, 0xe0781010u,
+    0x80010e03u, 0xbf810000u,
+};
+
+constexpr std::uint32_t Rows[32][4] = {
+    {0x3f800800u, 0x3f800c00u, 0xbf801401u, 0x00000000u},
+    {0x3f801800u, 0x3f801400u, 0xbf802c04u, 0x00000000u},
+    {0x3f802800u, 0x3f801c00u, 0xbf804409u, 0x00000000u},
+    {0x3f803800u, 0x3f802400u, 0xbf805c10u, 0x00000000u},
+    {0x3f804800u, 0x3f802c00u, 0xbf807419u, 0x00000000u},
+    {0x3f805800u, 0x3f803400u, 0xbf808c24u, 0x00000000u},
+    {0x3f806800u, 0x3f803c00u, 0xbf80a431u, 0x00000000u},
+    {0x3f807800u, 0x3f804400u, 0xbf80bc40u, 0x00000000u},
+    {0x3f808800u, 0x3f804c00u, 0xbf80d451u, 0x00000000u},
+    {0x3f809800u, 0x3f805400u, 0xbf80ec64u, 0x00000000u},
+    {0x3f80a800u, 0x3f805c00u, 0xbf810479u, 0x00000000u},
+    {0x3f80b800u, 0x3f806400u, 0xbf811c90u, 0x00000000u},
+    {0x3d801000u, 0xc1001800u, 0x3f002804u, 0x00000000u},
+    {0x3e005000u, 0xc0803800u, 0x3f008822u, 0x00000000u},
+    {0x3e809000u, 0xc0005800u, 0x3f00e864u, 0x00000000u},
+    {0x3f00d000u, 0xbf807800u, 0x3f0148c2u, 0x00000000u},
+    {0x3f811000u, 0xbf009800u, 0x3f01a944u, 0x00000000u},
+    {0x40015000u, 0xbe80b800u, 0x3f0209e2u, 0x00000000u},
+    {0x40819000u, 0xbe00d800u, 0x3f026aa4u, 0x00000000u},
+    {0x4101d000u, 0xbd80f800u, 0x3f02cb82u, 0x00000000u},
+    {0x40400000u, 0x3eaaaaabu, 0xbf800000u, 0x00000000u},
+    {0x40540000u, 0x3eabaaabu, 0xc0000000u, 0x00000000u},
+    {0x40680000u, 0x3eacaaabu, 0xc0400000u, 0x00000000u},
+    {0x407c0000u, 0x3eadaaabu, 0xc0800000u, 0x00000000u},
+    {0x40880000u, 0x3eaeaaabu, 0xc0a00000u, 0x00000000u},
+    {0x40920000u, 0x3eafaaabu, 0xc0c00000u, 0x00000000u},
+    {0x00000000u, 0xbf800000u, 0x00000000u, 0x00000000u},
+    {0x80000000u, 0x3f800000u, 0x80000000u, 0x00000000u},
+    {0x3f800000u, 0x3f800000u, 0xbf800000u, 0x00000000u},
+    {0x4e800001u, 0x4e7ffffeu, 0xdd800000u, 0x00000000u},
+    {0x3f800001u, 0x3f7fffffu, 0xbf800000u, 0x00000000u},
+    {0xbfc00000u, 0x35800000u, 0x35000000u, 0x00000000u},
+};
+constexpr std::uint32_t Expected[32][8] = {
+    {0x3f801401u, 0x00000000u, 0x40001401u, 0xc0001401u, 0x00000000u, 0xc0001401u, 0xb9c01000u, 0xb9c02202u},
+    {0x3f802c04u, 0x00000000u, 0x40002c04u, 0xc0002c04u, 0x00000000u, 0xc0002c04u, 0xba202000u, 0xba203905u},
+    {0x3f804409u, 0x00000000u, 0x40004409u, 0xc0004409u, 0x00000000u, 0xc0004409u, 0xba604800u, 0xba607910u},
+    {0x3f805c10u, 0x00000000u, 0x40005c10u, 0xc0005c10u, 0x00000000u, 0xc0005c10u, 0xba904000u, 0xba906892u},
+    {0x3f807419u, 0x00000000u, 0x40007419u, 0xc0007419u, 0x00000000u, 0xc0007419u, 0xbab06400u, 0xbab0a0a2u},
+    {0x3f808c24u, 0x00000000u, 0x40008c24u, 0xc0008c24u, 0x00000000u, 0xc0008c24u, 0xbad09000u, 0xbad0e4bau},
+    {0x3f80a431u, 0x00000000u, 0x4000a431u, 0xc000a431u, 0x00000000u, 0xc000a431u, 0xbaf0c400u, 0xbaf134dcu},
+    {0x3f80bc40u, 0x00000000u, 0x4000bc40u, 0xc000bc40u, 0x00000000u, 0xc000bc40u, 0xbb088000u, 0xbb08c884u},
+    {0x3f80d451u, 0x00000000u, 0x4000d451u, 0xc000d451u, 0x00000000u, 0xc000d451u, 0xbb18a200u, 0xbb18fca0u},
+    {0x3f80ec64u, 0x00000000u, 0x4000ec64u, 0xc000ec64u, 0x00000000u, 0xc000ec64u, 0xbb28c800u, 0xbb2936c3u},
+    {0x3f810479u, 0x00000000u, 0x40010479u, 0xc0010479u, 0x00000000u, 0xc0010479u, 0xbb38f200u, 0xbb3976eeu},
+    {0x3f811c90u, 0x00000000u, 0x40011c90u, 0xc0011c90u, 0x00000000u, 0xc0011c90u, 0xbb492000u, 0xbb49bd21u},
+    {0xbf002803u, 0x33800000u, 0xbf802804u, 0x3f802804u, 0x33800000u, 0x3f802804u, 0x3f102a04u, 0xc090450cu},
+    {0xbf008823u, 0xb3800000u, 0xbf808822u, 0x3f808822u, 0xb3800000u, 0x3f808822u, 0x3f209c22u, 0xc020e266u},
+    {0xbf00e863u, 0x33800000u, 0xbf80e864u, 0x3f80e864u, 0x33800000u, 0x3f80e864u, 0x3f413064u, 0xbfc1b535u},
+    {0xbf0148c3u, 0xb3800000u, 0xbf8148c2u, 0x3f8148c2u, 0xb3800000u, 0x3f8148c2u, 0x3f810c61u, 0xbf81855du},
+    {0xbf01a943u, 0x33800000u, 0xbf81a944u, 0x3f81a944u, 0x33800000u, 0x3f81a944u, 0x3fc1e4a2u, 0xbf42cae2u},
+    {0xbf0209e3u, 0xb3800000u, 0xbf8209e2u, 0x3f8209e2u, 0xb3800000u, 0x3f8209e2u, 0x4021d278u, 0xbf22bb17u},
+    {0xbf026aa3u, 0x33800000u, 0xbf826aa4u, 0x3f826aa4u, 0x33800000u, 0x3f826aa4u, 0x4091dd54u, 0xbf12d379u},
+    {0xbf02cb83u, 0xb3800000u, 0xbf82cb82u, 0x3f82cb82u, 0xb3800000u, 0x3f82cb82u, 0x4109fcb8u, 0xbf0b0812u},
+    {0x3f800000u, 0x00000000u, 0x40000000u, 0xc0000000u, 0x00000000u, 0xc0000000u, 0x40000000u, 0x3f2aaaabu},
+    {0x3f8e2956u, 0xbf63ad54u, 0x404714abu, 0xc04714abu, 0xbf63ad54u, 0xc04714abu, 0x3fa80000u, 0x3ee15000u},
+    {0x3f9c7aabu, 0xbfe38555u, 0x40871eabu, 0xc0871eabu, 0xbfe38555u, 0xc0871eabu, 0x3f200000u, 0x3e57d556u},
+    {0x3faaf400u, 0xc02a8600u, 0x40aabd00u, 0xc0aabd00u, 0xc02a8600u, 0xc0aabd00u, 0xbd800000u, 0xbcadaaabu},
+    {0x3fb99556u, 0xc0633555u, 0x40ce6556u, 0xc0ce6556u, 0xc0633555u, 0xc0ce6556u, 0xbf400000u, 0xbe830000u},
+    {0x3fc85eabu, 0xc08de855u, 0x40f217abu, 0xc0f217abu, 0xc08de855u, 0xc0f217abu, 0xbfb80000u, 0xbefc8556u},
+    {0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u},
+    {0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u},
+    {0x3f800000u, 0x00000000u, 0x40000000u, 0xc0000000u, 0x00000000u, 0xc0000000u, 0x00000000u, 0x00000000u},
+    {0x5d800000u, 0x00000000u, 0x5e000000u, 0xde000000u, 0x00000000u, 0xde000000u, 0xdd800000u, 0xec7ffffeu},
+    {0x3f800000u, 0x00000000u, 0x40000000u, 0xc0000000u, 0x00000000u, 0xc0000000u, 0x34000000u, 0x33ffffffu},
+    {0xb5c00000u, 0xb5800000u, 0xb6000000u, 0x36000000u, 0xb5800000u, 0x36000000u, 0xbfbffffcu, 0xb5bffffcu},
+};
+
+constexpr const char* Names[Results] = {
+    "v_mul_f32 v10, v4, v5",
+    "v_add_f32 v11, v10, v6",
+    "v_sub_f32 v12, v10, v6",
+    "v_subrev_f32 v13, v10, v6",
+    "v_add_f32 v14, v6, v10",
+    "v_sub_f32 v15, v6, v10",
+    "v_add_f32 v16, v4, v6",
+    "v_mul_f32 v17, v16, v5",
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Run(AgcDriver::VulkanDevice& device, const std::optional<ShaderRecompiler::ShaderFloatMode>& floatMode) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::span<const std::uint32_t> code(Code);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    request.context.floatMode = floatMode;
+    request.target.fusedExactArithmetic = FusedExactArithmetic;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check(const char* mode) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        for (std::uint32_t i = 0; i < Results; ++i) {
+            const auto actual = Output[tid * Results + i];
+            Require(actual == Expected[tid][i], std::string("f32 separate rounding: ") + (FusedExactArithmetic ? "fused " : "") + "lane " + std::to_string(tid) + " " + mode + " " + Names[i] + " is " + Hex(actual) + ", expected " + Hex(Expected[tid][i]));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        for (const bool fused : {false, true}) {
+            if (!fused && device->Target().fusedExactArithmetic) continue;
+            FusedExactArithmetic = fused;
+            Run(*device, std::nullopt);
+            Check("no float mode");
+            Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, false});
+            Check("IEEE=1");
+        }
+        std::puts("f32 separate rounding tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/shader/recompiler/CacheKey.hpp

@@ -297,6 +297,7 @@ private:
         append(key, value.float32FlushToZero);
         append(key, value.waveLdsMemoryBarriers);
         append(key, value.float32DenormPreserve);
+        append(key, value.fusedExactArithmetic);
     }
 };
 

core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp

@@ -832,7 +832,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const
     std::string buffer;
     Writer writer(buffer);
     writer.WriteU32(0x41505335u);
-    writer.WriteU32(43u);
+    writer.WriteU32(44u);
     writeShaderBinary(writer, request.shader);
     writeGuestContext(writer, request.context);
     writeSpirvTarget(writer, request.target);
@@ -870,6 +870,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const
         writer.WriteU32(std::bit_cast<std::uint32_t>(request.context.vertex->pointSizeMin));
         writer.WriteU32(std::bit_cast<std::uint32_t>(request.context.vertex->pointSizeMax));
     }
+    writer.WriteBool(request.target.fusedExactArithmetic);
     return base64Encode(buffer);
 }
 
@@ -878,7 +879,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const
     Reader reader(decoded);
     if (reader.ReadU32() != 0x41505335u) throw std::runtime_error("invalid recompile request signature");
     const auto version = reader.ReadU32();
-    if (version < 1u || version > 43u) throw std::runtime_error("unsupported recompile request serialization version");
+    if (version < 1u || version > 44u) throw std::runtime_error("unsupported recompile request serialization version");
     DeserializedRequest result{};
     result.request.shader = readShaderBinary(reader, result.shaderCode, result.shaderHeader);
     result.request.context = readGuestContext(reader, result, version);
@@ -932,6 +933,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const
         vertex.pointSizeMax = std::bit_cast<float>(reader.ReadU32());
         result.vertex = vertex;
     }
+    if (version >= 44u) result.request.target.fusedExactArithmetic = reader.ReadBool();
     if (version >= 21u && reader.Remaining() != 0u) throw std::runtime_error("RequestSerializer: unexpected trailing data");
     return result;
 }

core/shader/recompiler/Recompiler.cpp

@@ -605,6 +605,7 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program)
     targetOptions.robustBufferAccess2 = request.target.robustBufferAccess2;
     targetOptions.float32FlushToZero = request.target.float32FlushToZero;
     targetOptions.float32DenormPreserve = request.target.float32DenormPreserve;
+    targetOptions.fusedExactArithmetic = request.target.fusedExactArithmetic;
     targetOptions.meshBarycentricPixel = request.shader.stage == ShaderStage::Fragment && request.graphics && request.graphics->mesh;
     if (request.shader.stage == ShaderStage::Mesh && request.graphics && request.graphics->meshBarycentrics) {
         const auto& paired = *request.graphics->meshBarycentrics;

core/shader/recompiler/Recompiler.hpp

@@ -298,6 +298,7 @@ struct SpirvTarget {
     bool float32FlushToZero = false;
     bool waveLdsMemoryBarriers = false;
     bool float32DenormPreserve = false;
+    bool fusedExactArithmetic = false;
 };
 
 struct BindingLayout {

core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp

@@ -30,6 +30,7 @@ struct SpirvTargetOptions {
     bool robustBufferAccess2 = false;
     bool float32FlushToZero = false;
     bool float32DenormPreserve = false;
+    bool fusedExactArithmetic = false;
     bool meshBarycentricPixel = false;
     bool meshBarycentricVertex = false;
     std::uint32_t barycentricSmoothLocation = 0xffffffffu;

core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp

@@ -113,6 +113,7 @@ struct SpirvEmitterState {
     bool robustBufferAccess2 = false;
     bool float32FlushToZero = false;
     bool float32DenormPreserve = false;
+    bool fusedExactArithmetic = false;
     bool meshBarycentricPixel = false;
     bool meshBarycentricVertex = false;
     std::uint32_t barycentricSmoothLocation = 0xffffffffu;

core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp

@@ -140,6 +140,20 @@ std::uint32_t EmitExt(SpirvEmitterState& state, std::uint32_t type, std::uint32_
     return result;
 }
 
+namespace {
+
+std::uint32_t ExactMul(SpirvEmitterState& state, std::uint32_t lhs, std::uint32_t rhs) {
+    if (state.fusedExactArithmetic) return EmitExt(state, TypeF32(state), GLSLstd450Fma, {lhs, rhs, ConstantF32(state, 0x80000000u)});
+    return Exact(state, Binary(state, spv::OpFMul, TypeF32(state), lhs, rhs));
+}
+
+std::uint32_t ExactSub(SpirvEmitterState& state, std::uint32_t lhs, std::uint32_t rhs) {
+    if (state.fusedExactArithmetic) return EmitExt(state, TypeF32(state), GLSLstd450Fma, {ConstantF32(state, 0xbf800000u), rhs, lhs});
+    return Exact(state, Binary(state, spv::OpFSub, TypeF32(state), lhs, rhs));
+}
+
+}
+
 std::uint32_t EmitDppWriteCondition(SpirvValueEmitContext& ctx, const IrValue& inst) {
     auto& state = ctx.state;
     const auto flags = inst.Flags<DppMoveFlags>();
@@ -994,13 +1008,13 @@ std::uint32_t Log2SeriesNearOne(SpirvEmitterState& state, std::uint32_t differen
     for (const std::uint32_t coefficient : {0x3e124925u, 0xbe2aaaabu, 0x3e4ccccdu, 0xbe800000u, 0x3eaaaaabu, 0xbf000000u, 0x3f800000u}) {
         polynomial = Exact(state, EmitExt(state, TypeF32(state), GLSLstd450Fma, {polynomial, difference, ConstantF32(state, coefficient)}));
     }
-    const auto product = Exact(state, Binary(state, spv::OpFMul, TypeF32(state), difference, polynomial));
-    return Exact(state, Binary(state, spv::OpFMul, TypeF32(state), product, ConstantF32(state, 0x3fb8aa3bu)));
+    const auto product = ExactMul(state, difference, polynomial);
+    return ExactMul(state, product, ConstantF32(state, 0x3fb8aa3bu));
 }
 
 std::uint32_t EmitFPLog2(SpirvEmitterState& state, std::uint32_t arg0) {
     const auto value = EmitExt(state, TypeF32(state), GLSLstd450Log2, {arg0});
-    const auto difference = Exact(state, Binary(state, spv::OpFSub, TypeF32(state), arg0, ConstantF32(state, 0x3f800000u)));
+    const auto difference = ExactSub(state, arg0, ConstantF32(state, 0x3f800000u));
     const auto series = Log2SeriesNearOne(state, difference);
     const auto valueBits = Unary(state, spv::OpBitcast, TypeU32(state), value);
     const auto seriesBits = Unary(state, spv::OpBitcast, TypeU32(state), series);
@@ -1011,16 +1025,16 @@ std::uint32_t EmitFPLog2(SpirvEmitterState& state, std::uint32_t arg0) {
 }
 
 std::uint32_t SinTurnSeries(SpirvEmitterState& state, std::uint32_t turns) {
-    const auto square = Exact(state, Binary(state, spv::OpFMul, TypeF32(state), turns, turns));
+    const auto square = ExactMul(state, turns, turns);
     auto polynomial = ConstantF32(state, 0xc17183a8u);
     for (const std::uint32_t coefficient : {0x42283c1au, 0xc2996966u, 0x42a335e3u, 0xc2255de7u, 0x40c90fdbu}) {
         polynomial = Exact(state, EmitExt(state, TypeF32(state), GLSLstd450Fma, {polynomial, square, ConstantF32(state, coefficient)}));
     }
-    return Exact(state, Binary(state, spv::OpFMul, TypeF32(state), turns, polynomial));
+    return ExactMul(state, turns, polynomial);
 }
 
 std::uint32_t CosTurnSeries(SpirvEmitterState& state, std::uint32_t turns) {
-    const auto square = Exact(state, Binary(state, spv::OpFMul, TypeF32(state), turns, turns));
+    const auto square = ExactMul(state, turns, turns);
     auto polynomial = ConstantF32(state, 0x40fce9c5u);
     for (const std::uint32_t coefficient : {0xc1d368f9u, 0x4270fa83u, 0xc2aae9e4u, 0x4281e0f8u, 0xc19de9e6u, 0x3f800000u}) {
         polynomial = Exact(state, EmitExt(state, TypeF32(state), GLSLstd450Fma, {polynomial, square, ConstantF32(state, coefficient)}));
@@ -1047,9 +1061,9 @@ std::uint32_t EmitFPSin(SpirvEmitterState& state, std::uint32_t arg0) {
     const auto absolute = EmitExt(state, TypeF32(state), GLSLstd450FAbs, {cycle});
     const auto signBits = Binary(state, spv::OpBitwiseAnd, TypeU32(state), Unary(state, spv::OpBitcast, TypeU32(state), cycle), ConstantU32(state, 0x80000000u));
     const auto nearHalf = Binary(state, spv::OpFOrdGreaterThan, TypeBool(state), absolute, ConstantF32(state, 0x3e800000u));
-    const auto distance = Select(state, TypeF32(state), nearHalf, Exact(state, Binary(state, spv::OpFSub, TypeF32(state), ConstantF32(state, 0x3f000000u), absolute)), absolute);
+    const auto distance = Select(state, TypeF32(state), nearHalf, ExactSub(state, ConstantF32(state, 0x3f000000u), absolute), absolute);
     const auto pastEighth = Binary(state, spv::OpFOrdGreaterThan, TypeBool(state), distance, ConstantF32(state, 0x3e000000u));
-    const auto series = Select(state, TypeF32(state), pastEighth, CosTurnSeries(state, Exact(state, Binary(state, spv::OpFSub, TypeF32(state), ConstantF32(state, 0x3e800000u), distance))), SinTurnSeries(state, distance));
+    const auto series = Select(state, TypeF32(state), pastEighth, CosTurnSeries(state, ExactSub(state, ConstantF32(state, 0x3e800000u), distance)), SinTurnSeries(state, distance));
     const auto value = SeriesWhenFar(state, EmitExt(state, TypeF32(state), GLSLstd450Sin, {Binary(state, spv::OpFMul, TypeF32(state), absolute, ConstantF32(state, 0x40c90fdbu))}), series);
     const auto linearConstant = Select(state, TypeF32(state), nearHalf, ConstantF32(state, 0x40c90fdbu), ConstantF32(state, 0x40c90fd5u));
     const auto result = TrigLinearNearZero(state, value, distance, linearConstant);
@@ -1059,10 +1073,10 @@ std::uint32_t EmitFPSin(SpirvEmitterState& state, std::uint32_t arg0) {
 std::uint32_t EmitFPCos(SpirvEmitterState& state, std::uint32_t arg0) {
     const auto cycle = EmitTrigCycleF32(state, arg0, false);
     const auto absolute = EmitExt(state, TypeF32(state), GLSLstd450FAbs, {cycle});
-    const auto distance = Exact(state, Binary(state, spv::OpFSub, TypeF32(state), ConstantF32(state, 0x3e800000u), absolute));
+    const auto distance = ExactSub(state, ConstantF32(state, 0x3e800000u), absolute);
     const auto low = CosTurnSeries(state, absolute);
     const auto middle = SinTurnSeries(state, distance);
-    const auto high = Unary(state, spv::OpFNegate, TypeF32(state), CosTurnSeries(state, Exact(state, Binary(state, spv::OpFSub, TypeF32(state), ConstantF32(state, 0x3f000000u), absolute))));
+    const auto high = Unary(state, spv::OpFNegate, TypeF32(state), CosTurnSeries(state, ExactSub(state, ConstantF32(state, 0x3f000000u), absolute)));
     const auto pastEighth = Binary(state, spv::OpFOrdGreaterThan, TypeBool(state), absolute, ConstantF32(state, 0x3e000000u));
     const auto pastThreeEighths = Binary(state, spv::OpFOrdGreaterThan, TypeBool(state), absolute, ConstantF32(state, 0x3ec00000u));
     const auto series = Select(state, TypeF32(state), pastThreeEighths, high, Select(state, TypeF32(state), pastEighth, middle, low));
@@ -1467,14 +1481,17 @@ std::uint32_t EmitFPUnordGreaterThanEqual32(SpirvEmitterState& state, std::uint3
 }
 
 std::uint32_t EmitFPAdd32(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1) {
+    if (state.fusedExactArithmetic) return EmitGlsl<GLSLstd450Fma, IrType::F32>(state, ConstantF32(state, 0x3f800000u), arg0, arg1);
     return Exact(state, EmitNative<spv::OpFAdd, IrType::F32>(state, arg0, arg1));
 }
 
 std::uint32_t EmitFPSub32(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1) {
+    if (state.fusedExactArithmetic) return EmitGlsl<GLSLstd450Fma, IrType::F32>(state, ConstantF32(state, 0xbf800000u), arg1, arg0);
     return Exact(state, EmitNative<spv::OpFSub, IrType::F32>(state, arg0, arg1));
 }
 
 std::uint32_t EmitFPMul32(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1) {
+    if (state.fusedExactArithmetic) return EmitGlsl<GLSLstd450Fma, IrType::F32>(state, arg0, arg1, ConstantF32(state, 0x80000000u));
     return Exact(state, EmitNative<spv::OpFMul, IrType::F32>(state, arg0, arg1));
 }
 

core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp

@@ -279,6 +279,7 @@ std::vector<std::uint32_t> SpirvEmitter::Emit(const IrProgram& program, const Sh
     state.float32FlushToZero = target.float32FlushToZero;
     state.waveLdsMemoryBarriers = target.waveLdsMemoryBarriers;
     state.float32DenormPreserve = target.float32DenormPreserve;
+    state.fusedExactArithmetic = target.fusedExactArithmetic;
     state.hostSubgroupSize = target.subgroupSize;
     if (state.requirements.ldsLock && LdsInDeviceMemory(state) && std::find(target.supportedCapabilities.begin(), target.supportedCapabilities.end(), static_cast<std::uint32_t>(spv::CapabilityInt64Atomics)) != target.supportedCapabilities.end()) {
         state.requirements.ldsLock = false;

core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp

@@ -493,7 +493,21 @@ std::uint32_t FoldedOffsetCoord(SpirvValueEmitContext& ctx, const ImageEmitAcces
     auto& state = ctx.state;
     const auto components = setup.dimensionInfo.spatialComponents;
     const auto type = components == 1u ? TypeF32(state) : TypeF32Vector(state, components);
-    const auto sum = Binary(state, spv::OpFAdd, type, setup.coord, Unary(state, spv::OpConvertSToF, type, PackedOffset(ctx, access, setup.layout)));
+    const auto offset = Unary(state, spv::OpConvertSToF, type, PackedOffset(ctx, access, setup.layout));
+    if (state.fusedExactArithmetic) {
+        auto one = ConstantF32(state, 0x3f800000u);
+        if (components != 1u) {
+            const auto ones = state.module.AllocateId();
+            std::vector<std::uint32_t> words{spv::OpCompositeConstruct, type, ones};
+            words.insert(words.end(), components, one);
+            state.module.AddFunction(words);
+            one = ones;
+        }
+        const auto fused = state.module.AllocateId();
+        state.module.AddFunction(spv::OpExtInst, type, fused, GlslStd450(state), GLSLstd450Fma, one, setup.coord, offset);
+        return fused;
+    }
+    const auto sum = Binary(state, spv::OpFAdd, type, setup.coord, offset);
     state.module.AddAnnotation(spv::OpDecorate, sum, spv::DecorationNoContraction);
     return sum;
 }
@@ -1201,8 +1215,16 @@ void EmitPackedHorizontalGatherOp(SpirvValueEmitContext& ctx, const ImageEmitAcc
             return value;
         };
         const auto texelFloor = [&](std::uint32_t address, std::uint32_t length, bool centre) {
-            auto scaled = Binary(state, spv::OpFMul, TypeF32(state), AddressF32(ctx, access, address), Unary(state, spv::OpConvertSToF, TypeF32(state), length));
-            state.module.AddAnnotation(spv::OpDecorate, scaled, spv::DecorationNoContraction);
+            const auto addressF32 = AddressF32(ctx, access, address);
+            const auto lengthF32 = Unary(state, spv::OpConvertSToF, TypeF32(state), length);
+            std::uint32_t scaled = 0;
+            if (state.fusedExactArithmetic) {
+                scaled = state.module.AllocateId();
+                state.module.AddFunction(spv::OpExtInst, TypeF32(state), scaled, GlslStd450(state), GLSLstd450Fma, addressF32, lengthF32, ConstantF32(state, 0x80000000u));
+            } else {
+                scaled = Binary(state, spv::OpFMul, TypeF32(state), addressF32, lengthF32);
+                state.module.AddAnnotation(spv::OpDecorate, scaled, spv::DecorationNoContraction);
+            }
             if (centre) {
                 scaled = Binary(state, spv::OpFSub, TypeF32(state), scaled, ConstantF32(state, 0x3f000000u));
             }

core/shader/recompiler/tests/RequestSerializerTests.cpp

@@ -219,12 +219,13 @@ int main() {
             Require(request.graphics->draw.indexAddress == 0xabcdef3400u && request.graphics->draw.indexCount == 57u && request.graphics->draw.indexElementBytes == 4u && request.graphics->draw.instanceCount == 9u);
             Require(request.target.float16DenormPreserve == preserve);
             auto upgraded = bytes;
-            upgraded[4] = 43u;
+            upgraded[4] = 44u;
             InsertMeshBarycentricFlag(upgraded);
             if (fixture.pixel) upgraded.insert(upgraded.begin() + 258u, {0u, 0u, 0u});
             if (fixture.pixel) InsertPixelDefaultsAfterFrontFace(upgraded);
             if (fixture.tessellation) upgraded.insert(upgraded.end() - 35u, 6u, 0u);
             AppendVertexTail(upgraded);
+            upgraded.push_back(0u);
             Require(serializer.Serialize(request) == EncodeBase64(upgraded));
             auto truncated = bytes;
             truncated.pop_back();
@@ -259,12 +260,13 @@ int main() {
                     layerRejected = std::string_view(error.what()) == "RequestSerializer: truncated data";
                 }
                 Require(layerRejected);
-                layered[4] = 43u;
+                layered[4] = 44u;
                 InsertMeshBarycentricFlag(layered);
                 layered.insert(layered.begin() + 259u, {0u, 0u});
                 InsertPixelDefaultsAfterFrontFace(layered);
                 if (fixture.tessellation) layered.insert(layered.end() - 35u, 6u, 0u);
                 AppendVertexTail(layered);
+                layered.push_back(0u);
                 Require(serializer.Serialize(layerReplay.request) == EncodeBase64(layered));
             }
         }
@@ -285,11 +287,12 @@ int main() {
             Require(replay.request.context.pixel->rasterPrimitive == ShaderRecompiler::RasterPrimitive::Rectangle);
             Require(replay.request.context.vertex->paClVsOutCntl == 0x00050a05u);
             Require(replay.request.target.float16DenormPreserve == preserve);
-            bytes[4] = 43u;
+            bytes[4] = 44u;
             InsertMeshBarycentricFlag(bytes);
             bytes.insert(bytes.begin() + 259u, {0u, 0u});
             InsertPixelDefaultsAfterFrontFace(bytes);
             AppendVertexTail(bytes);
+            bytes.push_back(0u);
             Require(serializer.Serialize(replay.request) == EncodeBase64(bytes));
 
             auto truncated23 = DecodeBase64(encoded23);
@@ -415,11 +418,12 @@ int main() {
                 if (fixture.pixel) main38.insert(main38.begin() + 259u, {0u, 0u});
                 if (fixture.pixel) InsertPixelDefaultsAfterFrontFace(main38, false);
                 auto upgraded = main38;
-                upgraded[4] = 43u;
+                upgraded[4] = 44u;
                 if (fixture.pixel) upgraded.insert(upgraded.begin() + 317u, 16u, 0u);
                 if (fixture.tessellation) upgraded.insert(upgraded.end() - 35u, 1u, 0u);
                 main38.insert(main38.end(), sizeof(std::uint64_t) + 1u + 5u + 1u, 0u);
                 AppendVertexTail(upgraded);
+                upgraded.push_back(0u);
                 Require(serializer.Serialize(request) == EncodeBase64(upgraded));
                 const auto replay38 = serializer.Deserialize(EncodeBase64(main38));
                 Require(replay38.request.graphics.has_value() && replay38.request.graphics->tessellation.has_value() == fixture.tessellation);
@@ -435,8 +439,11 @@ int main() {
                         pointed.context.vertex->pointSize = 3.0f;
                         pointed.context.vertex->pointSizeMin = 1.0f;
                         pointed.context.vertex->pointSizeMax = 8.0f;
-                        auto version43 = DecodeBase64(serializer.Serialize(pointed));
-                        Require(version43.size() > 14u && version43[4] == 43u);
+                        auto version44 = DecodeBase64(serializer.Serialize(pointed));
+                        Require(version44.size() > 15u && version44[4] == 44u);
+                        auto version43 = version44;
+                        version43.pop_back();
+                        version43[4] = 43u;
                         const auto replay43 = serializer.Deserialize(EncodeBase64(version43));
                         Require(replay43.request.graphics->tessellation->mergedHull == merged && replay43.request.context.vertex->pointList && replay43.request.context.vertex->pointSize == 3.0f && replay43.request.context.vertex->pointSizeMin == 1.0f && replay43.request.context.vertex->pointSizeMax == 8.0f);
                         auto version42 = version43;
@@ -464,7 +471,7 @@ int main() {
                     const auto replay25 = serializer.Deserialize(EncodeBase64(main25));
                     Require(!fixture.pixel || (replay25.request.context.pixel->layerExported == layer && replay25.request.context.pixel->ditheredAlphaToCoverage == dither && replay25.request.context.pixel->reversedBlend == 0u));
                     Require(replay25.request.target.float16DenormPreserve == preserve && (!fixture.tessellation || replay25.request.graphics->tessellation->primitiveGroupSize == 37u));
-                    main25[4] = 43u;
+                    main25[4] = 44u;
                     InsertMeshBarycentricFlag(main25);
                     if (fixture.pixel) {
                         main25.insert(main25.begin() + 260u, 0u);
@@ -472,6 +479,7 @@ int main() {
                     }
                     if (fixture.tessellation) main25.insert(main25.end() - 35u, 1u, 0u);
                     AppendVertexTail(main25);
+                    main25.push_back(0u);
                     Require(serializer.Serialize(replay25.request) == EncodeBase64(main25));
                 }
             }
@@ -487,14 +495,15 @@ int main() {
         const auto replay = serializer.Deserialize(encoded);
         Require(replay.request.target.float16DenormPreserve == preserve && replay.request.target.float32FlushToZero == !preserve && replay.request.target.float32DenormPreserve == preserve);
         auto bytes = DecodeBase64(encoded);
-        Require(bytes.size() >= 8u && bytes[4] == 43u && bytes[5] == 0u && bytes[6] == 0u && bytes[7] == 0u);
+        Require(bytes.size() >= 8u && bytes[4] == 44u && bytes[5] == 0u && bytes[6] == 0u && bytes[7] == 0u);
         auto version40 = bytes;
         version40.pop_back();
+        version40.pop_back();
         version40[4] = 40u;
         const auto replay40 = serializer.Deserialize(EncodeBase64(version40));
         Require(!replay40.request.target.float32DenormPreserve && replay40.request.target.float32FlushToZero == !preserve);
         auto future = bytes;
-        future[4] = 44u;
+        future[4] = 45u;
         bool futureRejected = false;
         try {
             static_cast<void>(serializer.Deserialize(EncodeBase64(future)));
@@ -540,6 +549,7 @@ int main() {
     const auto order = std::search(version41.begin(), version41.end(), pixel.sampleLocations.begin(), pixel.sampleLocations.end()) + pixel.sampleLocations.size() + 1;
     Require(version41.end() - order > static_cast<std::ptrdiff_t>(sizeof(pixel.sampleOrder)) && *order == 0x10u);
     version41.erase(order, order + sizeof(pixel.sampleOrder));
+    version41.pop_back();
     version41[4] = 41u;
     const auto replay41 = serializer.Deserialize(EncodeBase64(version41));
     Require(replay41.request.context.pixel.has_value() && replay41.request.context.pixel->sampleLocations == pixel.sampleLocations && replay41.request.context.pixel->sampleOrder == std::array<std::uint32_t, 4>{});
@@ -557,6 +567,7 @@ int main() {
         version35.erase(stencilOperationValue, stencilOperationValue + pixelTail);
         version35[4] = 35u;
         version35.pop_back();
+        version35.pop_back();
         const auto replay35 = serializer.Deserialize(EncodeBase64(version35));
         Require(replay35.request.target.spannedWaves == spanned && !replay35.request.target.float32FlushToZero);
     }
@@ -574,6 +585,7 @@ int main() {
     Require(serializer.Serialize(pointReplay.request) == pointEncoded);
     auto pointBytes = DecodeBase64(pointEncoded);
     pointBytes.pop_back();
+    pointBytes.pop_back();
     bool pointRejected = false;
     try {
         static_cast<void>(serializer.Deserialize(EncodeBase64(pointBytes)));
@@ -593,7 +605,7 @@ int main() {
         for (const std::uint8_t version : {32u, 33u, 34u, 35u, 36u, 37u, 38u}) {
             pointRequest.target.spannedWaves = spanned;
             auto legacy = DecodeBase64(serializer.Serialize(pointRequest));
-            legacy.resize(legacy.size() - (version >= 36u ? 14u : version >= 35u ? 15u : 20u));
+            legacy.resize(legacy.size() - (version >= 36u ? 15u : version >= 35u ? 16u : 21u));
             legacy[4] = version;
             const auto replayLegacy = serializer.Deserialize(EncodeBase64(legacy));
             Require(replayLegacy.request.target.spannedWaves == spanned);
@@ -605,7 +617,7 @@ int main() {
             Require(!replayLegacy.request.context.vertex->pointList && replayLegacy.request.context.vertex->pointSize == 1.0f);
             Require(replayLegacy.request.context.vertex->pointSizeMin == 0.0f && replayLegacy.request.context.vertex->pointSizeMax == 0.0f);
             const auto upgradedLegacy = DecodeBase64(serializer.Serialize(replayLegacy.request));
-            Require(upgradedLegacy[4] == 43u && upgradedLegacy.size() == legacy.size() + (version >= 36u ? 14u : version >= 35u ? 15u : 20u));
+            Require(upgradedLegacy[4] == 44u && upgradedLegacy.size() == legacy.size() + (version >= 36u ? 15u : version >= 35u ? 16u : 21u));
         }
     }
 
@@ -614,14 +626,14 @@ int main() {
     const auto sampledPoint = serializer.Deserialize(serializer.Serialize(pointRequest));
     Require(sampledPoint.request.context.vertex->pointList && sampledPoint.request.context.pixel->sampleOrder == pixel.sampleOrder);
     auto legacy42 = DecodeBase64(serializer.Serialize(pointRequest));
-    legacy42.resize(legacy42.size() - 13u);
+    legacy42.resize(legacy42.size() - 14u);
     legacy42[4] = 42u;
     const auto replay42 = serializer.Deserialize(EncodeBase64(legacy42));
     Require(replay42.request.context.pixel->sampleOrder == pixel.sampleOrder && !replay42.request.context.vertex->pointList && replay42.request.context.vertex->pointSize == 1.0f);
     pixel.sampleOrder = {};
     pointRequest.context.pixel = pixel;
     auto legacy34 = DecodeBase64(serializer.Serialize(pointRequest));
-    legacy34.resize(legacy34.size() - 20u);
+    legacy34.resize(legacy34.size() - 21u);
     const auto stencil34 = std::search(legacy34.begin(), legacy34.end(), pixel.sampleLocations.begin(), pixel.sampleLocations.begin() + 4) + pixel.sampleLocations.size();
     const auto legacyPixelTail = static_cast<std::ptrdiff_t>(1u + sizeof(pixel.sampleOrder));
     Require(legacy34.end() - stencil34 > legacyPixelTail && std::all_of(stencil34, stencil34 + legacyPixelTail, [](std::uint8_t value) { return value == 0u; }));
@@ -631,5 +643,14 @@ int main() {
     Require(replay34.request.context.pixel.has_value() && replay34.request.context.pixel->iteratesSamples && replay34.request.context.pixel->positionLocation == pixel.positionLocation && replay34.request.context.pixel->sampleLocations == pixel.sampleLocations);
     Require(replay34.request.context.vertex.has_value() && !replay34.request.context.vertex->pointList && replay34.request.context.vertex->pointSize == 1.0f);
 
+    for (const bool fused : {false, true}) {
+        request.target.fusedExactArithmetic = fused;
+        auto bytes = DecodeBase64(serializer.Serialize(request));
+        Require(serializer.Deserialize(EncodeBase64(bytes)).request.target.fusedExactArithmetic == fused && bytes.back() == (fused ? 1u : 0u));
+        bytes[4] = 43u;
+        bytes.pop_back();
+        Require(!serializer.Deserialize(EncodeBase64(bytes)).request.target.fusedExactArithmetic);
+    }
+
     return 0;
 }