Merge pull request #7038 from thuanquang/fix/packed-f16-denormal-modes

feat(shader): translate packed f16 in the f16 denormal modes that flush

core/libs/prx/libSceAgcDriver/tests/execution/Float16MinMaxBits.cpp

@@ -85,7 +85,6 @@ const std::array<Check, 4> PackedChecks{{
 struct Mode {
     const char* name;
     std::optional<ShaderRecompiler::ShaderFloatMode> floatMode;
-    bool packed = true;
 };
 
 const std::array<Mode, 7> Modes{{
@@ -94,8 +93,8 @@ const std::array<Mode, 7> Modes{{
     {"dx10", ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}},
     {"ieee", ShaderRecompiler::ShaderFloatMode{0xc0u, false, true, false}},
     {"none", ShaderRecompiler::ShaderFloatMode{0xc0u, false, false, false}},
-    {"f16 denormals flushed, dx10 ieee", ShaderRecompiler::ShaderFloatMode{0x30u, true, true, false}, false},
-    {"f16 input denormals flushed", ShaderRecompiler::ShaderFloatMode{0xb0u, false, false, false}, false},
+    {"f16 denormals flushed, dx10 ieee", ShaderRecompiler::ShaderFloatMode{0x30u, true, true, false}},
+    {"f16 input denormals flushed", ShaderRecompiler::ShaderFloatMode{0xb0u, false, false, false}},
 }};
 
 std::uint64_t Seed = 0x2545f4914f6cdd1dull;
@@ -214,14 +213,7 @@ struct Model {
                 bits[index] = (registers[source.reg] >> (selectHigh ? 16u : 0u)) & 0xffffu;
                 if (high != 0u ? source.negateHi : source.negate) bits[index] ^= 0x8000u;
             }
-            std::uint32_t lane = Numeric(check.kind == Kind::Max, Read(bits[0]), Read(bits[1])).bits;
-            lane = IsNan(bits[0]) ? bits[1] : lane;
-            lane = IsNan(bits[1]) ? bits[0] : lane;
-            if (ieee) {
-                lane = IsSnan(bits[1]) ? Quiet(bits[1]) : lane;
-                lane = IsSnan(bits[0]) ? Quiet(bits[0]) : lane;
-            }
-            result |= Clamp(check.clamp, lane) << (16u * high);
+            result |= Clamp(check.clamp, Pair(check.kind == Kind::Max, Read(bits[0]), Read(bits[1])).bits) << (16u * high);
         }
         return result;
     }
@@ -267,8 +259,7 @@ int main() {
             const bool flushInputs = mode.floatMode.has_value() && (mode.floatMode->floatMode & 0x40u) == 0u;
             const Model model{mode.floatMode.has_value() && mode.floatMode->ieeeMode, !mode.floatMode.has_value() || mode.floatMode->dx10Clamp, flushInputs};
             const auto scalar = compile(ScalarCode, mode);
-            std::optional<ShaderRecompiler::RecompileResult> packed;
-            if (mode.packed) packed = compile(PackedCode, mode);
+            const auto packed = compile(PackedCode, mode);
             for (std::uint32_t batch = 0; batch < Batches; ++batch) {
                 for (std::uint32_t tid = 0; tid < Threads; ++tid) {
                     std::uint32_t* row = &Input[tid * Inputs];
@@ -293,7 +284,7 @@ int main() {
                     }
                 };
                 run(scalar, ScalarCode, ScalarChecks, false);
-                if (packed) run(*packed, PackedCode, PackedChecks, true);
+                run(packed, PackedCode, PackedChecks, true);
             }
         }
         std::printf("f16 min/max tests passed (%llu results)\n", static_cast<unsigned long long>(compared));

core/libs/prx/libSceAgcDriver/tests/execution/PackedAlu.cpp

@@ -196,7 +196,7 @@ constexpr const char* Names[48] = {
 };
 
 bool Ieee = false;
-bool FlushHalf = false;
+bool DirectedHalf = false;
 
 void Fill(std::uint32_t tid, std::uint32_t* words) {
     std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words);
@@ -236,7 +236,7 @@ void Run(AgcDriver::VulkanDevice& device) {
     };
     request.useCache = false;
     if (Ieee) request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false};
-    if (FlushHalf) request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0x30u, true, false, false};
+    if (DirectedHalf) request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf4u, true, false, false};
     const auto result = ShaderRecompiler::Recompile(request);
     device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
     device.WaitIdle();
@@ -264,14 +264,14 @@ int main() {
             Check();
         }
         Ieee = false;
-        FlushHalf = true;
+        DirectedHalf = true;
         bool refused = false;
         try {
             Run(*device);
         } catch (const std::exception& error) {
-            refused = std::string(error.what()).find("f16 denormals not kept") != std::string::npos;
+            refused = std::string(error.what()).find("f16 rounding not to nearest even") != std::string::npos;
         }
-        Require(refused, "packed alu: flushed f16 denormals were not refused");
+        Require(refused, "packed alu: directed f16 rounding was not refused");
         std::puts("packed alu tests passed");
         return 0;
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/execution/PackedF16DenormalModes.cpp

@@ -0,0 +1,203 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 16;
+constexpr std::uint32_t Inputs = 4;
+constexpr std::uint32_t Results = 4;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+alignas(256) constexpr std::array<std::uint32_t, 14> Code{
+    0x34020084u, 0x34040084u, 0xe03c1000u, 0x80000401u, 0xbf8c3f70u, 0xcc0f400au, 0x18020b04u, 0xcc10400bu,
+    0x18020b04u, 0xcc0e400cu, 0x1c1a0b04u, 0xe07c1000u, 0x80010a02u, 0xbf810000u,
+};
+constexpr std::uint32_t Rows[Threads][3] = {
+    {0x02000001u, 0x02000000u, 0x02000000u},
+    {0x83ff8001u, 0x03ff0001u, 0x00000000u},
+    {0x7c0003ffu, 0x00010001u, 0x00000000u},
+    {0x7bff0400u, 0x3c008001u, 0x00010000u},
+    {0x04013bffu, 0xb8000400u, 0x02000000u},
+    {0xbc003c01u, 0x000103ffu, 0x04000000u},
+    {0x3c000400u, 0x84010400u, 0x04008400u},
+    {0x04000400u, 0x3bff8400u, 0x83ff0400u},
+    {0x84000001u, 0x3bff0001u, 0x03ff8400u},
+    {0x35553c00u, 0x05550001u, 0x00000000u},
+    {0x01553800u, 0x44000001u, 0x00000001u},
+    {0x81553800u, 0x44000400u, 0x04000000u},
+    {0x00008400u, 0x80003800u, 0x80000000u},
+    {0x80003c00u, 0x80003c00u, 0x00000001u},
+    {0x5bff1fffu, 0x04002001u, 0x00000000u},
+    {0x3bff2001u, 0x3bff1fffu, 0x00018001u},
+};
+constexpr std::uint32_t Expected[4][Threads][3] = {
+    {
+        {0x00000000u, 0x00000000u, 0x00000000u},
+        {0x00000000u, 0x80008000u, 0x00000000u},
+        {0x7c000000u, 0xfe000000u, 0xfe000000u},
+        {0x7bff0400u, 0x7bff8000u, 0x7bff0000u},
+        {0xb8003bffu, 0x80000000u, 0x80000000u},
+        {0xbc003c01u, 0x80000000u, 0x04000000u},
+        {0x3c000800u, 0x84010000u, 0x80008400u},
+        {0x3bff0000u, 0x00008000u, 0x00000400u},
+        {0x3bff0000u, 0x80000000u, 0x80008400u},
+        {0x35553c00u, 0x00000000u, 0x00000000u},
+        {0x44003800u, 0x00000000u, 0x00000000u},
+        {0x44003800u, 0x80000000u, 0x04000000u},
+        {0x00003800u, 0x80008000u, 0x80008000u},
+        {0x80004000u, 0x00003c00u, 0x00003c00u},
+        {0x5bff2400u, 0x23ff0400u, 0x23ff0400u},
+        {0x3fff2400u, 0x3bfe0400u, 0x3bfe0400u},
+    },
+    {
+        {0x04000000u, 0x00000000u, 0x00000000u},
+        {0x00000000u, 0x80008000u, 0x80008000u},
+        {0x7c000400u, 0x7c000000u, 0x7c000000u},
+        {0x7bff0000u, 0x7bff8000u, 0x7bff8000u},
+        {0xb8003bffu, 0x80000000u, 0x80000000u},
+        {0xbc003c01u, 0x80000400u, 0x00000400u},
+        {0x3c000800u, 0x84010000u, 0x80008400u},
+        {0x3bff0000u, 0x00008000u, 0x00000400u},
+        {0x3bff0000u, 0x80000000u, 0x80008400u},
+        {0x35553c00u, 0x00000000u, 0x00000000u},
+        {0x44003800u, 0x05540000u, 0x05540000u},
+        {0x44003800u, 0x85540000u, 0x80000000u},
+        {0x00003800u, 0x80008000u, 0x80008000u},
+        {0x80004000u, 0x00003c00u, 0x00003c00u},
+        {0x5bff2400u, 0x23ff0400u, 0x23ff0400u},
+        {0x3fff2400u, 0x3bfe0400u, 0x3bfe0000u},
+    },
+    {
+        {0x00000000u, 0x00000000u, 0x00000000u},
+        {0x00000000u, 0x80008000u, 0x00000000u},
+        {0x7c000000u, 0xfe000000u, 0xfe000000u},
+        {0x7bff0400u, 0x7bff8000u, 0x7bff0000u},
+        {0xb8003bffu, 0x82000400u, 0x82000400u},
+        {0xbc003c01u, 0x80000000u, 0x04000000u},
+        {0x3c000800u, 0x84010000u, 0x80018400u},
+        {0x3bff0000u, 0x04008000u, 0x04000400u},
+        {0x3bff0000u, 0x84000000u, 0x84008400u},
+        {0x35553c00u, 0x01c70000u, 0x01c70000u},
+        {0x44003800u, 0x00000000u, 0x00000000u},
+        {0x44003800u, 0x80000200u, 0x04000200u},
+        {0x00003800u, 0x80008200u, 0x80008200u},
+        {0x80004000u, 0x00003c00u, 0x00003c00u},
+        {0x5bff2400u, 0x23ff0400u, 0x23ff0400u},
+        {0x3fff2400u, 0x3bfe0400u, 0x3bfe0400u},
+    },
+    {
+        {0x04000001u, 0x00000000u, 0x02000000u},
+        {0x00000000u, 0x80008000u, 0x80008000u},
+        {0x7c000400u, 0x7c000000u, 0x7c000000u},
+        {0x7bff03ffu, 0x7bff8000u, 0x7bff8000u},
+        {0xb8003bffu, 0x82000400u, 0x80000400u},
+        {0xbc003c01u, 0x80010400u, 0x03ff0400u},
+        {0x3c000800u, 0x84010000u, 0x80018400u},
+        {0x3bff0000u, 0x04008000u, 0x00000400u},
+        {0x3bff0002u, 0x84000000u, 0x80008400u},
+        {0x35553c00u, 0x01c70001u, 0x01c70001u},
+        {0x44003800u, 0x05540000u, 0x05540002u},
+        {0x44003800u, 0x85540200u, 0x81540200u},
+        {0x00003800u, 0x80008200u, 0x80008200u},
+        {0x80004000u, 0x00003c00u, 0x00003c00u},
+        {0x5bff2400u, 0x23ff0400u, 0x23ff0400u},
+        {0x3fff2400u, 0x3bfe0400u, 0x3bfe03ffu},
+    },
+};
+
+struct Mode {
+    const char* name;
+    ShaderRecompiler::ShaderFloatMode value;
+    std::uint32_t field;
+};
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x31016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+void Run(AgcDriver::VulkanDevice& device, const Mode& mode) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]);
+        Input[tid * Inputs + 3u] = 0u;
+    }
+    Output.fill(0xdeadbeefu);
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::span<const std::uint32_t> code(Code);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    request.context.floatMode = mode.value;
+    const auto result = ShaderRecompiler::Recompile(request);
+    device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
+    device.WaitIdle();
+}
+
+void Check(const Mode& mode) {
+    constexpr const char* Names[3] = {"v_pk_add_f16", "v_pk_mul_f16", "v_pk_fma_f16"};
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        for (std::uint32_t form = 0; form < 3u; ++form) {
+            const std::uint32_t actual = Output[tid * Results + form];
+            const std::uint32_t expected = Expected[mode.field][tid][form];
+            Require(actual == expected, std::string(Names[form]) + " " + mode.name + " of " + Hex(Rows[tid][0]) + ", " + Hex(Rows[tid][1]) + ", " + Hex(Rows[tid][2]) + " is " + Hex(actual) + ", expected " + Hex(expected));
+        }
+    }
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        const Mode modes[] = {
+            {"f16 denormal mode 0, IEEE mode on", ShaderRecompiler::ShaderFloatMode{0x30u, true, true, false}, 0u},
+            {"f16 denormal mode 0, IEEE mode off", ShaderRecompiler::ShaderFloatMode{0x30u, true, false, false}, 0u},
+            {"f16 denormal mode 1, IEEE mode on", ShaderRecompiler::ShaderFloatMode{0x70u, true, true, false}, 1u},
+            {"f16 denormal mode 1, IEEE mode off", ShaderRecompiler::ShaderFloatMode{0x70u, true, false, false}, 1u},
+            {"f16 denormal mode 2, IEEE mode on", ShaderRecompiler::ShaderFloatMode{0xb0u, true, true, false}, 2u},
+            {"f16 denormal mode 2, IEEE mode off", ShaderRecompiler::ShaderFloatMode{0xb0u, true, false, false}, 2u},
+            {"f16 denormal mode 3, IEEE mode on", ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}, 3u},
+            {"f16 denormal mode 3, IEEE mode off", ShaderRecompiler::ShaderFloatMode{0xf0u, true, false, false}, 3u},
+        };
+        for (const Mode& mode : modes) {
+            Run(*device, mode);
+            Check(mode);
+        }
+        std::puts("packed f16 denormal mode tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/shader/recompiler/Translation/src/FloatInstructions.cpp

@@ -11,25 +11,33 @@ void TranslateFloatInstruction(IrBuilder& builder, const RdnaInstruction& instru
 }
 
 bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opcode, bool accumulator, bool quietSnan) {
-    if (floatMode.has_value() && (floatMode->floatMode & 0xccu) != 0xc0u) {
-        throw std::runtime_error("packed f16 at pc " + std::to_string(inst.programCounter) + " in FLOAT_MODE " + std::to_string(floatMode->floatMode) + " (f16 denormals not kept or rounding not to nearest even) is not implemented");
+    if (floatMode.has_value() && (floatMode->floatMode & 0x0cu) != 0u) {
+        throw std::runtime_error("packed f16 at pc " + std::to_string(inst.programCounter) + " in FLOAT_MODE " + std::to_string(floatMode->floatMode) + " (f16 rounding not to nearest even) is not implemented");
     }
+    const bool flushResults = floatMode.has_value() && (floatMode->floatMode & 0x80u) == 0u;
+    const auto flushTinyResult = [&](IrF32 value) {
+        if (!flushResults) return value;
+        const IrU32 bits(ir.BitCastU32(value.Value()));
+        const IrU1 tiny(ir.ULessThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffffffu)), ir.Constant(opcode == IrOpcode::FPAdd32 ? 0x38800000u : 0x387ff000u)));
+        return IrF32(ir.BitCastF32(ir.Select(tiny.Value(), ir.BitwiseAnd(bits.Value(), ir.Constant(0x80000000u)), bits.Value())));
+    };
+    const auto readLane = [&](const RdnaOperand& operand, bool high) { return flushF16InputValue(readF16LaneAsF32(operand, high, true)); };
     const auto translateLane = [&](bool high) -> IrF32 {
-        const IrF32 lhs = readF16LaneAsF32(sourceAt(inst, 0u), high, true);
-        const IrF32 rhs = readF16LaneAsF32(sourceAt(inst, 1u), high, true);
+        const IrF32 lhs = readLane(sourceAt(inst, 0u), high);
+        const IrF32 rhs = readLane(sourceAt(inst, 1u), high);
         const auto ternary = [&](const IrF32& third) -> IrF32 {
-            if (opcode == IrOpcode::FPFma32) return fmaF16RoundedToOdd(lhs, rhs, third);
+            if (opcode == IrOpcode::FPFma32) return flushTinyResult(fmaF16RoundedToOdd(lhs, rhs, third));
             return IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value(), &third.Value()}));
         };
         if (accumulator) {
-            const IrF32 acc = readF16LaneAsF32(accumulatorOperand(inst), high, true);
+            const IrF32 acc = readLane(accumulatorOperand(inst), high);
             return applyF32ResultModifiers(inst.destination, clampF16Overflow(ternary(acc), {&lhs.Value(), &rhs.Value(), &acc.Value()}));
         }
         if (inst.sourceCount == 3u) {
-            const IrF32 third = readF16LaneAsF32(sourceAt(inst, 2u), high, true);
+            const IrF32 third = readLane(sourceAt(inst, 2u), high);
             return applyF32ResultModifiers(inst.destination, clampF16Overflow(ternary(third), {&lhs.Value(), &rhs.Value(), &third.Value()}));
         }
-        return applyF32ResultModifiers(inst.destination, clampF16Overflow(IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value()})), {&lhs.Value(), &rhs.Value()}));
+        return applyF32ResultModifiers(inst.destination, clampF16Overflow(flushTinyResult(IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value()}))), {&lhs.Value(), &rhs.Value()}));
     };
     const auto laneBits = [&](const RdnaOperand& operand, bool high) {
         const IrU32 source = readF16SourceBits(operand);
@@ -37,7 +45,7 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc
         IrU32 bits(ir.Emit(IrOpcode::BitFieldUExtract, IrType::U32, {&source.Value(), &ir.Constant(selectHigh ? 16u : 0u), &ir.Constant(16u)}));
         if (operand.absolute) bits = IrU32(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffu)));
         if (high ? operand.negateHi : operand.negate) bits = IrU32(ir.BitwiseXor(bits.Value(), ir.Constant(0x8000u)));
-        return bits;
+        return flushF16Input(bits);
     };
     std::array<std::array<IrU32, 2>, 3> sourceBits{};
     std::array<std::array<bool, 2>, 3> sourceBitsReady{};
@@ -51,7 +59,7 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc
         return sourceBits[index][lane];
     };
     const auto minMaxLane = [&](bool high) { return minMaxF16Bits(opcode == IrOpcode::FPMax32, sourceLaneBits(0u, high), sourceLaneBits(1u, high)); };
-    const bool native = nativeFloat16 && inst.destination.omod == 0u && (opcode == IrOpcode::FPAdd32 || opcode == IrOpcode::FPMul32);
+    const bool native = nativeFloat16 && !flushResults && inst.destination.omod == 0u && (opcode == IrOpcode::FPAdd32 || opcode == IrOpcode::FPMul32);
     const auto nativeResult = [&]() {
         const IrU32 lhs = packU16Lanes(sourceLaneBits(0u, false), sourceLaneBits(0u, true));
         const IrU32 rhs = packU16Lanes(sourceLaneBits(1u, false), sourceLaneBits(1u, true));

docs/dev/debt/shader.md

@@ -92,12 +92,12 @@
 - [v_div_fixup_f64](../../../core/shader/recompiler/Translation/src/Float64Instructions.cpp) flushes its src1 and src2 to a zero of the same sign when bit 0 of the `FLOAT_MODE` f16/f64 denormal field is clear (0 and 2) and keeps a denormal src0 and result in every mode, as `v_div_fixup_f32` does with the f32 field. Measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) over 1728 triples of 12 f64 values (zeros, denormals, the minimum normal, ±1, the largest finite value, infinity and quiet and signaling NaNs) at IEEE=0 and IEEE=1 with denorm16 0..3. Shaders without a known float mode keep the previous lowering. The PS5's gfx1013 is not measured.
 - [v_mullit_f32](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follows the hardware measured with the MODE register at IEEE=1 and denormals kept (a signaling NaN src0 is quieted only with IEEE=1, as measured at IEEE=0 too; a positive denormal src2 counts as positive); with f32 denormals flushed it follows the f32 denormals entry below, and denormal modes 1 and 2 throw.
 - [v_mad_legacy_f32 and v_mac_legacy_f32](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (VOP3 0x140, VOP2 0x06) round the product to f32 before the add, give a +0 product for a zero factor, and flush their f32 denormal sources, their product and their result to a zero of the same sign in every f32 denormal mode, an exception to the `FLOAT_MODE` model in [f32DenormalFlushFor](../../../core/shader/recompiler/Translation/src/VectorInstructions.cpp). Measured on gfx1010 (RX 5700 XT) with `tools/hw-oracle/template.s`, assembled by the AMD Windows driver's `amd_comgr_2.dll` and launched through its `amdhip64_6.dll` instead of `oracle.c` and ROCr, over the 32 rows of the five `FmaLegacy.cpp` and the four `MacLegacy.cpp` encodings: identical in wave32 and wave64 at IEEE 0 and 1 with f32 denormal modes 0 to 3, except that the two signaling NaN rows are quieted only with IEEE=1 (#1281, #1290). A sum of two zeros takes its sign on the bits (+0 unless both are -0), as lavapipe returned -0 for +0 + -0. The PS5's gfx1013 is not measured; LLVM decodes the encoding as `v_mad_legacy_f32` there, as on gfx1010. RDNA2 differs: on gfx1031 (RX 6700 XT, measured by pandather in #1281) the encoding is `v_fma_legacy_f32` and fused, with the same zero rule and denormal-source flush, so an RDNA2 measurement is not evidence for the console.
-- [v_fma_mixlo_f16 and v_fma_mixhi_f16](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) round the exact product-sum once to f16 with the f64 arithmetic of v_interp_p2_f16, so they need the Float64 capability. On RDNA2 (gfx1036) they follow the `FLOAT_MODE` f16 round and denormal fields, not the f32 round field: -1.5 * 2^-24 + 2^-126 from f32 sources (0xbfc00000, 0x33800000, 0x00800000) gives 0x8001 with f16 denormals kept and round to nearest even, 0x8000 with f16 denormals flushed and 0x8002 with f16 round toward -infinity. With f16 denormals kept, every f16 rounding mode is translated: the exact product-sum is rounded once in the mode's direction. All four modes match RDNA2 (gfx1036) on the 2,233 rows of the #2631 sweep, including `FLOAT_MODE` 0xcc (toward zero, used by ps5-opengl) and the exact cancellations toward -infinity; the PS5's gfx1013 is not measured. A result beyond 65504 becomes infinity or 65504 as IEEE 754 gives for the mode. Flushed f16 denormals still throw, and packed f16 still throws in every mode but round to nearest even with f16 denormals kept. `FP16_OVFL` set is not measured.
+- [v_fma_mixlo_f16 and v_fma_mixhi_f16](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) round the exact product-sum once to f16 with the f64 arithmetic of v_interp_p2_f16, so they need the Float64 capability. On RDNA2 (gfx1036) they follow the `FLOAT_MODE` f16 round and denormal fields, not the f32 round field: -1.5 * 2^-24 + 2^-126 from f32 sources (0xbfc00000, 0x33800000, 0x00800000) gives 0x8001 with f16 denormals kept and round to nearest even, 0x8000 with f16 denormals flushed and 0x8002 with f16 round toward -infinity. With f16 denormals kept, every f16 rounding mode is translated: the exact product-sum is rounded once in the mode's direction. All four modes match RDNA2 (gfx1036) on the 2,233 rows of the #2631 sweep, including `FLOAT_MODE` 0xcc (toward zero, used by ps5-opengl) and the exact cancellations toward -infinity; the PS5's gfx1013 is not measured. A result beyond 65504 becomes infinity or 65504 as IEEE 754 gives for the mode. Flushed f16 denormals still throw, and packed f16 throws with any f16 rounding but nearest even. `FP16_OVFL` set is not measured.
 - [Signaling NaN inputs](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) of the f32 and f16 unary instructions (`v_sqrt/rcp/rsq/log/exp/sin/cos/fract/floor/ceil/trunc/rndne/rcp_iflag/frexp_mant`), `v_ldexp_f32/f16`, `v_frexp_mant_f16`, `v_div_fixup_f32/f16` and `v_mullit_f32` are returned quieted only with IEEE=1 and unchanged with IEEE=0, measured on RDNA2 (gfx1035) in both modes, and so are those of the f64 forms (`v_sqrt/rcp/rsq/fract/floor/ceil/trunc/rndne/frexp_mant/ldexp/div_fixup_f64`) and `v_cvt_f64_f32`, measured on RDNA2 (gfx1036) in both modes; shaders without a known float mode follow IEEE=0. `v_pack_b32_f16` quiets each signaling NaN half by the same rule, measured on RDNA2 (gfx1036) in both modes. A signaling NaN result of `v_cubesc/cubetc/cubema_f32` is quieted by the same rule and keeps the sign the instruction gives it, measured on RDNA2 (gfx1036) in both modes. `v_add/sub/subrev/mul/fma/fmac/fmamk/fmaak/mul_legacy_f32` and `v_fma_mix_f32` with f32 sources choose their NaN result on the bits, measured over 1000 operand triples in both modes: the first NaN source in operand order after its modifiers (the subtrahend negated, as the hardware adds its negation), quieted only with IEEE=1, or the default NaN 0xffc00000 for an invalid operation; in the fma forms an invalid product (inf * 0) gives the default NaN even when src2 is a NaN, and in the legacy forms a zero factor makes the product 0 even when the other factor is a NaN (a NaN src2 still wins). `v_mad_legacy_f32` and `v_mac_legacy_f32` follow the same rule with their separately rounded product, as measured on gfx1010 (#1281, #1290): a NaN source quieted only with IEEE=1, inf - inf giving 0xffc00000, the zero rule hiding a NaN factor and a NaN src2 winning. The f16 and f64 arithmetic, `v_fma_mix_f32` with f16 sources and the separately rounded mad/mac forms (not measurable on RDNA2) follow the same rule on hardware but their NaN bits are left to the driver. `v_cvt_f16_f32`, `v_cvt_f32_f64`, `v_dot2_f32_f16`, `v_fma_mixlo_f16` and `v_fma_mixhi_f16` always quiet and `v_div_scale_f32` never does: the f32 signaling NaN 0x7f812345 as src0 or src2 gives 0x7e09 from mixlo and mixhi at IEEE=0 and IEEE=1 on RDNA2 (gfx1036), where `v_fma_mix_f32` gives 0x7f812345 at IEEE=0 and 0x7fc12345 at IEEE=1. The payload of an f16 NaN source of mixlo and mixhi is left to the driver's f16 to f32 conversion, which NVIDIA replaces.
 - [Scalar f16 add, sub, subrev and mul denormals](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follow the `FLOAT_MODE` f16 denormal field at nearest-even: bit 0 preserves inputs and bit 1 preserves the result; otherwise a denormal is flushed to its signed zero. Measured on RDNA2 (gfx1036) with IEEE=0/1 and FP16_OVFL=0/1, over 32 operand pairs in all four denormal modes, including signed zeros, denormals, minimum normals, infinities and NaNs. Multiply detects tininess after rounding to 11 significant bits with an unbounded exponent, before gradual half packing: `0x3bff * 0x0400` flushes, while `0x3c01 * 0x03ff` rounds to `0x0400` and is kept. `v_fma_f16`, `v_fmac_f16`, `v_fmamk_f16` and `v_fmaak_f16` follow the same field at nearest-even: inputs are flushed when bit 0 is clear, and the result is flushed when bit 1 is clear if the exact product-sum rounded to 11 significant bits with an unbounded exponent is below 2^-14, so `0x0400 * 0x0400 + 0x8400` rounds to `0x8400` and is kept. Measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) at IEEE=0 and IEEE=1 with denorm16 0..3 over 5075 operand triples of `v_fma_f16` and `v_fmac_f16` (4000 from 20 halves around the denormal range, 75 whose product-sum lies just below 2^-14 and 1000 random), and over the first 4075 of them for `v_fmamk_f16` and `v_fmaak_f16`, FP16_OVFL=1, the clamp, neg and abs modifiers and op_sel. Directed rounding and `v_fma_mixlo/mixhi_f16` keep their previous denormal lowering. Shaders without a known float mode keep the previous lowering. Scalar `v_sub_f16`/`v_subrev_f16` share `float16Binary` with add/mul and were measured on gfx1036 over the same operand set as add (1156 non-NaN-input rows per mode); NaN result bits still depend on the driver as recorded above.
 - [f16 unary, frexp and conversion denormals](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follow the `FLOAT_MODE` f16 denormal field: `v_ceil_f16`, `v_floor_f16`, `v_fract_f16`, `v_rcp_f16`, `v_rsq_f16`, `v_sqrt_f16`, `v_log_f16`, `v_exp_f16`, `v_sin_f16`, `v_frexp_mant_f16`, `v_frexp_exp_i16_f16`, `v_cvt_norm_i16_f16`, `v_cvt_norm_u16_f16` and `v_cvt_f32_f16` flush a denormal half input to its signed zero when bit 0 is clear (0 and 2), and the f16-result ones flush a denormal half result to its signed zero when bit 1 is clear (0 and 1), after the host rounds it to half. Measured on RDNA2 (gfx1036) at IEEE=0 and IEEE=1 with denorm16 0..3 over every non-NaN f16 input (63,490 rows per mode and instruction): the only mismatches are the 1-ULP driver differences of `v_exp_f16` (2 inputs) and `v_sin_f16` (14), which are the same with denormals kept. NaN inputs are not compared. `v_ldexp_f16` shares the input rule. `v_cvt_pknorm_i16_f16` and `v_cvt_pknorm_u16_f16` flush both half inputs by the same bit, measured on RDNA2 (gfx1030) over every f16 input (each half also negated in the other source) at IEEE=0 and IEEE=1 with denorm16 0..3 and at f16 rounding 1..3: every row matches. Shaders without a known float mode keep the previous lowering.
 - [v_div_fixup_f16](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) flushes its src1 and src2 to a zero of the same sign when bit 0 of the `FLOAT_MODE` f16 denormal field is clear (0 and 2) and keeps a denormal src0 and result in every mode, as `v_div_fixup_f32` does with the f32 field. Measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) over 6859 triples of 19 halves (zeros, denormals, minimum normals, ±0.5, ±1, the largest finite value, infinities and quiet and signaling NaNs) at IEEE=0 and IEEE=1 with denorm16 0..3 and FP16_OVFL 0 and 1, also with neg and abs source modifiers and with op_sel high halves. Shaders without a known float mode keep the previous lowering. The PS5's gfx1013 is not measured.
-- [Packed f16 NaN results](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (`v_pk_add/mul/fma/min/max_f16`) follow the hardware measured on RDNA2 (gfx1035) at IEEE=0 and IEEE=1 with f16 round-to-nearest-even and f16 denormals kept: with IEEE=1 a signaling NaN input is quieted and min/max return the other operand only for a quiet NaN; with IEEE=0 NaN inputs pass through unquieted and min/max ignore signaling. In `v_pk_fma_f16` an invalid product (inf * 0) gives the default NaN even when src2 is a NaN. A shader without a known float mode is translated as IEEE=0, the mode titles use; a mode with another f16 rounding or that flushes f16 denormals throws.
+- [Packed f16 NaN results](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (`v_pk_add/mul/fma/min/max_f16`) follow the hardware measured on RDNA2 (gfx1035) at IEEE=0 and IEEE=1 with f16 round-to-nearest-even and f16 denormals kept: with IEEE=1 a signaling NaN input is quieted and min/max return the other operand only for a quiet NaN; with IEEE=0 NaN inputs pass through unquieted and min/max ignore signaling. In `v_pk_fma_f16` an invalid product (inf * 0) gives the default NaN even when src2 is a NaN. A shader without a known float mode is translated as IEEE=0, the mode titles use; a mode with another f16 rounding throws. In the f16 denormal modes that flush, each denormal input lane is flushed to a zero of its sign when bit 0 of the field is clear (0 and 2), so `inf * 0x0001` is an invalid product there, and when bit 1 is clear (0 and 1) a `v_pk_add_f16` result below 2^-14, or a `v_pk_mul_f16`, `v_pk_fma_f16` or `v_pk_fmac_f16` result whose exact value rounded to 11 significant bits with an unbounded exponent is below 2^-14, is flushed to a zero of its sign; `v_pk_min/max_f16` flush only their inputs. The native f16x2 add and multiply is not used while results are flushed. Measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) at IEEE=0 and IEEE=1 with denorm16 0..3, over 19,192 to 162,204 lanes per instruction with op_sel, op_sel_hi, neg_lo, neg_hi and clamp variants and products and sums at the 2^-14 boundary.
 - [f32 denormals](../../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follow the shader's `FLOAT_MODE` f32 denormal field (bits 5:4) when it flushes them (0, and shaders without a known float mode, as in the 0xc0 most titles use), measured on RDNA2 (gfx1035) over 1444 rows per instruction at IEEE=0 and IEEE=1. `v_add/sub/subrev/mul/mul_legacy/mullit/fma/fmac/fmamk/fmaak/min/max/min3/max3/med3/fract/floor/ceil/frexp_mant/sin/cos_f32` flush their f32 inputs and their result to a zero of the same sign (`v_mullit_f32` measured on RDNA2 (gfx1036) over 6907 rows per mode at IEEE=0 and IEEE=1, so a positive denormal src2 counts as zero); the f32 compares, `v_cvt_i32/u32/rpi_i32/flr_i32_f32`, `v_frexp_exp_i32_f32`, `v_cvt_f64_f32`, `v_cvt_f16_f32`, `v_cvt_pkrtz_f16_f32`, `v_cvt_pknorm_i16/u16_f32` and `v_cvt_pk_u8_f32` flush their f32 inputs. `v_div_scale_f32` flushes all three inputs to signed zeros, measured on RDNA2 (gfx1036) over 2744 operand triples per mode at IEEE=0 and IEEE=1 with f32 denormals flushed and kept; its scale-down behaviour and host denormal preservation remain incomplete. `v_div_fixup_f32` flushes its src1 and src2 and keeps a denormal src0 and result, measured on RDNA2 (gfx1036) over 6859 rows per mode at IEEE=0 and IEEE=1. Tininess is detected after rounding, measured on RDNA2 (gfx1036) for `v_mul_f32` and `v_mullit_f32` at IEEE=0 and IEEE=1: a product is flushed when its value rounded to 24 bits with an unbounded exponent is below 2^-126, so `0x3f7ffffe * 0x00800001` (2^-126 * (1 - 2^-46)) gives `0x00800000` and `0x3f7fffff * 0x00800000` (2^-126 * (1 - 2^-24)) gives zero. The check rounds in the f32 round mode of `FLOAT_MODE` (bits 1:0), measured in all four modes, while the products themselves are still rounded to nearest even. This applies to `v_mul_f32`, `v_mul_legacy_f32`, `v_mullit_f32`, and to `v_fma_f32`, `v_fmac_f32`, `v_fmamk_f32`, `v_fmaak_f32` and `v_fma_mix_f32` when the addend is zero; with a nonzero addend the fused result is kept, so an exact result just below 2^-126 that rounds to it is not flushed. In the three directed modes the check is computed exactly from the significands. To nearest even (0, the mode of shaders without a known float mode) one factor is multiplied by 2^64, which is exact below 2^64, and the product with the other factor is compared with 2^-62. That multiply is a plain `OpFMul` like the product, and no f32 rounding mode is requested in the SPIR-V, so the check relies on the driver rounding f32 to nearest even by default, as the products do, and a product that rounds up to 2^-126 is the emitted product. `v_fma_mix_f32` follows this rule with mixed f16/f32 sources too, measured on RDNA2 (gfx1036) with f16 in either multiplicand slot, low and high half selection, and an f32 or f16 zero addend: f32 `0x00801002` times f16 `0x3bff` gives `0x00800000` to nearest at IEEE=0 and IEEE=1, while the neighbours `0x00801001` and `0x00801003` give zero and `0x00800001`. The tiny-product boundary was also measured for both signs in all four f32 round modes at IEEE=0; Its f16 sources are flushed to a zero of their sign when bit 0 of the f16/f64 denormal field is clear (0 and 2), whatever the f32 field, measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) at IEEE=0 and IEEE=1 with f32 denormal modes 0 and 3 and f16 denormal modes 0 to 3, over 2744 triples of 14 halves with all three sources f16 and 1960 rows with an f32 multiplicand. `v_fma_mixlo_f16` and `v_fma_mixhi_f16` flush their f32 sources too and keep a product below 2^-126, measured on RDNA2 (gfx1036). `v_mad_legacy_f32` and `v_mac_legacy_f32` flush their inputs, their product and their result in every mode, see its entry; their product is still checked for tininess on its exact value, so one below 2^-126 gives zero even when it rounds to 2^-126, as the console's separately rounded encoding is not measured at this boundary. Denormal modes 1 and 2 throw for the other instructions. Not covered, and not measurable on RDNA2 (no such opcodes) or not measured: `v_mac_f32`, `v_mad_f32`, `v_madak_f32` and `v_madmk_f32` (rounded separately), `v_dot2_f32_f16` and `v_div_fmas_f32` (on hardware `v_div_fmas_f32` flushes only its result), f32 interpolation and `ds_add_f32` keep the previous lowering; `ds_min/max/cmpst_f32` flush both operands by the field, see the LDS float atomics entry. With f32 denormals kept (3) the results depend on the driver's default denormal behaviour, as no `DenormPreserve` execution mode is requested; `v_ldexp_f32` and the f32 compares are the exceptions. `v_cmp_X_f32` and `v_cmpx_X_f32` compare the operands as sign-magnitude integers with the NaN cases handled apart, so a denormal is not read as zero, measured on RDNA2 (gfx1036) over 256 operand pairs per opcode at IEEE=0 and IEEE=1 with f32 denormals kept and flushed. `v_ldexp_f32` preserves subnormal inputs and rounds a subnormal result in the f32 round mode of `FLOAT_MOLine truncated
 - [v_rcp, v_rsq, v_sqrt, v_log, v_exp, v_sin and v_cos](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (f32 and f16) use the driver's approximation for finite results, not the console's: the low bits can differ. Only NaN, invalid, zero and infinite results and the exact sin/cos zeros are chosen on the bits. Measured on RDNA2 (gfx1036) over every f16 input and 32,832 f32 inputs: RADV returns the hardware's bits for every input of rcp, rsq, sqrt and log and all but one of exp (f16 0x11c5, 1 ULP); NVIDIA (615.71.09) stays within 2 ULP except `v_log_f32` near 1.0 (next entry). [v_sin and v_cos](../../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) take the driver's sine or cosine of the radians of the exact cycle, except within 2^-12 of a turn from a zero of the function, where the result is the linear term of the exact distance to the zero: the hardware's constant 6.2831826 (0x40c90fd5) at the integers, which the hardware applies exactly to inputs below 2^-12 of a turn, and the f32 2π at the half and quarter turns, where the hardware is within 7 ULP of it. Outside those bands, a driver result more than 4 ULP from a Taylor series in turns of the exact cycle is replaced by the series, which is within 2 ULP of the true sine and cosine; the hardware is up to 9 ULP from them (gfx1035, 243,092 f32 inputs), so results stay within 13 ULP of the hardware on any driver. RADV returns the hardware's bits for every f16 input and for 32,422 sine and 32,486 cosine results of the 32,832 f32 inputs; the rest are 1 ULP off, except up to 14 ULP on either side of non-zero integers and up to 15 ULP beyond ±256 cycles, where the hardware's own reduction differs from the exact fraction, and up to 6 ULP inside the half- and quarter-turn bands. NVIDIA's own sine is within 1 ULP on 93% of the f32 inputs and up to 1,166 ULP off just outside the bands, its cosine within 163 ULP; the series replaces those results. A denormal f32 input with f32 denormals kept gives the hardware the input times 6.2831826; both hosts flush it.
 - [v_ldexp_f32 in denormal modes 1 and 2](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follows the `FLOAT_MODE` f32 denormal field: mode 1 keeps a subnormal input and flushes a result below 2^-126 to a zero of its sign in every f32 round mode, also one that rounds to 2^-126 at subnormal precision (`0x3fffffff` scaled by 2^-127); mode 2 flushes a subnormal input to a zero of its sign and rounds a subnormal result in the f32 round mode of `FLOAT_MODE`, as mode 3 does. Measured on RDNA2 (gfx1030, Radeon RX 6800, AMD Windows driver 32.0.21045.5002 through HIP) over 10,260 rows (54 inputs of both signs: zeros, subnormals, normals near 2^-126 and 1.0, the largest finite value, infinities and NaNs, each scaled by 190 exponents from -2^31 to 2^31-1) at IEEE=0 and IEEE=1 with each f32 denormal mode 0 to 3 and f32 round mode 0 to 3, DX10_CLAMP=1, f16 denormals kept, f16 round to nearest even and FP16_OVFL=0; with the `clamp` modifier the result is clamped as in the other modes. The PS5's gfx1013 is not measured.