Merge pull request #464 from oneandonlydean/fix/relinker-exact-reciprocals

fix(relinker): lower VRSQRTPS and VRCPPS to correctly rounded results for --to-intel

core/relinker/CMakeLists.txt

@@ -44,6 +44,8 @@ add_executable(relinker
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sha1Lowering.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroOperands.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalOperands.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/CallSiteResolver.cpp
@@ -110,6 +112,8 @@ if(WIN32)
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sha1Lowering.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroOperands.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalOperands.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp
     )
     target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows)
     if(BUILD_TESTING)
@@ -158,6 +162,8 @@ if(BUILD_TESTING)
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sha1Lowering.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroOperands.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalOperands.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp

core/relinker/codegen/include/codegen/CodegenTypes.hpp

@@ -43,6 +43,7 @@ struct ConvertResult {
     std::size_t ReplacedCount;
     std::vector<TrampolineSite> Trampolines;
     std::vector<Amd64OnlySubstitutionReport> Reports;
+    std::size_t KeptCount = 0;
 };
 
 enum class ControlFlowKind : std::uint8_t {

core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp

@@ -44,6 +44,8 @@ inline constexpr Entry kExtrqRegisterForm = {"EXTRQ register form", nullptr, 0};
 inline constexpr Entry kInsertqRegisterForm = {"INSERTQ register form", nullptr, 0};
 inline constexpr Entry kMovntss = {"MOVNTSS", nullptr, 0};
 inline constexpr Entry kMovntsd = {"MOVNTSD", nullptr, 0};
+inline constexpr Entry kVrcpps = {"VRCPPS", nullptr, 0};
+inline constexpr Entry kVrsqrtps = {"VRSQRTPS", nullptr, 0};
 inline constexpr Entry kSha256rnds2 = {"SHA256RNDS2", nullptr, 0};
 inline constexpr Entry kSha256msg1 = {"SHA256MSG1", nullptr, 0};
 inline constexpr Entry kSha256msg2 = {"SHA256MSG2", nullptr, 0};

core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp

@@ -11,7 +11,8 @@ namespace Codegen {
 enum class Amd64OnlyLowering : std::uint8_t {
     InPlace,
     Trampoline,
-    Unsupported
+    Unsupported,
+    Kept
 };
 
 struct Amd64OnlyMatch {
@@ -21,6 +22,7 @@ struct Amd64OnlyMatch {
     std::vector<std::uint8_t> ReplacementBytes;
     std::vector<std::uint8_t> StubBody;
     std::size_t ReturnBranchOffset;
+    bool Optional = false;
 };
 
 struct Amd64OnlySubstitutionReport {

core/relinker/codegen/include/codegen/x86/ReciprocalLowering.hpp

@@ -0,0 +1,18 @@
+#ifndef CODEGEN_X86_RECIPROCALLOWERING_HPP
+#define CODEGEN_X86_RECIPROCALLOWERING_HPP
+
+#include <codegen/x86/ReciprocalOperands.hpp>
+#include <codegen/x86/StubBodyBuilder.hpp>
+#include <span>
+
+namespace Codegen {
+
+class ReciprocalLowering {
+public:
+    void EmitOutOfLine(StubBodyBuilder& body, const ReciprocalOperands& operands) const;
+    [[nodiscard]] LoweredBody LowerOutOfLine(const ReciprocalOperands& operands, std::span<const std::uint8_t> trailing = {}) const;
+};
+
+}
+
+#endif

core/relinker/codegen/include/codegen/x86/ReciprocalOperands.hpp

@@ -0,0 +1,25 @@
+#ifndef CODEGEN_X86_RECIPROCALOPERANDS_HPP
+#define CODEGEN_X86_RECIPROCALOPERANDS_HPP
+
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+
+namespace Codegen {
+
+enum class ReciprocalOperation : std::uint8_t {
+    Reciprocal,
+    ReciprocalSquareRoot
+};
+
+struct ReciprocalOperands {
+    ReciprocalOperation Operation;
+    std::uint8_t Destination;
+    std::uint8_t Source;
+};
+
+[[nodiscard]] std::optional<ReciprocalOperands> DecodeVexReciprocal(const std::uint8_t* data, std::size_t length);
+
+}
+
+#endif

core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp

@@ -23,6 +23,7 @@ void EmitShiftImm(std::vector<std::uint8_t>& out, std::uint8_t extension, std::u
 class StubBodyBuilder {
 public:
     void Sse(std::uint8_t prefix, std::initializer_list<std::uint8_t> opcode, std::uint8_t dst, std::uint8_t src);
+    void SsePlain(std::initializer_list<std::uint8_t> opcode, std::uint8_t dst, std::uint8_t src);
     void SseImm(std::uint8_t prefix, std::initializer_list<std::uint8_t> opcode, std::uint8_t dst, std::uint8_t src, std::uint8_t imm);
     void ShiftImm(std::uint8_t extension, std::uint8_t reg, std::uint8_t imm);
     void ShiftDwordImm(std::uint8_t extension, std::uint8_t reg, std::uint8_t imm);

core/relinker/codegen/src/Amd64OnlyConverter.cpp

@@ -126,50 +126,61 @@ void Amd64OnlyConverter::_convertSegment(
             break;
         }
         case Amd64OnlyLowering::Trampoline: {
-            std::size_t siteLength = match.Length;
-            auto stub = substitution;
-            if (siteLength < Amd64OnlySubstitutionTable::kJmpRel32.Size) {
-                const X64InstructionDecoder decoder;
-                std::vector<std::span<const std::uint8_t>> sequence{std::span<const std::uint8_t>(seg.data() + match.Offset, match.Length)};
-                std::size_t trailingBytes = 0;
-                for (auto next = item.Index + 1; siteLength < Amd64OnlySubstitutionTable::kJmpRel32.Size; ++next) {
-                    if (next >= matches.size() || matches[next].Offset != match.Offset + siteLength)
-                        throw CodegenException("AMD-only instruction too short for a jump and not followed by an instruction", fileOffset);
-                    const auto& following = matches[next];
-                    const std::span<const std::uint8_t> bytes(seg.data() + following.Offset, following.Length);
-                    const auto info = decoder.DecodeInstruction(bytes.data(), bytes.size());
-                    const bool amdOnly = _matcher->Match(bytes.data(), bytes.size()).has_value();
-                    if (amdOnly && trailingBytes == 0) {
-                        sequence.push_back(bytes);
-                        consumed.insert(next);
-                    } else if (amdOnly || info.FlowKind != ControlFlowKind::Sequential || info.HasRipRelativeDisp || info.HasBranchTarget) {
-                        throw CodegenException("AMD-only instruction too short for a jump is followed by an instruction that cannot move", ph.Offset + following.Offset);
-                    } else {
-                        trailingBytes += following.Length;
+            std::vector<std::size_t> taken;
+            try {
+                std::size_t siteLength = match.Length;
+                auto stub = substitution;
+                if (siteLength < Amd64OnlySubstitutionTable::kJmpRel32.Size) {
+                    const X64InstructionDecoder decoder;
+                    std::vector<std::span<const std::uint8_t>> sequence{std::span<const std::uint8_t>(seg.data() + match.Offset, match.Length)};
+                    std::size_t trailingBytes = 0;
+                    for (auto next = item.Index + 1; siteLength < Amd64OnlySubstitutionTable::kJmpRel32.Size; ++next) {
+                        if (next >= matches.size() || matches[next].Offset != match.Offset + siteLength)
+                            throw CodegenException("AMD-only instruction too short for a jump and not followed by an instruction", fileOffset);
+                        const auto& following = matches[next];
+                        const std::span<const std::uint8_t> bytes(seg.data() + following.Offset, following.Length);
+                        const auto info = decoder.DecodeInstruction(bytes.data(), bytes.size());
+                        const bool amdOnly = _matcher->Match(bytes.data(), bytes.size()).has_value();
+                        if (amdOnly && trailingBytes == 0) {
+                            sequence.push_back(bytes);
+                            taken.push_back(next);
+                        } else if (amdOnly || info.FlowKind != ControlFlowKind::Sequential || info.HasRipRelativeDisp || info.HasBranchTarget) {
+                            throw CodegenException("AMD-only instruction too short for a jump is followed by an instruction that cannot move", ph.Offset + following.Offset);
+                        } else {
+                            trailingBytes += following.Length;
+                        }
+                        siteLength += following.Length;
                     }
-                    siteLength += following.Length;
+                    const std::span<const std::uint8_t> trailing(seg.data() + match.Offset + siteLength - trailingBytes, trailingBytes);
+                    auto relocated = _atFileOffset(fileOffset, [&] { return _matcher->MatchSequence(sequence, trailing); });
+                    if (!relocated.has_value() || relocated->Lowering != Amd64OnlyLowering::Trampoline)
+                        throw CodegenException("AMD-only instruction sequence has no out-of-line lowering", fileOffset);
+                    stub = std::move(*relocated);
                 }
-                const std::span<const std::uint8_t> trailing(seg.data() + match.Offset + siteLength - trailingBytes, trailingBytes);
-                auto relocated = _atFileOffset(fileOffset, [&] { return _matcher->MatchSequence(sequence, trailing); });
-                if (!relocated.has_value() || relocated->Lowering != Amd64OnlyLowering::Trampoline)
-                    throw CodegenException("AMD-only instruction sequence has no out-of-line lowering", fileOffset);
-                stub = std::move(*relocated);
+                const auto hit = branchTargets.upper_bound(address);
+                if (hit != branchTargets.end() && *hit < address + siteLength)
+                    throw CodegenException("Branch enters an AMD-only instruction", ph.Offset + (*hit - ph.MappedAddress));
+                const auto begin = seg.begin() + static_cast<std::ptrdiff_t>(match.Offset);
+                result.Trampolines.push_back({
+                    fileOffset,
+                    address,
+                    siteLength,
+                    std::vector<std::uint8_t>(begin, begin + static_cast<std::ptrdiff_t>(siteLength)),
+                    stub.StubBody,
+                    stub.ReturnBranchOffset
+                });
+                replacementLength = stub.StubBody.size();
+                consumed.insert(taken.begin(), taken.end());
+            } catch (const CodegenException&) {
+                if (!substitution.Optional)
+                    throw;
+                ++result.KeptCount;
+                result.Reports.push_back({substitution.InstructionName, fileOffset, match.Length, 0, Amd64OnlyLowering::Kept});
+                continue;
             }
-            const auto hit = branchTargets.upper_bound(address);
-            if (hit != branchTargets.end() && *hit < address + siteLength)
-                throw CodegenException("Branch enters an AMD-only instruction", ph.Offset + (*hit - ph.MappedAddress));
-            const auto begin = seg.begin() + static_cast<std::ptrdiff_t>(match.Offset);
-            result.Trampolines.push_back({
-                fileOffset,
-                address,
-                siteLength,
-                std::vector<std::uint8_t>(begin, begin + static_cast<std::ptrdiff_t>(siteLength)),
-                stub.StubBody,
-                stub.ReturnBranchOffset
-            });
-            replacementLength = stub.StubBody.size();
             break;
         }
+        case Amd64OnlyLowering::Kept:
         case Amd64OnlyLowering::Unsupported:
             throw CodegenException("AMD-only instruction without Intel lowering: " + substitution.InstructionName, fileOffset);
         }
@@ -186,7 +197,7 @@ ConvertResult Amd64OnlyConverter::Convert(
     std::vector<std::uint8_t> fileBytes,
     const std::vector<Domain::ProgramHeader>& codeSegments
 ) const {
-    ConvertResult result{{}, 0, {}, {}};
+    ConvertResult result{};
     for (const auto& ph : codeSegments)
         _convertSegment(fileBytes, ph, result);
     result.Bytes = std::move(fileBytes);

core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp

@@ -5,6 +5,8 @@
 #include <codegen/x86/Sse4aOperands.hpp>
 #include <codegen/x86/ClzeroLowering.hpp>
 #include <codegen/x86/ClzeroOperands.hpp>
+#include <codegen/x86/ReciprocalLowering.hpp>
+#include <codegen/x86/ReciprocalOperands.hpp>
 #include <codegen/x86/Sha256Lowering.hpp>
 #include <codegen/x86/Sha256Operands.hpp>
 #include <codegen/x86/Sha1Lowering.hpp>
@@ -35,6 +37,10 @@ const Entry& _sse4aEntry(const Sse4aOperands& operands) {
     return operands.Insertq ? kInsertq : kExtrq;
 }
 
+const Entry& _reciprocalEntry(const ReciprocalOperands& operands) {
+    return operands.Operation == ReciprocalOperation::ReciprocalSquareRoot ? kVrsqrtps : kVrcpps;
+}
+
 const Entry& _sha256Entry(const Sha256Operands& operands) {
     switch (operands.Operation) {
     case Sha256Operation::Rnds2:
@@ -97,6 +103,7 @@ private:
     Sha256Lowering _sha256Lowering;
     Sha1Lowering _sha1Lowering;
     ClzeroLowering _clzeroLowering;
+    ReciprocalLowering _reciprocalLowering;
 
     [[nodiscard]] Amd64OnlyMatch _matchMovnts(const DecodedInstruction& instr, const Entry& entry) const;
     [[nodiscard]] Amd64OnlyMatch _matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span<const std::uint8_t> trailing) const;
@@ -176,13 +183,18 @@ std::optional<Amd64OnlyMatch> Amd64OnlyInstructionMatcher::MatchSequence(
             if (name == nullptr)
                 name = kClzero.Name;
             _clzeroLowering.EmitOutOfLine(body, operands);
+        } else if (const auto reciprocal = DecodeVexReciprocal(instr.Data, instr.Length)) {
+            if (name == nullptr)
+                name = _reciprocalEntry(*reciprocal).Name;
+            _reciprocalLowering.EmitOutOfLine(body, *reciprocal);
         } else {
             return std::nullopt;
         }
     }
     body.Raw(trailing);
     auto lowered = body.Finish();
-    return Amd64OnlyMatch{name, instructions.front().size(), Amd64OnlyLowering::Trampoline, {}, std::move(lowered.Bytes), lowered.ReturnBranchOffset};
+    const bool optional = DecodeVexReciprocal(instructions.front().data(), instructions.front().size()).has_value();
+    return Amd64OnlyMatch{name, instructions.front().size(), Amd64OnlyLowering::Trampoline, {}, std::move(lowered.Bytes), lowered.ReturnBranchOffset, optional};
 }
 
 std::optional<Amd64OnlyMatch> Amd64OnlyInstructionMatcher::Match(
@@ -216,6 +228,11 @@ std::optional<Amd64OnlyMatch> Amd64OnlyInstructionMatcher::Match(
     if (instr.IsMwaitx())
         return _validWait(instr) ? _inPlace(kMwaitx, length, std::vector<std::uint8_t>(kPause.Bytes, kPause.Bytes + kPause.Size)) : _unsupported(kMwaitx, length);
 
+    if (const auto reciprocal = DecodeVexReciprocal(data, length)) {
+        auto body = _reciprocalLowering.LowerOutOfLine(*reciprocal, trailing);
+        return Amd64OnlyMatch{_reciprocalEntry(*reciprocal).Name, length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset, true};
+    }
+
     if (instr.IsClzero())
         return _matchClzero(instr, trailing);
 

core/relinker/codegen/src/x86/ReciprocalLowering.cpp

@@ -0,0 +1,65 @@
+#include <codegen/x86/ReciprocalLowering.hpp>
+#include <array>
+#include <cstring>
+
+namespace Codegen {
+
+namespace {
+
+constexpr std::uint8_t kSqrtps = 0x51;
+constexpr std::uint8_t kDivps = 0x5E;
+constexpr std::uint8_t kMovaps = 0x28;
+constexpr float kOne = 1.0f;
+
+StubConstant _ones() {
+    StubConstant constant{};
+    for (std::size_t lane = 0; lane < 4; ++lane)
+        std::memcpy(constant.data() + lane * sizeof(kOne), &kOne, sizeof(kOne));
+    return constant;
+}
+
+std::uint8_t _scratch(const ReciprocalOperands& operands) {
+    std::uint8_t reg = 0;
+    while (reg == operands.Destination || reg == operands.Source)
+        ++reg;
+    return reg;
+}
+
+void _vexMovaps(StubBodyBuilder& body, const std::uint8_t dst, const std::uint8_t src) {
+    const std::array<std::uint8_t, 5> bytes{
+        0xC4,
+        static_cast<std::uint8_t>(((dst & 8) != 0 ? 0x00 : 0x80) | 0x40 | ((src & 8) != 0 ? 0x00 : 0x20) | 0x01),
+        0x78,
+        kMovaps,
+        static_cast<std::uint8_t>(0xC0 | ((dst & 7) << 3) | (src & 7))
+    };
+    body.Raw(bytes);
+}
+
+}
+
+void ReciprocalLowering::EmitOutOfLine(StubBodyBuilder& body, const ReciprocalOperands& operands) const {
+    const auto scratch = _scratch(operands);
+    const auto dst = operands.Destination;
+    body.Spill(scratch);
+    if (operands.Operation == ReciprocalOperation::ReciprocalSquareRoot) {
+        body.SsePlain({0x0F, kSqrtps}, scratch, operands.Source);
+        body.RipOperand({0x0F, kMovaps}, dst, _ones());
+        body.SsePlain({0x0F, kDivps}, dst, scratch);
+        _vexMovaps(body, dst, dst);
+    } else {
+        body.RipOperand({0x0F, kMovaps}, scratch, _ones());
+        body.SsePlain({0x0F, kDivps}, scratch, operands.Source);
+        _vexMovaps(body, dst, scratch);
+    }
+    body.Restore(scratch);
+}
+
+LoweredBody ReciprocalLowering::LowerOutOfLine(const ReciprocalOperands& operands, std::span<const std::uint8_t> trailing) const {
+    StubBodyBuilder body;
+    EmitOutOfLine(body, operands);
+    body.Raw(trailing);
+    return body.Finish();
+}
+
+}

core/relinker/codegen/src/x86/ReciprocalOperands.cpp

@@ -0,0 +1,55 @@
+#include <codegen/x86/ReciprocalOperands.hpp>
+#include <codegen/x86/X64OpcodeConstants.hpp>
+
+namespace Codegen {
+
+namespace {
+
+constexpr std::uint8_t kVexMap0F = 0x01;
+constexpr std::uint8_t kVexUnusedRegister = 0x0F;
+constexpr std::uint8_t kOpcodeRsqrtps = 0x52;
+constexpr std::uint8_t kOpcodeRcpps = 0x53;
+
+}
+
+std::optional<ReciprocalOperands> DecodeVexReciprocal(const std::uint8_t* data, const std::size_t length) {
+    using namespace X64OpcodeConstants;
+    if (length < 4)
+        return std::nullopt;
+    bool registerExtension = false;
+    bool rmExtension = false;
+    std::uint8_t payload = 0;
+    std::size_t opcodeOffset = 0;
+    if (data[0] == OneByteVex2 && length == 4) {
+        registerExtension = (data[1] & 0x80) == 0;
+        payload = data[1];
+        opcodeOffset = 2;
+    } else if (data[0] == OneByteVex3 && length == 5) {
+        if ((data[1] & Vex3MapMask) != kVexMap0F)
+            return std::nullopt;
+        registerExtension = (data[1] & 0x80) == 0;
+        rmExtension = (data[1] & 0x20) == 0;
+        payload = data[2];
+        opcodeOffset = 3;
+    } else {
+        return std::nullopt;
+    }
+    const auto vvvv = static_cast<std::uint8_t>((payload >> 3) & 0x0F);
+    const bool wide = (payload & 0x04) != 0;
+    const auto pp = static_cast<std::uint8_t>(payload & 0x03);
+    if (vvvv != kVexUnusedRegister || wide || pp != 0)
+        return std::nullopt;
+    const auto opcode = data[opcodeOffset];
+    if (opcode != kOpcodeRsqrtps && opcode != kOpcodeRcpps)
+        return std::nullopt;
+    const auto modrm = data[opcodeOffset + 1];
+    if (((modrm >> ModRmModShift) & ModRmModMask) != ModRmModRegister)
+        return std::nullopt;
+    ReciprocalOperands operands{};
+    operands.Operation = opcode == kOpcodeRsqrtps ? ReciprocalOperation::ReciprocalSquareRoot : ReciprocalOperation::Reciprocal;
+    operands.Destination = static_cast<std::uint8_t>(((modrm >> ModRmRegShift) & ModRmRegMask) | (registerExtension ? 8 : 0));
+    operands.Source = static_cast<std::uint8_t>((modrm & ModRmRmMask) | (rmExtension ? 8 : 0));
+    return operands;
+}
+
+}

core/relinker/codegen/src/x86/StubBodyBuilder.cpp

@@ -51,6 +51,14 @@ void StubBodyBuilder::Sse(const std::uint8_t prefix, const std::initializer_list
     EmitSse(_bytes, prefix, opcode, dst, src);
 }
 
+void StubBodyBuilder::SsePlain(const std::initializer_list<std::uint8_t> opcode, const std::uint8_t dst, const std::uint8_t src) {
+    const auto rex = _rex(dst, src);
+    if (rex != kRexBase)
+        _bytes.push_back(rex);
+    _bytes.insert(_bytes.end(), opcode.begin(), opcode.end());
+    _bytes.push_back(static_cast<std::uint8_t>(kModRmRegister | ((dst & 7) << 3) | (src & 7)));
+}
+
 void StubBodyBuilder::SseImm(const std::uint8_t prefix, const std::initializer_list<std::uint8_t> opcode, const std::uint8_t dst, const std::uint8_t src, const std::uint8_t imm) {
     EmitSse(_bytes, prefix, opcode, dst, src);
     _bytes.push_back(imm);

core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp

@@ -7,6 +7,7 @@
 #include <codegen/x86/Sha1Operands.hpp>
 #include <codegen/x86/ClzeroOperands.hpp>
 #include <codegen/x86/ClzeroLowering.hpp>
+#include <codegen/x86/ReciprocalOperands.hpp>
 #include <codegen/x86/DecodedInstruction.hpp>
 #include <codegen/x86/X64InstructionDecoder.hpp>
 #include <codegen/x86/X64InstructionRewriter.hpp>
@@ -18,6 +19,7 @@
 #include <elfpatcher/linux/LinuxElfPatcher.hpp>
 #include <io/ByteWriter.hpp>
 #include <array>
+#include <cmath>
 #include <cstring>
 #include <cstdint>
 #ifdef __linux__
@@ -495,6 +497,51 @@ void converterClzero() {
     require(failureOffset([&] { (void)converter->Convert(segmentRelative, {segmentHeader(text.size())}); }, "FS-relative CLZERO was accepted") == 0x20A, "CLZERO operand failure does not carry the file offset");
 }
 
+void reciprocalOperands() {
+    const auto vex2 = Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xF8, 0x52, 0xD5}.data(), 4);
+    require(vex2 && vex2->Operation == Codegen::ReciprocalOperation::ReciprocalSquareRoot && vex2->Destination == 2 && vex2->Source == 5, "VEX2 VRSQRTPS was not decoded");
+    const auto vex2High = Codegen::DecodeVexReciprocal(Bytes{0xC5, 0x78, 0x53, 0xC1}.data(), 4);
+    require(vex2High && vex2High->Operation == Codegen::ReciprocalOperation::Reciprocal && vex2High->Destination == 8 && vex2High->Source == 1, "VEX2 VRCPPS with a high destination was not decoded");
+    const auto vex3 = Codegen::DecodeVexReciprocal(Bytes{0xC4, 0x41, 0x78, 0x52, 0xC9}.data(), 5);
+    require(vex3 && vex3->Destination == 9 && vex3->Source == 9, "VEX3 VRSQRTPS with high registers was not decoded");
+    require(!Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xFC, 0x52, 0xD5}.data(), 4), "256-bit VRSQRTPS must stay native");
+    require(!Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xFA, 0x52, 0xD5}.data(), 4), "VRSQRTSS must stay native");
+    require(!Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xF8, 0x52, 0x10}.data(), 4), "Memory form must stay native");
+    require(!Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xF8, 0x51, 0xD5}.data(), 4), "VSQRTPS is not a reciprocal");
+    require(!Codegen::DecodeVexReciprocal(Bytes{0x0F, 0x52, 0xD5}.data(), 3), "Legacy RSQRTPS must stay native");
+}
+
+void converterReciprocal() {
+    const auto converter = Codegen::MakeAmd64OnlyConverter();
+    Bytes file(0x300, 0xCC);
+    const Bytes text = {
+        0xC5, 0xF8, 0x52, 0xD5,
+        0xC5, 0xE8, 0x59, 0xD1,
+        0xC5, 0xF8, 0x53, 0xC1,
+        0xC3};
+    std::copy(text.begin(), text.end(), file.begin() + 0x200);
+    const auto result = converter->Convert(file, {segmentHeader(text.size())});
+    require(result.Trampolines.size() == 1 && result.KeptCount == 1 && result.Bytes == file, "VRSQRTPS was not lowered or the unmovable VRCPPS was not kept");
+    const auto& site = result.Trampolines[0];
+    require(site.Offset == 0x200 && site.Length == 8 && result.Reports[0].InstructionName == "VRSQRTPS", "VRSQRTPS did not absorb the following VMULPS");
+    require(site.Body[site.ReturnBranchOffset - 4] == 0xC5 && site.Body[site.ReturnBranchOffset - 1] == 0xD1, "Absorbed VMULPS does not run before the return jump");
+    require(result.Reports[1].InstructionName == "VRCPPS" && result.Reports[1].Lowering == Codegen::Amd64OnlyLowering::Kept && result.Reports[1].Offset == 0x208, "VRCPPS before a return was not reported as kept");
+    auto branchInto = file;
+    const Bytes jump = {0xEB, 0x00, 0xC5, 0xF8, 0x52, 0xD5, 0xC5, 0xE8, 0x59, 0xD1, 0xC3};
+    std::copy(jump.begin(), jump.end(), branchInto.begin() + 0x200);
+    const auto branched = converter->Convert(branchInto, {segmentHeader(jump.size())});
+    require(branched.Trampolines.size() == 1 && branched.KeptCount == 0, "A branch to the reciprocal itself must still allow a stub");
+    const Bytes jumpInside = {0xEB, 0x04, 0xC5, 0xF8, 0x52, 0xD5, 0xC5, 0xE8, 0x59, 0xD1, 0xC3};
+    std::copy(jumpInside.begin(), jumpInside.end(), branchInto.begin() + 0x200);
+    const auto inside = converter->Convert(branchInto, {segmentHeader(jumpInside.size())});
+    require(inside.Trampolines.empty() && inside.KeptCount == 1 && inside.Bytes == branchInto, "A branch into the absorbed instruction must keep the reciprocal native");
+    auto ripRelative = file;
+    const Bytes load = {0xC5, 0xF8, 0x52, 0xD5, 0x8B, 0x05, 0x10, 0x00, 0x00, 0x00, 0xC3};
+    std::copy(load.begin(), load.end(), ripRelative.begin() + 0x200);
+    const auto kept = converter->Convert(ripRelative, {segmentHeader(load.size())});
+    require(kept.Trampolines.empty() && kept.KeptCount == 1 && kept.Bytes == ripRelative, "A following RIP-relative instruction was moved into a stub");
+}
+
 void converterStrayRex() {
     const auto converter = Codegen::MakeAmd64OnlyConverter();
     const auto convert = [&](const Bytes& text) {
@@ -718,6 +765,53 @@ std::array<std::uint64_t, 2> runRegisterFormStub(const Bytes& site, const std::u
     return {out[0], out[1]};
 }
 
+std::uint64_t packLanes(const float (&lanes)[4], const std::size_t first) {
+    std::uint64_t packed = 0;
+    std::memcpy(&packed, lanes + first, sizeof(packed));
+    return packed;
+}
+
+bool sameLane(const float expected, const float actual) {
+    if (std::isnan(expected))
+        return std::isnan(actual);
+    std::uint32_t a = 0;
+    std::uint32_t b = 0;
+    std::memcpy(&a, &expected, sizeof(a));
+    std::memcpy(&b, &actual, sizeof(b));
+    return a == b;
+}
+
+void reciprocalExecution() {
+    const float inputs[][4] = {
+        {1.0f, 4.0f, 0.25f, 2.0f},
+        {0.0f, -0.0f, INFINITY, -1.0f},
+        {1.00000012f, 0.99999994f, 3.0e-38f, 1.0e30f}};
+    for (const auto& lanes : inputs) {
+        const std::uint64_t source[2] = {packLanes(lanes, 0), packLanes(lanes, 2)};
+        const std::uint64_t destination[2] = {0x1111111111111111ull, 0x2222222222222222ull};
+        for (const bool squareRoot : {true, false}) {
+            const std::uint8_t opcode = squareRoot ? 0x52 : 0x53;
+            const Bytes distinct = {0xC5, 0xF8, opcode, 0xD5};
+            const Bytes same = {0xC5, 0xF8, opcode, 0xD2};
+            const auto distinctOut = runRegisterFormStub(distinct, destination, source);
+            const auto sameOut = runRegisterFormStub(same, source, source);
+            float distinctLanes[4];
+            float sameLanes[4];
+            std::memcpy(distinctLanes, distinctOut.data(), sizeof(distinctLanes));
+            std::memcpy(sameLanes, sameOut.data(), sizeof(sameLanes));
+            for (std::size_t lane = 0; lane < 4; ++lane) {
+                const float expected = squareRoot ? 1.0f / std::sqrt(lanes[lane]) : 1.0f / lanes[lane];
+                require(sameLane(expected, distinctLanes[lane]), "VRSQRTPS/VRCPPS stub is not correctly rounded");
+                require(sameLane(expected, sameLanes[lane]), "VRSQRTPS/VRCPPS stub with equal operands is not correctly rounded");
+            }
+        }
+    }
+    const float one[4] = {1.0f, 1.0f, 1.0f, 1.0f};
+    const std::uint64_t ones[2] = {packLanes(one, 0), packLanes(one, 2)};
+    const auto unit = runRegisterFormStub({0xC5, 0xF8, 0x52, 0xD5}, ones, ones);
+    require(unit[0] == ones[0] && unit[1] == ones[1], "VRSQRTPS of 1.0 must be exactly 1.0 so a renormalised unit quaternion stays unit");
+}
+
 void registerFormExecution() {
     const Bytes extrqDistinct = {0x66, 0x0F, 0x79, 0xD5};
     const Bytes extrqSame = {0x66, 0x0F, 0x79, 0xD2};
@@ -939,6 +1033,7 @@ void sha1Execution() {
     }
 }
 #else
+void reciprocalExecution() {}
 void registerFormExecution() {}
 void sha1Execution() {}
 void sha256Execution() {}
@@ -962,6 +1057,7 @@ int main() {
         sha256Operands();
         sha1Operands();
         clzeroOperands();
+        reciprocalOperands();
         matcherSubstitutions();
         goldenBodies();
         registerFormExecution();
@@ -969,12 +1065,14 @@ int main() {
         sha256Execution();
         sha1Execution();
         clzeroExecution();
+        reciprocalExecution();
         converterSegment();
         converterRipRelativeFollower();
         converterSha256();
         converterSha1();
         converterMonitorWait();
         converterClzero();
+        converterReciprocal();
         converterStrayRex();
         rewriterStrayRex();
         converterFailureOffsets();

core/relinker/main.cpp

@@ -19,6 +19,7 @@
 #include <relinker/pipeline/RelinkerPipeline.hpp>
 #include <relinker/guest/GuestImage.hpp>
 #include <codegen/IAmd64OnlyConverter.hpp>
+#include <map>
 #include <codegen/CodegenException.hpp>
 #include <filesystem>
 #include <iostream>
@@ -51,9 +52,18 @@ int main(const int argc, char* argv[]) {
             auto converted = Codegen::MakeAmd64OnlyConverter()->Convert(std::move(sourceBytes), codeSegments);
             sourceBytes = std::move(converted.Bytes);
             trampolines = std::move(converted.Trampolines);
-            for (const auto& report : converted.Reports)
-                std::cout << "Intel substitution: " << report.InstructionName << " at 0x" << std::hex << report.Offset << std::dec << " (" << report.OriginalLength << " bytes) -> " << (report.Lowering == Codegen::Amd64OnlyLowering::InPlace ? "in place " : "stub ") << report.ReplacementLength << " bytes\n";
-            std::cout << "Intel conversion: " << converted.ReplacedCount << " in place, " << trampolines.size() << " stubs\n";
+            std::map<std::string, std::size_t> stubsByName;
+            for (const auto& report : converted.Reports) {
+                if (report.Lowering == Codegen::Amd64OnlyLowering::Kept)
+                    std::cout << "Intel substitution: " << report.InstructionName << " at 0x" << std::hex << report.Offset << std::dec << " (" << report.OriginalLength << " bytes) kept: no room for a jump\n";
+                else if (report.InstructionName == "VRSQRTPS" || report.InstructionName == "VRCPPS")
+                    ++stubsByName[report.InstructionName];
+                else
+                    std::cout << "Intel substitution: " << report.InstructionName << " at 0x" << std::hex << report.Offset << std::dec << " (" << report.OriginalLength << " bytes) -> " << (report.Lowering == Codegen::Amd64OnlyLowering::InPlace ? "in place " : "stub ") << report.ReplacementLength << " bytes\n";
+            }
+            for (const auto& [name, count] : stubsByName)
+                std::cout << "Intel substitution: " << name << " -> stub at " << count << " sites\n";
+            std::cout << "Intel conversion: " << converted.ReplacedCount << " in place, " << trampolines.size() << " stubs, " << converted.KeptCount << " kept\n";
         }
 
         auto elfReader = std::make_shared<Relinker::ElfReader>(sourceBytes);

docs/dev/TechnicalDebt.md

@@ -60,6 +60,7 @@ Throughout the project, every function at every stage either **does exactly what
 - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test.
 - The libc [SSE4a trap emulation](../../core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp) on Windows is superseded by `--to-intel` and remains only until the relinked title has been verified without it.
 - `--to-intel` guest module trampolines are covered only by a synthetic relinker test; no game title has been verified with them on Linux or Windows.
+- `--to-intel` replaces the approximate VEX.128 register forms of VRSQRTPS and VRCPPS with [correctly rounded](../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). The 256-bit, scalar, legacy and memory forms keep the native instruction, as does a 4-byte site with no movable instruction after it for the jump (5 sites in PPSA21564).
 - [sceKeyboardGetKey2Char](../../core/libs/prx/libSceKeyboard/src/keyboard_impl.cpp) (libSceKeyboard) translates only the 101-key (US) arrangement and throws for the 106-key (Japanese) one; Ctrl and Alt do not change the character.
 - [libSceAudiodec](../../core/libs/prx/libSceAudiodec/Export.cpp) throws for the 24-bit PCM word size (`iBwPcm` 0): its sample layout is unknown. ATRAC9 decoding is covered only by configuration and error tests, as no ATRAC9 encoder is available for a fixture.
 - [libScePngEnc](../../core/libs/prx/libScePngEnc/Export.cpp) honours `filter_type` only as all filters (adaptive) or a single filter: the [PNG encoder](../../core/Decoder/Png/src/Png.cpp) (stb) cannot restrict adaptive filtering to a subset, so the first filter in the mask is used.