diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index a01c7cd0e3..840596b49f 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -44,6 +44,8 @@ add_executable(relinker ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sha1Lowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalOperands.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/CallSiteResolver.cpp @@ -110,6 +112,8 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sha1Lowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalOperands.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp ) target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows) if(BUILD_TESTING) @@ -158,6 +162,8 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sha1Lowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalOperands.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp diff --git a/core/relinker/codegen/include/codegen/CodegenTypes.hpp b/core/relinker/codegen/include/codegen/CodegenTypes.hpp index 51d90c0bc5..ee19633333 100644 --- a/core/relinker/codegen/include/codegen/CodegenTypes.hpp +++ b/core/relinker/codegen/include/codegen/CodegenTypes.hpp @@ -43,6 +43,7 @@ struct ConvertResult { std::size_t ReplacedCount; std::vector Trampolines; std::vector Reports; + std::size_t KeptCount = 0; }; enum class ControlFlowKind : std::uint8_t { diff --git a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp index b87b6a6bfc..8a89186717 100644 --- a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp +++ b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp @@ -44,6 +44,8 @@ inline constexpr Entry kExtrqRegisterForm = {"EXTRQ register form", nullptr, 0}; inline constexpr Entry kInsertqRegisterForm = {"INSERTQ register form", nullptr, 0}; inline constexpr Entry kMovntss = {"MOVNTSS", nullptr, 0}; inline constexpr Entry kMovntsd = {"MOVNTSD", nullptr, 0}; +inline constexpr Entry kVrcpps = {"VRCPPS", nullptr, 0}; +inline constexpr Entry kVrsqrtps = {"VRSQRTPS", nullptr, 0}; inline constexpr Entry kSha256rnds2 = {"SHA256RNDS2", nullptr, 0}; inline constexpr Entry kSha256msg1 = {"SHA256MSG1", nullptr, 0}; inline constexpr Entry kSha256msg2 = {"SHA256MSG2", nullptr, 0}; diff --git a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp index 98c740afa9..f54e2e1e0d 100644 --- a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp +++ b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp @@ -11,7 +11,8 @@ namespace Codegen { enum class Amd64OnlyLowering : std::uint8_t { InPlace, Trampoline, - Unsupported + Unsupported, + Kept }; struct Amd64OnlyMatch { @@ -21,6 +22,7 @@ struct Amd64OnlyMatch { std::vector ReplacementBytes; std::vector StubBody; std::size_t ReturnBranchOffset; + bool Optional = false; }; struct Amd64OnlySubstitutionReport { diff --git a/core/relinker/codegen/include/codegen/x86/ReciprocalLowering.hpp b/core/relinker/codegen/include/codegen/x86/ReciprocalLowering.hpp new file mode 100644 index 0000000000..a4f20ea25c --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/ReciprocalLowering.hpp @@ -0,0 +1,18 @@ +#ifndef CODEGEN_X86_RECIPROCALLOWERING_HPP +#define CODEGEN_X86_RECIPROCALLOWERING_HPP + +#include +#include +#include + +namespace Codegen { + +class ReciprocalLowering { +public: + void EmitOutOfLine(StubBodyBuilder& body, const ReciprocalOperands& operands) const; + [[nodiscard]] LoweredBody LowerOutOfLine(const ReciprocalOperands& operands, std::span trailing = {}) const; +}; + +} + +#endif diff --git a/core/relinker/codegen/include/codegen/x86/ReciprocalOperands.hpp b/core/relinker/codegen/include/codegen/x86/ReciprocalOperands.hpp new file mode 100644 index 0000000000..31102844ec --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/ReciprocalOperands.hpp @@ -0,0 +1,25 @@ +#ifndef CODEGEN_X86_RECIPROCALOPERANDS_HPP +#define CODEGEN_X86_RECIPROCALOPERANDS_HPP + +#include +#include +#include + +namespace Codegen { + +enum class ReciprocalOperation : std::uint8_t { + Reciprocal, + ReciprocalSquareRoot +}; + +struct ReciprocalOperands { + ReciprocalOperation Operation; + std::uint8_t Destination; + std::uint8_t Source; +}; + +[[nodiscard]] std::optional DecodeVexReciprocal(const std::uint8_t* data, std::size_t length); + +} + +#endif diff --git a/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp b/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp index ee200c20b6..7cebc4d558 100644 --- a/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp +++ b/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp @@ -23,6 +23,7 @@ void EmitShiftImm(std::vector& out, std::uint8_t extension, std::u class StubBodyBuilder { public: void Sse(std::uint8_t prefix, std::initializer_list opcode, std::uint8_t dst, std::uint8_t src); + void SsePlain(std::initializer_list opcode, std::uint8_t dst, std::uint8_t src); void SseImm(std::uint8_t prefix, std::initializer_list opcode, std::uint8_t dst, std::uint8_t src, std::uint8_t imm); void ShiftImm(std::uint8_t extension, std::uint8_t reg, std::uint8_t imm); void ShiftDwordImm(std::uint8_t extension, std::uint8_t reg, std::uint8_t imm); diff --git a/core/relinker/codegen/src/Amd64OnlyConverter.cpp b/core/relinker/codegen/src/Amd64OnlyConverter.cpp index 887d4154f4..aa0bc0d09b 100644 --- a/core/relinker/codegen/src/Amd64OnlyConverter.cpp +++ b/core/relinker/codegen/src/Amd64OnlyConverter.cpp @@ -126,50 +126,61 @@ void Amd64OnlyConverter::_convertSegment( break; } case Amd64OnlyLowering::Trampoline: { - std::size_t siteLength = match.Length; - auto stub = substitution; - if (siteLength < Amd64OnlySubstitutionTable::kJmpRel32.Size) { - const X64InstructionDecoder decoder; - std::vector> sequence{std::span(seg.data() + match.Offset, match.Length)}; - std::size_t trailingBytes = 0; - for (auto next = item.Index + 1; siteLength < Amd64OnlySubstitutionTable::kJmpRel32.Size; ++next) { - if (next >= matches.size() || matches[next].Offset != match.Offset + siteLength) - throw CodegenException("AMD-only instruction too short for a jump and not followed by an instruction", fileOffset); - const auto& following = matches[next]; - const std::span bytes(seg.data() + following.Offset, following.Length); - const auto info = decoder.DecodeInstruction(bytes.data(), bytes.size()); - const bool amdOnly = _matcher->Match(bytes.data(), bytes.size()).has_value(); - if (amdOnly && trailingBytes == 0) { - sequence.push_back(bytes); - consumed.insert(next); - } else if (amdOnly || info.FlowKind != ControlFlowKind::Sequential || info.HasRipRelativeDisp || info.HasBranchTarget) { - throw CodegenException("AMD-only instruction too short for a jump is followed by an instruction that cannot move", ph.Offset + following.Offset); - } else { - trailingBytes += following.Length; + std::vector taken; + try { + std::size_t siteLength = match.Length; + auto stub = substitution; + if (siteLength < Amd64OnlySubstitutionTable::kJmpRel32.Size) { + const X64InstructionDecoder decoder; + std::vector> sequence{std::span(seg.data() + match.Offset, match.Length)}; + std::size_t trailingBytes = 0; + for (auto next = item.Index + 1; siteLength < Amd64OnlySubstitutionTable::kJmpRel32.Size; ++next) { + if (next >= matches.size() || matches[next].Offset != match.Offset + siteLength) + throw CodegenException("AMD-only instruction too short for a jump and not followed by an instruction", fileOffset); + const auto& following = matches[next]; + const std::span bytes(seg.data() + following.Offset, following.Length); + const auto info = decoder.DecodeInstruction(bytes.data(), bytes.size()); + const bool amdOnly = _matcher->Match(bytes.data(), bytes.size()).has_value(); + if (amdOnly && trailingBytes == 0) { + sequence.push_back(bytes); + taken.push_back(next); + } else if (amdOnly || info.FlowKind != ControlFlowKind::Sequential || info.HasRipRelativeDisp || info.HasBranchTarget) { + throw CodegenException("AMD-only instruction too short for a jump is followed by an instruction that cannot move", ph.Offset + following.Offset); + } else { + trailingBytes += following.Length; + } + siteLength += following.Length; } - siteLength += following.Length; + const std::span trailing(seg.data() + match.Offset + siteLength - trailingBytes, trailingBytes); + auto relocated = _atFileOffset(fileOffset, [&] { return _matcher->MatchSequence(sequence, trailing); }); + if (!relocated.has_value() || relocated->Lowering != Amd64OnlyLowering::Trampoline) + throw CodegenException("AMD-only instruction sequence has no out-of-line lowering", fileOffset); + stub = std::move(*relocated); } - const std::span trailing(seg.data() + match.Offset + siteLength - trailingBytes, trailingBytes); - auto relocated = _atFileOffset(fileOffset, [&] { return _matcher->MatchSequence(sequence, trailing); }); - if (!relocated.has_value() || relocated->Lowering != Amd64OnlyLowering::Trampoline) - throw CodegenException("AMD-only instruction sequence has no out-of-line lowering", fileOffset); - stub = std::move(*relocated); + const auto hit = branchTargets.upper_bound(address); + if (hit != branchTargets.end() && *hit < address + siteLength) + throw CodegenException("Branch enters an AMD-only instruction", ph.Offset + (*hit - ph.MappedAddress)); + const auto begin = seg.begin() + static_cast(match.Offset); + result.Trampolines.push_back({ + fileOffset, + address, + siteLength, + std::vector(begin, begin + static_cast(siteLength)), + stub.StubBody, + stub.ReturnBranchOffset + }); + replacementLength = stub.StubBody.size(); + consumed.insert(taken.begin(), taken.end()); + } catch (const CodegenException&) { + if (!substitution.Optional) + throw; + ++result.KeptCount; + result.Reports.push_back({substitution.InstructionName, fileOffset, match.Length, 0, Amd64OnlyLowering::Kept}); + continue; } - const auto hit = branchTargets.upper_bound(address); - if (hit != branchTargets.end() && *hit < address + siteLength) - throw CodegenException("Branch enters an AMD-only instruction", ph.Offset + (*hit - ph.MappedAddress)); - const auto begin = seg.begin() + static_cast(match.Offset); - result.Trampolines.push_back({ - fileOffset, - address, - siteLength, - std::vector(begin, begin + static_cast(siteLength)), - stub.StubBody, - stub.ReturnBranchOffset - }); - replacementLength = stub.StubBody.size(); break; } + case Amd64OnlyLowering::Kept: case Amd64OnlyLowering::Unsupported: throw CodegenException("AMD-only instruction without Intel lowering: " + substitution.InstructionName, fileOffset); } @@ -186,7 +197,7 @@ ConvertResult Amd64OnlyConverter::Convert( std::vector fileBytes, const std::vector& codeSegments ) const { - ConvertResult result{{}, 0, {}, {}}; + ConvertResult result{}; for (const auto& ph : codeSegments) _convertSegment(fileBytes, ph, result); result.Bytes = std::move(fileBytes); diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index a0732af9cc..48bde0aafa 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include @@ -35,6 +37,10 @@ const Entry& _sse4aEntry(const Sse4aOperands& operands) { return operands.Insertq ? kInsertq : kExtrq; } +const Entry& _reciprocalEntry(const ReciprocalOperands& operands) { + return operands.Operation == ReciprocalOperation::ReciprocalSquareRoot ? kVrsqrtps : kVrcpps; +} + const Entry& _sha256Entry(const Sha256Operands& operands) { switch (operands.Operation) { case Sha256Operation::Rnds2: @@ -97,6 +103,7 @@ private: Sha256Lowering _sha256Lowering; Sha1Lowering _sha1Lowering; ClzeroLowering _clzeroLowering; + ReciprocalLowering _reciprocalLowering; [[nodiscard]] Amd64OnlyMatch _matchMovnts(const DecodedInstruction& instr, const Entry& entry) const; [[nodiscard]] Amd64OnlyMatch _matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span trailing) const; @@ -176,13 +183,18 @@ std::optional Amd64OnlyInstructionMatcher::MatchSequence( if (name == nullptr) name = kClzero.Name; _clzeroLowering.EmitOutOfLine(body, operands); + } else if (const auto reciprocal = DecodeVexReciprocal(instr.Data, instr.Length)) { + if (name == nullptr) + name = _reciprocalEntry(*reciprocal).Name; + _reciprocalLowering.EmitOutOfLine(body, *reciprocal); } else { return std::nullopt; } } body.Raw(trailing); auto lowered = body.Finish(); - return Amd64OnlyMatch{name, instructions.front().size(), Amd64OnlyLowering::Trampoline, {}, std::move(lowered.Bytes), lowered.ReturnBranchOffset}; + const bool optional = DecodeVexReciprocal(instructions.front().data(), instructions.front().size()).has_value(); + return Amd64OnlyMatch{name, instructions.front().size(), Amd64OnlyLowering::Trampoline, {}, std::move(lowered.Bytes), lowered.ReturnBranchOffset, optional}; } std::optional Amd64OnlyInstructionMatcher::Match( @@ -216,6 +228,11 @@ std::optional Amd64OnlyInstructionMatcher::Match( if (instr.IsMwaitx()) return _validWait(instr) ? _inPlace(kMwaitx, length, std::vector(kPause.Bytes, kPause.Bytes + kPause.Size)) : _unsupported(kMwaitx, length); + if (const auto reciprocal = DecodeVexReciprocal(data, length)) { + auto body = _reciprocalLowering.LowerOutOfLine(*reciprocal, trailing); + return Amd64OnlyMatch{_reciprocalEntry(*reciprocal).Name, length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset, true}; + } + if (instr.IsClzero()) return _matchClzero(instr, trailing); diff --git a/core/relinker/codegen/src/x86/ReciprocalLowering.cpp b/core/relinker/codegen/src/x86/ReciprocalLowering.cpp new file mode 100644 index 0000000000..b6f8fcadce --- /dev/null +++ b/core/relinker/codegen/src/x86/ReciprocalLowering.cpp @@ -0,0 +1,65 @@ +#include +#include +#include + +namespace Codegen { + +namespace { + +constexpr std::uint8_t kSqrtps = 0x51; +constexpr std::uint8_t kDivps = 0x5E; +constexpr std::uint8_t kMovaps = 0x28; +constexpr float kOne = 1.0f; + +StubConstant _ones() { + StubConstant constant{}; + for (std::size_t lane = 0; lane < 4; ++lane) + std::memcpy(constant.data() + lane * sizeof(kOne), &kOne, sizeof(kOne)); + return constant; +} + +std::uint8_t _scratch(const ReciprocalOperands& operands) { + std::uint8_t reg = 0; + while (reg == operands.Destination || reg == operands.Source) + ++reg; + return reg; +} + +void _vexMovaps(StubBodyBuilder& body, const std::uint8_t dst, const std::uint8_t src) { + const std::array bytes{ + 0xC4, + static_cast(((dst & 8) != 0 ? 0x00 : 0x80) | 0x40 | ((src & 8) != 0 ? 0x00 : 0x20) | 0x01), + 0x78, + kMovaps, + static_cast(0xC0 | ((dst & 7) << 3) | (src & 7)) + }; + body.Raw(bytes); +} + +} + +void ReciprocalLowering::EmitOutOfLine(StubBodyBuilder& body, const ReciprocalOperands& operands) const { + const auto scratch = _scratch(operands); + const auto dst = operands.Destination; + body.Spill(scratch); + if (operands.Operation == ReciprocalOperation::ReciprocalSquareRoot) { + body.SsePlain({0x0F, kSqrtps}, scratch, operands.Source); + body.RipOperand({0x0F, kMovaps}, dst, _ones()); + body.SsePlain({0x0F, kDivps}, dst, scratch); + _vexMovaps(body, dst, dst); + } else { + body.RipOperand({0x0F, kMovaps}, scratch, _ones()); + body.SsePlain({0x0F, kDivps}, scratch, operands.Source); + _vexMovaps(body, dst, scratch); + } + body.Restore(scratch); +} + +LoweredBody ReciprocalLowering::LowerOutOfLine(const ReciprocalOperands& operands, std::span trailing) const { + StubBodyBuilder body; + EmitOutOfLine(body, operands); + body.Raw(trailing); + return body.Finish(); +} + +} diff --git a/core/relinker/codegen/src/x86/ReciprocalOperands.cpp b/core/relinker/codegen/src/x86/ReciprocalOperands.cpp new file mode 100644 index 0000000000..c9e2037684 --- /dev/null +++ b/core/relinker/codegen/src/x86/ReciprocalOperands.cpp @@ -0,0 +1,55 @@ +#include +#include + +namespace Codegen { + +namespace { + +constexpr std::uint8_t kVexMap0F = 0x01; +constexpr std::uint8_t kVexUnusedRegister = 0x0F; +constexpr std::uint8_t kOpcodeRsqrtps = 0x52; +constexpr std::uint8_t kOpcodeRcpps = 0x53; + +} + +std::optional DecodeVexReciprocal(const std::uint8_t* data, const std::size_t length) { + using namespace X64OpcodeConstants; + if (length < 4) + return std::nullopt; + bool registerExtension = false; + bool rmExtension = false; + std::uint8_t payload = 0; + std::size_t opcodeOffset = 0; + if (data[0] == OneByteVex2 && length == 4) { + registerExtension = (data[1] & 0x80) == 0; + payload = data[1]; + opcodeOffset = 2; + } else if (data[0] == OneByteVex3 && length == 5) { + if ((data[1] & Vex3MapMask) != kVexMap0F) + return std::nullopt; + registerExtension = (data[1] & 0x80) == 0; + rmExtension = (data[1] & 0x20) == 0; + payload = data[2]; + opcodeOffset = 3; + } else { + return std::nullopt; + } + const auto vvvv = static_cast((payload >> 3) & 0x0F); + const bool wide = (payload & 0x04) != 0; + const auto pp = static_cast(payload & 0x03); + if (vvvv != kVexUnusedRegister || wide || pp != 0) + return std::nullopt; + const auto opcode = data[opcodeOffset]; + if (opcode != kOpcodeRsqrtps && opcode != kOpcodeRcpps) + return std::nullopt; + const auto modrm = data[opcodeOffset + 1]; + if (((modrm >> ModRmModShift) & ModRmModMask) != ModRmModRegister) + return std::nullopt; + ReciprocalOperands operands{}; + operands.Operation = opcode == kOpcodeRsqrtps ? ReciprocalOperation::ReciprocalSquareRoot : ReciprocalOperation::Reciprocal; + operands.Destination = static_cast(((modrm >> ModRmRegShift) & ModRmRegMask) | (registerExtension ? 8 : 0)); + operands.Source = static_cast((modrm & ModRmRmMask) | (rmExtension ? 8 : 0)); + return operands; +} + +} diff --git a/core/relinker/codegen/src/x86/StubBodyBuilder.cpp b/core/relinker/codegen/src/x86/StubBodyBuilder.cpp index 5546e29b2c..2ab183011c 100644 --- a/core/relinker/codegen/src/x86/StubBodyBuilder.cpp +++ b/core/relinker/codegen/src/x86/StubBodyBuilder.cpp @@ -51,6 +51,14 @@ void StubBodyBuilder::Sse(const std::uint8_t prefix, const std::initializer_list EmitSse(_bytes, prefix, opcode, dst, src); } +void StubBodyBuilder::SsePlain(const std::initializer_list opcode, const std::uint8_t dst, const std::uint8_t src) { + const auto rex = _rex(dst, src); + if (rex != kRexBase) + _bytes.push_back(rex); + _bytes.insert(_bytes.end(), opcode.begin(), opcode.end()); + _bytes.push_back(static_cast(kModRmRegister | ((dst & 7) << 3) | (src & 7))); +} + void StubBodyBuilder::SseImm(const std::uint8_t prefix, const std::initializer_list opcode, const std::uint8_t dst, const std::uint8_t src, const std::uint8_t imm) { EmitSse(_bytes, prefix, opcode, dst, src); _bytes.push_back(imm); diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp index 92590fa0b0..1ac2dbcb44 100644 --- a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #include #include #ifdef __linux__ @@ -495,6 +497,51 @@ void converterClzero() { require(failureOffset([&] { (void)converter->Convert(segmentRelative, {segmentHeader(text.size())}); }, "FS-relative CLZERO was accepted") == 0x20A, "CLZERO operand failure does not carry the file offset"); } +void reciprocalOperands() { + const auto vex2 = Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xF8, 0x52, 0xD5}.data(), 4); + require(vex2 && vex2->Operation == Codegen::ReciprocalOperation::ReciprocalSquareRoot && vex2->Destination == 2 && vex2->Source == 5, "VEX2 VRSQRTPS was not decoded"); + const auto vex2High = Codegen::DecodeVexReciprocal(Bytes{0xC5, 0x78, 0x53, 0xC1}.data(), 4); + require(vex2High && vex2High->Operation == Codegen::ReciprocalOperation::Reciprocal && vex2High->Destination == 8 && vex2High->Source == 1, "VEX2 VRCPPS with a high destination was not decoded"); + const auto vex3 = Codegen::DecodeVexReciprocal(Bytes{0xC4, 0x41, 0x78, 0x52, 0xC9}.data(), 5); + require(vex3 && vex3->Destination == 9 && vex3->Source == 9, "VEX3 VRSQRTPS with high registers was not decoded"); + require(!Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xFC, 0x52, 0xD5}.data(), 4), "256-bit VRSQRTPS must stay native"); + require(!Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xFA, 0x52, 0xD5}.data(), 4), "VRSQRTSS must stay native"); + require(!Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xF8, 0x52, 0x10}.data(), 4), "Memory form must stay native"); + require(!Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xF8, 0x51, 0xD5}.data(), 4), "VSQRTPS is not a reciprocal"); + require(!Codegen::DecodeVexReciprocal(Bytes{0x0F, 0x52, 0xD5}.data(), 3), "Legacy RSQRTPS must stay native"); +} + +void converterReciprocal() { + const auto converter = Codegen::MakeAmd64OnlyConverter(); + Bytes file(0x300, 0xCC); + const Bytes text = { + 0xC5, 0xF8, 0x52, 0xD5, + 0xC5, 0xE8, 0x59, 0xD1, + 0xC5, 0xF8, 0x53, 0xC1, + 0xC3}; + std::copy(text.begin(), text.end(), file.begin() + 0x200); + const auto result = converter->Convert(file, {segmentHeader(text.size())}); + require(result.Trampolines.size() == 1 && result.KeptCount == 1 && result.Bytes == file, "VRSQRTPS was not lowered or the unmovable VRCPPS was not kept"); + const auto& site = result.Trampolines[0]; + require(site.Offset == 0x200 && site.Length == 8 && result.Reports[0].InstructionName == "VRSQRTPS", "VRSQRTPS did not absorb the following VMULPS"); + require(site.Body[site.ReturnBranchOffset - 4] == 0xC5 && site.Body[site.ReturnBranchOffset - 1] == 0xD1, "Absorbed VMULPS does not run before the return jump"); + require(result.Reports[1].InstructionName == "VRCPPS" && result.Reports[1].Lowering == Codegen::Amd64OnlyLowering::Kept && result.Reports[1].Offset == 0x208, "VRCPPS before a return was not reported as kept"); + auto branchInto = file; + const Bytes jump = {0xEB, 0x00, 0xC5, 0xF8, 0x52, 0xD5, 0xC5, 0xE8, 0x59, 0xD1, 0xC3}; + std::copy(jump.begin(), jump.end(), branchInto.begin() + 0x200); + const auto branched = converter->Convert(branchInto, {segmentHeader(jump.size())}); + require(branched.Trampolines.size() == 1 && branched.KeptCount == 0, "A branch to the reciprocal itself must still allow a stub"); + const Bytes jumpInside = {0xEB, 0x04, 0xC5, 0xF8, 0x52, 0xD5, 0xC5, 0xE8, 0x59, 0xD1, 0xC3}; + std::copy(jumpInside.begin(), jumpInside.end(), branchInto.begin() + 0x200); + const auto inside = converter->Convert(branchInto, {segmentHeader(jumpInside.size())}); + require(inside.Trampolines.empty() && inside.KeptCount == 1 && inside.Bytes == branchInto, "A branch into the absorbed instruction must keep the reciprocal native"); + auto ripRelative = file; + const Bytes load = {0xC5, 0xF8, 0x52, 0xD5, 0x8B, 0x05, 0x10, 0x00, 0x00, 0x00, 0xC3}; + std::copy(load.begin(), load.end(), ripRelative.begin() + 0x200); + const auto kept = converter->Convert(ripRelative, {segmentHeader(load.size())}); + require(kept.Trampolines.empty() && kept.KeptCount == 1 && kept.Bytes == ripRelative, "A following RIP-relative instruction was moved into a stub"); +} + void converterStrayRex() { const auto converter = Codegen::MakeAmd64OnlyConverter(); const auto convert = [&](const Bytes& text) { @@ -718,6 +765,53 @@ std::array runRegisterFormStub(const Bytes& site, const std::u return {out[0], out[1]}; } +std::uint64_t packLanes(const float (&lanes)[4], const std::size_t first) { + std::uint64_t packed = 0; + std::memcpy(&packed, lanes + first, sizeof(packed)); + return packed; +} + +bool sameLane(const float expected, const float actual) { + if (std::isnan(expected)) + return std::isnan(actual); + std::uint32_t a = 0; + std::uint32_t b = 0; + std::memcpy(&a, &expected, sizeof(a)); + std::memcpy(&b, &actual, sizeof(b)); + return a == b; +} + +void reciprocalExecution() { + const float inputs[][4] = { + {1.0f, 4.0f, 0.25f, 2.0f}, + {0.0f, -0.0f, INFINITY, -1.0f}, + {1.00000012f, 0.99999994f, 3.0e-38f, 1.0e30f}}; + for (const auto& lanes : inputs) { + const std::uint64_t source[2] = {packLanes(lanes, 0), packLanes(lanes, 2)}; + const std::uint64_t destination[2] = {0x1111111111111111ull, 0x2222222222222222ull}; + for (const bool squareRoot : {true, false}) { + const std::uint8_t opcode = squareRoot ? 0x52 : 0x53; + const Bytes distinct = {0xC5, 0xF8, opcode, 0xD5}; + const Bytes same = {0xC5, 0xF8, opcode, 0xD2}; + const auto distinctOut = runRegisterFormStub(distinct, destination, source); + const auto sameOut = runRegisterFormStub(same, source, source); + float distinctLanes[4]; + float sameLanes[4]; + std::memcpy(distinctLanes, distinctOut.data(), sizeof(distinctLanes)); + std::memcpy(sameLanes, sameOut.data(), sizeof(sameLanes)); + for (std::size_t lane = 0; lane < 4; ++lane) { + const float expected = squareRoot ? 1.0f / std::sqrt(lanes[lane]) : 1.0f / lanes[lane]; + require(sameLane(expected, distinctLanes[lane]), "VRSQRTPS/VRCPPS stub is not correctly rounded"); + require(sameLane(expected, sameLanes[lane]), "VRSQRTPS/VRCPPS stub with equal operands is not correctly rounded"); + } + } + } + const float one[4] = {1.0f, 1.0f, 1.0f, 1.0f}; + const std::uint64_t ones[2] = {packLanes(one, 0), packLanes(one, 2)}; + const auto unit = runRegisterFormStub({0xC5, 0xF8, 0x52, 0xD5}, ones, ones); + require(unit[0] == ones[0] && unit[1] == ones[1], "VRSQRTPS of 1.0 must be exactly 1.0 so a renormalised unit quaternion stays unit"); +} + void registerFormExecution() { const Bytes extrqDistinct = {0x66, 0x0F, 0x79, 0xD5}; const Bytes extrqSame = {0x66, 0x0F, 0x79, 0xD2}; @@ -939,6 +1033,7 @@ void sha1Execution() { } } #else +void reciprocalExecution() {} void registerFormExecution() {} void sha1Execution() {} void sha256Execution() {} @@ -962,6 +1057,7 @@ int main() { sha256Operands(); sha1Operands(); clzeroOperands(); + reciprocalOperands(); matcherSubstitutions(); goldenBodies(); registerFormExecution(); @@ -969,12 +1065,14 @@ int main() { sha256Execution(); sha1Execution(); clzeroExecution(); + reciprocalExecution(); converterSegment(); converterRipRelativeFollower(); converterSha256(); converterSha1(); converterMonitorWait(); converterClzero(); + converterReciprocal(); converterStrayRex(); rewriterStrayRex(); converterFailureOffsets(); diff --git a/core/relinker/main.cpp b/core/relinker/main.cpp index 0802223bf0..54bb2f815b 100644 --- a/core/relinker/main.cpp +++ b/core/relinker/main.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -51,9 +52,18 @@ int main(const int argc, char* argv[]) { auto converted = Codegen::MakeAmd64OnlyConverter()->Convert(std::move(sourceBytes), codeSegments); sourceBytes = std::move(converted.Bytes); trampolines = std::move(converted.Trampolines); - for (const auto& report : converted.Reports) - std::cout << "Intel substitution: " << report.InstructionName << " at 0x" << std::hex << report.Offset << std::dec << " (" << report.OriginalLength << " bytes) -> " << (report.Lowering == Codegen::Amd64OnlyLowering::InPlace ? "in place " : "stub ") << report.ReplacementLength << " bytes\n"; - std::cout << "Intel conversion: " << converted.ReplacedCount << " in place, " << trampolines.size() << " stubs\n"; + std::map stubsByName; + for (const auto& report : converted.Reports) { + if (report.Lowering == Codegen::Amd64OnlyLowering::Kept) + std::cout << "Intel substitution: " << report.InstructionName << " at 0x" << std::hex << report.Offset << std::dec << " (" << report.OriginalLength << " bytes) kept: no room for a jump\n"; + else if (report.InstructionName == "VRSQRTPS" || report.InstructionName == "VRCPPS") + ++stubsByName[report.InstructionName]; + else + std::cout << "Intel substitution: " << report.InstructionName << " at 0x" << std::hex << report.Offset << std::dec << " (" << report.OriginalLength << " bytes) -> " << (report.Lowering == Codegen::Amd64OnlyLowering::InPlace ? "in place " : "stub ") << report.ReplacementLength << " bytes\n"; + } + for (const auto& [name, count] : stubsByName) + std::cout << "Intel substitution: " << name << " -> stub at " << count << " sites\n"; + std::cout << "Intel conversion: " << converted.ReplacedCount << " in place, " << trampolines.size() << " stubs, " << converted.KeptCount << " kept\n"; } auto elfReader = std::make_shared(sourceBytes); diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 7b044a5a4c..a18df58177 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -60,6 +60,7 @@ Throughout the project, every function at every stage either **does exactly what - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. - The libc [SSE4a trap emulation](../../core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp) on Windows is superseded by `--to-intel` and remains only until the relinked title has been verified without it. - `--to-intel` guest module trampolines are covered only by a synthetic relinker test; no game title has been verified with them on Linux or Windows. +- `--to-intel` replaces the approximate VEX.128 register forms of VRSQRTPS and VRCPPS with [correctly rounded](../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). The 256-bit, scalar, legacy and memory forms keep the native instruction, as does a 4-byte site with no movable instruction after it for the jump (5 sites in PPSA21564). - [sceKeyboardGetKey2Char](../../core/libs/prx/libSceKeyboard/src/keyboard_impl.cpp) (libSceKeyboard) translates only the 101-key (US) arrangement and throws for the 106-key (Japanese) one; Ctrl and Alt do not change the character. - [libSceAudiodec](../../core/libs/prx/libSceAudiodec/Export.cpp) throws for the 24-bit PCM word size (`iBwPcm` 0): its sample layout is unknown. ATRAC9 decoding is covered only by configuration and error tests, as no ATRAC9 encoder is available for a fixture. - [libScePngEnc](../../core/libs/prx/libScePngEnc/Export.cpp) honours `filter_type` only as all filters (adaptive) or a single filter: the [PNG encoder](../../core/Decoder/Png/src/Png.cpp) (stb) cannot restrict adaptive filtering to a subset, so the first filter in the mask is used.