diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 8b9643c5e1..784fc11985 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -50,6 +50,7 @@ add_library(relinker_core STATIC ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/CallSiteResolver.cpp @@ -167,6 +168,7 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ) target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows) if(BUILD_TESTING) @@ -350,6 +352,7 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp diff --git a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp index 7f09e114ad..41948aec85 100644 --- a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp +++ b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp @@ -42,6 +42,7 @@ inline constexpr Entry kRdseed = {"RDSEED", nullptr, 0}; inline constexpr Entry kRdpid = {"RDPID", nullptr, 0}; inline constexpr Entry kClwb = {"CLWB", nullptr, 0}; inline constexpr Entry kWideStore = {"256-bit store", nullptr, 0}; +inline constexpr Entry kScalarCompare = {"scalar compare", nullptr, 0}; inline constexpr Entry kExtrq = {"EXTRQ", nullptr, 0}; inline constexpr Entry kInsertq = {"INSERTQ", nullptr, 0}; inline constexpr Entry kExtrqRegisterForm = {"EXTRQ register form", nullptr, 0}; diff --git a/core/relinker/codegen/include/codegen/x86/ScalarCompareOperands.hpp b/core/relinker/codegen/include/codegen/x86/ScalarCompareOperands.hpp new file mode 100644 index 0000000000..458f799ce4 --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/ScalarCompareOperands.hpp @@ -0,0 +1,19 @@ +#ifndef CODEGEN_X86_SCALARCOMPAREOPERANDS_HPP +#define CODEGEN_X86_SCALARCOMPAREOPERANDS_HPP + +#include +#include +#include + +namespace Codegen { + +struct ScalarCompareOperands { + std::optional RipDisplacementOffset; + std::int64_t RipTarget = 0; +}; + +[[nodiscard]] std::optional DecodeScalarCompare(const std::uint8_t* data, std::size_t length); + +} + +#endif diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index edf59c514c..112fed6de7 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -106,6 +107,14 @@ Amd64OnlyMatch _inPlace(const Entry& entry, const std::size_t length, std::vecto return Amd64OnlyMatch{entry.Name, length, Amd64OnlyLowering::InPlace, std::move(replacement), {}, 0}; } +void _emitScalarCompare(StubBodyBuilder& body, const DecodedInstruction& instr, const ScalarCompareOperands& operands) { + const std::span bytes(instr.Data, instr.Length); + if (operands.RipDisplacementOffset) + body.RipRelative(bytes, *operands.RipDisplacementOffset, operands.RipTarget); + else + body.Raw(bytes); +} + bool _isClzeroOpcode(const DecodedInstruction& instr) { const auto pos = instr.OpcodeOffset(); return pos + 2 < instr.Length && instr.Data[pos] == X64OpcodeConstants::TwoByteOpcodeEscape && instr.Data[pos + 1] == X64OpcodeConstants::TwoByteGrp7 && instr.Data[pos + 2] == 0xFC; @@ -219,6 +228,8 @@ std::optional Amd64OnlyInstructionMatcher::_matchRosetta(const D return _trampoline(kRdpid.Name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { body.Raw(_rdpidReplacement(instr)); }), true); if (const auto wide = DecodeVexWideStore(instr.Data, instr.Length)) return _trampoline(kWideStore.Name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { _wideStoreLowering.EmitOutOfLine(body, *wide); }), false, true); + if (const auto compare = DecodeScalarCompare(instr.Data, instr.Length)) + return _trampoline(kScalarCompare.Name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { _emitScalarCompare(body, instr, *compare); }), true); return std::nullopt; } @@ -254,6 +265,10 @@ const char* Amd64OnlyInstructionMatcher::_emitOutOfLine(StubBodyBuilder& body, c _wideStoreLowering.EmitOutOfLine(body, *wide); return kWideStore.Name; } + if (const auto compare = _target == Amd64OnlyTarget::Rosetta ? DecodeScalarCompare(instr.Data, instr.Length) : std::nullopt) { + _emitScalarCompare(body, instr, *compare); + return kScalarCompare.Name; + } return nullptr; } diff --git a/core/relinker/codegen/src/x86/ScalarCompareOperands.cpp b/core/relinker/codegen/src/x86/ScalarCompareOperands.cpp new file mode 100644 index 0000000000..c0c31a92e6 --- /dev/null +++ b/core/relinker/codegen/src/x86/ScalarCompareOperands.cpp @@ -0,0 +1,63 @@ +#include +#include + +namespace Codegen { + +namespace { + +constexpr std::uint8_t kVexMap0F = 0x01; +constexpr std::uint8_t kPpF3 = 0x02; +constexpr std::uint8_t kCompareOpcode = 0xC2; +constexpr std::uint8_t kPredicateLeOs = 0x02; + +bool _segmentPrefix(const std::uint8_t prefix) { + using namespace X64OpcodeConstants; + return prefix == PrefixSegCs || prefix == PrefixSegSs || prefix == PrefixSegDs || prefix == PrefixSegEs || prefix == PrefixSegFs || prefix == PrefixSegGs; +} + +} + +std::optional DecodeScalarCompare(const std::uint8_t* data, const std::size_t length) { + using namespace X64OpcodeConstants; + std::size_t pos = 0; + while (pos < length && _segmentPrefix(data[pos])) + ++pos; + std::size_t modRmOffset = 0; + if (pos < length && data[pos] == OneByteVex2) { + if (pos + 4 > length || (data[pos + 1] & 0x03) < kPpF3 || data[pos + 2] != kCompareOpcode) + return std::nullopt; + modRmOffset = pos + 3; + } else if (pos < length && data[pos] == OneByteVex3) { + if (pos + 5 > length || (data[pos + 1] & Vex3MapMask) != kVexMap0F || (data[pos + 2] & 0x03) < kPpF3 || data[pos + 3] != kCompareOpcode) + return std::nullopt; + modRmOffset = pos + 4; + } else { + return std::nullopt; + } + const auto modrm = data[modRmOffset]; + const auto mod = static_cast((modrm >> ModRmModShift) & ModRmModMask); + const auto rm = static_cast(modrm & ModRmRmMask); + auto cursor = modRmOffset + 1; + std::optional sib; + if (mod != ModRmModRegister && rm == ModRmRmSibPresent) { + if (cursor >= length) + return std::nullopt; + sib = data[cursor++]; + } + const bool ripRelative = mod == ModRmModIndirect && rm == ModRmRmRipRelative; + const bool absolute = mod == ModRmModIndirect && sib.has_value() && (*sib & SibBaseMask) == SibBaseDisp32; + const std::size_t displacementSize = mod == ModRmModDisp8 ? Disp8Size : (mod == ModRmModDisp32 || ripRelative || absolute) ? Disp32Size : 0; + if (cursor + displacementSize + ImmSize8 != length || data[length - 1] <= kPredicateLeOs) + return std::nullopt; + ScalarCompareOperands operands; + if (ripRelative) { + std::uint32_t displacement = 0; + for (std::size_t index = 0; index < Disp32Size; ++index) + displacement |= static_cast(data[cursor + index]) << (index * 8); + operands.RipDisplacementOffset = cursor; + operands.RipTarget = static_cast(length) + static_cast(displacement); + } + return operands; +} + +} diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp index 8c4e4e0090..dcb82e697d 100644 --- a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -952,6 +952,61 @@ void converterRosetta() { require(intel.Reports.empty() && intel.Bytes == file, "The Intel target converted RDSEED, RDPID or CLWB"); } +void rosettaScalarCompares() { + const auto rosetta = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + const auto match = [&](const Bytes& bytes) { return rosetta->Match(bytes.data(), bytes.size()); }; + const Bytes compare = {0xC5, 0xEA, 0xC2, 0xCA, 0x03}; + const auto lowered = match(compare); + require(lowered && lowered->Lowering == Codegen::Amd64OnlyLowering::Trampoline && lowered->Optional && !lowered->RelocatableWindow && lowered->InstructionName == "scalar compare", "A Rosetta-hazard scalar compare was not lowered through an optional stub"); + require(lowered->ReturnBranchOffset == 5 && lowered->StubBody.size() == 10 && Bytes(lowered->StubBody.begin(), lowered->StubBody.begin() + 5) == compare && lowered->StubBody[5] == 0xE9 && lowered->Relocations.empty(), "The scalar compare stub does not run the compare unchanged before its return jump"); + const auto unaligned = match(Bytes{0xC5, 0xEA, 0xC2, 0x47, 0x68, 0x03}); + require(unaligned && Bytes(unaligned->StubBody.begin(), unaligned->StubBody.begin() + 6) == Bytes{0xC5, 0xEA, 0xC2, 0x47, 0x68, 0x03} && unaligned->Relocations.empty(), "A memory-form scalar compare does not move unchanged"); + const Bytes rip = {0xC5, 0xEA, 0xC2, 0x0D, 0x00, 0x01, 0x00, 0x00, 0x03}; + const auto ripLowered = match(rip); + require(ripLowered && ripLowered->Relocations.size() == 1 && ripLowered->Relocations[0].DisplacementOffset == 4 && ripLowered->Relocations[0].InstructionEnd == 9 && ripLowered->Relocations[0].SiteTarget == 9 + 0x100, "A RIP-relative scalar compare was not relocated from its place in the site"); + require(Bytes(ripLowered->StubBody.begin() + 4, ripLowered->StubBody.begin() + 8) == Bytes(4, 0), "The RIP-relative scalar compare keeps its displacement in the stub"); + const auto vex3 = match(Bytes{0xC4, 0xE1, 0xEB, 0xC2, 0xCB, 0x03}); + require(vex3 && Bytes(vex3->StubBody.begin(), vex3->StubBody.begin() + 6) == Bytes{0xC4, 0xE1, 0xEB, 0xC2, 0xCB, 0x03}, "A VEX3 VCMPSD with a hazard predicate does not move unchanged"); + for (const Bytes& kept : {Bytes{0xC5, 0xEA, 0xC2, 0xCA, 0x00}, Bytes{0xC5, 0xEA, 0xC2, 0xCA, 0x01}, Bytes{0xC5, 0xEA, 0xC2, 0xCA, 0x02}, Bytes{0xC4, 0xE1, 0xEB, 0xC2, 0xCB, 0x01}}) + require(!match(kept).has_value(), "An EQ_OQ, LT_OS or LE_OS scalar compare must stay inline"); + for (const Bytes& other : {Bytes{0xC5, 0xE9, 0xC2, 0xCA, 0x03}, Bytes{0xC5, 0xE8, 0xC2, 0xCA, 0x03}, Bytes{0xC5, 0xEA, 0x2E, 0xC8}, Bytes{0xC5, 0xEA, 0xC2, 0xCA}, Bytes{0xC4, 0xE2, 0xEB, 0xC2, 0xCB, 0x03}, Bytes{0x62, 0xD1, 0x76, 0x08, 0xC2, 0xCB, 0x03}, Bytes{0xF3, 0x0F, 0xC2, 0xCA, 0x03}}) + require(!match(other).has_value(), "An instruction other than a Rosetta-hazard scalar compare was moved"); + require(!Codegen::MakeAmd64OnlyInstructionMatcher()->Match(compare.data(), compare.size()).has_value(), "A scalar compare was moved for an Intel host"); + const std::vector> pair = {std::span(compare), std::span(rip)}; + const auto sequence = rosetta->MatchSequence(pair, {}); + require(sequence && sequence->InstructionName == "scalar compare" && sequence->ReturnBranchOffset == 14 && sequence->Relocations.size() == 1 && sequence->Relocations[0].SiteTarget == 5 + 9 + 0x100, "A scalar compare after another is not emitted out of line in one stub"); +} + +void converterRosettaScalarCompare() { + const auto converter = Codegen::MakeAmd64OnlyConverter(Codegen::Amd64OnlyTarget::Rosetta); + Bytes file(0x300, 0xCC); + const Bytes text = { + 0xC5, 0xFA, 0x10, 0x15, 0x10, 0x00, 0x00, 0x00, + 0xC5, 0xF8, 0x28, 0xC2, + 0xC5, 0xEA, 0xC2, 0xCA, 0x03, + 0xC5, 0xF8, 0x28, 0x0D, 0x20, 0x00, 0x00, 0x00, + 0xC5, 0xF8, 0x2E, 0xC8, + 0xC3}; + std::copy(text.begin(), text.end(), file.begin() + 0x200); + const auto result = converter->Convert(file, {segmentHeader(text.size())}); + require(result.ReplacedCount == 0 && result.KeptCount == 0 && result.Trampolines.size() == 1 && result.Reports.size() == 1 && result.Reports[0].InstructionName == "scalar compare" && result.Reports[0].Offset == 0x20C && result.Reports[0].Lowering == Codegen::Amd64OnlyLowering::Trampoline, "The Rosetta-hazard compare in the fmax shape was not the only moved instruction"); + const auto& site = result.Trampolines[0]; + require(site.Offset == 0x20C && site.Length == 5 && site.ReturnBranchOffset == 5 && Bytes(site.Body.begin(), site.Body.begin() + 5) == Bytes(text.begin() + 12, text.begin() + 17) && Bytes(site.Body.begin() + 5, site.Body.begin() + 10) == Bytes{0xE9, 0, 0, 0, 0} && site.Relocations.empty(), "The compare stub does not run the compare unchanged and return"); + require(result.Bytes == file, "Moving the compare changed the file image"); + auto branchInside = file; + const Bytes jumped = {0xEB, 0x02, 0xC5, 0xEA, 0xC2, 0xCA, 0x03, 0xC3}; + std::copy(jumped.begin(), jumped.end(), branchInside.begin() + 0x200); + const auto kept = converter->Convert(branchInside, {segmentHeader(jumped.size())}); + require(kept.Trampolines.empty() && kept.KeptCount == 1 && kept.Reports.size() == 1 && kept.Reports[0].Lowering == Codegen::Amd64OnlyLowering::Kept && kept.Bytes == branchInside, "A branch into the compare did not keep it native"); + auto ripSource = file; + const Bytes ripText = {0xC5, 0xEA, 0xC2, 0x0D, 0x00, 0x01, 0x00, 0x00, 0x03, 0xC3}; + std::copy(ripText.begin(), ripText.end(), ripSource.begin() + 0x200); + const auto ripMoved = converter->Convert(ripSource, {segmentHeader(ripText.size())}); + require(ripMoved.Trampolines.size() == 1 && ripMoved.Trampolines[0].Relocations.size() == 1 && ripMoved.Trampolines[0].Relocations[0].SiteTarget == 9 + 0x100, "A RIP-relative compare source was not relocated"); + const auto intel = Codegen::MakeAmd64OnlyConverter()->Convert(file, {segmentHeader(text.size())}); + require(intel.Reports.empty() && intel.Bytes == file, "The Intel target moved a scalar compare"); +} + void reciprocalOperands() { const auto vex2 = Codegen::DecodeReciprocal(Bytes{0xC5, 0xF8, 0x52, 0xD5}.data(), 4); require(vex2 && vex2->Operation == Codegen::ReciprocalOperation::ReciprocalSquareRoot && vex2->Destination == 2 && vex2->Source == 5, "VEX2 VRSQRTPS was not decoded"); @@ -1672,6 +1727,8 @@ int main() { converterClzero(); rosettaSubstitutions(); converterRosetta(); + rosettaScalarCompares(); + converterRosettaScalarCompare(); wideStoreSplit(); converterWideStoreWindows(); converterJumpTableTargets(); diff --git a/core/relinker/relinker/src/pipeline/Conversion.cpp b/core/relinker/relinker/src/pipeline/Conversion.cpp index 18c4012cb9..a0b9e36493 100644 --- a/core/relinker/relinker/src/pipeline/Conversion.cpp +++ b/core/relinker/relinker/src/pipeline/Conversion.cpp @@ -56,7 +56,7 @@ ConversionOutput Conversion::Convert(ConversionInput input, const ConversionOpti sourceBytes = std::move(converted.Bytes); trampolines = std::move(converted.Trampolines); std::map stubsByName; - const auto counted = [](const std::string& name) { return name == "VRSQRTPS" || name == "VRCPPS" || name == "256-bit store"; }; + const auto counted = [](const std::string& name) { return name == "VRSQRTPS" || name == "VRCPPS" || name == "256-bit store" || name == "scalar compare"; }; for (const auto& report : converted.Reports) { if (report.Lowering == Codegen::Amd64OnlyLowering::Kept) { std::ostringstream message; message << "Intel substitution: " << report.InstructionName << " at 0x" << std::hex << report.Offset << std::dec << " (" << report.OriginalLength << " bytes) kept: no room for a jump\n"; inputProgress.Message(message.str()); }