From d70efacd6eae43107717f81882c773dfed557665 Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Sun, 25 Oct 2026 17:49:04 +0530 Subject: [PATCH 1/6] feat(relinker): lower SHLX, SHRX, SARX and RORX under --to-rosetta --- core/relinker/CMakeLists.txt | 8 + .../include/codegen/x86/ScalarLowering.hpp | 4 +- .../include/codegen/x86/ShiftLowering.hpp | 31 +++ .../src/x86/Amd64OnlyInstructionMatcher.cpp | 7 + .../codegen/src/x86/ShiftLowering.cpp | 127 +++++++++ .../codegen/tests/LoweringExecution.hpp | 10 +- .../codegen/tests/ShiftLoweringTests.cpp | 245 ++++++++++++++++++ 7 files changed, 428 insertions(+), 4 deletions(-) create mode 100644 core/relinker/codegen/include/codegen/x86/ShiftLowering.hpp create mode 100644 core/relinker/codegen/src/x86/ShiftLowering.cpp create mode 100644 core/relinker/codegen/tests/ShiftLoweringTests.cpp diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 4432faa105..f708e064af 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -55,6 +55,7 @@ add_library(relinker_core STATIC ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/MaskedStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ShiftLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/RelrRelocations.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp @@ -106,6 +107,11 @@ if(BUILD_TESTING) add_test(NAME bls_lowering COMMAND bls_lowering_tests) set_tests_properties(bls_lowering PROPERTIES TIMEOUT 20) + add_test_executable(shift_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/ShiftLoweringTests.cpp) + target_link_libraries(shift_lowering_tests PRIVATE relinker_core) + add_test(NAME shift_lowering COMMAND shift_lowering_tests) + set_tests_properties(shift_lowering PROPERTIES TIMEOUT 60) + add_test_executable(masked_store_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/MaskedStoreLoweringTests.cpp) target_link_libraries(masked_store_lowering_tests PRIVATE relinker_core) add_test(NAME masked_store_lowering COMMAND masked_store_lowering_tests) @@ -258,6 +264,7 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/MaskedStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ShiftLowering.cpp ) target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows) if(BUILD_TESTING) @@ -548,6 +555,7 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/MaskedStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ShiftLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp diff --git a/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp b/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp index 363e8f4181..5627419ebc 100644 --- a/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp +++ b/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp @@ -72,9 +72,9 @@ inline std::uint8_t Scratch(std::array& used, const char* instruction) throw CodegenException(std::string(instruction) + " has no scratch register"); } -inline void Memory(StubBodyBuilder& body, std::uint8_t opcode, bool wide, const MemoryOperand& memory, std::uint8_t reg, const char* instruction) { +inline void Memory(StubBodyBuilder& body, std::uint8_t opcode, bool wide, const MemoryOperand& memory, std::uint8_t reg, const char* instruction, std::int32_t frame = FrameSize) { auto displacement = static_cast(memory.Displacement); - if (memory.StackBase) displacement += FrameSize; + if (memory.StackBase) displacement += frame; const bool address32 = std::find(memory.Prefixes.begin(), memory.Prefixes.end(), 0x67) != memory.Prefixes.end(); if (!address32 && displacement > std::numeric_limits::max()) throw CodegenException(std::string(instruction) + " stack displacement does not fit after spilling"); diff --git a/core/relinker/codegen/include/codegen/x86/ShiftLowering.hpp b/core/relinker/codegen/include/codegen/x86/ShiftLowering.hpp new file mode 100644 index 0000000000..abc086f8e1 --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/ShiftLowering.hpp @@ -0,0 +1,31 @@ +#ifndef CODEGEN_X86_SHIFTLOWERING_HPP +#define CODEGEN_X86_SHIFTLOWERING_HPP + +#include +#include + +namespace Codegen { + +enum class ShiftOperation : std::uint8_t { + Left, + LogicalRight, + ArithmeticRight, + RotateRight, +}; + +struct ShiftOperands { + ShiftOperation Operation; + std::uint8_t Destination; + std::uint8_t Source; + std::uint8_t Count; + bool Wide; + std::optional Memory; +}; + +[[nodiscard]] const char* ShiftName(ShiftOperation operation); +[[nodiscard]] std::optional DecodeShift(const std::uint8_t* data, std::size_t length); +void EmitShift(StubBodyBuilder& body, const ShiftOperands& operands); + +} + +#endif diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index 73c91595e3..496121d571 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -259,6 +260,8 @@ std::optional Amd64OnlyInstructionMatcher::_matchRosetta(const D return _trampoline("ANDN", instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitAndn(body, *andn); })); if (const auto bls = DecodeBls(instr.Data, instr.Length)) return _trampoline(BlsName(bls->Operation), instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitBls(body, *bls); })); + if (const auto shift = DecodeShift(instr.Data, instr.Length)) + return _trampoline(ShiftName(shift->Operation), instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitShift(body, *shift); })); if (instr.IsRdseed()) { std::vector replacement(instr.Data, instr.Data + instr.Length); replacement[instr.OpcodeOffset() + 2] &= static_cast(~(1u << X64OpcodeConstants::ModRmRegShift)); @@ -334,6 +337,10 @@ const char* Amd64OnlyInstructionMatcher::_emitOutOfLine(StubBodyBuilder& body, c EmitBls(body, *bls); return BlsName(bls->Operation); } + if (const auto shift = _target == Amd64OnlyTarget::Rosetta ? DecodeShift(instr.Data, instr.Length) : std::nullopt) { + EmitShift(body, *shift); + return ShiftName(shift->Operation); + } if (_target == Amd64OnlyTarget::Rosetta && instr.IsRdtscp()) { body.Raw(_rdtscpReplacement(instr)); return kRdtscp.Name; diff --git a/core/relinker/codegen/src/x86/ShiftLowering.cpp b/core/relinker/codegen/src/x86/ShiftLowering.cpp new file mode 100644 index 0000000000..16dab8f686 --- /dev/null +++ b/core/relinker/codegen/src/x86/ShiftLowering.cpp @@ -0,0 +1,127 @@ +#include +#include + +namespace Codegen { + +using namespace ScalarLowering; + +namespace { + +constexpr std::uint8_t kCountRegister = 1; +constexpr std::int32_t kResultSlot = 0; +constexpr std::int32_t kCountSlot = 8; +constexpr std::int32_t kFlagsSlot = 16; +constexpr std::int32_t kStackResultSlot = 24; +constexpr std::uint8_t kPushFlags = 0x9C; +constexpr std::uint8_t kPopFlags = 0x9D; +constexpr std::int32_t kFrame = FrameSize + 16; + +std::uint8_t _shiftDigit(const ShiftOperation operation) { + switch (operation) { + case ShiftOperation::Left: return 4; + case ShiftOperation::LogicalRight: return 5; + case ShiftOperation::ArithmeticRight: return 7; + case ShiftOperation::RotateRight: return 1; + } + throw CodegenException("unknown shift operation"); +} + +void _loadSource(StubBodyBuilder& body, const ShiftOperands& operands, const std::uint8_t result, const char* name) { + if (operands.Memory) + Memory(body, 0x8B, operands.Wide, *operands.Memory, result, name, kFrame); + else if (operands.Source == Stack) + StackSlot(body, 0x8D, true, result, kFrame); + else + Register(body, 0x8B, operands.Wide, result, operands.Source); +} + +} + +const char* ShiftName(const ShiftOperation operation) { + switch (operation) { + case ShiftOperation::Left: return "SHLX"; + case ShiftOperation::LogicalRight: return "SHRX"; + case ShiftOperation::ArithmeticRight: return "SARX"; + case ShiftOperation::RotateRight: return "RORX"; + } + throw CodegenException("unknown shift operation"); +} + +std::optional DecodeShift(const std::uint8_t* data, const std::size_t length) { + std::size_t vex = 0; + while (vex < length && LegacyPrefix(data[vex])) ++vex; + if (vex + 5 > length || data[vex] != 0xC4) return std::nullopt; + const auto map = static_cast(data[vex + 1] & 0x1F); + const auto pp = static_cast(data[vex + 2] & 3); + ShiftOperation operation; + if (map == 2 && data[vex + 3] == 0xF7 && pp != 0) + operation = pp == 1 ? ShiftOperation::Left : pp == 2 ? ShiftOperation::ArithmeticRight : ShiftOperation::LogicalRight; + else if (map == 3 && data[vex + 3] == 0xF0 && pp == 3) + operation = ShiftOperation::RotateRight; + else + return std::nullopt; + const auto name = ShiftName(operation); + if ((data[vex + 2] & 4) != 0) throw CodegenException(std::string(name) + " requires VEX.L=0"); + const auto extensions = static_cast(~data[vex + 1]); + const auto modrm = data[vex + 4]; + const auto vvvv = static_cast((~data[vex + 2] >> 3) & 15); + if (operation == ShiftOperation::RotateRight && vvvv != 0) throw CodegenException("RORX requires VEX.vvvv=1111"); + ShiftOperands operands{ + operation, + static_cast(((modrm >> 3) & 7) | ((extensions & 0x80) >> 4)), + static_cast((modrm & 7) | ((extensions & 0x20) >> 2)), + vvvv, + (data[vex + 2] & 0x80) != 0, std::nullopt}; + std::size_t end = vex + 5; + if ((modrm >> 6) != 3) { + const auto rex = static_cast(((extensions & 0x20) >> 5) | ((extensions & 0x40) >> 5)); + operands.Memory = DecodeMemoryOperand(data, length, vex + 4, rex, {data, data + vex}); + end = vex + 4 + operands.Memory->EncodedSize; + } + if (operation == ShiftOperation::RotateRight) { + if (end >= length) throw CodegenException("RORX is missing its immediate"); + operands.Count = data[end]; + ++end; + } + if (end != length) throw CodegenException(std::string(name) + " has trailing bytes"); + return operands; +} + +void EmitShift(StubBodyBuilder& body, const ShiftOperands& operands) { + const auto name = ShiftName(operands.Operation); + const bool rotate = operands.Operation == ShiftOperation::RotateRight; + auto used = operands.Memory ? UsedRegisters({operands.Destination, operands.Count, kCountRegister}, operands.Memory) + : UsedRegisters({operands.Destination, operands.Source, operands.Count, kCountRegister}, std::nullopt); + const auto result = Scratch(used, name); + StackSlot(body, 0x8D, true, Stack, -kFrame); + StackSlot(body, 0x89, true, result, kResultSlot); + if (!rotate) StackSlot(body, 0x89, true, kCountRegister, kCountSlot); + body.Raw(std::array{kPushFlags}); + StackSlot(body, 0x8F, false, 0, kFlagsSlot); + _loadSource(body, operands, result, name); + if (rotate) { + Register(body, 0xC1, operands.Wide, _shiftDigit(operands.Operation), result); + body.Raw(std::array{operands.Count}); + } else { + if (operands.Count == Stack) + StackSlot(body, 0x8D, true, kCountRegister, kFrame); + else if (operands.Count != kCountRegister) + Register(body, 0x8B, true, kCountRegister, operands.Count); + Register(body, 0xD3, operands.Wide, _shiftDigit(operands.Operation), result); + } + if (operands.Destination == Stack) + StackSlot(body, 0x89, operands.Wide, result, kStackResultSlot); + else if (!rotate && operands.Destination == kCountRegister) + StackSlot(body, 0x89, true, result, kCountSlot); + else + Register(body, 0x8B, operands.Wide, operands.Destination, result); + StackSlot(body, 0xFF, false, 6, kFlagsSlot); + body.Raw(std::array{kPopFlags}); + if (!rotate) StackSlot(body, 0x8B, true, kCountRegister, kCountSlot); + StackSlot(body, 0x8B, true, result, kResultSlot); + StackSlot(body, 0x8D, true, Stack, kFrame); + if (operands.Destination == Stack) + StackSlot(body, 0x8B, operands.Wide, Stack, kStackResultSlot - kFrame); +} + +} diff --git a/core/relinker/codegen/tests/LoweringExecution.hpp b/core/relinker/codegen/tests/LoweringExecution.hpp index c020a91375..28eb241865 100644 --- a/core/relinker/codegen/tests/LoweringExecution.hpp +++ b/core/relinker/codegen/tests/LoweringExecution.hpp @@ -25,6 +25,7 @@ struct State { std::array Output{}; std::uint64_t Flags = 0; std::array RedZone{}; + std::uint64_t InputFlags = 0; }; inline constexpr std::uint64_t kSentinel = 0xBADC0FFEE0DDF00Dull; @@ -33,9 +34,12 @@ inline constexpr std::uint64_t kCarry = 0x1u; inline constexpr std::uint64_t kZero = 0x40u; inline constexpr std::uint64_t kSign = 0x80u; +inline constexpr std::uint64_t kBaseFlags = 0x202u; + inline State InitialState() { State state; for (std::size_t reg = 0; reg < 16; ++reg) state.Input[reg] = 0xABCD123456789000ull + reg * 0x1234567ull; + state.InputFlags = kBaseFlags | kFlagMask; return state; } @@ -45,12 +49,14 @@ class Execution { public: using Emit = std::function; - Execution(const Bytes& site, std::uint64_t memoryValue, bool ripRelative, const Emit& emit) { + Execution(const Bytes& site, std::uint64_t memoryValue, bool ripRelative, const Emit& emit, std::size_t trailing = 0) { for (auto reg : {3, 5, 12, 13, 14, 15}) push(static_cast(reg)); memory(0x8D, 4, 4, -128); rip({0x48, 0x89, 0x3D}, 0); rip({0x48, 0x89, 0x25}, 8); memory(0x89, 4, 7, 32); + memory(0xFF, 6, 7, 49 * 8); + bytes.push_back(0x9D); bytes.insert(bytes.end(), {0x48, 0xB8}); append64(kSentinel); for (int displacement = -128; displacement < 0; displacement += 8) @@ -64,7 +70,7 @@ public: auto relocated = site; if (ripRelative) { const auto displacement = static_cast(4096 + 24 - (siteOffset + site.size())); - std::memcpy(relocated.data() + relocated.size() - 4, &displacement, sizeof(displacement)); + std::memcpy(relocated.data() + relocated.size() - 4 - trailing, &displacement, sizeof(displacement)); } Codegen::StubBodyBuilder builder; emit(builder, relocated); diff --git a/core/relinker/codegen/tests/ShiftLoweringTests.cpp b/core/relinker/codegen/tests/ShiftLoweringTests.cpp new file mode 100644 index 0000000000..850330713c --- /dev/null +++ b/core/relinker/codegen/tests/ShiftLoweringTests.cpp @@ -0,0 +1,245 @@ +#include "LoweringExecution.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using LoweringExecution::Bytes; +using Codegen::ShiftOperation; + +constexpr std::array kShifts{ShiftOperation::Left, ShiftOperation::LogicalRight, ShiftOperation::ArithmeticRight}; +constexpr std::array kOperations{ShiftOperation::Left, ShiftOperation::LogicalRight, ShiftOperation::ArithmeticRight, ShiftOperation::RotateRight}; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +std::uint8_t pp(ShiftOperation operation) { + switch (operation) { + case ShiftOperation::Left: return 1; + case ShiftOperation::ArithmeticRight: return 2; + case ShiftOperation::LogicalRight: return 3; + case ShiftOperation::RotateRight: return 3; + } + return 0; +} + +Bytes shift(ShiftOperation operation, bool wide, std::uint8_t dst, std::uint8_t src, std::uint8_t count) { + return {0xC4, static_cast(0xE2 ^ ((dst & 8) << 4) ^ ((src & 8) << 2)), + static_cast((wide ? 0x80 : 0) | ((~count & 15) << 3) | pp(operation)), 0xF7, + static_cast(0xC0 | ((dst & 7) << 3) | (src & 7))}; +} + +Bytes rotate(bool wide, std::uint8_t dst, std::uint8_t src, std::uint8_t immediate) { + return {0xC4, static_cast(0xE3 ^ ((dst & 8) << 4) ^ ((src & 8) << 2)), + static_cast((wide ? 0x80 : 0) | 0x78 | 3), 0xF0, + static_cast(0xC0 | ((dst & 7) << 3) | (src & 7)), immediate}; +} + +Bytes memorySite(ShiftOperation operation, bool wide, const Bytes& tail, std::uint8_t immediate) { + const bool rorx = operation == ShiftOperation::RotateRight; + Bytes site{0xC4, static_cast(rorx ? 0xE3 : 0xE2), static_cast((wide ? 0x80 : 0) | (rorx ? 0x78 : 0x70) | pp(operation)), static_cast(rorx ? 0xF0 : 0xF7)}; + site.insert(site.end(), tail.begin(), tail.end()); + if (rorx) site.push_back(immediate); + return site; +} + +void decoding() { + for (const auto operation : kShifts) { + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (std::uint8_t src = 0; src < 16; ++src) { + for (std::uint8_t count = 0; count < 16; ++count) { + const auto bytes = shift(operation, wide, dst, src, count); + const auto operands = Codegen::DecodeShift(bytes.data(), bytes.size()); + require(operands && operands->Operation == operation && operands->Wide == wide && operands->Destination == dst && operands->Source == src && operands->Count == count && !operands->Memory, "shift register decoding changed an operand"); + } + } + } + } + } + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (std::uint8_t src = 0; src < 16; ++src) { + for (const std::uint8_t immediate : std::array{0, 1, 63, 255}) { + const auto bytes = rotate(wide, dst, src, immediate); + const auto operands = Codegen::DecodeShift(bytes.data(), bytes.size()); + require(operands && operands->Operation == ShiftOperation::RotateRight && operands->Wide == wide && operands->Destination == dst && operands->Source == src && operands->Count == immediate && !operands->Memory, "RORX register decoding changed an operand"); + } + } + } + } + const Bytes shiftMemory{0x64, 0x67, 0xC4, 0x02, 0xA9, 0xF7, 0x8C, 0xAC, 0x10, 0, 0, 0}; + auto operands = Codegen::DecodeShift(shiftMemory.data(), shiftMemory.size()); + require(operands && operands->Operation == ShiftOperation::Left && operands->Destination == 9 && operands->Count == 10 && operands->Memory && operands->Memory->RexIndexBase == 3 && operands->Memory->Prefixes == Bytes{0x64, 0x67}, "shift memory extensions or prefixes were lost"); + const Bytes rotateMemory{0xC4, 0x03, 0xFB, 0xF0, 0x8C, 0xAC, 0x10, 0, 0, 0, 0x21}; + operands = Codegen::DecodeShift(rotateMemory.data(), rotateMemory.size()); + require(operands && operands->Operation == ShiftOperation::RotateRight && operands->Destination == 9 && operands->Count == 0x21 && operands->Memory && operands->Memory->InstructionLength == rotateMemory.size(), "RORX memory operand or immediate was lost"); + for (const Bytes bytes : {Bytes{0x90}, Bytes{0xC4, 0xE2, 0xF1}, Bytes{0xC4, 0xE1, 0xF1, 0xF7, 0xC0}, Bytes{0xC4, 0xE2, 0xF0, 0xF7, 0xC0}, Bytes{0xC4, 0xE3, 0xFA, 0xF0, 0xC0, 0}, Bytes{0xC4, 0xE2, 0xFB, 0xF0, 0xC0, 0}}) + require(!Codegen::DecodeShift(bytes.data(), bytes.size()), "Unrelated, truncated, BEXTR or wrong-map opcode matched a shift"); + for (const Bytes bytes : {Bytes{0xC4, 0xE2, 0xF5, 0xF7, 0xC0}, Bytes{0xC4, 0xE2, 0xF1, 0xF7, 0xC0, 0x90}, Bytes{0xC4, 0xE2, 0xF1, 0xF7, 0x84}, Bytes{0xC4, 0xE3, 0xF3, 0xF0, 0xC0, 0}, Bytes{0xC4, 0xE3, 0xFB, 0xF0, 0xC0}, Bytes{0xC4, 0xE3, 0xFB, 0xF0, 0xC0, 0, 0}}) { + bool rejected = false; + try { (void)Codegen::DecodeShift(bytes.data(), bytes.size()); } + catch (const Codegen::CodegenException&) { rejected = true; } + require(rejected, "Malformed shift was accepted"); + } + const Bytes overflow{0xC4, 0xE2, 0xF1, 0xF7, 0x8C, 0x24, 0xFF, 0xFF, 0xFF, 0x7F}; + bool rejected = false; + try { + Codegen::StubBodyBuilder body; + Codegen::EmitShift(body, *Codegen::DecodeShift(overflow.data(), overflow.size())); + } catch (const Codegen::CodegenException&) { rejected = true; } + require(rejected, "An overflowing stack displacement was silently wrapped"); +} + +void matching() { + const auto intel = Codegen::MakeAmd64OnlyInstructionMatcher(); + const auto rosetta = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + for (const auto operation : kOperations) { + const auto bytes = operation == ShiftOperation::RotateRight ? rotate(true, 0, 6, 7) : shift(operation, true, 0, 6, 2); + const auto name = Codegen::ShiftName(operation); + require(!intel->Match(bytes.data(), bytes.size()), std::string("Intel mode rewrote ") + name); + const auto match = rosetta->Match(bytes.data(), bytes.size()); + require(match && match->InstructionName == name && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline && !match->Optional, std::string("Rosetta mode did not require a trampoline for ") + name); + const std::array, 2> sequence{bytes, bytes}; + require(rosetta->MatchSequence(sequence, {}).has_value(), std::string("A sequence could not move out of line for ") + name); + } +} + +#if LOWERING_EXECUTION_AVAILABLE + +using LoweringExecution::Execution; +using LoweringExecution::State; + +Execution executable(const Bytes& site, std::uint64_t memoryValue) { + const auto operands = Codegen::DecodeShift(site.data(), site.size()); + require(operands.has_value(), "Execution site was not a shift"); + return Execution(site, memoryValue, operands->Memory && operands->Memory->RipRelative, [](Codegen::StubBodyBuilder& body, const Bytes& relocated) { + Codegen::EmitShift(body, *Codegen::DecodeShift(relocated.data(), relocated.size())); + }, operands->Operation == ShiftOperation::RotateRight ? 1 : 0); +} + +std::uint64_t expected(const Codegen::ShiftOperands& operands, std::uint64_t source, std::uint64_t count) { + const unsigned bits = operands.Wide ? 64 : 32; + const auto mask = operands.Wide ? ~std::uint64_t{0} : 0xFFFFFFFFull; + source &= mask; + const unsigned amount = static_cast(count & (bits - 1)); + switch (operands.Operation) { + case ShiftOperation::Left: return (source << amount) & mask; + case ShiftOperation::LogicalRight: return source >> amount; + case ShiftOperation::ArithmeticRight: { + const bool negative = (source >> (bits - 1)) != 0; + const auto shifted = source >> amount; + return negative && amount != 0 ? (shifted | (mask << (bits - amount))) & mask : shifted; + } + case ShiftOperation::RotateRight: return amount == 0 ? source : ((source >> amount) | (source << (bits - amount))) & mask; + } + return 0; +} + +void check(const State& state, const Codegen::ShiftOperands& operands, std::uint64_t source, std::uint64_t count) { + const auto name = Codegen::ShiftName(operands.Operation); + const auto result = expected(operands, source, count); + for (std::size_t reg = 0; reg < 16; ++reg) { + const auto wanted = reg == operands.Destination ? result : state.Input[reg]; + if (state.Output[reg] != wanted) + throw std::runtime_error(std::string(name) + (operands.Wide ? " 64" : " 32") + " dst " + std::to_string(operands.Destination) + " src " + std::to_string(operands.Source) + " count " + std::to_string(operands.Count) + (operands.Memory ? " memory" : "") + ": register " + std::to_string(reg) + " is " + std::to_string(state.Output[reg]) + ", wanted " + std::to_string(wanted) + " (source " + std::to_string(source) + ", count value " + std::to_string(count) + ")"); + } + require((state.Flags & LoweringExecution::kFlagMask) == (state.InputFlags & LoweringExecution::kFlagMask), std::string(name) + " did not preserve the flags"); + for (const auto value : state.RedZone) require(value == LoweringExecution::kSentinel, std::string(name) + " overwrote the caller's red zone"); +} + +void run(const Execution& execution, const Codegen::ShiftOperands& operands, State state, std::uint64_t memoryValue) { + for (const auto flags : {LoweringExecution::kBaseFlags | LoweringExecution::kFlagMask, LoweringExecution::kBaseFlags}) { + state.InputFlags = flags; + auto trial = state; + execution.Run(trial); + const auto source = operands.Memory ? memoryValue : trial.Input[operands.Source]; + const auto count = operands.Operation == ShiftOperation::RotateRight ? operands.Count : trial.Input[operands.Count]; + check(trial, operands, source, count); + } +} + +void execution() { + constexpr std::array values{0, 1, 0x80000000ull, 0x8000000000000000ull, 0xFFFFFFFFFFFFFFFFull, 0xABCD123456789000ull}; + constexpr std::array counts{0, 1, 31, 32, 63, 64, 0xFF}; + for (const auto operation : kShifts) { + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (const auto pair : std::array, 9>{{{6, 2}, {dst, 10}, {11, dst}, {dst, dst}, {4, 9}, {9, 4}, {1, 6}, {6, 1}, {1, 1}}}) { + const auto site = shift(operation, wide, dst, pair.first, pair.second); + const auto operands = *Codegen::DecodeShift(site.data(), site.size()); + const auto execution = executable(site, 0); + for (const auto value : values) { + for (const auto count : counts) { + auto state = LoweringExecution::InitialState(); + state.Input[pair.first] = value; + state.Input[pair.second] = count; + run(execution, operands, state, 0); + } + } + } + } + } + } + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (const std::uint8_t src : std::array{6, dst, 4, 10}) { + for (const std::uint8_t immediate : std::array{0, 1, 31, 32, 63, 255}) { + const auto site = rotate(wide, dst, src, immediate); + const auto operands = *Codegen::DecodeShift(site.data(), site.size()); + const auto execution = executable(site, 0); + for (const auto value : values) { + auto state = LoweringExecution::InitialState(); + state.Input[src] = value; + run(execution, operands, state, 0); + } + } + } + } + } + for (const auto operation : kOperations) { + for (bool wide : {false, true}) { + for (const Bytes tail : {Bytes{0x06}, Bytes{0x46, 0x10}, Bytes{0x84, 0x24, 0x40, 0, 0, 0}, Bytes{0x04, 0xCA}, Bytes{0x85, 0x10, 0, 0, 0}, Bytes{0x05, 0, 0, 0, 0}}) { + for (const auto value : std::array{0, 0xFEDCBA9876543210ull, 0xFFFFFFFF00000000ull}) { + const auto site = memorySite(operation, wide, tail, 13); + const auto operands = *Codegen::DecodeShift(site.data(), site.size()); + const auto execution = executable(site, value); + auto state = LoweringExecution::InitialState(); + state.Input[6] = reinterpret_cast(execution.Memory()) - (tail[0] == 0x46 ? 16 : 0); + state.Input[2] = reinterpret_cast(execution.Memory()) - 3 * 8; + state.Input[1] = 3; + state.Input[5] = reinterpret_cast(execution.Memory()) - 16; + state.Output[0] = value; + run(execution, operands, state, value); + } + } + } + } +} +#else +void execution() { std::cout << "shift execution checks require a Linux/macOS x86-64 build\n"; } +#endif + +} + +int main() { + try { + decoding(); + matching(); + execution(); + std::cout << "shift lowering checks passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} From bc6710735a46e78c03c77df816dde8e2dab90de5 Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Sun, 25 Oct 2026 17:50:04 +0530 Subject: [PATCH 2/6] docs(relinker): record the shift lowering --- docs/dev/debt/relinker.md | 7 ++++--- docs/user/USAGE.md | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index 746cb8a474..1464883fd2 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -46,14 +46,15 @@ - In macOS output, a guest instruction that addresses memory through GS becomes `UD2` and NOP padding of its own length ([macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)): the host owns GS, which holds the pthread TSD base the guest TLS stubs read, and the guest GS base that FreeBSD's `sysarch(AMD64_SET_GSBASE)` would set is not emulated. A converted module runs until such an instruction is reached, which then faults. Prospero Win (PPSA99995) has six in its Wine `ntdll`. - macOS output still writes pointer fixups into executable segments. `dyld_info -fixups` reports each as `pointer rebase is in executable segment`, next to `pointer rebase is in non-writable segment` for a fixup in a segment without write access, which the [macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp) rejects. A segment is executable and writable when its `PT_LOAD` is `PF_R|PF_W|PF_X`, or when a writable `PT_LOAD` shares a 4 KiB page with an executable one, because each page takes the union of the flags of the `PT_LOAD`s that cover it. dyld on an M1 Pro running macOS 27.0.1 (x86-64 output under Rosetta 2, `LC_BUILD_VERSION` minos and sdk 11.0) loads such an executable, applies the fixup and runs it; a fixup into a `PF_R|PF_X` segment ended the process there with SIGBUS in `dyld4::Loader::applyFixupsGeneric` before `main`, not with a refusal. Not measured for a dylib or under another macOS or SDK version. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. -- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR and MULX are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. -- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, MULX, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. +- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX and RORX are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. +- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. - `--to-intel` replaces the approximate legacy SSE and VEX.128 register forms of RSQRTPS/RCPPS and VRSQRTPS/VRCPPS with [correctly rounded](../../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). Legacy SSE forms preserve the destination's upper YMM bits; VEX forms clear them. The 256-bit, scalar and memory forms keep the native instruction, as does a site shorter than 5 bytes with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). - XSAVEC raises an illegal instruction under Rosetta and is not converted, because it writes the compacted format. XSAVEOPT, which also faults there, is rewritten to XSAVE in place by the [matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) under `--to-rosetta` (`0F AE /6` becomes `/4`; prefixes and operand are kept). The two differ only in the init and modified optimizations, which let XSAVEOPT skip writing unchanged or initial state ([Intel SDM volume 1](https://cdrdv2-public.intel.com/812382/253665-sdm-vol-1.pdf), chapter 13), so XRSTOR restores the same state from either area. Measured on one M5 with macOS 27.0.1: XSAVE and XRSTOR handle x87, SSE and AVX state in the standard format, although CPUID leaf 1 reports neither XSAVE nor OSXSAVE and leaf 0xD sub-leaf 1 reports neither XSAVEOPT nor XSAVEC. XSAVE also executes on an M5 with macOS 26.4, where it writes an XSTATE_BV of 0x7 and XGETBV returns the same x87, SSE and AVX mask although leaf 1 still reports neither XSAVE nor OSXSAVE; XRSTOR was not exercised there. The converter tests' x86-64 build executes the replacement XSAVE under Rosetta on that host. - `--trace-imports` ([Windows entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp), [Linux entry stub](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp), [macOS entry](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)) leaves every import that is not a PLT slot or a function GOT slot untraced, and the same for a function reached through any other relocation, `sceKernelDlsym`, a weak import, or a module loaded at run time. On Linux the entry stub makes the page of every traced slot readable, writable and executable with `mprotect` before it stores the thunk, so a slot inside `PT_GNU_RELRO` can be replaced; those pages stay writable, and data pages become executable, while tracing is on. On macOS the stub writes the thunk into the bound slot and does not change its protection, because that slot stays in a writable segment - [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. -- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. BEXTR, BZHI, PDEP, PEXT, RORX, SARX, SHLX and SHRX are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. BEXTR, BZHI, MULX, PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. BEXTR (the `pp=0` form of the same opcode), BZHI, MULX, PDEP and PEXT are not lowered. Balatro (PPSA21402, #1142) stopped on `shlxq` after the ANDN lowering; this lowering has not been run on that title. The execution test runs on an x86-64 host, natively or under Rosetta, and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. - Relative pointer-width unwind pointer decoding in the [exception frame reader](../../../core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp) (`DW_EH_PE_absptr` and `DW_EH_PE_udata8`) wraps modulo 2^64 to accommodate backward offsets, as standard DWARF unwinders add them. Console unwind table validation for out-of-range displacements is unmeasured. Unsigned offsets narrower than pointer width (`DW_EH_PE_udata2` and `DW_EH_PE_udata4`) cannot express negative displacements and throw when address addition overflows. diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index 94e7c27b2a..d63dd7299e 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -74,7 +74,7 @@ All switches are disabled by default. `unused-filter` defaults to `0`; `--rpath` | `--linux-fixed-base
` | Load the Linux executable at this fixed address instead of the default `0x400000`, given in decimal or hexadecimal (`0x`): every virtual address of the image is offset by it, so virtual address 0 is placed at the address, as the console places an executable. Requires a 64 KiB-aligned address from `0x10000` through `0x7fffffff0000` that is a multiple of every `PT_LOAD` alignment, and an image that ends in the user address space. The executable is written as `ET_EXEC`, without ASLR, as it is at the default address. Bundled modules keep their normal placement. Conflicts with `--windows`, `--macos` and `--lazy-binding`. | | `--macos-fixed-base
` | Map the lowest ELF load page of the macOS executable at this fixed address, given in decimal or hexadecimal (`0x`). Requires `--macos`, a 64 KiB-aligned address from `0x20000` through `0x7fffffff0000`, and an image that ends in the macOS user address space. `__PAGEZERO` ends 64 KiB below the address and the Mach-O header occupies those 64 KiB. The executable has no ASLR; an address the process cannot map fails to load. Bundled modules keep their normal placement. | | `--to-intel` | Convert supported AMD-only instructions in the executable and bundled modules. Unsupported instructions or unreachable conversion stubs cause an error. | -| `--to-rosetta` | Convert as `--to-intel` does and also lower what Rosetta lacks: ANDN becomes scalar NOT/AND, BLSI, BLSMSK and BLSR become scalar LEA/AND/XOR/NEG sequences, RDSEED becomes RDRAND, CLWB a NOP, XSAVEOPT becomes XSAVE, and RDPID and the ECX of RDTSCP return processor 0. Implies `--to-intel`. Independent of `--macos`: `--macos` alone leaves these instructions as they are. Conflicts with `--windows`. | +| `--to-rosetta` | Convert as `--to-intel` does and also lower what Rosetta lacks: ANDN becomes scalar NOT/AND, BLSI, BLSMSK and BLSR become scalar LEA/AND/XOR/NEG sequences, SHLX, SHRX, SARX and RORX become flag-preserving scalar shifts and rotates, RDSEED becomes RDRAND, CLWB a NOP, XSAVEOPT becomes XSAVE, and RDPID and the ECX of RDTSCP return processor 0. Implies `--to-intel`. Independent of `--macos`: `--macos` alone leaves these instructions as they are. Conflicts with `--windows`. | | `unused-filter=0` | Keep all imported NID references. | | `unused-filter=1` | Filter unused non-PLT imports using control-flow and GOT access analysis, treating address-taken code as reachable; preserve PLT imports. | | `unused-filter=2` | Apply strict unused-import analysis and compact the PLT. Unsupported analysis cases cause an error. | From 5d3b53fc831fc047e3a05ac7700c03a97b2176bd Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Sun, 25 Oct 2026 19:12:08 +0530 Subject: [PATCH 3/6] fix(relinker): keep the RORX immediate out of the shift lowering's register set --- .../codegen/include/codegen/x86/ScalarLowering.hpp | 5 ++++- core/relinker/codegen/src/x86/ShiftLowering.cpp | 8 ++++++-- core/relinker/codegen/tests/ShiftLoweringTests.cpp | 5 ++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp b/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp index 5627419ebc..f510a32ad8 100644 --- a/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp +++ b/core/relinker/codegen/include/codegen/x86/ScalarLowering.hpp @@ -50,7 +50,10 @@ inline void StackSlot(StubBodyBuilder& body, std::uint8_t opcode, bool wide, std inline std::array UsedRegisters(std::initializer_list registers, const std::optional& memory) { std::array used{}; used[Stack] = true; - for (const auto reg : registers) used[reg] = true; + for (const auto reg : registers) { + if (reg >= used.size()) throw CodegenException("register index " + std::to_string(reg) + " is out of range"); + used[reg] = true; + } if (!memory) return used; if (memory->Rm == 4) { const auto base = static_cast((memory->Sib & 7) | ((memory->RexIndexBase & 1) << 3)); diff --git a/core/relinker/codegen/src/x86/ShiftLowering.cpp b/core/relinker/codegen/src/x86/ShiftLowering.cpp index 16dab8f686..925731e57d 100644 --- a/core/relinker/codegen/src/x86/ShiftLowering.cpp +++ b/core/relinker/codegen/src/x86/ShiftLowering.cpp @@ -90,8 +90,12 @@ std::optional DecodeShift(const std::uint8_t* data, const std::si void EmitShift(StubBodyBuilder& body, const ShiftOperands& operands) { const auto name = ShiftName(operands.Operation); const bool rotate = operands.Operation == ShiftOperation::RotateRight; - auto used = operands.Memory ? UsedRegisters({operands.Destination, operands.Count, kCountRegister}, operands.Memory) - : UsedRegisters({operands.Destination, operands.Source, operands.Count, kCountRegister}, std::nullopt); + std::array used{}; + if (rotate) + used = operands.Memory ? UsedRegisters({operands.Destination}, operands.Memory) : UsedRegisters({operands.Destination, operands.Source}, std::nullopt); + else + used = operands.Memory ? UsedRegisters({operands.Destination, operands.Count, kCountRegister}, operands.Memory) + : UsedRegisters({operands.Destination, operands.Source, operands.Count, kCountRegister}, std::nullopt); const auto result = Scratch(used, name); StackSlot(body, 0x8D, true, Stack, -kFrame); StackSlot(body, 0x89, true, result, kResultSlot); diff --git a/core/relinker/codegen/tests/ShiftLoweringTests.cpp b/core/relinker/codegen/tests/ShiftLoweringTests.cpp index 850330713c..d0f3b04f2d 100644 --- a/core/relinker/codegen/tests/ShiftLoweringTests.cpp +++ b/core/relinker/codegen/tests/ShiftLoweringTests.cpp @@ -194,7 +194,10 @@ void execution() { for (bool wide : {false, true}) { for (std::uint8_t dst = 0; dst < 16; ++dst) { for (const std::uint8_t src : std::array{6, dst, 4, 10}) { - for (const std::uint8_t immediate : std::array{0, 1, 31, 32, 63, 255}) { + const bool every = (dst == 0 && src == 6) || dst == src; + for (int value = 0; value < 256; ++value) { + const auto immediate = static_cast(value); + if (!every && immediate > 1 && immediate != 31 && immediate != 32 && immediate != 63 && immediate != 255) continue; const auto site = rotate(wide, dst, src, immediate); const auto operands = *Codegen::DecodeShift(site.data(), site.size()); const auto execution = executable(site, 0); From 7f0b73575bd7787a76eba6e74f22ddd8720fb737 Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Thu, 29 Oct 2026 18:44:28 +0530 Subject: [PATCH 4/6] fix(relinker): restore an RSP destination in one load in the shift stub, list MULX as lowered and drop the title note --- core/relinker/codegen/src/x86/ShiftLowering.cpp | 5 +++-- docs/dev/debt/relinker.md | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/relinker/codegen/src/x86/ShiftLowering.cpp b/core/relinker/codegen/src/x86/ShiftLowering.cpp index 925731e57d..22f42615e3 100644 --- a/core/relinker/codegen/src/x86/ShiftLowering.cpp +++ b/core/relinker/codegen/src/x86/ShiftLowering.cpp @@ -123,9 +123,10 @@ void EmitShift(StubBodyBuilder& body, const ShiftOperands& operands) { body.Raw(std::array{kPopFlags}); if (!rotate) StackSlot(body, 0x8B, true, kCountRegister, kCountSlot); StackSlot(body, 0x8B, true, result, kResultSlot); - StackSlot(body, 0x8D, true, Stack, kFrame); if (operands.Destination == Stack) - StackSlot(body, 0x8B, operands.Wide, Stack, kStackResultSlot - kFrame); + StackSlot(body, 0x8B, operands.Wide, Stack, kStackResultSlot); + else + StackSlot(body, 0x8D, true, Stack, kFrame); } } diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index 1464883fd2..ca3116a64a 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -46,15 +46,15 @@ - In macOS output, a guest instruction that addresses memory through GS becomes `UD2` and NOP padding of its own length ([macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)): the host owns GS, which holds the pthread TSD base the guest TLS stubs read, and the guest GS base that FreeBSD's `sysarch(AMD64_SET_GSBASE)` would set is not emulated. A converted module runs until such an instruction is reached, which then faults. Prospero Win (PPSA99995) has six in its Wine `ntdll`. - macOS output still writes pointer fixups into executable segments. `dyld_info -fixups` reports each as `pointer rebase is in executable segment`, next to `pointer rebase is in non-writable segment` for a fixup in a segment without write access, which the [macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp) rejects. A segment is executable and writable when its `PT_LOAD` is `PF_R|PF_W|PF_X`, or when a writable `PT_LOAD` shares a 4 KiB page with an executable one, because each page takes the union of the flags of the `PT_LOAD`s that cover it. dyld on an M1 Pro running macOS 27.0.1 (x86-64 output under Rosetta 2, `LC_BUILD_VERSION` minos and sdk 11.0) loads such an executable, applies the fixup and runs it; a fixup into a `PF_R|PF_X` segment ended the process there with SIGBUS in `dyld4::Loader::applyFixupsGeneric` before `main`, not with a refusal. Not measured for a dylib or under another macOS or SDK version. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. -- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX and RORX are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. -- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. +- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX and MULX are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. +- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, MULX, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. - `--to-intel` replaces the approximate legacy SSE and VEX.128 register forms of RSQRTPS/RCPPS and VRSQRTPS/VRCPPS with [correctly rounded](../../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). Legacy SSE forms preserve the destination's upper YMM bits; VEX forms clear them. The 256-bit, scalar and memory forms keep the native instruction, as does a site shorter than 5 bytes with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). - XSAVEC raises an illegal instruction under Rosetta and is not converted, because it writes the compacted format. XSAVEOPT, which also faults there, is rewritten to XSAVE in place by the [matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) under `--to-rosetta` (`0F AE /6` becomes `/4`; prefixes and operand are kept). The two differ only in the init and modified optimizations, which let XSAVEOPT skip writing unchanged or initial state ([Intel SDM volume 1](https://cdrdv2-public.intel.com/812382/253665-sdm-vol-1.pdf), chapter 13), so XRSTOR restores the same state from either area. Measured on one M5 with macOS 27.0.1: XSAVE and XRSTOR handle x87, SSE and AVX state in the standard format, although CPUID leaf 1 reports neither XSAVE nor OSXSAVE and leaf 0xD sub-leaf 1 reports neither XSAVEOPT nor XSAVEC. XSAVE also executes on an M5 with macOS 26.4, where it writes an XSTATE_BV of 0x7 and XGETBV returns the same x87, SSE and AVX mask although leaf 1 still reports neither XSAVE nor OSXSAVE; XRSTOR was not exercised there. The converter tests' x86-64 build executes the replacement XSAVE under Rosetta on that host. - `--trace-imports` ([Windows entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp), [Linux entry stub](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp), [macOS entry](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)) leaves every import that is not a PLT slot or a function GOT slot untraced, and the same for a function reached through any other relocation, `sceKernelDlsym`, a weak import, or a module loaded at run time. On Linux the entry stub makes the page of every traced slot readable, writable and executable with `mprotect` before it stores the thunk, so a slot inside `PT_GNU_RELRO` can be replaced; those pages stay writable, and data pages become executable, while tracing is on. On macOS the stub writes the thunk into the bound slot and does not change its protection, because that slot stays in a writable segment - [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. -- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. BEXTR, BZHI, MULX, PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. -- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. BEXTR (the `pp=0` form of the same opcode), BZHI, MULX, PDEP and PEXT are not lowered. Balatro (PPSA21402, #1142) stopped on `shlxq` after the ANDN lowering; this lowering has not been run on that title. The execution test runs on an x86-64 host, natively or under Rosetta, and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. BEXTR, BZHI, PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. BEXTR (the `pp=0` form of the same opcode), BZHI, PDEP and PEXT are not lowered. The execution test runs natively on x86-64 or under Rosetta and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. - Relative pointer-width unwind pointer decoding in the [exception frame reader](../../../core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp) (`DW_EH_PE_absptr` and `DW_EH_PE_udata8`) wraps modulo 2^64 to accommodate backward offsets, as standard DWARF unwinders add them. Console unwind table validation for out-of-range displacements is unmeasured. Unsigned offsets narrower than pointer width (`DW_EH_PE_udata2` and `DW_EH_PE_udata4`) cannot express negative displacements and throw when address addition overflows. From cf78130042ef92fbb185a7efbf0fd7da056b4bea Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Sun, 25 Oct 2026 23:03:33 +0530 Subject: [PATCH 5/6] feat(relinker): lower BZHI and BEXTR under --to-rosetta --- core/relinker/CMakeLists.txt | 8 + .../include/codegen/x86/BitfieldLowering.hpp | 29 +++ .../src/x86/Amd64OnlyInstructionMatcher.cpp | 7 + .../codegen/src/x86/BitfieldLowering.cpp | 158 +++++++++++++ .../codegen/tests/BitfieldLoweringTests.cpp | 211 ++++++++++++++++++ .../relinker/tests/test_to_rosetta.py | 2 +- docs/dev/debt/relinker.md | 9 +- docs/user/USAGE.md | 2 +- 8 files changed, 420 insertions(+), 6 deletions(-) create mode 100644 core/relinker/codegen/include/codegen/x86/BitfieldLowering.hpp create mode 100644 core/relinker/codegen/src/x86/BitfieldLowering.cpp create mode 100644 core/relinker/codegen/tests/BitfieldLoweringTests.cpp diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index f708e064af..91a96ed9ba 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -56,6 +56,7 @@ add_library(relinker_core STATIC ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ShiftLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BitfieldLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/RelrRelocations.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp @@ -112,6 +113,11 @@ if(BUILD_TESTING) add_test(NAME shift_lowering COMMAND shift_lowering_tests) set_tests_properties(shift_lowering PROPERTIES TIMEOUT 60) + add_test_executable(bitfield_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/BitfieldLoweringTests.cpp) + target_link_libraries(bitfield_lowering_tests PRIVATE relinker_core) + add_test(NAME bitfield_lowering COMMAND bitfield_lowering_tests) + set_tests_properties(bitfield_lowering PROPERTIES TIMEOUT 60) + add_test_executable(masked_store_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/MaskedStoreLoweringTests.cpp) target_link_libraries(masked_store_lowering_tests PRIVATE relinker_core) add_test(NAME masked_store_lowering COMMAND masked_store_lowering_tests) @@ -265,6 +271,7 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ShiftLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BitfieldLowering.cpp ) target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows) if(BUILD_TESTING) @@ -556,6 +563,7 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BlsLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ShiftLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/BitfieldLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp diff --git a/core/relinker/codegen/include/codegen/x86/BitfieldLowering.hpp b/core/relinker/codegen/include/codegen/x86/BitfieldLowering.hpp new file mode 100644 index 0000000000..6765d45a9c --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/BitfieldLowering.hpp @@ -0,0 +1,29 @@ +#ifndef CODEGEN_X86_BITFIELDLOWERING_HPP +#define CODEGEN_X86_BITFIELDLOWERING_HPP + +#include +#include + +namespace Codegen { + +enum class BitfieldOperation : std::uint8_t { + ZeroHigh, + Extract, +}; + +struct BitfieldOperands { + BitfieldOperation Operation; + std::uint8_t Destination; + std::uint8_t Source; + std::uint8_t Control; + bool Wide; + std::optional Memory; +}; + +[[nodiscard]] const char* BitfieldName(BitfieldOperation operation); +[[nodiscard]] std::optional DecodeBitfield(const std::uint8_t* data, std::size_t length); +void EmitBitfield(StubBodyBuilder& body, const BitfieldOperands& operands); + +} + +#endif diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index 496121d571..4749d20165 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -262,6 +263,8 @@ std::optional Amd64OnlyInstructionMatcher::_matchRosetta(const D return _trampoline(BlsName(bls->Operation), instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitBls(body, *bls); })); if (const auto shift = DecodeShift(instr.Data, instr.Length)) return _trampoline(ShiftName(shift->Operation), instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitShift(body, *shift); })); + if (const auto bitfield = DecodeBitfield(instr.Data, instr.Length)) + return _trampoline(BitfieldName(bitfield->Operation), instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitBitfield(body, *bitfield); })); if (instr.IsRdseed()) { std::vector replacement(instr.Data, instr.Data + instr.Length); replacement[instr.OpcodeOffset() + 2] &= static_cast(~(1u << X64OpcodeConstants::ModRmRegShift)); @@ -341,6 +344,10 @@ const char* Amd64OnlyInstructionMatcher::_emitOutOfLine(StubBodyBuilder& body, c EmitShift(body, *shift); return ShiftName(shift->Operation); } + if (const auto bitfield = _target == Amd64OnlyTarget::Rosetta ? DecodeBitfield(instr.Data, instr.Length) : std::nullopt) { + EmitBitfield(body, *bitfield); + return BitfieldName(bitfield->Operation); + } if (_target == Amd64OnlyTarget::Rosetta && instr.IsRdtscp()) { body.Raw(_rdtscpReplacement(instr)); return kRdtscp.Name; diff --git a/core/relinker/codegen/src/x86/BitfieldLowering.cpp b/core/relinker/codegen/src/x86/BitfieldLowering.cpp new file mode 100644 index 0000000000..1d9fb3e545 --- /dev/null +++ b/core/relinker/codegen/src/x86/BitfieldLowering.cpp @@ -0,0 +1,158 @@ +#include +#include + +namespace Codegen { + +using namespace ScalarLowering; + +namespace { + +constexpr std::uint8_t kControlRegister = 1; +constexpr std::int32_t kResultSlot = 0; +constexpr std::int32_t kControlSlot = 8; +constexpr std::int32_t kMaskSlot = 16; +constexpr std::int32_t kStackResultSlot = 24; +constexpr std::int32_t kFrame = FrameSize + 16; +constexpr std::uint8_t kJumpBelow = 0x72; +constexpr std::uint8_t kJumpAboveOrEqual = 0x73; +constexpr std::uint8_t kJump = 0xEB; +constexpr std::uint8_t kSetCarry = 0xF9; + +std::uint8_t _bits(const BitfieldOperands& operands) { + return operands.Wide ? 64 : 32; +} + +std::vector _compareControlByte(const std::uint8_t rm, const std::uint8_t bits) { + return {0x80, static_cast(0xF8 | rm), bits}; +} + +std::vector _lowMask(const BitfieldOperands& operands, const std::uint8_t mask) { + auto bytes = RegisterBytes(0xD3, operands.Wide, 4, mask); + const auto invert = RegisterBytes(0xF7, operands.Wide, 2, mask); + bytes.insert(bytes.end(), invert.begin(), invert.end()); + return bytes; +} + +void _append(std::vector& bytes, const std::vector& more) { + bytes.insert(bytes.end(), more.begin(), more.end()); +} + +void _loadSource(StubBodyBuilder& body, const BitfieldOperands& operands, const std::uint8_t result, const char* name) { + if (operands.Memory) + Memory(body, 0x8B, operands.Wide, *operands.Memory, result, name, kFrame); + else if (operands.Source == Stack) + StackSlot(body, 0x8D, true, result, kFrame); + else + Register(body, 0x8B, operands.Wide, result, operands.Source); +} + +void _zeroHigh(StubBodyBuilder& body, const BitfieldOperands& operands, const std::uint8_t result, const std::uint8_t mask) { + auto high = operands.Wide ? std::vector{} : RegisterBytes(0x8B, false, result, result); + _append(high, RegisterBytes(0x85, operands.Wide, result, result)); + high.push_back(kSetCarry); + auto low = _lowMask(operands, mask); + _append(low, RegisterBytes(0x21, operands.Wide, mask, result)); + body.Raw(_compareControlByte(kControlRegister, _bits(operands))); + body.Raw(std::array{kJumpBelow, static_cast(high.size() + 2)}); + body.Raw(high); + body.Raw(std::array{kJump, static_cast(low.size())}); + body.Raw(low); +} + +void _extract(StubBodyBuilder& body, const BitfieldOperands& operands, const std::uint8_t result, const std::uint8_t mask) { + const auto bits = _bits(operands); + std::vector length{0x88, 0xE9}; + _append(length, _lowMask(operands, mask)); + const auto zero = RegisterBytes(0x31, operands.Wide, result, result); + auto shifted = RegisterBytes(0xD3, operands.Wide, 5, result); + _append(shifted, _compareControlByte(5, bits)); + shifted.push_back(kJumpAboveOrEqual); + shifted.push_back(static_cast(length.size() + 2 + zero.size())); + _append(shifted, length); + shifted.push_back(kJump); + shifted.push_back(static_cast(zero.size())); + body.Raw(_compareControlByte(kControlRegister, bits)); + body.Raw(std::array{kJumpAboveOrEqual, static_cast(shifted.size())}); + body.Raw(shifted); + body.Raw(zero); + Register(body, 0x21, operands.Wide, mask, result); +} + +} + +const char* BitfieldName(const BitfieldOperation operation) { + switch (operation) { + case BitfieldOperation::ZeroHigh: return "BZHI"; + case BitfieldOperation::Extract: return "BEXTR"; + } + throw CodegenException("unknown bitfield operation"); +} + +std::optional DecodeBitfield(const std::uint8_t* data, const std::size_t length) { + std::size_t vex = 0; + while (vex < length && LegacyPrefix(data[vex])) ++vex; + if (vex + 5 > length || data[vex] != 0xC4 || (data[vex + 1] & 0x1F) != 2 || (data[vex + 2] & 3) != 0) return std::nullopt; + BitfieldOperation operation; + if (data[vex + 3] == 0xF5) + operation = BitfieldOperation::ZeroHigh; + else if (data[vex + 3] == 0xF7) + operation = BitfieldOperation::Extract; + else + return std::nullopt; + const auto name = BitfieldName(operation); + if ((data[vex + 2] & 4) != 0) throw CodegenException(std::string(name) + " requires VEX.L=0"); + const auto extensions = static_cast(~data[vex + 1]); + const auto modrm = data[vex + 4]; + BitfieldOperands operands{ + operation, + static_cast(((modrm >> 3) & 7) | ((extensions & 0x80) >> 4)), + static_cast((modrm & 7) | ((extensions & 0x20) >> 2)), + static_cast((~data[vex + 2] >> 3) & 15), + (data[vex + 2] & 0x80) != 0, std::nullopt}; + std::size_t end = vex + 5; + if ((modrm >> 6) != 3) { + const auto rex = static_cast(((extensions & 0x20) >> 5) | ((extensions & 0x40) >> 5)); + operands.Memory = DecodeMemoryOperand(data, length, vex + 4, rex, {data, data + vex}); + end = vex + 4 + operands.Memory->EncodedSize; + } + if (end != length) throw CodegenException(std::string(name) + " has trailing bytes"); + return operands; +} + +void EmitBitfield(StubBodyBuilder& body, const BitfieldOperands& operands) { + const auto name = BitfieldName(operands.Operation); + auto used = operands.Memory ? UsedRegisters({operands.Destination, operands.Control, kControlRegister}, operands.Memory) + : UsedRegisters({operands.Destination, operands.Source, operands.Control, kControlRegister}, std::nullopt); + const auto result = Scratch(used, name); + const auto mask = Scratch(used, name); + StackSlot(body, 0x8D, true, Stack, -kFrame); + StackSlot(body, 0x89, true, result, kResultSlot); + StackSlot(body, 0x89, true, kControlRegister, kControlSlot); + StackSlot(body, 0x89, true, mask, kMaskSlot); + _loadSource(body, operands, result, name); + if (operands.Control == Stack) + StackSlot(body, 0x8D, true, kControlRegister, kFrame); + else if (operands.Control != kControlRegister) + Register(body, 0x8B, true, kControlRegister, operands.Control); + auto allOnes = RegisterBytes(0xC7, operands.Wide, 0, mask); + Append32(allOnes, -1); + body.Raw(allOnes); + if (operands.Operation == BitfieldOperation::ZeroHigh) + _zeroHigh(body, operands, result, mask); + else + _extract(body, operands, result, mask); + if (operands.Destination == Stack) + StackSlot(body, 0x89, operands.Wide, result, kStackResultSlot); + else if (operands.Destination == kControlRegister) + StackSlot(body, 0x89, true, result, kControlSlot); + else + Register(body, 0x8B, operands.Wide, operands.Destination, result); + StackSlot(body, 0x8B, true, mask, kMaskSlot); + StackSlot(body, 0x8B, true, kControlRegister, kControlSlot); + StackSlot(body, 0x8B, true, result, kResultSlot); + StackSlot(body, 0x8D, true, Stack, kFrame); + if (operands.Destination == Stack) + StackSlot(body, 0x8B, operands.Wide, Stack, kStackResultSlot - kFrame); +} + +} diff --git a/core/relinker/codegen/tests/BitfieldLoweringTests.cpp b/core/relinker/codegen/tests/BitfieldLoweringTests.cpp new file mode 100644 index 0000000000..acfc306750 --- /dev/null +++ b/core/relinker/codegen/tests/BitfieldLoweringTests.cpp @@ -0,0 +1,211 @@ +#include "LoweringExecution.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using LoweringExecution::Bytes; +using Codegen::BitfieldOperation; + +constexpr std::array kOperations{BitfieldOperation::ZeroHigh, BitfieldOperation::Extract}; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +std::uint8_t opcode(BitfieldOperation operation) { + return operation == BitfieldOperation::ZeroHigh ? 0xF5 : 0xF7; +} + +Bytes instruction(BitfieldOperation operation, bool wide, std::uint8_t dst, std::uint8_t src, std::uint8_t control) { + return {0xC4, static_cast(0xE2 ^ ((dst & 8) << 4) ^ ((src & 8) << 2)), + static_cast((wide ? 0x80 : 0) | ((~control & 15) << 3)), opcode(operation), + static_cast(0xC0 | ((dst & 7) << 3) | (src & 7))}; +} + +Bytes memorySite(BitfieldOperation operation, bool wide, const Bytes& tail) { + Bytes site{0xC4, 0xE2, static_cast((wide ? 0x80 : 0) | 0x40), opcode(operation)}; + site.insert(site.end(), tail.begin(), tail.end()); + return site; +} + +void decoding() { + for (const auto operation : kOperations) { + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (std::uint8_t src = 0; src < 16; ++src) { + for (std::uint8_t control = 0; control < 16; ++control) { + const auto bytes = instruction(operation, wide, dst, src, control); + const auto operands = Codegen::DecodeBitfield(bytes.data(), bytes.size()); + require(operands && operands->Operation == operation && operands->Wide == wide && operands->Destination == dst && operands->Source == src && operands->Control == control && !operands->Memory, "bitfield register decoding changed an operand"); + } + } + } + } + } + const Bytes memory{0x64, 0x67, 0xC4, 0x02, 0xA8, 0xF5, 0x8C, 0xAC, 0x10, 0, 0, 0}; + const auto operands = Codegen::DecodeBitfield(memory.data(), memory.size()); + require(operands && operands->Operation == BitfieldOperation::ZeroHigh && operands->Destination == 9 && operands->Control == 10 && operands->Memory && operands->Memory->RexIndexBase == 3 && operands->Memory->Prefixes == Bytes{0x64, 0x67}, "bitfield memory extensions or prefixes were lost"); + for (const Bytes bytes : {Bytes{0x90}, Bytes{0xC4, 0xE2, 0xF0}, Bytes{0xC4, 0xE1, 0xF0, 0xF5, 0xC0}, Bytes{0xC4, 0xE2, 0xF1, 0xF7, 0xC0}, Bytes{0xC4, 0xE2, 0xF3, 0xF5, 0xC0}, Bytes{0xC4, 0xE2, 0xF2, 0xF5, 0xC0}, Bytes{0xC4, 0xE2, 0xF3, 0xF6, 0xC0}, Bytes{0xC4, 0xE2, 0xF0, 0xF3, 0xC8}}) + require(!Codegen::DecodeBitfield(bytes.data(), bytes.size()), "Unrelated, truncated, SHLX, PDEP, PEXT, MULX or BLSR opcode matched a bitfield instruction"); + for (const Bytes bytes : {Bytes{0xC4, 0xE2, 0xF4, 0xF5, 0xC0}, Bytes{0xC4, 0xE2, 0xF0, 0xF7, 0xC0, 0x90}, Bytes{0xC4, 0xE2, 0xF0, 0xF5, 0x84}}) { + bool rejected = false; + try { (void)Codegen::DecodeBitfield(bytes.data(), bytes.size()); } + catch (const Codegen::CodegenException&) { rejected = true; } + require(rejected, "Malformed bitfield instruction was accepted"); + } + const Bytes overflow{0xC4, 0xE2, 0xF0, 0xF7, 0x8C, 0x24, 0xFF, 0xFF, 0xFF, 0x7F}; + bool rejected = false; + try { + Codegen::StubBodyBuilder body; + Codegen::EmitBitfield(body, *Codegen::DecodeBitfield(overflow.data(), overflow.size())); + } catch (const Codegen::CodegenException&) { rejected = true; } + require(rejected, "An overflowing stack displacement was silently wrapped"); +} + +void matching() { + const auto intel = Codegen::MakeAmd64OnlyInstructionMatcher(); + const auto rosetta = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + for (const auto operation : kOperations) { + const auto bytes = instruction(operation, true, 0, 6, 2); + const auto name = Codegen::BitfieldName(operation); + require(!intel->Match(bytes.data(), bytes.size()), std::string("Intel mode rewrote ") + name); + const auto match = rosetta->Match(bytes.data(), bytes.size()); + require(match && match->InstructionName == name && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline && !match->Optional, std::string("Rosetta mode did not require a trampoline for ") + name); + const std::array, 2> sequence{bytes, bytes}; + require(rosetta->MatchSequence(sequence, {}).has_value(), std::string("A sequence could not move out of line for ") + name); + } +} + +#if LOWERING_EXECUTION_AVAILABLE + +using LoweringExecution::Execution; +using LoweringExecution::State; + +Execution executable(const Bytes& site, std::uint64_t memoryValue) { + const auto operands = Codegen::DecodeBitfield(site.data(), site.size()); + require(operands.has_value(), "Execution site was not a bitfield instruction"); + return Execution(site, memoryValue, operands->Memory && operands->Memory->RipRelative, [](Codegen::StubBodyBuilder& body, const Bytes& relocated) { + Codegen::EmitBitfield(body, *Codegen::DecodeBitfield(relocated.data(), relocated.size())); + }); +} + +struct Expected { + std::uint64_t Result; + std::uint64_t Flags; + std::uint64_t FlagMask; +}; + +Expected expected(const Codegen::BitfieldOperands& operands, std::uint64_t source, std::uint64_t control) { + const unsigned bits = operands.Wide ? 64 : 32; + const auto mask = operands.Wide ? ~std::uint64_t{0} : 0xFFFFFFFFull; + source &= mask; + const auto lowBits = [&](unsigned count) { return count >= bits ? mask : (std::uint64_t{1} << count) - 1; }; + if (operands.Operation == BitfieldOperation::ZeroHigh) { + const auto index = static_cast(control & 0xFF); + const auto result = index >= bits ? source : source & lowBits(index); + const auto flags = (result == 0 ? LoweringExecution::kZero : 0u) | (index >= bits ? LoweringExecution::kCarry : 0u) | ((result >> (bits - 1)) != 0 ? LoweringExecution::kSign : 0u); + return {result, flags, LoweringExecution::kFlagMask}; + } + const auto start = static_cast(control & 0xFF); + const auto length = static_cast((control >> 8) & 0xFF); + const auto result = (start >= bits ? 0 : source >> start) & lowBits(length); + return {result, result == 0 ? LoweringExecution::kZero : 0u, LoweringExecution::kFlagMask & ~LoweringExecution::kSign}; +} + +void check(const State& state, const Codegen::BitfieldOperands& operands, std::uint64_t source, std::uint64_t control) { + const auto name = Codegen::BitfieldName(operands.Operation); + const auto wanted = expected(operands, source, control); + const auto describe = [&] { + return std::string(name) + (operands.Wide ? " 64" : " 32") + " dst " + std::to_string(operands.Destination) + " src " + std::to_string(operands.Source) + " control " + std::to_string(operands.Control) + (operands.Memory ? " memory" : "") + " (source " + std::to_string(source) + ", control value " + std::to_string(control) + ")"; + }; + for (std::size_t reg = 0; reg < 16; ++reg) { + const auto value = reg == operands.Destination ? wanted.Result : state.Input[reg]; + if (state.Output[reg] != value) + throw std::runtime_error(describe() + ": register " + std::to_string(reg) + " is " + std::to_string(state.Output[reg]) + ", wanted " + std::to_string(value)); + } + if ((state.Flags & wanted.FlagMask) != wanted.Flags) + throw std::runtime_error(describe() + ": flags are " + std::to_string(state.Flags & wanted.FlagMask) + ", wanted " + std::to_string(wanted.Flags)); + for (const auto value : state.RedZone) require(value == LoweringExecution::kSentinel, std::string(name) + " overwrote the caller's red zone"); +} + +void run(const Execution& execution, const Codegen::BitfieldOperands& operands, State state, std::uint64_t memoryValue) { + for (const auto flags : {LoweringExecution::kBaseFlags | LoweringExecution::kFlagMask, LoweringExecution::kBaseFlags}) { + state.InputFlags = flags; + auto trial = state; + execution.Run(trial); + const auto source = operands.Memory ? memoryValue : trial.Input[operands.Source]; + check(trial, operands, source, trial.Input[operands.Control]); + } +} + +void execution() { + constexpr std::array values{0, 1, 0x80000000ull, 0x8000000000000000ull, 0xFFFFFFFFFFFFFFFFull, 0xABCD123456789000ull}; + constexpr std::array controls{0, 1, 31, 32, 63, 64, 0xFF, 0x100, 0x801, 0x1F05, 0x2000, 0x201F, 0x3F3F, 0x4000, 0x4020, 0xFF00, 0xFFFF, 0x0F10, 0x12345678, 0xFFFFFFFFFFFF0405ull}; + for (const auto operation : kOperations) { + for (bool wide : {false, true}) { + for (std::uint8_t dst = 0; dst < 16; ++dst) { + for (const auto pair : std::array, 9>{{{6, 2}, {dst, 10}, {11, dst}, {dst, dst}, {4, 9}, {9, 4}, {1, 6}, {6, 1}, {1, 1}}}) { + const auto site = instruction(operation, wide, dst, pair.first, pair.second); + const auto operands = *Codegen::DecodeBitfield(site.data(), site.size()); + const auto execution = executable(site, 0); + for (const auto value : values) { + for (const auto control : controls) { + auto state = LoweringExecution::InitialState(); + state.Input[pair.first] = value; + state.Input[pair.second] = control; + run(execution, operands, state, 0); + } + } + } + } + } + } + for (const auto operation : kOperations) { + for (bool wide : {false, true}) { + for (const Bytes tail : {Bytes{0x06}, Bytes{0x46, 0x10}, Bytes{0x84, 0x24, 0x40, 0, 0, 0}, Bytes{0x04, 0xCA}, Bytes{0x85, 0x10, 0, 0, 0}, Bytes{0x05, 0, 0, 0, 0}}) { + for (const auto value : std::array{0, 0xFEDCBA9876543210ull, 0xFFFFFFFF00000000ull}) { + for (const auto control : std::array{0, 0x0C04, 0x3F, 0x40}) { + const auto site = memorySite(operation, wide, tail); + const auto operands = *Codegen::DecodeBitfield(site.data(), site.size()); + const auto execution = executable(site, value); + auto state = LoweringExecution::InitialState(); + state.Input[6] = reinterpret_cast(execution.Memory()) - (tail[0] == 0x46 ? 16 : 0); + state.Input[2] = reinterpret_cast(execution.Memory()) - 3 * 8; + state.Input[1] = 3; + state.Input[5] = reinterpret_cast(execution.Memory()) - 16; + state.Input[7] = control; + state.Output[0] = value; + run(execution, operands, state, value); + } + } + } + } + } +} +#else +void execution() { std::cout << "bitfield execution checks require a Linux/macOS x86-64 build\n"; } +#endif + +} + +int main() { + try { + decoding(); + matching(); + execution(); + std::cout << "bitfield lowering checks passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/relinker/relinker/tests/test_to_rosetta.py b/core/relinker/relinker/tests/test_to_rosetta.py index 93024c8457..d3a1c6b157 100644 --- a/core/relinker/relinker/tests/test_to_rosetta.py +++ b/core/relinker/relinker/tests/test_to_rosetta.py @@ -64,7 +64,7 @@ def main(): assert (andn not in output.read_bytes()) == lowered, f"{options} {'did not lower' if lowered else 'changed'} ANDN" if lowered: assert "ANDN" in result.stdout, result.stdout - for name, bls in (("BLSR", "c4e2f8f3ce"), ("BLSMSK", "c4e2f8f3d6"), ("BLSI", "c4e2f8f3de")): + for name, bls in (("BLSR", "c4e2f8f3ce"), ("BLSMSK", "c4e2f8f3d6"), ("BLSI", "c4e2f8f3de"), ("BZHI", "c4e2e8f5c6"), ("BEXTR", "c4e2e8f7c6")): code = bytes.fromhex(bls + "c3") for target in ((), ("--macos",)): for conversion in ((), ("--to-intel",), ("--to-rosetta",)): diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index ca3116a64a..db4bee72a4 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -46,15 +46,16 @@ - In macOS output, a guest instruction that addresses memory through GS becomes `UD2` and NOP padding of its own length ([macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)): the host owns GS, which holds the pthread TSD base the guest TLS stubs read, and the guest GS base that FreeBSD's `sysarch(AMD64_SET_GSBASE)` would set is not emulated. A converted module runs until such an instruction is reached, which then faults. Prospero Win (PPSA99995) has six in its Wine `ntdll`. - macOS output still writes pointer fixups into executable segments. `dyld_info -fixups` reports each as `pointer rebase is in executable segment`, next to `pointer rebase is in non-writable segment` for a fixup in a segment without write access, which the [macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp) rejects. A segment is executable and writable when its `PT_LOAD` is `PF_R|PF_W|PF_X`, or when a writable `PT_LOAD` shares a 4 KiB page with an executable one, because each page takes the union of the flags of the `PT_LOAD`s that cover it. dyld on an M1 Pro running macOS 27.0.1 (x86-64 output under Rosetta 2, `LC_BUILD_VERSION` minos and sdk 11.0) loads such an executable, applies the fixup and runs it; a fixup into a `PF_R|PF_X` segment ended the process there with SIGBUS in `dyld4::Loader::applyFixupsGeneric` before `main`, not with a refusal. Not measured for a dylib or under another macOS or SDK version. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. -- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX and MULX are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. -- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, MULX, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. +- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, BZHI and BEXTR are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. +- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, BZHI, BEXTR, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. - `--to-intel` replaces the approximate legacy SSE and VEX.128 register forms of RSQRTPS/RCPPS and VRSQRTPS/VRCPPS with [correctly rounded](../../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). Legacy SSE forms preserve the destination's upper YMM bits; VEX forms clear them. The 256-bit, scalar and memory forms keep the native instruction, as does a site shorter than 5 bytes with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). - XSAVEC raises an illegal instruction under Rosetta and is not converted, because it writes the compacted format. XSAVEOPT, which also faults there, is rewritten to XSAVE in place by the [matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) under `--to-rosetta` (`0F AE /6` becomes `/4`; prefixes and operand are kept). The two differ only in the init and modified optimizations, which let XSAVEOPT skip writing unchanged or initial state ([Intel SDM volume 1](https://cdrdv2-public.intel.com/812382/253665-sdm-vol-1.pdf), chapter 13), so XRSTOR restores the same state from either area. Measured on one M5 with macOS 27.0.1: XSAVE and XRSTOR handle x87, SSE and AVX state in the standard format, although CPUID leaf 1 reports neither XSAVE nor OSXSAVE and leaf 0xD sub-leaf 1 reports neither XSAVEOPT nor XSAVEC. XSAVE also executes on an M5 with macOS 26.4, where it writes an XSTATE_BV of 0x7 and XGETBV returns the same x87, SSE and AVX mask although leaf 1 still reports neither XSAVE nor OSXSAVE; XRSTOR was not exercised there. The converter tests' x86-64 build executes the replacement XSAVE under Rosetta on that host. - `--trace-imports` ([Windows entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp), [Linux entry stub](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp), [macOS entry](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)) leaves every import that is not a PLT slot or a function GOT slot untraced, and the same for a function reached through any other relocation, `sceKernelDlsym`, a weak import, or a module loaded at run time. On Linux the entry stub makes the page of every traced slot readable, writable and executable with `mprotect` before it stores the thunk, so a slot inside `PT_GNU_RELRO` can be replaced; those pages stay writable, and data pages become executable, while tracing is on. On macOS the stub writes the thunk into the bound slot and does not change its protection, because that slot stays in a writable segment - [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. -- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. BEXTR, BZHI, PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. -- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. BEXTR (the `pp=0` form of the same opcode), BZHI, PDEP and PEXT are not lowered. The execution test runs natively on x86-64 or under Rosetta and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. MULX, PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. MULX, PDEP and PEXT are not lowered. Balatro (PPSA21402, #1142) stopped on `shlxq` after the ANDN lowering; this lowering has not been run on that title. The execution test runs on an x86-64 host, natively or under Rosetta, and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [BZHI and BEXTR lowering](../../../core/relinker/codegen/src/x86/BitfieldLowering.cpp) builds the mask with `shl` by CL and `not` on a scratch register and applies it with `and` under `--to-rosetta`, with RCX spilled to carry the control operand in a 160-byte frame below the red zone; the control's low byte is compared with the operand size, so an index or start at or past it takes the SDM's unmasked (BZHI) or zero (BEXTR) branch instead of the masked count a scalar shift would use. BZHI sets CF through STC on that branch, and ZF and SF follow the result through `test` or `and`; BEXTR's ZF follows the result and CF and OF clear through `and`, while its SF, which the SDM leaves undefined, follows the result here. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. MULX, PDEP and PEXT are not lowered. The execution test runs on an x86-64 host, natively or under Rosetta, and checks the result and flags against a model of the SDM semantics that matched a Zen 2's BZHI and BEXTR on 122,880 cases, not against the hardware instruction in the same run; precise fault delivery inside a spill sequence has not been verified. - Relative pointer-width unwind pointer decoding in the [exception frame reader](../../../core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp) (`DW_EH_PE_absptr` and `DW_EH_PE_udata8`) wraps modulo 2^64 to accommodate backward offsets, as standard DWARF unwinders add them. Console unwind table validation for out-of-range displacements is unmeasured. Unsigned offsets narrower than pointer width (`DW_EH_PE_udata2` and `DW_EH_PE_udata4`) cannot express negative displacements and throw when address addition overflows. diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index d63dd7299e..e48958bc5e 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -74,7 +74,7 @@ All switches are disabled by default. `unused-filter` defaults to `0`; `--rpath` | `--linux-fixed-base
` | Load the Linux executable at this fixed address instead of the default `0x400000`, given in decimal or hexadecimal (`0x`): every virtual address of the image is offset by it, so virtual address 0 is placed at the address, as the console places an executable. Requires a 64 KiB-aligned address from `0x10000` through `0x7fffffff0000` that is a multiple of every `PT_LOAD` alignment, and an image that ends in the user address space. The executable is written as `ET_EXEC`, without ASLR, as it is at the default address. Bundled modules keep their normal placement. Conflicts with `--windows`, `--macos` and `--lazy-binding`. | | `--macos-fixed-base
` | Map the lowest ELF load page of the macOS executable at this fixed address, given in decimal or hexadecimal (`0x`). Requires `--macos`, a 64 KiB-aligned address from `0x20000` through `0x7fffffff0000`, and an image that ends in the macOS user address space. `__PAGEZERO` ends 64 KiB below the address and the Mach-O header occupies those 64 KiB. The executable has no ASLR; an address the process cannot map fails to load. Bundled modules keep their normal placement. | | `--to-intel` | Convert supported AMD-only instructions in the executable and bundled modules. Unsupported instructions or unreachable conversion stubs cause an error. | -| `--to-rosetta` | Convert as `--to-intel` does and also lower what Rosetta lacks: ANDN becomes scalar NOT/AND, BLSI, BLSMSK and BLSR become scalar LEA/AND/XOR/NEG sequences, SHLX, SHRX, SARX and RORX become flag-preserving scalar shifts and rotates, RDSEED becomes RDRAND, CLWB a NOP, XSAVEOPT becomes XSAVE, and RDPID and the ECX of RDTSCP return processor 0. Implies `--to-intel`. Independent of `--macos`: `--macos` alone leaves these instructions as they are. Conflicts with `--windows`. | +| `--to-rosetta` | Convert as `--to-intel` does and also lower what Rosetta lacks: ANDN becomes scalar NOT/AND, BLSI, BLSMSK and BLSR become scalar LEA/AND/XOR/NEG sequences, SHLX, SHRX, SARX and RORX become flag-preserving scalar shifts and rotates, BZHI and BEXTR become scalar shift-and-mask sequences, RDSEED becomes RDRAND, CLWB a NOP, XSAVEOPT becomes XSAVE, and RDPID and the ECX of RDTSCP return processor 0. Implies `--to-intel`. Independent of `--macos`: `--macos` alone leaves these instructions as they are. Conflicts with `--windows`. | | `unused-filter=0` | Keep all imported NID references. | | `unused-filter=1` | Filter unused non-PLT imports using control-flow and GOT access analysis, treating address-taken code as reachable; preserve PLT imports. | | `unused-filter=2` | Apply strict unused-import analysis and compact the PLT. Unsupported analysis cases cause an error. | From 6ad23bffdd542a255807696258f2b93dbe0b6061 Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Thu, 29 Oct 2026 18:45:15 +0530 Subject: [PATCH 6/6] fix(relinker): restore an RSP destination in one load in the BZHI and BEXTR stub and list MULX as lowered --- core/relinker/codegen/src/x86/BitfieldLowering.cpp | 5 +++-- docs/dev/debt/relinker.md | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/core/relinker/codegen/src/x86/BitfieldLowering.cpp b/core/relinker/codegen/src/x86/BitfieldLowering.cpp index 1d9fb3e545..bc9cf32679 100644 --- a/core/relinker/codegen/src/x86/BitfieldLowering.cpp +++ b/core/relinker/codegen/src/x86/BitfieldLowering.cpp @@ -150,9 +150,10 @@ void EmitBitfield(StubBodyBuilder& body, const BitfieldOperands& operands) { StackSlot(body, 0x8B, true, mask, kMaskSlot); StackSlot(body, 0x8B, true, kControlRegister, kControlSlot); StackSlot(body, 0x8B, true, result, kResultSlot); - StackSlot(body, 0x8D, true, Stack, kFrame); if (operands.Destination == Stack) - StackSlot(body, 0x8B, operands.Wide, Stack, kStackResultSlot - kFrame); + StackSlot(body, 0x8B, operands.Wide, Stack, kStackResultSlot); + else + StackSlot(body, 0x8D, true, Stack, kFrame); } } diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index db4bee72a4..ad959bbf72 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -46,16 +46,16 @@ - In macOS output, a guest instruction that addresses memory through GS becomes `UD2` and NOP padding of its own length ([macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)): the host owns GS, which holds the pthread TSD base the guest TLS stubs read, and the guest GS base that FreeBSD's `sysarch(AMD64_SET_GSBASE)` would set is not emulated. A converted module runs until such an instruction is reached, which then faults. Prospero Win (PPSA99995) has six in its Wine `ntdll`. - macOS output still writes pointer fixups into executable segments. `dyld_info -fixups` reports each as `pointer rebase is in executable segment`, next to `pointer rebase is in non-writable segment` for a fixup in a segment without write access, which the [macOS image writer](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp) rejects. A segment is executable and writable when its `PT_LOAD` is `PF_R|PF_W|PF_X`, or when a writable `PT_LOAD` shares a 4 KiB page with an executable one, because each page takes the union of the flags of the `PT_LOAD`s that cover it. dyld on an M1 Pro running macOS 27.0.1 (x86-64 output under Rosetta 2, `LC_BUILD_VERSION` minos and sdk 11.0) loads such an executable, applies the fixup and runs it; a fixup into a `PF_R|PF_X` segment ended the process there with SIGBUS in `dyld4::Loader::applyFixupsGeneric` before `main`, not with a refusal. Not measured for a dylib or under another macOS or SDK version. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. -- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, BZHI and BEXTR are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. -- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, BZHI, BEXTR, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. +- `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). AVX2, FMA, BMI1/2, F16C and MOVBE were observed to execute on an M5 Max without being reported through CPUID; this is not a guarantee for other Rosetta hosts. ANDN and RDRAND fault on an M1 running macOS 15.3.1 and execute on an M1 running macOS 26.6.2, so on the M1 the difference follows the macOS version; on that host BEXTR, BLSI, BLSMSK, BLSR, TZCNT, LZCNT, POPCNT, BZHI, MULX, PDEP, PEXT, RORX, SARX, SHLX, SHRX, MOVBE, FMA, AVX2 and F16C execute as well, its CPUID leaf 1 reports none of MOVBE, FMA, AVX or F16C, and RDSEED, RDPID, CLWB, ADCX and ADOX raise SIGILL. They also execute on an M5 running macOS 26.4, where the rest of BMI1 and BMI2, LZCNT, PREFETCHW, CLFLUSHOPT, AVX, AVX2, FMA, F16C and MOVBE execute as well although CPUID leaf 7 reads zero and leaves 1 and 0x80000001 report none of them, and where RDSEED, CLWB, RDPID, ADCX, ADOX, SHA and AVX-512F raise an illegal instruction. On that host the BMI1 and BMI2 results match the Intel SDM definitions over 200,000 random inputs each for PDEP, PEXT, BEXTR, BZHI, LZCNT and TZCNT, so they compute the documented result and do not only avoid a fault. ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, BZHI, BEXTR and MULX are lowered, but RDRAND and the other BMI instructions still depend on the host. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, and RDTSCP runs unchanged with ECX set to 0 after it, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB (also measured on an M5, macOS 27.0.1). An RDPID or RDTSCP site with no movable instruction after it for the jump is kept; a kept RDTSCP reports 0xAAAAAAAB. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests. ANDN lowering is also executed by an x86-64 test build under Rosetta on the M1; the other conversions were originally checked on an M5 Max, outside this repository's CI. +- `--to-rosetta` output faults on an M2 running macOS 14.8.9 when it reaches AVX, AVX2, FMA, F16C, RDRAND or a BMI1/BMI2 instruction other than ANDN, BLSI, BLSMSK, BLSR, SHLX, SHRX, SARX, RORX, BZHI, BEXTR, MULX, TZCNT and LZCNT: Rosetta on that host raises SIGILL for them and for RDSEED, RDPID, CLWB, ADCX and ADOX ([matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Only TZCNT, LZCNT, POPCNT, MOVBE, CLFLUSHOPT, PREFETCHW and RDTSCP execute, and RDTSCP sets ECX to 0xAAAAAAAB. CPUID leaf 1 reports POPCNT and none of MOVBE, AVX, FMA, F16C or RDRAND, leaf 7 reads zero, and leaf 0x80000001 reports RDTSCP but neither LZCNT nor PREFETCHW. An M2 Max on macOS 27.2 and an M2 Pro on macOS 27.0.1 report the same CPUID but execute everything except RDSEED, RDPID, CLWB, ADCX and ADOX, so the faults follow the macOS version, not the chip. - `--to-intel` replaces the approximate legacy SSE and VEX.128 register forms of RSQRTPS/RCPPS and VRSQRTPS/VRCPPS with [correctly rounded](../../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). Legacy SSE forms preserve the destination's upper YMM bits; VEX forms clear them. The 256-bit, scalar and memory forms keep the native instruction, as does a site shorter than 5 bytes with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). - XSAVEC raises an illegal instruction under Rosetta and is not converted, because it writes the compacted format. XSAVEOPT, which also faults there, is rewritten to XSAVE in place by the [matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) under `--to-rosetta` (`0F AE /6` becomes `/4`; prefixes and operand are kept). The two differ only in the init and modified optimizations, which let XSAVEOPT skip writing unchanged or initial state ([Intel SDM volume 1](https://cdrdv2-public.intel.com/812382/253665-sdm-vol-1.pdf), chapter 13), so XRSTOR restores the same state from either area. Measured on one M5 with macOS 27.0.1: XSAVE and XRSTOR handle x87, SSE and AVX state in the standard format, although CPUID leaf 1 reports neither XSAVE nor OSXSAVE and leaf 0xD sub-leaf 1 reports neither XSAVEOPT nor XSAVEC. XSAVE also executes on an M5 with macOS 26.4, where it writes an XSTATE_BV of 0x7 and XGETBV returns the same x87, SSE and AVX mask although leaf 1 still reports neither XSAVE nor OSXSAVE; XRSTOR was not exercised there. The converter tests' x86-64 build executes the replacement XSAVE under Rosetta on that host. - `--trace-imports` ([Windows entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp), [Linux entry stub](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp), [macOS entry](../../../core/relinker/elfpatcher/src/macos/MacOsImage.cpp)) leaves every import that is not a PLT slot or a function GOT slot untraced, and the same for a function reached through any other relocation, `sceKernelDlsym`, a weak import, or a module loaded at run time. On Linux the entry stub makes the page of every traced slot readable, writable and executable with `mprotect` before it stores the thunk, so a slot inside `PT_GNU_RELRO` can be replaced; those pages stay writable, and data pages become executable, while tracing is on. On macOS the stub writes the thunk into the bound slot and does not change its protection, because that slot stays in a writable segment - [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. -- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. MULX, PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. -- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. MULX, PDEP and PEXT are not lowered. Balatro (PPSA21402, #1142) stopped on `shlxq` after the ANDN lowering; this lowering has not been run on that title. The execution test runs on an x86-64 host, natively or under Rosetta, and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. -- [BZHI and BEXTR lowering](../../../core/relinker/codegen/src/x86/BitfieldLowering.cpp) builds the mask with `shl` by CL and `not` on a scratch register and applies it with `and` under `--to-rosetta`, with RCX spilled to carry the control operand in a 160-byte frame below the red zone; the control's low byte is compared with the operand size, so an index or start at or past it takes the SDM's unmasked (BZHI) or zero (BEXTR) branch instead of the masked count a scalar shift would use. BZHI sets CF through STC on that branch, and ZF and SF follow the result through `test` or `and`; BEXTR's ZF follows the result and CF and OF clear through `and`, while its SF, which the SDM leaves undefined, follows the result here. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. MULX, PDEP and PEXT are not lowered. The execution test runs on an x86-64 host, natively or under Rosetta, and checks the result and flags against a model of the SDM semantics that matched a Zen 2's BZHI and BEXTR on 122,880 cases, not against the hardware instruction in the same run; precise fault delivery inside a spill sequence has not been verified. +- [BLSI, BLSMSK and BLSR lowering](../../../core/relinker/codegen/src/x86/BlsLowering.cpp) uses scalar LEA, AND, XOR, NEG and OR after a TEST on the source, with two scratch registers saved below the red zone, under `--to-rosetta`, following their SDM entries: SF follows the result, ZF follows the result for BLSI and BLSR and clears for BLSMSK, CF is set for a zero source (BLSR, BLSMSK) or a nonzero source (BLSI) through STC on the branch that needs it, OF clears, and AF and PF are undefined. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws, as for ANDN. PDEP and PEXT are not lowered. The execution test runs only on an x86-64 host and checks the SDM semantics, not the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [SHLX, SHRX, SARX and RORX lowering](../../../core/relinker/codegen/src/x86/ShiftLowering.cpp) uses `shl`, `shr`, `sar` by CL and `ror` by immediate under `--to-rosetta`; the count is masked to the operand size by both the BMI2 instruction and the scalar one. These four leave RFLAGS unchanged where the scalar instructions write them, so the stub saves the flags with `pushfq` into a slot of a 160-byte frame below the red zone and restores them with `popfq` after the result is written; RCX is spilled to carry the count. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. PDEP and PEXT are not lowered. The execution test runs natively on x86-64 or under Rosetta and checks the result against the SDM semantics and the flags against the values loaded before the stub, not against the hardware instruction; precise fault delivery inside a spill sequence has not been verified. +- [BZHI and BEXTR lowering](../../../core/relinker/codegen/src/x86/BitfieldLowering.cpp) builds the mask with `shl` by CL and `not` on a scratch register and applies it with `and` under `--to-rosetta`, with RCX spilled in a 160-byte frame below the red zone. An index or start at or past the operand size takes the SDM's unmasked (BZHI) or zero (BEXTR) branch, not a scalar shift's masked count. BZHI sets CF through STC there, and ZF and SF follow the result; BEXTR clears CF and OF, ZF follows the result, and its SF, undefined in the SDM, follows the result here. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the frame throws, as for ANDN. PDEP and PEXT are not lowered. The execution test runs natively on x86-64 or under Rosetta and checks result and flags against an SDM model that matched a Zen 2's BZHI and BEXTR on 122,880 cases, not against the hardware instruction in the same run; precise fault delivery inside a spill sequence has not been verified. - Relative pointer-width unwind pointer decoding in the [exception frame reader](../../../core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp) (`DW_EH_PE_absptr` and `DW_EH_PE_udata8`) wraps modulo 2^64 to accommodate backward offsets, as standard DWARF unwinders add them. Console unwind table validation for out-of-range displacements is unmeasured. Unsigned offsets narrower than pointer width (`DW_EH_PE_udata2` and `DW_EH_PE_udata4`) cannot express negative displacements and throw when address addition overflows.