#ifndef CODEGEN_TESTS_LOWERINGEXECUTION_HPP #define CODEGEN_TESTS_LOWERINGEXECUTION_HPP #include #include #include #include #include #include #include #include #if (defined(__linux__) || defined(__APPLE__)) && defined(__x86_64__) #include #define LOWERING_EXECUTION_AVAILABLE 1 #else #define LOWERING_EXECUTION_AVAILABLE 0 #endif namespace LoweringExecution { using Bytes = std::vector; struct State { std::array Input{}; std::array Output{}; std::uint64_t Flags = 0; std::array RedZone{}; std::uint64_t InputFlags = 0; }; inline constexpr std::uint64_t kSentinel = 0xBADC0FFEE0DDF00Dull; inline constexpr std::uint64_t kFlagMask = 0x8C1u; inline constexpr std::uint64_t kCarry = 0x1u; inline constexpr std::uint64_t kZero = 0x40u; inline constexpr std::uint64_t kSign = 0x80u; inline constexpr std::uint64_t kBaseFlags = 0x202u; inline State InitialState() { State state; for (std::size_t reg = 0; reg < 16; ++reg) state.Input[reg] = 0xABCD123456789000ull + reg * 0x1234567ull; state.InputFlags = kBaseFlags | kFlagMask; return state; } #if LOWERING_EXECUTION_AVAILABLE class Execution { public: using Emit = std::function; Execution(const Bytes& site, std::uint64_t memoryValue, bool ripRelative, const Emit& emit, std::size_t trailing = 0) { for (auto reg : {3, 5, 12, 13, 14, 15}) push(static_cast(reg)); memory(0x8D, 4, 4, -128); rip({0x48, 0x89, 0x3D}, 0); rip({0x48, 0x89, 0x25}, 8); memory(0x89, 4, 7, 32); memory(0xFF, 6, 7, 49 * 8); bytes.push_back(0x9D); bytes.insert(bytes.end(), {0x48, 0xB8}); append64(kSentinel); for (int displacement = -128; displacement < 0; displacement += 8) memory(0x89, 0, 4, displacement); memory(0x8B, 0, 7, 16 * 8); memory(0x89, 0, 4, 64); for (std::uint8_t reg = 0; reg < 16; ++reg) if (reg != 4 && reg != 7) memory(0x8B, reg, 7, reg * 8); memory(0x8B, 7, 7, 7 * 8); const auto siteOffset = bytes.size(); auto relocated = site; if (ripRelative) { const auto displacement = static_cast(4096 + 24 - (siteOffset + site.size())); std::memcpy(relocated.data() + relocated.size() - 4 - trailing, &displacement, sizeof(displacement)); } Codegen::StubBodyBuilder builder; emit(builder, relocated); auto body = builder.Finish(); const auto stubOffset = bytes.size(); Codegen::ApplyStubRelocations(body.Bytes, body.Relocations, siteOffset, stubOffset, 0); bytes.insert(bytes.end(), body.Bytes.begin(), body.Bytes.begin() + static_cast(body.ReturnBranchOffset)); rip({0x48, 0x89, 0x25}, 16); rip({0x48, 0x8B, 0x25}, 8); memory(0x8D, 4, 4, -256); for (std::uint8_t reg = 0; reg < 16; ++reg) if (reg != 4) push(reg); bytes.push_back(0x9C); rip({0x48, 0x8B, 0x3D}, 0); memory(0x8B, 0, 4, 0); memory(0x89, 0, 7, 32 * 8); int offset = 8; for (int reg = 15; reg >= 0; --reg) { if (reg == 4) continue; memory(0x8B, 0, 4, offset); memory(0x89, 0, 7, (16 + reg) * 8); offset += 8; } rip({0x48, 0x8B, 0x05}, 16); memory(0x89, 0, 7, 20 * 8); for (int slot = 0; slot < 16; ++slot) { memory(0x8B, 0, 4, 256 + 128 - 128 + slot * 8); memory(0x89, 0, 7, (33 + slot) * 8); } rip({0x48, 0x8B, 0x25}, 8); memory(0x8D, 4, 4, 128); for (auto reg : {15, 14, 13, 12, 5, 3}) pop(static_cast(reg)); bytes.push_back(0xC3); if (bytes.size() >= 4096) throw std::runtime_error("Execution wrapper exceeded its code page"); code = static_cast(mmap(nullptr, 8192, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); if (code == MAP_FAILED) throw std::runtime_error("Cannot allocate execution test pages"); std::memcpy(code, bytes.data(), bytes.size()); std::memcpy(code + 4096 + 24, &memoryValue, sizeof(memoryValue)); if (mprotect(code, 4096, PROT_READ | PROT_EXEC) != 0) throw std::runtime_error("Cannot make lowering test code executable"); } ~Execution() { if (code != MAP_FAILED) munmap(code, 8192); } Execution(const Execution&) = delete; Execution& operator=(const Execution&) = delete; void Run(State& state) const { reinterpret_cast(code)(&state); } const void* Memory() const { return code + 4096 + 24; } private: Bytes bytes; std::uint8_t* code = reinterpret_cast(MAP_FAILED); void append32(std::int32_t value) { for (unsigned shift = 0; shift < 32; shift += 8) bytes.push_back(static_cast(static_cast(value) >> shift)); } void append64(std::uint64_t value) { for (unsigned shift = 0; shift < 64; shift += 8) bytes.push_back(static_cast(value >> shift)); } void rip(std::initializer_list opcode, int slot) { bytes.insert(bytes.end(), opcode); append32(static_cast(4096 + slot - (bytes.size() + 4))); } void push(std::uint8_t reg) { if (reg >= 8) bytes.push_back(0x41); bytes.push_back(static_cast(0x50 | (reg & 7))); } void pop(std::uint8_t reg) { if (reg >= 8) bytes.push_back(0x41); bytes.push_back(static_cast(0x58 | (reg & 7))); } void memory(std::uint8_t opcode, std::uint8_t reg, std::uint8_t base, int displacement) { bytes.insert(bytes.end(), {static_cast(0x48 | ((reg & 8) >> 1) | ((base & 8) >> 3)), opcode, static_cast(0x80 | ((reg & 7) << 3) | (base & 7))}); if ((base & 7) == 4) bytes.push_back(0x24); append32(displacement); } }; #endif } #endif