From 07798774b60d668231e577b00f2e2e60bd0a0ed3 Mon Sep 17 00:00:00 2001 From: Chronyyx Date: Sun, 25 Oct 2026 03:23:34 -0400 Subject: [PATCH] fix(relinker): frame red-zone leaves whose only push is rbp The check that refuses leaves without pushes that push or pop also refused `push rbp; mov rbp, rsp` leaves without other pushes, whose `pop rbp` it counted, since rbp is not in their list of pushes. A unit test and an execution test now cover such a leaf with a 4-byte prologue and a 2-byte exit. The debt entry also notes that the check refuses shrink-wrapped leaves whose later pushes and pops balance. Co-Authored-By: Claude Opus 5.5 --- .../codegen/src/x86/RedZoneFramer.cpp | 2 +- .../tests/RedZoneFramerExecutionTests.cpp | 7 ++++++- .../codegen/tests/RedZoneFramerTests.cpp | 21 +++++++++++++++++++ docs/dev/debt/relinker.md | 2 +- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/core/relinker/codegen/src/x86/RedZoneFramer.cpp b/core/relinker/codegen/src/x86/RedZoneFramer.cpp index 4b0e48755e..a168da568b 100644 --- a/core/relinker/codegen/src/x86/RedZoneFramer.cpp +++ b/core/relinker/codegen/src/x86/RedZoneFramer.cpp @@ -330,7 +330,7 @@ void frameSegment( if (use.RspBased && use.RspDisp < 0) redZone = true; if (reason != nullptr) continue; if (code[insn.Info.OpcodeOffset] == 0xc9 || code[insn.Info.OpcodeOffset] == 0xc8) { reason = "leave or enter"; continue; } - if (pushed.empty() && allocation == 0 && pushesOrPops(code, insn)) { reason = "push or pop in a leaf without pushes"; continue; } + if (!rbpPrologue && pushed.empty() && allocation == 0 && pushesOrPops(code, insn)) { reason = "push or pop in a leaf without pushes"; continue; } if (rbpPrologue) { if (use.UsesRsp) { reason = "other use of rsp"; continue; } } else { diff --git a/core/relinker/codegen/tests/RedZoneFramerExecutionTests.cpp b/core/relinker/codegen/tests/RedZoneFramerExecutionTests.cpp index 2a744c9f8e..f890466561 100644 --- a/core/relinker/codegen/tests/RedZoneFramerExecutionTests.cpp +++ b/core/relinker/codegen/tests/RedZoneFramerExecutionTests.cpp @@ -139,6 +139,11 @@ std::vector widenedLeaves() { 0x48, 0x03, 0x44, 0x24, 0xf0, 0x48, 0x01, 0xd8, 0xff, 0xc9, 0x75, 0xf4, 0x48, 0x01, 0xd8, 0x5b, 0xc3, }, 8 * (kA + kB) + kB}, + {"frame-pointer leaf without other pushes spilling below rbp", { + 0x55, 0x48, 0x89, 0xe5, 0x48, 0x89, 0x7d, 0xf0, 0x48, 0x89, 0x75, 0xf8, 0x31, 0xc0, 0xb9, 0x08, 0x00, 0x00, 0x00, + 0x48, 0x03, 0x45, 0xf0, 0x48, 0x03, 0x45, 0xf8, 0xff, 0xc9, 0x75, 0xf4, + 0x48, 0x03, 0x45, 0xf0, 0x5d, 0xc3, + }, 8 * (kA + kB) + kA}, }; } @@ -284,7 +289,7 @@ int main() { const auto backward = runLeaves(backwardStubbedLeaves()); require(backward.OperandStubs == 2 && backward.StubbedOperands == 3 && backward.MovedOperands == 5, "the leaves comparing at rsp before a branch were framed with " + std::to_string(backward.OperandStubs) + " operand stubs carrying " + std::to_string(backward.StubbedOperands) + " rsp operands and " + std::to_string(backward.MovedOperands) + " moved in place, not 2 carrying 3 and 5"); const auto widened = runLeaves(widenedLeaves()); - require(widened.WidenedPrologues == 2 && widened.WidenedExits == 2 && widened.WidenedOperands == 3, "the leaves with short prologues and exits were framed with " + std::to_string(widened.WidenedPrologues) + " widened prologues, " + std::to_string(widened.WidenedExits) + " widened exits and " + std::to_string(widened.WidenedOperands) + " rsp operands in them, not 2, 2 and 3"); + require(widened.WidenedPrologues == 3 && widened.WidenedExits == 3 && widened.WidenedOperands == 3, "the leaves with short prologues and exits were framed with " + std::to_string(widened.WidenedPrologues) + " widened prologues, " + std::to_string(widened.WidenedExits) + " widened exits and " + std::to_string(widened.WidenedOperands) + " rsp operands in them, not 3, 3 and 3"); std::cout << "red-zone framer execution tests passed\n"; return 0; } catch (const std::exception& error) { diff --git a/core/relinker/codegen/tests/RedZoneFramerTests.cpp b/core/relinker/codegen/tests/RedZoneFramerTests.cpp index 153ff37690..8464880a98 100644 --- a/core/relinker/codegen/tests/RedZoneFramerTests.cpp +++ b/core/relinker/codegen/tests/RedZoneFramerTests.cpp @@ -313,6 +313,26 @@ void widensShortProloguesAndExits() { require(displacements.size() == 1 && displacements[0].Offset == 0x40 + PushRbx.size() + StoreBelowRsp.size() + 4 && displacements[0].Bytes == std::vector{0x00}, "the load between the widened sites was not moved in place"); } +void widensAShortFramePointerPrologue() { + const std::vector framePrologue{0x55, 0x48, 0x89, 0xe5}; + const std::vector popRbpExit{0x5d, 0xc3}; + std::vector body = StoreBelowPushes; + append(body, LoadBelowPushes); + const auto input = image({framelessLeaf(framePrologue, body, popRbpExit)}); + Codegen::RedZoneFrameReport report; + std::vector displacements; + const auto sites = Codegen::FrameRedZoneFunctions(input.bytes, input.segments, input.functions, {}, report, displacements); + require(report.Framed == 1 && report.FramedWithoutFramePointer == 0 && report.WidenedPrologues == 1 && report.WidenedExits == 1 && report.WidenedOperands == 0 && sites.size() == 2 && displacements.empty(), "a frame-pointer leaf without other pushes was not framed by widening its 4-byte prologue and 2-byte exit"); + std::vector entryBody = framePrologue; + append(entryBody, {0x48, 0x8d, 0x64, 0x24, 0x80}); + append(entryBody, StoreBelowPushes); + require(sites[0].OriginalBytes.size() == framePrologue.size() + StoreBelowPushes.size() && sites[0].Body == stubBody(entryBody), "the store through rbp was not copied unchanged after rsp is lowered by the whole red zone"); + std::vector exitBody = LoadBelowPushes; + append(exitBody, {0x48, 0x8d, 0xa4, 0x24, 0x80, 0x00, 0x00, 0x00}); + append(exitBody, popRbpExit); + require(sites[1].OriginalBytes.size() == LoadBelowPushes.size() + popRbpExit.size() && sites[1].Body == stubBody(exitBody), "the load through rbp was not copied unchanged ahead of raising rsp and popping rbp"); +} + void framesLeavesWithoutPushes() { std::vector body = StoreBelowRsp; append(body, LoadBelowRsp); @@ -676,6 +696,7 @@ int main() { skipsShortOperandsAtRspWhoseWidenedStubCoversABranchTarget(); skipsLeavesWithOperandsThatCannotMove(); widensShortProloguesAndExits(); + widensAShortFramePointerPrologue(); framesLeavesWithoutPushes(); skipsFragmentsThatPopWithoutPushing(); skipsWideningOverBranchTargets(); diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index 53b174ffc5..ac11efa2f2 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -14,7 +14,7 @@ - [SDK memory-management syscall traps](../../../core/relinker/relinker/src/analysis/SyscallScanner.cpp) under `--trap-sdk-syscall-stubs` recognize three exact instruction sequences from [ps5-payload-dev SDK libc/mman.c](https://github.com/ps5-payload-dev/sdk/blob/12cf3dda40/libc/mman.c), observed in Nuvio Maxd 0.5.15 (PPSA99288): syscall 74 and the two syscall 477 paths, with their argument preparation and carry-flag error handling. Matching requires a decoded instruction boundary and the complete sequence inside the scanned code range. Other SDK/compiler output remains rejected. The matched syscall becomes `UD2`; this does not implement mmap or mprotect and a call still faults. - The [instruction decoder](../../../core/relinker/codegen/src/x86/X64InstructionDecoder.cpp) reads a 16-bit displacement for `66 E8`, `66 E9` and `66 0F 80`-`8F` without REX.W, as AMD processors, the console included, do (binutils `objdump -M amd64`; REX.W selects the 32-bit form). AMD also truncates the target of every near relative branch with that prefix, rel8, LOOP and JRCXZ forms included, to 16 bits. Instruction metadata marks these branches as truncated and reports no branch target for them, and stubs refuse to move them. Compilers do not emit these forms. - Converted Linux main executables are [placed at `0x400000`](../../../core/relinker/relinker/src/pipeline/Conversion.cpp) by default, as `ET_EXEC` without ASLR, not only with `--linux-fixed-base`: the console executable lies inside the 40-bit window that the title's own `sceLibcMspaceCreate` checks (#3043). Where the console loads an executable that does not require an address is not measured; `0x400000` is the base that executables reading their GOT through `app0/imports.txt` assume (#4013). The relinker refuses an image that spans 2 GiB or more, so the executable ends below the guest arena at `0x200000000`. With `--lazy-binding` the executable stays position independent. -- `--frame-red-zone` frames only leaf functions that start with pushes (`push rbp; mov rbp, rsp; push ...`, or pushes alone, which may be followed by `sub $n, %rsp` or `lea -n(%rsp), %rsp`) or with neither pushes nor an allocation, and leave through exits that undo that allocation with `add` or `lea`, pop the same registers and `ret` ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)). A prologue or exit shorter than its five-byte jump takes in the straight-line instructions after or before it. The leaf keeps its red zone when those cannot make five bytes, when a branch targets one of them or the first instruction of a widened prologue, or when a leaf without pushes pushes or pops, as a function fragment does. Leaves with a frame pointer that allocate, exits that free another size, and functions with indirect jumps or other RSP arithmetic keep their red zone too. +- `--frame-red-zone` frames only leaf functions that start with pushes (`push rbp; mov rbp, rsp; push ...`, or pushes alone followed or not by a `sub` or `lea` allocation) or with neither pushes nor an allocation, and leave through exits that undo the allocation, pop the same registers and `ret` ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)). A prologue or exit shorter than its five-byte jump takes in the straight-line instructions after or before it. The leaf keeps its red zone when those cannot make five bytes, when a branch targets one of them or the first instruction of a widened prologue, or when a leaf without pushes pushes or pops: that refuses fragments that pop registers pushed before them, and also shrink-wrapped leaves whose later pushes and pops balance, which could be framed. Leaves with a frame pointer that allocate, exits that free another size and functions with indirect jumps or other RSP arithmetic keep their red zone too. - In a red-zone leaf without a frame pointer, `--frame-red-zone` lowers RSP after the prologue by the deepest displacement below RSP, rounded up to 16, and moves every RSP operand by the same amount ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)). A leaf that takes the address of a slot below RSP, or uses an RSP operand with an index register, is lowered by the whole 128 bytes, since a pointer or an index can reach deeper than any displacement; values below RSP reached through any other register are not seen. Leaves with a frame pointer are lowered by 128 bytes. Inside a moved frame the CFA rules of the guest's unwind tables are off by the shift. - An RSP operand that `--frame-red-zone` cannot move in place, an 8-bit displacement the shift takes past 0x7f or one without a displacement, moves to a stub with the instructions after it up to five bytes and is re-encoded there with the moved displacement ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)). The leaf is skipped when that site would take in a branch target, a branch, ret or RIP-relative instruction, or another site, and when the operand is EVEX, whose 8-bit displacement is scaled. A game patch over a framed site or a moved displacement is refused. - In `--frame-red-zone`, function bounds come from the executable's `.eh_frame` FDEs: each function is decoded from its FDE start to its FDE end, or to int3 padding before it, and the relinker stops with an error when the executable has none. Code outside every FDE, such as a function without unwind information, is not framed ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)).