mirror of
https://github.com/boykopovar/AnyPS5.git
synced 2026-10-27 00:28:14 +00:00
feat(relinker): frame red-zone leaves whose prologue or exit is shorter than a jump
--frame-red-zone replaces a leaf's prologue and each exit with a 5-byte jump to a stub, so it skipped leaves whose pushes or pop-and-ret exit were shorter than that, and leaves with neither pushes nor an allocation. A short prologue now takes in the straight-line instructions after it and a short exit those before it, re-encoded in the stub with the moved displacement. Widening stops at branches, rets, RIP-relative instructions, pushes and pops, and the leaf is skipped when a branch targets the instructions taken in or the first instruction of a widened prologue, or when a leaf without pushes pushes or pops. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
af49a07885
commit
c697fb0ba9
7 files changed
+178
-20
No files matched your search
@@ -22,6 +22,9 @@ struct RedZoneFrameReport {
|
||||
std::size_t Framed = 0;
|
||||
std::size_t FramedWithoutFramePointer = 0;
|
||||
std::size_t Exits = 0;
|
||||
std::size_t WidenedPrologues = 0;
|
||||
std::size_t WidenedExits = 0;
|
||||
std::size_t WidenedOperands = 0;
|
||||
std::size_t MovedOperands = 0;
|
||||
std::size_t OperandStubs = 0;
|
||||
std::size_t StubbedOperands = 0;
|
||||
|
||||
@@ -47,6 +47,16 @@ std::optional<std::uint8_t> poppedRegister(const std::uint8_t* code, const Insn&
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
bool pushesOrPops(const std::uint8_t* code, const Insn& insn) {
|
||||
if (pushedRegister(code, insn) || poppedRegister(code, insn)) return true;
|
||||
const auto& info = insn.Info;
|
||||
const auto lead = code[info.OpcodeOffset];
|
||||
if (info.RexPrefix == 0 && (lead == 0xc4 || lead == 0xc5 || lead == 0x62 || lead == 0x8f) && info.Opcode != 0x8f) return false;
|
||||
if (info.IsTwoByteOpcode) return info.Opcode == 0xa0 || info.Opcode == 0xa1 || info.Opcode == 0xa8 || info.Opcode == 0xa9;
|
||||
const auto reg = static_cast<std::uint8_t>((info.ModRmByte >> 3) & 7);
|
||||
return (info.Opcode >= 0x50 && info.Opcode <= 0x5f) || info.Opcode == 0x68 || info.Opcode == 0x6a || info.Opcode == 0x9c || info.Opcode == 0x9d || (info.Opcode == 0x8f && info.HasModRm && reg == 0) || (info.Opcode == 0xff && info.HasModRm && reg == 6);
|
||||
}
|
||||
|
||||
std::optional<std::int64_t> rspAdjustment(const std::uint8_t* code, const Insn& insn) {
|
||||
std::int32_t value = 0;
|
||||
if (insn.Length == 4 && code[0] == 0x48 && code[1] == 0x83 && (code[2] == 0xec || code[2] == 0xc4)) {
|
||||
@@ -208,6 +218,16 @@ struct OperandStub {
|
||||
std::size_t Operands = 0;
|
||||
};
|
||||
|
||||
struct ExitSite {
|
||||
std::size_t First;
|
||||
std::size_t Tail;
|
||||
std::size_t Begin;
|
||||
std::size_t TailBegin;
|
||||
std::size_t End;
|
||||
std::vector<std::uint8_t> Lead;
|
||||
std::size_t Operands = 0;
|
||||
};
|
||||
|
||||
void frameSegment(
|
||||
const std::vector<std::uint8_t>& fileBytes,
|
||||
const Domain::ProgramHeader& ph,
|
||||
@@ -286,7 +306,7 @@ void frameSegment(
|
||||
bool leaf = true;
|
||||
bool redZone = false;
|
||||
const char* reason = nullptr;
|
||||
std::vector<std::pair<std::size_t, std::size_t>> exits;
|
||||
std::vector<ExitSite> exits;
|
||||
std::map<std::size_t, std::pair<std::size_t, std::vector<std::uint8_t>>> moves;
|
||||
std::vector<std::size_t> stubbed;
|
||||
const auto functionEnd = limit;
|
||||
@@ -310,6 +330,7 @@ void frameSegment(
|
||||
if (use.RspBased && use.RspDisp < 0) redZone = true;
|
||||
if (reason != nullptr) continue;
|
||||
if (code[insn.Info.OpcodeOffset] == 0xc9 || code[insn.Info.OpcodeOffset] == 0xc8) { reason = "leave or enter"; continue; }
|
||||
if (pushed.empty() && allocation == 0 && pushesOrPops(code, insn)) { reason = "push or pop in a leaf without pushes"; continue; }
|
||||
if (rbpPrologue) {
|
||||
if (use.UsesRsp) { reason = "other use of rsp"; continue; }
|
||||
} else {
|
||||
@@ -335,24 +356,60 @@ void frameSegment(
|
||||
if (q == p || !frees.contains(q - 1)) { reason = "exit does not free the prologue's allocation"; continue; }
|
||||
--q;
|
||||
}
|
||||
const auto siteStart = body[q].Offset;
|
||||
const auto siteEnd = insn.Offset + 1;
|
||||
if (siteEnd - siteStart < kJumpBytes) { reason = "exit shorter than a jump"; continue; }
|
||||
if (targetInside(siteStart, siteEnd)) { reason = "branch into an exit"; continue; }
|
||||
exits.emplace_back(siteStart, siteEnd);
|
||||
ExitSite exit{q, q, body[q].Offset, body[q].Offset, insn.Offset + 1, {}};
|
||||
while (exit.End - exit.Begin < kJumpBytes && exit.First > p) {
|
||||
const auto j = exit.First - 1;
|
||||
const auto& prior = body[j];
|
||||
if (prior.Offset + prior.Length != exit.Begin || prior.Info.FlowKind != ControlFlowKind::Sequential || prior.Info.HasRipRelativeDisp || frees.contains(j) || pushesOrPops(at(j), prior)) break;
|
||||
const auto priorUse = memoryUse(at(j), prior);
|
||||
const auto bytes = priorUse.RspBased ? withDisplacement(at(j), prior, priorUse, priorUse.RspDisp + shift) : std::vector<std::uint8_t>(at(j), at(j) + prior.Length);
|
||||
if (!bytes) break;
|
||||
exit.Lead.insert(exit.Lead.begin(), bytes->begin(), bytes->end());
|
||||
if (priorUse.RspBased) ++exit.Operands;
|
||||
exit.Begin = prior.Offset;
|
||||
exit.First = j;
|
||||
}
|
||||
if (exit.End - exit.Begin < kJumpBytes) { reason = "exit shorter than a jump"; continue; }
|
||||
if (targetInside(exit.Begin, exit.End)) { reason = exit.First == exit.Tail ? "branch into an exit" : "branch into the instructions before a short exit"; continue; }
|
||||
exits.push_back(std::move(exit));
|
||||
}
|
||||
}
|
||||
if (!leaf || !redZone) continue;
|
||||
++report.RedZoneFunctions;
|
||||
const auto prologueEnd = p > 0 ? body[p - 1].Offset + body[p - 1].Length : start;
|
||||
if (reason == nullptr && ((pushed.empty() && allocation == 0) || prologueEnd - start < kJumpBytes)) reason = "prologue shorter than a jump";
|
||||
if (reason == nullptr && targetInside(start, prologueEnd)) reason = "branch into the prologue";
|
||||
auto entryEnd = prologueEnd;
|
||||
auto entryLast = p;
|
||||
std::vector<std::uint8_t> entryTail;
|
||||
std::size_t entryOperands = 0;
|
||||
while (entryEnd - start < kJumpBytes && entryLast < body.size()) {
|
||||
const auto& covered = body[entryLast];
|
||||
if (covered.Offset != entryEnd || covered.Info.FlowKind != ControlFlowKind::Sequential || covered.Info.HasRipRelativeDisp || frees.contains(entryLast) || pushesOrPops(at(entryLast), covered)) break;
|
||||
if (std::any_of(exits.begin(), exits.end(), [&](const ExitSite& exit) { return covered.Offset < exit.End && exit.Begin < covered.Offset + covered.Length; })) break;
|
||||
const auto use = memoryUse(at(entryLast), covered);
|
||||
const auto bytes = use.RspBased ? withDisplacement(at(entryLast), covered, use, use.RspDisp + shift) : std::vector<std::uint8_t>(at(entryLast), at(entryLast) + covered.Length);
|
||||
if (!bytes) break;
|
||||
entryTail.insert(entryTail.end(), bytes->begin(), bytes->end());
|
||||
if (use.RspBased) ++entryOperands;
|
||||
entryEnd += covered.Length;
|
||||
++entryLast;
|
||||
}
|
||||
if (reason == nullptr && entryEnd - start < kJumpBytes) reason = "prologue shorter than a jump";
|
||||
if (reason == nullptr && targetInside(start, entryEnd)) reason = entryEnd == prologueEnd ? "branch into the prologue" : "branch into the instructions after a short prologue";
|
||||
if (reason == nullptr && entryEnd != prologueEnd && targets.contains(start)) reason = "branch to the start of a widened prologue";
|
||||
if (reason == nullptr && exits.empty()) reason = "no ret exit";
|
||||
if (reason == nullptr && (overlapsExisting(start, prologueEnd) || std::any_of(exits.begin(), exits.end(), [&](const auto& exit) { return overlapsExisting(exit.first, exit.second); }))) reason = "overlaps another stub";
|
||||
if (reason == nullptr && std::any_of(exits.begin(), exits.end(), [&](const ExitSite& exit) { return exit.Begin < entryEnd; })) reason = "prologue shorter than a jump";
|
||||
if (reason == nullptr && (overlapsExisting(start, entryEnd) || std::any_of(exits.begin(), exits.end(), [&](const ExitSite& exit) { return overlapsExisting(exit.Begin, exit.End); }))) reason = "overlaps another stub";
|
||||
const auto carried = [&](const std::size_t k) {
|
||||
moves.erase(k);
|
||||
stubbed.erase(std::remove(stubbed.begin(), stubbed.end(), k), stubbed.end());
|
||||
};
|
||||
for (auto k = p; k < entryLast; ++k) carried(k);
|
||||
for (const auto& exit : exits)
|
||||
for (auto k = exit.First; k < exit.Tail; ++k) carried(k);
|
||||
std::vector<OperandStub> stubs;
|
||||
const auto overlapsSite = [&](const std::size_t begin, const std::size_t end) {
|
||||
if (begin < prologueEnd && start < end) return true;
|
||||
if (std::any_of(exits.begin(), exits.end(), [&](const auto& exit) { return begin < exit.second && exit.first < end; })) return true;
|
||||
if (begin < entryEnd && start < end) return true;
|
||||
if (std::any_of(exits.begin(), exits.end(), [&](const ExitSite& exit) { return begin < exit.End && exit.Begin < end; })) return true;
|
||||
return std::any_of(stubs.begin(), stubs.end(), [&](const OperandStub& stub) { return begin < stub.End && stub.Begin < end; });
|
||||
};
|
||||
for (const auto k : stubbed) {
|
||||
@@ -405,18 +462,27 @@ void frameSegment(
|
||||
auto entry = original(start, prologueEnd);
|
||||
const auto lower = lowerRsp(shift);
|
||||
entry.insert(entry.end(), lower.begin(), lower.end());
|
||||
addSite(start, prologueEnd, std::move(entry));
|
||||
for (const auto& exit : exits) {
|
||||
auto code = raiseRsp(shift);
|
||||
const auto tail = original(exit.first, exit.second);
|
||||
entry.insert(entry.end(), entryTail.begin(), entryTail.end());
|
||||
addSite(start, entryEnd, std::move(entry));
|
||||
for (auto& exit : exits) {
|
||||
auto code = std::move(exit.Lead);
|
||||
const auto raise = raiseRsp(shift);
|
||||
code.insert(code.end(), raise.begin(), raise.end());
|
||||
const auto tail = original(exit.TailBegin, exit.End);
|
||||
code.insert(code.end(), tail.begin(), tail.end());
|
||||
addSite(exit.first, exit.second, std::move(code));
|
||||
addSite(exit.Begin, exit.End, std::move(code));
|
||||
}
|
||||
for (auto& stub : stubs) addSite(stub.Begin, stub.End, std::move(stub.Body));
|
||||
for (auto& [k, move] : moves) displacements.push_back({ph.Offset + move.first, std::move(move.second)});
|
||||
++report.Framed;
|
||||
if (!rbpPrologue) ++report.FramedWithoutFramePointer;
|
||||
report.Exits += exits.size();
|
||||
if (entryLast != p) ++report.WidenedPrologues;
|
||||
report.WidenedOperands += entryOperands;
|
||||
for (const auto& exit : exits) {
|
||||
if (exit.First != exit.Tail) ++report.WidenedExits;
|
||||
report.WidenedOperands += exit.Operands;
|
||||
}
|
||||
report.MovedOperands += moves.size();
|
||||
report.OperandStubs += stubs.size();
|
||||
for (const auto& stub : stubs) report.StubbedOperands += stub.Operands;
|
||||
|
||||
@@ -127,6 +127,21 @@ std::vector<Leaf> backwardStubbedLeaves() {
|
||||
};
|
||||
}
|
||||
|
||||
std::vector<Leaf> widenedLeaves() {
|
||||
return {
|
||||
{"leaf without pushes spilling 0x10 below rsp and adding a spill before its ret", {
|
||||
0x48, 0x89, 0x7c, 0x24, 0xf0, 0x48, 0x89, 0x74, 0x24, 0xf8, 0x31, 0xc0, 0xb9, 0x08, 0x00, 0x00, 0x00,
|
||||
0x48, 0x03, 0x44, 0x24, 0xf0, 0x48, 0x03, 0x44, 0x24, 0xf8, 0xff, 0xc9, 0x75, 0xf2,
|
||||
0x48, 0x03, 0x44, 0x24, 0xf0, 0xc3,
|
||||
}, 8 * (kA + kB) + kA},
|
||||
{"leaf pushing only rbx, spilling below rsp and adding rbx before its pop", {
|
||||
0x53, 0x48, 0x89, 0x7c, 0x24, 0xf0, 0x48, 0x89, 0xf3, 0x31, 0xc0, 0xb9, 0x08, 0x00, 0x00, 0x00,
|
||||
0x48, 0x03, 0x44, 0x24, 0xf0, 0x48, 0x01, 0xd8, 0xff, 0xc9, 0x75, 0xf4,
|
||||
0x48, 0x01, 0xd8, 0x5b, 0xc3,
|
||||
}, 8 * (kA + kB) + kB},
|
||||
};
|
||||
}
|
||||
|
||||
void writeRel32(std::uint8_t* at, const std::uint64_t target, const std::uint64_t next) {
|
||||
const auto displacement = static_cast<std::int64_t>(target) - static_cast<std::int64_t>(next);
|
||||
require(displacement >= std::numeric_limits<std::int32_t>::min() && displacement <= std::numeric_limits<std::int32_t>::max(), "a stub jump is out of rel32 range");
|
||||
@@ -268,6 +283,8 @@ int main() {
|
||||
require(stubbed.OperandStubs == 3 && stubbed.StubbedOperands == 5, "the leaves were framed with " + std::to_string(stubbed.OperandStubs) + " operand stubs carrying " + std::to_string(stubbed.StubbedOperands) + " rsp operands, not 3 carrying 5");
|
||||
const auto backward = runLeaves(backwardStubbedLeaves());
|
||||
require(backward.OperandStubs == 2 && backward.StubbedOperands == 3 && backward.MovedOperands == 5, "the leaves comparing at rsp before a branch were framed with " + std::to_string(backward.OperandStubs) + " operand stubs carrying " + std::to_string(backward.StubbedOperands) + " rsp operands and " + std::to_string(backward.MovedOperands) + " moved in place, not 2 carrying 3 and 5");
|
||||
const auto widened = runLeaves(widenedLeaves());
|
||||
require(widened.WidenedPrologues == 2 && widened.WidenedExits == 2 && widened.WidenedOperands == 3, "the leaves with short prologues and exits were framed with " + std::to_string(widened.WidenedPrologues) + " widened prologues, " + std::to_string(widened.WidenedExits) + " widened exits and " + std::to_string(widened.WidenedOperands) + " rsp operands in them, not 2, 2 and 3");
|
||||
std::cout << "red-zone framer execution tests passed\n";
|
||||
return 0;
|
||||
} catch (const std::exception& error) {
|
||||
|
||||
@@ -275,14 +275,81 @@ void skipsLeavesWithOperandsThatCannotMove() {
|
||||
std::vector<std::uint8_t> beforeTarget = StoreBelowRsp;
|
||||
append(beforeTarget, LoadAtRsp);
|
||||
append(beforeTarget, {0xff, 0xc0, 0x3b, 0x44, 0x24, 0xf0, 0x72, 0xf8});
|
||||
const auto input = image({framelessLeaf(FramelessPrologue, evex, FramelessExit), framelessLeaf(FramelessPrologue, beforeExit, FramelessExit), framelessLeaf(FramelessPrologue, beforeTarget, FramelessExit), framelessLeaf({0x41, 0x56, 0x41, 0x57}, StoreBelowRsp, {0x41, 0x5f, 0x41, 0x5e, 0xc3})});
|
||||
std::vector<std::uint8_t> afterShortPrologue = LoadRipRelative;
|
||||
append(afterShortPrologue, StoreBelowRsp);
|
||||
const auto input = image({framelessLeaf(FramelessPrologue, evex, FramelessExit), framelessLeaf(FramelessPrologue, beforeExit, FramelessExit), framelessLeaf(FramelessPrologue, beforeTarget, FramelessExit), framelessLeaf({0x41, 0x56, 0x41, 0x57}, afterShortPrologue, {0x41, 0x5f, 0x41, 0x5e, 0xc3})});
|
||||
Codegen::RedZoneFrameReport report;
|
||||
std::vector<Codegen::RedZoneDisplacement> displacements;
|
||||
const auto sites = Codegen::FrameRedZoneFunctions(input.bytes, input.segments, input.functions, {}, report, displacements);
|
||||
require(sites.empty() && displacements.empty() && report.RedZoneFunctions == 4, "a leaf whose rsp operands can neither move nor be stubbed, or whose prologue is too short, was framed");
|
||||
require(report.Skipped.at("rsp operand that cannot move with rsp") == 1, "an EVEX operand, whose 8-bit displacement is scaled, was not refused");
|
||||
require(report.Skipped.at("rsp operand that cannot move and has no room for a stub") == 2, "an operand at rsp followed by an exit or a branch target was not refused for want of room");
|
||||
require(report.Skipped.at("prologue shorter than a jump") == 1, "the leaf with a short prologue was not reported with its reason");
|
||||
require(report.Skipped.at("prologue shorter than a jump") == 1, "the leaf whose short prologue is followed by a rip-relative instruction was not reported with its reason");
|
||||
}
|
||||
|
||||
const std::vector<std::uint8_t> PushRbx{0x53};
|
||||
const std::vector<std::uint8_t> PopRbxExit{0x5b, 0xc3};
|
||||
const std::vector<std::uint8_t> Ret{0xc3};
|
||||
|
||||
void widensShortProloguesAndExits() {
|
||||
std::vector<std::uint8_t> body = StoreBelowRsp;
|
||||
append(body, LoadBelowRsp);
|
||||
append(body, AddRdxToRax);
|
||||
const auto input = image({framelessLeaf(PushRbx, body, PopRbxExit)});
|
||||
Codegen::RedZoneFrameReport report;
|
||||
std::vector<Codegen::RedZoneDisplacement> displacements;
|
||||
const auto sites = Codegen::FrameRedZoneFunctions(input.bytes, input.segments, input.functions, {}, report, displacements);
|
||||
require(report.Framed == 1 && report.WidenedPrologues == 1 && report.WidenedExits == 1 && report.WidenedOperands == 1 && report.MovedOperands == 1 && sites.size() == 2 && report.Skipped.empty(), "a leaf with a 1-byte push and a 2-byte exit was not framed by widening both sites");
|
||||
const auto& entry = sites[0];
|
||||
require(entry.Address == Base + 0x40 && entry.Length == PushRbx.size() + StoreBelowRsp.size(), "the prologue site does not run on over the store after the push");
|
||||
require(entry.Body == stubBody({0x53, 0x48, 0x8d, 0x64, 0x24, 0xf0, 0x48, 0x89, 0x7c, 0x24, 0x00}) && entry.ReturnBranchOffset == 11, "the store after the push was not moved by the shift after rsp is lowered");
|
||||
const auto& exit = sites[1];
|
||||
const auto add = Base + 0x40 + PushRbx.size() + StoreBelowRsp.size() + LoadBelowRsp.size();
|
||||
require(exit.Address == add && exit.Length == AddRdxToRax.size() + PopRbxExit.size(), "the exit site does not start at the add before the pop");
|
||||
std::vector<std::uint8_t> exitBody = AddRdxToRax;
|
||||
append(exitBody, {0x48, 0x8d, 0xa4, 0x24, 0x10, 0x00, 0x00, 0x00});
|
||||
append(exitBody, PopRbxExit);
|
||||
require(exit.Body == stubBody(exitBody), "the add was not copied ahead of raising rsp and the pop");
|
||||
require(displacements.size() == 1 && displacements[0].Offset == 0x40 + PushRbx.size() + StoreBelowRsp.size() + 4 && displacements[0].Bytes == std::vector<std::uint8_t>{0x00}, "the load between the widened sites was not moved in place");
|
||||
}
|
||||
|
||||
void framesLeavesWithoutPushes() {
|
||||
std::vector<std::uint8_t> body = StoreBelowRsp;
|
||||
append(body, LoadBelowRsp);
|
||||
const auto input = image({framelessLeaf({}, body, Ret)});
|
||||
Codegen::RedZoneFrameReport report;
|
||||
std::vector<Codegen::RedZoneDisplacement> displacements;
|
||||
const auto sites = Codegen::FrameRedZoneFunctions(input.bytes, input.segments, input.functions, {}, report, displacements);
|
||||
require(report.Framed == 1 && report.WidenedPrologues == 1 && report.WidenedExits == 1 && report.WidenedOperands == 2 && sites.size() == 2 && displacements.empty(), "a leaf that only spills below rsp and returns was not framed");
|
||||
require(sites[0].Address == Base + 0x40 && sites[0].OriginalBytes == StoreBelowRsp, "the entry site of a leaf without pushes does not cover its first instruction");
|
||||
require(sites[0].Body == stubBody({0x48, 0x8d, 0x64, 0x24, 0xf0, 0x48, 0x89, 0x7c, 0x24, 0x00}), "the entry stub does not lower rsp before the store moved by the shift");
|
||||
std::vector<std::uint8_t> exitBody{0x48, 0x8b, 0x44, 0x24, 0x00, 0x48, 0x8d, 0xa4, 0x24, 0x10, 0x00, 0x00, 0x00};
|
||||
append(exitBody, Ret);
|
||||
require(sites[1].OriginalBytes.size() == LoadBelowRsp.size() + Ret.size() && sites[1].Body == stubBody(exitBody), "the exit stub does not run the load moved by the shift, raise rsp and return");
|
||||
}
|
||||
|
||||
void skipsFragmentsThatPopWithoutPushing() {
|
||||
std::vector<std::uint8_t> body = StoreBelowRsp;
|
||||
append(body, LoadBelowRsp);
|
||||
const auto input = image({framelessLeaf({}, body, PopRbxExit)});
|
||||
Codegen::RedZoneFrameReport report;
|
||||
std::vector<Codegen::RedZoneDisplacement> displacements;
|
||||
const auto sites = Codegen::FrameRedZoneFunctions(input.bytes, input.segments, input.functions, {}, report, displacements);
|
||||
require(sites.empty() && displacements.empty() && report.Skipped.at("push or pop in a leaf without pushes") == 1, "a fragment without pushes that pops a register its caller pushed was framed");
|
||||
}
|
||||
|
||||
void skipsWideningOverBranchTargets() {
|
||||
const std::vector<std::uint8_t> loopToStart{0x48, 0x89, 0x7c, 0x24, 0xf0, 0xff, 0xc9, 0x75, 0xf7, 0x48, 0x8b, 0x44, 0x24, 0xf0, 0xc3};
|
||||
std::vector<std::uint8_t> jumpToPop = StoreBelowRsp;
|
||||
append(jumpToPop, {0x85, 0xff, 0x74, 0x03});
|
||||
append(jumpToPop, AddRdxToRax);
|
||||
const auto input = image({loopToStart, framelessLeaf(PushRbx, jumpToPop, PopRbxExit)});
|
||||
Codegen::RedZoneFrameReport report;
|
||||
std::vector<Codegen::RedZoneDisplacement> displacements;
|
||||
const auto sites = Codegen::FrameRedZoneFunctions(input.bytes, input.segments, input.functions, {}, report, displacements);
|
||||
require(sites.empty() && displacements.empty() && report.RedZoneFunctions == 2, "a leaf with a branch into a widened site was framed");
|
||||
require(report.Skipped.at("branch to the start of a widened prologue") == 1, "a leaf without pushes whose loop branches back to its first instruction was not refused");
|
||||
require(report.Skipped.at("branch into the instructions before a short exit") == 1, "a short exit that a branch reaches past the add before it was not refused");
|
||||
}
|
||||
|
||||
void skipsLeavesThatCopyRsp() {
|
||||
@@ -608,6 +675,10 @@ int main() {
|
||||
stubsShortOperandsAtRspBeforeABranchWithTheInstructionsBeforeThem();
|
||||
skipsShortOperandsAtRspWhoseWidenedStubCoversABranchTarget();
|
||||
skipsLeavesWithOperandsThatCannotMove();
|
||||
widensShortProloguesAndExits();
|
||||
framesLeavesWithoutPushes();
|
||||
skipsFragmentsThatPopWithoutPushing();
|
||||
skipsWideningOverBranchTargets();
|
||||
skipsLeavesThatCopyRsp();
|
||||
framesEachFunctionFromItsOwnFde();
|
||||
treatsATailCallPastTheFdeEndAsLeaving();
|
||||
|
||||
@@ -125,6 +125,7 @@ ConversionOutput Conversion::Convert(ConversionInput input, const ConversionOpti
|
||||
std::copy(moved.Bytes.begin(), moved.Bytes.end(), sourceBytes.begin() + static_cast<std::ptrdiff_t>(moved.Offset));
|
||||
}
|
||||
{ std::ostringstream message; message << "Red-zone framing: " << report.Functions << " functions, " << report.RedZoneFunctions << " leaf functions keep values below RSP, " << report.Framed << " framed (" << report.FramedWithoutFramePointer << " without a frame pointer, " << report.MovedOperands << " rsp operands moved, " << report.StubbedOperands << " in " << report.OperandStubs << " operand stubs; " << report.Exits << " exits)\n"; inputProgress.Message(message.str()); }
|
||||
if (report.WidenedPrologues != 0 || report.WidenedExits != 0) { std::ostringstream message; message << "Red-zone framing: " << report.WidenedPrologues << " prologues widened over the instructions after them and " << report.WidenedExits << " exits over the instructions before them, as they were shorter than a jump (" << report.WidenedOperands << " rsp operands moved into their stubs)\n"; inputProgress.Message(message.str()); }
|
||||
for (const auto& [reason, count] : report.Skipped) { std::ostringstream message; message << "Red-zone framing: skipped " << count << ": " << reason << '\n'; inputProgress.Message(message.str()); }
|
||||
trampolines.insert(trampolines.end(), std::make_move_iterator(framed.begin()), std::make_move_iterator(framed.end()));
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
- [SDK memory-management syscall traps](../../../core/relinker/relinker/src/analysis/SyscallScanner.cpp) under `--trap-sdk-syscall-stubs` recognize three exact instruction sequences from [ps5-payload-dev SDK libc/mman.c](https://github.com/ps5-payload-dev/sdk/blob/12cf3dda40/libc/mman.c), observed in Nuvio Maxd 0.5.15 (PPSA99288): syscall 74 and the two syscall 477 paths, with their argument preparation and carry-flag error handling. Matching requires a decoded instruction boundary and the complete sequence inside the scanned code range. Other SDK/compiler output remains rejected. The matched syscall becomes `UD2`; this does not implement mmap or mprotect and a call still faults.
|
||||
- The [instruction decoder](../../../core/relinker/codegen/src/x86/X64InstructionDecoder.cpp) reads a 16-bit displacement for `66 E8`, `66 E9` and `66 0F 80`-`8F` without REX.W, as AMD processors, the console included, do (binutils `objdump -M amd64`; REX.W selects the 32-bit form). AMD also truncates the target of every near relative branch with that prefix, rel8, LOOP and JRCXZ forms included, to 16 bits. Instruction metadata marks these branches as truncated and reports no branch target for them, and stubs refuse to move them. Compilers do not emit these forms.
|
||||
- Converted Linux main executables are [placed at `0x400000`](../../../core/relinker/relinker/src/pipeline/Conversion.cpp) by default, as `ET_EXEC` without ASLR, not only with `--linux-fixed-base`: the console executable lies inside the 40-bit window that the title's own `sceLibcMspaceCreate` checks (#3043). Where the console loads an executable that does not require an address is not measured; `0x400000` is the base that executables reading their GOT through `app0/imports.txt` assume (#4013). The relinker refuses an image that spans 2 GiB or more, so the executable ends below the guest arena at `0x200000000`. With `--lazy-binding` the executable stays position independent.
|
||||
- `--frame-red-zone` frames only leaf functions that start with a chain of pushes of at least five bytes (`push rbp; mov rbp, rsp; push ...`, or pushes alone, which may be followed by `sub $n, %rsp` or `lea -n(%rsp), %rsp`) and leave through exits of at least five bytes that undo that allocation with `add` or `lea`, pop the same registers and `ret` ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)). Leaves with a frame pointer that allocate, leaves with neither pushes nor an allocation, shorter prologues and exits, exits that free another size, and functions with indirect jumps or other RSP arithmetic keep their red zone.
|
||||
- `--frame-red-zone` frames only leaf functions that start with pushes (`push rbp; mov rbp, rsp; push ...`, or pushes alone, which may be followed by `sub $n, %rsp` or `lea -n(%rsp), %rsp`) or with neither pushes nor an allocation, and leave through exits that undo that allocation with `add` or `lea`, pop the same registers and `ret` ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)). A prologue or exit shorter than its five-byte jump takes in the straight-line instructions after or before it. The leaf keeps its red zone when those cannot make five bytes, when a branch targets one of them or the first instruction of a widened prologue, or when a leaf without pushes pushes or pops, as a function fragment does. Leaves with a frame pointer that allocate, exits that free another size, and functions with indirect jumps or other RSP arithmetic keep their red zone too.
|
||||
- In a red-zone leaf without a frame pointer, `--frame-red-zone` lowers RSP after the prologue by the deepest displacement below RSP, rounded up to 16, and moves every RSP operand by the same amount ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)). A leaf that takes the address of a slot below RSP, or uses an RSP operand with an index register, is lowered by the whole 128 bytes, since a pointer or an index can reach deeper than any displacement; values below RSP reached through any other register are not seen. Leaves with a frame pointer are lowered by 128 bytes. Inside a moved frame the CFA rules of the guest's unwind tables are off by the shift.
|
||||
- An RSP operand that `--frame-red-zone` cannot move in place, an 8-bit displacement the shift takes past 0x7f or one without a displacement, moves to a stub with the instructions after it up to five bytes and is re-encoded there with the moved displacement ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)). The leaf is skipped when that site would take in a branch target, a branch, ret or RIP-relative instruction, or another site, and when the operand is EVEX, whose 8-bit displacement is scaled. A game patch over a framed site or a moved displacement is refused.
|
||||
- In `--frame-red-zone`, function bounds come from the executable's `.eh_frame` FDEs: each function is decoded from its FDE start to its FDE end, or to int3 padding before it, and the relinker stops with an error when the executable has none. Code outside every FDE, such as a function without unwind information, is not framed ([framer](../../../core/relinker/codegen/src/x86/RedZoneFramer.cpp)).
|
||||
|
||||
+1
-1
@@ -87,7 +87,7 @@ All switches are disabled by default. `unused-filter` defaults to `0`; `--rpath`
|
||||
| `--exclude-sce-module <file>` | Deprecated. Exclude a bundled module by exact filename, not path. Repeat for multiple files; a missing filename is an error. Conflicts with `--skip-sce-module`. |
|
||||
| `--skip-syscall-check` | Deprecated. Disable syscall scanning in the executable and bundled modules. |
|
||||
| `--trap-sdk-syscall-stubs` | Replace the exact ps5-payload-dev generic and numbered syscall wrapper stubs and the recognized inline mmap/mprotect sequences from SDK libc/mman.c with `UD2`, so unused wrappers do not block relinking. Other syscall instructions still cause an error. If a trapped wrapper runs, it raises an illegal-instruction exception. For debugging only. |
|
||||
| `--frame-red-zone` | Windows only: give leaf functions that keep values below RSP (the System V red zone) a frame, so an exception Windows delivers on a guest thread cannot overwrite them. Covers leaves with a `push rbp; mov rbp, rsp; push ...` prologue and leaves that only push registers, with or without a `sub` allocation after the pushes; in those, RSP is lowered only as far as the leaf reaches below it and their RSP operands are moved with it, in place or, when their encoding cannot hold the moved displacement, in a stub. Function bounds come from the executable's `.eh_frame`, and the relinker stops with an error when it has none. The relinker prints how many it framed and why it skipped the others, and refuses a game patch over a framed site. |
|
||||
| `--frame-red-zone` | Windows only: give leaf functions that keep values below RSP (the System V red zone) a frame, so an exception Windows delivers on a guest thread cannot overwrite them. Covers leaves with a `push rbp; mov rbp, rsp; push ...` prologue, leaves that only push registers, with or without a `sub` allocation after the pushes, and leaves with neither; a prologue or exit shorter than a jump takes in the instructions next to it. In leaves without a frame pointer, RSP is lowered only as far as the leaf reaches below it and their RSP operands are moved with it, in place or, when their encoding cannot hold the moved displacement, in a stub. Function bounds come from the executable's `.eh_frame`, and the relinker stops with an error when it has none. The relinker prints how many it framed and why it skipped the others, and refuses a game patch over a framed site. |
|
||||
| `--lazy-binding` | Deprecated. Enable lazy symbol binding instead of eager binding. Incompatible with bundled ELF modules. The Linux executable stays position independent (`ET_DYN`) instead of being placed at `0x400000`. |
|
||||
|
||||
Specify `unused-filter=0|1|2` without `--`, at most once. Unknown options and extra positional arguments are errors. Use `relinker --help` to print the usage syntax to stdout and exit with code `0` without converting files. Invoking `relinker` without arguments prints the usage syntax to stderr and exits with code `1`.
|
||||
|
||||
Reference in new issue
Block a user