fix(relinker): resolve symbol-name imports through NIDs on Linux

Homebrew built with the ps5-payload-dev SDK names its system
libraries with .sprx and imports by symbol name, while the libraries
in libs/ are .prx files that export NIDs. On Linux such an executable
converted but stopped at load time on the first .sprx dependency and,
once that was found, on the first symbol name.

The Linux patcher now loads a dependency named with .sprx, .suprx or
.debug_prx as the .prx library of the same stem. In an executable that
names such a dependency, an import without a module that is named by
symbol gets the NID of that name. A name that a bundled module exports
still binds to that module. There is no lookup by the name as written,
because the host loader also searches the host C library. Other
executables keep their unqualified imports as written.

core/relinker/CMakeLists.txt

@@ -369,12 +369,15 @@ if(BUILD_TESTING)
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/StubBodyBuilder.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/io/src/ByteWriter.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/io/src/BufferUtils.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/../libs/nid/src/NidCompute.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/../libs/nid/src/Sha1.cpp
     )
     target_include_directories(linux_load_pages_tests PRIVATE
         ${CMAKE_CURRENT_SOURCE_DIR}/domain/include
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include
         ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include
         ${CMAKE_CURRENT_SOURCE_DIR}/io/include
+        ${CMAKE_CURRENT_SOURCE_DIR}/../libs/nid/include
     )
     add_test(NAME linux_load_pages COMMAND linux_load_pages_tests)
     set_tests_properties(linux_load_pages PROPERTIES TIMEOUT 20)
@@ -409,12 +412,15 @@ if(BUILD_TESTING)
         ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SectionHeaderTableBuilder.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/io/src/ByteWriter.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/io/src/BufferUtils.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/../libs/nid/src/NidCompute.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/../libs/nid/src/Sha1.cpp
     )
     target_include_directories(amd64_only_converter_tests PRIVATE
         ${CMAKE_CURRENT_SOURCE_DIR}/domain/include
         ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include
         ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include
         ${CMAKE_CURRENT_SOURCE_DIR}/io/include
+        ${CMAKE_CURRENT_SOURCE_DIR}/../libs/nid/include
     )
     if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR (NOT CMAKE_OSX_ARCHITECTURES AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")))
         target_link_options(amd64_only_converter_tests PRIVATE "LINKER:-pagezero_size,0x10000")
@@ -595,6 +601,8 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_host_libc.py $<TARGET_FILE:relinker>)
         add_test(NAME windows_named_imports COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_windows_named_imports.py $<TARGET_FILE:relinker>)
+        add_test(NAME linux_named_imports COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_linux_named_imports.py $<TARGET_FILE:relinker>)
         add_library(guest_module_args_host SHARED elfpatcher/tests/GuestModuleArgsHost.cpp)
         target_include_directories(guest_module_args_host PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../libs)
         set_target_properties(guest_module_args_host PROPERTIES

core/relinker/domain/include/domain/Types.hpp

@@ -106,6 +106,7 @@ struct SysVDynamicSection {
     std::vector<GuestRuntime> GuestModules;
     std::map<VirtualAddress, std::string> ImportModules;
     std::set<VirtualAddress> FunctionImports;
+    std::set<VirtualAddress> NamedImports;
     std::set<std::string> UnmatchedImportModules;
     std::vector<std::string> NeededLibraries;
 };

core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp

@@ -4,10 +4,13 @@
 #include <elfpatcher/general/SectionHeaderTableRequest.hpp>
 #include <codegen/x86/Amd64OnlySubstitutionTable.hpp>
 #include <codegen/x86/StubBodyBuilder.hpp>
+#include <domain/ImportModule.hpp>
 #include <io/BufferUtils.hpp>
+#include <nid/NidCompute.hpp>
 #include <algorithm>
 #include <limits>
 #include <map>
+#include <set>
 #include <span>
 #include <sstream>
 #include <string>
@@ -190,6 +193,46 @@ std::vector<TracedImport> TracedImports(const Domain::SysVDynamicSection& dynSec
     return imports;
 }
 
+Domain::SysVDynamicSection ImportsByNid(Domain::SysVDynamicSection dynSection) {
+    RequireWholeEntries(dynSection.DynamicSegmentData, kDynEntrySize, "Executable dependency table");
+    for (std::size_t entry = 0; entry < dynSection.DynamicSegmentData.size(); entry += kDynEntrySize) {
+        if (static_cast<std::int64_t>(Io::ReadU64(dynSection.DynamicSegmentData, entry)) != DT_NEEDED)
+            continue;
+        const auto offset = Io::ReadU64(dynSection.DynamicSegmentData, entry + 8);
+        if (offset >= dynSection.DynStrData.size())
+            throw Domain::RelinkerException("DT_NEEDED string offset is out of bounds", offset);
+        const auto name = ReadDynamicString(dynSection.DynStrData, static_cast<std::uint32_t>(offset));
+        const auto library = Domain::NeededLibraryName(name);
+        if (library == name)
+            continue;
+        Io::WriteU64(dynSection.DynamicSegmentData, entry + 8, dynSection.DynStrData.size());
+        Io::AppendString(dynSection.DynStrData, library);
+    }
+    for (auto& library : dynSection.NeededLibraries)
+        library = Domain::NeededLibraryName(library);
+
+    std::set<std::uint64_t> symbols;
+    for (const auto* table : {&dynSection.RelaData, &dynSection.RelaPltData}) {
+        RequireWholeEntries(*table, kRelaEntrySize, "Relocation table");
+        for (std::size_t entry = 0; entry < table->size(); entry += kRelaEntrySize)
+            if (dynSection.NamedImports.contains(Io::ReadU64(*table, entry)))
+                symbols.insert(Io::ReadU64(*table, entry + 8) >> 32);
+    }
+    for (const auto symbol : symbols) {
+        if (symbol >= dynSection.DynSymData.size() / kSymEntrySize)
+            throw Domain::RelinkerException("Invalid import symbol index", symbol);
+        const auto symbolOffset = static_cast<std::size_t>(symbol * kSymEntrySize);
+        const auto name = ReadDynamicString(dynSection.DynStrData, Io::ReadU32(dynSection.DynSymData, symbolOffset));
+        if (name.empty() || name.find('#') != std::string::npos)
+            continue;
+        if (dynSection.DynStrData.size() > std::numeric_limits<std::uint32_t>::max())
+            throw Domain::RelinkerException("Dynamic string table too large");
+        Io::WriteU32(dynSection.DynSymData, symbolOffset, static_cast<std::uint32_t>(dynSection.DynStrData.size()));
+        Io::AppendString(dynSection.DynStrData, Nid::ComputeNid(name, ""));
+    }
+    return dynSection;
+}
+
 std::vector<std::uint8_t> TracePrologue(const std::uint64_t vaddr, const std::uint64_t table, const std::uint64_t tableEnd, const std::uint64_t function) {
     std::vector<std::uint8_t> code = {0x52, 0x53, 0x41, 0x54, 0x48, 0x83, 0xec, 0x08};
     const auto rip = [&](std::initializer_list<std::uint8_t> opcode, const std::uint64_t target) {
@@ -297,13 +340,14 @@ void LinuxElfPatcher::_appendTrampoline(
 std::vector<std::uint8_t> LinuxElfPatcher::Patch(
     std::vector<std::uint8_t> sourceElf,
     const std::vector<Domain::ProgramHeader>& originalHeaders,
-    const Domain::SysVDynamicSection& dynSection,
+    const Domain::SysVDynamicSection& sourceDynSection,
     const std::uint64_t originalPltGotVaddr,
     const std::string& runPath,
     const bool lazyBinding,
     const bool dependencyDiagnostics,
     const std::vector<Codegen::TrampolineSite>& trampolines)
 {
+    const auto dynSection = ImportsByNid(sourceDynSection);
     if (dependencyDiagnostics)
         throw Domain::RelinkerException("Linux target does not support --windows-diagnostics");
     if (_fixedBase && lazyBinding)

core/relinker/relinker/src/output/SysVDynamicSectionBuilder.cpp

@@ -1,5 +1,7 @@
 #include <relinker/output/SysVDynamicSectionBuilder.hpp>
 #include <elfpatcher/general/ElfConstants.hpp>
+#include <domain/ImportModule.hpp>
+#include <algorithm>
 #include <cstring>
 
 namespace Relinker {
@@ -77,6 +79,9 @@ SysVDynamicSection SysVDynamicSectionBuilder::BuildDynamicSection(
 
     _appendElfSym(result.DynSymData, 0, 0, 0, 0, 0, 0);
 
+    const bool consoleFileNames = std::any_of(neededLibraries.begin(), neededLibraries.end(),
+        [](const std::string& name) { return Domain::NeededLibraryName(name) != name; });
+
     auto stripHashSuffix = [](const std::string& value) -> std::string {
         const auto hashPos = value.find('#');
         if (hashPos == std::string::npos) return value;
@@ -98,6 +103,7 @@ SysVDynamicSection SysVDynamicSectionBuilder::BuildDynamicSection(
     for (const auto& ref : nidReferences) {
         if (!ref.Library.empty()) result.ImportModules.emplace(ref.RelocationAddress, ref.Library);
         if (ref.Function) result.FunctionImports.insert(ref.RelocationAddress);
+        if (consoleFileNames && ref.Library.empty() && !ref.Nid.empty() && ref.Nid.find('#') == std::string::npos) result.NamedImports.insert(ref.RelocationAddress);
         std::uint32_t relType = ref.RelocationTypeValue;
         if (relType == 0) relType = R_X86_64_JUMP_SLOT;
 

core/relinker/relinker/tests/test_guest_module_identity.py

@@ -98,7 +98,7 @@ def main():
                     assert not artifact.exists(), artifact
                     assert len(list((output.parent / 'app0').rglob('*.guest.prx'))) == 1
                     if not windows:
-                        assert 'libGuest.suprx' in needed_libraries(output.read_bytes())
+                        assert 'libGuest.prx' in needed_libraries(output.read_bytes())
 
             result, output = convert(f'outside-ambiguous-{windows}', windows, 'libGuest.suprx', module_directory='.',
                 providers={'one.prx': declared_provider(22, ('libGuest',))},
@@ -187,7 +187,7 @@ def main():
                 if windows:
                     run(output, error='Failed to load module:')
                 else:
-                    assert request in needed_libraries(output.read_bytes())
+                    assert request.replace('.suprx', '.prx') in needed_libraries(output.read_bytes())
 
         providers = {'wrong.prx': declared_provider(11, ('other',)),
                      'right.prx': declared_provider(22, ('libGuest',))}

core/relinker/relinker/tests/test_guest_needed_modules.py

@@ -186,7 +186,7 @@ def main():
             assert output.exists(), output
             assert not list((case / "app0").rglob("*.guest.prx"))
             if not windows:
-                assert needed_libraries(output.read_bytes()) == ["needed.debug_prx" if "debug-name" in case.name else "needed.prx"]
+                assert needed_libraries(output.read_bytes()) == ["needed.prx"]
 
             case = work / f"{windows}-ambiguous"
             for name in ("first", "second"):

core/relinker/relinker/tests/test_guest_needed_nested_modules.py

@@ -62,7 +62,7 @@ def main():
                     assert output.is_file()
                     assert not list((output.parent / 'app0').rglob('*.guest.prx'))
                     if not windows:
-                        assert needed_libraries(output.read_bytes()) == ['libGuest.suprx']
+                        assert needed_libraries(output.read_bytes()) == ['libGuest.prx']
                     result, output = convert(f'{label}-{identity}-excluded', windows, 'libGuest.suprx', files, (filename,))
                     assert result.returncode == 2 and 'Excluded guest module file not found' in result.stderr, result.stderr
                     assert not output.exists()

core/relinker/relinker/tests/test_linux_named_imports.py

@@ -0,0 +1,84 @@
+import os
+from pathlib import Path
+import platform
+import subprocess
+import sys
+import tempfile
+
+sys.path.insert(0, str(Path(__file__).resolve().parent / "macos"))
+
+from guest_import_fixtures import executable
+from nid import nid
+from test_guest_module_directories import module_with_symbol, needed_libraries
+from test_guest_symbol_names import imported_symbols
+
+
+def library(work, name, symbol, value):
+    source = work / (name + ".c")
+    source.write_text("#include <unistd.h>\n"
+                      f"int exported(void) __asm__(\"\\\"{symbol}\\\"\");\n"
+                      f"int exported(void) {{ _exit({value}); }}\n")
+    output = work / (name + ".so")
+    subprocess.run([os.environ.get("CC", "cc"), "-shared", "-fPIC", str(source), "-o", str(output)], check=True)
+    return output.read_bytes()
+
+
+def convert(relinker, case, image, module=None):
+    case.mkdir()
+    if module is not None:
+        (case / "sce_module").mkdir()
+        (case / "sce_module" / "bundled.prx").write_bytes(module)
+    source = case / "input.elf"
+    source.write_bytes(image)
+    output = case / "output.elf"
+    options = [] if module is not None else ["--skip-sce-module"]
+    result = subprocess.run([str(relinker), *options, str(source), str(output)], capture_output=True, text=True, timeout=30)
+    assert result.returncode == 0, (case.name, result.stdout, result.stderr)
+    return output
+
+
+def run(output, library):
+    (output.parent / "libs").mkdir()
+    (output.parent / "libs" / "available.prx").write_bytes(library)
+    output.chmod(0o755)
+    return subprocess.run([str(output)], capture_output=True, text=True, timeout=30)
+
+
+def main():
+    relinker = Path(sys.argv[1]).resolve()
+    runnable = sys.platform.startswith("linux") and platform.machine() in ("x86_64", "AMD64")
+    with tempfile.TemporaryDirectory(prefix="anyps5-linux-named-imports-") as directory:
+        work = Path(directory)
+        cases = {
+            "sprx": convert(relinker, work / "sprx", executable("available.sprx", "shared")),
+            "prx": convert(relinker, work / "prx", executable("available.prx", "shared")),
+            "qualified": convert(relinker, work / "qualified", executable("available.sprx", "shared#A#B")),
+            "name": convert(relinker, work / "name", executable("available.sprx", "shared")),
+            "guest": convert(relinker, work / "guest", executable("bundled.prx", "shared", extra_dependencies=("available.sprx",)),
+                             module_with_symbol(True)),
+        }
+        image = cases["sprx"].read_bytes()
+        assert imported_symbols(image) == {nid("shared")}, imported_symbols(image)
+        assert needed_libraries(image) == ["available.prx"], needed_libraries(image)
+        assert imported_symbols(cases["prx"].read_bytes()) == {"shared"}
+        assert imported_symbols(cases["qualified"].read_bytes()) == {"shared"}
+        image = cases["guest"].read_bytes()
+        assert imported_symbols(image) == {"shared#guest"}, imported_symbols(image)
+        assert "available.prx" in needed_libraries(image), needed_libraries(image)
+        if not runnable:
+            return
+        by_nid = library(work, "by-nid", nid("shared"), 22)
+        by_name = library(work, "by-name", "shared", 44)
+        result = run(cases["sprx"], by_nid)
+        assert result.returncode == 22, (result.returncode, result.stdout, result.stderr)
+        result = run(cases["prx"], by_name)
+        assert result.returncode == 44, (result.returncode, result.stdout, result.stderr)
+        result = run(cases["qualified"], by_nid)
+        assert result.returncode != 0 and "undefined symbol: shared" in result.stderr, (result.returncode, result.stderr)
+        result = run(cases["name"], by_name)
+        assert result.returncode != 0 and f"undefined symbol: {nid('shared')}" in result.stderr, (result.returncode, result.stderr)
+    print("Linux named import tests passed")
+
+
+if __name__ == "__main__":
+    main()

docs/dev/debt/relinker.md

@@ -12,7 +12,8 @@
 - [SDK memory-management syscall traps](../../../core/relinker/relinker/src/analysis/SyscallScanner.cpp) under `--trap-sdk-syscall-stubs` recognize three exact instruction sequences from [ps5-payload-dev SDK libc/mman.c](https://github.com/ps5-payload-dev/sdk/blob/12cf3dda40/libc/mman.c), observed in Nuvio Maxd 0.5.15 (PPSA99288): syscall 74 and the two syscall 477 paths, with their argument preparation and carry-flag error handling. Matching requires a decoded instruction boundary and the complete sequence inside the scanned code range. Other SDK/compiler output remains rejected. The matched syscall becomes `UD2`; this does not implement mmap or mprotect and a call still faults.
 - The [instruction decoder](../../../core/relinker/codegen/src/x86/X64InstructionDecoder.cpp) reads a 16-bit displacement for `66 E8`, `66 E9` and `66 0F 80`-`8F` without REX.W, as AMD processors, the console included, do (binutils `objdump -M amd64`; REX.W selects the 32-bit form). AMD also truncates the target of every near relative branch with that prefix, rel8, LOOP and JRCXZ forms included, to 16 bits. Instruction metadata marks these branches as truncated and reports no branch target for them, and stubs refuse to move them. Compilers do not emit these forms.
 - `--to-intel` lowers MCOMMIT to STC plus NOP padding and RDPRU through a runtime-checked stub in the [matcher](../../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp): AMD APM Vol. 3 defines RDPRU ECX 0 as MPERF and ECX 1 as APERF, neither readable on Intel hosts, so both selectors return the host TSC as a disclosed monotonic-counter approximation (their ratio always reads 1.0) while other selectors yield zero EDX:EAX. STC asserts CF=1 because under x86-TSO prior stores are already ordered and guest memory is host-DRAM-backed, leaving the commit-success case as the only observable one. Non-zero RDPRU selectors, the TSC-for-MPERF/APERF substitution, and any console path reporting MCOMMIT CF=0 are unverified.
-- [Unscoped imports on Windows](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp) - an import without a module that no searched library exports under its name is searched again under the NID of that name, because homebrew built with the [ps5-payload-dev SDK](https://github.com/ps5-payload-dev/sdk) imports by symbol name (Quake, from the websrv homebrew launcher; Mari0, #6239) while the libraries in `libs/` export NIDs. A dependency named with `.sprx`, `.suprx` or `.debug_prx` loads the `.prx` library of the same stem from `libs/`, and `libkernel_web` also loads `libkernel.prx`, as `libSceLibcInternal` already loads `libc.prx`: AnyPS5's `libkernel_web.prx` exports only two functions, where the console's `libkernel_web.sprx` is a libkernel variant. A name that matches a library's export as written still binds to that export first. Linux and macOS resolve these imports through the host loader by name and are unchanged.
+- [Unscoped imports on Windows](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp) - an import without a module that no searched library exports under its name is searched again under the NID of that name, because homebrew built with the [ps5-payload-dev SDK](https://github.com/ps5-payload-dev/sdk) imports by symbol name (Quake, from the websrv homebrew launcher; Mari0, #6239) while the libraries in `libs/` export NIDs. A dependency named with `.sprx`, `.suprx` or `.debug_prx` loads the `.prx` library of the same stem from `libs/`, and `libkernel_web` also loads `libkernel.prx`, as `libSceLibcInternal` already loads `libc.prx`: AnyPS5's `libkernel_web.prx` exports only two functions, where the console's `libkernel_web.sprx` is a libkernel variant. A name that matches a library's export as written still binds to that export first. macOS resolves these imports through the host loader by name and is unchanged.
+- [Unscoped imports on Linux](../../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) - in an executable that names a dependency with `.sprx`, `.suprx` or `.debug_prx`, as homebrew built with the [ps5-payload-dev SDK](https://github.com/ps5-payload-dev/sdk) does, an import without a module that is named by symbol is bound to the NID of that name only. Unlike Windows, the name as written is not tried first, because the host loader also searches the host C library, which exports `malloc` or `pthread_create` under their names. A name that a bundled module exports still binds to that module, and such a dependency loads the `.prx` library of the same stem. Unqualified imports of other executables, and imports of bundled modules, are still bound by name.
 - The length-changing path of the [instruction rewriter](../../../core/relinker/codegen/src/x86/X64InstructionRewriter.cpp) is not used by the [converter](../../../core/relinker/codegen/src/Amd64OnlyConverter.cpp): it does not adjust data-to-code references (relocations, FDEs, jump tables) or segment sizes, so every substitution keeps the instruction length.
 - The [converter](../../../core/relinker/codegen/src/Amd64OnlyConverter.cpp) takes the branch targets that keep an instruction in place from a linear scan of the code segment, so data in that segment decodes as branches too: OpenSSL's x86-64 assembly keeps strings such as "SHA1 block transform for x86_64, CRYPTOGAMS by <appro@openssl.org>" in its code, and in SuperTuxKart (PPSA69421) and Mortalfin (PPSA99141) the `jae` read from "sf" points into the VEX prefix of a 256-bit store. A direct branch target inside a scanned instruction counts only where every byte before it is a legacy prefix or REX, which a real branch can skip (a `lock` prefix, for example); a target in a VEX prefix, opcode or operand is ignored, as a jump-table entry that starts no instruction already is. A real branch to such a target, overlapping instructions as hand-written code can, is not seen, and the instruction it enters can be moved.
 - A `PT_SCE_RELRO` segment that no `PT_LOAD` covers, the PS4 layout, is [loaded](../../../core/relinker/relinker/src/parsing/ElfReader.cpp) as a read-write `PT_LOAD` and stays writable after relocation; FreeBSD rtld makes `PT_GNU_RELRO` read-only once relocation is done, and what the console does with `PT_SCE_RELRO` is not checked.