Merge pull request #9663 from AdnanKhan-ak47/feat/relinker-report-unconverted-needed

feat(relinker): report needed modules shipped outside the module directories

core/relinker/cli/include/NativeModules.hpp

@@ -1,11 +1,13 @@
 #pragma once
 
 #include <relinker/guest/GuestImage.hpp>
+#include <iosfwd>
 
 namespace Cli {
 
 struct NativeModules {
     std::filesystem::path Root;
+    std::vector<std::filesystem::path> Directories;
     std::vector<std::filesystem::path> RuntimeDirectories;
     std::vector<Relinker::GuestModuleInput> Modules;
 };
@@ -14,4 +16,6 @@ bool SameDirectory(const std::filesystem::path& first, const std::filesystem::pa
 
 NativeModules ReadModules(const std::filesystem::path& inputPath, const std::filesystem::path& sceModulePath, const std::set<std::string>& excludedModules, const std::vector<std::string>& moduleDirectories);
 
+void ReportUnconvertedNeeded(const NativeModules& modules, const std::vector<std::string>& neededLibraries, bool foldCase, std::ostream& out);
+
 }

core/relinker/cli/src/NativeModules.cpp

@@ -2,13 +2,59 @@
 #include <io/FileReader.hpp>
 #include <io/NativePath.hpp>
 #include <io/FileIdentity.hpp>
+#include <domain/ImportModule.hpp>
 #include <algorithm>
+#include <cstdint>
 #include <fstream>
 #include <iostream>
 #include <map>
+#include <optional>
+#include <string_view>
+#include <system_error>
+#include <tuple>
 
 namespace Cli {
 
+namespace {
+
+struct CandidateHeader {
+    bool Elf = false;
+    bool Self = false;
+    std::uint16_t Type = 0;
+};
+
+std::optional<CandidateHeader> TryReadCandidateHeader(const std::filesystem::path& path) {
+    std::ifstream stream(path, std::ios::binary);
+    if (!stream) return std::nullopt;
+    char header[18]{};
+    stream.read(header, sizeof header);
+    if (stream.bad()) return std::nullopt;
+    CandidateHeader result;
+    if (stream.gcount() < 4) return result;
+    const auto byte = [&](const std::size_t index) { return static_cast<unsigned char>(header[index]); };
+    result.Self = (byte(0) == 0x4f && byte(1) == 0x15 && byte(2) == 0x3d && byte(3) == 0x1d) || (byte(0) == 0x54 && byte(1) == 0x14 && byte(2) == 0xf5 && byte(3) == 0xee);
+    result.Elf = byte(0) == 0x7f && header[1] == 'E' && header[2] == 'L' && header[3] == 'F';
+    if (result.Elf && stream.gcount() == sizeof header) result.Type = static_cast<std::uint16_t>(byte(16) | (byte(17) << 8));
+    return result;
+}
+
+CandidateHeader ReadCandidateHeader(const std::filesystem::path& path) {
+    const auto header = TryReadCandidateHeader(path);
+    if (!header) throw Domain::RelinkerException("Cannot read guest candidate: " + Io::Utf8Path(path));
+    return *header;
+}
+
+std::string FoldedName(std::string name, const bool foldCase) {
+    if (foldCase) for (auto& character : name) if (character >= 'A' && character <= 'Z') character = static_cast<char>(character + ('a' - 'A'));
+    return name;
+}
+
+std::string_view ModuleStem(const std::string_view name) {
+    return name.substr(0, name.size() - Domain::ModuleSuffix(name).size());
+}
+
+}
+
 bool SameDirectory(const std::filesystem::path& first, const std::filesystem::path& second) {
     if (std::filesystem::exists(first) && std::filesystem::exists(second)) return std::filesystem::equivalent(first, second);
     return std::filesystem::weakly_canonical(first) == std::filesystem::weakly_canonical(second);
@@ -45,17 +91,6 @@ NativeModules ReadModules(const std::filesystem::path& inputPath, const std::fil
     }
     std::vector<std::filesystem::path> paths;
     std::set<std::string> unmatchedExclusions = excludedModules;
-    const auto isElf = [](const std::filesystem::path& path, bool& self) {
-        std::ifstream stream(path, std::ios::binary);
-        if (!stream) throw Domain::RelinkerException("Cannot read guest candidate: " + Io::Utf8Path(path));
-        char magic[4]{};
-        stream.read(magic, 4);
-        if (stream.bad()) throw Domain::RelinkerException("Cannot read guest candidate magic: " + Io::Utf8Path(path));
-        if (stream.gcount() != 4) return false;
-        const auto byte = [&](const std::size_t index) { return static_cast<unsigned char>(magic[index]); };
-        self = (byte(0) == 0x4f && byte(1) == 0x15 && byte(2) == 0x3d && byte(3) == 0x1d) || (byte(0) == 0x54 && byte(1) == 0x14 && byte(2) == 0xf5 && byte(3) == 0xee);
-        return byte(0) == 0x7f && magic[1] == 'E' && magic[2] == 'L' && magic[3] == 'F';
-    };
     for (const auto& directory : directories) {
         if (!std::filesystem::is_directory(directory)) throw Domain::RelinkerException("Guest module path is not a directory: " + Io::Utf8Path(directory));
         for (const auto& entry : std::filesystem::directory_iterator(directory)) {
@@ -66,9 +101,9 @@ NativeModules ReadModules(const std::filesystem::path& inputPath, const std::fil
             }
             if (!entry.is_regular_file()) continue;
             if (Io::SameFile(entry.path(), inputPath)) continue;
-            bool self = false;
-            if (isElf(entry.path(), self)) paths.push_back(entry.path());
-            else if (self)
+            const auto header = ReadCandidateHeader(entry.path());
+            if (header.Elf) paths.push_back(entry.path());
+            else if (header.Self)
                 std::cout << "Skipping SELF container: " << entry.path().lexically_relative(root).generic_string()
                           << " (unwrap it before relinking)\n";
         }
@@ -77,6 +112,7 @@ NativeModules ReadModules(const std::filesystem::path& inputPath, const std::fil
     std::sort(paths.begin(), paths.end());
     NativeModules result;
     result.Root = root;
+    result.Directories = std::move(directories);
     result.RuntimeDirectories = std::move(runtimeDirectories);
     for (const auto& path : paths) result.Modules.push_back({path, Io::FileReader().Read(Io::Utf8Path(path))});
     std::map<std::size_t, std::string> sonames;
@@ -107,4 +143,49 @@ NativeModules ReadModules(const std::filesystem::path& inputPath, const std::fil
     return result;
 }
 
+void ReportUnconvertedNeeded(const NativeModules& modules, const std::vector<std::string>& neededLibraries, const bool foldCase, std::ostream& out) {
+    std::map<std::string, std::vector<std::string>> stems;
+    for (const auto& name : neededLibraries) {
+        const auto folded = FoldedName(name, foldCase);
+        if (!Domain::ModuleSuffix(folded).empty()) stems[std::string(ModuleStem(folded))].push_back(name);
+    }
+    if (stems.empty()) return;
+    struct Found {
+        std::filesystem::path Relative;
+        std::string Name;
+        bool Self;
+    };
+    std::vector<Found> found;
+    std::error_code error;
+    for (auto entry = std::filesystem::recursive_directory_iterator(modules.Root, std::filesystem::directory_options::skip_permission_denied, error); !error && entry != std::filesystem::recursive_directory_iterator(); entry.increment(error)) {
+        const bool directory = entry->is_directory(error);
+        if (error) {
+            error.clear();
+            continue;
+        }
+        if (directory) {
+            if (std::any_of(modules.Directories.begin(), modules.Directories.end(), [&](const auto& scanned) { return std::filesystem::equivalent(scanned, entry->path(), error) && !error; })) entry.disable_recursion_pending();
+            error.clear();
+            continue;
+        }
+        const auto filename = FoldedName(Io::Utf8Path(entry->path().filename()), foldCase);
+        if (Domain::ModuleSuffix(filename).empty()) continue;
+        const auto stem = stems.find(std::string(ModuleStem(filename)));
+        if (stem == stems.end()) continue;
+        if (!entry->is_regular_file(error) || error) {
+            error.clear();
+            continue;
+        }
+        const auto header = TryReadCandidateHeader(entry->path());
+        if (!header || (!header->Self && !(header->Elf && header->Type == 0xfe18))) continue;
+        for (const auto& name : stem->second) found.push_back({entry->path().lexically_relative(modules.Root), name, header->Self});
+    }
+    std::sort(found.begin(), found.end(), [](const auto& left, const auto& right) { return std::tie(left.Relative, left.Name) < std::tie(right.Relative, right.Name); });
+    for (const auto& [relative, name, self] : found) {
+        const auto directory = relative.parent_path().generic_string();
+        out << "Needed module not converted: " << name << " at " << relative.generic_string() << "; add --module-dir " << (directory.empty() ? "." : directory)
+            << (self ? " and unwrap the SELF container" : "") << '\n';
+    }
+}
+
 }

core/relinker/main.cpp

@@ -45,11 +45,12 @@ ConvertedExecutable Convert(const Cli::Args& args, const std::string& inputPath,
     input.Executable = fileReader.Read(inputPath);
     input.InputPath = std::filesystem::absolute(Io::NativePath(inputPath)).lexically_normal();
     input.OutputPath = Io::NativePath(absPath);
+    Cli::NativeModules modules;
     if (!args.skipSceModule) {
-        auto modules = Cli::ReadModules(std::filesystem::absolute(Io::NativePath(args.inputPath)).lexically_normal(), Io::NativePath(args.sceModulePath), args.excludedSceModules, args.moduleDirectories);
-        input.ModuleRoot = std::move(modules.Root);
+        modules = Cli::ReadModules(std::filesystem::absolute(Io::NativePath(args.inputPath)).lexically_normal(), Io::NativePath(args.sceModulePath), args.excludedSceModules, args.moduleDirectories);
+        input.ModuleRoot = modules.Root;
         input.Modules = std::move(modules.Modules);
-        input.RuntimeDirectories = std::move(modules.RuntimeDirectories);
+        input.RuntimeDirectories = modules.RuntimeDirectories;
     }
     const auto iconPath = Io::NativePath(args.inputPath).parent_path() / "sce_sys" / "icon0.png";
     if (args.toWindows && std::filesystem::exists(iconPath)) input.Icon = fileReader.Read(Io::Utf8Path(iconPath));
@@ -76,6 +77,7 @@ ConvertedExecutable Convert(const Cli::Args& args, const std::string& inputPath,
     progress.IncludeFile(Io::Utf8Path(input.OutputPath));
     for (const auto& module : input.Modules) progress.IncludeFile(Io::Utf8Path(module.Path));
     auto converted = Relinker::Conversion(progress).Convert(std::move(input), options);
+    if (!args.skipSceModule) Cli::ReportUnconvertedNeeded(modules, converted.Analysis.DynamicSection.NeededLibraries, args.toWindows, std::cout);
     return {inputPath, absPath, std::move(converted.Analysis), std::move(converted.Modules), std::move(converted.Executable), std::move(converted.WindowsDiagnostics), std::move(converted.AppliedPatches), converted.LinuxBase};
 }
 

core/relinker/relinker/tests/test_guest_needed_modules.py

@@ -86,11 +86,110 @@ def check_guest_needed(relinker, work):
             assert not output.exists() and not (case / "app0").exists(), output
 
 
+def check_needed_outside(relinker, work):
+    self_container = bytes.fromhex("4f153d1d") + b"\0" * 60
+    sce_module = module_with_symbol(True)
+    struct.pack_into("<H", sce_module, 16, 0xFE18)
+    message = "Needed module not converted: "
+
+    def convert(case, options=(), needed=NEEDED):
+        source = case / "input.elf"
+        source.write_bytes(executable_with_needed(needed))
+        result = subprocess.run([str(relinker), *options, str(source), str(case / "output.elf")],
+                                capture_output=True, text=True, timeout=30)
+        assert result.returncode == 0, (case.name, result.stdout, result.stderr)
+        return [line for line in result.stdout.splitlines() if line.startswith(message)]
+
+    case = work / "outside-self-root"
+    (case / "sce_module").mkdir(parents=True)
+    (case / "sce_module" / "wrapped.prx").write_bytes(self_container)
+    (case / "needed.prx").write_bytes(self_container)
+    assert convert(case) == [message + "needed.prx at needed.prx; add --module-dir . and unwrap the SELF container"]
+
+    case = work / "outside-alias-nested"
+    (case / "sce_module").mkdir(parents=True)
+    (case / "Media" / "Modules").mkdir(parents=True)
+    (case / "Media" / "Modules" / "needed.sprx").write_bytes(sce_module)
+    (case / "Media" / "Modules" / "needed.prx.guest.prx").write_bytes(sce_module)
+    (case / "Media" / "other.prx").write_bytes(sce_module)
+    assert convert(case) == [message + "needed.prx at Media/Modules/needed.sprx; add --module-dir Media/Modules"]
+    assert convert(case, ["--module-dir", "Media/Modules"]) == []
+    assert (case / "app0" / "Media" / "Modules" / "needed.sprx.guest.prx").is_file()
+
+    case = work / "outside-two-copies"
+    (case / "sce_module").mkdir(parents=True)
+    for name in ("first", "second"):
+        (case / name).mkdir()
+        (case / name / "needed.prx").write_bytes(sce_module)
+    assert convert(case) == [message + "needed.prx at first/needed.prx; add --module-dir first",
+                             message + "needed.prx at second/needed.prx; add --module-dir second"]
+
+    case = work / "outside-not-sce"
+    (case / "sce_module").mkdir(parents=True)
+    (case / "libs").mkdir()
+    (case / "libs" / "needed.prx").write_bytes(module_with_symbol(True))
+    (case / "needed.prx").write_text("not ELF")
+    (case / "sce_module" / "needed.prx").mkdir()
+    assert convert(case) == []
+
+    case = work / "outside-converted"
+    (case / "sce_module").mkdir(parents=True)
+    (case / "sce_module" / "needed.prx").write_bytes(sce_module)
+    (case / "needed.prx").write_bytes(sce_module)
+    assert convert(case) == []
+
+    case = work / "outside-skipped"
+    case.mkdir()
+    (case / "needed.prx").write_bytes(sce_module)
+    assert convert(case, ["--skip-sce-module"]) == []
+
+    case = work / "outside-case"
+    (case / "sce_module").mkdir(parents=True)
+    (case / "Media").mkdir()
+    (case / "Media" / "NEEDED.PRX").write_bytes(sce_module)
+    assert convert(case) == []
+    assert convert(case, ["--windows"]) == [message + "needed.prx at Media/NEEDED.PRX; add --module-dir Media"]
+
+    if os.name != "nt" and os.geteuid() != 0:
+        case = work / "outside-unreadable"
+        (case / "sce_module").mkdir(parents=True)
+        (case / "Media").mkdir()
+        (case / "Media" / "needed.prx").write_bytes(sce_module)
+        (case / "needed.prx").write_bytes(self_container)
+        (case / "needed.prx").chmod(0)
+        try:
+            assert convert(case) == [message + "needed.prx at Media/needed.prx; add --module-dir Media"]
+        finally:
+            (case / "needed.prx").chmod(0o644)
+
+    if os.name != "nt":
+        case = work / "outside-loops"
+        (case / "sce_module").mkdir(parents=True)
+        (case / "Media").mkdir()
+        (case / "Media" / "needed.prx").write_bytes(sce_module)
+        os.symlink("loop", case / "loop")
+        os.symlink("needed.prx", case / "needed.prx")
+        os.symlink("missing.prx", case / "Media" / "needed.sprx")
+        (case / "Media" / "Nested").mkdir()
+        os.symlink("..", case / "Media" / "Nested" / "needed.suprx")
+        expected = [message + "needed.prx at Media/needed.prx; add --module-dir Media"]
+        assert convert(case) == expected
+        if os.geteuid() != 0:
+            (case / "locked").mkdir()
+            (case / "locked" / "needed.prx").write_bytes(sce_module)
+            (case / "locked").chmod(0)
+            try:
+                assert convert(case) == expected
+            finally:
+                (case / "locked").chmod(0o755)
+
+
 def main():
     relinker = Path(sys.argv[1]).resolve()
     with tempfile.TemporaryDirectory(prefix="anyps5-needed-modules-") as directory:
         work = Path(directory)
         check_guest_needed(relinker, work)
+        check_needed_outside(relinker, work)
 
         def convert(case, windows, needed=NEEDED, repeats=1):
             if not (case / "sce_modules").exists():

docs/user/USAGE.md

@@ -29,6 +29,8 @@ relinker [options] <input.elf> <output>
 
 An explicit `--module-dir` also works when the module parent has no `sce_module`, `sce_modules` or `prx` directory. For a homebrew bundle with shared libraries beside its executable, use `--module-dir .`; no empty standard directory is required. Without an explicit module directory, the standard-directory requirement remains.
 
+A module the executable needs that is not converted is looked for under the module parent directory, outside the module directories. Each SELF container or `ET_SCE_DYNAMIC` ELF found there whose name matches, up to the `.prx`, `.sprx`, `.suprx` or `.debug_prx` suffix (ignoring case with `--windows`, as the guest module match does), is reported after the conversion with the `--module-dir` that would include it; the output still refers to a host library of that name. A file that cannot be read is not reported.
+
 Linux output:
 
 ```sh