fix(libkernel): fail dlopen of an unrelinked module without the Windows Bad Image dialog

On Windows, dlopen passed a module that was never relinked to LoadLibraryExW, which raised the
modal Bad Image hard-error dialog and blocked the calling thread until someone answered it. The
load now runs with SEM_FAILCRITICALERRORS set for that thread only, so it fails at once with the
same result as before. Every failed load is reported once per file on stderr with a [module]
prefix; a file that exists with no relinked .guest.prx names the relinker's --module-dir option.

core/libs/prx/libkernel/CMakeLists.txt

@@ -239,7 +239,7 @@ if(BUILD_TESTING)
         target_sources(guest_dynamic_loader_tests PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/GuestDynamicLoader.def")
     endif()
     add_test(NAME guest_dynamic_loader COMMAND guest_dynamic_loader_tests $<TARGET_FILE_NAME:guest_module_fixture>)
-    set_tests_properties(guest_dynamic_loader PROPERTIES
+    set_tests_properties(guest_dynamic_loader PROPERTIES TIMEOUT 30
         WORKING_DIRECTORY "$<TARGET_FILE_DIR:guest_module_fixture>" RESOURCE_LOCK guest_dynamic_loader_files)
     if(WIN32)
         set_tests_properties(guest_dynamic_loader PROPERTIES ENVIRONMENT_MODIFICATION

core/libs/prx/libkernel/Module/src/DynamicLoader.cpp

@@ -11,6 +11,7 @@
 #include <memory>
 #include <limits>
 #include <mutex>
+#include <set>
 #include <string>
 #include <stdexcept>
 #include <system_error>
@@ -36,6 +37,29 @@ void Error(const char* message) {
     std::snprintf(loaderError.data(), loaderError.size(), "%s", message);
     pendingError = true;
 }
+void LoadFailure(const std::filesystem::path& source, const std::filesystem::path& resolved, const std::string& reason) {
+    std::error_code status;
+    const auto message = resolved == source && std::filesystem::is_regular_file(source, status)
+        ? "dlopen: " + source.string() + " was not relinked: no " + source.filename().string() + ".guest.prx was found (" + reason +
+            "); relink its directory with the relinker's --module-dir <dir> option"
+        : "dlopen: cannot load " + resolved.string() + " (" + reason + ")";
+    Error(message.c_str());
+    static std::mutex reportedLock;
+    static std::set<std::filesystem::path> reported;
+    std::lock_guard lock(reportedLock);
+    if (reported.insert(resolved).second) std::fprintf(stderr, "[module] %s\n", message.c_str());
+}
+#ifdef _WIN32
+HMODULE LoadWithoutErrorDialog(const std::filesystem::path& path) {
+    const auto mode = GetThreadErrorMode();
+    SetThreadErrorMode(mode | SEM_FAILCRITICALERRORS, nullptr);
+    const auto loaded = LoadLibraryExW(path.c_str(), nullptr, LOAD_WITH_ALTERED_SEARCH_PATH);
+    const auto error = GetLastError();
+    SetThreadErrorMode(mode, nullptr);
+    SetLastError(error);
+    return loaded;
+}
+#endif
 struct Module {
     void* native = nullptr;
     bool owned = true;
@@ -179,9 +203,15 @@ void* APS5_VABI dlopen_nid_postfix(const char* path, int flags) {
             module->owned = false;
         } else {
             if (!*path) { Error("dlopen: empty module path"); return nullptr; }
-            const auto resolved = RelinkedModulePath_nid_no_patch(ResolvePath_nid_no_patch(path));
-            module->native = LoadLibraryExW(resolved.c_str(), nullptr, LOAD_WITH_ALTERED_SEARCH_PATH);
-            if (module->native) InitializeDeferredModule(static_cast<HMODULE>(module->native));
+            const auto source = ResolvePath_nid_no_patch(path);
+            const auto resolved = RelinkedModulePath_nid_no_patch(source);
+            module->native = LoadWithoutErrorDialog(resolved);
+            if (!module->native) {
+                const auto error = GetLastError();
+                LoadFailure(source, resolved, "Windows loader error " + std::to_string(error));
+                return nullptr;
+            }
+            InitializeDeferredModule(static_cast<HMODULE>(module->native));
         }
         if (!module->native) {
             char message[128];
@@ -189,11 +219,15 @@ void* APS5_VABI dlopen_nid_postfix(const char* path, int flags) {
             Error(message); return nullptr;
         }
 #else
-        const auto resolved = path ? RelinkedModulePath_nid_no_patch(ResolvePath_nid_no_patch(path)).string() : std::string{};
+        const auto source = path ? ResolvePath_nid_no_patch(path) : std::filesystem::path{};
+        const auto resolved = path ? RelinkedModulePath_nid_no_patch(source) : std::filesystem::path{};
         const int nativeFlags = ((flags & 3) == 1 ? RTLD_LAZY : RTLD_NOW) |
             ((flags & 0x100) ? RTLD_GLOBAL : RTLD_LOCAL);
         module->native = ::dlopen(path ? resolved.c_str() : nullptr, nativeFlags);
-        if (!module->native) { Error(::dlerror()); return nullptr; }
+        if (!module->native) {
+            if (path) LoadFailure(source, resolved, ::dlerror()); else Error(::dlerror());
+            return nullptr;
+        }
 #endif
         std::lock_guard lock(modulesMutex);
         const auto handle = nextHandle++;

core/libs/tests/GuestDynamicLoader.cpp

@@ -3,9 +3,15 @@
 #include <cstdlib>
 #include <cstdint>
 #include <cstdio>
+#include <cstring>
 #include <filesystem>
 #include <fstream>
-#ifndef _WIN32
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
 #include <dlfcn.h>
 #endif
 #include <stdexcept>
@@ -37,6 +43,28 @@ static bool ThrowsRuntimeError(TFunction function) {
     }
     return false;
 }
+static bool FailsAsUnrelinked(const std::filesystem::path& module) {
+    void* handle = dlopen_nid_postfix(module.string().c_str(), 2);
+    const char* error = dlerror_nid_postfix();
+    return handle == nullptr && error != nullptr && std::strstr(error, module.filename().string().c_str()) != nullptr &&
+        std::strstr(error, "--module-dir") != nullptr;
+}
+static void CheckUnrelinkedModule(const std::filesystem::path& module) {
+#ifdef _WIN32
+    const UINT processMode = SetErrorMode(0);
+    DWORD threadMode = 0;
+    const bool threadModeSet = SetThreadErrorMode(SEM_NOGPFAULTERRORBOX, &threadMode) != 0;
+#endif
+    const bool first = FailsAsUnrelinked(module);
+    const bool again = FailsAsUnrelinked(module);
+#ifdef _WIN32
+    const DWORD threadModeAfter = GetThreadErrorMode();
+    SetThreadErrorMode(threadMode, nullptr);
+    SetErrorMode(processMode);
+    Require(threadModeSet && threadModeAfter == SEM_NOGPFAULTERRORBOX);
+#endif
+    Require(first && again);
+}
 #ifdef _WIN32
 static void CheckGlobalLookupOrder(const char* path) {
     const std::filesystem::path other = "anyps5-global-order-for-test.guest.prx";
@@ -108,7 +136,8 @@ int main(int argc, char** argv) {
     Require(dlerror_nid_postfix() != nullptr);
     Require(dlerror_nid_postfix() == nullptr);
     Require(dlopen_nid_postfix("anyps5-missing-module-for-test.prx", 2) == nullptr);
-    Require(dlerror_nid_postfix() != nullptr);
+    const char* missing = dlerror_nid_postfix();
+    Require(missing != nullptr && std::strstr(missing, "--module-dir") == nullptr);
     void* mainSymbol = nullptr;
     const int mainResult = sceKernelDlsym(0, "GuestDefaultScopeOnly", &mainSymbol);
     std::fprintf(stderr, "sceKernelDlsym(0, GuestDefaultScopeOnly): result=0x%08x, symbol=%p\n", static_cast<unsigned>(mainResult), mainSymbol);
@@ -155,10 +184,12 @@ int main(int argc, char** argv) {
     const std::filesystem::path guest = "anyps5-relinked-module-for-test.prx";
     auto relinked = guest;
     relinked += ".guest.prx";
+    std::filesystem::remove(relinked);
     {
         std::ofstream elf(guest, std::ios::binary);
         elf.write("\x7f" "ELF", 4);
     }
+    CheckUnrelinkedModule(guest);
     std::filesystem::copy_file(argv[1], relinked, std::filesystem::copy_options::overwrite_existing);
     void* redirected = dlopen_nid_postfix(guest.string().c_str(), 2);
     Require(redirected != nullptr);

docs/user/USAGE.md

@@ -172,7 +172,7 @@ Some applications also need their original `app0/sce_sys/param.json` at runtime.
 
 Resources can also be packed into `app0.zar` beside `app0/`, a [ZArchive](https://github.com/Exzap/ZArchive) whose root is the `app0/` directory. A file present in `app0/` is read from there and any other file from the archive; the executable, `sce_module/` and `sce_sys/` stay in `app0/`. The archive is read only through `sceKernelOpen`/`open`, the read, seek and stat calls on its descriptors and `sceKernelStat`; other calls on an archived path, listing a directory that the archive also contains and opening a file in it for writing fail as described in [technical debt](../dev/debt/libkernel.md). An archive that cannot be opened stops the title at its first file access.
 
-Use the generated files printed as `Guest module:` for bundled title modules. `libs/` is for AnyPS5 system libraries, not the original PS5 `.prx` files. Placing an original PS5 module in `libs/` on Windows makes Windows try to load it as a DLL and can fail with error 193 (not a valid Win32 application).
+Use the generated files printed as `Guest module:` for bundled title modules. `libs/` is for AnyPS5 system libraries, not the original PS5 `.prx` files. Placing an original PS5 module in `libs/` on Windows makes Windows try to load it as a DLL and can fail with error 193 (not a valid Win32 application). A module the title loads while it runs that was not relinked, such as one in a directory not given to `--module-dir`, is not loaded either: the title gets an error and continues, and the runtime prints `[module] dlopen: <path> was not relinked: ...` once for each file. Relink that module's directory with `--module-dir <dir>`.
 
 On Windows, a self-built `libs/` also needs `libgcc_s_seh-1.dll`, `libstdc++-6.dll` and `libwinpthread-1.dll` from the `mingw64/bin` directory of the toolchain that built the libraries; the release archives already contain them. The libraries are loaded without searching `PATH`, so a copy elsewhere on the system is not used, and a library that needs one of them fails with error 126 (the specified module could not be found) although its `.prx` file is present.