Merge pull request #8550 from Retr0G28/fix/fopen-directory-read-windows

fix(libc): open a directory with fopen in a read mode on Windows

core/libs/prx/libc/include/FileStream.hpp

@@ -23,6 +23,8 @@ struct UcrtStreamState {
     int count;
     long flags;
 };
+extern "C" void LibcRegisterDirectoryStreams_nid_no_patch(int (*open)(const wchar_t* path), void (*forget)(int descriptor));
+extern "C" void ForgetDirectoryStream_nid_no_patch(int descriptor);
 #else
 #include <unistd.h>
 #endif
@@ -66,6 +68,7 @@ class FileStream {
     bool _dynamic;
     bool guestError = false;
     bool randomDevice = false;
+    bool directory = false;
 
 public:
     explicit FileStream(std::FILE* handle, bool dynamic = false) : _handle(handle), _dynamic(dynamic) {
@@ -130,6 +133,14 @@ public:
         return randomDevice;
     }
 
+    void MarkDirectory() {
+        directory = true;
+    }
+
+    bool IsDirectory() const {
+        return directory;
+    }
+
     bool Reopen(const std::filesystem::path& filename, const char* mode,
         [[maybe_unused]] std::FILE* (*open)(const std::filesystem::path&, const std::string&) = nullptr) {
         auto* previous = GetHandle();
@@ -140,6 +151,7 @@ public:
         std::wstring wideMode;
         for (const char character : std::string_view(mode)) wideMode.push_back(static_cast<unsigned char>(character));
         const int replaced = previous ? _fileno(previous) : -1;
+        if (std::exchange(directory, false)) ForgetDirectoryStream_nid_no_patch(replaced);
         auto* const standard = HostStandardHandle_nid_no_patch(replaced);
         GuestFileLockForget_nid_no_patch(replaced);
         if (open && !filename.empty()) {
@@ -260,6 +272,10 @@ public:
 #endif
     }
 
+    void KeepDirectoryReadError(bool failed) {
+        if (failed && directory && (_guest.flags & 0x40) != 0) errno = 21;
+    }
+
     void SetError() {
         guestError = true;
         SyncStatus();
@@ -276,6 +292,7 @@ public:
         GetHandle();
 #ifdef _WIN32
         GuestFileLockForget_nid_no_patch(::_fileno(_handle));
+        if (std::exchange(directory, false)) ForgetDirectoryStream_nid_no_patch(::_fileno(_handle));
 #endif
         const int result = std::fclose(std::exchange(_handle, nullptr));
         _guest.flags = 0;

core/libs/prx/libc/src/FileAndHeap.cpp

@@ -65,6 +65,19 @@ static std::optional<GuestFileMode> ParseFileMode(const char* mode) {
 
 #ifdef _WIN32
 static constexpr DWORD AppendOnlyAccess = FILE_GENERIC_WRITE & ~FILE_WRITE_DATA;
+
+static std::optional<std::wstring> HostOpenName(const std::filesystem::path& path) {
+    auto filename = path.native();
+    if (!filename.starts_with(L"\\\\?\\") && !filename.starts_with(L"\\\\.\\")) {
+        std::error_code pathError;
+        auto absolutePath = std::filesystem::absolute(path, pathError);
+        if (pathError) { errno = pathError.default_error_condition().value(); return std::nullopt; }
+        const auto absolute = absolutePath.lexically_normal().make_preferred().native();
+        if (absolute.size() >= MAX_PATH)
+            filename = absolute.starts_with(L"\\\\") ? L"\\\\?\\UNC\\" + absolute.substr(2) : L"\\\\?\\" + absolute;
+    }
+    return filename;
+}
 #endif
 
 static std::FILE* OpenNative(const std::filesystem::path& path, const std::string& nativeMode) {
@@ -76,16 +89,9 @@ static std::FILE* OpenNative(const std::filesystem::path& path, const std::strin
     const DWORD disposition = access == 'r' ? OPEN_EXISTING
         : nativeMode.find('x') != std::string::npos ? CREATE_NEW : access == 'w' ? CREATE_ALWAYS : OPEN_ALWAYS;
     SECURITY_ATTRIBUTES security{sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE};
-    auto filename = path.native();
-    if (!filename.starts_with(L"\\\\?\\") && !filename.starts_with(L"\\\\.\\")) {
-        std::error_code pathError;
-        auto absolutePath = std::filesystem::absolute(path, pathError);
-        if (pathError) { errno = pathError.default_error_condition().value(); return nullptr; }
-        const auto absolute = absolutePath.lexically_normal().make_preferred().native();
-        if (absolute.size() >= MAX_PATH)
-            filename = absolute.starts_with(L"\\\\") ? L"\\\\?\\UNC\\" + absolute.substr(2) : L"\\\\?\\" + absolute;
-    }
-    const auto handle = CreateFileW(filename.c_str(), desired, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, &security,
+    const auto filename = HostOpenName(path);
+    if (!filename) return nullptr;
+    const auto handle = CreateFileW(filename->c_str(), desired, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, &security,
         disposition, FILE_ATTRIBUTE_NORMAL, nullptr);
     if (handle == INVALID_HANDLE_VALUE) {
         errno = WindowsOpenError_nid_no_patch(GetLastError());
@@ -307,6 +313,47 @@ static int OpenFailure(const std::filesystem::path& path, bool writes, int hostE
     return writes && error == 13 && std::filesystem::is_directory(path, status) ? 21 : error;
 }
 
+#ifdef _WIN32
+static int (*g_openDirectoryDescriptor)(const wchar_t*) = nullptr;
+static void (*g_forgetDirectoryDescriptor)(int) = nullptr;
+
+void LibcRegisterDirectoryStreams_nid_no_patch(int (*open)(const wchar_t*), void (*forget)(int)) {
+    g_openDirectoryDescriptor = open;
+    g_forgetDirectoryDescriptor = forget;
+}
+
+void ForgetDirectoryStream_nid_no_patch(int descriptor) {
+    if (!g_forgetDirectoryDescriptor) throw std::runtime_error("ForgetDirectoryStream: libkernel has not registered its directory descriptors");
+    g_forgetDirectoryDescriptor(descriptor);
+}
+
+static FileStream* OpenDirectoryStream(const std::filesystem::path& path) {
+    if (!g_openDirectoryDescriptor) throw std::runtime_error("fopen: libkernel has not registered its directory descriptors");
+    const auto filename = HostOpenName(path);
+    if (!filename) { errno = GuestErrnoFromHost_nid_no_patch(errno); return nullptr; }
+    const auto release = [](void* pointer) { ::operator delete(pointer); };
+    std::unique_ptr<void, decltype(release)> storage(::operator new(sizeof(FileStream)), release);
+    const int descriptor = g_openDirectoryDescriptor(filename->c_str());
+    if (descriptor < 0) {
+        const int error = GuestErrnoFromHost_nid_no_patch(errno);
+        errno = error == 2 ? NotFoundError_nid_no_patch(path) : error;
+        return nullptr;
+    }
+    auto* native = ::_fdopen(descriptor, "rb");
+    if (!native) {
+        const int error = GuestErrnoFromHost_nid_no_patch(errno);
+        ForgetDirectoryStream_nid_no_patch(descriptor);
+        ::_close(descriptor);
+        errno = error;
+        return nullptr;
+    }
+    auto* stream = new (storage.get()) FileStream(native, true);
+    storage.release();
+    stream->MarkDirectory();
+    return stream;
+}
+#endif
+
 FileStream* APS5_VABI freopen_nid_postfix(const char* filename, const char* mode, FileStream* stream) {
     if (!stream || !mode) { errno = 22; return nullptr; }
     if (!filename) { errno = 45; return nullptr; } // Mode-only reopening is not supported.
@@ -370,7 +417,13 @@ FileStream* APS5_VABI fopen_nid_postfix(const char* filename, const char* mode)
     if (const int error = ResolvePathError_nid_no_patch(filename, fpath)) { errno = error; return nullptr; }
     std::unique_ptr<std::FILE, int (*)(std::FILE*)> handle(OpenNative(fpath, parsed->native), std::fclose);
     if (!handle) {
-        errno = OpenFailure(fpath, parsed->writes, errno);
+        const int hostError = errno;
+#ifdef _WIN32
+        std::error_code status;
+        if (!parsed->writes && GuestErrnoFromHost_nid_no_patch(hostError) == 13 && std::filesystem::is_directory(fpath, status))
+            return OpenDirectoryStream(fpath);
+#endif
+        errno = OpenFailure(fpath, parsed->writes, hostError);
         if (errno == 2) errno = NotFoundError_nid_no_patch(fpath);
         return nullptr;
     }
@@ -445,6 +498,7 @@ size_t APS5_VABI fread_nid_postfix(void* buffer, size_t size, size_t count, File
     if (!stream->SwitchToReading()) return 0;
     const auto result = std::fread(buffer, size, count, handle);
     stream->SyncStatus();
+    stream->KeepDirectoryReadError(result < count);
     return result;
 }
 
@@ -529,7 +583,7 @@ int APS5_VABI fputs_nid_postfix(const char* str, FileStream* stream) {
 int APS5_VABI fflush_nid_postfix(FileStream* stream) {
     if (stream && !stream->IsOpen()) { errno = 9; return -1; }
     std::FILE* native = stream ? GetStreamHandle(stream) : nullptr;
-    if (stream && stream->IsRandomDevice()) return 0;
+    if (stream && (stream->IsRandomDevice() || stream->IsDirectory())) return 0;
 #ifdef __linux__
     if (native && !__fwriting(native)) return 0;
 #endif

core/libs/prx/libc/src/Formatting.cpp

@@ -378,6 +378,7 @@ int APS5_VABI fscanf_nid_postfix(FileStream* stream, const char* format, ...) {
     va_end(args);
 #endif
     stream->SyncStatus();
+    stream->KeepDirectoryReadError(result == -1);
     return result;
 }
 
@@ -656,6 +657,7 @@ int APS5_VABI vfscanf_nid_postfix(FileStream* stream, const char* format, VaList
     va_end(copy);
 #endif
     stream->SyncStatus();
+    stream->KeepDirectoryReadError(result == -1);
     return result;
 }
 

core/libs/prx/libc/src/StandardStreams.cpp

@@ -39,6 +39,7 @@ int APS5_VABI fgetc_nid_postfix(FileStream* stream) {
     if (!stream->SwitchToReading()) return -1;
     const int result = std::fgetc(native);
     stream->SyncStatus();
+    stream->KeepDirectoryReadError(result == -1);
     return result;
 }
 std::int32_t APS5_VABI fgetwc_nid_postfix(FileStream* stream) {
@@ -94,6 +95,7 @@ int APS5_VABI getc_unlocked_nid_postfix(FileStream* stream) {
     const int result = ::getc_unlocked(native);
 #endif
     stream->SyncUnlockedStatus();
+    stream->KeepDirectoryReadError(result == -1);
     return result;
 }
 int APS5_VABI __srget_nid_postfix(FileStream* stream) { return fgetc_nid_postfix(stream); }
@@ -140,6 +142,7 @@ char* APS5_VABI fgets_nid_postfix(char* buffer, int size, FileStream* stream) {
     if (!stream->SwitchToReading()) return nullptr;
     auto* result = std::fgets(buffer, size, native);
     stream->SyncStatus();
+    stream->KeepDirectoryReadError(result == nullptr);
     return result;
 }
 int APS5_VABI feof_nid_postfix(FileStream* stream) {

core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp

@@ -8,6 +8,7 @@
 
 #ifdef _WIN32
 
+#include "prx/libc/include/FileStream.hpp"
 #include "prx/libc/include/General.hpp"
 
 #include <cerrno>
@@ -63,6 +64,19 @@ void Load(DirectoryState& state, int fd) {
     }
 }
 
+int OpenStreamDirectory(const wchar_t* path) {
+    return File::OpenDirectoryDescriptor(path);
+}
+
+struct StreamRegistration {
+    StreamRegistration() { LibcRegisterDirectoryStreams_nid_no_patch(&OpenStreamDirectory, &File::ForgetDirectoryDescriptor); }
+    ~StreamRegistration() { LibcRegisterDirectoryStreams_nid_no_patch(nullptr, nullptr); }
+    StreamRegistration(const StreamRegistration&) = delete;
+    StreamRegistration& operator=(const StreamRegistration&) = delete;
+};
+
+const StreamRegistration g_streamRegistration;
+
 }
 
 namespace File {

core/libs/tests/FileStreams.cpp

@@ -234,18 +234,11 @@ int main(int argc, char** argv) {
     errno = 0;
     Require(fopen_nid_postfix(argv[1], "rb") == nullptr);
     Require(errno == ENOENT);
-#ifdef _WIN32
-    const std::string directoryMode = "rb";
-    constexpr int directoryError = EACCES;
-#else
-    const std::string directoryMode = "wb";
-    constexpr int directoryError = EISDIR;
-#endif
     const std::string directoryName = std::string(argv[1]) + ".dir";
     Require(std::filesystem::create_directory(directoryName));
     errno = 0;
-    Require(fopen_nid_postfix(directoryName.c_str(), directoryMode.c_str()) == nullptr);
-    Require(errno == directoryError);
+    Require(fopen_nid_postfix(directoryName.c_str(), "wb") == nullptr);
+    Require(errno == EISDIR);
     Require(std::filesystem::remove(directoryName));
     const std::u8string unicodeName = u8"セーブé.tmp";
     const std::string guestName(unicodeName.begin(), unicodeName.end());

core/libs/tests/GuestFilesystem.cpp

@@ -85,6 +85,13 @@ FileStream* APS5_VABI freopen_nid_postfix(const char*, const char*, FileStream*)
 std::size_t APS5_VABI fread_nid_postfix(void*, std::size_t, std::size_t, FileStream*);
 int APS5_VABI fclose_nid_postfix(FileStream*);
 int APS5_VABI fgetc_nid_postfix(FileStream*);
+int APS5_VABI fileno_nid_postfix(FileStream*);
+char* APS5_VABI fgets_nid_postfix(char*, int, FileStream*);
+int APS5_VABI fscanf_nid_postfix(FileStream*, const char*, ...);
+int APS5_VABI fputc_nid_postfix(int, FileStream*);
+int APS5_VABI fseek_nid_postfix(FileStream*, std::int64_t, int);
+std::int64_t APS5_VABI ftell_nid_postfix(FileStream*);
+void APS5_VABI clearerr_nid_postfix(FileStream*);
 int APS5_VABI feof_nid_postfix(FileStream*);
 int APS5_VABI ferror_nid_postfix(FileStream*);
 int APS5_VABI setvbuf_nid_postfix(FileStream*, char*, int, std::size_t);
@@ -606,6 +613,45 @@ int main() {
     for (const char* openMode : {"w", "wb", "r+", "a"}) {
         Require(fopen_nid_postfix(root.string().c_str(), openMode) == nullptr && *__error_nid_postfix() == 21);
     }
+    {
+        const auto listed = root / "listed.txt";
+        std::ofstream(listed, std::ios::binary) << "listed";
+        for (const char* openMode : {"r", "rb", "re"}) {
+            auto* directoryStream = fopen_nid_postfix(root.string().c_str(), openMode);
+            Require(directoryStream != nullptr);
+            const int directoryDescriptor = fileno_nid_postfix(directoryStream);
+            FileStat directoryStatus{};
+            Require(directoryDescriptor >= 0 && sceKernelFstat(directoryDescriptor, &directoryStatus) == 0 && (directoryStatus.st_mode & 0170000) == 0040000);
+            Require(feof_nid_postfix(directoryStream) == 0 && ferror_nid_postfix(directoryStream) == 0 && fflush_nid_postfix(directoryStream) == 0);
+            char unread[8] = {};
+            int scanned = 0;
+            *__error_nid_postfix() = 0;
+            Require(fread_nid_postfix(unread, 1, sizeof(unread), directoryStream) == 0 && *__error_nid_postfix() == 21);
+            Require(ferror_nid_postfix(directoryStream) != 0 && feof_nid_postfix(directoryStream) == 0);
+            clearerr_nid_postfix(directoryStream);
+            Require(ferror_nid_postfix(directoryStream) == 0);
+            *__error_nid_postfix() = 0;
+            Require(fgetc_nid_postfix(directoryStream) == -1 && *__error_nid_postfix() == 21 && ferror_nid_postfix(directoryStream) != 0);
+            clearerr_nid_postfix(directoryStream);
+            *__error_nid_postfix() = 0;
+            Require(fgets_nid_postfix(unread, sizeof(unread), directoryStream) == nullptr && *__error_nid_postfix() == 21 && ferror_nid_postfix(directoryStream) != 0);
+            clearerr_nid_postfix(directoryStream);
+            *__error_nid_postfix() = 0;
+            Require(fscanf_nid_postfix(directoryStream, "%d", &scanned) == -1 && *__error_nid_postfix() == 21 && ferror_nid_postfix(directoryStream) != 0);
+            clearerr_nid_postfix(directoryStream);
+            Require(fseek_nid_postfix(directoryStream, 0, 0) == 0 && ftell_nid_postfix(directoryStream) == 0);
+            *__error_nid_postfix() = 0;
+            Require(fputc_nid_postfix('x', directoryStream) == -1 && *__error_nid_postfix() == 9 && ferror_nid_postfix(directoryStream) != 0);
+            Require(fclose_nid_postfix(directoryStream) == 0);
+#ifdef _WIN32
+            const int reused = open_nid_postfix(listed.string().c_str(), 0, 0);
+            char text[6]{};
+            Require(reused == directoryDescriptor && read_nid_postfix(reused, text, sizeof(text)) == 6 && std::memcmp(text, "listed", 6) == 0);
+            Require(close_nid_postfix(reused) == 0);
+#endif
+        }
+        Require(std::filesystem::remove(listed));
+    }
     Require(mkdir_nid_postfix((root / "missing" / "child").string().c_str(), 0700) == -1 && *__error_nid_postfix() == 2);
     Require(mkdir_nid_postfix(nullptr, 0700) == -1 && *__error_nid_postfix() == 14);
     const auto file = root / "file.txt";

docs/dev/debt/libc.md

@@ -95,7 +95,7 @@
 - On macOS the [guest arena](../../../core/libs/prx/libc/src/GuestArena.cpp) finds host regions with `mach_vm_region` and reserves the rest with a fixed `mach_vm_map`, which fails instead of replacing, as macOS has no `/proc/self/maps` or `MAP_FIXED_NOREPLACE`. The kernel keeps the range up to 0x70_0000_0000 unmappable for every process (max protection none). It starts at 0x10_0000_0000 on macOS 26.5 (M1 Pro, arm64 and x86_64 under Rosetta) and at 0xF_C000_0000 on macOS 27 (27.0.1 on an M4 Pro, arm64 and x86_64 under Rosetta; 27.2 on an M2 Max, arm64). With dyld and the Rosetta runtime near 0x2_0200_0000 and 0x3_0400_0000, a guest mapping fixed inside those ranges fails where the console places it; mappings without a fixed address are placed around them.
 - [sceKernelSync](../../../core/libs/prx/libc/src/General.cpp) (libkernel) flushes only the host files and directories recorded through `RecordWrittenPath_nid_no_patch` (the libc, libkernel, save data and app content file functions) and their changed parent directories. A host write in a library that does not record its path is not flushed by `sceKernelSync`
 - [sceKernelSync](../../../core/libs/prx/libc/src/General.cpp) (libkernel) on Windows skips a recorded file it cannot open for writing (read-only, or held open by a handle that denies write sharing), since `FlushFileBuffers` needs a writable handle. On Linux, it skips a recorded file it cannot open for reading. Such a file's data may not reach the disk before a host crash; the guest reads the same data either way
-- [`fopen` and `freopen`](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) on Windows fail to open a directory in a read mode with `EACCES`, where FreeBSD opens it; directory streams are not emulated
+- [`fopen` and `freopen`](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) on Windows: `freopen` onto a directory in a read mode fails with `EACCES`, where FreeBSD opens it. Reading a directory stream from `fopen` fails with `EISDIR` and sets the error indicator on every host, as FreeBSD 11 tmpfs does; FreeBSD 11 FFS returns the directory's raw blocks instead, and what the console's filesystems return is unverified
 - [readdir and readdir_r](../../../core/libs/prx/libc/src/Directory.cpp) (libc) on Windows report, on a file system whose directory listing carries no file ID (the entry's ID is 0, or the volume refuses `FileIdBothDirectoryInfo`), a nonzero `d_fileno` hashed from the entry name and the directory's file index, which differs from `lstat`'s `st_ino`. Not reproduced: NTFS lists an ID for every entry
 - [Writes on an update stream](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) on Windows go through the UCRT, which drops a write that directly follows a read without `fflush`, `fseek`, `fsetpos` or `rewind` unless the read reached end of file: `fwrite` returns 0, and `fputc` and `fprintf` report success, without the error indicator. FreeBSD 11.4's [`__swsetup`](https://github.com/freebsd/freebsd-src/blob/d106002967eff712894ab8b5337bbbff106ce5e0/lib/libc/stdio/wsetup.c#L59-L74) discards the read buffer and writes at the descriptor's offset, past a read-ahead of `st_blksize` bytes for a regular file, so where the console writes depends on its block size. Reads directly after writes follow FreeBSD's `__srefill`
 - [`fopen` of /dev/random and /dev/urandom](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) in a read mode returns a stream over the host null device whose `fread` fills the buffer with host OS entropy, as the libkernel [random device](../../../core/libs/prx/libkernel/File/src/RandomDevice.cpp) descriptors do. Only the exact paths are recognised, the stream has no descriptor, `setvbuf`, `feof`, `ferror`, `clearerr`, `flockfile`, `ftrylockfile`, `funlockfile` and `fclose` work on it, `fflush` returns 0 as on a FreeBSD read-only stream, and every other stream function throws, as does opening the paths for writing. `freopen` onto these paths still fails with `ENOENT`. ProsperoTV (PPSA99003) reads its phone pairing code this way