Merge pull request #10545 from Bram910/fix/file-prefix-allocation

fix(io): avoid allocating the entire file prefix limit

core/relinker/CMakeLists.txt

@@ -364,6 +364,18 @@ if(BUILD_TESTING)
     add_test(NAME file_identity COMMAND file_identity_tests)
     set_tests_properties(file_identity PROPERTIES TIMEOUT 20)
 
+    add_test_executable(file_reader_tests
+        ${CMAKE_CURRENT_SOURCE_DIR}/io/tests/FileReaderTests.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/io/src/FileReader.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/io/src/NativePath.cpp
+    )
+    target_include_directories(file_reader_tests PRIVATE
+        ${CMAKE_CURRENT_SOURCE_DIR}/io/include
+        ${CMAKE_CURRENT_SOURCE_DIR}/domain/include
+    )
+    add_test(NAME file_reader_prefix COMMAND file_reader_tests)
+    set_tests_properties(file_reader_prefix PROPERTIES TIMEOUT 20)
+
     add_test_executable(param_json_utf8_tests
         ${CMAKE_CURRENT_SOURCE_DIR}/../libs/tests/ParamJsonParser.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libkernel/AppMetadata/src/ParamJsonParser.cpp)

core/relinker/io/src/FileReader.cpp

@@ -1,6 +1,8 @@
 #include <io/FileReader.hpp>
 #include <io/NativePath.hpp>
 #include <domain/Types.hpp>
+#include <algorithm>
+#include <array>
 #include <filesystem>
 #include <fstream>
 #include <limits>
@@ -17,12 +19,17 @@ std::vector<std::uint8_t> FileReader::ReadPrefix(const std::string& path, const
     std::ifstream f(native, std::ios::binary);
     if (!f)
         throw Domain::RelinkerException("Cannot open file: " + path);
-    std::vector<std::uint8_t> buf(size);
-    f.read(reinterpret_cast<char*>(buf.data()), static_cast<std::streamsize>(size));
-    const auto count = f.gcount();
-    if (f.bad())
-        throw Domain::RelinkerException("Cannot read file: " + path);
-    buf.resize(static_cast<std::size_t>(count));
+    std::vector<std::uint8_t> buf;
+    std::array<std::uint8_t, 4096> chunk;
+    while (buf.size() < size) {
+        const auto requested = std::min(size - buf.size(), chunk.size());
+        f.read(reinterpret_cast<char*>(chunk.data()), static_cast<std::streamsize>(requested));
+        const auto count = static_cast<std::size_t>(f.gcount());
+        if (f.bad())
+            throw Domain::RelinkerException("Cannot read file: " + path);
+        buf.insert(buf.end(), chunk.begin(), chunk.begin() + count);
+        if (count < requested) break;
+    }
     return buf;
 }
 

core/relinker/io/tests/FileReaderTests.cpp

@@ -0,0 +1,107 @@
+#include <io/FileReader.hpp>
+#include <io/NativePath.hpp>
+#include <domain/Types.hpp>
+#include <algorithm>
+#include <chrono>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <iterator>
+#include <limits>
+#include <stdexcept>
+
+namespace {
+
+using Bytes = std::vector<std::uint8_t>;
+
+void require(bool condition, const std::string& message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+template<typename TAction>
+void requireError(const TAction& action, const std::string& message) {
+    try {
+        action();
+    } catch (const Domain::RelinkerException& error) {
+        require(error.what() == message, "Unexpected prefix-read error: " + std::string(error.what()));
+        return;
+    }
+    throw std::runtime_error("Invalid prefix read was accepted: " + message);
+}
+
+void prefixes(const std::filesystem::path& path, const Bytes& expected) {
+    const Io::FileReader reader;
+    const auto name = Io::Utf8Path(path);
+    const auto maximum = static_cast<std::size_t>(std::numeric_limits<std::streamsize>::max());
+    for (const std::size_t limit : {std::size_t{0}, std::size_t{1}, std::size_t{2}, std::size_t{4095},
+            std::size_t{4096}, std::size_t{4097}, std::size_t{8191}, std::size_t{8192}, std::size_t{8193},
+            expected.size(), expected.size() + 1, maximum}) {
+        const auto count = std::min(expected.size(), limit);
+        require(reader.ReadPrefix(name, limit) == Bytes(expected.begin(), expected.begin() + count),
+            "Incorrect file prefix for limit " + std::to_string(limit));
+    }
+}
+
+void fileContents(const std::filesystem::path& directory) {
+    const auto path = directory / Io::NativePath("prefix-\xc3\xa9-\xe6\x97\xa5.bin");
+    for (const std::size_t size : {std::size_t{0}, std::size_t{3}, std::size_t{4096}, std::size_t{4097},
+            std::size_t{8192}, std::size_t{10003}}) {
+        Bytes expected(size);
+        for (std::size_t index = 0; index < size; ++index)
+            expected[index] = static_cast<std::uint8_t>(index * 37 + 13);
+        std::ofstream output(path, std::ios::binary);
+        if (!expected.empty())
+            output.write(reinterpret_cast<const char*>(expected.data()), static_cast<std::streamsize>(expected.size()));
+        output.close();
+        require(static_cast<bool>(output), "Cannot prepare the prefix-read fixture");
+        prefixes(path, expected);
+        require(Io::FileReader().Read(Io::Utf8Path(path)) == expected, "Prefix reads changed the input file");
+    }
+}
+
+void invalidRequests(const std::filesystem::path& directory) {
+    const Io::FileReader reader;
+    const auto missing = Io::Utf8Path(directory / "missing.bin");
+    const auto folder = Io::Utf8Path(directory);
+    for (const std::size_t size : {std::size_t{0}, std::size_t{1}, std::size_t{4097}}) {
+        requireError([&] { reader.ReadPrefix(missing, size); }, "Cannot open file: " + missing);
+        requireError([&] { reader.ReadPrefix(folder, size); }, "Cannot open file: " + folder);
+    }
+    const auto maximum = static_cast<std::size_t>(std::numeric_limits<std::streamsize>::max());
+    if (maximum < std::numeric_limits<std::size_t>::max()) {
+        requireError([&] { reader.ReadPrefix(missing, maximum + 1); }, "File prefix is too large: " + missing);
+    }
+}
+
+void zeroReportedFileSize() {
+#ifdef __linux__
+    const std::filesystem::path path = "/proc/self/cmdline";
+    require(std::filesystem::is_regular_file(path) && std::filesystem::file_size(path) == 0,
+        "Expected a regular proc file with a zero reported size");
+    std::ifstream input(path, std::ios::binary);
+    const Bytes expected{std::istreambuf_iterator<char>(input), std::istreambuf_iterator<char>()};
+    require(!expected.empty(), "Cannot read the proc prefix fixture");
+    prefixes(path, expected);
+#endif
+}
+
+}
+
+int main() {
+    const auto directory = std::filesystem::temp_directory_path() /
+        ("anyps5-file-reader-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()));
+    try {
+        std::filesystem::create_directory(directory);
+        fileContents(directory);
+        invalidRequests(directory);
+        zeroReportedFileSize();
+        std::filesystem::remove_all(directory);
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        std::error_code ignored;
+        std::filesystem::remove_all(directory, ignored);
+        return 1;
+    }
+    std::cout << "File prefix-read tests passed\n";
+    return 0;
+}