Merge pull request #9631 from UnhealthyKraken/feat/libc-fetch-seqcst

feat(libc): implement sequential bitwise fetch helpers

core/libs/CMakeLists.txt

@@ -278,6 +278,32 @@ if(BUILD_TESTING)
     add_test(NAME nid_buffer_validation COMMAND nid_buffer_validation_tests)
     set_tests_properties(nid_buffer_validation PROPERTIES TIMEOUT 10)
 
+    add_test_executable(guest_dinkumware_fetch_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/GuestDinkumwareFetch.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/nid/src/NidCompute.cpp ${CMAKE_CURRENT_SOURCE_DIR}/nid/src/Sha1.cpp)
+    target_include_directories(guest_dinkumware_fetch_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/nid/include)
+    target_link_libraries(guest_dinkumware_fetch_tests PRIVATE ${CMAKE_DL_LIBS})
+    configure_windows_unwind(guest_dinkumware_fetch_tests)
+    if(MINGW)
+        target_link_libraries(guest_dinkumware_fetch_tests PRIVATE libc)
+    endif()
+    add_custom_target(guest_dinkumware_fetch_libraries DEPENDS ${LIBC_PATCHED})
+    add_dependencies(guest_dinkumware_fetch_tests guest_dinkumware_fetch_libraries)
+    foreach(operation IN ITEMS and or xor)
+        foreach(width IN ITEMS 1 2 4)
+            add_test(NAME guest_dinkumware_fetch_${operation}_${width} COMMAND guest_dinkumware_fetch_tests
+                $<TARGET_FILE_DIR:libc> unpatched ${operation} ${width})
+            add_test(NAME guest_dinkumware_fetch_${operation}_${width}_patched COMMAND guest_dinkumware_fetch_tests
+                ${PATCHED_DIR} patched ${operation} ${width})
+            set_tests_properties(guest_dinkumware_fetch_${operation}_${width}
+                guest_dinkumware_fetch_${operation}_${width}_patched PROPERTIES TIMEOUT 20)
+            if(WIN32)
+                set_tests_properties(guest_dinkumware_fetch_${operation}_${width} PROPERTIES ENVIRONMENT_MODIFICATION
+                    "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+                set_tests_properties(guest_dinkumware_fetch_${operation}_${width}_patched PROPERTIES ENVIRONMENT_MODIFICATION
+                    "PATH=path_list_prepend:${PATCHED_DIR}")
+            endif()
+        endforeach()
+    endforeach()
     add_test_executable(pe_import_first_tests allocator/tests/PeImportFirstTests.cpp allocator/PeImportFirst.cpp)
     target_include_directories(pe_import_first_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_compile_features(pe_import_first_tests PRIVATE cxx_std_20)

core/libs/prx/libc/include/Api.hpp

@@ -163,6 +163,15 @@ int APS5_VABI _Atomic_compare_exchange_strong_8_nid_postfix(volatile std::uint64
 int APS5_VABI _Atomic_compare_exchange_weak_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t* expected, std::uint8_t desired, int, int);
 int APS5_VABI _Atomic_compare_exchange_weak_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t* expected, std::uint16_t desired, int, int);
 int APS5_VABI _Atomic_compare_exchange_weak_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t* expected, std::uint64_t desired, int, int);
+std::uint8_t APS5_VABI _Fetch_and_seq_cst_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value);
+std::uint16_t APS5_VABI _Fetch_and_seq_cst_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value);
+std::uint32_t APS5_VABI _Fetch_and_seq_cst_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value);
+std::uint8_t APS5_VABI _Fetch_or_seq_cst_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value);
+std::uint16_t APS5_VABI _Fetch_or_seq_cst_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value);
+std::uint32_t APS5_VABI _Fetch_or_seq_cst_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value);
+std::uint8_t APS5_VABI _Fetch_xor_seq_cst_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value);
+std::uint16_t APS5_VABI _Fetch_xor_seq_cst_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value);
+std::uint32_t APS5_VABI _Fetch_xor_seq_cst_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value);
 std::uint8_t APS5_VABI _Atomic_fetch_add_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int);
 std::uint16_t APS5_VABI _Atomic_fetch_add_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int);
 std::uint64_t APS5_VABI _Atomic_fetch_add_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int order);

core/libs/prx/libc/src/DinkumwareAtomics.cpp

@@ -63,6 +63,18 @@ int LockedCompareExchange(volatile unsigned char* flag, std::size_t size, volati
 
 extern "C" {
 
+std::uint8_t APS5_VABI _Fetch_and_seq_cst_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value) { return __atomic_fetch_and(target, value, __ATOMIC_SEQ_CST); }
+std::uint16_t APS5_VABI _Fetch_and_seq_cst_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value) { return __atomic_fetch_and(target, value, __ATOMIC_SEQ_CST); }
+std::uint32_t APS5_VABI _Fetch_and_seq_cst_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value) { return __atomic_fetch_and(target, value, __ATOMIC_SEQ_CST); }
+
+std::uint8_t APS5_VABI _Fetch_or_seq_cst_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value) { return __atomic_fetch_or(target, value, __ATOMIC_SEQ_CST); }
+std::uint16_t APS5_VABI _Fetch_or_seq_cst_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value) { return __atomic_fetch_or(target, value, __ATOMIC_SEQ_CST); }
+std::uint32_t APS5_VABI _Fetch_or_seq_cst_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value) { return __atomic_fetch_or(target, value, __ATOMIC_SEQ_CST); }
+
+std::uint8_t APS5_VABI _Fetch_xor_seq_cst_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value) { return __atomic_fetch_xor(target, value, __ATOMIC_SEQ_CST); }
+std::uint16_t APS5_VABI _Fetch_xor_seq_cst_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value) { return __atomic_fetch_xor(target, value, __ATOMIC_SEQ_CST); }
+std::uint32_t APS5_VABI _Fetch_xor_seq_cst_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value) { return __atomic_fetch_xor(target, value, __ATOMIC_SEQ_CST); }
+
 void APS5_VABI _Atomic_store_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { Store(target, value); }
 void APS5_VABI _Atomic_store_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { Store(target, value); }
 void APS5_VABI _Atomic_store_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value, int) { Store(target, value); }

core/libs/tests/GuestDinkumwareFetch.cpp

@@ -0,0 +1,133 @@
+#include "prx/libc/include/general/VabiMacros.hpp"
+#include <nid/NidCompute.hpp>
+#include <array>
+#include <barrier>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <filesystem>
+#include <limits>
+#include <string>
+#include <thread>
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <dlfcn.h>
+#endif
+
+static void Require(bool value, const char* message) {
+    if (value) return;
+    std::fprintf(stderr, "%s\n", message);
+    std::exit(1);
+}
+
+template<typename TValue>
+using Fetch = TValue (APS5_VABI*)(volatile TValue*, TValue);
+
+template<typename TValue>
+static TValue Expected(TValue before, TValue operand, const std::string& operation) {
+    if (operation == "and") return before & operand;
+    if (operation == "or") return before | operand;
+    return before ^ operand;
+}
+
+template<typename TValue>
+static void Check(Fetch<TValue> fetch, const std::string& operation) {
+    constexpr TValue maximum = std::numeric_limits<TValue>::max();
+    std::array<TValue, 3> guarded{maximum, 0, maximum};
+    const auto one = [&](TValue before, TValue operand) {
+        guarded[1] = before;
+        Require(fetch(&guarded[1], operand) == before, "fetch must return the original value");
+        Require(guarded[1] == Expected(before, operand, operation), "bitwise result");
+        Require(guarded[0] == maximum && guarded[2] == maximum, "neighboring storage changed");
+    };
+    if constexpr (sizeof(TValue) == 1) {
+        for (unsigned before = 0; before < 256; ++before)
+            for (unsigned operand = 0; operand < 256; ++operand)
+                one(static_cast<TValue>(before), static_cast<TValue>(operand));
+    } else {
+        constexpr std::array<TValue, 6> values{0, 1, maximum, static_cast<TValue>(maximum >> 1), static_cast<TValue>(TValue{1} << (sizeof(TValue) * 8 - 1)), static_cast<TValue>(0xA55AA55A)};
+        for (TValue before : values) for (TValue operand : values) one(before, operand);
+        std::uint32_t random = 0x31415926;
+        for (unsigned i = 0; i < 4096; ++i) {
+            random = random * 1664525u + 1013904223u;
+            const TValue before = static_cast<TValue>(random);
+            random = random * 1664525u + 1013904223u;
+            one(before, static_cast<TValue>(random));
+        }
+    }
+    guarded[1] = operation == "and" ? maximum : 0;
+    std::array<std::thread, 4> workers;
+    std::barrier start(4);
+    for (unsigned index = 0; index < workers.size(); ++index) {
+        workers[index] = std::thread([&, index] {
+            const TValue bit = static_cast<TValue>(TValue{1} << index);
+            const TValue operand = operation == "and" ? static_cast<TValue>(~bit) : bit;
+            start.arrive_and_wait();
+            for (unsigned iteration = 0; iteration < 1025; ++iteration) {
+                const TValue previous = fetch(&guarded[1], operand);
+                const bool expected = operation == "and" ? iteration == 0 : operation == "or" ? iteration != 0 : iteration % 2 != 0;
+                Require(((previous & bit) != 0) == expected, "concurrent original-bit history");
+            }
+        });
+    }
+    for (auto& worker : workers) worker.join();
+    Require(guarded[1] == (operation == "and" ? static_cast<TValue>(maximum & ~TValue{15}) : TValue{15}), "concurrent updates lost");
+    Require(guarded[0] == maximum && guarded[2] == maximum, "concurrent neighboring storage changed");
+    std::array<TValue, 2> flags{};
+    std::array<TValue, 2> observed{};
+    std::barrier phase(3);
+    const auto litmus = [&](unsigned index) {
+        for (unsigned i = 0; i < 512; ++i) {
+            phase.arrive_and_wait();
+            fetch(&flags[index], operation == "and" ? TValue{0} : TValue{1});
+            observed[index] = fetch(&flags[1 - index], operation == "and" ? maximum : TValue{0});
+            phase.arrive_and_wait();
+        }
+    };
+    std::thread first(litmus, 0), second(litmus, 1);
+    for (unsigned i = 0; i < 512; ++i) {
+        flags.fill(operation == "and" ? maximum : TValue{0});
+        phase.arrive_and_wait();
+        phase.arrive_and_wait();
+        const TValue initial = operation == "and" ? maximum : TValue{0};
+        Require(observed[0] != initial || observed[1] != initial, "forbidden sequentially consistent outcome");
+    }
+    first.join();
+    second.join();
+}
+
+int main(int argc, char** argv) {
+    Require(argc == 5, "library directory, patched or unpatched, operation, width");
+    const auto path = std::filesystem::absolute(argv[1]) / "libc.prx";
+    const std::string operation = argv[3];
+    const int width = std::atoi(argv[4]);
+    Require(operation == "and" || operation == "or" || operation == "xor", "operation");
+    Require(width == 1 || width == 2 || width == 4, "width");
+    const std::string name = "_Fetch_" + operation + "_seq_cst_" + std::to_string(width);
+    const std::string symbol = std::string(argv[2]) == "patched" ? Nid::ComputeNid(name, "") : name + "_nid_postfix";
+#ifdef _WIN32
+    SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX);
+    const auto startup = GetModuleHandleW(L"libc.prx");
+    const auto library = LoadLibraryExW(path.c_str(), nullptr, LOAD_WITH_ALTERED_SEARCH_PATH);
+    Require(library != nullptr, "load libc");
+    std::array<wchar_t, 32768> loaded{};
+    const auto length = GetModuleFileNameW(library, loaded.data(), static_cast<DWORD>(loaded.size()));
+    Require(length != 0 && length < loaded.size(), "loaded libc path");
+    Require(std::filesystem::equivalent(path, std::filesystem::path(loaded.data())), "wrong libc loaded");
+    if (startup) Require(startup == library, "startup libc differs from selected libc");
+    std::printf("Selected libc: %ls (startup module %s)\n", loaded.data(), startup ? "same" : "not imported");
+    const auto function = reinterpret_cast<void*>(GetProcAddress(library, symbol.c_str()));
+#else
+    const auto library = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL);
+    if (!library) std::fprintf(stderr, "%s\n", dlerror());
+    Require(library != nullptr, "load libc");
+    const auto function = dlsym(library, symbol.c_str());
+#endif
+    if (!function) std::fprintf(stderr, "Missing %s (%s)\n", name.c_str(), symbol.c_str());
+    Require(function != nullptr, "cannot exercise missing helper");
+    if (width == 1) Check(reinterpret_cast<Fetch<std::uint8_t>>(function), operation);
+    if (width == 2) Check(reinterpret_cast<Fetch<std::uint16_t>>(function), operation);
+    if (width == 4) Check(reinterpret_cast<Fetch<std::uint32_t>>(function), operation);
+    std::printf("%s: value, original result, bounds, concurrency and ordering checks passed\n", name.c_str());
+}

docs/dev/debt/libc.md

@@ -34,6 +34,7 @@
 - [acosl](../../../core/libs/prx/libc/src/Math.cpp) (libc) - computed by the host's math library, like the other libc math functions here, not by the console's (FreeBSD msun ld80 code in Sony's `libc.prx`): results can differ from the console's in the last bit, and on Linux glibc sets `errno` to `EDOM` for `acosl` outside [-1, 1] where msun only raises the floating-point exception. `long double` is the x87 80-bit format, as Sony's `frexpl` uses it (`fldt 0x8(%rsp)`, result in `st(0)`)
 - [atanl, sinl, cosl, tanl, sinhl, coshl, tanhl, expl, exp2l, logl, log10l and powl](../../../core/libs/prx/libc/src/Math.cpp) (libc) - computed by the host's math library, as `acosl` is, not by the console's FreeBSD msun ld80 code: results can differ from the console's in the last bit (glibc's `log10l(1000)` is not exactly 3), and on Linux glibc sets `errno` to `EDOM` or `ERANGE` where msun only raises the floating-point exception. `ldexpl` and `truncl` are exact operations and give the console's result; `logb` and `remainder` give the console's value too, but on Linux glibc sets `errno` (`ERANGE` for `logb(0)`, `EDOM` for `remainder` by zero or of an infinity) where msun only raises the floating-point exception
 - [Dinkumware atomics](../../../core/libs/prx/libc/src/DinkumwareAtomics.cpp) (libc) - every operation is sequentially consistent whatever memory order it is given, which is never weaker than the console. Matching the `libc.prx` bundled with PPSA08427 (that binary run side by side on every order from 0 to 7), `_Atomic_load_8` returns 0 for a release, acq_rel or invalid order and the 8-byte exchange and fetch operations return 0 without touching memory for an order above `memory_order_seq_cst`; the generic `_Atomic_copy`, `_Atomic_exchange` and `_Atomic_compare_exchange_*` lock the byte flag passed first, as the console does
+- [_Fetch_{and,or,xor}_seq_cst_{1,2,4}](../../../core/libs/prx/libc/src/DinkumwareAtomics.cpp) (libc) follow the two-argument unsigned 8-, 16- and 32-bit helpers in Dinkumware [VC12 xatomic.h](https://github.com/ganboing/crt.vc12/blob/f1d47126bf13d1c9dccaa6c53b72217fd0474968/src/xatomic.h), using sequentially consistent atomic read-modify-write operations and returning the original value. The firmware 3.20 export inventory supplies the names and NIDs; the reference defines inline namespace helpers, so the equivalence to the console exports and their calling convention are inferred, not console-measured. A live, suitably aligned object is required as in the reference; invalid-pointer and misalignment fault behavior on the console is unverified. Windows tests cover raw and patched exports, values, neighboring storage and concurrent updates; a finite ordering litmus cannot prove the memory model.
 - [std_execute_once](../../../core/libs/prx/libc/Export.cpp) (libc) - assumed to be the plain-named twin of `std::_Execute_once` (`_ZSt13_Execute_onceRSt9once_flagPFiPvS1_PS1_ES1_`, the callee of Dinkumware `call_once`): it delegates there and gets the [thread.once.callonce] semantics already verified for that symbol (one invocation per flag, concurrent callers block until it completes, and a failing or throwing invocation leaves the flag unready so the next caller runs it). The name equivalence is not confirmed on a title, and the callback convention (called with (null, arg, null), non-zero return meaning success) is the twin's, unverified for this symbol
 - [__atomic_load, __atomic_store, __atomic_exchange and __atomic_compare_exchange](../../../core/libs/prx/libc/src/AtomicBuiltins.cpp) (libc) - the generic, size-parameterised libatomic entry points are a single `ud2` in the `libc.prx` bundled with PPSA08427 (disassembly), so the console traps on any call; these throw instead
 - [Sized __atomic calls](../../../core/libs/prx/libc/src/DinkumwareAtomics.cpp) (libc) - the 36 `__atomic_load_N`, `__atomic_store_N`, `__atomic_exchange_N`, `__atomic_compare_exchange_N` and `__atomic_fetch_{add,sub,and,or,xor}_N` calls (N = 1, 2, 4, 8) are sequentially consistent whatever order they are given. The `libc.prx` bundled with PPSA08427, run side by side on random values and on every order from -1 to 7, switches on the order but always uses a locked instruction or CAS loop, except that a store with an order other than `memory_order_seq_cst` is a plain store, so these are never weaker than the console. Unlike `_Atomic_load_8` and the 8-byte `_Atomic_*` exchange and fetch calls above, the console accepts every order here, including release, acq_rel and invalid ones, and these do the same