Merge pull request #11340 from harisibrar1/feat/libressl-ex-data-indices

feat(libSceLibreSsl): implement the SSL and SSL_CTX ex_data index registry as LibreSSL 2.5.4 does

core/libs/prx/libSceLibreSsl/CMakeLists.txt

@@ -2,7 +2,7 @@ add_library(libSceLibreSsl SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/E
 target_include_directories(libSceLibreSsl PRIVATE ${LIBS_INCLUDE_DIR})
 target_link_libraries(libSceLibreSsl PRIVATE mbedcrypto)
 configure_windows_unwind(libSceLibreSsl)
-target_sources(libSceLibreSsl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/Asn1.cpp)
+target_sources(libSceLibreSsl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/Asn1.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/ExData.cpp)
 
 if(BUILD_TESTING)
     add_test_executable(guest_libressl_x509_errors_tests ${LIBS_INCLUDE_DIR}/tests/GuestLibreSslX509Errors.cpp)
@@ -29,8 +29,14 @@ if(BUILD_TESTING)
     add_test(NAME guest_libressl_asn1 COMMAND guest_libressl_asn1_tests)
     add_test(NAME guest_libressl_asn1_allocator COMMAND guest_libressl_asn1_tests allocator)
     set_tests_properties(guest_libressl_asn1 guest_libressl_asn1_allocator PROPERTIES TIMEOUT 30)
+    add_test_executable(guest_libressl_ex_data_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/ExData.cpp)
+    target_include_directories(guest_libressl_ex_data_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_libressl_ex_data_tests PRIVATE libSceLibreSsl libc)
+    configure_windows_unwind(guest_libressl_ex_data_tests)
+    add_test(NAME guest_libressl_ex_data COMMAND guest_libressl_ex_data_tests)
+    set_tests_properties(guest_libressl_ex_data PROPERTIES TIMEOUT 30)
     if(WIN32)
-        set_tests_properties(guest_libressl_digest guest_libressl_asn1 guest_libressl_asn1_allocator PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(guest_libressl_digest guest_libressl_asn1 guest_libressl_asn1_allocator guest_libressl_ex_data PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceLibreSsl>")
     endif()
 

core/libs/prx/libSceLibreSsl/Export.cpp

@@ -223,11 +223,6 @@ void APS5_VABI CRYPTO_THREADID_set_numeric_nid_postfix(CryptoThreadId* id, std::
     id->val = value;
 }
 
-int APS5_VABI CRYPTO_cleanup_all_ex_data_nid_postfix(void) {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
-}
-
 int APS5_VABI CRYPTO_num_locks_nid_postfix(void) {
     return CryptoNumLocks;
 }
@@ -994,11 +989,6 @@ int APS5_VABI SSL_CTX_get_ex_data_nid_postfix(void) {
     return 0;
 }
 
-int APS5_VABI SSL_CTX_get_ex_new_index_nid_postfix(void) {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
-}
-
 int APS5_VABI SSL_CTX_get_verify_mode_nid_postfix(void) {
     NotImplemented_nid_no_patch(__func__);
     return 0;
@@ -1209,16 +1199,6 @@ int APS5_VABI SSL_get_ex_data_nid_postfix(void) {
     return 0;
 }
 
-int APS5_VABI SSL_get_ex_data_X509_STORE_CTX_idx_nid_postfix(void) {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
-}
-
-int APS5_VABI SSL_get_ex_new_index_nid_postfix(void) {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
-}
-
 int APS5_VABI SSL_get_fd_nid_postfix(void) {
     NotImplemented_nid_no_patch(__func__);
     return 0;

core/libs/prx/libSceLibreSsl/include/Api.hpp

@@ -9,6 +9,14 @@ struct Asn1String {
     std::int64_t flags;
 };
 
+struct LibreSslExData {
+    void* stack;
+};
+
+using LibreSslExNew = int (APS5_VABI*)(void* parent, void* pointer, LibreSslExData* data, int index, std::int64_t argl, void* argp);
+using LibreSslExFree = void (APS5_VABI*)(void* parent, void* pointer, LibreSslExData* data, int index, std::int64_t argl, void* argp);
+using LibreSslExDup = int (APS5_VABI*)(LibreSslExData* to, LibreSslExData* from, void* fromData, int index, std::int64_t argl, void* argp);
+
 inline constexpr int MaxRounds = 14;
 
 struct AesKey {
@@ -169,7 +177,7 @@ int APS5_VABI CONF_modules_free_nid_postfix(void);
 int APS5_VABI CONF_modules_load_file_nid_postfix(void);
 int APS5_VABI CRYPTO_THREADID_set_callback_nid_postfix(void* callback);
 void APS5_VABI CRYPTO_THREADID_set_numeric_nid_postfix(CryptoThreadId* id, std::uint64_t value);
-int APS5_VABI CRYPTO_cleanup_all_ex_data_nid_postfix(void);
+void APS5_VABI CRYPTO_cleanup_all_ex_data_nid_postfix(void);
 int APS5_VABI CRYPTO_num_locks_nid_postfix(void);
 void APS5_VABI CRYPTO_set_add_lock_callback_nid_postfix(void* callback);
 void APS5_VABI CRYPTO_set_locking_callback_nid_postfix(void* callback);
@@ -335,7 +343,7 @@ int APS5_VABI SSL_CTX_ctrl_nid_postfix(void);
 int APS5_VABI SSL_CTX_free_nid_postfix(void);
 int APS5_VABI SSL_CTX_get_cert_store_nid_postfix(void);
 int APS5_VABI SSL_CTX_get_ex_data_nid_postfix(void);
-int APS5_VABI SSL_CTX_get_ex_new_index_nid_postfix(void);
+int APS5_VABI SSL_CTX_get_ex_new_index_nid_postfix(std::int64_t argl, void* argp, LibreSslExNew newFunction, LibreSslExDup dupFunction, LibreSslExFree freeFunction);
 int APS5_VABI SSL_CTX_get_verify_mode_nid_postfix(void);
 int APS5_VABI SSL_CTX_load_verify_locations_nid_postfix(void);
 int APS5_VABI SSL_CTX_load_verify_mem_nid_postfix(void);
@@ -380,7 +388,7 @@ int APS5_VABI SSL_get_current_cipher_nid_postfix(void);
 int APS5_VABI SSL_get_error_nid_postfix(void);
 int APS5_VABI SSL_get_ex_data_nid_postfix(void);
 int APS5_VABI SSL_get_ex_data_X509_STORE_CTX_idx_nid_postfix(void);
-int APS5_VABI SSL_get_ex_new_index_nid_postfix(void);
+int APS5_VABI SSL_get_ex_new_index_nid_postfix(std::int64_t argl, void* argp, LibreSslExNew newFunction, LibreSslExDup dupFunction, LibreSslExFree freeFunction);
 int APS5_VABI SSL_get_fd_nid_postfix(void);
 int APS5_VABI SSL_get_peer_cert_chain_nid_postfix(void);
 int APS5_VABI SSL_get_peer_certificate_nid_postfix(void);

core/libs/prx/libSceLibreSsl/src/ExData.cpp

@@ -0,0 +1,66 @@
+#include "prx/libSceLibreSsl/include/Api.hpp"
+#include "prx/libSceLibreSsl/include/Error.hpp"
+#include <array>
+#include <cstdint>
+#include <mutex>
+#include <new>
+#include <vector>
+
+namespace {
+constexpr int CryptoLibrary = 15;
+constexpr int AnyFunction = 0xfff;
+constexpr int MallocFailure = 65;
+constexpr int SslClass = 1;
+constexpr int SslContextClass = 2;
+constexpr int StoreContextClass = 5;
+constexpr int ClassCount = 16;
+
+struct ExDataFunctions {
+    std::int64_t argl;
+    void* argp;
+    LibreSslExNew newFunction;
+    LibreSslExDup dupFunction;
+    LibreSslExFree freeFunction;
+};
+
+std::mutex classesLock;
+std::array<std::vector<ExDataFunctions>, ClassCount> classes;
+std::mutex storeContextLock;
+int storeContextIndex = -1;
+
+int NewIndex(int classIndex, const ExDataFunctions& functions) {
+    std::lock_guard lock(classesLock);
+    auto& registered = classes[classIndex];
+    try {
+        registered.push_back(functions);
+    } catch (const std::bad_alloc&) {
+        LibreSslPutError_nid_no_patch(CryptoLibrary, AnyFunction, MallocFailure);
+        return -1;
+    }
+    return static_cast<int>(registered.size() - 1);
+}
+}
+
+extern "C" {
+
+int APS5_VABI SSL_get_ex_new_index_nid_postfix(std::int64_t argl, void* argp, LibreSslExNew newFunction, LibreSslExDup dupFunction, LibreSslExFree freeFunction) {
+    return NewIndex(SslClass, {argl, argp, newFunction, dupFunction, freeFunction});
+}
+
+int APS5_VABI SSL_CTX_get_ex_new_index_nid_postfix(std::int64_t argl, void* argp, LibreSslExNew newFunction, LibreSslExDup dupFunction, LibreSslExFree freeFunction) {
+    return NewIndex(SslContextClass, {argl, argp, newFunction, dupFunction, freeFunction});
+}
+
+int APS5_VABI SSL_get_ex_data_X509_STORE_CTX_idx_nid_postfix(void) {
+    static char verifyCallback[] = "SSL for verify callback";
+    std::lock_guard lock(storeContextLock);
+    if (storeContextIndex < 0) storeContextIndex = NewIndex(StoreContextClass, {0, verifyCallback, nullptr, nullptr, nullptr});
+    return storeContextIndex;
+}
+
+void APS5_VABI CRYPTO_cleanup_all_ex_data_nid_postfix(void) {
+    std::lock_guard lock(classesLock);
+    for (auto& registered : classes) std::vector<ExDataFunctions>().swap(registered);
+}
+
+}

core/libs/prx/libSceLibreSsl/tests/ExData.cpp

@@ -0,0 +1,72 @@
+#include "prx/libSceLibreSsl/libSceLibreSsl.hpp"
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <thread>
+#include <vector>
+
+namespace {
+
+void Require(bool condition, const char* message) {
+    if (condition) return;
+    std::fprintf(stderr, "failed: %s\n", message);
+    std::abort();
+}
+
+int APS5_VABI NewCallback(void*, void*, LibreSslExData*, int, std::int64_t, void*) {
+    std::abort();
+}
+
+void APS5_VABI FreeCallback(void*, void*, LibreSslExData*, int, std::int64_t, void*) {
+    std::abort();
+}
+
+int APS5_VABI DupCallback(LibreSslExData*, LibreSslExData*, void*, int, std::int64_t, void*) {
+    std::abort();
+}
+
+int NewSslIndex() {
+    return SSL_get_ex_new_index_nid_postfix(7, nullptr, nullptr, nullptr, nullptr);
+}
+
+int NewContextIndex() {
+    int marker = 0;
+    return SSL_CTX_get_ex_new_index_nid_postfix(-1, &marker, nullptr, nullptr, nullptr);
+}
+
+}
+
+int main() {
+    Require(NewSslIndex() == 0 && NewSslIndex() == 1 && NewSslIndex() == 2, "SSL indices count from 0");
+    Require(NewContextIndex() == 0 && NewContextIndex() == 1, "SSL_CTX indices are counted apart from SSL ones");
+    Require(SSL_get_ex_data_X509_STORE_CTX_idx_nid_postfix() == 0, "the verify-callback index is the first X509_STORE_CTX index");
+    Require(SSL_get_ex_data_X509_STORE_CTX_idx_nid_postfix() == 0, "the verify-callback index is allocated once");
+    Require(NewSslIndex() == 3, "the verify-callback index does not use an SSL index");
+
+    CRYPTO_cleanup_all_ex_data_nid_postfix();
+    Require(NewSslIndex() == 0 && NewContextIndex() == 0, "cleanup releases every class");
+    Require(SSL_get_ex_data_X509_STORE_CTX_idx_nid_postfix() == 0, "cleanup keeps the cached verify-callback index");
+    CRYPTO_cleanup_all_ex_data_nid_postfix();
+    CRYPTO_cleanup_all_ex_data_nid_postfix();
+    Require(NewContextIndex() == 0, "cleanup can be repeated");
+
+    CRYPTO_cleanup_all_ex_data_nid_postfix();
+    int argument = 0;
+    Require(SSL_get_ex_new_index_nid_postfix(-5, &argument, NewCallback, DupCallback, FreeCallback) == 0, "typed SSL callbacks are accepted and stored");
+    Require(SSL_CTX_get_ex_new_index_nid_postfix(9, &argument, NewCallback, DupCallback, FreeCallback) == 0, "typed SSL_CTX callbacks are accepted and stored");
+    Require(NewSslIndex() == 1 && NewContextIndex() == 1, "indices with callbacks count like any other");
+    CRYPTO_cleanup_all_ex_data_nid_postfix();
+    std::vector<std::thread> threads;
+    std::vector<int> indices(64, -1);
+    for (int thread = 0; thread < 8; ++thread)
+        threads.emplace_back([&indices, thread] {
+            for (int round = 0; round < 8; ++round) indices[thread * 8 + round] = NewSslIndex();
+        });
+    for (auto& thread : threads) thread.join();
+    std::vector<bool> seen(indices.size(), false);
+    for (const int index : indices) {
+        Require(index >= 0 && index < static_cast<int>(indices.size()) && !seen[index], "concurrent callers get distinct indices");
+        seen[index] = true;
+    }
+    return 0;
+}

docs/dev/debt/libSceLibreSsl.md

@@ -15,6 +15,7 @@
 - [AES_set_encrypt_key, AES_set_decrypt_key, AES_encrypt, AES_decrypt, AES_ecb_encrypt, AES_cbc_encrypt, AES_cfb128_encrypt, AES_ctr128_encrypt](../../../core/libs/prx/libSceLibreSsl/src/Aes.cpp) (libSceLibreSsl) - follow LibreSSL's [aes.h](https://github.com/libressl/openbsd/blob/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/aes/aes.h) and [mode code](https://github.com/libressl/openbsd/tree/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/modes). Nothing is checked on the console. Key setup follows [LibreSSL 2.5.4's aes_core.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/aes/aes_core.c), the version the `X509_verify_cert_error_string` entry names: a null pointer returns -1 and a key length other than 128, 192 or 256 bits returns -2 before the key is written, and only the 44, 52 or 60 words the key length needs are written. The `AES_KEY` schedule uses that file's generic C layout (big-endian words, `rounds` 10, 12 or 14). The 2.5.4 amd64 build uses [aes-x86_64](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/aes/asm/aes-x86_64.pl) instead, which copies the key bytes without swapping, so each word is byte-reversed compared with the C layout (OpenSSL 3.0.13's x86-64 code gives the same schedules byte-reversed); the console's build options are unknown, so a schedule the title reads or fills is unverified. CTR leaves the last keystream block in `ecount_buf` even at a block boundary. Null pointers in the cipher and mode functions, other `rounds` values and a CFB or CTR position outside 0-15 throw
 - [ERR_get_error, ERR_peek_error, ERR_clear_error, ERR_remove_thread_state](../../../core/libs/prx/libSceLibreSsl/src/Error.cpp) (libSceLibreSsl) - no export of this library adds an error yet, so the queue stays empty for guest code. The per-thread queue of 16 slots holding 15 codes, its order and the `ERR_PACK` layout follow LibreSSL's [err.c](https://github.com/libressl/openbsd/blob/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/err/err.c) and [err.h](https://github.com/libressl/openbsd/blob/bc9483725c5815efb64b6d9acc52fc5fba139212/src/lib/libcrypto/err/err.h); the module's LibreSSL version is unknown and nothing is checked on the console. The queue keeps no file, line or data, as no export reads them. `ERR_remove_thread_state` with a thread id throws: that version asserts a null id, while older versions remove the state of the thread the id names
 - [OCSP_response_status_str, OCSP_cert_status_str and OCSP_crl_reason_str](../../../core/libs/prx/libSceLibreSsl/src/Ocsp.cpp) (libSceLibreSsl) - the module's strings are not measured on the console. The tables follow LibreSSL [ocsp_prn.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/ocsp/ocsp_prn.c), unchanged from revision 1.8 (2015) to 1.12 (2025), with codes from `ocsp.h`; the argument is a 64-bit `long` and every value not in a table returns `(UNKNOWN)`
+- [SSL_get_ex_new_index, SSL_CTX_get_ex_new_index, SSL_get_ex_data_X509_STORE_CTX_idx and CRYPTO_cleanup_all_ex_data](../../../core/libs/prx/libSceLibreSsl/src/ExData.cpp) (libSceLibreSsl) - unverified on a PS5. The stored `argl`, `argp` and callbacks are never called, because no export creates an `SSL` or `SSL_CTX` and `SSL_get_ex_data`/`SSL_set_ex_data` still throw. Index numbering follows LibreSSL 2.5.4's [ex_data.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/ex_data.c#L310-L406), [ssl_lib.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libssl/ssl_lib.c#L2828-L2854) and [ssl_cert.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libssl/ssl_cert.c#L133-L160): per class from 0, the verify-callback index allocated once and kept across `CRYPTO_cleanup_all_ex_data`, which restarts every class at 0.
 
 - [SSL_CIPHER_get_bits, SSL_CIPHER_get_name and SSL_CIPHER_get_version](../../../core/libs/prx/libSceLibreSsl/src/Cipher.cpp) - the 88-byte guest cipher layout and these calls are not console-verified. They follow [LibreSSL 2.5.4 ssl.h](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libssl/ssl.h) and [ssl_ciph.c](https://github.com/libressl/openbsd/blob/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libssl/ssl_ciph.c), matching the version in [published firmware 13.40 measurements](https://github.com/boykopovar/AnyPS5/discussions/8981#discussioncomment-19111214). Console modifications and other firmware layouts remain unverified. Protocol labels use the full 64-bit cipher ID, not the protocol flags; TLS 1.2 ciphers still report `TLSv1/SSLv3`, as in that release.
 - [sk_num, sk_value, sk_pop, sk_pop_free](../../../core/libs/prx/libSceLibreSsl/src/Stack.cpp) (libSceLibreSsl) - the 32-byte `_STACK` layout (`num`, `data`, `sorted`, `num_alloc`, `comp`) and the results follow [LibreSSL 2.5.4](https://github.com/libressl/openbsd/tree/2e2c612066aa993717528a0249c15c51128d61cb/src/lib/libcrypto/stack) `stack.h` and `stack.c`, the version identified in [published firmware 13.40 measurements](https://github.com/boykopovar/AnyPS5/discussions/8981#discussioncomment-19111214); nothing is checked on the console. `sk_pop_free` releases the element array and the stack with the guest `free`, as the module would through libc. No export of this module creates a stack yet, so only stacks the caller builds reach them. A null element array that LibreSSL would read through, and a null free function with a non-null element, throw where LibreSSL faults