fix(libkernel): answer a join of the calling thread before its detach state

scePthreadJoin checked whether the thread was detached before whether it
was the calling thread. So a detached thread joining itself got EINVAL
instead of EDEADLK, and so did the main thread, which scePthreadSelf
adopts as detached. FreeBSD's join checks the calling thread right after
the null check and before it looks at the detach state. The self check
now comes first; the null, detached and already-joined answers are
unchanged.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

core/libs/prx/libkernel/Pthread/src/Thread.cpp

@@ -830,9 +830,9 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread
 
 int APS5_VABI scePthreadJoin(Pthread thread, void** retval) {
     if (!thread) return SCE_KERNEL_ERROR_EINVAL;
+    if (thread == CurrentGuestState().current) return SCE_KERNEL_ERROR_EDEADLK;
     if (thread->_detached) return SCE_KERNEL_ERROR_EINVAL;
     if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_ESRCH;
-    if (thread == CurrentGuestState().current) return SCE_KERNEL_ERROR_EDEADLK;
     ThreadCancel::Check();
     {
         std::unique_lock lock(thread->_join_mtx);

core/libs/tests/GuestPthreadJoinSelf.cpp

@@ -1,12 +1,16 @@
 #include "SceTypes.hpp"
+#include <atomic>
+#include <chrono>
 #include <cstdint>
 #include <cstdlib>
+#include <thread>
 
 extern "C" {
 int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name);
 int APS5_VABI scePthreadJoin(Pthread thread, void** retval);
 Pthread APS5_VABI scePthreadSelf(void);
 int APS5_VABI pthread_join_nid_postfix(Pthread thread, void** retval);
+int APS5_VABI scePthreadDetach(Pthread thread);
 }
 
 static constexpr int SCE_OK = 0;
@@ -23,6 +27,18 @@ static void* APS5_VABI JoinSelf(void*) {
     return reinterpret_cast<void*>(std::uintptr_t{0x1234});
 }
 
+static std::atomic<int> detachedChecks{0};
+
+static void* APS5_VABI DetachThenJoinSelf(void*) {
+    void* result = reinterpret_cast<void*>(std::uintptr_t{0x66});
+    Require(scePthreadDetach(scePthreadSelf()) == SCE_OK);
+    Require(scePthreadJoin(scePthreadSelf(), &result) == SCE_KERNEL_ERROR_EDEADLK);
+    Require(pthread_join_nid_postfix(scePthreadSelf(), &result) == GUEST_EDEADLK);
+    Require(result == reinterpret_cast<void*>(std::uintptr_t{0x66}));
+    detachedChecks.store(1);
+    return nullptr;
+}
+
 int main() {
     Pthread thread = nullptr;
     void* result = nullptr;
@@ -36,4 +52,14 @@ int main() {
     Require(scePthreadCreate(&thread, &unset, JoinSelf, nullptr, "unset-attr") == SCE_OK);
     Require(scePthreadJoin(thread, &result) == SCE_OK);
     Require(result == reinterpret_cast<void*>(std::uintptr_t{0x1234}));
+
+    thread = nullptr;
+    Require(scePthreadCreate(&thread, nullptr, DetachThenJoinSelf, nullptr, "detach-join-self") == SCE_OK);
+    for (int wait = 0; wait < 5000 && detachedChecks.load() == 0; ++wait) std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    Require(detachedChecks.load() == 1);
+
+    result = reinterpret_cast<void*>(std::uintptr_t{0x77});
+    Require(scePthreadJoin(scePthreadSelf(), &result) == SCE_KERNEL_ERROR_EDEADLK);
+    Require(pthread_join_nid_postfix(scePthreadSelf(), &result) == GUEST_EDEADLK);
+    Require(result == reinterpret_cast<void*>(std::uintptr_t{0x77}));
 }

core/libs/tests/GuestThreadSelf.cpp

@@ -110,7 +110,8 @@ int main() {
     Require(pthread_getthreadid_np_nid_postfix() == static_cast<int>(mainTid));
     int local = 0;
     Require(StackContains(mainSelf, &local));
-    Require(scePthreadJoin(mainSelf, nullptr) == SCE_KERNEL_ERROR_EINVAL);
+    Require(scePthreadJoin(mainSelf, nullptr) == SCE_KERNEL_ERROR_EDEADLK);
+    Require(pthread_join_nid_postfix(mainSelf, nullptr) == GUEST_EDEADLK);
     Require(scePthreadDetach(mainSelf) == SCE_KERNEL_ERROR_EINVAL);
     Require(scePthreadJoin(nullptr, nullptr) == SCE_KERNEL_ERROR_EINVAL);
     Require(scePthreadDetach(nullptr) == SCE_KERNEL_ERROR_EINVAL);