Linux: Fixes hangs due to mutexes locked while fork happens.

When a fork occurs FEX needs to be incredibly careful as any thread
(that isn't forking) that holds a lock will vanish when the fork occurs.

At this point if the newly forked process tries to use these mutexes
then the process hangs indefinitely.

The three major mutexes that need to be held during a fork:
- Code Invalidation mutex
  - This is the highest priority and causes us to hang frequently.
  - This is highly likely to occur when one thread is loading shared
    libraries and another thread is forking.
     - Happens frequently with Wine and steam.
- VMA tracking mutex
  - This one happens when one thread is allocating memory while a fork
    occurs.
  - This closely relates to the code invalidation mutex, just happens at
    the syscall layer instead of the FEXCore layer.
  - Happens as frequently as the code invalidation mutex.
- Allocation mutex
  - This mutex is used for FEX's 64-bit Allocator, this happens when FEX
    is allocating memory on one thread and a fork occurs.
  - Fairly infrequent because jemalloc doesn't allocate VMA regions that
    often.

While this likely doesn't hit all of the FEX mutexes, this hits the ones
that are burning fires and are happening frequently.

- FEXCore: Adds forkable mutex/locks

Necessary since we have a few locations in FEX that need to be locked
before and after a fork.

When a fork occurs the locks must be locked prior to the fork. Then
afterwards they either need to unlock or be set to default
initialization state.
- Parent
   - Does an unlock
- Child
   - Sets the lock to default initialization state
   - This is because it pthreads does TID based ownership checking on
     unique locks and refcount based waiting for shared locks.
   - No way to "unlock" after fork in this case other than default
     initializing.

External/FEXCore/Source/Interface/Context/Context.h

@@ -152,7 +152,9 @@ namespace FEXCore::Context {
        * @param Thread The internal FEX thread state object
        */
       void DestroyThread(FEXCore::Core::InternalThreadState *Thread) override;
-      void CleanupAfterFork(FEXCore::Core::InternalThreadState *Thread) override;
+
+      void LockBeforeFork(FEXCore::Core::InternalThreadState *Thread) override;
+      void UnlockAfterFork(FEXCore::Core::InternalThreadState *Thread, bool Child) override;
       void SetSignalDelegator(FEXCore::SignalDelegator *SignalDelegation) override;
       void SetSyscallHandler(FEXCore::HLE::SyscallHandler *Handler) override;
 
@@ -252,7 +254,7 @@ namespace FEXCore::Context {
     Event PauseWait;
     bool Running{};
 
-    std::shared_mutex CodeInvalidationMutex;
+    FEXCore::ForkableSharedMutex CodeInvalidationMutex;
 
     FEXCore::CPUIDEmu CPUID;
     FEXCore::HLE::SyscallHandler *SyscallHandler{};
@@ -289,7 +291,7 @@ namespace FEXCore::Context {
     template<auto Fn>
     static uint64_t ThreadExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame, uint64_t *record) {
       auto Thread = Frame->Thread;
-      ScopedDeferredSignalWithSharedLock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
+      ScopedDeferredSignalWithForkableSharedLock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
 
       return Fn(Frame, record);
     }
@@ -300,8 +302,7 @@ namespace FEXCore::Context {
       auto Thread = Frame->Thread;
 
       LogMan::Throw::AFmt(Thread->ThreadManager.GetTID() == FHU::Syscalls::gettid(), "Must be called from owning thread {}, not {}", Thread->ThreadManager.GetTID(), FHU::Syscalls::gettid());
-
-      ScopedDeferredSignalWithUniqueLock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
+      ScopedDeferredSignalWithForkableUniqueLock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
 
       ThreadRemoveCodeEntry(Thread, GuestRIP);
     }

External/FEXCore/Source/Interface/Core/Core.cpp

@@ -8,6 +8,7 @@ $end_info$
 */
 
 #include <cstdint>
+#include "FEXCore/Utils/DeferredSignalMutex.h"
 #include "Interface/Context/Context.h"
 #include "Interface/Core/LookupCache.h"
 #include "Interface/Core/Core.h"
@@ -24,6 +25,7 @@ $end_info$
 #include "Interface/IR/Passes/RegisterAllocationPass.h"
 #include "Interface/IR/Passes.h"
 #include "Interface/IR/PassManager.h"
+#include "Utils/Allocator.h"
 #include "Utils/Allocator/HostAllocator.h"
 
 #include <FEXCore/Config/Config.h>
@@ -663,7 +665,17 @@ namespace FEXCore::Context {
     delete Thread;
   }
 
-  void ContextImpl::CleanupAfterFork(FEXCore::Core::InternalThreadState *LiveThread) {
+  void ContextImpl::UnlockAfterFork(FEXCore::Core::InternalThreadState *LiveThread, bool Child) {
+    Allocator::UnlockAfterFork(LiveThread, Child);
+
+    if (Child) {
+      CodeInvalidationMutex.StealAndDropActiveLocks();
+    }
+    else {
+      CodeInvalidationMutex.unlock();
+      return;
+    }
+
     // This function is called after fork
     // We need to cleanup some of the thread data that is dead
     for (auto &DeadThread : Threads) {
@@ -702,6 +714,11 @@ namespace FEXCore::Context {
     FEXCore::Threads::Thread::CleanupAfterFork();
   }
 
+  void ContextImpl::LockBeforeFork(FEXCore::Core::InternalThreadState *Thread) {
+    CodeInvalidationMutex.lock();
+    Allocator::LockBeforeFork(Thread);
+  }
+
   void ContextImpl::AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr) {
     Thread->LookupCache->AddBlockMapping(Address, Ptr);
   }
@@ -1048,7 +1065,7 @@ namespace FEXCore::Context {
     auto Thread = Frame->Thread;
 
     // Invalidate might take a unique lock on this, to guarantee that during invalidation no code gets compiled
-    std::shared_lock lk(CodeInvalidationMutex);
+    ScopedDeferredSignalWithForkableSharedLock lk(CodeInvalidationMutex, Thread);
 
     // Is the code in the cache?
     // The backends only check L1 and L2, not L3
@@ -1228,7 +1245,7 @@ namespace FEXCore::Context {
     // Potential deferred since Thread might not be valid.
     // Thread object isn't valid very early in frontend's initialization.
     // To be more optimal the frontend should provide this code with a valid Thread object earlier.
-    ScopedPotentialDeferredSignalWithUniqueLock CodeInvalidationLock(CodeInvalidationMutex, Thread);
+    ScopedPotentialDeferredSignalWithForkableUniqueLock lk(CodeInvalidationMutex, Thread);
 
     InvalidateGuestCodeRangeInternal(this, Start, Length);
   }
@@ -1237,7 +1254,7 @@ namespace FEXCore::Context {
     // Potential deferred since Thread might not be valid.
     // Thread object isn't valid very early in frontend's initialization.
     // To be more optimal the frontend should provide this code with a valid Thread object earlier.
-    ScopedPotentialDeferredSignalWithUniqueLock CodeInvalidationLock(CodeInvalidationMutex, Thread);
+    ScopedPotentialDeferredSignalWithForkableUniqueLock lk(CodeInvalidationMutex, Thread);
 
     InvalidateGuestCodeRangeInternal(this, Start, Length);
     CallAfter(Start, Length);
@@ -1265,7 +1282,7 @@ namespace FEXCore::Context {
   }
 
   void ContextImpl::ThreadAddBlockLink(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestDestination, uintptr_t HostLink, const std::function<void()> &delinker) {
-    std::shared_lock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex);
+    ScopedDeferredSignalWithForkableSharedLock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
 
     Thread->LookupCache->AddBlockLink(GuestDestination, HostLink, delinker);
   }

External/FEXCore/Source/Utils/Allocator.cpp

@@ -340,5 +340,17 @@ namespace FEXCore::Allocator {
       ::munmap(Region.Ptr, Region.Size);
     }
   }
+
+  void LockBeforeFork(FEXCore::Core::InternalThreadState *Thread) {
+    if (Alloc64) {
+      Alloc64->LockBeforeFork(Thread);
+    }
+  }
+
+  void UnlockAfterFork(FEXCore::Core::InternalThreadState *Thread, bool Child) {
+    if (Alloc64) {
+      Alloc64->UnlockAfterFork(Thread, Child);
+    }
+  }
 }
 #endif

External/FEXCore/Source/Utils/Allocator.h

@@ -0,0 +1,10 @@
+#pragma once
+
+namespace FEXCore::Core {
+struct InternalThreadState;
+}
+
+namespace FEXCore::Allocator {
+  void LockBeforeFork(FEXCore::Core::InternalThreadState *Thread);
+  void UnlockAfterFork(FEXCore::Core::InternalThreadState *Thread, bool Child);
+}

External/FEXCore/Source/Utils/Allocator/64BitAllocator.cpp

@@ -49,6 +49,19 @@ namespace Alloc::OSAllocator {
       void *Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) override;
       int Munmap(void *addr, size_t length) override;
 
+      void LockBeforeFork(FEXCore::Core::InternalThreadState *Thread) override {
+        AllocationMutex.lock();
+      }
+
+      void UnlockAfterFork(FEXCore::Core::InternalThreadState *Thread, bool Child) override {
+        if (Child) {
+          AllocationMutex.StealAndDropActiveLocks();
+        }
+        else {
+          AllocationMutex.unlock();
+        }
+      }
+
     private:
       // Upper bound is the maximum virtual address space of the host processor
       uintptr_t UPPER_BOUND = (1ULL << 57);
@@ -139,7 +152,7 @@ namespace Alloc::OSAllocator {
       LiveRegionListType *LiveRegions{};
 
       Alloc::ForwardOnlyIntrusiveArenaAllocator *ObjectAlloc{};
-      std::mutex AllocationMutex{};
+      FEXCore::ForkableUniqueMutex AllocationMutex;
       void DetermineVASize();
 
       LiveVMARegion *MakeRegionActive(ReservedRegionListType::iterator ReservedIterator, uint64_t UsedSize) {
@@ -258,7 +271,7 @@ void *OSAllocator_64Bit::Mmap(void *addr, size_t length, int prot, int flags, in
   size_t NumberOfPages = length / FHU::FEX_PAGE_SIZE;
 
   // This needs a mutex to be thread safe
-  FEXCore::ScopedPotentialDeferredSignalWithMutex lk(AllocationMutex, TLSThread);
+  FEXCore::ScopedPotentialDeferredSignalWithForkableMutex lk(AllocationMutex, TLSThread);
 
   uint64_t AllocatedOffset{};
   LiveVMARegion *LiveRegion{};
@@ -446,7 +459,7 @@ int OSAllocator_64Bit::Munmap(void *addr, size_t length) {
   }
 
   // This needs a mutex to be thread safe
-  FEXCore::ScopedPotentialDeferredSignalWithMutex lk(AllocationMutex, TLSThread);
+  FEXCore::ScopedPotentialDeferredSignalWithForkableMutex lk(AllocationMutex, TLSThread);
 
   length = FEXCore::AlignUp(length, FHU::FEX_PAGE_SIZE);
 
@@ -571,7 +584,7 @@ OSAllocator_64Bit::OSAllocator_64Bit() {
 
 OSAllocator_64Bit::~OSAllocator_64Bit() {
   // This needs a mutex to be thread safe
-  FEXCore::ScopedPotentialDeferredSignalWithMutex lk(AllocationMutex, TLSThread);
+  FEXCore::ScopedPotentialDeferredSignalWithForkableMutex lk(AllocationMutex, TLSThread);
 
   // Walk the pages and deallocate
   // First walk the live regions

External/FEXCore/Source/Utils/Allocator/HostAllocator.h

@@ -22,6 +22,9 @@ namespace Alloc {
 
       virtual void *Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) { return nullptr; }
       virtual int Munmap(void *addr, size_t length) { return -1; }
+
+      virtual void LockBeforeFork(FEXCore::Core::InternalThreadState *Thread) {}
+      virtual void UnlockAfterFork(FEXCore::Core::InternalThreadState *Thread, bool Child) {}
   };
 
   class GlobalAllocator {

External/FEXCore/include/FEXCore/Core/Context.h

@@ -243,7 +243,8 @@ namespace FEXCore::Context {
       FEX_DEFAULT_VISIBILITY virtual void RunThread(FEXCore::Core::InternalThreadState *Thread) = 0;
       FEX_DEFAULT_VISIBILITY virtual void StopThread(FEXCore::Core::InternalThreadState *Thread) = 0;
       FEX_DEFAULT_VISIBILITY virtual void DestroyThread(FEXCore::Core::InternalThreadState *Thread) = 0;
-      FEX_DEFAULT_VISIBILITY virtual void CleanupAfterFork(FEXCore::Core::InternalThreadState *Thread) = 0;
+      FEX_DEFAULT_VISIBILITY virtual void LockBeforeFork(FEXCore::Core::InternalThreadState *Thread) {}
+      FEX_DEFAULT_VISIBILITY virtual void UnlockAfterFork(FEXCore::Core::InternalThreadState *Thread, bool Child) {}
       FEX_DEFAULT_VISIBILITY virtual void SetSignalDelegator(FEXCore::SignalDelegator *SignalDelegation) = 0;
       FEX_DEFAULT_VISIBILITY virtual void SetSyscallHandler(FEXCore::HLE::SyscallHandler *Handler) = 0;
 

External/FEXCore/include/FEXCore/Utils/DeferredSignalMutex.h

@@ -11,6 +11,91 @@
 #include <unistd.h>
 
 namespace FEXCore {
+#ifndef _WIN32
+  // Replacement for std::mutexes to deal with unlocking issues in the face of Linux fork() semantics.
+  //
+  // A fork() only clones the parent's calling thread. Other threads are silently dropped, which permanently leaves any mutexes owned by them locked.
+  // To address this issue, ForkableUniqueMutex and ForkableSharedMutex provide a way to forcefully remove any dangling locks and reset the mutexes to their default state.
+  class ForkableUniqueMutex final {
+    public:
+      ForkableUniqueMutex()
+        : Mutex (PTHREAD_MUTEX_INITIALIZER) {
+      }
+
+      // Move-only type
+      ForkableUniqueMutex(const ForkableUniqueMutex&) = delete;
+      ForkableUniqueMutex& operator=(const ForkableUniqueMutex&) = delete;
+      ForkableUniqueMutex(ForkableUniqueMutex &&rhs) = default;
+      ForkableUniqueMutex& operator=(ForkableUniqueMutex &&) = default;
+
+      void lock() {
+        [[maybe_unused]] const auto Result = pthread_mutex_lock(&Mutex);
+        LOGMAN_THROW_A_FMT(Result == 0, "{} failed to lock with {}", __func__, Result);
+      }
+      void unlock() {
+        [[maybe_unused]] const auto Result = pthread_mutex_unlock(&Mutex);
+        LOGMAN_THROW_A_FMT(Result == 0, "{} failed to unlock with {}", __func__, Result);
+      }
+      // Initialize the internal pthread object to its default initializer state.
+      // Should only ever be used in the child process when a Linux fork() has occured.
+      void StealAndDropActiveLocks() {
+        Mutex = PTHREAD_MUTEX_INITIALIZER;
+      }
+    private:
+      pthread_mutex_t Mutex;
+  };
+
+  class ForkableSharedMutex final {
+    public:
+      ForkableSharedMutex()
+        : Mutex (PTHREAD_RWLOCK_INITIALIZER) {
+      }
+
+      // Move-only type
+      ForkableSharedMutex(const ForkableSharedMutex&) = delete;
+      ForkableSharedMutex& operator=(const ForkableSharedMutex&) = delete;
+      ForkableSharedMutex(ForkableSharedMutex &&rhs) = default;
+      ForkableSharedMutex& operator=(ForkableSharedMutex &&) = default;
+
+      void lock() {
+        [[maybe_unused]] const auto Result = pthread_rwlock_wrlock(&Mutex);
+        LOGMAN_THROW_A_FMT(Result == 0, "{} failed to lock with {}", __func__, Result);
+      }
+      void unlock() {
+        [[maybe_unused]] const auto Result = pthread_rwlock_unlock(&Mutex);
+        LOGMAN_THROW_A_FMT(Result == 0, "{} failed to unlock with {}", __func__, Result);
+      }
+      void lock_shared() {
+        [[maybe_unused]] const auto Result = pthread_rwlock_rdlock(&Mutex);
+        LOGMAN_THROW_A_FMT(Result == 0, "{} failed to lock with {}", __func__, Result);
+      }
+
+      void unlock_shared() {
+        unlock();
+      }
+
+      bool try_lock() {
+        const auto Result = pthread_rwlock_trywrlock(&Mutex);
+        return Result == 0;
+      }
+
+      bool try_lock_shared() {
+        const auto Result = pthread_rwlock_tryrdlock(&Mutex);
+        return Result == 0;
+      }
+      // Initialize the internal pthread object to its default initializer state.
+      // Should only ever be used in the child process when a Linux fork() has occured.
+      void StealAndDropActiveLocks() {
+        Mutex = PTHREAD_RWLOCK_INITIALIZER;
+      }
+    private:
+      pthread_rwlock_t Mutex;
+  };
+#else
+  // Windows doesn't support forking, so these can be standard mutexes.
+  using ForkableUniqueMutex = std::mutex;
+  using ForkableSharedMutex = std::shared_mutex;
+#endif
 
   template<typename MutexType, void (MutexType::*lock_fn)(), void (MutexType::*unlock_fn)()>
   class ScopedDeferredSignalWithMutexBase final {
@@ -66,6 +151,20 @@ namespace FEXCore {
   using ScopedDeferredSignalWithSharedLock = ScopedDeferredSignalWithMutexBase<std::shared_mutex, &std::shared_mutex::lock_shared, &std::shared_mutex::unlock_shared>;
   using ScopedDeferredSignalWithUniqueLock = ScopedDeferredSignalWithMutexBase<std::shared_mutex, &std::shared_mutex::lock, &std::shared_mutex::unlock>;
 
+  // Forkable variant
+  using ScopedDeferredSignalWithForkableMutex      = ScopedDeferredSignalWithMutexBase<
+    FEXCore::ForkableUniqueMutex,
+    &FEXCore::ForkableUniqueMutex::lock,
+    &FEXCore::ForkableUniqueMutex::unlock>;
+  using ScopedDeferredSignalWithForkableSharedLock = ScopedDeferredSignalWithMutexBase<
+    FEXCore::ForkableSharedMutex,
+    &FEXCore::ForkableSharedMutex::lock_shared,
+    &FEXCore::ForkableSharedMutex::unlock_shared>;
+  using ScopedDeferredSignalWithForkableUniqueLock = ScopedDeferredSignalWithMutexBase<
+    FEXCore::ForkableSharedMutex,
+    &FEXCore::ForkableSharedMutex::lock,
+    &FEXCore::ForkableSharedMutex::unlock>;
+
   template<typename MutexType, void (MutexType::*lock_fn)(), void (MutexType::*unlock_fn)()>
   class ScopedPotentialDeferredSignalWithMutexBase final {
     public:
@@ -131,4 +230,18 @@ namespace FEXCore {
   using ScopedPotentialDeferredSignalWithMutex      = ScopedPotentialDeferredSignalWithMutexBase<std::mutex, &std::mutex::lock, &std::mutex::unlock>;
   using ScopedPotentialDeferredSignalWithSharedLock = ScopedPotentialDeferredSignalWithMutexBase<std::shared_mutex, &std::shared_mutex::lock_shared, &std::shared_mutex::unlock_shared>;
   using ScopedPotentialDeferredSignalWithUniqueLock = ScopedPotentialDeferredSignalWithMutexBase<std::shared_mutex, &std::shared_mutex::lock, &std::shared_mutex::unlock>;
+
+  // Forkable variant
+  using ScopedPotentialDeferredSignalWithForkableMutex      = ScopedPotentialDeferredSignalWithMutexBase<
+    FEXCore::ForkableUniqueMutex,
+    &FEXCore::ForkableUniqueMutex::lock,
+    &FEXCore::ForkableUniqueMutex::unlock>;
+  using ScopedPotentialDeferredSignalWithForkableSharedLock = ScopedPotentialDeferredSignalWithMutexBase<
+    FEXCore::ForkableSharedMutex,
+    &FEXCore::ForkableSharedMutex::lock_shared,
+    &FEXCore::ForkableSharedMutex::unlock_shared>;
+  using ScopedPotentialDeferredSignalWithForkableUniqueLock = ScopedPotentialDeferredSignalWithMutexBase<
+    FEXCore::ForkableSharedMutex,
+    &FEXCore::ForkableSharedMutex::lock,
+    &FEXCore::ForkableSharedMutex::unlock>;
 }

FEXHeaderUtils/FEXHeaderUtils/ScopedSignalMask.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <FEXCore/Utils/CompilerDefs.h>
+#include <FEXCore/Utils/DeferredSignalMutex.h>
 
 #include <atomic>
 #include <cstdint>
@@ -106,4 +106,17 @@ namespace FHU {
   using ScopedSignalMaskWithMutex = ScopedSignalMaskWithMutexBase<std::mutex, &std::mutex::lock, &std::mutex::unlock>;
   using ScopedSignalMaskWithSharedLock = ScopedSignalMaskWithMutexBase<std::shared_mutex, &std::shared_mutex::lock_shared, &std::shared_mutex::unlock_shared>;
   using ScopedSignalMaskWithUniqueLock = ScopedSignalMaskWithMutexBase<std::shared_mutex, &std::shared_mutex::lock, &std::shared_mutex::unlock>;
+
+  using ScopedSignalMaskWithForkableMutex = ScopedSignalMaskWithMutexBase<
+    FEXCore::ForkableUniqueMutex,
+    &FEXCore::ForkableUniqueMutex::lock,
+    &FEXCore::ForkableUniqueMutex::unlock>;
+  using ScopedSignalMaskWithForkableSharedLock = ScopedSignalMaskWithMutexBase<
+    FEXCore::ForkableSharedMutex,
+    &FEXCore::ForkableSharedMutex::lock_shared,
+    &FEXCore::ForkableSharedMutex::unlock_shared>;
+  using ScopedSignalMaskWithForkableUniqueLock = ScopedSignalMaskWithMutexBase<
+    FEXCore::ForkableSharedMutex,
+    &FEXCore::ForkableSharedMutex::lock,
+    &FEXCore::ForkableSharedMutex::unlock>;
 }

Source/Tools/FEXLoader/LinuxSyscalls/Syscalls.cpp

@@ -580,12 +580,33 @@ uint64_t CloneHandler(FEXCore::Core::CpuStateFrame *Frame, FEX::HLE::clone3_args
     if (!AnyFlagsSet(flags, CLONE_THREAD)) {
       // Has an unsupported flag
       // Fall to a handler that can handle this case
+      auto Thread = Frame->Thread;
+
+      args->SignalMask = ~0ULL;
+      ::syscall(SYS_rt_sigprocmask, SIG_SETMASK, &args->SignalMask, &args->SignalMask, sizeof(args->SignalMask));
+      Thread->CTX->LockBeforeFork(Frame->Thread);
+
+      FEX::HLE::_SyscallHandler->LockBeforeFork();
+
+      uint64_t Result{};
       if (args->Type == TYPE_CLONE2) {
-        return Clone2Handler(Frame, args);
+        Result = Clone2Handler(Frame, args);
       }
       else {
-        return Clone3Handler(Frame, args);
+        Result = Clone3Handler(Frame, args);
       }
+
+      if (Result != 0) {
+        // Parent
+        // Unlock the mutexes on both sides of the fork
+        FEX::HLE::_SyscallHandler->UnlockAfterFork(false);
+
+        // Clear all the other threads that are being tracked
+        Thread->CTX->UnlockAfterFork(Frame->Thread, false);
+
+        ::syscall(SYS_rt_sigprocmask, SIG_SETMASK, &args->SignalMask, nullptr, sizeof(args->SignalMask));
+      }
+      return Result;
     }
     else {
       LogMan::Msg::IFmt("Unsupported flag with CLONE_THREAD. This breaks TLS, falling down classic thread path");
@@ -634,6 +655,7 @@ uint64_t CloneHandler(FEXCore::Core::CpuStateFrame *Frame, FEX::HLE::clone3_args
       // Normally a thread cleans itself up on exit. But because we need to join, we are now responsible
       Thread->CTX->DestroyThread(NewThread);
     }
+
     SYSCALL_ERRNO();
   }
 };
@@ -808,17 +830,16 @@ uint64_t UnimplementedSyscallSafe(FEXCore::Core::CpuStateFrame *Frame, uint64_t
 }
 
 void SyscallHandler::LockBeforeFork() {
-  // XXX shared_mutex has issues with locking and forks
-  // VMATracking.Mutex.lock();
-
-  // Add other mutexes here
+  VMATracking.Mutex.lock();
 }
 
-void SyscallHandler::UnlockAfterFork() {
-  // Add other mutexes here
-
-  // XXX shared_mutex has issues with locking and forks
-  // VMATracking.Mutex.unlock();
+void SyscallHandler::UnlockAfterFork(bool Child) {
+  if (Child) {
+    VMATracking.Mutex.StealAndDropActiveLocks();
+  }
+  else {
+    VMATracking.Mutex.unlock();
+  }
 }
 
 static bool isHEX(char c) {

Source/Tools/FEXLoader/LinuxSyscalls/Syscalls.h

@@ -15,6 +15,7 @@ $end_info$
 #include <FEXCore/HLE/SourcecodeResolver.h>
 #include <FEXCore/IR/IR.h>
 #include <FEXCore/Utils/CompilerDefs.h>
+#include <FEXCore/Utils/DeferredSignalMutex.h>
 #include <FEXCore/fextl/fmt.h>
 #include <FEXCore/fextl/map.h>
 #include <FEXCore/fextl/memory.h>
@@ -220,7 +221,7 @@ public:
 
   ///// FORK tracking /////
   void LockBeforeFork();
-  void UnlockAfterFork();
+  void UnlockAfterFork(bool Child);
 
   SourcecodeResolver *GetSourcecodeResolver() override { return this; }
 
@@ -327,7 +328,7 @@ private:
   struct VMATracking {
     using VMAEntry = SyscallHandler::VMAEntry;
     // Held while reading/writing this struct
-    std::shared_mutex Mutex;
+    FEXCore::ForkableSharedMutex Mutex;
 
     // Memory ranges indexed by page aligned starting address
     fextl::map<uint64_t, VMAEntry> VMAs;
@@ -430,6 +431,7 @@ enum TypeOfClone {
 
 struct clone3_args {
   TypeOfClone Type;
+  uint64_t SignalMask;
   kernel_clone3_args args;
 };
 

Source/Tools/FEXLoader/LinuxSyscalls/Syscalls/Thread.cpp

@@ -140,7 +140,13 @@ namespace FEX::HLE {
       // If we don't have CLONE_THREAD then we are effectively a fork
       // Clear all the other threads that are being tracked
       // Frame->Thread is /ONLY/ safe to access when CLONE_THREAD flag is not set
-      CTX->CleanupAfterFork(Frame->Thread);
+      // Unlock the mutexes on both sides of the fork
+      FEX::HLE::_SyscallHandler->UnlockAfterFork(true);
+
+      // Clear all the other threads that are being tracked
+      Thread->CTX->UnlockAfterFork(Frame->Thread, true);
+
+      ::syscall(SYS_rt_sigprocmask, SIG_SETMASK, &CloneArgs->SignalMask, nullptr, sizeof(CloneArgs->SignalMask));
 
       Thread->CurrentFrame->State.gregs[FEXCore::X86State::REG_RAX] = 0;
       Thread->CurrentFrame->State.gregs[FEXCore::X86State::REG_RBX] = 0;
@@ -187,6 +193,11 @@ namespace FEX::HLE {
 
   uint64_t ForkGuest(FEXCore::Core::InternalThreadState *Thread, FEXCore::Core::CpuStateFrame *Frame, uint32_t flags, void *stack, size_t StackSize, pid_t *parent_tid, pid_t *child_tid, void *tls) {
     // Just before we fork, we lock all syscall mutexes so that both processes will end up with a locked mutex
+
+    uint64_t Mask{~0ULL};
+    ::syscall(SYS_rt_sigprocmask, SIG_SETMASK, &Mask, &Mask, sizeof(Mask));
+    Thread->CTX->LockBeforeFork(Frame->Thread);
+
     FEX::HLE::_SyscallHandler->LockBeforeFork();
 
     const bool IsVFork = flags & CLONE_VFORK;
@@ -215,11 +226,17 @@ namespace FEX::HLE {
     else {
       Result = fork();
     }
+    const bool IsChild = Result == 0;
 
-    // Unlock the mutexes on both sides of the fork
-    FEX::HLE::_SyscallHandler->UnlockAfterFork();
+    if (IsChild) {
+      // Unlock the mutexes on both sides of the fork
+      FEX::HLE::_SyscallHandler->UnlockAfterFork(IsChild);
+
+      // Clear all the other threads that are being tracked
+      Thread->CTX->UnlockAfterFork(Frame->Thread, IsChild);
+
+      ::syscall(SYS_rt_sigprocmask, SIG_SETMASK, &Mask, nullptr, sizeof(Mask));
 
-    if (Result == 0) {
       // Child
       // update the internal TID
       Thread->ThreadManager.TID = FHU::Syscalls::gettid();
@@ -227,9 +244,6 @@ namespace FEX::HLE {
       FEX::HLE::_SyscallHandler->FM.UpdatePID(Thread->ThreadManager.PID);
       Thread->ThreadManager.clear_child_tid = nullptr;
 
-      // Clear all the other threads that are being tracked
-      Thread->CTX->CleanupAfterFork(Frame->Thread);
-
       // only a  single thread running so no need to remove anything from the thread array
 
       // Handle child setup now
@@ -275,6 +289,14 @@ namespace FEX::HLE {
         }
       }
 
+      // Unlock the mutexes on both sides of the fork
+      FEX::HLE::_SyscallHandler->UnlockAfterFork(IsChild);
+
+      // Clear all the other threads that are being tracked
+      Thread->CTX->UnlockAfterFork(Frame->Thread, IsChild);
+
+      ::syscall(SYS_rt_sigprocmask, SIG_SETMASK, &Mask, nullptr, sizeof(Mask));
+
       // VFork needs the parent to wait for the child to exit.
       if (IsVFork) {
         // Wait for the read end of the pipe to close.

Source/Tools/FEXLoader/LinuxSyscalls/SyscallsSMCTracking.cpp

@@ -54,7 +54,7 @@ bool SyscallHandler::HandleSegfault(FEXCore::Core::InternalThreadState *Thread,
 
   {
     // Can't use the deferred signal lock in the SIGSEGV handler.
-    FHU::ScopedSignalMaskWithSharedLock lk(_SyscallHandler->VMATracking.Mutex);
+    FHU::ScopedSignalMaskWithForkableSharedLock lk(_SyscallHandler->VMATracking.Mutex);
 
     auto VMATracking = &_SyscallHandler->VMATracking;
 
@@ -111,7 +111,7 @@ void SyscallHandler::MarkGuestExecutableRange(FEXCore::Core::InternalThreadState
       return;
     }
 
-    FEXCore::ScopedDeferredSignalWithSharedLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedDeferredSignalWithForkableSharedLock lk(VMATracking.Mutex, Thread);
 
     // Find the first mapping at or after the range ends, or ::end().
     // Top points to the address after the end of the range
@@ -166,7 +166,7 @@ void SyscallHandler::MarkGuestExecutableRange(FEXCore::Core::InternalThreadState
 
 // Used for AOT
 FEXCore::HLE::AOTIRCacheEntryLookupResult SyscallHandler::LookupAOTIRCacheEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestAddr) {
-  FEXCore::ScopedDeferredSignalWithSharedLock lk(VMATracking.Mutex, Thread);
+  FEXCore::ScopedDeferredSignalWithForkableSharedLock lk(VMATracking.Mutex, Thread);
 
   // Get the first mapping after GuestAddr, or end
   // GuestAddr is inclusive
@@ -194,7 +194,7 @@ void SyscallHandler::TrackMmap(FEXCore::Core::InternalThreadState *Thread, uintp
     // NOTE: Frontend calls this with a nullptr Thread during initialization, but
     //       providing this code with a valid Thread object earlier would allow
     //       us to be more optimal by using ScopedDeferredSignalWithUniqueLock instead
-    FEXCore::ScopedPotentialDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedPotentialDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
 
     static uint64_t AnonSharedId = 1;
 
@@ -233,6 +233,7 @@ void SyscallHandler::TrackMmap(FEXCore::Core::InternalThreadState *Thread, uintp
   }
 
   if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) {
+    // VMATracking.Mutex can't be held while executing this, otherwise it hangs if the JIT is in the process of looking up code in the AOT JIT.
     CTX->InvalidateGuestCodeRange(Thread, (uintptr_t)Base, Size);
   }
 }
@@ -244,7 +245,7 @@ void SyscallHandler::TrackMunmap(FEXCore::Core::InternalThreadState *Thread, uin
     // Frontend calls this with nullptr Thread during initialization.
     // This is why `ScopedPotentialDeferredSignalWithUniqueLock` is used here.
     // To be more optimal the frontend should provide this code with a valid Thread object earlier.
-    FEXCore::ScopedPotentialDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedPotentialDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
 
     VMATracking.ClearUnsafe(CTX, Base, Size);
   }
@@ -258,7 +259,7 @@ void SyscallHandler::TrackMprotect(FEXCore::Core::InternalThreadState *Thread, u
   Size = FEXCore::AlignUp(Size, FHU::FEX_PAGE_SIZE);
 
   {
-    FEXCore::ScopedDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
 
     VMATracking.ChangeUnsafe(Base, Size, VMAProt::fromProt(Prot));
   }
@@ -273,7 +274,7 @@ void SyscallHandler::TrackMremap(FEXCore::Core::InternalThreadState *Thread, uin
   NewSize = FEXCore::AlignUp(NewSize, FHU::FEX_PAGE_SIZE);
 
   {
-    FEXCore::ScopedDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
 
     const auto OldVMA = VMATracking.LookupVMAUnsafe(OldAddress);
 
@@ -331,7 +332,7 @@ void SyscallHandler::TrackShmat(FEXCore::Core::InternalThreadState *Thread, int
   uint64_t Length = stat.shm_segsz;
 
   {
-    FEXCore::ScopedDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
 
     // TODO
     MRID mrid{SpecialDev::SHM, static_cast<uint64_t>(shmid)};
@@ -353,7 +354,7 @@ void SyscallHandler::TrackShmat(FEXCore::Core::InternalThreadState *Thread, int
 void SyscallHandler::TrackShmdt(FEXCore::Core::InternalThreadState *Thread, uintptr_t Base) {
   uintptr_t Length = 0;
   {
-    FEXCore::ScopedDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
 
     Length = VMATracking.ClearShmUnsafe(CTX, Base);
   }
@@ -367,8 +368,7 @@ void SyscallHandler::TrackShmdt(FEXCore::Core::InternalThreadState *Thread, uint
 void SyscallHandler::TrackMadvise(FEXCore::Core::InternalThreadState *Thread, uintptr_t Base, uintptr_t Size, int advice) {
   Size = FEXCore::AlignUp(Size, FHU::FEX_PAGE_SIZE);
   {
-    FEXCore::ScopedDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
-
+    FEXCore::ScopedDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
     // TODO
   }
 }