[WIP] Fork syscall support

Bit buggy

External/FEXCore/Source/Interface/Context/Context.h

@@ -131,6 +131,9 @@ namespace FEXCore::Context {
     void CopyMemoryMapping(FEXCore::Core::InternalThreadState *ParentThread, FEXCore::Core::InternalThreadState *ChildThread);
     void RunThread(FEXCore::Core::InternalThreadState *Thread);
 
+    void HandleForkChildSide(uint64_t prevTID);
+    uint64_t GetPIDHack();
+
   protected:
     void ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
 

External/FEXCore/Source/Interface/Core/Core.cpp

@@ -489,7 +489,7 @@ namespace FEXCore::Context {
     Thread->BlockCache->ClearCache();
     Thread->CPUBackend->ClearCache();
     Thread->IntBackend->ClearCache();
-
+  
     if (GuestRIP != 0) {
       auto IR = Thread->IRLists.find(GuestRIP)->second.release();
       Thread->IRLists.clear();
@@ -497,6 +497,34 @@ namespace FEXCore::Context {
     }
   }
 
+  static int indent = 0;
+
+  void Context::HandleForkChildSide(uint64_t PrevTID) {
+    // Whoa okay, we have forked. Time to clean up
+
+    // Every thread that thinks it exists right now? Not real. Delete them
+    for(auto iter = Threads.begin(); iter != Threads.end(); ++iter) {
+      // Walk the thread list and delete every thread that isn't this one
+      auto Thread = *iter;
+      LogMan::Msg::D("[Child] Deleted a thread");
+      //Thread->ExecutionThread.detach();
+      //delete Thread;
+      //Threads.erase(iter);
+    }
+    indent = 1;
+  }
+
+  uint64_t Context::GetPIDHack() {
+    std::lock_guard<std::mutex> lk(ThreadCreationMutex);
+    for (auto &Thread : Threads) {
+      if (Thread->State.ThreadManager.parent_tid == 0) {
+        return Thread->State.ThreadManager.GetTID();
+      }
+    }
+
+    return ~0ULL;
+  }
+
   uintptr_t Context::CompileBlock(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
     void *CodePtr {nullptr};
     uint8_t const *GuestCode{};
@@ -681,6 +709,8 @@ namespace FEXCore::Context {
     Thread->State.ThreadManager.TID = ::gettid();
     ++IdleWaitRefCount;
 
+    LogMan::Msg::D("[%d] Waiting to run", Thread->State.ThreadManager.TID.load());
+
     // Now notify the thread that we are initialized
     Thread->ThreadWaiting.NotifyAll();
 
@@ -689,7 +719,11 @@ namespace FEXCore::Context {
       Thread->StartRunning.Wait();
     }
 
+    LogMan::Msg::D("[%d] Running", Thread->State.ThreadManager.TID.load());
+
     if (ShouldStop.load() || Thread->State.RunningEvents.ShouldStop.load()) {
+      LogMan::Msg::D("[%d] Early exiting", Thread->State.ThreadManager.TID.load());
+
       ShouldStop = true;
       Thread->State.RunningEvents.ShouldStop.store(true);
       Thread->State.RunningEvents.Running.store(false);
@@ -714,14 +748,6 @@ namespace FEXCore::Context {
     }
 
     uint64_t InitializationStep = 0;
-    if (Initializing) {
-      Thread->State.State.rip = ~0ULL;
-    }
-    else {
-      if (Thread->State.ThreadManager.GetTID() == 1) {
-        Thread->State.State.rip = StartingRIP;
-      }
-    }
 
     if (Thread->CPUBackend->HasCustomDispatch()) {
       Thread->CPUBackend->ExecuteCustomDispatch(&Thread->State);
@@ -745,8 +771,9 @@ namespace FEXCore::Context {
         uint64_t GuestRIP = Thread->State.State.rip;
 
         if (CoreDebugLevel >= 1) {
-          char const *Name = LocalLoader->FindSymbolNameInRange(GuestRIP - MemoryBase);
-          LogMan::Msg::D(">>>>RIP: 0x%lx(0x%lx): '%s'", GuestRIP, GuestRIP - MemoryBase, Name ? Name : "<Unknown>");
+          char const *Name = LocalLoader->FindSymbolNameInRange(GuestRIP);
+          std::string Index[2] = {"", "\t"};
+          LogMan::Msg::D("%s[%d] >>>>RIP: 0x%lx(0x%lx): '%s'", Index[indent].c_str(), ::gettid(), GuestRIP, GuestRIP - MemoryBase, Name ? Name : "<Unknown>");
         }
 
         if (!Thread->CPUBackend->NeedsOpDispatch()) {
@@ -822,9 +849,12 @@ namespace FEXCore::Context {
         }
 
         if (Thread->State.RunningEvents.ShouldStop.load()) {
+          LogMan::Msg::D("[%d] Thread Event Should Stop", Thread->State.ThreadManager.TID.load());
+
           // If it is the parent thread that died then just leave
           // XXX: This doesn't make sense when the parent thread doesn't outlive its children
           if (Thread->State.ThreadManager.parent_tid == 0) {
+            LogMan::Msg::D("[%d] Thread Event Everything should stop", Thread->State.ThreadManager.TID.load());
             ShouldStop = true;
             Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_SHUTDOWN;
           }

External/FEXCore/Source/Interface/HLE/Syscalls/Thread.cpp

@@ -3,6 +3,7 @@
 #include "Interface/HLE/Syscalls.h"
 #include "Interface/HLE/x64/Syscalls.h"
 
+#include <FEXCore/Core/CodeLoader.h>
 #include <FEXCore/Core/X86Enums.h>
 
 #include <limits.h>
@@ -24,14 +25,113 @@ struct InternalThreadState;
 
 namespace FEXCore::HLE {
 
+  FEXCore::Core::InternalThreadState *CreateNewThread(FEXCore::Core::InternalThreadState *Thread, uint32_t flags, void *stack, pid_t *parent_tid, pid_t *child_tid, void *tls) {
+    FEXCore::Core::CPUState NewThreadState{};
+    // Clone copies the parent thread's state
+    memcpy(&NewThreadState, &Thread->State.State, sizeof(FEXCore::Core::CPUState));
+
+    NewThreadState.gregs[FEXCore::X86State::REG_RAX] = 0;
+    NewThreadState.gregs[FEXCore::X86State::REG_RBX] = 0;
+    NewThreadState.gregs[FEXCore::X86State::REG_RBP] = 0;
+    NewThreadState.gregs[FEXCore::X86State::REG_RSP] = reinterpret_cast<uint64_t>(stack);
+
+    if (flags & CLONE_SETTLS) {
+      NewThreadState.fs = reinterpret_cast<uint64_t>(tls);
+    }
+
+    // Set us to start just after the syscall instruction
+    NewThreadState.rip += 2;
+
+    auto NewThread = Thread->CTX->CreateThread(&NewThreadState, reinterpret_cast<uint64_t>(parent_tid));
+    Thread->CTX->CopyMemoryMapping(Thread, NewThread);
+
+    Thread->CTX->InitializeThread(NewThread);
+
+    // Return the new threads TID
+    uint64_t Result = NewThread->State.ThreadManager.GetTID();
+
+    // Sets the child TID to pointer in ParentTID
+    if (flags & CLONE_PARENT_SETTID) {
+      *parent_tid = Result;
+    }
+
+    // Sets the child TID to the pointer in ChildTID
+    if (flags & CLONE_CHILD_SETTID) {
+      NewThread->State.ThreadManager.set_child_tid = child_tid;
+      *child_tid = Result;
+    }
+
+    // When the thread exits, clear the child thread ID at ChildTID
+    // Additionally wakeup a futex at that address
+    // Address /may/ be changed with SET_TID_ADDRESS syscall
+    if (flags & CLONE_CHILD_CLEARTID) {
+      NewThread->State.ThreadManager.clear_child_tid = child_tid;
+    }
+
+    return NewThread;
+  }
+
+  uint64_t DoFork(FEXCore::Core::InternalThreadState *Thread, uint32_t flags, void *stack, pid_t *parent_tid, pid_t *child_tid, void *tls) {
+    LogMan::Msg::E("FORKING!");
+    uint64_t PrevTID = Thread->State.ThreadManager.TID;
+    pid_t pid = fork();
+    if (pid == 0) {
+      // Child
+      // uhhh...
+      Thread->CTX->HandleForkChildSide(PrevTID);
+      Thread->State.ThreadManager.TID = ::gettid();
+    }
+    else {
+      // Parent
+      return pid;
+    }
+
+    // Handle child setup now
+    if (stack == nullptr) {
+      // In the case of fork and nullptr stack then the child uses the same stack space as the parent
+      // Same virtual address, different addressspace
+      LogMan::Msg::D("@@@@@@@ Had to setup custom stack");
+      stack = (void*)Thread->State.State.gregs[X86State::REG_RSP];
+    }
+
+    LogMan::Msg::D("Time to spin up fork thread with Stack: 0x%lx\n", stack);
+    auto NewThread = CreateNewThread(Thread, flags, stack, 0, child_tid, tls);
+
+    NewThread->State.State.gregs[FEXCore::X86State::REG_RAX] = 0;
+    // Return the new threads TID
+    uint64_t Result = NewThread->State.ThreadManager.GetTID();
+    LogMan::Msg::D("Child [%d] starting at: 0x%lx. Parent was at 0x%lx", Result, NewThread->State.State.rip, Thread->State.State.rip);
+
+    // Actually start the thread
+    Thread->CTX->RunThread(NewThread);
+
+    // This thread needs to die now
+    // New thread becomes the new thread
+    Thread->State.ThreadManager.parent_tid = ~0ULL;
+    Thread->State.RunningEvents.ShouldStop = true;
+
+    return 0;
+  }
+
+
   void RegisterThread() {
     REGISTER_SYSCALL_IMPL(getpid, [](FEXCore::Core::InternalThreadState *Thread) -> uint64_t {
+  #if 1
+      uint64_t Result = Thread->CTX->GetPIDHack();
+      if (Result == ~0ULL) {
+        Result = ::getpid();
+        SYSCALL_ERRNO();
+      }
+      return Result;
+
+  #else
       uint64_t Result = ::getpid();
       SYSCALL_ERRNO();
+  #endif
     });
 
     REGISTER_SYSCALL_IMPL(clone, [](FEXCore::Core::InternalThreadState *Thread, uint32_t flags, void *stack, pid_t *parent_tid, pid_t *child_tid, void *tls) -> uint64_t {
-  #define FLAGPRINT(x, y) if (flags & (y)) LogMan::Msg::I("\tFlag: " #x)
+    #define FLAGPRINT(x, y) if (flags & (y)) LogMan::Msg::I("\tFlag: " #x)
       FLAGPRINT(CSIGNAL,              0x000000FF);
       FLAGPRINT(CLONE_VM,             0x00000100);
       FLAGPRINT(CLONE_FS,             0x00000200);
@@ -58,50 +158,15 @@ namespace FEXCore::HLE {
       FLAGPRINT(CLONE_IO,             0x80000000);
 
       if (!(flags & CLONE_VM)) {
-        LogMan::Msg::E("Unsupported clone without CLONE_VM");
-        return -ENOSPC;
-      }
-      FEXCore::Core::CPUState NewThreadState{};
-      // Clone copies the parent thread's state
-      memcpy(&NewThreadState, &Thread->State.State, sizeof(FEXCore::Core::CPUState));
-
-      NewThreadState.gregs[FEXCore::X86State::REG_RAX] = 0;
-      NewThreadState.gregs[FEXCore::X86State::REG_RBX] = 0;
-      NewThreadState.gregs[FEXCore::X86State::REG_RBP] = 0;
-      NewThreadState.gregs[FEXCore::X86State::REG_RSP] = reinterpret_cast<uint64_t>(stack);
-
-      if (flags & CLONE_SETTLS) {
-        NewThreadState.fs = reinterpret_cast<uint64_t>(tls);
+        LogMan::Msg::D("Parent [%d] Leaving syscall at rip: 0x%lx", Thread->State.ThreadManager.GetTID(), Thread->State.State.rip);
+        return DoFork(Thread, flags, stack, parent_tid, child_tid, tls);
       }
 
-      // Set us to start just after the syscall instruction
-      NewThreadState.rip += 2;
-
-      auto NewThread = Thread->CTX->CreateThread(&NewThreadState, reinterpret_cast<uint64_t>(parent_tid));
-      Thread->CTX->CopyMemoryMapping(Thread, NewThread);
-
-      Thread->CTX->InitializeThread(NewThread);
+      auto NewThread = CreateNewThread(Thread, flags, stack, parent_tid, child_tid, tls);
 
       // Return the new threads TID
       uint64_t Result = NewThread->State.ThreadManager.GetTID();
-
-      // Sets the child TID to pointer in ParentTID
-      if (flags & CLONE_PARENT_SETTID) {
-        *parent_tid = Result;
-      }
-
-      // Sets the child TID to the pointer in ChildTID
-      if (flags & CLONE_CHILD_SETTID) {
-        NewThread->State.ThreadManager.set_child_tid = child_tid;
-        *child_tid = Result;
-      }
-
-      // When the thread exits, clear the child thread ID at ChildTID
-      // Additionally wakeup a futex at that address
-      // Address /may/ be changed with SET_TID_ADDRESS syscall
-      if (flags & CLONE_CHILD_CLEARTID) {
-        NewThread->State.ThreadManager.clear_child_tid = child_tid;
-      }
+      LogMan::Msg::D("Child [%d] starting at: 0x%lx. Parent was at 0x%lx", Result, NewThread->State.State.rip, Thread->State.State.rip);
 
       // Actually start the thread
       Thread->CTX->RunThread(NewThread);
@@ -110,6 +175,15 @@ namespace FEXCore::HLE {
         // If VFORK is set then the calling process is suspended until the thread exits with execve or exit
         NewThread->ExecutionThread.join();
       }
+
+      if (!(flags & CLONE_VM)) {
+        // This thread needs to die now
+        // New thread becomes the new thread
+        NewThread->ExecutionThread.join();
+        Thread->State.ThreadManager.parent_tid = ~0ULL;
+        Thread->State.RunningEvents.ShouldStop = true;
+      }
+
       SYSCALL_ERRNO();
     });
 

External/FEXCore/Source/Interface/Memory/MemMapper.cpp

@@ -18,7 +18,7 @@ namespace FEXCore::Memory {
     }
 
     void *Ptr = mmap(reinterpret_cast<void*>(PtrOffset), Size, Flags,
-      MAP_POPULATE | MAP_SHARED | (Fixed ? MAP_FIXED : 0), SHM->SHMFD, Offset);
+      MAP_POPULATE | MAP_PRIVATE | (Fixed ? MAP_FIXED : 0), SHM->SHMFD, Offset);
 
     if (Ptr == MAP_FAILED) {
       LogMan::Msg::A("Failed to map memory region [0x%lx, 0x%lx)", Offset, Offset + Size);
@@ -36,7 +36,7 @@ namespace FEXCore::Memory {
     }
 
     void *Ptr = mmap(reinterpret_cast<void*>(PtrOffset), Size, Flags,
-      MAP_SHARED | (Fixed ? MAP_FIXED : 0), SHM->SHMFD, Offset);
+      MAP_PRIVATE | (Fixed ? MAP_FIXED : 0), SHM->SHMFD, Offset);
 
     if (Ptr == MAP_FAILED) {
       LogMan::Msg::A("Failed to map memory region [0x%lx, 0x%lx)", Offset, Offset + Size);