FEXCore: Start changing how thread creation works

The frontend needs to be in control of how threads are created. This is
inherent to the fact that OS threads are OS specific. We currently have
this weird split that when initializing the FEXCore context, we create a
parent thread at all times.

This does some initial cleanup that gets the core initialization nearly
decoupled.

FEXCore/Source/Interface/Context/Context.h

@@ -111,25 +111,31 @@ namespace FEXCore::Context {
       /**
        * @brief Used to create FEX thread objects in preparation for creating a true OS thread. Does set a TID or PID.
        *
-       * @param NewThreadState The initial thread state to setup for our state
+       * @param InitialRIP The starting RIP of this thread
+       * @param StackPointer The starting RSP of this thread
+       * @param NewThreadState The initial thread state to setup for our state, if inheriting.
        * @param ParentTID The PID that was the parent thread that created this
        *
        * @return The InternalThreadState object that tracks all of the emulated thread's state
        *
        * Usecases:
+       *  Parent thread Creation:
+       *    - Thread = CreateThread(InitialRIP, InitialStack, nullptr, 0);
+       *    - CTX->RunUntilExit(Thread);
        *  OS thread Creation:
-       *    - Thread = CreateThread(NewState, PPID);
+       *    - Thread = CreateThread(0, 0, NewState, PPID);
        *    - InitializeThread(Thread);
        *  OS fork (New thread created with a clone of thread state):
        *    - clone{2, 3}
-       *    - Thread = CreateThread(CopyOfThreadState, PPID);
+       *    - Thread = CreateThread(0, 0, CopyOfThreadState, PPID);
        *    - ExecutionThread(Thread); // Starts executing without creating another host thread
        *  Thunk callback executing guest code from native host thread
-       *    - Thread = CreateThread(NewState, PPID);
+       *    - Thread = CreateThread(0, 0, NewState, PPID);
        *    - InitializeThreadTLSData(Thread);
        *    - HandleCallback(Thread, RIP);
        */
-      FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) override;
+
+      FEXCore::Core::InternalThreadState* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) override;
 
       // Public for threading
       void ExecutionThread(FEXCore::Core::InternalThreadState *Thread) override;
@@ -417,15 +423,6 @@ namespace FEXCore::Context {
     }
 
   private:
-    /**
-     * @brief Does some final thread initialization
-     *
-     * @param Thread The internal FEX thread state object
-     *
-     * InitCore and CreateThread both call this to finish up thread object initialization
-     */
-    void InitializeThreadData(FEXCore::Core::InternalThreadState *Thread);
-
     /**
      * @brief Initializes the JIT compilers for the thread
      *

FEXCore/Source/Interface/Core/Core.cpp

@@ -387,18 +387,11 @@ namespace FEXCore::Context {
       StartPaused = true;
     }
 
-    using namespace FEXCore::Core;
-
-    FEXCore::Core::InternalThreadState *Thread = CreateThread(nullptr, 0);
+    FEXCore::Core::InternalThreadState *Thread = CreateThread(InitialRIP, StackPointer, nullptr, 0);
 
     // We are the parent thread
     ParentThread = Thread;
 
-    Thread->CurrentFrame->State.gregs[X86State::REG_RSP] = StackPointer;
-
-    Thread->CurrentFrame->State.rip = InitialRIP;
-
-    InitializeThreadData(Thread);
     return Thread;
   }
 
@@ -578,10 +571,6 @@ namespace FEXCore::Context {
     return ParentThread->StatusCode;
   }
 
-  void ContextImpl::InitializeThreadData(FEXCore::Core::InternalThreadState *Thread) {
-    Thread->CPUBackend->Initialize();
-  }
-
   struct ExecutionThreadHandler {
     ContextImpl *This;
     FEXCore::Core::InternalThreadState *Thread;
@@ -680,20 +669,22 @@ namespace FEXCore::Context {
     Thread->PassManager->Finalize();
   }
 
-  FEXCore::Core::InternalThreadState* ContextImpl::CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) {
+  FEXCore::Core::InternalThreadState* ContextImpl::CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) {
     FEXCore::Core::InternalThreadState *Thread = new FEXCore::Core::InternalThreadState{};
 
+    Thread->CurrentFrame->State.gregs[X86State::REG_RSP] = StackPointer;
+    Thread->CurrentFrame->State.rip = InitialRIP;
+
     // Copy over the new thread state to the new object
     if (NewThreadState) {
-      memcpy(Thread->CurrentFrame, NewThreadState, sizeof(FEXCore::Core::CPUState));
+      memcpy(&Thread->CurrentFrame->State, NewThreadState, sizeof(FEXCore::Core::CPUState));
     }
-    Thread->CurrentFrame->Thread = Thread;
 
     // Set up the thread manager state
     Thread->ThreadManager.parent_tid = ParentTID;
+    Thread->CurrentFrame->Thread = Thread;
 
     InitializeCompiler(Thread);
-    InitializeThreadData(Thread);
 
     Thread->CurrentFrame->State.DeferredSignalRefCount.Store(0);
     Thread->CurrentFrame->State.DeferredSignalFaultAddress = reinterpret_cast<Core::NonAtomicRefCounter<uint64_t>*>(FEXCore::Allocator::VirtualAlloc(4096));

FEXCore/include/FEXCore/Core/CPUBackend.h

@@ -155,13 +155,6 @@ namespace CPU {
      */
     [[nodiscard]] virtual void *MapRegion(void *HostPtr, uint64_t GuestPtr, uint64_t Size) = 0;
 
-    /**
-     * @brief This is post-setup initialization that is called just before code executino
-     *
-     * Guest memory is available at this point and ThreadState is valid
-     */
-    virtual void Initialize() {}
-
     /**
      * @brief Lets FEXCore know if this CPUBackend needs IR and DebugData for CompileCode
      *

FEXCore/include/FEXCore/Core/Context.h

@@ -278,7 +278,19 @@ namespace FEXCore::Context {
       ///< Sets FEX's internal EFLAGS representation to the passed in compacted form.
       FEX_DEFAULT_VISIBILITY virtual void SetFlagsFromCompactedEFLAGS(FEXCore::Core::InternalThreadState *Thread, uint32_t EFLAGS) = 0;
 
-      FEX_DEFAULT_VISIBILITY virtual FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) = 0;
+      /**
+       * @brief Create a new thread object that doesn't inherit any state.
+       * Used to create FEX thread objects in preparation for creating a true OS thread.
+       *
+       * @param InitialRIP The starting RIP of this thread
+       * @param StackPointer The starting RSP of this thread
+       * @param NewThreadState The thread state to inherit from if not nullptr.
+       * @param ParentTID The thread ID that the parent is inheriting from
+       *
+       * @return A new InternalThreadState object for using with a new guest thread.
+       */
+      FEX_DEFAULT_VISIBILITY virtual FEXCore::Core::InternalThreadState* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState *NewThreadState = nullptr, uint64_t ParentTID = 0) = 0;
+
       FEX_DEFAULT_VISIBILITY virtual void ExecutionThread(FEXCore::Core::InternalThreadState *Thread) = 0;
       FEX_DEFAULT_VISIBILITY virtual void InitializeThread(FEXCore::Core::InternalThreadState *Thread) = 0;
       FEX_DEFAULT_VISIBILITY virtual void RunThread(FEXCore::Core::InternalThreadState *Thread) = 0;

Source/Tools/FEXLoader/AOT/AOTGenerator.cpp

@@ -106,8 +106,7 @@ void AOTGenSection(FEXCore::Context::Context *CTX, ELFCodeLoader::LoadedSection
       setpriority(PRIO_PROCESS, FHU::Syscalls::gettid(), 19);
 
       // Setup thread - Each compilation thread uses its own backing FEX thread
-      FEXCore::Core::CPUState state;
-      auto Thread = CTX->CreateThread(&state, FHU::Syscalls::gettid());
+      auto Thread = CTX->CreateThread(0, 0);
       fextl::set<uint64_t> ExternalBranchesLocal;
       CTX->ConfigureAOTGen(Thread, &ExternalBranchesLocal, SectionMaxAddress);
 

Source/Tools/FEXLoader/LinuxSyscalls/Syscalls/Thread.cpp

@@ -58,7 +58,7 @@ namespace FEX::HLE {
       NewThreadState.gregs[FEXCore::X86State::REG_RSP] = args->args.stack;
     }
 
-    auto NewThread = CTX->CreateThread(&NewThreadState, args->args.parent_tid);
+    auto NewThread = CTX->CreateThread(0, 0, &NewThreadState, args->args.parent_tid);
     CTX->InitializeThread(NewThread);
 
     if (FEX::HLE::_SyscallHandler->Is64BitMode()) {
@@ -131,7 +131,7 @@ namespace FEX::HLE {
       }
 
       // Overwrite thread
-      NewThread = CTX->CreateThread(&NewThreadState, GuestArgs->parent_tid);
+      NewThread = CTX->CreateThread(0, 0, &NewThreadState, GuestArgs->parent_tid);
 
       // CLONE_PARENT_SETTID, CLONE_CHILD_SETTID, CLONE_CHILD_CLEARTID, CLONE_PIDFD will be handled by kernel
       // Call execution thread directly since we already are on the new thread

Source/Windows/WOW64/Module.cpp

@@ -555,7 +555,7 @@ void BTCpuProcessInit() {
 }
 
 NTSTATUS BTCpuThreadInit() {
-  GetTLS().ThreadState() = CTX->CreateThread(nullptr, 0);
+  GetTLS().ThreadState() = CTX->CreateThread(0, 0);
 
   std::scoped_lock Lock(ThreadSuspendLock);
   InitializedWOWThreads.emplace(GetCurrentThreadId());