From e227f1343fb41ab0456a2e13b91dd46d825367a8 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Wed, 22 Nov 2023 07:17:53 -0800 Subject: [PATCH] FEXCore: Start changing how thread creation works The frontend needs to be in control of how threads are created. This is inherent to the fact that OS threads are OS specific. We currently have this weird split that when initializing the FEXCore context, we create a parent thread at all times. This does some initial cleanup that gets the core initialization nearly decoupled. --- FEXCore/Source/Interface/Context/Context.h | 25 ++++++++----------- FEXCore/Source/Interface/Core/Core.cpp | 23 ++++++----------- FEXCore/include/FEXCore/Core/CPUBackend.h | 7 ------ FEXCore/include/FEXCore/Core/Context.h | 14 ++++++++++- Source/Tools/FEXLoader/AOT/AOTGenerator.cpp | 3 +-- .../LinuxSyscalls/Syscalls/Thread.cpp | 4 +-- Source/Windows/WOW64/Module.cpp | 2 +- 7 files changed, 35 insertions(+), 43 deletions(-) diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index 4de7f5dea..38072a226 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -111,25 +111,31 @@ namespace FEXCore::Context { /** * @brief Used to create FEX thread objects in preparation for creating a true OS thread. Does set a TID or PID. * - * @param NewThreadState The initial thread state to setup for our state + * @param InitialRIP The starting RIP of this thread + * @param StackPointer The starting RSP of this thread + * @param NewThreadState The initial thread state to setup for our state, if inheriting. * @param ParentTID The PID that was the parent thread that created this * * @return The InternalThreadState object that tracks all of the emulated thread's state * * Usecases: + * Parent thread Creation: + * - Thread = CreateThread(InitialRIP, InitialStack, nullptr, 0); + * - CTX->RunUntilExit(Thread); * OS thread Creation: - * - Thread = CreateThread(NewState, PPID); + * - Thread = CreateThread(0, 0, NewState, PPID); * - InitializeThread(Thread); * OS fork (New thread created with a clone of thread state): * - clone{2, 3} - * - Thread = CreateThread(CopyOfThreadState, PPID); + * - Thread = CreateThread(0, 0, CopyOfThreadState, PPID); * - ExecutionThread(Thread); // Starts executing without creating another host thread * Thunk callback executing guest code from native host thread - * - Thread = CreateThread(NewState, PPID); + * - Thread = CreateThread(0, 0, NewState, PPID); * - InitializeThreadTLSData(Thread); * - HandleCallback(Thread, RIP); */ - FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) override; + + FEXCore::Core::InternalThreadState* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) override; // Public for threading void ExecutionThread(FEXCore::Core::InternalThreadState *Thread) override; @@ -417,15 +423,6 @@ namespace FEXCore::Context { } private: - /** - * @brief Does some final thread initialization - * - * @param Thread The internal FEX thread state object - * - * InitCore and CreateThread both call this to finish up thread object initialization - */ - void InitializeThreadData(FEXCore::Core::InternalThreadState *Thread); - /** * @brief Initializes the JIT compilers for the thread * diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index a0a99d1fa..f18757339 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -387,18 +387,11 @@ namespace FEXCore::Context { StartPaused = true; } - using namespace FEXCore::Core; - - FEXCore::Core::InternalThreadState *Thread = CreateThread(nullptr, 0); + FEXCore::Core::InternalThreadState *Thread = CreateThread(InitialRIP, StackPointer, nullptr, 0); // We are the parent thread ParentThread = Thread; - Thread->CurrentFrame->State.gregs[X86State::REG_RSP] = StackPointer; - - Thread->CurrentFrame->State.rip = InitialRIP; - - InitializeThreadData(Thread); return Thread; } @@ -578,10 +571,6 @@ namespace FEXCore::Context { return ParentThread->StatusCode; } - void ContextImpl::InitializeThreadData(FEXCore::Core::InternalThreadState *Thread) { - Thread->CPUBackend->Initialize(); - } - struct ExecutionThreadHandler { ContextImpl *This; FEXCore::Core::InternalThreadState *Thread; @@ -680,20 +669,22 @@ namespace FEXCore::Context { Thread->PassManager->Finalize(); } - FEXCore::Core::InternalThreadState* ContextImpl::CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) { + FEXCore::Core::InternalThreadState* ContextImpl::CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) { FEXCore::Core::InternalThreadState *Thread = new FEXCore::Core::InternalThreadState{}; + Thread->CurrentFrame->State.gregs[X86State::REG_RSP] = StackPointer; + Thread->CurrentFrame->State.rip = InitialRIP; + // Copy over the new thread state to the new object if (NewThreadState) { - memcpy(Thread->CurrentFrame, NewThreadState, sizeof(FEXCore::Core::CPUState)); + memcpy(&Thread->CurrentFrame->State, NewThreadState, sizeof(FEXCore::Core::CPUState)); } - Thread->CurrentFrame->Thread = Thread; // Set up the thread manager state Thread->ThreadManager.parent_tid = ParentTID; + Thread->CurrentFrame->Thread = Thread; InitializeCompiler(Thread); - InitializeThreadData(Thread); Thread->CurrentFrame->State.DeferredSignalRefCount.Store(0); Thread->CurrentFrame->State.DeferredSignalFaultAddress = reinterpret_cast*>(FEXCore::Allocator::VirtualAlloc(4096)); diff --git a/FEXCore/include/FEXCore/Core/CPUBackend.h b/FEXCore/include/FEXCore/Core/CPUBackend.h index 42fae4fdf..f0c400d3a 100644 --- a/FEXCore/include/FEXCore/Core/CPUBackend.h +++ b/FEXCore/include/FEXCore/Core/CPUBackend.h @@ -155,13 +155,6 @@ namespace CPU { */ [[nodiscard]] virtual void *MapRegion(void *HostPtr, uint64_t GuestPtr, uint64_t Size) = 0; - /** - * @brief This is post-setup initialization that is called just before code executino - * - * Guest memory is available at this point and ThreadState is valid - */ - virtual void Initialize() {} - /** * @brief Lets FEXCore know if this CPUBackend needs IR and DebugData for CompileCode * diff --git a/FEXCore/include/FEXCore/Core/Context.h b/FEXCore/include/FEXCore/Core/Context.h index 3d41685c3..7b047a7e7 100644 --- a/FEXCore/include/FEXCore/Core/Context.h +++ b/FEXCore/include/FEXCore/Core/Context.h @@ -278,7 +278,19 @@ namespace FEXCore::Context { ///< Sets FEX's internal EFLAGS representation to the passed in compacted form. FEX_DEFAULT_VISIBILITY virtual void SetFlagsFromCompactedEFLAGS(FEXCore::Core::InternalThreadState *Thread, uint32_t EFLAGS) = 0; - FEX_DEFAULT_VISIBILITY virtual FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) = 0; + /** + * @brief Create a new thread object that doesn't inherit any state. + * Used to create FEX thread objects in preparation for creating a true OS thread. + * + * @param InitialRIP The starting RIP of this thread + * @param StackPointer The starting RSP of this thread + * @param NewThreadState The thread state to inherit from if not nullptr. + * @param ParentTID The thread ID that the parent is inheriting from + * + * @return A new InternalThreadState object for using with a new guest thread. + */ + FEX_DEFAULT_VISIBILITY virtual FEXCore::Core::InternalThreadState* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState *NewThreadState = nullptr, uint64_t ParentTID = 0) = 0; + FEX_DEFAULT_VISIBILITY virtual void ExecutionThread(FEXCore::Core::InternalThreadState *Thread) = 0; FEX_DEFAULT_VISIBILITY virtual void InitializeThread(FEXCore::Core::InternalThreadState *Thread) = 0; FEX_DEFAULT_VISIBILITY virtual void RunThread(FEXCore::Core::InternalThreadState *Thread) = 0; diff --git a/Source/Tools/FEXLoader/AOT/AOTGenerator.cpp b/Source/Tools/FEXLoader/AOT/AOTGenerator.cpp index 416e60634..88c4d44ef 100644 --- a/Source/Tools/FEXLoader/AOT/AOTGenerator.cpp +++ b/Source/Tools/FEXLoader/AOT/AOTGenerator.cpp @@ -106,8 +106,7 @@ void AOTGenSection(FEXCore::Context::Context *CTX, ELFCodeLoader::LoadedSection setpriority(PRIO_PROCESS, FHU::Syscalls::gettid(), 19); // Setup thread - Each compilation thread uses its own backing FEX thread - FEXCore::Core::CPUState state; - auto Thread = CTX->CreateThread(&state, FHU::Syscalls::gettid()); + auto Thread = CTX->CreateThread(0, 0); fextl::set ExternalBranchesLocal; CTX->ConfigureAOTGen(Thread, &ExternalBranchesLocal, SectionMaxAddress); diff --git a/Source/Tools/FEXLoader/LinuxSyscalls/Syscalls/Thread.cpp b/Source/Tools/FEXLoader/LinuxSyscalls/Syscalls/Thread.cpp index e5d47dfec..22f8adcd4 100644 --- a/Source/Tools/FEXLoader/LinuxSyscalls/Syscalls/Thread.cpp +++ b/Source/Tools/FEXLoader/LinuxSyscalls/Syscalls/Thread.cpp @@ -58,7 +58,7 @@ namespace FEX::HLE { NewThreadState.gregs[FEXCore::X86State::REG_RSP] = args->args.stack; } - auto NewThread = CTX->CreateThread(&NewThreadState, args->args.parent_tid); + auto NewThread = CTX->CreateThread(0, 0, &NewThreadState, args->args.parent_tid); CTX->InitializeThread(NewThread); if (FEX::HLE::_SyscallHandler->Is64BitMode()) { @@ -131,7 +131,7 @@ namespace FEX::HLE { } // Overwrite thread - NewThread = CTX->CreateThread(&NewThreadState, GuestArgs->parent_tid); + NewThread = CTX->CreateThread(0, 0, &NewThreadState, GuestArgs->parent_tid); // CLONE_PARENT_SETTID, CLONE_CHILD_SETTID, CLONE_CHILD_CLEARTID, CLONE_PIDFD will be handled by kernel // Call execution thread directly since we already are on the new thread diff --git a/Source/Windows/WOW64/Module.cpp b/Source/Windows/WOW64/Module.cpp index 1d5177c7c..170a8158d 100644 --- a/Source/Windows/WOW64/Module.cpp +++ b/Source/Windows/WOW64/Module.cpp @@ -555,7 +555,7 @@ void BTCpuProcessInit() { } NTSTATUS BTCpuThreadInit() { - GetTLS().ThreadState() = CTX->CreateThread(nullptr, 0); + GetTLS().ThreadState() = CTX->CreateThread(0, 0); std::scoped_lock Lock(ThreadSuspendLock); InitializedWOWThreads.emplace(GetCurrentThreadId());