Windows: add Thread implementation

So we can make a worker thread that the guest (hopefully) won't see.

Source/Windows/ARM64EC/Module.cpp

@@ -36,6 +36,7 @@ $end_info$
 #include "Common/Exception.h"
 #include "Common/ImageTracker.h"
 #include "Common/InvalidationTracker.h"
+#include "Common/Threads.h"
 #include "Common/OvercommitTracker.h"
 #include "Common/TSOHandlerConfig.h"
 #include "Common/CPUFeatures.h"
@@ -581,6 +582,7 @@ NTSTATUS ProcessInit() {
   InitSyscalls();
 
   FEX::Windows::InitCRTProcess();
+  FEX::Windows::SetupThreadHandlers();
   const auto ExecutablePath = FEX::Windows::GetExecutableFilePath();
   const auto ExecutableName = FEX::Windows::BaseName(ExecutablePath);
   FEX::Config::LoadConfig(fextl::string {ExecutableName}, _environ, FEX::ReadPortabilityInformation());

Source/Windows/Common/CMakeLists.txt

@@ -14,7 +14,8 @@ add_library(CommonWindows STATIC
   SHMStats.cpp
   InvalidationTracker.cpp
   ImageTracker.cpp
-  Logging.cpp)
+  Logging.cpp
+  Threads.cpp)
 
 target_link_libraries(CommonWindows FEXCore_Base)
 target_include_directories(CommonWindows PRIVATE "${CMAKE_SOURCE_DIR}/Source/Windows/include/")

Source/Windows/Common/Threads.cpp

@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: MIT
+#include "Threads.h"
+#include "CRT/CRT.h"
+
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/Threads.h>
+#include <FEXCore/fextl/memory.h>
+
+#include <winternl.h>
+#include <windows.h>
+
+namespace FEX::Windows {
+namespace WinThreadImpl {
+  class Thread final : public FEXCore::Threads::Thread {
+  public:
+    Thread(FEXCore::Threads::ThreadFunc Func, void* Arg)
+      : UserFunc {Func}
+      , UserArg {Arg} {
+      // hide everything from guest, don't initialize anything, we'll do that manually in RunThread()
+      const ULONG CreateFlags = THREAD_CREATE_FLAGS_SKIP_THREAD_ATTACH | THREAD_CREATE_FLAGS_HIDE_FROM_DEBUGGER |
+                                THREAD_CREATE_FLAGS_SKIP_LOADER_INIT | THREAD_CREATE_FLAGS_BYPASS_PROCESS_FREEZE;
+      NTSTATUS Status = NtCreateThreadEx(&Handle, THREAD_ALL_ACCESS, nullptr, GetCurrentProcess(),
+                                         reinterpret_cast<PRTL_THREAD_START_ROUTINE>(&Thread::RunThread), this, CreateFlags, 0, 0, 0, nullptr);
+      if (Status < 0) {
+        LogMan::Msg::EFmt("NtCreateThreadEx failed: 0x{:x}", static_cast<uint32_t>(Status));
+        Handle = nullptr;
+      }
+    }
+
+    bool joinable() override {
+      return Handle != nullptr;
+    }
+
+    bool join(void** ret) override {
+      if (!Handle) {
+        return false;
+      }
+      NtWaitForSingleObject(Handle, FALSE, nullptr);
+      if (ret) {
+        *ret = ReturnValue;
+      }
+      return true;
+    }
+
+    bool detach() override {
+      // almost no-op, it's already detached, just deref handle
+      if (Handle) {
+        NtClose(Handle);
+        Handle = nullptr;
+      }
+      return true;
+    }
+
+    bool IsSelf() override {
+      return TID == GetCurrentThreadId();
+    }
+
+    ~Thread() override {
+      if (Handle) {
+        NtClose(Handle);
+      }
+    }
+
+  private:
+    static void RunThread(Thread* This) {
+      This->TID = GetCurrentThreadId();
+      // do initialization we skipped earlier here around the user entrypoint
+      FEX::Windows::InitCRTThread();
+      This->ReturnValue = This->UserFunc(This->UserArg);
+      FEX::Windows::DeinitCRTThread();
+      NtTerminateThread(GetCurrentThread(), 0);
+    }
+
+    FEXCore::Threads::ThreadFunc UserFunc;
+    void* UserArg;
+    HANDLE Handle {};
+    DWORD TID {};
+    void* ReturnValue {};
+  };
+
+  fextl::unique_ptr<FEXCore::Threads::Thread> CreateThread(FEXCore::Threads::ThreadFunc Func, void* Arg) {
+    return fextl::make_unique<Thread>(Func, Arg);
+  }
+
+  void CleanupAfterFork() {}
+} // namespace WinThreadImpl
+
+void SetupThreadHandlers() {
+  FEXCore::Threads::Pointers Ptrs = {
+    .CreateThread = WinThreadImpl::CreateThread,
+    .CleanupAfterFork = WinThreadImpl::CleanupAfterFork,
+  };
+  FEXCore::Threads::Thread::SetInternalPointers(Ptrs);
+}
+} // namespace FEX::Windows

Source/Windows/Common/Threads.h

@@ -0,0 +1,6 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+
+namespace FEX::Windows {
+void SetupThreadHandlers();
+} // namespace FEX::Windows

Source/Windows/WOW64/Module.cpp

@@ -38,6 +38,7 @@ $end_info$
 #include "Common/TSOHandlerConfig.h"
 #include "Common/ImageTracker.h"
 #include "Common/InvalidationTracker.h"
+#include "Common/Threads.h"
 #include "Common/OvercommitTracker.h"
 #include "Common/CPUFeatures.h"
 #include "Common/Logging.h"
@@ -515,6 +516,7 @@ public:
 
 void BTCpuProcessInit() {
   FEX::Windows::InitCRTProcess();
+  FEX::Windows::SetupThreadHandlers();
   const auto ExecutablePath = FEX::Windows::GetExecutableFilePath();
   const auto ExecutableName = FEX::Windows::BaseName(ExecutablePath);
   FEX::Config::LoadConfig(fextl::string {ExecutableName}, _environ, FEX::ReadPortabilityInformation());

Source/Windows/include/winternl.h

@@ -407,6 +407,23 @@ typedef enum _SECTION_INHERIT {
   ViewUnmap = 2,
 } SECTION_INHERIT;
 
+typedef NTSTATUS(WINAPI* PRTL_THREAD_START_ROUTINE)(void* Parameter);
+
+typedef struct _PS_ATTRIBUTE {
+  ULONG_PTR Attribute;
+  SIZE_T Size;
+  union {
+    ULONG_PTR Value;
+    void* ValuePtr;
+  };
+  PSIZE_T ReturnLength;
+} PS_ATTRIBUTE, *PPS_ATTRIBUTE;
+
+typedef struct _PS_ATTRIBUTE_LIST {
+  SIZE_T TotalLength;
+  PS_ATTRIBUTE Attributes[1];
+} PS_ATTRIBUTE_LIST, *PPS_ATTRIBUTE_LIST;
+
 /* definitions of bits in the Feature set for the x86 processors */
 #define CPU_FEATURE_VME 0x00000005    /* Virtual 86 Mode Extensions */
 #define CPU_FEATURE_TSC 0x00000002    /* Time Stamp Counter available */
@@ -518,6 +535,12 @@ NTSTATUS WINAPI NtAllocateVirtualMemoryEx(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG,
 NTSTATUS WINAPI NtAllocateVirtualMemory(HANDLE, PVOID*, ULONG_PTR, SIZE_T*, ULONG, ULONG);
 NTSTATUS WINAPI NtContinue(PCONTEXT, BOOLEAN);
 NTSTATUS WINAPI NtCreateSection(HANDLE*, ACCESS_MASK, const OBJECT_ATTRIBUTES*, const LARGE_INTEGER*, ULONG, ULONG, HANDLE);
+#define THREAD_CREATE_FLAGS_SKIP_THREAD_ATTACH 0x00000002
+#define THREAD_CREATE_FLAGS_HIDE_FROM_DEBUGGER 0x00000004
+#define THREAD_CREATE_FLAGS_SKIP_LOADER_INIT 0x00000020
+#define THREAD_CREATE_FLAGS_BYPASS_PROCESS_FREEZE 0x00000040
+NTSTATUS WINAPI NtCreateThreadEx(HANDLE*, ACCESS_MASK, OBJECT_ATTRIBUTES*, HANDLE, PRTL_THREAD_START_ROUTINE, void*, ULONG, ULONG_PTR,
+                                 SIZE_T, SIZE_T, PPS_ATTRIBUTE_LIST);
 NTSTATUS WINAPI NtDelayExecution(BOOLEAN, const LARGE_INTEGER*);
 NTSTATUS WINAPI NtDuplicateObject(HANDLE, HANDLE, HANDLE, PHANDLE, ACCESS_MASK, ULONG, ULONG);
 NTSTATUS WINAPI NtFlushInstructionCache(HANDLE, LPCVOID, SIZE_T);
@@ -538,6 +561,7 @@ NTSTATUS WINAPI NtReadFile(HANDLE, HANDLE, PIO_APC_ROUTINE, PVOID, PIO_STATUS_BL
 NTSTATUS WINAPI NtSetContextThread(HANDLE, const CONTEXT*);
 NTSTATUS WINAPI NtSuspendThread(HANDLE, PULONG);
 NTSTATUS WINAPI NtTerminateProcess(HANDLE, LONG);
+NTSTATUS WINAPI NtTerminateThread(HANDLE, LONG);
 NTSTATUS WINAPI NtUnlockFile(HANDLE, PIO_STATUS_BLOCK, PLARGE_INTEGER, PLARGE_INTEGER, ULONG);
 NTSTATUS WINAPI NtWriteFile(HANDLE, HANDLE, PIO_APC_ROUTINE, PVOID, PIO_STATUS_BLOCK, const void*, ULONG, PLARGE_INTEGER, PULONG);
 void WINAPI ProcessPendingCrossProcessEmulatorWork();