Docs: Update for release FEX-2107

ARM64: Fixes bugs in unaligned atomic signal handlers

CMakeLists.txt

@@ -252,7 +252,7 @@ add_compile_options(-Wall)
 
 configure_file(
   ${CMAKE_CURRENT_SOURCE_DIR}/include/Config.h.in
-  ${CMAKE_BINARY_DIR}/generated/Config.h)
+  ${CMAKE_BINARY_DIR}/generated/ConfigDefines.h)
 
 if (BUILD_TESTS)
   include(CTest)
@@ -261,6 +261,9 @@ if (BUILD_TESTS)
 endif()
 add_subdirectory(External/FEXCore)
 
+# Binfmt_misc files must be installed prior to Source/ installs
+add_subdirectory(Data/binfmts/)
+
 add_subdirectory(Source/)
 add_subdirectory(Data/AppConfig/)
 
@@ -307,3 +310,63 @@ if (BUILD_THUNKS)
     DEPENDS guest-libs
   )
 endif()
+
+set(FEX_VERSION_MAJOR "0")
+set(FEX_VERSION_MINOR "0")
+set(FEX_VERSION_PATCH "0")
+
+find_package(Git)
+if (GIT_FOUND)
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} describe --abbrev=0
+    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    OUTPUT_VARIABLE GIT_DESCRIBE_STRING
+    RESULT_VARIABLE GIT_ERROR
+    ERROR_QUIET
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+
+  if (NOT ${GIT_ERROR} EQUAL 0)
+    # Likely built in a way that doesn't have tags
+    # Setup a version tag that is unknown
+    set(GIT_DESCRIBE_STRING "FEX-0000")
+  endif()
+
+  # Change something like `FEX-2106.1-76-<hash>` in to a list
+  string(REPLACE "-" ";" DESCRIBE_LIST ${GIT_DESCRIBE_STRING})
+
+  # Extract the `2106.1` element
+  list(GET DESCRIBE_LIST 1 DESCRIBE_LIST)
+
+  # Change `2106.1` in to a list
+  string(REPLACE "." ";" DESCRIBE_LIST ${DESCRIBE_LIST})
+
+  # Calculate list size
+  list(LENGTH DESCRIBE_LIST LIST_SIZE)
+
+  # Pull out the major version
+  list(GET DESCRIBE_LIST 0 FEX_VERSION_MAJOR)
+
+  # Minor version only exists if there is a .1 at the end
+  # eg: 2106 versus 2106.1
+  if (LIST_SIZE GREATER 1)
+    list(GET DESCRIBE_LIST 1 FEX_VERSION_MINOR)
+  endif()
+endif()
+
+# Package creation
+set (CPACK_GENERATOR "DEB")
+set (CPACK_PACKAGE_CONTACT "team@fex-emu.org")
+set (CPACK_PACKAGE_VERSION_MAJOR "${FEX_VERSION_MAJOR}")
+set (CPACK_PACKAGE_VERSION_MINOR "${FEX_VERSION_MINOR}")
+set (CPACK_PACKAGE_VERSION_PATCH "${FEX_VERSION_PATCH}")
+
+# Debian defines
+set (CPACK_DEBIAN_PACKAGE_DEPENDS "libstdc++6")
+set (CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/CPack/postinst;${CMAKE_CURRENT_SOURCE_DIR}/CPack/prerm")
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
+  # binfmt_misc conflicts with qemu-user-static
+  # We also only install binfmt_misc on aarch64 hosts
+  set (CPACK_DEBIAN_PACKAGE_CONFLICTS "qemu-user-static")
+endif()
+include (CPack)

CPack/postinst

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -e
+update_binfmt() {
+  # Check for update-binfmts
+  command -v update-binfmts >/dev/null || return 0
+
+  # Setup binfmt_misc
+  update-binfmts --import FEX-x86
+  update-binfmts --import FEX-x86_64
+}
+
+# Install FEXInterpreter hardlink
+# Needs to be done before setting up binfmt_misc
+ln -f /usr/bin/FEXLoader /usr/bin/FEXInterpreter
+
+if [ $(uname -m) = 'aarch64' ]; then
+  update_binfmt
+fi

CPack/prerm

@@ -0,0 +1,17 @@
+#!/bin/sh
+set -e
+update_binfmt() {
+  # Check for update-binfmts
+  command -v update-binfmts >/dev/null || return 0
+
+  # Uninstall
+  update-binfmts --unimport FEX-x86
+  update-binfmts --unimport FEX-x86_64
+}
+
+if [ $(uname -m) = 'aarch64' ]; then
+  update_binfmt
+fi
+
+# Remove FEXInterpreter hardlink
+unlink /usr/bin/FEXInterpreter

Data/binfmts/CMakeLists.txt

@@ -0,0 +1,4 @@
+install(FILES FEX-x86
+  DESTINATION ${CMAKE_INSTALL_PREFIX}/share/binfmts/)
+install(FILES FEX-x86_64
+  DESTINATION ${CMAKE_INSTALL_PREFIX}/share/binfmts/)

Data/binfmts/FEX-x86

@@ -0,0 +1,9 @@
+package fex
+interpreter /usr/bin/FEXInterpreter
+magic \x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00
+offset 0
+mask \xff\xff\xff\xff\xff\xfe\xfe\x00\x00\x00\x00\xff\xff\xff\xff\xff\xfe\xff\xff\xff
+credentials yes
+fix_binary yes
+preserve no
+

Data/binfmts/FEX-x86_64

@@ -0,0 +1,8 @@
+package fex
+interpreter /usr/bin/FEXInterpreter
+magic \x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00
+offset 0
+mask \xff\xff\xff\xff\xff\xfe\xfe\x00\x00\x00\x00\xff\xff\xff\xff\xff\xfe\xff\xff\xff
+credentials yes
+fix_binary yes
+preserve no

Dockerfile

@@ -3,7 +3,7 @@ FROM ubuntu:20.04 as builder
 
 RUN DEBIAN_FRONTEND="noninteractive" apt-get update
 RUN DEBIAN_FRONTEND="noninteractive" apt install -y cmake \
-clang-10 llvm-10 nasm ninja-build libnuma-dev \
+clang-10 llvm-10 nasm ninja-build \
 libcap-dev libglfw3-dev libepoxy-dev python3-dev \
 python3 linux-headers-generic
 
@@ -23,7 +23,7 @@ FROM ubuntu:20.04
 
 RUN DEBIAN_FRONTEND="noninteractive" apt-get update
 RUN DEBIAN_FRONTEND="noninteractive" apt install -y \
-libnuma-dev libcap-dev libglfw3-dev libepoxy-dev
+libcap-dev libglfw3-dev libepoxy-dev
 
 COPY --from=builder /opt/FEX/build/Bin/* /usr/bin/
 

External/FEXCore/Source/CMakeLists.txt

@@ -124,8 +124,6 @@ set (SRCS
   Interface/IR/Passes/SyscallOptimization.cpp
   Utils/Allocator.cpp
   Utils/Allocator/64BitAllocator.cpp
-  Utils/ELFContainer.cpp
-  Utils/ELFSymbolDatabase.cpp
   Utils/LogManager.cpp
   Utils/Threads.cpp
   )
@@ -286,6 +284,8 @@ function(AddObject Name Type)
   target_compile_options(${Name}
     PRIVATE
     -Wall
+    -Werror=cast-qual
+    -Werror=ignored-qualifiers
     -Werror=implicit-fallthrough
 
     -Wno-trigraphs

External/FEXCore/Source/Common/SoftFloat.h

@@ -1,4 +1,6 @@
 #pragma once
+
+#include <FEXCore/Utils/BitUtils.h>
 #include <FEXCore/Utils/LogManager.h>
 
 #include <cmath>
@@ -158,18 +160,18 @@ struct X80SoftFloat {
   }
 
   operator float() const {
-    float32_t Result = extF80_to_f32(*this);
-    return *(float*)&Result;
+    const float32_t Result = extF80_to_f32(*this);
+    return FEXCore::BitCast<float>(Result);
   }
 
   operator double() const {
-    float64_t Result = extF80_to_f64(*this);
-    return *(double*)&Result;
+    const float64_t Result = extF80_to_f64(*this);
+    return FEXCore::BitCast<double>(Result);
   }
 
   operator BIGFLOAT() const {
-    float128_t Result = extF80_to_f128(*this);
-    return *(BIGFLOAT*)&Result;
+    const float128_t Result = extF80_to_f128(*this);
+    return FEXCore::BitCast<BIGFLOAT>(Result);
   }
 
   operator int16_t() const {
@@ -196,11 +198,11 @@ struct X80SoftFloat {
   }
 
   void operator=(const float rhs) {
-    *this = f32_to_extF80(*(float32_t*)&rhs);
+    *this = f32_to_extF80(FEXCore::BitCast<float32_t>(rhs));
   }
 
   void operator=(const double rhs) {
-    *this = f64_to_extF80(*(float64_t*)&rhs);
+    *this = f64_to_extF80(FEXCore::BitCast<float64_t>(rhs));
   }
 
   void operator=(const int16_t rhs) {
@@ -226,15 +228,15 @@ struct X80SoftFloat {
   }
 
   X80SoftFloat(const float rhs) {
-    *this = f32_to_extF80(*(float32_t*)&rhs);
+    *this = f32_to_extF80(FEXCore::BitCast<float32_t>(rhs));
   }
 
   X80SoftFloat(const double rhs) {
-    *this = f64_to_extF80(*(float64_t*)&rhs);
+    *this = f64_to_extF80(FEXCore::BitCast<float64_t>(rhs));
   }
 
   X80SoftFloat(BIGFLOAT rhs) {
-    *this = f128_to_extF80(*(float128_t*)&rhs);
+    *this = f128_to_extF80(FEXCore::BitCast<float128_t>(rhs));
   }
 
   X80SoftFloat(const int16_t rhs) {

External/FEXCore/Source/Interface/Config/Config.cpp

@@ -90,10 +90,21 @@ namespace FEXCore::Config {
         !std::filesystem::create_directories(ConfigFile)) {
       LogMan::Msg::D("Couldn't create config directory: '%s'", ConfigFile.c_str());
       // Let's go local in this case
-      return "./";
+      return "./" + Filename + ".json";
     }
 
-    ConfigFile += "AppConfig/" + Filename + ".json";
+    ConfigFile += "AppConfig/";
+
+    // Attempt to create the local folder if it doesn't exist
+    if (!Global &&
+        !std::filesystem::exists(ConfigFile) &&
+        !std::filesystem::create_directories(ConfigFile)) {
+      LogMan::Msg::D("Couldn't create AppConfig directory: '%s'", ConfigFile.c_str());
+      // Let's go local in this case
+      return "./" + Filename + ".json";
+    }
+
+    ConfigFile += Filename + ".json";
     return ConfigFile;
   }
 

External/FEXCore/Source/Interface/Context/Context.cpp

@@ -37,12 +37,11 @@ namespace FEXCore::Context {
     return CTX->InitCore(Loader);
   }
 
-  void SetExitHandler(FEXCore::Context::Context *CTX,
-      std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)> handler) {
-    CTX->CustomExitHandler = handler;
+  void SetExitHandler(FEXCore::Context::Context *CTX, ExitHandler handler) {
+    CTX->CustomExitHandler = std::move(handler);
   }
 
-  std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)> GetExitHandler(FEXCore::Context::Context *CTX) {
+  ExitHandler GetExitHandler(FEXCore::Context::Context *CTX) {
     return CTX->CustomExitHandler;
   }
 
@@ -105,12 +104,12 @@ namespace FEXCore::Context {
     CTX->HandleCallback(RIP);
   }
 
-  void RegisterHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func) {
-      CTX->RegisterHostSignalHandler(Signal, Func);
+  void RegisterHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func, bool Required) {
+      CTX->RegisterHostSignalHandler(Signal, Func, Required);
   }
 
-  void RegisterFrontendHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func) {
-    CTX->RegisterFrontendHostSignalHandler(Signal, Func);
+  void RegisterFrontendHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func, bool Required) {
+    CTX->RegisterFrontendHostSignalHandler(Signal, Func, Required);
   }
 
   FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Context::Context *CTX, FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) {

External/FEXCore/Source/Interface/Context/Context.h

@@ -147,7 +147,7 @@ namespace FEXCore::Context {
     std::unique_ptr<FEXCore::ThunkHandler> ThunkHandler;
 
     CustomCPUFactoryType CustomCPUFactory;
-    std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)> CustomExitHandler;
+    FEXCore::Context::ExitHandler CustomExitHandler;
 
     struct AOTIRCacheEntry {
       AOTIRInlineIndex *Array;
@@ -200,8 +200,8 @@ namespace FEXCore::Context {
     void StartGdbServer();
     void StopGdbServer();
     void HandleCallback(uint64_t RIP);
-    void RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func);
-    void RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func);
+    void RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required);
+    void RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required);
 
     static void RemoveCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
 
@@ -217,9 +217,28 @@ namespace FEXCore::Context {
     bool GetDebugDataForRIP(uint64_t RIP, FEXCore::Core::DebugData *Data);
     bool FindHostCodeForRIP(uint64_t RIP, uint8_t **Code);
 
-    std::tuple<FEXCore::IR::IRListView *, FEXCore::IR::RegisterAllocationData *, uint64_t, uint64_t, uint64_t, uint64_t> GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
+    struct GenerateIRResult {
+      FEXCore::IR::IRListView* IRList;
+      // User's responsibility to deallocate this.
+      FEXCore::IR::RegisterAllocationData* RAData;
+      uint64_t TotalInstructions;
+      uint64_t TotalInstructionsLength;
+      uint64_t StartAddr;
+      uint64_t Length;
+    };
+    [[nodiscard]] GenerateIRResult GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
 
-    std::tuple<void *, FEXCore::IR::IRListView *, FEXCore::Core::DebugData *, FEXCore::IR::RegisterAllocationData *, bool, uint64_t, uint64_t> CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
+    struct CompileCodeResult {
+      void* CompiledCode;
+      FEXCore::IR::IRListView* IRData;
+      FEXCore::Core::DebugData* DebugData;
+      // User's responsibility to deallocate this.
+      FEXCore::IR::RegisterAllocationData* RAData;
+      bool GeneratedIR;
+      uint64_t StartAddr;
+      uint64_t Length;
+    };
+    [[nodiscard]] CompileCodeResult CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
     uintptr_t CompileBlock(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP);
     
     // same as CompileBlock, but aborts on failure
@@ -240,7 +259,9 @@ namespace FEXCore::Context {
     void DestroyThread(FEXCore::Core::InternalThreadState *Thread);
     void CleanupAfterFork(FEXCore::Core::InternalThreadState *ExceptForThread);
 
-    std::vector<FEXCore::Core::InternalThreadState*> *const GetThreads() { return &Threads; }
+    std::vector<FEXCore::Core::InternalThreadState*>* GetThreads() { return &Threads; }
+
+    uint8_t GetGPRSize() const { return Config.Is64BitMode ? 8 : 4; }
 
     void AddNamedRegion(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename);
     void RemoveNamedRegion(uintptr_t Base, uintptr_t Size);

External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64.cpp

@@ -157,6 +157,13 @@ bool HandleCASPAL(void *_ucontext, void *_info, uint32_t Instr) {
           mcontext->regs[ExpectedReg2] = FailedResult >> 32;
           return true;
         }
+
+        // This happens in the case that between Load and CAS that something has store our desired in to the memory location
+        // This means our CAS fails because what we wanted to store was already stored
+        uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
+        mcontext->regs[ExpectedReg1] = FailedResult & ~0U;
+        mcontext->regs[ExpectedReg2] = FailedResult >> 32;
+        return true;
       }
     }
     else {
@@ -200,20 +207,18 @@ bool HandleCASPAL(void *_ucontext, void *_info, uint32_t Instr) {
           __uint128_t FailedResultOurBits = TmpExpected & Mask;
           __uint128_t FailedResultNotOurBits = TmpExpected & NegMask;
 
-          __uint128_t FailedDesiredOurBits = TmpDesired & Mask;
           __uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;
           if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
             // If the bits changed that weren't part of our regular CAS then we need to try again
             continue;
           }
-          if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
-            // If the bits changed that we were wanting to change then we have failed and can return
-            // We need to extract the bits and return them in EXPECTED
-            uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
-            mcontext->regs[ExpectedReg1] = FailedResult & ~0U;
-            mcontext->regs[ExpectedReg2] = FailedResult >> 32;
-            return true;
-          }
+
+          // This happens in the case that between Load and CAS that something has store our desired in to the memory location
+          // This means our CAS fails because what we wanted to store was already stored
+          uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
+          mcontext->regs[ExpectedReg1] = FailedResult & ~0U;
+          mcontext->regs[ExpectedReg2] = FailedResult >> 32;
+          return true;
         }
       }
     }
@@ -221,13 +226,19 @@ bool HandleCASPAL(void *_ucontext, void *_info, uint32_t Instr) {
   return false;
 }
 
+template <typename T>
+using CASExpectedFn = T (*)(T Src, T Expected);
+template <typename T>
+using CASDesiredFn = T (*)(T Src, T Desired);
+
+template<bool Retry>
 static
-std::tuple<uint16_t, bool> DoCAS16(
+uint16_t DoCAS16(
   uint16_t DesiredSrc,
   uint16_t ExpectedSrc,
   uint64_t Addr,
-  std::function<uint16_t(uint16_t SrcVal, uint16_t Expected)> ExpectedFunction,
-  std::function<uint16_t(uint16_t SrcVal, uint16_t Desired)> DesiredFunction) {
+  CASExpectedFn<uint16_t> ExpectedFunction,
+  CASDesiredFn<uint16_t> DesiredFunction) {
   // 16 bit
   uint64_t AlignmentMask = 0b1111;
   if ((Addr & AlignmentMask) == 15) {
@@ -235,49 +246,66 @@ std::tuple<uint16_t, bool> DoCAS16(
     // Need a dual 8bit CAS loop
     uint64_t AddrUpper = Addr + 1;
 
-    uint8_t ActualUpper{};
-    uint8_t ActualLower{};
-    // Careful ordering here
-    ActualUpper = LoadAcquire8(AddrUpper);
-    ActualLower = LoadAcquire8(Addr);
+    while (1) {
+      uint8_t ActualUpper{};
+      uint8_t ActualLower{};
+      // Careful ordering here
+      ActualUpper = LoadAcquire8(AddrUpper);
+      ActualLower = LoadAcquire8(Addr);
 
-    uint16_t Actual = ActualUpper;
-    Actual <<= 8;
-    Actual |= ActualLower;
+      uint16_t Actual = ActualUpper;
+      Actual <<= 8;
+      Actual |= ActualLower;
 
-    uint16_t Desired = DesiredFunction(Actual, DesiredSrc);
-    uint8_t DesiredLower = Desired;
-    uint8_t DesiredUpper = Desired >> 8;
+      uint16_t Desired = DesiredFunction(Actual, DesiredSrc);
+      uint8_t DesiredLower = Desired;
+      uint8_t DesiredUpper = Desired >> 8;
 
-    uint16_t Expected = ExpectedFunction(Actual, ExpectedSrc);
-    uint8_t ExpectedLower = Expected;
-    uint8_t ExpectedUpper = Expected >> 8;
+      uint16_t Expected = ExpectedFunction(Actual, ExpectedSrc);
+      uint8_t ExpectedLower = Expected;
+      uint8_t ExpectedUpper = Expected >> 8;
 
-    if (ActualUpper == ExpectedUpper &&
-        ActualLower == ExpectedLower) {
-      if (StoreCAS8(ExpectedUpper, DesiredUpper, AddrUpper)) {
-        if (StoreCAS8(ExpectedLower, DesiredLower, Addr)) {
-          // Stored successfully
-          return std::make_tuple(Expected, true);
-        }
-        else {
-          // CAS managed to tear, we can't really solve this
-          // Continue down the path to let the guest know values weren't expected
+      bool Tear = false;
+      if (ActualUpper == ExpectedUpper &&
+          ActualLower == ExpectedLower) {
+        if (StoreCAS8(ExpectedUpper, DesiredUpper, AddrUpper)) {
+          if (StoreCAS8(ExpectedLower, DesiredLower, Addr)) {
+            // Stored successfully
+            return Expected;
+          }
+          else {
+            // CAS managed to tear, we can't really solve this
+            // Continue down the path to let the guest know values weren't expected
+            Tear = true;
+          }
         }
+
+        ActualLower = ExpectedLower;
+        ActualUpper = ExpectedUpper;
       }
 
-      ActualLower = ExpectedLower;
-      ActualUpper = ExpectedUpper;
+      // If the bits changed that we were wanting to change then we have failed and can return
+      // We need to extract the bits and return them in EXPECTED
+      uint16_t FailedResult = ActualUpper;
+      FailedResult <<= 8;
+      FailedResult |= ActualLower;
+
+      if constexpr (Retry) {
+        if (Tear) {
+          // If we are retrying and tearing then we can't do anything here
+          // XXX: Resolve with TME
+          return FailedResult;
+        }
+        else {
+          // We can retry safely
+        }
+      }
+      else {
+        // Without Retry (CAS) then we have failed regardless of tear
+        // CAS failed but handled successfully
+        return FailedResult;
+      }
     }
-
-    // If the bits changed that we were wanting to change then we have failed and can return
-    // We need to extract the bits and return them in EXPECTED
-    uint16_t FailedResult = ActualUpper;
-    FailedResult <<= 8;
-    FailedResult |= ActualLower;
-
-    // CAS failed but handled successfully
-    return std::make_tuple(FailedResult, false);
   }
   else {
     AlignmentMask = 0b111;
@@ -316,28 +344,30 @@ std::tuple<uint16_t, bool> DoCAS16(
         bool CASResult = Atomic128->compare_exchange_strong(TmpExpected, TmpDesired);
         if (CASResult) {
           // Successful, so we are done
-          return std::make_tuple(Expected >> (Alignment * 8), true);
+          return Expected >> (Alignment * 8);
         }
         else {
+          if constexpr (Retry) {
+            // If we failed but we have enabled retry then just retry without checking results
+            // CAS can't retry but atomic memory ops need to retry until passing
+            continue;
+          }
           // Not successful
           // Now we need to check the results to see if we need to try again
           __uint128_t FailedResultOurBits = TmpExpected & Mask;
           __uint128_t FailedResultNotOurBits = TmpExpected & NegMask;
 
-          __uint128_t FailedDesiredOurBits = TmpDesired & Mask;
           __uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;
           if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
             // If the bits changed that weren't part of our regular CAS then we need to try again
             continue;
           }
-          if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
-            // If the bits changed that we were wanting to change then we have failed and can return
-            // We need to extract the bits and return them in EXPECTED
-            uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);
-            LogMan::Msg::D("Expected 0x%04x, Desired 0x%04x, Result 0x%04x", (uint16_t)(Expected >> (Alignment * 8)), DesiredSrc, FailedResult);
-            // CAS failed but handled successfully
-            return std::make_tuple(FailedResult, false);
-          }
+
+          // This happens in the case that between Load and CAS that something has store our desired in to the memory location
+          // This means our CAS fails because what we wanted to store was already stored
+          uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);
+          // CAS failed but handled successfully
+          return FailedResult;
         }
       }
     }
@@ -379,28 +409,31 @@ std::tuple<uint16_t, bool> DoCAS16(
           bool CASResult = Atomic->compare_exchange_strong(TmpExpected, TmpDesired);
           if (CASResult) {
             // Successful, so we are done
-            return std::make_tuple(Expected >> (Alignment * 8), true);
+            return Expected >> (Alignment * 8);
           }
           else {
+            if constexpr (Retry) {
+              // If we failed but we have enabled retry then just retry without checking results
+              // CAS can't retry but atomic memory ops need to retry until passing
+              continue;
+            }
             // Not successful
             // Now we need to check the results to see if we can try again
             uint64_t FailedResultOurBits = TmpExpected & Mask;
             uint64_t FailedResultNotOurBits = TmpExpected & NegMask;
 
-            uint64_t FailedDesiredOurBits = TmpDesired & Mask;
             uint64_t FailedDesiredNotOurBits = TmpDesired & NegMask;
 
             if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
               // If the bits changed that weren't part of our regular CAS then we need to try again
               continue;
             }
-            if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
-              // If the bits changed that we were wanting to change then we have failed and can return
-              // We need to extract the bits and return them in EXPECTED
-              uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);
-              // CAS failed but handled successfully
-              return std::make_tuple(FailedResult, false);
-            }
+
+            // This happens in the case that between Load and CAS that something has store our desired in to the memory location
+            // This means our CAS fails because what we wanted to store was already stored
+            uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);
+            // CAS failed but handled successfully
+            return FailedResult;
           }
         }
       }
@@ -442,28 +475,31 @@ std::tuple<uint16_t, bool> DoCAS16(
           bool CASResult = Atomic->compare_exchange_strong(TmpExpected, TmpDesired);
           if (CASResult) {
             // Successful, so we are done
-            return std::make_tuple(Expected >> (Alignment * 8), true);
+            return Expected >> (Alignment * 8);
           }
           else {
+            if constexpr (Retry) {
+              // If we failed but we have enabled retry then just retry without checking results
+              // CAS can't retry but atomic memory ops need to retry until passing
+              continue;
+            }
             // Not successful
             // Now we need to check the results to see if we can try again
             uint32_t FailedResultOurBits = TmpExpected & Mask;
             uint32_t FailedResultNotOurBits = TmpExpected & NegMask;
 
-            uint32_t FailedDesiredOurBits = TmpDesired & Mask;
             uint32_t FailedDesiredNotOurBits = TmpDesired & NegMask;
 
             if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
               // If the bits changed that weren't part of our regular CAS then we need to try again
               continue;
             }
-            if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
-              // If the bits changed that we were wanting to change then we have failed and can return
-              // We need to extract the bits and return them in EXPECTED
-              uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);
-              // CAS failed but handled successfully
-              return std::make_tuple(FailedResult, false);
-            }
+
+            // This happens in the case that between Load and CAS that something has store our desired in to the memory location
+            // This means our CAS fails because what we wanted to store was already stored
+            uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);
+            // CAS failed but handled successfully
+            return FailedResult;
           }
         }
       }
@@ -471,13 +507,14 @@ std::tuple<uint16_t, bool> DoCAS16(
   }
 }
 
+template<bool Retry>
 static
-std::tuple<uint32_t, bool> DoCAS32(
+uint32_t DoCAS32(
   uint32_t DesiredSrc,
   uint32_t ExpectedSrc,
   uint64_t Addr,
-  std::function<uint32_t(uint32_t SrcVal, uint32_t Expected)> ExpectedFunction,
-  std::function<uint32_t(uint32_t SrcVal, uint32_t Desired)> DesiredFunction) {
+  CASExpectedFn<uint32_t> ExpectedFunction,
+  CASDesiredFn<uint32_t> DesiredFunction) {
   // 32 bit
   uint64_t AlignmentMask = 0b1111;
   if ((Addr & AlignmentMask) > 12) {
@@ -509,6 +546,7 @@ std::tuple<uint32_t, bool> DoCAS32(
       TmpDesired &= NegMask;
       TmpDesired |= Desired << (Alignment * 8);
 
+      bool Tear = false;
       if (TmpExpected == TmpActual) {
         uint32_t TmpExpectedLower = TmpExpected;
         uint32_t TmpExpectedUpper = TmpExpected >> 32;
@@ -519,11 +557,12 @@ std::tuple<uint32_t, bool> DoCAS32(
         if (StoreCAS32(TmpExpectedUpper, TmpDesiredUpper, AddrUpper)) {
           if (StoreCAS32(TmpExpectedLower, TmpDesiredLower, Addr)) {
             // Stored successfully
-            return std::make_tuple(Expected, true);
+            return Expected;
           }
           else {
             // CAS managed to tear, we can't really solve this
             // Continue down the path to let the guest know values weren't expected
+            Tear = true;
           }
         }
 
@@ -541,18 +580,30 @@ std::tuple<uint32_t, bool> DoCAS32(
       uint64_t FailedResultOurBits = TmpExpected & Mask;
       uint64_t FailedResultNotOurBits = TmpExpected & NegMask;
 
-      uint64_t FailedDesiredOurBits = TmpDesired & Mask;
       uint64_t FailedDesiredNotOurBits = TmpDesired & NegMask;
       if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
         // If the bits changed that weren't part of our regular CAS then we need to try again
         continue;
       }
-      if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
-        // If the bits changed that we were wanting to change then we have failed and can return
-        // We need to extract the bits and return them in EXPECTED
-        uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);
+
+      // This happens in the case that between Load and CAS that something has store our desired in to the memory location
+      // This means our CAS fails because what we wanted to store was already stored
+      uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);
+
+      if constexpr (Retry) {
+        if (Tear) {
+          // If we are retrying and tearing then we can't do anything here
+          // XXX: Resolve with TME
+          return FailedResult;
+        }
+        else {
+          // We can retry safely
+        }
+      }
+      else {
+        // Without Retry (CAS) then we have failed regardless of tear
         // CAS failed but handled successfully
-        return std::make_tuple(FailedResult, false);
+        return FailedResult;
       }
     }
   }
@@ -591,27 +642,31 @@ std::tuple<uint32_t, bool> DoCAS32(
         bool CASResult = Atomic128->compare_exchange_strong(TmpExpected, TmpDesired);
         if (CASResult) {
           // Stored successfully
-          return std::make_tuple(Expected, true);
+          return Expected;
         }
         else {
+          if constexpr (Retry) {
+            // If we failed but we have enabled retry then just retry without checking results
+            // CAS can't retry but atomic memory ops need to retry until passing
+            continue;
+          }
+
           // Not successful
           // Now we need to check the results to see if we need to try again
           __uint128_t FailedResultOurBits = TmpExpected & Mask;
           __uint128_t FailedResultNotOurBits = TmpExpected & NegMask;
 
-          __uint128_t FailedDesiredOurBits = TmpDesired & Mask;
           __uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;
           if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
             // If the bits changed that weren't part of our regular CAS then we need to try again
             continue;
           }
-          if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
-            // If the bits changed that we were wanting to change then we have failed and can return
-            // We need to extract the bits and return them in EXPECTED
-            uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);
-            // CAS failed but handled successfully
-            return std::make_tuple(FailedResult, false);
-          }
+
+          // This happens in the case that between Load and CAS that something has store our desired in to the memory location
+          // This means our CAS fails because what we wanted to store was already stored
+          uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);
+          // CAS failed but handled successfully
+          return FailedResult;
         }
       }
     }
@@ -650,41 +705,46 @@ std::tuple<uint32_t, bool> DoCAS32(
         bool CASResult = Atomic->compare_exchange_strong(TmpExpected, TmpDesired);
         if (CASResult) {
           // Stored successfully
-          return std::make_tuple(Expected, true);
+          return Expected;
         }
         else {
+          if constexpr (Retry) {
+            // If we failed but we have enabled retry then just retry without checking results
+            // CAS can't retry but atomic memory ops need to retry until passing
+            continue;
+          }
+
           // Not successful
           // Now we need to check the results to see if we can try again
           uint64_t FailedResultOurBits = TmpExpected & Mask;
           uint64_t FailedResultNotOurBits = TmpExpected & NegMask;
 
-          uint64_t FailedDesiredOurBits = TmpDesired & Mask;
           uint64_t FailedDesiredNotOurBits = TmpDesired & NegMask;
 
           if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
             // If the bits changed that weren't part of our regular CAS then we need to try again
             continue;
           }
-          if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
-            // If the bits changed that we were wanting to change then we have failed and can return
-            // We need to extract the bits and return them in EXPECTED
-            uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);
-            // CAS failed but handled successfully
-            return std::make_tuple(FailedResult, false);
-          }
+
+          // This happens in the case that between Load and CAS that something has store our desired in to the memory location
+          // This means our CAS fails because what we wanted to store was already stored
+          uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);
+          // CAS failed but handled successfully
+          return FailedResult;
         }
       }
     }
   }
 }
 
+template<bool Retry>
 static
-std::tuple<uint64_t, bool> DoCAS64(
+uint64_t DoCAS64(
   uint64_t DesiredSrc,
   uint64_t ExpectedSrc,
   uint64_t Addr,
-  std::function<uint64_t(uint64_t SrcVal, uint64_t Expected)> ExpectedFunction,
-  std::function<uint64_t(uint64_t SrcVal, uint64_t Desired)> DesiredFunction) {
+  CASExpectedFn<uint64_t> ExpectedFunction,
+  CASDesiredFn<uint64_t> DesiredFunction) {
   // 64bit
   uint64_t AlignmentMask = 0b1111;
   if ((Addr & AlignmentMask) > 8) {
@@ -724,15 +784,17 @@ std::tuple<uint64_t, bool> DoCAS64(
       uint64_t TmpDesiredLower = TmpDesired;
       uint64_t TmpDesiredUpper = TmpDesired >> 64;
 
+      bool Tear = false;
       if (TmpExpected == TmpActual) {
         if (StoreCAS64(TmpExpectedUpper, TmpDesiredUpper, AddrUpper)) {
           if (StoreCAS64(TmpExpectedLower, TmpDesiredLower, Addr)) {
             // Stored successfully
-            return std::make_tuple(Expected, true);
+            return Expected;
           }
           else {
             // CAS managed to tear, we can't really solve this
             // Continue down the path to let the guest know values weren't expected
+            Tear = true;
           }
         }
 
@@ -750,18 +812,30 @@ std::tuple<uint64_t, bool> DoCAS64(
       __uint128_t FailedResultOurBits = TmpExpected & Mask;
       __uint128_t FailedResultNotOurBits = TmpExpected & NegMask;
 
-      __uint128_t FailedDesiredOurBits = TmpDesired & Mask;
       __uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;
       if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
         // If the bits changed that weren't part of our regular CAS then we need to try again
         continue;
       }
-      if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
-        // If the bits changed that we were wanting to change then we have failed and can return
-        // We need to extract the bits and return them in EXPECTED
-        uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
+
+      // This happens in the case that between Load and CAS that something has store our desired in to the memory location
+      // This means our CAS fails because what we wanted to store was already stored
+      uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
+
+      if constexpr (Retry) {
+        if (Tear) {
+          // If we are retrying and tearing then we can't do anything here
+          // XXX: Resolve with TME
+          return FailedResult;
+        }
+        else {
+          // We can retry safely
+        }
+      }
+      else {
+        // Without Retry (CAS) then we have failed regardless of tear
         // CAS failed but handled successfully
-        return std::make_tuple(FailedResult, false);
+        return FailedResult;
       }
     }
   }
@@ -796,35 +870,34 @@ std::tuple<uint64_t, bool> DoCAS64(
       bool CASResult = Atomic128->compare_exchange_strong(TmpExpected, TmpDesired);
       if (CASResult) {
         // Stored successfully
-        return std::make_tuple(Expected, true);
+        return Expected;
       }
       else {
+        if constexpr (Retry) {
+          // If we failed but we have enabled retry then just retry without checking results
+          // CAS can't retry but atomic memory ops need to retry until passing
+          continue;
+        }
+
         // Not successful
         // Now we need to check the results to see if we need to try again
         __uint128_t FailedResultOurBits = TmpExpected & Mask;
         __uint128_t FailedResultNotOurBits = TmpExpected & NegMask;
 
-        __uint128_t FailedDesiredOurBits = TmpDesired & Mask;
         __uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;
         if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
           // If the bits changed that weren't part of our regular CAS then we need to try again
           continue;
         }
-        if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
-          // If the bits changed that we were wanting to change then we have failed and can return
-          // We need to extract the bits and return them in EXPECTED
-          uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
-          // CAS failed but handled successfully
-          return std::make_tuple(FailedResult, false);
-        }
 
-        // If we got here, that means the CAS failed
-        // NotOurBits didn't change and bits we cared about didn't change
-        ERROR_AND_DIE("Impossible");
+        // This happens in the case that between Load and CAS that something has store our desired in to the memory location
+        // This means our CAS fails because what we wanted to store was already stored
+        uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
+        // CAS failed but handled successfully
+        return FailedResult;
       }
     }
   }
-
 }
 
 bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr) {
@@ -855,7 +928,7 @@ bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr) {
   // 8bit can't be unaligned
   // Only need to handle 16, 32, 64
   if (Size == 2) {
-    auto Res = DoCAS16(
+    auto Res = DoCAS16<false>(
       mcontext->regs[DesiredReg],
       mcontext->regs[ExpectedReg],
       Addr,
@@ -871,12 +944,12 @@ bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr) {
     // Regardless of pass or fail
     // We set the result register if it isn't a zero register
     if (ExpectedReg != 31) {
-      mcontext->regs[ExpectedReg] = std::get<0>(Res);
+      mcontext->regs[ExpectedReg] = Res;
     }
     return true;
   }
   else if (Size == 4) {
-    auto Res = DoCAS32(
+    auto Res = DoCAS32<false>(
       mcontext->regs[DesiredReg],
       mcontext->regs[ExpectedReg],
       Addr,
@@ -892,12 +965,12 @@ bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr) {
     // Regardless of pass or fail
     // We set the result register if it isn't a zero register
     if (ExpectedReg != 31) {
-      mcontext->regs[ExpectedReg] = std::get<0>(Res);
+      mcontext->regs[ExpectedReg] = Res;
     }
     return true;
   }
   else if (Size == 8) {
-    auto Res = DoCAS64(
+    auto Res = DoCAS64<false>(
       mcontext->regs[DesiredReg],
       mcontext->regs[ExpectedReg],
       Addr,
@@ -913,7 +986,7 @@ bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr) {
     // Regardless of pass or fail
     // We set the result register if it isn't a zero register
     if (ExpectedReg != 31) {
-      mcontext->regs[ExpectedReg] = std::get<0>(Res);
+      mcontext->regs[ExpectedReg] = Res;
     }
     return true;
   }
@@ -964,7 +1037,7 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
       return Desired;
     };
 
-    std::function<uint16_t(uint16_t SrcVal, uint16_t Desired)> DesiredFunction;
+    CASDesiredFn<uint16_t> DesiredFunction{};
 
     switch (Op) {
       case ATOMIC_ADD_OP:
@@ -988,21 +1061,16 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
         break;
     }
 
-    bool Passed = false;
-    while (!Passed) {
-      auto Res = DoCAS16(
-        mcontext->regs[SourceReg],
-        0, // Unused
-        Addr,
-        NOPExpected,
-        DesiredFunction);
-      Passed = std::get<1>(Res);
-      // If we passed and our destination register is not zero
-      // Then we need to update the result register with what was in memory
-      if (Passed &&
-          ResultReg != 31) {
-        mcontext->regs[ResultReg] = std::get<0>(Res);
-      }
+    auto Res = DoCAS16<true>(
+      mcontext->regs[SourceReg],
+      0, // Unused
+      Addr,
+      NOPExpected,
+      DesiredFunction);
+    // If we passed and our destination register is not zero
+    // Then we need to update the result register with what was in memory
+    if (ResultReg != 31) {
+      mcontext->regs[ResultReg] = Res;
     }
     return true;
   }
@@ -1031,7 +1099,7 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
       return Desired;
     };
 
-    std::function<uint32_t(uint32_t SrcVal, uint32_t Desired)> DesiredFunction;
+    CASDesiredFn<uint32_t> DesiredFunction{};
 
     switch (Op) {
       case ATOMIC_ADD_OP:
@@ -1055,21 +1123,16 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
         break;
     }
 
-    bool Passed = false;
-    while (!Passed) {
-      auto Res = DoCAS32(
-        mcontext->regs[SourceReg],
-        0, // Unused
-        Addr,
-        NOPExpected,
-        DesiredFunction);
-      Passed = std::get<1>(Res);
-      // If we passed and our destination register is not zero
-      // Then we need to update the result register with what was in memory
-      if (Passed &&
-          ResultReg != 31) {
-        mcontext->regs[ResultReg] = std::get<0>(Res);
-      }
+    auto Res = DoCAS32<true>(
+      mcontext->regs[SourceReg],
+      0, // Unused
+      Addr,
+      NOPExpected,
+      DesiredFunction);
+    // If we passed and our destination register is not zero
+    // Then we need to update the result register with what was in memory
+    if (ResultReg != 31) {
+      mcontext->regs[ResultReg] = Res;
     }
     return true;
   }
@@ -1098,7 +1161,7 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
       return Desired;
     };
 
-    std::function<uint64_t(uint64_t SrcVal, uint64_t Desired)> DesiredFunction;
+    CASDesiredFn<uint64_t> DesiredFunction{};
 
     switch (Op) {
       case ATOMIC_ADD_OP:
@@ -1122,21 +1185,16 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
         break;
     }
 
-    bool Passed = false;
-    while (!Passed) {
-      auto Res = DoCAS64(
-        mcontext->regs[SourceReg],
-        0, // Unused
-        Addr,
-        NOPExpected,
-        DesiredFunction);
-      Passed = std::get<1>(Res);
-      // If we passed and our destination register is not zero
-      // Then we need to update the result register with what was in memory
-      if (Passed &&
-          ResultReg != 31) {
-        mcontext->regs[ResultReg] = std::get<0>(Res);
-      }
+    auto Res = DoCAS64<true>(
+      mcontext->regs[SourceReg],
+      0, // Unused
+      Addr,
+      NOPExpected,
+      DesiredFunction);
+    // If we passed and our destination register is not zero
+    // Then we need to update the result register with what was in memory
+    if (ResultReg != 31) {
+      mcontext->regs[ResultReg] = Res;
     }
     return true;
   }

External/FEXCore/Source/Interface/Core/CPUID.cpp

@@ -15,7 +15,26 @@ $end_info$
 #endif
 
 namespace FEXCore {
-//#define CPUID_AMD
+constexpr uint32_t SUPPORTS_AVX = 0;
+// #define CPUID_AMD
+#ifdef CPUID_AMD
+constexpr uint32_t FAMILY_IDENTIFIER =
+  0 |          // Stepping
+  (0xA << 4) | // Model
+  (0xF << 8) | // Family ID
+  (0 << 12) |  // Processor type
+  (0 << 16) |  // Extended model ID
+  (1 << 20);   // Extended family ID
+#else
+constexpr uint32_t FAMILY_IDENTIFIER =
+  0 |          // Stepping
+  (0x7 << 4) |   // Model
+  (0x6 << 8) | // Family ID
+  (0 << 12) |  // Processor type
+  (1 << 16) |  // Extended model ID
+  (0x0 << 20);   // Extended family ID
+#endif
+
 #ifdef _M_ARM_64
 static uint32_t GetCycleCounterFrequency() {
   uint64_t Result{};
@@ -38,7 +57,7 @@ static uint32_t GetCycleCounterFrequency() {
 }
 #endif
 
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_0h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_0h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
 
   // EBX, EDX, ECX become the manufacturer id string
@@ -57,25 +76,23 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_0h() {
 }
 
 // Processor Info and Features bits
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
+  uint32_t CoreCount = Cores();
+
+  Res.eax = FAMILY_IDENTIFIER;
 
-  Res.eax = 0 | // Stepping
-    (0 << 4) | // Model
-    (0xF << 8) | // Family ID
-    (0 << 12) | // Processor type
-    (0 << 16) | // Extended model ID
-    (0 << 20); // Extended family ID
   Res.ebx = 0 | // Brand index
     (8 << 8) | // Cache line size in bytes
-    (8 << 16) | // Number of addressable IDs for the logical cores in the physical CPU
+    (CoreCount << 16) | // Number of addressable IDs for the logical cores in the physical CPU
     (0 << 24); // Local APIC ID
+
   Res.ecx =
     (1 <<  0) | // SSE3
     (0 <<  1) | // PCLMULQDQ
     (1 <<  2) | // DS area supports 64bit layout
     (1 <<  3) | // MWait
-    (1 <<  4) | // DS-CPL
+    (0 <<  4) | // DS-CPL
     (0 <<  5) | // VMX
     (0 <<  6) | // SMX
     (0 <<  7) | // Intel SpeedStep
@@ -89,7 +106,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h() {
     (0 << 15) | // Perfmon and debug capability
     (0 << 16) | // Reserved
     (0 << 17) | // Process-context identifiers
-    (1 << 18) | // Prefetching from memory mapped device
+    (0 << 18) | // Prefetching from memory mapped device
     (1 << 19) | // SSE4.1
     (0 << 20) | // SSE4.2
     (0 << 21) | // X2APIC
@@ -99,7 +116,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h() {
     (CTX->HostFeatures.SupportsAES << 25) | // AES
     (0 << 26) | // XSAVE
     (0 << 27) | // OSXSAVE
-    (0 << 28) | // AVX
+    (SUPPORTS_AVX << 28) | // AVX
     (0 << 29) | // F16C
     (0 << 30) | // RDRAND
     (0 << 31);  // Hypervisor always returns zero
@@ -132,16 +149,16 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h() {
     (1 << 24) | // FXSAVE/FXRSTOR
     (1 << 25) | // SSE
     (1 << 26) | // SSE2
-    (1 << 27) | // Self Snoop
+    (0 << 27) | // Self Snoop
     (1 << 28) | // Max APIC IDs reserved field is valid
-    (1 << 29) | // Thermal monitor
+    (0 << 29) | // Thermal monitor
     (0 << 30) | // Reserved
-    (1 << 31);  // Pending break enable
+    (0 << 31);  // Pending break enable
   return Res;
 }
 
 // 2: Cache and TLB information
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_02h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_02h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
 
   // returns default values from i7 model 1Ah
@@ -165,124 +182,286 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_02h() {
   return Res;
 }
 
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_06h() {
+// 4: Deterministic cache parameters for each level
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_04h(uint32_t Leaf) {
+  FEXCore::CPUID::FunctionResults Res{};
+  constexpr uint32_t CacheType_Data = 1;
+  constexpr uint32_t CacheType_Instruction = 2;
+  constexpr uint32_t CacheType_Unified = 3;
+
+  if (Leaf == 0) {
+    // Report L1D
+    uint32_t CoreCount = Cores() - 1;
+
+    Res.eax = CacheType_Data | // Cache type
+      (0b001 << 5) |      // Cache level
+      (1 << 8)  |         // Self initializing cache level
+      (0 << 9)  |         // Fully associative
+      (0 << 14) |         // Maximum number of addressable IDs for logical processors sharing this cache (With SMT this would be 1)
+      (CoreCount << 26);  // Maximum number of addressable IDs for processor cores in the physical package
+
+    Res.ebx =
+      (63 << 0) | // Line Size - 1 : Claiming 64 byte
+      (0 << 12) | // Physical Line partitions
+      (7 << 22);  // Associativity - 1 : Claiming 8 way
+
+    // 32KB
+    Res.ecx = 63; // Number of sets - 1 : Claiming 64 sets
+
+    Res.edx =
+      (0 << 0) | // Write-back invalidate
+      (0 << 1) | // Cache inclusiveness - Includes lower caches
+      (0 << 2);  // Complex cache indexing - 0: Direct, 1: Complex
+  }
+  else if (Leaf == 1) {
+    // Report L1I
+    uint32_t CoreCount = Cores() - 1;
+
+    Res.eax = CacheType_Instruction | // Cache type
+      (0b001 << 5) |      // Cache level
+      (1 << 8)  |         // Self initializing cache level
+      (0 << 9)  |         // Fully associative
+      (0 << 14) |         // Maximum number of addressable IDs for logical processors sharing this cache (With SMT this would be 1)
+      (CoreCount << 26); // Maximum number of addressable IDs for processor cores in the physical package
+
+    Res.ebx =
+      (63 << 0) | // Line Size - 1 : Claiming 64 byte
+      (0 << 12) | // Physical Line partitions
+      (7 << 22);  // Associativity - 1 : Claiming 8 way
+
+    // 32KB
+    Res.ecx = 63; // Number of sets - 1 : Claiming 64 sets
+
+    Res.edx =
+      (0 << 0) | // Write-back invalidate
+      (0 << 1) | // Cache inclusiveness - Includes lower caches
+      (0 << 2);  // Complex cache indexing - 0: Direct, 1: Complex
+  }
+  else if (Leaf == 2) {
+    // Report L2
+    uint32_t CoreCount = Cores() - 1;
+
+    Res.eax = CacheType_Unified | // Cache type
+      (0b010 << 5) |      // Cache level
+      (1 << 8)  |         // Self initializing cache level
+      (0 << 9)  |         // Fully associative
+      (0 << 14) |         // Maximum number of addressable IDs for logical processors sharing this cache
+      (CoreCount << 26);  // Maximum number of addressable IDs for processor cores in the physical package
+
+    Res.ebx =
+      (63 << 0) | // Line Size - 1 : Claiming 64 byte
+      (0 << 12) | // Physical Line partitions
+      (7 << 22);  // Associativity - 1 : Claiming 8 way
+
+    // 512KB
+    Res.ecx = 0x3FF; // Number of sets - 1 : Claiming 1024 sets
+
+    Res.edx =
+      (0 << 0) | // Write-back invalidate
+      (0 << 1) | // Cache inclusiveness - Includes lower caches
+      (0 << 2);  // Complex cache indexing - 0: Direct, 1: Complex
+  }
+  else if (Leaf == 3) {
+    // Report L3
+    uint32_t CoreCount = Cores() - 1;
+
+    Res.eax = CacheType_Unified | // Cache type
+      (0b011 << 5) |      // Cache level
+      (1 << 8)  |         // Self initializing cache level
+      (0 << 9)  |         // Fully associative
+      (CoreCount << 14) | // Maximum number of addressable IDs for logical processors sharing this cache
+      (CoreCount << 26);  // Maximum number of addressable IDs for processor cores in the physical package
+
+    Res.ebx =
+      (63 << 0) | // Line Size - 1 : Claiming 64 byte
+      (0 << 12) | // Physical Line partitions
+      (7 << 22);  // Associativity - 1 : Claiming 8 way
+
+    // 8MB
+    Res.ecx = 0x4000; // Number of sets - 1 : Claiming 16384 sets
+
+    Res.edx =
+      (0 << 0) | // Write-back invalidate
+      (0 << 1) | // Cache inclusiveness - Includes lower caches
+      (1 << 2);  // Complex cache indexing - 0: Direct, 1: Complex
+  }
+
+  return Res;
+}
+
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_06h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
   Res.eax = (1 << 2); // Always running APIC
   Res.ecx = (0 << 3); // Intel performance energy bias preference (EPB)
   return Res;
 }
 
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
+  if (Leaf == 0) {
+    // Number of subfunctions
+    Res.eax = 0x0;
+    Res.ebx =
+      (1 <<  0) | // FS/GS support
+      (0 <<  1) | // TSC adjust MSR
+      (0 <<  2) | // SGX
+      (0 <<  3) | // BMI1
+      (0 <<  4) | // Intel Hardware Lock Elison
+      (0 <<  5) | // AVX2 support
+      (1 <<  6) | // FPU data pointer updated only on exception
+      (1 <<  7) | // SMEP support
+      (0 <<  8) | // BMI2
+      (0 <<  9) | // Enhanced REP MOVSB/STOSB
+      (1 << 10) | // INVPCID for system software control of process-context
+      (0 << 11) | // Restricted transactional memory
+      (0 << 12) | // Intel resource directory technology Monitoring
+      (1 << 13) | // Deprecates FPU CS and DS
+      (0 << 14) | // Intel MPX
+      (0 << 15) | // Intel Resource Directory Technology Allocation
+      (0 << 16) | // Reserved
+      (0 << 17) | // Reserved
+      (0 << 18) | // RDSEED
+      (0 << 19) | // ADCX and ADOX instructions
+      (0 << 20) | // SMAP Supervisor mode access prevention and CLAC/STAC instructions
+      (0 << 21) | // Reserved
+      (0 << 22) | // Reserved
+      (0 << 23) | // CLFLUSHOPT instruction
+      (0 << 24) | // CLWB instruction
+      (0 << 25) | // Intel processor trace
+      (0 << 26) | // Reserved
+      (0 << 27) | // Reserved
+      (0 << 28) | // Reserved
+      (0 << 29) | // SHA instructions
+      (0 << 30) | // Reserved
+      (0 << 31);  // Reserved
 
-  // Number of subfunctions
-  Res.eax = 0x0;
-  Res.ebx =
-    (1 <<  0) | // FS/GS support
-    (0 <<  1) | // TSC adjust MSR
-    (0 <<  2) | // SGX
-    (0 <<  3) | // BMI1
-    (0 <<  4) | // Intel Hardware Lock Elison
-    (0 <<  5) | // AVX2 support
-    (1 <<  6) | // FPU data pointer updated only on exception
-    (1 <<  7) | // SMEP support
-    (0 <<  8) | // BMI2
-    (0 <<  9) | // Enhanced REP MOVSB/STOSB
-    (1 << 10) | // INVPCID for system software control of process-context
-    (0 << 11) | // Restricted transactional memory
-    (0 << 12) | // Intel resource directory technology Monitoring
-    (1 << 13) | // Deprecates FPU CS and DS
-    (0 << 14) | // Intel MPX
-    (0 << 15) | // Intel Resource Directory Technology Allocation
-    (0 << 16) | // Reserved
-    (0 << 17) | // Reserved
-    (0 << 18) | // RDSEED
-    (0 << 19) | // ADCX and ADOX instructions
-    (0 << 20) | // SMAP Supervisor mode access prevention and CLAC/STAC instructions
-    (0 << 21) | // Reserved
-    (0 << 22) | // Reserved
-    (0 << 23) | // CLFLUSHOPT instruction
-    (0 << 24) | // CLWB instruction
-    (0 << 25) | // Intel processor trace
-    (0 << 26) | // Reserved
-    (0 << 27) | // Reserved
-    (0 << 28) | // Reserved
-    (0 << 29) | // SHA instructions
-    (0 << 30) | // Reserved
-    (0 << 31);  // Reserved
+    Res.ecx =
+      (1 <<  0) | // PREFETCHWT1
+      (0 <<  1) | // AVX512VBMI
+      (0 <<  2) | // Usermode instruction prevention
+      (0 <<  3) | // Protection keys for user mode pages
+      (0 <<  4) | // OS protection keys
+      (0 <<  5) | // waitpkg
+      (0 <<  6) | // AVX512_VBMI2
+      (0 <<  7) | // CET shadow stack
+      (0 <<  8) | // GFNI
+      (0 <<  9) | // VAES
+      (0 << 10) | // VPCLMULQDQ
+      (0 << 11) | // AVX512_VNNI
+      (0 << 12) | // AVX512_BITALG
+      (0 << 13) | // Intel Total Memory Encryption
+      (0 << 14) | // AVX512_VPOPCNTDQ
+      (0 << 15) | // Reserved
+      (0 << 16) | // 5 Level page tables
+      (0 << 17) | // MPX MAWAU
+      (0 << 18) | // MPX MAWAU
+      (0 << 19) | // MPX MAWAU
+      (0 << 20) | // MPX MAWAU
+      (0 << 21) | // MPX MAWAU
+      (0 << 22) | // RDPID Read Processor ID
+      (0 << 23) | // Reserved
+      (0 << 24) | // Reserved
+      (0 << 25) | // CLDEMOTE
+      (0 << 26) | // Reserved
+      (0 << 27) | // MOVDIRI
+      (0 << 28) | // MOVDIR64B
+      (0 << 29) | // Reserved
+      (0 << 30) | // SGX Launch configuration
+      (0 << 31);  // Reserved
 
-  Res.ecx =
-    (1 <<  0) | // PREFETCHWT1
-    (0 <<  1) | // AVX512VBMI
-    (0 <<  2) | // Usermode instruction prevention
-    (0 <<  3) | // Protection keys for user mode pages
-    (1 <<  4) | // OS protection keys
-    (0 <<  5) | // waitpkg
-    (0 <<  6) | // AVX512_VBMI2
-    (0 <<  7) | // CET shadow stack
-    (0 <<  8) | // GFNI
-    (0 <<  9) | // VAES
-    (0 << 10) | // VPCLMULQDQ
-    (0 << 11) | // AVX512_VNNI
-    (0 << 12) | // AVX512_BITALG
-    (0 << 13) | // Intel Total Memory Encryption
-    (0 << 14) | // AVX512_VPOPCNTDQ
-    (0 << 15) | // Reserved
-    (0 << 16) | // 5 Level page tables
-    (0 << 17) | // MPX MAWAU
-    (0 << 18) | // MPX MAWAU
-    (0 << 19) | // MPX MAWAU
-    (0 << 20) | // MPX MAWAU
-    (0 << 21) | // MPX MAWAU
-    (0 << 22) | // RDPID Read Processor ID
-    (0 << 23) | // Reserved
-    (0 << 24) | // Reserved
-    (0 << 25) | // CLDEMOTE
-    (0 << 26) | // Reserved
-    (0 << 27) | // MOVDIRI
-    (0 << 28) | // MOVDIR64B
-    (0 << 29) | // Reserved
-    (0 << 30) | // SGX Launch configuration
-    (0 << 31);  // Reserved
-
-  Res.edx =
-    (0 <<  0) | // Reserved
-    (0 <<  1) | // Reserved
-    (0 <<  2) | // AVX512_4VNNIW
-    (0 <<  3) | // AVX512_4FMAPS
-    (0 <<  4) | // Fast Short Rep Mov
-    (0 <<  5) | // Reserved
-    (0 <<  6) | // Reserved
-    (0 <<  7) | // Reserved
-    (0 <<  8) | // AVX512_VP2INTERSECT
-    (0 <<  9) | // Reserved
-    (0 << 10) | // VERW clears CPU buffers
-    (0 << 11) | // Reserved
-    (0 << 12) | // Reserved
-    (0 << 13) | // Reserved
-    (0 << 14) | // SERIALIZE instruction
-    (0 << 15) | // Reserved
-    (0 << 16) | // Reserved
-    (0 << 17) | // Reserved
-    (0 << 18) | // Intel PCONFIG
-    (0 << 19) | // Intel Architectural LBR
-    (0 << 20) | // Intel CET
-    (0 << 21) | // Reserved
-    (0 << 22) | // Reserved
-    (0 << 23) | // Reserved
-    (0 << 24) | // Reserved
-    (0 << 25) | // Reserved
-    (0 << 26) | // Reserved
-    (0 << 27) | // Reserved
-    (0 << 28) | // L1D Flush
-    (0 << 29) | // Arch capabilities
-    (0 << 30) | // Reserved
-    (0 << 31);  // Reserved
+    Res.edx =
+      (0 <<  0) | // Reserved
+      (0 <<  1) | // Reserved
+      (0 <<  2) | // AVX512_4VNNIW
+      (0 <<  3) | // AVX512_4FMAPS
+      (0 <<  4) | // Fast Short Rep Mov
+      (0 <<  5) | // Reserved
+      (0 <<  6) | // Reserved
+      (0 <<  7) | // Reserved
+      (0 <<  8) | // AVX512_VP2INTERSECT
+      (0 <<  9) | // Reserved
+      (0 << 10) | // VERW clears CPU buffers
+      (0 << 11) | // Reserved
+      (0 << 12) | // Reserved
+      (0 << 13) | // Reserved
+      (0 << 14) | // SERIALIZE instruction
+      (0 << 15) | // Reserved
+      (0 << 16) | // Reserved
+      (0 << 17) | // Reserved
+      (0 << 18) | // Intel PCONFIG
+      (0 << 19) | // Intel Architectural LBR
+      (0 << 20) | // Intel CET
+      (0 << 21) | // Reserved
+      (0 << 22) | // Reserved
+      (0 << 23) | // Reserved
+      (0 << 24) | // Reserved
+      (0 << 25) | // Reserved
+      (0 << 26) | // Reserved
+      (0 << 27) | // Reserved
+      (0 << 28) | // L1D Flush
+      (0 << 29) | // Arch capabilities
+      (0 << 30) | // Reserved
+      (0 << 31);  // Reserved
+  }
 
   return Res;
 }
 
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_15h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_0Dh(uint32_t Leaf) {
+  // Leaf 0
+  FEXCore::CPUID::FunctionResults Res{};
+
+  uint32_t XFeatureSupportedSizeMax = SUPPORTS_AVX ? 0x0000'0340 : 0x0000'0240; // XFeatureEnabledSizeMax: Legacy Header + FPU/SSE + AVX
+  if (Leaf == 0) {
+    // XFeatureSupportedMask[31:0]
+    Res.eax =
+      (1 << 0) |            // X87 support
+      (1 << 1) |            // 128-bit SSE support
+      (SUPPORTS_AVX << 2) | // 256-bit AVX support
+      (0b00 << 3) |         // MPX State
+      (0b000 << 5) |        // AVX-512 state
+      (0 << 8) |            // "Used for IA32_XSS" ... Used for what?
+      (0 << 9);             // PKRU state
+
+    // EBX and ECX doesn't need to match if a feature is supported but not enabled
+    Res.ebx = XFeatureSupportedSizeMax;
+    Res.ecx = XFeatureSupportedSizeMax; // XFeatureSupportedSizeMax: Size in bytes of XSAVE/XRSTOR area
+
+    // XFeatureSupportedMask[63:32]
+    Res.edx = 0; // Upper 32-bits of XFeatureSupportedMask
+  }
+  else if (Leaf == 1) {
+    Res.eax =
+      (0 << 0) | // XSAVEOPT
+      (0 << 1) | // XSAVEC (and XRSTOR)
+      (0 << 2) | // XGETBV - XGETBV with ECX=1 supported
+      (0 << 3);  // XSAVES - XSAVES, XRSTORS, and IA32_XSS supported
+
+    // Same information as Leaf 0 for ebx
+    Res.ebx = XFeatureSupportedSizeMax;
+
+    // Lower supported 32bits of IA32_XSS MSR. IA32_XSS[n] can only be set to 1 if ECX[n] is 1
+    Res.ecx =
+      (0b0000'0000 << 0) | // Used for XCR0
+      (0 << 8) |           // PT state
+      (0 << 9);            // Used for XCR0
+
+    // Upper supported 32bits of IA32_XSS MSR. IA32_XSS[n+32] can only be set to 1 if EDX[n] is 1
+    // Entirely reserved atm
+    Res.edx = 0;
+  }
+  else if (Leaf == 2) {
+    Res.eax = SUPPORTS_AVX ? 0x0000'0100 : 0; // YmmSaveStateSize
+    Res.ebx = SUPPORTS_AVX ? 0x0000'0240 : 0; // YmmSaveStateOffset
+
+    // Reserved
+    Res.ecx = 0;
+    Res.edx = 0;
+  }
+  return Res;
+}
+
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_15h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
   // TSC frequency = ECX * EBX / EAX
   uint32_t FrequencyHz = GetCycleCounterFrequency();
@@ -295,7 +474,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_15h() {
 }
 
 // Highest extended function implemented
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0000h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0000h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
   Res.eax = 0x8000001F;
 
@@ -314,15 +493,10 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0000h() {
 }
 
 // Extended processor and feature bits
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0001h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0001h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
 
-  Res.eax = 0 | // Stepping
-    (0 << 4) | // Model
-    (0 << 8) | // Family ID
-    (0 << 12) | // Processor type
-    (0 << 16) | // Extended model ID
-    (0 << 20); // Extended family ID
+  Res.eax = FAMILY_IDENTIFIER;
 
   Res.ecx =
     (1 <<  0) | // LAHF/SAHF
@@ -347,13 +521,13 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0001h() {
     (0 << 19) | // Reserved
     (0 << 20) | // Reserved
     (0 << 21) | // Reserved
-    (1 << 22) | // Topology extensions support
-    (1 << 23) | // Core performance counter extensions
-    (1 << 24) | // NB performance counter extensions
+    (0 << 22) | // Topology extensions support
+    (0 << 23) | // Core performance counter extensions
+    (0 << 24) | // NB performance counter extensions
     (0 << 25) | // Reserved
     (0 << 26) | // Data breakpoints extensions
-    (1 << 27) | // Performance TSC
-    (1 << 28) | // L2 perf counter extensions
+    (0 << 27) | // Performance TSC
+    (0 << 28) | // L2 perf counter extensions
     (0 << 29) | // Reserved
     (0 << 30) | // Reserved
     (0 << 31);  // Reserved
@@ -385,7 +559,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0001h() {
     (1 << 23) | // MMX
     (1 << 24) | // FXSAVE/FXRSTOR
     (1 << 25) | // FXSAVE/FXRSTOR Optimizations
-    (1 << 26) | // 1 gigabit pages
+    (0 << 26) | // 1 gigabit pages
     (0 << 27) | // RDTSCP
     (0 << 28) | // Reserved
     (1 << 29) | // Long Mode
@@ -400,26 +574,26 @@ constexpr char ProcessorBrand[48] = {
 };
 
 //Processor brand string
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0002h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0002h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
   memcpy(&Res, &ProcessorBrand[0], sizeof(FEXCore::CPUID::FunctionResults));
   return Res;
 }
 
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0003h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0003h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
   memcpy(&Res, &ProcessorBrand[16], sizeof(FEXCore::CPUID::FunctionResults));
   return Res;
 }
 
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0004h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0004h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
   memcpy(&Res, &ProcessorBrand[32], sizeof(FEXCore::CPUID::FunctionResults));
   return Res;
 }
 
 // L1 Cache and TLB identifiers
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0005h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0005h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
 
   // L1 TLB Information for 2MB and 4MB pages
@@ -454,7 +628,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0005h() {
 }
 
 // L2 Cache identifiers
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0006h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0006h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
 
   // L2 TLB Information for 2MB and 4MB pages
@@ -488,7 +662,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0006h() {
 }
 
 // Advanced power management
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0007h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0007h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
   Res.eax = (1 << 2); // APIC timer not affected by p-state
   Res.edx =
@@ -497,7 +671,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0007h() {
 }
 
 // Virtual and physical address sizes
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0008h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0008h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
   Res.eax =
     (48 << 0) | // PhysAddrSize = 48-bit
@@ -512,14 +686,14 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0008h() {
   uint32_t CoreCount = Cores() - 1;
   Res.ecx =
     (0 << 16) |       // PerfTscSize: Performance timestamp count size
-    (0 << 12) |       // ApicIdSize: Number of bits in ApicID
+    ((uint32_t)std::log2(CoreCount + 1) << 12) |       // ApicIdSize: Number of bits in ApicID
     (CoreCount << 0); // Count count subtract one
 
   return Res;
 }
 
 // TLB 1GB page identifiers
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0019h() {
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0019h(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
   Res.eax =
     (0xF << 28) | // L1 DTLB associativity for 1GB pages
@@ -535,27 +709,128 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0019h() {
   return Res;
 }
 
-FEXCore::CPUID::FunctionResults CPUIDEmu::Function_Reserved() {
+// Deterministic cache parameters for each level
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_001Dh(uint32_t Leaf) {
+  // This is nearly a copy of CPUID function 4h
+  // There are some minor changes though
+
+  FEXCore::CPUID::FunctionResults Res{};
+  constexpr uint32_t CacheType_Data = 1;
+  constexpr uint32_t CacheType_Instruction = 2;
+  constexpr uint32_t CacheType_Unified = 3;
+
+  if (Leaf == 0) {
+    // Report L1D
+    Res.eax = CacheType_Data | // Cache type
+      (0b001 << 5) |      // Cache level
+      (1 << 8)  |         // Self initializing cache level
+      (0 << 9)  |         // Fully associative
+      (0 << 14);          // Maximum number of addressable IDs for logical processors sharing this cache (With SMT this would be 1)
+
+    Res.ebx =
+      (63 << 0) | // Line Size - 1 : Claiming 64 byte
+      (0 << 12) | // Physical Line partitions
+      (7 << 22);  // Associativity - 1 : Claiming 8 way
+
+    // 32KB
+    Res.ecx = 63; // Number of sets - 1 : Claiming 64 sets
+
+    Res.edx =
+      (0 << 0) | // Write-back invalidate
+      (0 << 1);  // Cache inclusiveness - Includes lower caches
+  }
+  else if (Leaf == 1) {
+    // Report L1I
+    Res.eax = CacheType_Instruction | // Cache type
+      (0b001 << 5) |      // Cache level
+      (1 << 8)  |         // Self initializing cache level
+      (0 << 9)  |         // Fully associative
+      (0 << 14);          // Maximum number of addressable IDs for logical processors sharing this cache (With SMT this would be 1)
+
+    Res.ebx =
+      (63 << 0) | // Line Size - 1 : Claiming 64 byte
+      (0 << 12) | // Physical Line partitions
+      (7 << 22);  // Associativity - 1 : Claiming 8 way
+
+    // 32KB
+    Res.ecx = 63; // Number of sets - 1 : Claiming 64 sets
+
+    Res.edx =
+      (0 << 0) | // Write-back invalidate
+      (0 << 1);  // Cache inclusiveness - Includes lower caches
+  }
+  else if (Leaf == 2) {
+    // Report L2
+    Res.eax = CacheType_Unified | // Cache type
+      (0b010 << 5) |      // Cache level
+      (1 << 8)  |         // Self initializing cache level
+      (0 << 9)  |         // Fully associative
+      (0 << 14);          // Maximum number of addressable IDs for logical processors sharing this cache
+
+    Res.ebx =
+      (63 << 0) | // Line Size - 1 : Claiming 64 byte
+      (0 << 12) | // Physical Line partitions
+      (7 << 22);  // Associativity - 1 : Claiming 8 way
+
+    // 512KB
+    Res.ecx = 0x3FF; // Number of sets - 1 : Claiming 1024 sets
+
+    Res.edx =
+      (0 << 0) | // Write-back invalidate
+      (0 << 1);  // Cache inclusiveness - Includes lower caches
+  }
+  else if (Leaf == 3) {
+    // Report L3
+    uint32_t CoreCount = Cores() - 1;
+
+    Res.eax = CacheType_Unified | // Cache type
+      (0b011 << 5) |      // Cache level
+      (1 << 8)  |         // Self initializing cache level
+      (0 << 9)  |         // Fully associative
+      (CoreCount << 14);  // Maximum number of addressable IDs for logical processors sharing this cache
+
+    Res.ebx =
+      (63 << 0) | // Line Size - 1 : Claiming 64 byte
+      (0 << 12) | // Physical Line partitions
+      (7 << 22);  // Associativity - 1 : Claiming 8 way
+
+    // 8MB
+    Res.ecx = 0x4000; // Number of sets - 1 : Claiming 16384 sets
+
+    Res.edx =
+      (0 << 0) | // Write-back invalidate
+      (0 << 1);  // Cache inclusiveness - Includes lower caches
+  }
+
+  return Res;
+}
+
+FEXCore::CPUID::FunctionResults CPUIDEmu::Function_Reserved(uint32_t Leaf) {
   FEXCore::CPUID::FunctionResults Res{};
   return Res;
 }
 
 void CPUIDEmu::Init(FEXCore::Context::Context *ctx) {
   CTX = ctx;
-  RegisterFunction(0, std::bind(&CPUIDEmu::Function_0h, this));
-  RegisterFunction(1, std::bind(&CPUIDEmu::Function_01h, this));
-  RegisterFunction(2, std::bind(&CPUIDEmu::Function_02h, this));
+  using namespace std::placeholders;
+  RegisterFunction(0, std::bind(&CPUIDEmu::Function_0h, this, _1));
+  RegisterFunction(1, std::bind(&CPUIDEmu::Function_01h, this, _1));
+  RegisterFunction(2, std::bind(&CPUIDEmu::Function_02h, this, _1));
   // 3: Serial Number(previously), now reserved
-  // 4: Deterministic cache parameters for each level
+#ifndef CPUID_AMD
+  // Deterministic cache parameters for each level
+  RegisterFunction(0x4, std::bind(&CPUIDEmu::Function_04h, this, _1));
+#endif
   // 5: Monitor/mwait
   // Thermal and power management
-  RegisterFunction(6, std::bind(&CPUIDEmu::Function_06h, this));
+  RegisterFunction(6, std::bind(&CPUIDEmu::Function_06h, this, _1));
   // Extended feature flags
-  RegisterFunction(7, std::bind(&CPUIDEmu::Function_07h, this));
+  RegisterFunction(7, std::bind(&CPUIDEmu::Function_07h, this, _1));
   // 9: Direct Cache Access information
   // 0x0A: Architectural performance monitoring
   // 0x0B: Extended topology enumeration
   // 0x0D: Processor extended state enumeration
+  RegisterFunction(0x0D, std::bind(&CPUIDEmu::Function_0Dh, this, _1));
   // 0x0F: Intel RDT monitoring
   // 0x10: Intel RDT allocation enumeration
   // 0x12: Intel SGX capability enumeration
@@ -564,43 +839,47 @@ void CPUIDEmu::Init(FEXCore::Context::Context *ctx) {
 #ifndef CPUID_AMD
   // Timestamp counter information
   // Doesn't exist on AMD hardware
-  RegisterFunction(0x15, std::bind(&CPUIDEmu::Function_15h, this));
+  RegisterFunction(0x15, std::bind(&CPUIDEmu::Function_15h, this, _1));
 #endif
   // 0x16: Processor frequency information
   // 0x17: SoC vendor attribute enumeration
 
   // Largest extended function number
-  RegisterFunction(0x8000'0000, std::bind(&CPUIDEmu::Function_8000_0000h, this));
+  RegisterFunction(0x8000'0000, std::bind(&CPUIDEmu::Function_8000_0000h, this, _1));
   // Processor vendor
-  RegisterFunction(0x8000'0001, std::bind(&CPUIDEmu::Function_8000_0001h, this));
+  RegisterFunction(0x8000'0001, std::bind(&CPUIDEmu::Function_8000_0001h, this, _1));
   // Processor brand string
-  RegisterFunction(0x8000'0002, std::bind(&CPUIDEmu::Function_8000_0002h, this));
+  RegisterFunction(0x8000'0002, std::bind(&CPUIDEmu::Function_8000_0002h, this, _1));
   // Processor brand string continued
-  RegisterFunction(0x8000'0003, std::bind(&CPUIDEmu::Function_8000_0003h, this));
+  RegisterFunction(0x8000'0003, std::bind(&CPUIDEmu::Function_8000_0003h, this, _1));
   // Processor brand string continued
-  RegisterFunction(0x8000'0004, std::bind(&CPUIDEmu::Function_8000_0004h, this));
+  RegisterFunction(0x8000'0004, std::bind(&CPUIDEmu::Function_8000_0004h, this, _1));
   // 0x8000'0005: L1 Cache and TLB identifiers
 #ifdef CPUID_AMD
-  RegisterFunction(0x8000'0005, std::bind(&CPUIDEmu::Function_8000_0005h, this));
+  RegisterFunction(0x8000'0005, std::bind(&CPUIDEmu::Function_8000_0005h, this, _1));
 #else
   // This is full reserved on Intel platforms
-  RegisterFunction(0x8000'0005, std::bind(&CPUIDEmu::Function_Reserved, this));
+  RegisterFunction(0x8000'0005, std::bind(&CPUIDEmu::Function_Reserved, this, _1));
 #endif
   // 0x8000'0006: L2 Cache identifiers
-  RegisterFunction(0x8000'0006, std::bind(&CPUIDEmu::Function_8000_0006h, this));
+  RegisterFunction(0x8000'0006, std::bind(&CPUIDEmu::Function_8000_0006h, this, _1));
   // Advanced power management information
-  RegisterFunction(0x8000'0007, std::bind(&CPUIDEmu::Function_8000_0007h, this));
+  RegisterFunction(0x8000'0007, std::bind(&CPUIDEmu::Function_8000_0007h, this, _1));
   // Virtual and physical address sizes
-  RegisterFunction(0x8000'0008, std::bind(&CPUIDEmu::Function_8000_0008h, this));
+  RegisterFunction(0x8000'0008, std::bind(&CPUIDEmu::Function_8000_0008h, this, _1));
 
   // 0x8000'000A: SVM Revision
   // TLB 1GB page identifiers
-  RegisterFunction(0x8000'0019, std::bind(&CPUIDEmu::Function_8000_0019h, this));
+  RegisterFunction(0x8000'0019, std::bind(&CPUIDEmu::Function_8000_0019h, this, _1));
 
   // 0x8000'001A: Performance optimization identifiers
   // 0x8000'001B: Instruction based sampling identifiers
   // 0x8000'001C: Lightweight profiling capabilities
   // 0x8000'001D: Cache properties
+#ifdef CPUID_AMD
+  // Deterministic cache parameters for each level
+  RegisterFunction(0x8000'001D, std::bind(&CPUIDEmu::Function_8000_001Dh, this, _1));
+#endif
   // 0x8000'001E: Extended APIC ID
   // 0x8000'001F: AMD Secure Encryption
 }

External/FEXCore/Source/Interface/Core/CPUID.h

@@ -24,23 +24,23 @@ private:
 public:
   void Init(FEXCore::Context::Context *ctx);
 
-  FEXCore::CPUID::FunctionResults RunFunction(uint32_t Function, [[maybe_unused]] uint32_t Leaf) {
+  FEXCore::CPUID::FunctionResults RunFunction(uint32_t Function, uint32_t Leaf) {
     auto Handler = FunctionHandlers.find(Function);
 
     if (Handler == FunctionHandlers.end()) {
       #ifndef NDEBUG
-        LogMan::Msg::E("Unhandled CPU ID function, 0x%x", Function);
+        LogMan::Msg::E("Unhandled CPU ID function, 0x%x-0x%x", Function, Leaf);
       #endif
-      return Function_Reserved();
+      return Function_Reserved(Leaf);
     }
 
-    return Handler->second();
+    return Handler->second(Leaf);
   }
 private:
   FEXCore::Context::Context *CTX;
   FEX_CONFIG_OPT(Cores, THREADS);
 
-  using FunctionHandler = std::function<FEXCore::CPUID::FunctionResults()>;
+  using FunctionHandler = std::function<FEXCore::CPUID::FunctionResults(uint32_t Leaf)>;
   void RegisterFunction(uint32_t Function, FunctionHandler Handler) {
     FunctionHandlers[Function] = Handler;
   }
@@ -48,23 +48,26 @@ private:
   std::unordered_map<uint32_t, FunctionHandler> FunctionHandlers;
 
   // Functions
-  FEXCore::CPUID::FunctionResults Function_0h();
-  FEXCore::CPUID::FunctionResults Function_01h();
-  FEXCore::CPUID::FunctionResults Function_02h();
-  FEXCore::CPUID::FunctionResults Function_06h();
-  FEXCore::CPUID::FunctionResults Function_07h();
-  FEXCore::CPUID::FunctionResults Function_15h();
-  FEXCore::CPUID::FunctionResults Function_8000_0000h();
-  FEXCore::CPUID::FunctionResults Function_8000_0001h();
-  FEXCore::CPUID::FunctionResults Function_8000_0002h();
-  FEXCore::CPUID::FunctionResults Function_8000_0003h();
-  FEXCore::CPUID::FunctionResults Function_8000_0004h();
-  FEXCore::CPUID::FunctionResults Function_8000_0005h();
-  FEXCore::CPUID::FunctionResults Function_8000_0006h();
-  FEXCore::CPUID::FunctionResults Function_8000_0007h();
-  FEXCore::CPUID::FunctionResults Function_8000_0008h();
-  FEXCore::CPUID::FunctionResults Function_8000_0009h();
-  FEXCore::CPUID::FunctionResults Function_8000_0019h();
-  FEXCore::CPUID::FunctionResults Function_Reserved();
+  FEXCore::CPUID::FunctionResults Function_0h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_01h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_02h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_04h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_06h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_07h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_0Dh(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_15h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_8000_0000h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_8000_0001h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_8000_0002h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_8000_0003h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_8000_0004h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_8000_0005h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_8000_0006h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_8000_0007h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_8000_0008h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_8000_0009h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_8000_0019h(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_8000_001Dh(uint32_t Leaf);
+  FEXCore::CPUID::FunctionResults Function_Reserved(uint32_t Leaf);
 };
 }

External/FEXCore/Source/Interface/Core/Core.cpp

@@ -110,39 +110,7 @@ constexpr std::array<std::string_view const, 16> RegNames = {
 std::string_view const& GetGRegName(unsigned Reg) {
   return RegNames[Reg];
 }
-
-namespace DefaultFallbackCore {
-  class DefaultFallbackCore final : public FEXCore::CPU::CPUBackend {
-  public:
-    explicit DefaultFallbackCore(FEXCore::Core::ThreadState *Thread)
-      : ThreadState {reinterpret_cast<FEXCore::Core::InternalThreadState*>(Thread)} {
-    }
-    ~DefaultFallbackCore() override = default;
-
-    std::string GetName() override { return "Default Fallback"; }
-
-    void *MapRegion(void *HostPtr, uint64_t VirtualGuestPtr, uint64_t Size) override {
-      return HostPtr;
-    }
-
-    void Initialize() override {}
-    bool NeedsOpDispatch() override { return false; }
-
-    void *CompileCode(uint64_t Entry, FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override {
-      LogMan::Msg::E("Fell back to default code handler at RIP: 0x%lx", ThreadState->CurrentFrame->State.rip);
-      return nullptr;
-    }
-
-  private:
-    FEXCore::Core::InternalThreadState *ThreadState;
-  };
-
-  FEXCore::CPU::CPUBackend *CPUCreationFactory(FEXCore::Context::Context* CTX, FEXCore::Core::ThreadState *Thread) {
-    return new DefaultFallbackCore(Thread);
-  }
-}
-
-}
+} // namespace FEXCore::Core
 
 namespace FEXCore::Context {
   void Context::AOTIRCaptureCacheWriteoutQueue_Flush() {
@@ -279,12 +247,12 @@ namespace FEXCore::Context {
     Thread->CPUBackend->CallbackPtr(Thread->CurrentFrame, RIP);
   }
 
-  void Context::RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func) {
-      SignalDelegation->RegisterHostSignalHandler(Signal, Func);
+  void Context::RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required) {
+      SignalDelegation->RegisterHostSignalHandler(Signal, Func, Required);
   }
 
-  void Context::RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func) {
-    SignalDelegation->RegisterFrontendHostSignalHandler(Signal, Func);
+  void Context::RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required) {
+    SignalDelegation->RegisterFrontendHostSignalHandler(Signal, Func, Required);
   }
 
   void Context::WaitForIdle() {
@@ -406,8 +374,6 @@ namespace FEXCore::Context {
         }
         if (Thread->RunningEvents.Running.load()) {
           StopThread(Thread);
-        } else {
-          LogMan::Msg::D("Skipping thread %p: Already stopped", Thread);
         }
       }
     }
@@ -617,6 +583,9 @@ namespace FEXCore::Context {
 
     // We now only have one thread
     IdleWaitRefCount = 1;
+
+    // Clean up dead stacks
+    FEXCore::Threads::Thread::CleanupAfterFork();
   }
 
   void Context::AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr, uint64_t Start, uint64_t Length) {
@@ -635,7 +604,7 @@ namespace FEXCore::Context {
     }
   }
 
-  std::tuple<FEXCore::IR::IRListView *, FEXCore::IR::RegisterAllocationData *, uint64_t, uint64_t, uint64_t, uint64_t> Context::GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
+  Context::GenerateIRResult Context::GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
     uint8_t const *GuestCode{};
     GuestCode = reinterpret_cast<uint8_t const*>(GuestRIP);
 
@@ -645,14 +614,14 @@ namespace FEXCore::Context {
     uint64_t TotalInstructionsLength {0};
 
     if (!Thread->FrontendDecoder->DecodeInstructionsAtEntry(GuestCode, GuestRIP)) {
-      return { nullptr, nullptr, 0, 0, 0, 0 };
+      return {};
     }
 
     auto CodeBlocks = Thread->FrontendDecoder->GetDecodedBlocks();
 
     Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks);
 
-    uint8_t GPRSize = Config.Is64BitMode ? 8 : 4;
+    const uint8_t GPRSize = GetGPRSize();
 
     for (size_t j = 0; j < CodeBlocks->size(); ++j) {
       FEXCore::Frontend::Decoder::DecodedBlocks const &Block = CodeBlocks->at(j);
@@ -732,7 +701,7 @@ namespace FEXCore::Context {
             return { nullptr, nullptr, 0, 0, 0, 0 };
           }
           else {
-            uint8_t GPRSize = Config.Is64BitMode ? 8 : 4;
+            const uint8_t GPRSize = GetGPRSize();
 
             // We had some instructions. Early exit
             Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_EntrypointOffset(Block.Entry + BlockInstructionsLength - GuestRIP, GPRSize));
@@ -823,7 +792,14 @@ namespace FEXCore::Context {
 
     Thread->OpDispatcher->ResetWorkingList();
 
-    return {IRList, RAData.release(), TotalInstructions, TotalInstructionsLength, Thread->FrontendDecoder->DecodedMinAddress, Thread->FrontendDecoder->DecodedMaxAddress - Thread->FrontendDecoder->DecodedMinAddress };
+    return {
+      .IRList = IRList,
+      .RAData = RAData.release(),
+      .TotalInstructions = TotalInstructions,
+      .TotalInstructionsLength = TotalInstructionsLength,
+      .StartAddr = Thread->FrontendDecoder->DecodedMinAddress,
+      .Length = Thread->FrontendDecoder->DecodedMaxAddress - Thread->FrontendDecoder->DecodedMinAddress,
+    };
   }
 
   AOTIRInlineEntry *AOTIRInlineIndex::GetInlineEntry(uint64_t DataOffset) {
@@ -866,16 +842,16 @@ namespace FEXCore::Context {
 
     if (Inserted.second) {
       //GuestHash
-      Stream->write((char*)&Hash, sizeof(Hash));
+      Stream->write((const char*)&Hash, sizeof(Hash));
 
       //GuestLength
-      Stream->write((char*)&Length, sizeof(Length));
+      Stream->write((const char*)&Length, sizeof(Length));
 
       // RAData (inline)
       // In file, IsShared is always set
       auto Shared = RAData->IsShared;
       RAData->IsShared = true;
-      Stream->write((char*)RAData, RAData->Size(RAData->MapCount));
+      Stream->write((const char*)RAData, RAData->Size(RAData->MapCount));
       RAData->IsShared = Shared;
 
       // IRData (inline)
@@ -883,7 +859,7 @@ namespace FEXCore::Context {
     }
   }
 
-  std::tuple<void *, FEXCore::IR::IRListView *, FEXCore::Core::DebugData *, FEXCore::IR::RegisterAllocationData *, bool, uint64_t, uint64_t> Context::CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
+  Context::CompileCodeResult Context::CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
     FEXCore::IR::IRListView *IRList {};
     FEXCore::Core::DebugData *DebugData {};
     FEXCore::IR::RegisterAllocationData *RAData {};
@@ -981,10 +957,18 @@ namespace FEXCore::Context {
     }
 
     if (IRList == nullptr) {
-      return { nullptr, nullptr, nullptr, nullptr, false, 0, 0 };
+      return {};
     }
     // Attempt to get the CPU backend to compile this code
-    return { Thread->CPUBackend->CompileCode(GuestRIP, IRList, DebugData, RAData), IRList, DebugData, RAData, GeneratedIR, StartAddr, Length};
+    return {
+      .CompiledCode = Thread->CPUBackend->CompileCode(GuestRIP, IRList, DebugData, RAData),
+      .IRData = IRList,
+      .DebugData = DebugData,
+      .RAData = RAData,
+      .GeneratedIR = GeneratedIR,
+      .StartAddr = StartAddr,
+      .Length = Length,
+    };
   }
 
   static bool readAll(int fd, void *data, size_t size) {
@@ -1057,41 +1041,41 @@ namespace FEXCore::Context {
 
     std::unique_lock lk(AOTIRCacheLock);
 
-    for (auto &AOTModule: AOTIRCaptureCache) {
-      if (!AOTModule.second.Stream) {
+    for (auto& [String, Entry] : AOTIRCaptureCache) {
+      if (!Entry.Stream) {
         continue;
       }
 
-      auto ModSize = AOTModule.first.size();
-      auto &stream = AOTModule.second.Stream;
+      const auto ModSize = String.size();
+      auto &stream = Entry.Stream;
 
       // pad to 32 bytes
-      char Zero = 0;
+      constexpr char Zero = 0;
       while(stream->tellp() & 31)
         stream->write(&Zero, 1);
 
       // AOTIRInlineIndex
-      auto FnCount = AOTModule.second.Index.size();
-      size_t DataBase = -stream->tellp();
+      const auto FnCount = Entry.Index.size();
+      const size_t DataBase = -stream->tellp();
 
-      stream->write((char*)&FnCount, sizeof(FnCount));
-      stream->write((char*)&DataBase, sizeof(DataBase));
+      stream->write((const char*)&FnCount, sizeof(FnCount));
+      stream->write((const char*)&DataBase, sizeof(DataBase));
 
-      for (auto entry: AOTModule.second.Index) {
+      for (const auto& [GuestStart, DataOffset] : Entry.Index) {
         //AOTIRInlineIndexEntry
 
         // GuestStart
-        stream->write((char*)&entry.first, sizeof(entry.first));
+        stream->write((const char*)&GuestStart, sizeof(GuestStart));
 
         // DataOffset
-        stream->write((char*)&entry.second, sizeof(entry.second));
+        stream->write((const char*)&DataOffset, sizeof(DataOffset));
       }
 
       // End of file header
-      auto IndexSize = FnCount * sizeof(AOTIRInlineIndexEntry) + sizeof(DataBase) + sizeof(FnCount);
-      stream->write((char*)&IndexSize, sizeof(IndexSize));
-      stream->write((char*)&AOTModule.first[0], ModSize);
-      stream->write((char*)&ModSize, sizeof(ModSize));
+      const auto IndexSize = FnCount * sizeof(AOTIRInlineIndexEntry) + sizeof(DataBase) + sizeof(FnCount);
+      stream->write((const char*)&IndexSize, sizeof(IndexSize));
+      stream->write(String.c_str(), ModSize);
+      stream->write((const char*)&ModSize, sizeof(ModSize));
     }
   }
 
@@ -1243,8 +1227,6 @@ namespace FEXCore::Context {
 
     ++IdleWaitRefCount;
 
-    LogMan::Msg::D("[%d] Waiting to run", Thread->ThreadManager.TID.load());
-
     // Now notify the thread that we are initialized
     Thread->ThreadWaiting.NotifyAll();
 
@@ -1253,8 +1235,6 @@ namespace FEXCore::Context {
       Thread->StartRunning.Wait();
     }
 
-    LogMan::Msg::D("[%d] Running", Thread->ThreadManager.TID.load());
-
     Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_NONE;
 
     Thread->RunningEvents.Running = true;

External/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp

@@ -175,24 +175,10 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
 
       // siginfo_t
       siginfo_t *HostSigInfo = reinterpret_cast<siginfo_t*>(info);
-      if (HostSigInfo->si_code == SI_USER) {
-        // If the signal was a user signal then we need to pass this struct through unaltered
-        // Guest might be doing something with it
-        *guest_siginfo = *HostSigInfo;
-      }
-      else {
-        guest_siginfo->si_signo = Signal;
-        switch (Signal) {
-        case SIGSEGV:
-        case SIGBUS:
-          guest_siginfo->si_code = HostSigInfo->si_code;
-          guest_siginfo->si_errno = HostSigInfo->si_errno;
-          // Macro expansion to get the si_addr
-          guest_siginfo->si_addr = HostSigInfo->si_addr;
-          break;
-        default: LogMan::Msg::D("Unhandled siginfo_t signal: %d", Signal); break;
-        }
-      }
+      // aarch64 and x86_64 siginfo_t matches. We can just copy this over
+      // SI_USER could also potentially have random data in it, needs to be bit perfect
+      // For guest faults we don't have a real way to reconstruct state to a real guest RIP
+      *guest_siginfo = *HostSigInfo;
 
       Frame->State.gregs[X86State::REG_RSI] = SigInfoLocation;
       Frame->State.gregs[X86State::REG_RDX] = UContextLocation;
@@ -202,7 +188,36 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
       NewGuestSP -= sizeof(FEXCore::x86::ucontext_t);
       uint64_t UContextLocation = 0; // NewGuestSP;
       NewGuestSP -= sizeof(FEXCore::x86::siginfo_t);
-      uint64_t SigInfoLocation = 0; // NewGuestSP;
+      uint64_t SigInfoLocation = NewGuestSP;
+
+      FEXCore::x86::siginfo_t *guest_siginfo = reinterpret_cast<FEXCore::x86::siginfo_t*>(SigInfoLocation);
+      siginfo_t *HostSigInfo = reinterpret_cast<siginfo_t*>(info);
+
+      // These three elements are in every siginfo
+      guest_siginfo->si_signo = HostSigInfo->si_signo;
+      guest_siginfo->si_errno = HostSigInfo->si_errno;
+      guest_siginfo->si_code = HostSigInfo->si_code;
+
+      switch (Signal) {
+        case SIGSEGV:
+        case SIGBUS:
+          // Macro expansion to get the si_addr
+          // Can't really give a real result here. Pull from the context for now
+          guest_siginfo->_sifields._sigfault.addr = Frame->State.rip;
+          break;
+        case SIGCHLD:
+          guest_siginfo->_sifields._sigchld.pid = HostSigInfo->si_pid;
+          guest_siginfo->_sifields._sigchld.uid = HostSigInfo->si_uid;
+          guest_siginfo->_sifields._sigchld.status = HostSigInfo->si_status;
+          guest_siginfo->_sifields._sigchld.utime = HostSigInfo->si_utime;
+          guest_siginfo->_sifields._sigchld.stime = HostSigInfo->si_stime;
+          break;
+      default:
+        LogMan::Msg::D("Unhandled siginfo_t signal: %d", Signal);
+        // Hope for the best, most things just copy over
+        memcpy(guest_siginfo, info, sizeof(siginfo_t));
+        break;
+      }
 
       NewGuestSP -= 4;
       *(uint32_t*)NewGuestSP = UContextLocation;

External/FEXCore/Source/Interface/Core/Frontend.cpp

@@ -21,7 +21,9 @@ namespace FEXCore::Frontend {
 using namespace FEXCore::X86Tables;
 
 static uint32_t MapModRMToReg(uint8_t REX, uint8_t bits, bool HighBits, bool HasREX, bool HasXMM, bool HasMM, uint8_t InvalidOffset = 16) {
-  constexpr std::array<uint64_t, 16> GPRIndexes = {
+  using GPRArray = std::array<uint32_t, 16>;
+
+  static constexpr GPRArray GPRIndexes = {
     // Classical ordering?
     FEXCore::X86State::REG_RAX,
     FEXCore::X86State::REG_RCX,
@@ -41,7 +43,7 @@ static uint32_t MapModRMToReg(uint8_t REX, uint8_t bits, bool HighBits, bool Has
     FEXCore::X86State::REG_R15,
   };
 
-  constexpr std::array<uint64_t, 16> GPR8BitHighIndexes = {
+  static constexpr GPRArray GPR8BitHighIndexes = {
     // Classical ordering?
     FEXCore::X86State::REG_RAX,
     FEXCore::X86State::REG_RCX,
@@ -61,7 +63,7 @@ static uint32_t MapModRMToReg(uint8_t REX, uint8_t bits, bool HighBits, bool Has
     FEXCore::X86State::REG_R15,
   };
 
-  constexpr std::array<uint64_t, 16> XMMIndexes = {
+  static constexpr GPRArray XMMIndexes = {
     FEXCore::X86State::REG_XMM_0,
     FEXCore::X86State::REG_XMM_1,
     FEXCore::X86State::REG_XMM_2,
@@ -80,7 +82,7 @@ static uint32_t MapModRMToReg(uint8_t REX, uint8_t bits, bool HighBits, bool Has
     FEXCore::X86State::REG_XMM_15,
   };
 
-  constexpr std::array<uint64_t, 16> MMIndexes = {
+  static constexpr GPRArray MMIndexes = {
     FEXCore::X86State::REG_MM_0,
     FEXCore::X86State::REG_MM_1,
     FEXCore::X86State::REG_MM_2,
@@ -99,7 +101,7 @@ static uint32_t MapModRMToReg(uint8_t REX, uint8_t bits, bool HighBits, bool Has
     FEXCore::X86State::REG_INVALID
   };
 
-  const std::array<uint64_t, 16> *GPRs = &GPRIndexes;
+  const GPRArray *GPRs = &GPRIndexes;
   if (HasXMM) {
     GPRs = &XMMIndexes;
   }
@@ -939,7 +941,7 @@ void Decoder::BranchTargetInMultiblockRange() {
 
   // If the RIP setting is conditional AND within our symbol range then it can be considered for multiblock
   uint64_t TargetRIP = 0;
-  uint8_t GPRSize = CTX->Config.Is64BitMode ? 8 : 4;
+  const uint8_t GPRSize = CTX->GetGPRSize();
   bool Conditional = true;
 
   switch (DecodeInst->OP) {

External/FEXCore/Source/Interface/Core/GdbServer.cpp

@@ -47,11 +47,11 @@ void GdbServer::Break(int signal) {
 }
 
 GdbServer::GdbServer(FEXCore::Context::Context *ctx) : CTX(ctx) {
-    ctx->CustomExitHandler = [this](uint64_t ThreadId, FEXCore::Context::ExitReason ExitReason) {
+    Context::SetExitHandler(ctx, [this](uint64_t ThreadId, FEXCore::Context::ExitReason ExitReason) {
         if (ExitReason == FEXCore::Context::ExitReason::EXIT_DEBUG) {
             this->Break(SIGTRAP);
         }
-    };
+    });
 
     // This is a total hack as there is currently no way to resume once hitting a segfault
     // But it's semi-useful for debugging.
@@ -64,7 +64,7 @@ GdbServer::GdbServer(FEXCore::Context::Context *ctx) : CTX(ctx) {
           usleep(100000);
 
         return true;
-    });
+    }, true);
 
     StartThread();
 }

External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp

@@ -93,12 +93,12 @@ InterpreterCore::InterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::
     CTX->SignalDelegation->RegisterHostSignalHandler(SignalDelegator::SIGNAL_FOR_PAUSE, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
       InterpreterCore *Core = reinterpret_cast<InterpreterCore*>(Thread->CPUBackend.get());
       return Core->Dispatcher->HandleSignalPause(Signal, info, ucontext);
-    });
+    }, true);
 
     CTX->SignalDelegation->RegisterHostSignalHandler(SIGBUS, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
       InterpreterCore *Core = reinterpret_cast<InterpreterCore*>(Thread->CPUBackend.get());
       return Core->HandleSIGBUS(Signal, info, ucontext);
-    });
+    }, true);
 
     auto GuestSignalHandler = [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack) -> bool {
       InterpreterCore *Core = reinterpret_cast<InterpreterCore*>(Thread->CPUBackend.get());

External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp

@@ -865,8 +865,7 @@ DEF_OP(Bfi) {
 
 DEF_OP(Bfe) {
   auto Op = IROp->C<IR::IROp_Bfe>();
-  uint8_t OpSize = IROp->Size;
-  LOGMAN_THROW_A(OpSize <= 8, "OpSize is too large for BFE: %d", OpSize);
+  LOGMAN_THROW_A(IROp->Size <= 8, "OpSize is too large for BFE: %d", IROp->Size);
   LOGMAN_THROW_A(Op->Width != 0, "Invalid BFE width of 0");
 
   auto Dst = GetReg<RA_64>(Node);

External/FEXCore/Source/Interface/Core/JIT/Arm64/BranchOps.cpp

@@ -257,7 +257,7 @@ DEF_OP(Thunk) {
 
 DEF_OP(ValidateCode) {
   auto Op = IROp->C<IR::IROp_ValidateCode>();
-  uint8_t *OldCode = (uint8_t *)&Op->CodeOriginalLow;
+  const auto *OldCode = (const uint8_t *)&Op->CodeOriginalLow;
   int len = Op->CodeLength;
   int idx = 0;
 
@@ -268,7 +268,7 @@ DEF_OP(ValidateCode) {
   while (len >= 8)
   {
     ldr(x2, MemOperand(x0, idx));
-    LoadConstant(x3, *(uint32_t *)(OldCode + idx));
+    LoadConstant(x3, *(const uint32_t *)(OldCode + idx));
     cmp(x2, x3);
     csel(GetReg<RA_64>(Node), GetReg<RA_64>(Node), x1, Condition::eq);
     len -= 8;
@@ -277,7 +277,7 @@ DEF_OP(ValidateCode) {
   while (len >= 4)
   {
     ldr(w2, MemOperand(x0, idx));
-    LoadConstant(w3, *(uint32_t *)(OldCode + idx));
+    LoadConstant(w3, *(const uint32_t *)(OldCode + idx));
     cmp(w2, w3);
     csel(GetReg<RA_64>(Node), GetReg<RA_64>(Node), x1, Condition::eq);
     len -= 4;
@@ -286,7 +286,7 @@ DEF_OP(ValidateCode) {
   while (len >= 2)
   {
     ldrh(w2, MemOperand(x0, idx));
-    LoadConstant(w3, *(uint16_t *)(OldCode + idx));
+    LoadConstant(w3, *(const uint16_t *)(OldCode + idx));
     cmp(w2, w3);
     csel(GetReg<RA_64>(Node), GetReg<RA_64>(Node), x1, Condition::eq);
     len -= 2;
@@ -295,7 +295,7 @@ DEF_OP(ValidateCode) {
   while (len >= 1)
   {
     ldrb(w2, MemOperand(x0, idx));
-    LoadConstant(w3, *(uint8_t *)(OldCode + idx));
+    LoadConstant(w3, *(const uint8_t *)(OldCode + idx));
     cmp(w2, w3);
     csel(GetReg<RA_64>(Node), GetReg<RA_64>(Node), x1, Condition::eq);
     len -= 1;

External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp

@@ -44,8 +44,10 @@ using namespace vixl::aarch64;
 void Arm64JITCore::Op_Unhandled(FEXCore::IR::IROp_Header *IROp, uint32_t Node) {
   FallbackInfo Info;
   if (!InterpreterOps::GetFallbackHandler(IROp, &Info)) {
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
     auto Name = FEXCore::IR::GetName(IROp->Op);
     LOGMAN_MSG_A("Unhandled IR Op: %s", std::string(Name).c_str());
+#endif
   } else {
     switch(Info.ABI) {
       case FABI_VOID_U16:{
@@ -292,8 +294,11 @@ void Arm64JITCore::Op_Unhandled(FEXCore::IR::IROp_Header *IROp, uint32_t Node) {
 
       case FABI_UNKNOWN:
       default:
-      auto Name = FEXCore::IR::GetName(IROp->Op);
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
+        auto Name = FEXCore::IR::GetName(IROp->Op);
         LOGMAN_MSG_A("Unhandled IR Fallback abi: %s %d", std::string(Name).c_str(), Info.ABI);
+#endif
+      break;
     }
   }
 }
@@ -502,17 +507,17 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern
     CTX->SignalDelegation->RegisterHostSignalHandler(SIGILL, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
       Arm64JITCore *Core = reinterpret_cast<Arm64JITCore*>(Thread->CPUBackend.get());
       return Core->Dispatcher->HandleSIGILL(Signal, info, ucontext);
-    });
+    }, true);
 
     CTX->SignalDelegation->RegisterHostSignalHandler(SIGBUS, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
       Arm64JITCore *Core = reinterpret_cast<Arm64JITCore*>(Thread->CPUBackend.get());
       return Core->HandleSIGBUS(Signal, info, ucontext);
-    });
+    }, true);
 
     CTX->SignalDelegation->RegisterHostSignalHandler(SignalDelegator::SIGNAL_FOR_PAUSE, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
       Arm64JITCore *Core = reinterpret_cast<Arm64JITCore*>(Thread->CPUBackend.get());
       return Core->Dispatcher->HandleSignalPause(Signal, info, ucontext);
-    });
+    }, true);
 
     auto GuestSignalHandler = [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack) -> bool {
       Arm64JITCore *Core = reinterpret_cast<Arm64JITCore*>(Thread->CPUBackend.get());
@@ -707,8 +712,6 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
   this->Entry = Entry;
   this->RAData = RAData;
 
-  auto HeaderOp = IR->GetHeader();
-
   #ifndef NDEBUG
   LoadConstant(x0, Entry);
   #endif
@@ -786,8 +789,10 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
 
   for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {
     using namespace FEXCore::IR;
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
     auto BlockIROp = BlockHeader->CW<FEXCore::IR::IROp_CodeBlock>();
     LOGMAN_THROW_A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
+#endif
 
     {
       uint32_t Node = IR->GetID(BlockNode);

External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp

@@ -766,7 +766,6 @@ DEF_OP(VFRSqrt) {
 
 DEF_OP(VNeg) {
   auto Op = IROp->C<IR::IROp_VNeg>();
-  uint8_t OpSize = IROp->Size;
   switch (Op->Header.ElementSize) {
   case 1:
     neg(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B());
@@ -780,13 +779,12 @@ DEF_OP(VNeg) {
   case 8:
     neg(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
     break;
-  default: LOGMAN_MSG_A("Unsupported Not size: %d", OpSize);
+  default: LOGMAN_MSG_A("Unsupported Not size: %d", IROp->Size);
   }
 }
 
 DEF_OP(VFNeg) {
   auto Op = IROp->C<IR::IROp_VFNeg>();
-  uint8_t OpSize = IROp->Size;
   switch (Op->Header.ElementSize) {
   case 4:
     fneg(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
@@ -794,7 +792,7 @@ DEF_OP(VFNeg) {
   case 8:
     fneg(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
     break;
-  default: LOGMAN_MSG_A("Unsupported Not size: %d", OpSize);
+  default: LOGMAN_MSG_A("Unsupported Not size: %d", IROp->Size);
   }
 }
 

External/FEXCore/Source/Interface/Core/JIT/x86_64/ALUOps.cpp

@@ -970,9 +970,7 @@ DEF_OP(Bfi) {
 
 DEF_OP(Bfe) {
   auto Op = IROp->C<IR::IROp_Bfe>();
-  uint8_t OpSize = IROp->Size;
-
-  LOGMAN_THROW_A(OpSize <= 8, "OpSize is too large for BFE: %d", OpSize);
+  LOGMAN_THROW_A(IROp->Size <= 8, "OpSize is too large for BFE: %d", IROp->Size);
 
   auto Dst = GetDst<RA_64>(Node);
 

External/FEXCore/Source/Interface/Core/JIT/x86_64/BranchOps.cpp

@@ -248,7 +248,7 @@ DEF_OP(Thunk) {
 
 DEF_OP(ValidateCode) {
   auto Op = IROp->C<IR::IROp_ValidateCode>();
-  uint8_t* OldCode = (uint8_t*)&Op->CodeOriginalLow;
+  const auto* OldCode = (const uint8_t*)&Op->CodeOriginalLow;
   int len = Op->CodeLength;
   int idx = 0;
 
@@ -256,20 +256,20 @@ DEF_OP(ValidateCode) {
   mov(rax, Entry + Op->Offset);
   mov(rbx, 1);
   while (len >= 4) {
-    cmp(dword[rax + idx], *(uint32_t*)(OldCode + idx));
+    cmp(dword[rax + idx], *(const uint32_t*)(OldCode + idx));
     cmovne(GetDst<RA_64>(Node), rbx);
     len-=4;
     idx+=4;
   }
   while (len >= 2) {
-    mov(rcx, *(uint16_t*)(OldCode + idx));
+    mov(rcx, *(const uint16_t*)(OldCode + idx));
     cmp(word[rax + idx], cx);
     cmovne(GetDst<RA_64>(Node), rbx);
     len-=2;
     idx+=2;
   }
   while (len >= 1) {
-    cmp(byte[rax + idx], *(uint8_t*)(OldCode + idx));
+    cmp(byte[rax + idx], *(const uint8_t*)(OldCode + idx));
     cmovne(GetDst<RA_64>(Node), rbx);
     len-=1;
     idx+=1;
@@ -319,8 +319,9 @@ DEF_OP(CPUID) {
   //
   // Result: RAX, RDX. 4xi32
 
-  mov (rsi, GetSrc<RA_64>(Op->Header.Args[0].ID()));
-  mov (rdx, GetSrc<RA_64>(Op->Header.Args[1].ID()));
+  // rsi can be in the source registers, so copy argument to edx first
+  mov (edx, GetSrc<RA_32>(Op->Header.Args[1].ID()));
+  mov (esi, GetSrc<RA_32>(Op->Header.Args[0].ID()));
   mov (rdi, reinterpret_cast<uint64_t>(&CTX->CPUID));
 
   auto NumPush = RA64.size();

External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp

@@ -84,8 +84,10 @@ void X86JITCore::PopRegs() {
 void X86JITCore::Op_Unhandled(FEXCore::IR::IROp_Header *IROp, uint32_t Node) {
   FallbackInfo Info;
   if (!InterpreterOps::GetFallbackHandler(IROp, &Info)) {
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
     auto Name = FEXCore::IR::GetName(IROp->Op);
     LOGMAN_MSG_A("Unhandled IR Op: %s", std::string(Name).c_str());
+#endif
   } else {
     switch(Info.ABI) {
       case FABI_VOID_U16: {
@@ -282,8 +284,11 @@ void X86JITCore::Op_Unhandled(FEXCore::IR::IROp_Header *IROp, uint32_t Node) {
 
       case FABI_UNKNOWN:
       default:
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
       auto Name = FEXCore::IR::GetName(IROp->Op);
         LOGMAN_MSG_A("Unhandled IR Fallback abi: %s %d", std::string(Name).c_str(), Info.ABI);
+#endif
+        break;
     }
   }
 }
@@ -343,12 +348,12 @@ X86JITCore::X86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalTh
     CTX->SignalDelegation->RegisterHostSignalHandler(SIGILL, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
       X86JITCore *Core = reinterpret_cast<X86JITCore*>(Thread->CPUBackend.get());
       return Core->Dispatcher->HandleSIGILL(Signal, info, ucontext);
-    });
+    }, true);
 
     CTX->SignalDelegation->RegisterHostSignalHandler(SignalDelegator::SIGNAL_FOR_PAUSE, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
       X86JITCore *Core = reinterpret_cast<X86JITCore*>(Thread->CPUBackend.get());
       return Core->Dispatcher->HandleSignalPause(Signal, info, ucontext);
-    });
+    }, true);
 
     auto GuestSignalHandler = [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack) -> bool {
       X86JITCore *Core = reinterpret_cast<X86JITCore*>(Thread->CPUBackend.get());
@@ -662,8 +667,10 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
   for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {
     using namespace FEXCore::IR;
     {
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
       auto BlockIROp = BlockHeader->CW<IROp_CodeBlock>();
       LOGMAN_THROW_A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
+#endif
 
       uint32_t Node = IR->GetID(BlockNode);
       auto IsTarget = JumpTargets.find(Node);

External/FEXCore/Source/Interface/Core/LookupCache.h

@@ -38,7 +38,10 @@ public:
   std::map<uint64_t, std::vector<uint64_t>> CodePages;
 
   void AddBlockMapping(uint64_t Address, void *HostCode, uint64_t Start, uint64_t Length) { 
-    auto InsertPoint = BlockList.emplace(Address, (uintptr_t)HostCode);
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
+    auto InsertPoint =
+#endif
+    BlockList.emplace(Address, (uintptr_t)HostCode);
     LOGMAN_THROW_A(InsertPoint.second == true, "Dupplicate block mapping added");
 
     for (auto CurrentPage = Start >> 12, EndPage = (Start + Length) >> 12; CurrentPage <= EndPage; CurrentPage++) {

External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h

@@ -85,7 +85,7 @@ public:
       auto it = JumpTargets.find(NextRIP);
       if (it == JumpTargets.end()) {
 
-        uint8_t GPRSize = CTX->Config.Is64BitMode ? 8 : 4;
+        const uint8_t GPRSize = CTX->GetGPRSize();
         // If we don't have a jump target to a new block then we have to leave
         // Set the RIP to the next instruction and leave
         auto RelocatedNextRIP = _EntrypointOffset(NextRIP - Entry, GPRSize);

External/FEXCore/Source/Interface/IR/PassManager.cpp

@@ -45,10 +45,9 @@ void PassManager::AddDefaultPasses(bool InlineConstants, bool StaticRegisterAllo
       InsertPass(CreateStaticRegisterAllocationPass());
   }
 
-  CompactionPass = CreateIRCompaction();
   // If the IR is compacted post-RA then the node indexing gets messed up and the backend isn't able to find the register assigned to a node
   // Compact before IR, don't worry about RA generating spills/fills
-  InsertPass(CompactionPass);
+  CompactionPass = InsertPass(CreateIRCompaction());
 }
 
 void PassManager::AddDefaultValidationPasses() {
@@ -60,8 +59,7 @@ void PassManager::AddDefaultValidationPasses() {
 }
 
 void PassManager::InsertRegisterAllocationPass(bool OptimizeSRA) {
-    RAPass = IR::CreateRegisterAllocationPass(CompactionPass, OptimizeSRA);
-    InsertPass(RAPass);
+  RAPass = InsertPass(IR::CreateRegisterAllocationPass(CompactionPass, OptimizeSRA));
 }
 
 bool PassManager::Run(IREmitter *IREmit) {

External/FEXCore/Source/Interface/IR/PassManager.h

@@ -42,9 +42,9 @@ class PassManager final {
 public:
   void AddDefaultPasses(bool InlineConstants, bool StaticRegisterAllocation);
   void AddDefaultValidationPasses();
-  void InsertPass(Pass *Pass) {
+  Pass* InsertPass(std::unique_ptr<Pass> Pass) {
     Pass->RegisterPassManager(this);
-    Passes.emplace_back(Pass);
+    return Passes.emplace_back(std::move(Pass)).get();
   }
 
   void InsertRegisterAllocationPass(bool OptimizeSRA);
@@ -52,7 +52,7 @@ public:
   bool Run(IREmitter *IREmit);
 
   void RegisterExitHandler(ShouldExitHandler Handler) {
-    ExitHandler = Handler;
+    ExitHandler = std::move(Handler);
   }
 
   bool HasRAPass() const {
@@ -73,15 +73,15 @@ protected:
 
 private:
   Pass *RAPass{};
-  FEXCore::IR::Pass *CompactionPass{};
+  Pass *CompactionPass{};
 
   std::vector<std::unique_ptr<Pass>> Passes;
 
 #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
   std::vector<std::unique_ptr<Pass>> ValidationPasses;
-  void InsertValidationPass(Pass *Pass) {
+  void InsertValidationPass(std::unique_ptr<Pass> Pass) {
     Pass->RegisterPassManager(this);
-    ValidationPasses.emplace_back(Pass);
+    ValidationPasses.emplace_back(std::move(Pass));
   }
 #endif
 

External/FEXCore/Source/Interface/IR/Passes.h

@@ -1,25 +1,27 @@
 #pragma once
 
+#include <memory>
+
 namespace FEXCore::IR {
 class Pass;
 class RegisterAllocationPass;
 class RegisterAllocationData;
 
-FEXCore::IR::Pass* CreateConstProp(bool InlineConstants);
-FEXCore::IR::Pass* CreateContextLoadStoreElimination();
-FEXCore::IR::Pass* CreateSyscallOptimization();
-FEXCore::IR::Pass* CreateDeadFlagCalculationEliminination();
-FEXCore::IR::Pass* CreateDeadStoreElimination();
-FEXCore::IR::Pass* CreatePassDeadCodeElimination();
-FEXCore::IR::Pass* CreateIRCompaction();
-FEXCore::IR::RegisterAllocationPass* CreateRegisterAllocationPass(FEXCore::IR::Pass* CompactionPass, bool OptimizeSRA);
-FEXCore::IR::Pass* CreateStaticRegisterAllocationPass();
-FEXCore::IR::Pass* CreateLongDivideEliminationPass();
+std::unique_ptr<FEXCore::IR::Pass> CreateConstProp(bool InlineConstants);
+std::unique_ptr<FEXCore::IR::Pass> CreateContextLoadStoreElimination();
+std::unique_ptr<FEXCore::IR::Pass> CreateSyscallOptimization();
+std::unique_ptr<FEXCore::IR::Pass> CreateDeadFlagCalculationEliminination();
+std::unique_ptr<FEXCore::IR::Pass> CreateDeadStoreElimination();
+std::unique_ptr<FEXCore::IR::Pass> CreatePassDeadCodeElimination();
+std::unique_ptr<FEXCore::IR::Pass> CreateIRCompaction();
+std::unique_ptr<FEXCore::IR::RegisterAllocationPass> CreateRegisterAllocationPass(FEXCore::IR::Pass* CompactionPass, bool OptimizeSRA);
+std::unique_ptr<FEXCore::IR::Pass> CreateStaticRegisterAllocationPass();
+std::unique_ptr<FEXCore::IR::Pass> CreateLongDivideEliminationPass();
 
 namespace Validation {
-FEXCore::IR::Pass* CreateIRValidation();
-FEXCore::IR::Pass* CreatePhiValidation();
-FEXCore::IR::Pass* CreateValueDominanceValidation();
+std::unique_ptr<FEXCore::IR::Pass> CreateIRValidation();
+std::unique_ptr<FEXCore::IR::Pass> CreatePhiValidation();
+std::unique_ptr<FEXCore::IR::Pass> CreateValueDominanceValidation();
 }
 }
 

External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp

@@ -992,8 +992,8 @@ bool ConstProp::Run(IREmitter *IREmit) {
   return Changed;
 }
 
-FEXCore::IR::Pass* CreateConstProp(bool InlineConstants) {
-  return new ConstProp(InlineConstants);
+std::unique_ptr<FEXCore::IR::Pass> CreateConstProp(bool InlineConstants) {
+  return std::make_unique<ConstProp>(InlineConstants);
 }
 
 }

External/FEXCore/Source/Interface/IR/Passes/DeadCodeElimination.cpp

@@ -59,10 +59,8 @@ void DeadCodeElimination::markUsed(OrderedNodeWrapper *CodeOp, IROp_Header *IROp
 
 }
 
-
-FEXCore::IR::Pass* CreatePassDeadCodeElimination() {
-  return new DeadCodeElimination{};
+std::unique_ptr<FEXCore::IR::Pass> CreatePassDeadCodeElimination() {
+  return std::make_unique<DeadCodeElimination>();
 }
 
-
 }

External/FEXCore/Source/Interface/IR/Passes/DeadContextStoreElimination.cpp

@@ -283,7 +283,7 @@ class RCLSE final : public FEXCore::IR::Pass {
 public:
   RCLSE() {
     ClassifyContextStruct(&ClassifiedStruct);
-    DCE.reset(FEXCore::IR::CreatePassDeadCodeElimination());
+    DCE = FEXCore::IR::CreatePassDeadCodeElimination();
   }
   bool Run(FEXCore::IR::IREmitter *IREmit) override;
 private:
@@ -636,8 +636,8 @@ bool RCLSE::Run(FEXCore::IR::IREmitter *IREmit) {
 
 namespace FEXCore::IR {
 
-FEXCore::IR::Pass* CreateContextLoadStoreElimination() {
-  return new RCLSE{};
+std::unique_ptr<FEXCore::IR::Pass> CreateContextLoadStoreElimination() {
+  return std::make_unique<RCLSE>();
 }
 
 }

External/FEXCore/Source/Interface/IR/Passes/DeadStoreElimination.cpp

@@ -331,8 +331,8 @@ bool DeadStoreElimination::Run(IREmitter *IREmit) {
   return Changed;
 }
 
-FEXCore::IR::Pass* CreateDeadStoreElimination() {
-  return new DeadStoreElimination{};
+std::unique_ptr<FEXCore::IR::Pass> CreateDeadStoreElimination() {
+  return std::make_unique<DeadStoreElimination>();
 }
 
 }

External/FEXCore/Source/Interface/IR/Passes/IRCompaction.cpp

@@ -153,8 +153,10 @@ bool IRCompaction::Run(IREmitter *IREmit) {
   {
     // Fixup the arguments of all the IROps
     for (auto &Block : GeneratedCodeBlocks) {
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
       auto BlockIROp = LocalIR.GetOp<FEXCore::IR::IROp_CodeBlock>(Block.NewNode);
       LOGMAN_THROW_A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
+#endif
 
       for (auto [LocalNode, LocalIROp] : LocalIR.GetCode(Block.NewNode)) {
 
@@ -199,8 +201,8 @@ bool IRCompaction::Run(IREmitter *IREmit) {
   return true;
 }
 
-FEXCore::IR::Pass* CreateIRCompaction() {
-  return new IRCompaction{};
+std::unique_ptr<FEXCore::IR::Pass> CreateIRCompaction() {
+  return std::make_unique<IRCompaction>();
 }
 
 }

External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp

@@ -11,7 +11,7 @@ $end_info$
 #include "Interface/Core/OpcodeDispatcher.h"
 #include "Common/BitSet.h"
 
-#include <iostream>
+#include <sstream>
 
 namespace {
   struct BlockInfo {
@@ -54,8 +54,10 @@ bool IRValidation::Run(IREmitter *IREmit) {
 
   std::vector<uint32_t> Uses(CurrentIR.GetSSACount(), 0);
 
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
   auto HeaderOp = CurrentIR.GetHeader();
   LOGMAN_THROW_A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
+#endif
 
   IR::RegisterAllocationData * RAData{};
   if (Manager->HasRAPass()) {
@@ -279,13 +281,13 @@ bool IRValidation::Run(IREmitter *IREmit) {
       Out << "Warnings:" << std::endl << Warnings.str() << std::endl;
     }
 
-    LogMan::Msg::E("%s", Out.str().c_str());
+    LogMan::Msg::EFmt("{}", Out.str());
   }
 
   return false;
 }
 
-FEXCore::IR::Pass* CreateIRValidation() {
-  return new IRValidation{};
+std::unique_ptr<FEXCore::IR::Pass> CreateIRValidation() {
+  return std::make_unique<IRValidation>();
 }
 }

External/FEXCore/Source/Interface/IR/Passes/LongDivideRemovalPass.cpp

@@ -106,7 +106,7 @@ bool LongDivideEliminationPass::Run(IREmitter *IREmit) {
   return Changed;
 }
 
-FEXCore::IR::Pass* CreateLongDivideEliminationPass() {
-  return new LongDivideEliminationPass{};
+std::unique_ptr<FEXCore::IR::Pass> CreateLongDivideEliminationPass() {
+  return std::make_unique<LongDivideEliminationPass>();
 }
 }

External/FEXCore/Source/Interface/IR/Passes/PhiValidation.cpp

@@ -8,7 +8,7 @@ $end_info$
 #include "Interface/IR/PassManager.h"
 #include "Interface/Core/OpcodeDispatcher.h"
 
-#include <iostream>
+#include <sstream>
 
 namespace FEXCore::IR::Validation {
 
@@ -59,15 +59,15 @@ bool PhiValidation::Run(IREmitter *IREmit) {
 
     Out << "Errors:" << std::endl << Errors.str() << std::endl;
 
-    LogMan::Msg::E(Out.str().c_str());
+    LogMan::Msg::EFmt("{}", Out.str());
   }
 
 
   return false;
 }
 
-FEXCore::IR::Pass* CreatePhiValidation() {
-  return new PhiValidation{};
+std::unique_ptr<FEXCore::IR::Pass> CreatePhiValidation() {
+  return std::make_unique<PhiValidation>();
 }
 
 }

External/FEXCore/Source/Interface/IR/Passes/RedundantFlagCalculationElimination.cpp

@@ -59,8 +59,8 @@ bool DeadFlagCalculationEliminination::Run(IREmitter *IREmit) {
   return Changed;
 }
 
-FEXCore::IR::Pass* CreateDeadFlagCalculationEliminination() {
-  return new DeadFlagCalculationEliminination{};
+std::unique_ptr<FEXCore::IR::Pass> CreateDeadFlagCalculationEliminination() {
+  return std::make_unique<DeadFlagCalculationEliminination>();
 }
 
 }

External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp

@@ -46,7 +46,7 @@ namespace {
       for (int i = 1; i < Size; i++)
         Items[i] = 0xDEADBEEF;
       #endif
-      Next.release();
+      Next.reset();
     }
 
     BucketList() {
@@ -144,7 +144,7 @@ namespace {
         }
         else if (++i == Size) {
           if (that->Next->Items[0] == 0) {
-            that->Next.release();
+            that->Next.reset();
             foundThat->Items[foundI] = that->Items[Size-1];
             that->Items[Size-1] = 0;
             break;
@@ -1399,11 +1399,13 @@ namespace FEXCore::IR {
         if (InterferenceNode != ~0U) {
           FEXCore::IR::RegisterClassType InterferenceRegClass = FEXCore::IR::RegisterClassType{Graph->AllocData->Map[InterferenceNode].Class};
           uint32_t SpillSlot = FindSpillSlot(InterferenceNode, InterferenceRegClass);
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
           RegisterNode *InterferenceRegisterNode = &Graph->Nodes[InterferenceNode];
           LOGMAN_THROW_A(SpillSlot != ~0U, "Interference Node doesn't have a spill slot!");
           //LOGMAN_THROW_A(InterferenceRegisterNode->Head.RegAndClass.Reg != INVALID_REG, "Interference node never assigned a register?");
           LOGMAN_THROW_A(InterferenceRegClass != ~0U, "Interference node never assigned a register class?");
           LOGMAN_THROW_A(InterferenceRegisterNode->Head.PhiPartner == nullptr, "We don't support spilling PHI nodes currently");
+#endif
 
           // This is the op that we need to dump
           auto [InterferenceOrderedNode, InterferenceIROp] = IR.at(InterferenceNode)();
@@ -1538,7 +1540,7 @@ namespace FEXCore::IR {
     return Changed;
   }
 
-  FEXCore::IR::RegisterAllocationPass* CreateRegisterAllocationPass(FEXCore::IR::Pass* CompactionPass, bool OptimizeSRA) {
-    return new ConstrainedRAPass{CompactionPass, OptimizeSRA};
+  std::unique_ptr<FEXCore::IR::RegisterAllocationPass> CreateRegisterAllocationPass(FEXCore::IR::Pass* CompactionPass, bool OptimizeSRA) {
+    return std::make_unique<ConstrainedRAPass>(CompactionPass, OptimizeSRA);
   }
 }

External/FEXCore/Source/Interface/IR/Passes/StaticRegisterAllocationPass.cpp

@@ -94,8 +94,8 @@ bool StaticRegisterAllocationPass::Run(IREmitter *IREmit) {
   return true;
 }
 
-FEXCore::IR::Pass* CreateStaticRegisterAllocationPass() {
-  return new StaticRegisterAllocationPass{};
+std::unique_ptr<FEXCore::IR::Pass> CreateStaticRegisterAllocationPass() {
+  return std::make_unique<StaticRegisterAllocationPass>();
 }
 
 }

External/FEXCore/Source/Interface/IR/Passes/SyscallOptimization.cpp

@@ -44,12 +44,11 @@ bool SyscallOptimization::Run(IREmitter *IREmit) {
     }
   }
 
-
   return Changed;
 }
 
-FEXCore::IR::Pass* CreateSyscallOptimization() {
-  return new SyscallOptimization{};
+std::unique_ptr<FEXCore::IR::Pass> CreateSyscallOptimization() {
+  return std::make_unique<SyscallOptimization>();
 }
 
 }

External/FEXCore/Source/Interface/IR/Passes/ValueDominanceValidation.cpp

@@ -8,9 +8,9 @@ $end_info$
 #include "Interface/IR/PassManager.h"
 #include "Interface/Core/OpcodeDispatcher.h"
 
-#include <iostream>
 #include <map>
 #include <list>
+#include <sstream>
 #include <unordered_map>
 
 namespace {
@@ -206,14 +206,14 @@ bool ValueDominanceValidation::Run(IREmitter *IREmit) {
       Out << "Warnings:" << std::endl << Warnings.str() << std::endl;
     }
 
-    LogMan::Msg::E(Out.str().c_str());
+    LogMan::Msg::EFmt("{}", Out.str());
   }
 
   return false;
 }
 
-FEXCore::IR::Pass* CreateValueDominanceValidation() {
-  return new ValueDominanceValidation{};
+std::unique_ptr<FEXCore::IR::Pass> CreateValueDominanceValidation() {
+  return std::make_unique<ValueDominanceValidation>();
 }
 
 }

External/FEXCore/Source/Utils/Allocator.cpp

@@ -74,7 +74,7 @@ namespace FEXCore::Allocator {
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
   void SetupHooks() {
-    Alloc64.reset(Alloc::OSAllocator::Create64BitAllocator());
+    Alloc64 = Alloc::OSAllocator::Create64BitAllocator();
     __mmap_hook   = FEX_mmap;
     __munmap_hook = FEX_munmap;
     FEXCore::Allocator::mmap = FEX_mmap;

External/FEXCore/Source/Utils/Allocator/64BitAllocator.cpp

@@ -666,6 +666,15 @@ OSAllocator_64Bit::OSAllocator_64Bit() {
         LiveRegions = ObjectAlloc->new_construct(LiveRegions, ObjectAlloc);
       }
       else {
+
+        // If the allocation size is large than a page, then try allowing it to be a huge page
+        // This enables the kernel to use transparent large pages in the allocator which can reduce memory pressure
+        // Considering we are allocating the entire VA space, this is a good thing
+        // If MADV_HUGEPAGE isn't support then this will fail harmlessly
+        if (AllocationSize > 4096) {
+          ::madvise(Ptr, AllocationSize, MADV_HUGEPAGE);
+        }
+
         bool Merged = false;
         if (PrevReserved) {
           Merged = MergeReservedRegionIfPossible(PrevReserved, reinterpret_cast<uint64_t>(Ptr), AllocationSize);
@@ -709,7 +718,7 @@ OSAllocator_64Bit::~OSAllocator_64Bit() {
   }
 }
 
-Alloc::HostAllocator *Create64BitAllocator() {
-  return new OSAllocator_64Bit{};
+std::unique_ptr<Alloc::HostAllocator> Create64BitAllocator() {
+  return std::make_unique<OSAllocator_64Bit>();
 }
 }

External/FEXCore/Source/Utils/Allocator/HostAllocator.h

@@ -1,6 +1,8 @@
 #pragma once
+
 #include <cstddef>
 #include <cstdint>
+#include <memory>
 #include <sys/types.h>
 
 constexpr static uint64_t PAGE_SIZE = 4096;
@@ -29,16 +31,15 @@ static inline uint64_t AlignUp(uint64_t value, uint64_t size) {
       GlobalAllocator(HostAllocator *_Alloc)
         : Alloc {_Alloc} {}
 
+      virtual ~GlobalAllocator() = default;
       virtual void *malloc(size_t Size) = 0;
       virtual void *calloc(size_t num, size_t size) = 0;
       virtual void *realloc(void *ptr, size_t size) = 0;
       virtual void *memalign(size_t alignment, size_t size) = 0;
       virtual void free(void *ptr) = 0;
   };
-
-  GlobalAllocator *CreateBasicAllocator(HostAllocator *Alloc);
 }
 
 namespace Alloc::OSAllocator {
-Alloc::HostAllocator *Create64BitAllocator();
+std::unique_ptr<Alloc::HostAllocator> Create64BitAllocator();
 }

External/FEXCore/Source/Utils/Threads.cpp

@@ -14,30 +14,48 @@ namespace FEXCore::Threads {
     void *Ptr;
     size_t Size;
   };
-  std::mutex StackPoolMutex{};
-  std::deque<StackPoolItem> StackPool;
+  std::mutex DeadStackPoolMutex{};
+  std::mutex LiveStackPoolMutex{};
+
+  std::deque<StackPoolItem> DeadStackPool;
+  std::deque<StackPoolItem> LiveStackPool;
 
   void *AllocateStackObject(size_t Size) {
-    std::unique_lock<std::mutex> lk{StackPoolMutex};
-    if (StackPool.size() == 0) {
+    std::lock_guard lk{DeadStackPoolMutex};
+    if (DeadStackPool.size() == 0) {
       // Nothing in the pool, just allocate
       return FEXCore::Allocator::mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_GROWSDOWN, -1, 0);
     }
 
     // Keep the first item in the stack pool
-    auto Result = StackPool.front().Ptr;
-    StackPool.pop_front();
+    auto Result = DeadStackPool.front().Ptr;
+    DeadStackPool.pop_front();
 
     // Erase the rest as a garbage collection step
-    for (auto &Item : StackPool) {
+    for (auto &Item : DeadStackPool) {
       FEXCore::Allocator::munmap(Item.Ptr, Item.Size);
     }
     return Result;
   }
 
-  void AddStackToPool(void *Ptr, size_t Size) {
-    std::unique_lock<std::mutex> lk{StackPoolMutex};
-    StackPool.emplace_back(StackPoolItem{Ptr, Size});
+  void AddStackToDeadPool(void *Ptr, size_t Size) {
+    std::lock_guard lk{DeadStackPoolMutex};
+    DeadStackPool.emplace_back(StackPoolItem{Ptr, Size});
+  }
+
+  void AddStackToLivePool(void *Ptr, size_t Size) {
+    std::lock_guard lk{LiveStackPoolMutex};
+    LiveStackPool.emplace_back(StackPoolItem{Ptr, Size});
+  }
+
+  void RemoveStackFromLivePool(void *Ptr) {
+    std::lock_guard lk{LiveStackPoolMutex};
+    for (auto it = LiveStackPool.begin(); it != LiveStackPool.end(); ++it) {
+      if (it->Ptr == Ptr) {
+        LiveStackPool.erase(it);
+        return;
+      }
+    }
   }
 
   void *InitializeThread(void *Ptr);
@@ -49,6 +67,7 @@ namespace FEXCore::Threads {
       , UserArg {Arg} {
       pthread_attr_t Attr{};
       Stack = AllocateStackObject(STACK_SIZE);
+      AddStackToLivePool(Stack, STACK_SIZE);
       pthread_attr_init(&Attr);
       pthread_attr_setstack(&Attr, Stack, STACK_SIZE);
       pthread_create(&Thread, &Attr, Func, Arg);
@@ -87,7 +106,8 @@ namespace FEXCore::Threads {
     }
 
     void FreeStack() {
-      AddStackToPool(Stack, STACK_SIZE);
+      RemoveStackFromLivePool(Stack);
+      AddStackToDeadPool(Stack, STACK_SIZE);
     }
 
     private:
@@ -115,8 +135,38 @@ namespace FEXCore::Threads {
     return std::make_unique<PThread>(Func, Arg);
   }
 
+  void CleanupAfterFork_PThread() {
+    // We don't need to pull the mutex here
+    // After a fork we are the only thread running
+    // Just need to make sure not to delete our own stack
+    uintptr_t StackLocation = reinterpret_cast<uintptr_t>(alloca(0));
+
+    auto ClearStackPool = [&](auto &StackPool) {
+      for (auto it = StackPool.begin(); it != StackPool.end(); ) {
+        StackPoolItem &Item = *it;
+        uintptr_t ItemStack = reinterpret_cast<uintptr_t>(Item.Ptr);
+        if (ItemStack <= StackLocation && (ItemStack + Item.Size) > StackLocation) {
+          // This is our stack item, skip it
+          ++it;
+        }
+        else {
+          // Untracked stack. Clean it up
+          FEXCore::Allocator::munmap(Item.Ptr, Item.Size);
+          it = StackPool.erase(it);
+        }
+      }
+    };
+
+    // Clear both dead stacks and live stacks
+    ClearStackPool(DeadStackPool);
+    ClearStackPool(LiveStackPool);
+
+    LogMan::Throw::A((DeadStackPool.size() + LiveStackPool.size()) <= 1, "After fork we should only have zero or one tracked stacks!");
+  }
+
   static FEXCore::Threads::Pointers Ptrs = {
     .CreateThread = CreateThread_PThread,
+    .CleanupAfterFork = CleanupAfterFork_PThread,
   };
 
   std::unique_ptr<FEXCore::Threads::Thread> FEXCore::Threads::Thread::Create(
@@ -125,6 +175,10 @@ namespace FEXCore::Threads {
     return Ptrs.CreateThread(Func, Arg);
   }
 
+  void FEXCore::Threads::Thread::CleanupAfterFork() {
+    return Ptrs.CleanupAfterFork();
+  }
+
   void FEXCore::Threads::Thread::SetInternalPointers(Pointers const &_Ptrs) {
     memcpy(&Ptrs, &_Ptrs, sizeof(FEXCore::Threads::Pointers));
   }

External/FEXCore/include/FEXCore/Core/Context.h

@@ -46,8 +46,11 @@ namespace FEXCore::Context {
     MODE_32BIT,
     MODE_64BIT,
   };
+
   using CustomCPUFactoryType = std::function<std::unique_ptr<FEXCore::CPU::CPUBackend> (FEXCore::Context::Context*, FEXCore::Core::InternalThreadState *Thread)>;
 
+  using ExitHandler = std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)>;
+
   /**
    * @brief This initializes internal FEXCore state that is shared between contexts and requires overhead to setup
    */
@@ -90,8 +93,8 @@ namespace FEXCore::Context {
    */
   FEX_DEFAULT_VISIBILITY bool InitCore(FEXCore::Context::Context *CTX, FEXCore::CodeLoader *Loader);
 
-  FEX_DEFAULT_VISIBILITY void SetExitHandler(FEXCore::Context::Context *CTX, std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)> handler);
-  FEX_DEFAULT_VISIBILITY std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)> GetExitHandler(FEXCore::Context::Context *CTX);
+  FEX_DEFAULT_VISIBILITY void SetExitHandler(FEXCore::Context::Context *CTX, ExitHandler handler);
+  FEX_DEFAULT_VISIBILITY ExitHandler GetExitHandler(FEXCore::Context::Context *CTX);
 
   /**
    * @brief Pauses execution on the CPU core
@@ -216,8 +219,8 @@ namespace FEXCore::Context {
 
   FEX_DEFAULT_VISIBILITY void HandleCallback(FEXCore::Context::Context *CTX, uint64_t RIP);
 
-  FEX_DEFAULT_VISIBILITY void RegisterHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func);
-  FEX_DEFAULT_VISIBILITY void RegisterFrontendHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func);
+  FEX_DEFAULT_VISIBILITY void RegisterHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func, bool Required);
+  FEX_DEFAULT_VISIBILITY void RegisterFrontendHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func, bool Required);
 
   FEX_DEFAULT_VISIBILITY FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Context::Context *CTX, FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID);
   FEX_DEFAULT_VISIBILITY void InitializeThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);

External/FEXCore/include/FEXCore/Core/SignalDelegator.h

@@ -54,8 +54,8 @@ namespace Core {
      *
      * It's a process level signal handler so one must be careful
      */
-    virtual void RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func) = 0;
-    virtual void RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func) = 0;
+    virtual void RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required) = 0;
+    virtual void RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required) = 0;
 
     /**
      * @brief Registers a signal handler for the host to handle a signal specifically for guest handling

External/FEXCore/include/FEXCore/Core/UContext.h

@@ -96,7 +96,24 @@ namespace FEXCore {
 
   namespace x86 {
     struct FEX_PACKED siginfo_t {
-      uint32_t pad[32];
+      int si_signo;
+      int si_errno;
+      int si_code;
+      union {
+        uint32_t pad[29];
+        /* SIGILL, SIGFPE, SIGSEGV, SIBUS */
+        struct {
+          uint32_t addr;
+        } _sigfault;
+        /* SIGCHLD */
+        struct {
+          int32_t pid;
+          int32_t uid;
+          int32_t status;
+          int32_t utime;
+          int32_t stime;
+        } _sigchld;
+      } _sifields;
     };
     static_assert(sizeof(FEXCore::x86::siginfo_t) == 128, "This needs to be the right size");
 

External/FEXCore/include/FEXCore/IR/IREmitter.h

@@ -583,7 +583,11 @@ friend class FEXCore::IR::PassManager;
    * @{ */
   /**  @} */
   void LinkCodeBlocks(OrderedNode *CodeNode, OrderedNode *Next) {
-    FEXCore::IR::IROp_CodeBlock *CurrentIROp = CodeNode->Op(DualListData.DataBegin())->CW<FEXCore::IR::IROp_CodeBlock>();
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
+    FEXCore::IR::IROp_CodeBlock *CurrentIROp =
+#endif
+    CodeNode->Op(DualListData.DataBegin())->CW<FEXCore::IR::IROp_CodeBlock>();
+
     LOGMAN_THROW_A(CurrentIROp->Header.Op == IROps::OP_CODEBLOCK, "Invalid");
 
     CodeNode->append(DualListData.ListBegin(), Next);

External/FEXCore/include/FEXCore/IR/IntrusiveIRList.h

@@ -69,8 +69,8 @@ class DualIntrusiveAllocator final {
     size_t ListSize() const { return ListCurrentOffset; }
     size_t ListBackingSize() const { return MemorySize; }
 
-    uintptr_t const DataBegin() const { return Data; }
-    uintptr_t const ListBegin() const { return List; }
+    uintptr_t DataBegin() const { return Data; }
+    uintptr_t ListBegin() const { return List; }
 
     void Reset() { DataCurrentOffset = 0; ListCurrentOffset = 0; }
 
@@ -159,7 +159,7 @@ public:
     stream.write((char*)GetListData(), ListSize);
   }
 
-  size_t GetInlineSize() {
+  size_t GetInlineSize() const {
     static_assert(sizeof(*this) == 40);
     return sizeof(*this) + DataSize + ListSize;
   }
@@ -317,11 +317,11 @@ public:
     return iterator(reinterpret_cast<uintptr_t>(GetListData()), reinterpret_cast<uintptr_t>(GetData()), Wrapped);
   }
 
-  uintptr_t const GetData() const {
+  uintptr_t GetData() const {
     return reinterpret_cast<uintptr_t>(IRDataInternal ? IRDataInternal : InlineData);
   }
 
-  uintptr_t const GetListData() const {
+  uintptr_t GetListData() const {
     return reinterpret_cast<uintptr_t>(ListDataInternal ? ListDataInternal : &InlineData[DataSize]);
   }
 

External/FEXCore/include/FEXCore/Utils/BitUtils.h

@@ -6,6 +6,7 @@
 #include <climits>
 #include <cstddef>
 #include <cstdint>
+#include <cstring>
 #include <type_traits>
 
 namespace FEXCore {
@@ -61,4 +62,20 @@ template <typename T>
     return trailing_zeroes + 1;
 }
 
+// Stand-in for std::bit_cast until libc++ implements it.
+template <typename To, typename From>
+[[nodiscard]] inline To BitCast(const From& source) noexcept
+{
+  static_assert(sizeof(From) == sizeof(To),
+                "BitCast source and destination types must be equal in size.");
+  static_assert(std::is_trivially_copyable_v<From>,
+                "BitCast source type must be trivially copyable.");
+  static_assert(std::is_trivially_copyable_v<To>,
+                "BitCast destination type must be trivially copyable.");
+
+  std::aligned_storage_t<sizeof(To), alignof(To)> storage;
+  std::memcpy(&storage, &source, sizeof(storage));
+  return reinterpret_cast<To&>(storage);
+}
+
 } // namespace FEXCore

External/FEXCore/include/FEXCore/Utils/Threads.h

@@ -7,8 +7,11 @@ namespace FEXCore::Threads {
 
   class Thread;
   using CreateThreadFunc = std::function<std::unique_ptr<Thread>(ThreadFunc Func, void* Arg)>;
+  using CleanupAfterForkFunc = std::function<void()>;
+
   struct Pointers {
     CreateThreadFunc CreateThread;
+    CleanupAfterForkFunc CleanupAfterFork;
   };
 
   // API
@@ -19,10 +22,19 @@ namespace FEXCore::Threads {
     virtual bool join(void **ret) = 0;
     virtual bool detach() = 0;
     virtual bool IsSelf() = 0;
+
+    /**
+     * @name Calls provided API functions
+     * @{ */
+
     static std::unique_ptr<Thread> Create(
       ThreadFunc Func,
       void* Arg);
 
+    static void CleanupAfterFork();
+    /**  @} */
+
+    // Set API functions
     static void SetInternalPointers(Pointers const &_Ptrs);
   };
 }

Readme.md

@@ -17,7 +17,6 @@ See the [Source Outline](docs/SourceOutline.md) for more information.
 * cmake (version 3.14 minimum)
 * ninja-build
 * clang (version 10 minimum for C++20)
-* libnuma-dev
 * libglfw3-dev (For GUI)
 * libsdl2-dev (For GUI)
 * libepoxy-dev (For GUI)

Source/CMakeLists.txt

@@ -1,4 +1,6 @@
 add_subdirectory(Common/)
 add_subdirectory(CommonCore/)
+add_subdirectory(Linux/)
 add_subdirectory(Tests/)
 add_subdirectory(Tools/)
+

Source/Common/CMakeLists.txt

@@ -1,8 +1,10 @@
 set(NAME Common)
 set(SRCS
   ArgumentLoader.cpp
-  EnvironmentLoader.cpp
   Config.cpp
+  EnvironmentLoader.cpp
+  FileFormatCheck.cpp
+  RootFSSetup.cpp
   StringUtil.cpp)
 
 add_library(${NAME} STATIC ${SRCS})

Source/Common/Config.cpp

@@ -60,15 +60,16 @@ namespace FEX::Config {
     jsonPool_t PoolObject;
     std::unique_ptr<std::list<json_t>> json_objects;
   };
+  static_assert(offsetof(JsonAllocator, PoolObject) == 0, "This needs to be at offset zero");
 
   json_t* PoolInit(jsonPool_t* Pool) {
-    JsonAllocator* alloc = json_containerOf(Pool, JsonAllocator, PoolObject);
+    JsonAllocator* alloc = reinterpret_cast<JsonAllocator*>(Pool);
     alloc->json_objects = std::make_unique<std::list<json_t>>();
     return &*alloc->json_objects->emplace(alloc->json_objects->end());
   }
 
   json_t* PoolAlloc(jsonPool_t* Pool) {
-    JsonAllocator* alloc = json_containerOf(Pool, JsonAllocator, PoolObject);
+    JsonAllocator* alloc = reinterpret_cast<JsonAllocator*>(Pool);
     return &*alloc->json_objects->emplace(alloc->json_objects->end());
   }
 

Source/Common/FileFormatCheck.cpp

@@ -0,0 +1,65 @@
+#include <fstream>
+#include <string>
+
+namespace FEX::FormatCheck {
+  bool IsSquashFS(std::string const &Filename) {
+    // If it is a regular file then we need to check if it is a valid archive
+    struct SquashFSHeader {
+      uint32_t magic;
+      uint32_t inode_count;
+      uint32_t mtime;
+      uint32_t block_size;
+      uint32_t fragment_entry_count;
+      uint16_t compression_id;
+      uint16_t block_log;
+      uint16_t flags;
+      uint16_t id_count;
+      uint16_t version_major;
+      uint16_t version_minor;
+      uint64_t More[8]; // More things that don't matter to us
+    };
+
+    SquashFSHeader Header{};
+    std::fstream File(Filename, std::ios::in);
+
+    if (!File.is_open()) {
+      return false;
+    }
+
+    if (!File.seekg(0, std::fstream::end)) {
+      return false;
+    }
+
+    auto FileSize = File.tellg();
+    if (File.fail()) {
+      return false;
+    }
+
+    if (FileSize <= 0) {
+      return false;
+    }
+
+    if (!File.seekg(0, std::fstream::beg)) {
+      return false;
+    }
+
+    if (FileSize < sizeof(SquashFSHeader)) {
+      return false;
+    }
+
+    if (!File.read(reinterpret_cast<char*>(&Header), sizeof(SquashFSHeader))) {
+      return false;
+    }
+
+    // Make sure the cookie matches
+    if (Header.magic == 0x73717368) {
+      // Sanity check the version
+      uint32_t version = (uint32_t)Header.version_major << 16 | Header.version_minor;
+      if (version >= 0x00040000) {
+        // Everything is sane, we can add it
+        return true;
+      }
+    }
+    return false;
+  }
+}

Source/Common/FileFormatCheck.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <string>
+
+namespace FEX::FormatCheck {
+  bool IsSquashFS(std::string const &Filename);
+}

Source/Common/RootFSSetup.cpp

@@ -0,0 +1,124 @@
+#include "ConfigDefines.h"
+#include "Common/Config.h"
+#include "Common/FileFormatCheck.h"
+
+#include <FEXCore/Config/Config.h>
+
+#include <filesystem>
+
+#include <poll.h>
+#include <unistd.h>
+#include <sys/eventfd.h>
+#include <sys/prctl.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+
+namespace FEX::RootFS {
+bool Setup(char **const envp) {
+  // We need to setup the rootfs here
+  // If the configuration is set to use a folder then there is nothing to do
+  // If it is setup to use a squashfs then we need to do something more complex
+
+  FEX_CONFIG_OPT(LDPath, ROOTFS);
+  if (FEX::FormatCheck::IsSquashFS(LDPath())) {
+    pid_t ParentTID = ::getpid();
+    std::string ParentTIDString = std::to_string(ParentTID);
+    std::string Tmp = "/tmp/.FEXMount" + ParentTIDString + "-XXXXXX";
+    char *TempFolder = Tmp.data();
+
+    // Make the temporary mount folder
+    if (mkdtemp(TempFolder) == nullptr) {
+      LogMan::Msg::E("Couldn't create temporary mount name: %s", TempFolder);
+      return false;
+    }
+
+    // Change the permissions
+    if (chmod(TempFolder, 0777) != 0) {
+      LogMan::Msg::E("Couldn't change permissions on temporary mount: %s", TempFolder);
+      rmdir(TempFolder);
+      return false;
+    }
+
+    // Open some pipes for communicating with the new processes
+    int fds[2]{};
+    if (pipe2(fds, 0) != 0) {
+      LogMan::Msg::E("Couldn't open pipe");
+      return false;
+    }
+
+    // Convert the write pipe to a string to pass to the child process
+    std::string PipeString;
+    PipeString = std::to_string(fds[1]);
+
+    pid_t pid = fork();
+    if (pid == 0) {
+      // Child
+      close(fds[0]); // Close read end of pipe
+      const char *argv[6];
+      argv[0] = FEX_INSTALL_PREFIX "/bin/FEXMountDaemon";
+      argv[1] = LDPath().c_str();
+      argv[2] = TempFolder;
+      argv[3] = ParentTIDString.c_str();
+      argv[4] = PipeString.c_str();
+      argv[5] = nullptr;
+
+      if (execve(argv[0], (char * const*)argv, envp) == -1) {
+        // Let the parent know that we couldn't execute for some reason
+        uint64_t error{1};
+        write(fds[1], &error, sizeof(error));
+
+        // Give a hopefully helpful error message for users
+        LogMan::Msg::E("Couldn't execute: %s", argv[0]);
+        LogMan::Msg::E("This means the squashFS rootfs won't be mounted.");
+        LogMan::Msg::E("Expect errors!");
+        // Destroy this fork
+        exit(1);
+      }
+    }
+    else {
+      // Parent
+      // Wait for the child to exit so we can check if it is mounted or not
+      close(fds[1]); // Close write end of the pipe
+
+      // Wait for a message from FEXMountDaemon
+      pollfd PollFD;
+      PollFD.fd = fds[0];
+      PollFD.events = POLLIN;
+
+      poll(&PollFD, 1, -1);
+
+      // Read a value from the pipe to get an expected result
+      // This will come from FEXMountDaemon or our local fork depending on results
+      uint64_t ChildResult{};
+      int Result = read(fds[0], &ChildResult, sizeof(ChildResult));
+
+      if (Result != sizeof(ChildResult)) {
+        LogMan::Msg::D("Spurious read error");
+        return false;
+      }
+
+      if (ChildResult == 1) {
+        // Error
+        LogMan::Msg::D("FEXMountDaemon couldn't mount child for some reason");
+        return false;
+      }
+
+      // Check if we have an directory inside our temp folder
+      std::string Path = TempFolder;
+      std::string PathUser = Path + "/usr";
+      if (!std::filesystem::exists(PathUser)) {
+        LogMan::Msg::D("Child couldn't mount rootfs, /usr doesn't exist");
+        rmdir(TempFolder);
+        return false;
+      }
+
+      // If everything has passed then we can now update the rootfs path
+      FEXCore::Config::EraseSet(FEXCore::Config::CONFIG_ROOTFS, Path);
+      return true;
+    }
+  }
+
+  // Nothing to do
+  return true;
+}
+}

Source/Common/RootFSSetup.h

@@ -0,0 +1,5 @@
+#pragma once
+
+namespace FEX::RootFS {
+  bool Setup(char **const envp);
+}

Source/CommonCore/HostFactory.cpp

@@ -56,12 +56,13 @@ namespace HostFactory {
         auto InternalThread = Thread;
         HostCore *Core = reinterpret_cast<HostCore*>(InternalThread->CPUBackend.get());
         return Core->HandleSIGSEGV(Thread, Signal, info, ucontext);
-      }
+      },
+      true
     );
 
     FEXCore::Context::RegisterHostSignalHandler(CTX, 63, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
       return true;
-    });
+    }, true);
   }
 
   bool HostCore::HandleSIGSEGV(FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) {

Source/Linux/CMakeLists.txt

@@ -0,0 +1,10 @@
+set (SRCS
+  Utils/ELFContainer.cpp
+  Utils/ELFSymbolDatabase.cpp
+  )
+
+add_library(FEX_Utils OBJECT ${SRCS})
+target_link_libraries(FEX_Utils FEXCore)
+
+target_include_directories(FEX_Utils PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/Source/)
+target_include_directories(FEX_Utils PRIVATE ${CMAKE_BINARY_DIR}/generated)

Source/Linux/Utils/ELFContainer.cpp

@@ -5,8 +5,8 @@ desc: Loads and parses an elf to memory. Also handles some loading & logic.
 $end_info$
 */
 
-#include <FEXCore/Utils/Common/MathUtils.h>
-#include <FEXCore/Utils/ELFContainer.h>
+#include "Common/MathUtils.h"
+#include "Linux/Utils/ELFContainer.h"
 #include <FEXCore/Utils/LogManager.h>
 #include <cstring>
 #include <elf.h>

Source/Linux/Utils/ELFParser.h

@@ -6,7 +6,7 @@
 #include <fcntl.h>
 #include <unistd.h>
 
-#include "ELFContainer.h"
+#include "Linux/Utils/ELFContainer.h"
 
 /*
   Simpler elf parser, checks for the elf MAGIC COOKIE
@@ -31,9 +31,9 @@ struct ELFParser {
     std::ifstream elf(file);
 
     fd = ::open(file.c_str(), O_RDONLY);
-    
+
     if (fd == -1) {
-      LogMan::Msg::E("Failed to open '%s'", file.c_str());
+      // Likely just doesn't exist
       return false;
     }
 
@@ -82,12 +82,12 @@ struct ELFParser {
         LogMan::Msg::E("Invalid e_phentsize32 from '%s'", file.c_str());
         return false;
       }
-      
+
       // Convert to 64 bit header
       for (int i = 0; i < EI_NIDENT; i++)
         ehdr.e_ident[i] = hdr32.e_ident[i];
 
-      #define COPY(name) ehdr.name = hdr32.name      
+      #define COPY(name) ehdr.name = hdr32.name
       COPY(e_type);
       COPY(e_machine);
       COPY(e_version);
@@ -112,7 +112,7 @@ struct ELFParser {
 
     } else if (header[EI_CLASS] == ELFCLASS64) {
       elf.read((char*)&ehdr, sizeof(ehdr));
-      
+
       if (!elf.good()) {
         LogMan::Msg::E("Failed to read Ehdr64 from '%s'", file.c_str());
         return false;
@@ -158,12 +158,12 @@ struct ELFParser {
     if (type == ::ELFLoader::ELFContainer::TYPE_X86_32) {
       Elf32_Phdr phdrs32[ehdr.e_phnum];
       elf.read((char*)phdrs32, sizeof(Elf32_Phdr) * ehdr.e_phnum);
-      
+
       if (!elf.good()) {
         LogMan::Msg::E("Failed to read phdr32 from '%s'", file.c_str());
         return false;
       }
-      
+
       // Convert to 64 bit program headers
       phdrs.resize(ehdr.e_phnum);
 
@@ -185,13 +185,13 @@ struct ELFParser {
       phdrs.resize(ehdr.e_phnum);
 
       elf.read((char*)&phdrs[0], sizeof(Elf64_Phdr) * ehdr.e_phnum);
-      
+
       if (!elf.good()) {
         LogMan::Msg::E("Failed to read phdr64 from '%s'", file.c_str());
         return false;
       }
     }
-    
+
     for (auto phdr : phdrs) {
       if (phdr.p_type == PT_INTERP) {
         elf.seekg(phdr.p_offset);
@@ -219,4 +219,4 @@ struct ELFParser {
   ~ELFParser() {
     Closefd();
   }
-};
+};

Source/Linux/Utils/ELFSymbolDatabase.cpp

@@ -5,10 +5,11 @@ desc: Part of our now defunct ld-linux replacement, keeps tracks of all symbols,
 $end_info$
 */
 
-#include <FEXCore/Utils/ELFSymbolDatabase.h>
+#include "Common/MathUtils.h"
+#include "Linux/Utils/ELFSymbolDatabase.h"
+
 #include <FEXCore/Utils/Allocator.h>
 #include <FEXCore/Utils/LogManager.h>
-#include <FEXCore/Utils/Common/MathUtils.h>
 
 #include <cstring>
 #include <elf.h>
@@ -82,7 +83,10 @@ ELFSymbolDatabase::ELFSymbolDatabase(::ELFLoader::ELFContainer *file)
     for (auto &Lib : UnfilledDependencies) {
       if (NameToELF.find(Lib) == NameToELF.end()) {
         std::string LibraryPath;
-        bool Found = FindLibraryFile(&LibraryPath, Lib.c_str());
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
+        bool Found =
+#endif
+        FindLibraryFile(&LibraryPath, Lib.c_str());
         LOGMAN_THROW_A(Found, "Couldn't find library '%s'", Lib.c_str());
         auto Info = DynamicELFInfo.emplace_back(new ELFInfo{});
         Info->Name = Lib;

Source/Linux/Utils/ELFSymbolDatabase.h

@@ -1,13 +1,12 @@
 #pragma once
 
-#include <FEXCore/Utils/CompilerDefs.h>
-#include <FEXCore/Utils/ELFContainer.h>
+#include "Linux/Utils/ELFContainer.h"
 
 #include <unordered_map>
 #include <vector>
 
 namespace ELFLoader {
-class FEX_DEFAULT_VISIBILITY ELFSymbolDatabase final {
+class ELFSymbolDatabase final {
 public:
   ELFSymbolDatabase(::ELFLoader::ELFContainer *file);
   ~ELFSymbolDatabase();

Source/Tests/CMakeLists.txt

@@ -39,29 +39,16 @@ install(PROGRAMS "${PROJECT_SOURCE_DIR}/Scripts/FEXUpdateAOTIRCache.sh" DESTINAT
 
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
   add_custom_target(binfmt_misc_32
-    COMMAND ${CMAKE_COMMAND} -E
-    echo "Attempting to remove FEX-x86 misc prior to install. Ignore permission denied"
-    COMMAND ${CMAKE_COMMAND} -E
-    echo -1 > /proc/sys/fs/binfmt_misc/FEX-x86 || (exit 0)
-    COMMAND ${CMAKE_COMMAND} -E
     echo "Attempting to install FEX-x86 misc now."
-    COMMAND ${CMAKE_COMMAND} -E
-      echo
-      ':FEX-x86:M:0:\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x03\\x00:\\xff\\xff\\xff\\xff\\xff\\xfe\\xfe\\x00\\x00\\x00\\x00\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:${CMAKE_INSTALL_PREFIX}/bin/FEXInterpreter:CF' > /proc/sys/fs/binfmt_misc/register
+    COMMAND "update-binfmts" "--import" "FEX-x86"
     COMMAND ${CMAKE_COMMAND} -E
     echo "binfmt_misc FEX-x86 installed"
   )
 
   add_custom_target(binfmt_misc_64
-    COMMAND ${CMAKE_COMMAND} -E
-    echo "Attempting to remove FEX-x86_64 misc prior to install. Ignore permission denied"
-    COMMAND ${CMAKE_COMMAND} -E
-    echo -1 > /proc/sys/fs/binfmt_misc/FEX-x86_64 || (exit 0)
     COMMAND ${CMAKE_COMMAND} -E
     echo "Attempting to install FEX-x86_64 misc now."
-    COMMAND ${CMAKE_COMMAND} -E
-      echo
-      ':FEX-x86_64:M:0:\\x7fELF\\x02\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x3e\\x00:\\xff\\xff\\xff\\xff\\xff\\xfe\\xfe\\x00\\x00\\x00\\x00\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:${CMAKE_INSTALL_PREFIX}/bin/FEXInterpreter:CF' > /proc/sys/fs/binfmt_misc/register
+    COMMAND "update-binfmts" "--import" "FEX-x86_64"
     COMMAND ${CMAKE_COMMAND} -E
     echo "binfmt_misc FEX-x86_64 installed"
   )

Source/Tests/ELFCodeLoader.h

@@ -1,6 +1,8 @@
 #pragma once
 #include "Common/Config.h"
 #include "Common/MathUtils.h"
+#include "Linux/Utils/ELFContainer.h"
+#include "Linux/Utils/ELFSymbolDatabase.h"
 
 #include <FEXCore/Core/CodeLoader.h>
 #include <array>
@@ -15,8 +17,6 @@
 #include <FEXCore/Core/X86Enums.h>
 #include <FEXCore/Utils/Allocator.h>
 #include <FEXCore/Utils/LogManager.h>
-#include <FEXCore/Utils/ELFContainer.h>
-#include <FEXCore/Utils/ELFSymbolDatabase.h>
 
 namespace FEX::HarnessHelper {
 
@@ -194,13 +194,7 @@ public:
 
   uint64_t GetStackPointer() override {
     uintptr_t StackPointer{};
-    if (File.GetMode() == ::ELFLoader::ELFContainer::MODE_64BIT) {
-      StackPointer = reinterpret_cast<uintptr_t>(FEXCore::Allocator::mmap(nullptr, StackSize(), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
-    }
-    else {
-      StackPointer = reinterpret_cast<uintptr_t>(FEXCore::Allocator::mmap(reinterpret_cast<void*>(STACK_OFFSET), StackSize(), PROT_READ | PROT_WRITE, MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
-      LOGMAN_THROW_A(StackPointer != ~0ULL, "Get Stack Pointer mmap failed");
-    }
+    StackPointer = reinterpret_cast<uintptr_t>(FEXCore::Allocator::mmap(nullptr, StackSize(), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
 
     StackPointer += StackSize();
     // Set up our initial CPU state
@@ -307,13 +301,13 @@ public:
   }
 
   char const *FindSymbolNameInRange(uint64_t Address) {
-    
+
     ELFLoader::ELFSymbol const *Sym;
     Sym = DB.GetSymbolInRange(std::make_pair(Address, 1));
     if (Sym) {
       return Sym->Name;
     }
-    
+
     return nullptr;
   }
 
@@ -356,7 +350,6 @@ private:
   uint64_t EnvironmentBackingSize{};
 
   constexpr static uint64_t STACK_SIZE = 8 * 1024 * 1024;
-  constexpr static uint64_t STACK_OFFSET = 0xc000'0000;
 };
 
 }

Source/Tests/ELFCodeLoader2.h

@@ -2,6 +2,8 @@
 #pragma once
 #include "Common/Config.h"
 #include "Common/MathUtils.h"
+#include "Linux/Utils/ELFParser.h"
+#include "Linux/Utils/ELFSymbolDatabase.h"
 
 #include <FEXCore/Core/CodeLoader.h>
 #include <array>
@@ -16,8 +18,6 @@
 #include <FEXCore/Core/CoreState.h>
 #include <FEXCore/Core/X86Enums.h>
 #include <FEXCore/Utils/LogManager.h>
-#include <FEXCore/Utils/ELFParser.h>
-#include <FEXCore/Utils/ELFSymbolDatabase.h>
 
 #include <elf.h>
 #include <sys/personality.h>
@@ -56,7 +56,7 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
 
     if (first == headers.end())
       return 0;
-    
+
     return PAGE_ALIGN(last->p_vaddr + last->p_memsz) - PAGE_START(first->p_vaddr);
   }
 
@@ -68,6 +68,11 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
     auto off = Header.p_offset - PAGE_OFFSET(Header.p_vaddr);
 
     size = PAGE_ALIGN(size);
+    if (size == 0) {
+      // PT_LOAD section without a file size
+      // Will need to have a memory size that is not zero instead
+      return true;
+    }
 
     void *rv;
 
@@ -106,7 +111,7 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
   std::optional<uintptr_t> LoadElfFile(ELFParser& Elf, uintptr_t *BrkBase, TMap Mapper, TUnmap Unmapper) {
 
     uintptr_t LoadBase = 0;
-       
+
     if (Elf.ehdr.e_type == ET_DYN) {
       // needs base address
       auto TotalSize  = CalculateTotalElfSize(Elf.phdrs) + (BrkBase ? BRK_SIZE : 0);
@@ -135,7 +140,7 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
 			if (!MapFile(Elf, LoadBase, Header, MapProt, MapType, Mapper)) {
         return {};
       }
-	
+
       if (Header.p_memsz > Header.p_filesz) {
         // clear bss
         auto BSSStart = LoadBase + Header.p_vaddr + Header.p_filesz;
@@ -297,12 +302,8 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
     // ADDR_LIMIT_3GB STACK -> 0xc0000000 else -> 0xFFFFe000
 
     // map stack here, so that nothing gets mapped there
-    if (Is64BitMode()) {
-      StackPointer = reinterpret_cast<uintptr_t>(Mapper(nullptr, StackSize(), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK | MAP_GROWSDOWN, -1, 0));
-    }
-    else {
-      StackPointer = reinterpret_cast<uintptr_t>(Mapper(reinterpret_cast<void*>(STACK_OFFSET), StackSize(), PROT_READ | PROT_WRITE, MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK | MAP_GROWSDOWN, -1, 0));
-    }
+    // This works with both 64-bit and 32-bit. The mapper will only give us a function in the correct region
+    StackPointer = reinterpret_cast<uintptr_t>(Mapper(nullptr, StackSize(), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK | MAP_GROWSDOWN, -1, 0));
 
     if (StackPointer == ~0ULL) {
       LogMan::Msg::E("Allocating stack failed");
@@ -325,7 +326,7 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
     // XXX Randomise brk?
 
     BrkStart = (uint64_t)Mapper((void*)BrkBase, BRK_SIZE, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, 0, 0);
-    
+
     if ((void*)BrkStart == MAP_FAILED) {
       LogMan::Msg::E("Failed to allocate BRK @ %lx, %d\n", BrkBase, errno);
       return false;
@@ -585,7 +586,6 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
 
   constexpr static uint64_t BRK_SIZE = 8 * 1024 * 1024;
   constexpr static uint64_t STACK_SIZE = 8 * 1024 * 1024;
-  constexpr static uint64_t STACK_OFFSET = 0xc000'0000;
 
   std::vector<std::string> Args;
   std::vector<std::string> EnvironmentVariables;

Source/Tests/FEXBash.cpp

@@ -5,10 +5,11 @@ desc: Launches bash under FEX and passes arguments via -c to it
 $end_info$
 */
 
-#include "Config.h"
+#include "ConfigDefines.h"
 #include "Common/ArgumentLoader.h"
 #include "Common/EnvironmentLoader.h"
 #include "Common/Config.h"
+#include "Common/RootFSSetup.h"
 
 #include <FEXCore/Config/Config.h>
 #include <filesystem>
@@ -32,6 +33,12 @@ int main(int argc, char **argv, char **const envp) {
     return 0;
   }
 
+  // Ensure RootFS is setup before config options try to pull CONFIG_ROOTFS
+  if (!FEX::RootFS::Setup(envp)) {
+    LogMan::Msg::E("RootFS failure");
+    return -1;
+  }
+
   FEX_CONFIG_OPT(RootFSPath, ROOTFS);
   std::vector<const char*> Argv;
   std::string BinShPath = RootFSPath() + "/bin/sh";

Source/Tests/FEXLoader.cpp

@@ -6,19 +6,20 @@ $end_info$
 */
 
 #include "Common/ArgumentLoader.h"
-#include "Common/EnvironmentLoader.h"
 #include "Common/Config.h"
+#include "Common/EnvironmentLoader.h"
+#include "Common/RootFSSetup.h"
 #include "ELFCodeLoader.h"
 #include "ELFCodeLoader2.h"
 #include "Tests/LinuxSyscalls/x32/Syscalls.h"
 #include "Tests/LinuxSyscalls/x64/Syscalls.h"
 #include "Tests/LinuxSyscalls/SignalDelegator.h"
+#include "Linux/Utils/ELFContainer.h"
 
 #include <FEXCore/Config/Config.h>
 #include <FEXCore/Core/CodeLoader.h>
 #include <FEXCore/Core/Context.h>
 #include <FEXCore/Utils/Allocator.h>
-#include <FEXCore/Utils/ELFContainer.h>
 #include <FEXCore/Utils/LogManager.h>
 
 #include <cstdint>
@@ -251,6 +252,8 @@ void AOTGenSection(FEXCore::Context::Context *CTX, ELFCodeLoader2::LoadedSection
 
   for (int i = 0; i < get_nprocs_conf(); i++) {
     std::thread thd([&BranchTargets, CTX, &counter, &Compiled, &Section, &QueueMutex, SectionMaxAddress]() {
+      // Set the priority of the thread so it doesn't overwhelm the system when running in the background
+      setpriority(PRIO_PROCESS, ::gettid(), 19);
 
       // Setup thread - Each compilation thread uses its own backing FEX thread
       FEXCore::Core::CPUState state;
@@ -357,6 +360,12 @@ int main(int argc, char **argv, char **const envp) {
   FEXCore::Config::Set(FEXCore::Config::CONFIG_IS_INTERPRETER, IsInterpreter ? "1" : "0");
   FEXCore::Config::Set(FEXCore::Config::CONFIG_INTERPRETER_INSTALLED, IsInterpreterInstalled() ? "1" : "0");
 
+  // Ensure RootFS is setup before config options try to pull CONFIG_ROOTFS
+  if (!FEX::RootFS::Setup(envp)) {
+    LogMan::Msg::E("RootFS failure");
+    return -1;
+  }
+
   FEX_CONFIG_OPT(SilentLog, SILENTLOG);
   FEX_CONFIG_OPT(AOTIRCapture, AOTIRCAPTURE);
   FEX_CONFIG_OPT(AOTIRGenerate, AOTIRGENERATE);
@@ -407,7 +416,7 @@ int main(int argc, char **argv, char **const envp) {
   FEXCore::Config::Set(FEXCore::Config::CONFIG_APP_FILENAME, std::filesystem::canonical(Program));
   FEXCore::Config::Set(FEXCore::Config::CONFIG_IS64BIT_MODE, Loader.Is64BitMode() ? "1" : "0");
 
-  FEX::HLE::x32::MemAllocator *Allocator = nullptr;
+  std::unique_ptr<FEX::HLE::x32::MemAllocator> Allocator;
 
   if (Loader.Is64BitMode()) {
     if (!Loader.MapMemory([](void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
@@ -433,9 +442,9 @@ int main(int argc, char **argv, char **const envp) {
 
     Allocator = FEX::HLE::x32::CreateAllocator(Use32BitAllocator);
 
-    if (!Loader.MapMemory([Allocator](void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
+    if (!Loader.MapMemory([&Allocator](void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
       return Allocator->mmap(addr, length, prot, flags, fd, offset);
-    }, [Allocator](void *addr, size_t length) {
+    }, [&Allocator](void *addr, size_t length) {
       return Allocator->munmap(addr, length);
     })) {
       // failed to map
@@ -450,13 +459,9 @@ int main(int argc, char **argv, char **const envp) {
   auto CTX = FEXCore::Context::CreateNewContext();
   FEXCore::Context::InitializeContext(CTX);
 
-  std::unique_ptr<FEX::HLE::SignalDelegator> SignalDelegation = std::make_unique<FEX::HLE::SignalDelegator>();
-
-  std::unique_ptr<FEX::HLE::SyscallHandler> SyscallHandler{
-    Loader.Is64BitMode() ?
-      FEX::HLE::x64::CreateHandler(CTX, SignalDelegation.get()) :
-      FEX::HLE::x32::CreateHandler(CTX, SignalDelegation.get(), Allocator)
-  };
+  auto SignalDelegation = std::make_unique<FEX::HLE::SignalDelegator>();
+  auto SyscallHandler = Loader.Is64BitMode() ? FEX::HLE::x64::CreateHandler(CTX, SignalDelegation.get())
+                                             : FEX::HLE::x32::CreateHandler(CTX, SignalDelegation.get(), std::move(Allocator));
 
   SyscallHandler->SetCodeLoader(&Loader);
 

Source/Tests/HarnessHelpers.h

@@ -2,6 +2,8 @@
 
 #include "Common/Config.h"
 #include "Common/MathUtils.h"
+#include "Linux/Utils/ELFContainer.h"
+#include "Linux/Utils/ELFSymbolDatabase.h"
 
 #include <array>
 #include <bitset>
@@ -17,8 +19,6 @@
 #include <FEXCore/Utils/Allocator.h>
 #include <FEXCore/Utils/BitUtils.h>
 #include <FEXCore/Utils/CompilerDefs.h>
-#include <FEXCore/Utils/ELFContainer.h>
-#include <FEXCore/Utils/ELFSymbolDatabase.h>
 #include <FEXCore/Utils/LogManager.h>
 
 namespace FEX::HarnessHelper {

Source/Tests/LinuxSyscalls/CMakeLists.txt

@@ -57,7 +57,7 @@ add_library(LinuxEmulation STATIC
     Syscalls/Stubs.cpp
 )
 
-target_link_libraries(LinuxEmulation FEXCore pthread numa)
+target_link_libraries(LinuxEmulation FEXCore pthread FEX_Utils)
 target_include_directories(LinuxEmulation PRIVATE ${CMAKE_BINARY_DIR}/generated)
 target_include_directories(LinuxEmulation PRIVATE ${PROJECT_SOURCE_DIR}/External/drm-headers/include/)
 

Source/Tests/LinuxSyscalls/EmulatedFiles/EmulatedFiles.cpp

@@ -19,6 +19,25 @@ $end_info$
 using string = std::string;
 
 namespace FEX::EmulatedFile {
+  /**
+   * @brief Generates a temporary file using raw FDs
+   *
+   * Since we are hooking syscalls that are expecting to use raw FDs, we need to make sure to also use raw FDs.
+   * The guest application can leave these FDs dangling.
+   *
+   * Using glibc tmpfile creates a FILE which glibc tracks and will try cleaning up on application exit.
+   * If we are running a 32-bit application then this dangling FILE will be allocated using the FEX allcator
+   * Which will have already been cleaned up on shutdown.
+   *
+   * Dangling raw FD is safe since if the guest doesn't close them, then the kernel cleans them up on application close.
+   *
+   * @return A temporary file that we can use
+   */
+  static int GenTmpFD() {
+    int fd = open("/tmp", O_RDWR | O_TMPFILE | O_EXCL |  S_IRUSR | S_IWUSR);
+    return fd;
+  }
+
   std::string GenerateCPUInfo(FEXCore::Context::Context *ctx, uint32_t CPUCores) {
     std::ostringstream cpu_stream{};
     auto res_0 = FEXCore::Context::RunCPUIDFunction(ctx, 0, 0);
@@ -591,48 +610,48 @@ namespace FEX::EmulatedFile {
   EmulatedFDManager::EmulatedFDManager(FEXCore::Context::Context *ctx)
     : CTX {ctx} {
     FDReadCreators["/proc/cpuinfo"] = [&](FEXCore::Context::Context *ctx, int32_t fd, const char *pathname, int32_t flags, mode_t mode) -> int32_t {
-      FILE *fp = tmpfile();
-      fwrite((void*)&cpu_info.at(0), sizeof(uint8_t), cpu_info.size(), fp);
-      fseek(fp, 0, SEEK_SET);
-      int32_t f = fileno(fp);
-      return f;
+      int FD = GenTmpFD();
+      write(FD, (void*)&cpu_info.at(0), cpu_info.size());
+      lseek(FD, 0, SEEK_SET);
+      return FD;
     };
 
     FDReadCreators["/proc/sys/kernel/osrelease"] = [&](FEXCore::Context::Context *ctx, int32_t fd, const char *pathname, int32_t flags, mode_t mode) -> int32_t {
-      FILE *fp = tmpfile();
+      int FD = GenTmpFD();
       uint32_t GuestVersion = FEX::HLE::_SyscallHandler->GetGuestKernelVersion();
-      fprintf(fp, "%d.%d.%d\n",
+      char Tmp[64]{};
+      snprintf(Tmp, sizeof(Tmp), "%d.%d.%d\n",
         FEX::HLE::SyscallHandler::KernelMajor(GuestVersion),
         FEX::HLE::SyscallHandler::KernelMinor(GuestVersion),
         FEX::HLE::SyscallHandler::KernelPatch(GuestVersion));
-      fputc('\0', fp);
-      fseek(fp, 0, SEEK_SET);
-      int32_t f = fileno(fp);
-      return f;
+      // + 1 to ensure null at the end
+      write(FD, Tmp, strlen(Tmp) + 1);
+      lseek(FD, 0, SEEK_SET);
+      return FD;
     };
 
     FDReadCreators["/proc/version"] = [&](FEXCore::Context::Context *ctx, int32_t fd, const char *pathname, int32_t flags, mode_t mode) -> int32_t {
-      FILE *fp = tmpfile();
+      int FD = GenTmpFD();
       // UTS version NEEDS to be in a format that can pass to `date -d`
       // Format of this is Linux version <Release> (<Compile By>@<Compile Host>) (<Linux Compiler>) #<version> {SMP, PREEMPT, PREEMPT_RT} <UTS version>\n"
       const char kernel_version[] = "Linux version %d.%d.%d (FEX@FEX) (clang) #" GIT_DESCRIBE_STRING " SMP " __DATE__ " " __TIME__ "\n";
       uint32_t GuestVersion = FEX::HLE::_SyscallHandler->GetGuestKernelVersion();
-      fprintf(fp, kernel_version,
+      char Tmp[sizeof(kernel_version) + 64]{};
+      snprintf(Tmp, sizeof(Tmp), kernel_version,
         FEX::HLE::SyscallHandler::KernelMajor(GuestVersion),
         FEX::HLE::SyscallHandler::KernelMinor(GuestVersion),
         FEX::HLE::SyscallHandler::KernelPatch(GuestVersion));
-      fputc('\0', fp);
-      fseek(fp, 0, SEEK_SET);
-      int32_t f = fileno(fp);
-      return f;
+      // + 1 to ensure null at the end
+      write(FD, Tmp, strlen(Tmp) + 1);
+      lseek(FD, 0, SEEK_SET);
+      return FD;
     };
 
     auto NumCPUCores = [&](FEXCore::Context::Context *ctx, int32_t fd, const char *pathname, int32_t flags, mode_t mode) -> int32_t {
-      FILE *fp = tmpfile();
-      fwrite((void*)&cpus_online.at(0), sizeof(uint8_t), cpus_online.size(), fp);
-      fseek(fp, 0, SEEK_SET);
-      int32_t f = fileno(fp);
-      return f;
+      int FD = GenTmpFD();
+      write(FD, (void*)&cpus_online.at(0), cpus_online.size());
+      lseek(FD, 0, SEEK_SET);
+      return FD;
     };
 
     FDReadCreators["/sys/devices/system/cpu/online"] = NumCPUCores;
@@ -644,23 +663,22 @@ namespace FEX::EmulatedFile {
     FDReadCreators["/proc/self/auxv"] = &EmulatedFDManager::ProcAuxv;
 
     auto cmdline_handler = [&](FEXCore::Context::Context *ctx, int32_t fd, const char *pathname, int32_t flags, mode_t mode) -> int32_t {
-      FILE *fp = tmpfile();
+      int FD = GenTmpFD();
       auto CodeLoader = FEX::HLE::_SyscallHandler->GetCodeLoader();
       auto Args = CodeLoader->GetApplicationArguments();
+      char NullChar{};
       // cmdline is an array of null terminated arguments
       for (size_t i = 1; i < Args->size(); ++i) {
         auto &Arg = Args->at(i);
-        fwrite(Arg.c_str(), sizeof(uint8_t), Arg.size(), fp);
+        write(FD, Arg.c_str(), Arg.size());
         // Finish off with a null terminator
-        fwrite("\0", sizeof(uint8_t), 1, fp);
+        write(FD, &NullChar, sizeof(uint8_t));
       }
 
       // One additional null terminator to finish the list
-      fwrite("\0", sizeof(uint8_t), 1, fp);
-
-      fseek(fp, 0, SEEK_SET);
-      int32_t f = fileno(fp);
-      return f;
+      write(FD, &NullChar, sizeof(uint8_t));
+      lseek(FD, 0, SEEK_SET);
+      return FD;
     };
 
     FDReadCreators["/proc/self/cmdline"] = cmdline_handler;
@@ -710,11 +728,10 @@ namespace FEX::EmulatedFile {
       return -1;
     }
 
-    FILE* fp = tmpfile();
-    fwrite((void*)auxvBase, 1, auxvSize, fp);
-    fseek(fp, 0, SEEK_SET);
-    int32_t f = fileno(fp);
-    return f;
+    int FD = GenTmpFD();
+    write(FD, (void*)auxvBase, auxvSize);
+    lseek(FD, 0, SEEK_SET);
+    return FD;
   }
 }
 

Source/Tests/LinuxSyscalls/FileManagement.cpp

@@ -31,41 +31,38 @@ $end_info$
 namespace FEX::HLE {
 
 static bool LoadFile(std::vector<char> &Data, const std::string &Filename) {
-  std::fstream File;
-  File.open(Filename, std::ios::in);
+  std::fstream File(Filename, std::ios::in);
 
   if (!File.is_open()) {
     return false;
   }
 
   if (!File.seekg(0, std::fstream::end)) {
-    LogMan::Msg::D("Couldn't load configuration file: Seek end");
+    LogMan::Msg::DFmt("Couldn't load configuration file: Seek end");
     return false;
   }
 
   auto FileSize = File.tellg();
   if (File.fail()) {
-    LogMan::Msg::D("Couldn't load configuration file: tellg");
+    LogMan::Msg::DFmt("Couldn't load configuration file: tellg");
     return false;
   }
 
   if (!File.seekg(0, std::fstream::beg)) {
-    LogMan::Msg::D("Couldn't load configuration file: Seek beginning");
+    LogMan::Msg::DFmt("Couldn't load configuration file: Seek beginning");
     return false;
   }
 
-  if (FileSize > 0) {
-    Data.resize(FileSize);
-    if (!File.read(&Data.at(0), FileSize)) {
-      // Probably means permissions aren't set. Just early exit
-      return false;
-    }
-    File.close();
-  }
-  else {
+  if (FileSize <= 0) {
+    LogMan::Msg::DFmt("FileSize less than or equal to zero specified");
     return false;
   }
 
+  Data.resize(FileSize);
+  if (!File.read(Data.data(), FileSize)) {
+    // Probably means permissions aren't set. Just early exit
+    return false;
+  }
   return true;
 }
 
@@ -349,8 +346,8 @@ uint64_t FileManager::Openat([[maybe_unused]] int dirfs, const char *pathname, i
   }
 
   if (fd != -1) {
-    std::lock_guard<std::mutex> lk(FDLock);
-    FDToNameMap[fd] = SelfPath;
+    std::lock_guard lk(FDLock);
+    FDToNameMap.insert_or_assign(fd, SelfPath);
   }
 
   return fd;
@@ -378,8 +375,8 @@ uint64_t FileManager::Openat2(int dirfs, const char *pathname, FEX::HLE::open_ho
   }
 
   if (fd != -1) {
-    std::lock_guard<std::mutex> lk(FDLock);
-    FDToNameMap[fd] = SelfPath;
+    std::lock_guard lk(FDLock);
+    FDToNameMap.insert_or_assign(fd, SelfPath);
   }
 
   return fd;

Source/Tests/LinuxSyscalls/FileManagement.h

@@ -63,7 +63,7 @@ private:
   std::mutex FDLock;
   std::unordered_map<int32_t, std::string> FDToNameMap;
   std::string GetEmulatedPath(const char *pathname, bool FollowSymlink = false);
-  std::map<std::string, std::string> ThunkOverlays;
+  std::map<std::string, std::string, std::less<>> ThunkOverlays;
 
   FEX_CONFIG_OPT(Filename, APP_FILENAME);
   FEX_CONFIG_OPT(LDPath, ROOTFS);

Source/Tests/LinuxSyscalls/SignalDelegator.cpp

@@ -214,33 +214,44 @@ namespace FEX::HLE {
       return false;
     }
 
-    // Now install the thunk handler
-    SignalHandler.HostAction.sa_sigaction = &SignalHandlerThunk;
+    // Default flags for us
     SignalHandler.HostAction.sa_flags = SA_SIGINFO | SA_RESTART | SA_ONSTACK;
 
+    if (HostHandlers[Signal].Required == false &&
+        (SignalHandler.GuestAction.sigaction_handler.handler == SIG_DFL ||
+         SignalHandler.GuestAction.sigaction_handler.handler == SIG_IGN)) {
+      // If getting set to DFL or IGN on first install then just install to those
+      SignalHandler.HostAction.sa_handler = SignalHandler.GuestAction.sigaction_handler.handler;
+    }
+    else {
+      // Now install the thunk handler
+      SignalHandler.HostAction.sa_sigaction = &SignalHandlerThunk;
+    }
+
     if (SignalHandler.GuestAction.sa_flags & SA_NODEFER) {
       // If the guest is using NODEFER then make sure to set it for the host as well
       SignalHandler.HostAction.sa_flags |= SA_NODEFER;
     }
 
-    /*
-     * XXX: This isn't quite as straightforward as a memcmp
-     * There are conflicting definitions between sigset_t and __sigset_t causing problems here
-    sigset_t EmptySet{};
-    sigemptyset(&EmptySet);
-    if (SignalHandler.GuestAction.sa_mask != EmptySet) {
-      // If the guest has masked some signals then we need to also mask those signals
-      SignalHandler.HostAction.sa_mask = SignalHandler.GuestAction.sa_mask;
+    // Walk the signals we have that are required and make sure to remove it from the mask
+    // This'll likely be SIGILL, SIGBUS, SIG63
 
-      // If the guest tried masking SIGILL or SIGBUS then too bad, we actually need this on the host
-      sigdelset(SignalHandler.HostAction.sa_mask, SIGILL);
-      sigdelset(SignalHandler.HostAction.sa_mask, SIGBUS);
+    // If the guest has masked some signals then we need to also mask those signals
+    sigemptyset(&SignalHandler.HostAction.sa_mask);
+    for (size_t i = 1; i < HostHandlers.size(); ++i) {
+      if (HostHandlers[i].Required) {
+        sigdelset(&SignalHandler.HostAction.sa_mask, i);
+      }
+      else if (SigIsMember(&SignalHandler.GuestAction.sa_mask, i)) {
+        sigaddset(&SignalHandler.HostAction.sa_mask, i);
+      }
     }
-    */
 
     // We don't care about the previous handler in this case
     int Result = sigaction(Signal, &SignalHandler.HostAction, &SignalHandler.OldAction);
-    if (Result < 0) {
+    if (Result < 0 &&
+        !(Signal == 32 || Signal == 33)) {
+      // Signal 32 and 33 are consumed by glibc. We don't handle this atm
       LogMan::Msg::E("Failed to install host signal thunk for signal %d: %s", Signal, strerror(errno));
       return false;
     }
@@ -251,32 +262,46 @@ namespace FEX::HLE {
 
   void SignalDelegator::UpdateHostThunk(int Signal) {
     SignalHandler &SignalHandler = HostHandlers[Signal];
-    bool Changed{};
 
     // This only gets called if a guest thunk was already installed and we need to check if we need to update the flags or signal mask
     if ((SignalHandler.GuestAction.sa_flags ^ SignalHandler.HostAction.sa_flags) & SA_NODEFER) {
       // NODEFER changed, we need to update this
       SignalHandler.HostAction.sa_flags |= SignalHandler.GuestAction.sa_flags & SA_NODEFER;
-      Changed = true;
     }
 
-    /*
-    if ((SignalHandler.GuestAction.sa_mask ^ SignalHandler.HostAction.sa_mask) & ~(SIGILL | SIGBUS)) {
-      // If the signal ignore mask has updated (avoiding the two we need for the host) then we need to update
-      SignalHandler.HostAction.sa_mask = SignalHandler.GuestAction.sa_mask;
-      sigdelset(SignalHandler.HostAction.sa_mask, SIGILL);
-      sigdelset(SignalHandler.HostAction.sa_mask, SIGBUS);
-      Changed = true;
+    if ((SignalHandler.GuestAction.sa_flags ^ SignalHandler.HostAction.sa_flags) & SA_RESTART) {
+      // RESTART changed, we need to update this
+      SignalHandler.HostAction.sa_flags |= SignalHandler.GuestAction.sa_flags & SA_RESTART;
     }
-    */
 
-    if (!Changed) {
-      return;
+    if (HostHandlers[Signal].Required == false &&
+        (SignalHandler.GuestAction.sigaction_handler.handler == SIG_DFL ||
+         SignalHandler.GuestAction.sigaction_handler.handler == SIG_IGN)) {
+      // If we are changing a none required signal back to DFL or IGN then we can allow this
+      SignalHandler.HostAction.sa_handler = SignalHandler.GuestAction.sigaction_handler.handler;
+    }
+    else {
+      // Set the handler to host handler
+      SignalHandler.HostAction.sa_sigaction = &SignalHandlerThunk;
+    }
+
+    // Walk the signals we have that are required and make sure to remove it from the mask
+    // This'll likely be SIGILL, SIGBUS, SIG63
+    sigemptyset(&SignalHandler.HostAction.sa_mask);
+    for (size_t i = 1; i < HostHandlers.size(); ++i) {
+      if (HostHandlers[i].Required) {
+        sigdelset(&SignalHandler.HostAction.sa_mask, i);
+      }
+      else if (SigIsMember(&SignalHandler.GuestAction.sa_mask, i)) {
+        sigaddset(&SignalHandler.HostAction.sa_mask, i);
+      }
     }
 
     // Only update our host signal here
     int Result = sigaction(Signal, &SignalHandler.HostAction, nullptr);
-    if (Result < 0) {
+    if (Result < 0 &&
+        !(Signal == 32 || Signal == 33)) {
+      // Signal 32 and 33 are consumed by glibc. We don't handle this atm
       LogMan::Msg::E("Failed to update host signal thunk for signal %d: %s", Signal, strerror(errno));
     }
   }
@@ -318,7 +343,7 @@ namespace FEX::HLE {
       {SIGWINCH,  DEFAULT_IGNORE},
     }};
 
-    for (const auto [Signal, Behaviour] : SignalDefaultBehaviours) {
+    for (const auto &[Signal, Behaviour] : SignalDefaultBehaviours) {
       HostHandlers[Signal].DefaultBehaviour = Behaviour;
     }
   }
@@ -418,26 +443,27 @@ namespace FEX::HLE {
     return true;
   }
 
-  void SignalDelegator::RegisterHostSignalHandler(int Signal, FEXCore::HostSignalDelegatorFunction Func) {
+  void SignalDelegator::RegisterHostSignalHandler(int Signal, FEXCore::HostSignalDelegatorFunction Func, bool Required) {
     // Linux signal handlers are per-process rather than per thread
     // Multiple threads could be calling in to this
     std::lock_guard lk(HostDelegatorMutex);
     HostHandlers[Signal].Handler = std::move(Func);
+    HostHandlers[Signal].Required = Required;
     InstallHostThunk(Signal);
   }
 
-  void SignalDelegator::RegisterFrontendHostSignalHandler(int Signal, FEXCore::HostSignalDelegatorFunction Func) {
+  void SignalDelegator::RegisterFrontendHostSignalHandler(int Signal, FEXCore::HostSignalDelegatorFunction Func, bool Required) {
     // Linux signal handlers are per-process rather than per thread
     // Multiple threads could be calling in to this
     std::lock_guard lk(HostDelegatorMutex);
     HostHandlers[Signal].FrontendHandler = std::move(Func);
+    HostHandlers[Signal].Required = Required;
     InstallHostThunk(Signal);
   }
 
   void SignalDelegator::RegisterHostSignalHandlerForGuest(int Signal, FEXCore::HostSignalDelegatorFunctionForGuest Func) {
     std::lock_guard lk(HostDelegatorMutex);
     HostHandlers[Signal].GuestHandler = std::move(Func);
-    InstallHostThunk(Signal);
   }
 
   uint64_t SignalDelegator::RegisterGuestSignalHandler(int Signal, const FEXCore::GuestSigAction *Action, FEXCore::GuestSigAction *OldAction) {
@@ -560,6 +586,24 @@ namespace FEX::HLE {
       else {
         return -EINVAL;
       }
+
+      // Now actually set the host mask
+      // This will hide from the guest that we are not actually setting all of the masks it wants
+      sigset_t HostSet{};
+      sigemptyset(&HostSet);
+
+      for (size_t i = 0; i < MAX_SIGNALS; ++i) {
+        if (HostHandlers[i + 1].Required) {
+          // If it is a required host signal then we can't mask it
+          continue;
+        }
+
+        if (ThreadData.CurrentSignalMask.Val & (1ULL << i)) {
+          sigaddset(&HostSet, i + 1);
+        }
+      }
+
+      pthread_sigmask(SIG_SETMASK, &HostSet, nullptr);
     }
 
     CheckForPendingSignals();
@@ -573,6 +617,19 @@ namespace FEX::HLE {
     }
 
     *set = ThreadData.PendingSignals;
+
+    sigset_t HostSet{};
+    if (sigpending(&HostSet) == 0) {
+      uint64_t HostSignals{};
+      for (size_t i = 0; i < MAX_SIGNALS; ++i) {
+        if (sigismember(&HostSet, i + 1)) {
+          HostSignals |= (1ULL << i);
+        }
+      }
+
+      // Merge the real pending signal mask as well
+      *set |= HostSignals;
+    }
     return 0;
   }
 

Source/Tests/LinuxSyscalls/SignalDelegator.h

@@ -62,8 +62,8 @@ namespace FEX::HLE {
      *
      * It's a process level signal handler so one must be careful
      */
-    void RegisterHostSignalHandler(int Signal, FEXCore::HostSignalDelegatorFunction Func) override;
-    void RegisterFrontendHostSignalHandler(int Signal, FEXCore::HostSignalDelegatorFunction Func) override;
+    void RegisterHostSignalHandler(int Signal, FEXCore::HostSignalDelegatorFunction Func, bool Required) override;
+    void RegisterFrontendHostSignalHandler(int Signal, FEXCore::HostSignalDelegatorFunction Func, bool Required) override;
 
     /**
      * @brief Registers a signal handler for the host to handle a signal specifically for guest handling
@@ -100,6 +100,7 @@ namespace FEX::HLE {
 
     struct SignalHandler {
       std::atomic<bool> Installed{};
+      bool Required{};
       struct sigaction HostAction{};
       struct sigaction OldAction{};
       FEXCore::HostSignalDelegatorFunction Handler{};

Source/Tests/LinuxSyscalls/Syscalls.cpp

@@ -8,6 +8,7 @@ $end_info$
 
 #include <FEXCore/Utils/LogManager.h>
 #include "Common/MathUtils.h"
+#include "Linux/Utils/ELFContainer.h"
 
 #include "Tests/LinuxSyscalls/Syscalls.h"
 #include "Tests/LinuxSyscalls/Syscalls/Thread.h"
@@ -18,7 +19,6 @@ $end_info$
 #include <FEXCore/Core/CodeLoader.h>
 #include <FEXCore/Debug/InternalThreadState.h>
 #include <FEXCore/Utils/Allocator.h>
-#include <FEXCore/Utils/ELFContainer.h>
 #include <fcntl.h>
 #include <filesystem>
 #include <fstream>
@@ -204,35 +204,39 @@ static bool AllFlagsSet(uint64_t Flags, uint64_t Mask) {
 
 uint64_t CloneHandler(FEXCore::Core::CpuStateFrame *Frame, FEX::HLE::clone3_args *args) {
   uint64_t flags = args->flags;
-#define FLAGPRINT(x, y) if (args->flags & (y)) LogMan::Msg::I("\tFlag: " #x)
-  FLAGPRINT(CSIGNAL,              0x000000FF);
-  FLAGPRINT(CLONE_VM,             0x00000100);
-  FLAGPRINT(CLONE_FS,             0x00000200);
-  FLAGPRINT(CLONE_FILES,          0x00000400);
-  FLAGPRINT(CLONE_SIGHAND,        0x00000800);
-  FLAGPRINT(CLONE_PTRACE,         0x00002000);
-  FLAGPRINT(CLONE_VFORK,          0x00004000);
-  FLAGPRINT(CLONE_PARENT,         0x00008000);
-  FLAGPRINT(CLONE_THREAD,         0x00010000);
-  FLAGPRINT(CLONE_NEWNS,          0x00020000);
-  FLAGPRINT(CLONE_SYSVSEM,        0x00040000);
-  FLAGPRINT(CLONE_SETTLS,         0x00080000);
-  FLAGPRINT(CLONE_PARENT_SETTID,  0x00100000);
-  FLAGPRINT(CLONE_CHILD_CLEARTID, 0x00200000);
-  FLAGPRINT(CLONE_DETACHED,       0x00400000);
-  FLAGPRINT(CLONE_UNTRACED,       0x00800000);
-  FLAGPRINT(CLONE_CHILD_SETTID,   0x01000000);
-  FLAGPRINT(CLONE_NEWCGROUP,      0x02000000);
-  FLAGPRINT(CLONE_NEWUTS,         0x04000000);
-  FLAGPRINT(CLONE_NEWIPC,         0x08000000);
-  FLAGPRINT(CLONE_NEWUSER,        0x10000000);
-  FLAGPRINT(CLONE_NEWPID,         0x20000000);
-  FLAGPRINT(CLONE_NEWNET,         0x40000000);
-  FLAGPRINT(CLONE_IO,             0x80000000);
+  auto PrintFlags = [](uint64_t Flags) -> void {
+#define FLAGPRINT(x, y) if (Flags & (y)) LogMan::Msg::I("\tFlag: " #x)
+    FLAGPRINT(CSIGNAL,              0x000000FF);
+    FLAGPRINT(CLONE_VM,             0x00000100);
+    FLAGPRINT(CLONE_FS,             0x00000200);
+    FLAGPRINT(CLONE_FILES,          0x00000400);
+    FLAGPRINT(CLONE_SIGHAND,        0x00000800);
+    FLAGPRINT(CLONE_PTRACE,         0x00002000);
+    FLAGPRINT(CLONE_VFORK,          0x00004000);
+    FLAGPRINT(CLONE_PARENT,         0x00008000);
+    FLAGPRINT(CLONE_THREAD,         0x00010000);
+    FLAGPRINT(CLONE_NEWNS,          0x00020000);
+    FLAGPRINT(CLONE_SYSVSEM,        0x00040000);
+    FLAGPRINT(CLONE_SETTLS,         0x00080000);
+    FLAGPRINT(CLONE_PARENT_SETTID,  0x00100000);
+    FLAGPRINT(CLONE_CHILD_CLEARTID, 0x00200000);
+    FLAGPRINT(CLONE_DETACHED,       0x00400000);
+    FLAGPRINT(CLONE_UNTRACED,       0x00800000);
+    FLAGPRINT(CLONE_CHILD_SETTID,   0x01000000);
+    FLAGPRINT(CLONE_NEWCGROUP,      0x02000000);
+    FLAGPRINT(CLONE_NEWUTS,         0x04000000);
+    FLAGPRINT(CLONE_NEWIPC,         0x08000000);
+    FLAGPRINT(CLONE_NEWUSER,        0x10000000);
+    FLAGPRINT(CLONE_NEWPID,         0x20000000);
+    FLAGPRINT(CLONE_NEWNET,         0x40000000);
+    FLAGPRINT(CLONE_IO,             0x80000000);
+#undef FLAGPRINT
+  };
 
   auto Thread = Frame->Thread;
 
   if (AnyFlagsSet(flags, CLONE_UNTRACED | CLONE_PTRACE)) {
+    PrintFlags(flags);
     LogMan::Msg::D("clone: Ptrace* not supported");
   }
 
@@ -245,10 +249,12 @@ uint64_t CloneHandler(FEXCore::Core::CpuStateFrame *Frame, FEX::HLE::clone3_args
 #endif
 
   if (AnyFlagsSet(flags, CLONE_CLEAR_SIGHAND)) {
+    PrintFlags(flags);
     LogMan::Msg::D("clone3: CLONE_CLEAR_SIGHAND unsupported");
   }
 
   if (AnyFlagsSet(flags, CLONE_INTO_CGROUP)) {
+    PrintFlags(flags);
     LogMan::Msg::D("clone3: CLONE_INTO_CGROUP unsupported");
     return -EOPNOTSUPP;
   }
@@ -261,6 +267,7 @@ uint64_t CloneHandler(FEXCore::Core::CpuStateFrame *Frame, FEX::HLE::clone3_args
   if (AnyFlagsSet(flags, CLONE_NEWNS | CLONE_NEWCGROUP | CLONE_NEWUTS | CLONE_NEWIPC | CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNET)) {
     // NEWUSER doesn't need any privileges from 3.8 onward
     // We just don't support it yet
+    PrintFlags(flags);
     LogMan::Msg::I("Unconditionally returning EPERM on clone namespace");
     return -EPERM;
   }
@@ -268,12 +275,14 @@ uint64_t CloneHandler(FEXCore::Core::CpuStateFrame *Frame, FEX::HLE::clone3_args
   if (!(flags & CLONE_THREAD)) {
 
     if (flags & CLONE_VFORK) {
+      PrintFlags(flags);
       flags &= ~CLONE_VFORK;
       flags &= ~CLONE_VM;
       LogMan::Msg::D("clone: WARNING: CLONE_VFORK w/o CLONE_THREAD");
     }
 
     if (AnyFlagsSet(flags, CLONE_SYSVSEM | CLONE_FS |  CLONE_FILES | CLONE_SIGHAND | CLONE_VM)) {
+      PrintFlags(flags);
       LogMan::Msg::I("clone: Unsuported flags w/o CLONE_THREAD (Shared Resources), %X", flags);
       return -EPERM;
     }
@@ -285,8 +294,8 @@ uint64_t CloneHandler(FEXCore::Core::CpuStateFrame *Frame, FEX::HLE::clone3_args
       reinterpret_cast<pid_t*>(args->child_tid),
       reinterpret_cast<void*>(args->tls));
   } else {
-
     if (!AllFlagsSet(flags, CLONE_SYSVSEM | CLONE_FS |  CLONE_FILES | CLONE_SIGHAND)) {
+      PrintFlags(flags);
       LogMan::Msg::I("clone: CLONE_THREAD: Unsuported flags w/ CLONE_THREAD (Shared Resources), %X", flags);
       return -EPERM;
     }

Source/Tests/LinuxSyscalls/Syscalls.h

@@ -123,6 +123,10 @@ public:
   uint32_t GetHostKernelVersion() const { return HostKernelVersion; }
   uint32_t GetGuestKernelVersion() const { return GuestKernelVersion; }
 
+  bool IsHostKernelVersionAtLeast(uint32_t Major, uint32_t Minor = 0, uint32_t Patch = 0) const {
+    return GetHostKernelVersion() >= KernelVersion(Major, Minor, Patch);
+  }
+
   static uint32_t CalculateHostKernelVersion();
   uint32_t CalculateGuestKernelVersion();
 

Source/Tests/LinuxSyscalls/Syscalls/FD.cpp

@@ -206,7 +206,7 @@ namespace FEX::HLE {
       SYSCALL_ERRNO();
     });
 
-    if (Handler->GetHostKernelVersion() >= FEX::HLE::SyscallHandler::KernelVersion(5, 8, 0)) {
+    if (Handler->IsHostKernelVersionAtLeast(5, 8, 0)) {
       // Only exists on kernel 5.8+
       REGISTER_SYSCALL_IMPL(faccessat2, [](FEXCore::Core::CpuStateFrame *Frame, int dirfd, const char *pathname, int mode, int flags) -> uint64_t {
         uint64_t Result = FEX::HLE::_SyscallHandler->FM.FAccessat2(dirfd, pathname, mode, flags);
@@ -331,7 +331,7 @@ namespace FEX::HLE {
       SYSCALL_ERRNO();
     });
 
-    if (Handler->GetHostKernelVersion() >= FEX::HLE::SyscallHandler::KernelVersion(5, 3, 0)) {
+    if (Handler->IsHostKernelVersionAtLeast(5, 3, 0)) {
       REGISTER_SYSCALL_IMPL(pidfd_open, [](FEXCore::Core::CpuStateFrame *Frame, pid_t pid, unsigned int flags) -> uint64_t {
         uint64_t Result = ::syscall(SYS_pidfd_open, pid, flags);
         SYSCALL_ERRNO();
@@ -341,7 +341,7 @@ namespace FEX::HLE {
       REGISTER_SYSCALL_IMPL(pidfd_open, UnimplementedSyscallSafe);
     }
 
-    if (Handler->GetHostKernelVersion() >= FEX::HLE::SyscallHandler::KernelVersion(5, 9, 0)) {
+    if (Handler->IsHostKernelVersionAtLeast(5, 9, 0)) {
       REGISTER_SYSCALL_IMPL(close_range, [](FEXCore::Core::CpuStateFrame *Frame, unsigned int first, unsigned int last, unsigned int flags) -> uint64_t {
         uint64_t Result = FEX::HLE::_SyscallHandler->FM.CloseRange(first, last, flags);
         SYSCALL_ERRNO();

Source/Tests/LinuxSyscalls/Syscalls/FS.cpp

@@ -15,7 +15,6 @@ $end_info$
 #include <sys/swap.h>
 #include <sys/syscall.h>
 #include <sys/types.h>
-#include <sys/vfs.h>
 #include <unistd.h>
 #include <fcntl.h>
 #include <sys/xattr.h>
@@ -110,16 +109,6 @@ namespace FEX::HLE {
 #endif
     });
 
-    REGISTER_SYSCALL_IMPL(statfs, [](FEXCore::Core::CpuStateFrame *Frame, const char *path, struct statfs *buf) -> uint64_t {
-      uint64_t Result = FEX::HLE::_SyscallHandler->FM.Statfs(path, buf);
-      SYSCALL_ERRNO();
-    });
-
-    REGISTER_SYSCALL_IMPL(fstatfs, [](FEXCore::Core::CpuStateFrame *Frame, int fd, struct statfs *buf) -> uint64_t {
-      uint64_t Result = ::fstatfs(fd, buf);
-      SYSCALL_ERRNO();
-    });
-
     REGISTER_SYSCALL_IMPL(truncate, [](FEXCore::Core::CpuStateFrame *Frame, const char *path, off_t length) -> uint64_t {
       uint64_t Result = ::truncate(path, length);
       SYSCALL_ERRNO();

Source/Tests/LinuxSyscalls/Syscalls/IOUring.cpp

@@ -20,7 +20,7 @@ namespace SignalDelegator {
 
 namespace FEX::HLE {
   void RegisterIOUring(FEX::HLE::SyscallHandler *const Handler) {
-    if (Handler->GetHostKernelVersion() >= FEX::HLE::SyscallHandler::KernelVersion(5, 1, 0)) {
+    if (Handler->IsHostKernelVersionAtLeast(5, 1, 0)) {
       REGISTER_SYSCALL_IMPL(io_uring_setup, [](FEXCore::Core::CpuStateFrame *Frame, uint32_t entries, void* params) -> uint64_t {
         uint64_t Result = ::syscall(SYS_io_uring_setup, entries, params);
         SYSCALL_ERRNO();

Source/Tests/LinuxSyscalls/Syscalls/SHM.cpp

@@ -23,10 +23,5 @@ namespace FEX::HLE {
       uint64_t Result = ::shmctl(shmid, cmd, buf);
       SYSCALL_ERRNO();
     });
-
-    REGISTER_SYSCALL_IMPL(shmdt, [](FEXCore::Core::CpuStateFrame *Frame, const void *shmaddr) -> uint64_t {
-      uint64_t Result = ::shmdt(shmaddr);
-      SYSCALL_ERRNO();
-    });
   }
 }

Source/Tests/LinuxSyscalls/Syscalls/Signals.cpp

@@ -42,9 +42,9 @@ namespace FEX::HLE {
       SYSCALL_ERRNO();
     });
 
-    if (Handler->GetHostKernelVersion() >= FEX::HLE::SyscallHandler::KernelVersion(5, 1, 0)) {
+    if (Handler->IsHostKernelVersionAtLeast(5, 1, 0)) {
       REGISTER_SYSCALL_IMPL(pidfd_send_signal, [](FEXCore::Core::CpuStateFrame *Frame, int pidfd, int sig, siginfo_t *info, unsigned int flags) -> uint64_t {
-        uint64_t Result = ::syscall(SYS_pidfd_send_signal);
+        uint64_t Result = ::syscall(SYS_pidfd_send_signal, pidfd, sig, info, flags);
         SYSCALL_ERRNO();
       });
     }

Source/Tests/LinuxSyscalls/Syscalls/Socket.cpp

@@ -40,16 +40,6 @@ namespace FEX::HLE {
       SYSCALL_ERRNO();
     });
 
-    REGISTER_SYSCALL_IMPL(sendmsg, [](FEXCore::Core::CpuStateFrame *Frame, int sockfd, const struct msghdr *msg, int flags) -> uint64_t {
-      uint64_t Result = ::sendmsg(sockfd, msg, flags);
-      SYSCALL_ERRNO();
-    });
-
-    REGISTER_SYSCALL_IMPL(recvmsg, [](FEXCore::Core::CpuStateFrame *Frame, int sockfd, struct msghdr *msg, int flags) -> uint64_t {
-      uint64_t Result = ::recvmsg(sockfd, msg, flags);
-      SYSCALL_ERRNO();
-    });
-
     REGISTER_SYSCALL_IMPL(shutdown, [](FEXCore::Core::CpuStateFrame *Frame, int sockfd, int how) -> uint64_t {
       uint64_t Result = ::shutdown(sockfd, how);
       SYSCALL_ERRNO();

Source/Tests/LinuxSyscalls/Syscalls/Thread.cpp

@@ -352,6 +352,12 @@ namespace FEX::HLE {
         case 0x3001: // ARCH_CET_STATUS
           Result = -EINVAL; // We don't support CET, return EINVAL
         break;
+        case 0x1011: // ARCH_GET_CPUID
+          return 1;
+        break;
+        case 0x1012: // ARCH_SET_CPUID
+          return -ENODEV; // Claim we don't support faulting on CPUID
+        break;
         default:
           LogMan::Msg::E("Unknown prctl: 0x%x", code);
           Result = -EINVAL;

Source/Tests/LinuxSyscalls/x32/EPoll.cpp

@@ -20,6 +20,7 @@ $end_info$
 #include <vector>
 
 ARG_TO_STR(FEX::HLE::x32::compat_ptr<FEX::HLE::epoll_event_x86>, "%lx")
+ARG_TO_STR(FEX::HLE::x32::compat_ptr<FEX::HLE::x32::timespec32>, "%lx")
 
 namespace FEX::HLE::x32 {
   void RegisterEpoll(FEX::HLE::SyscallHandler *const Handler) {
@@ -35,7 +36,7 @@ namespace FEX::HLE::x32 {
       SYSCALL_ERRNO();
     });
 
-    REGISTER_SYSCALL_IMPL_X32(epoll_ctl, [](FEXCore::Core::CpuStateFrame *Frame, int epfd, int op, int fd, epoll_event_x86 *event) -> uint64_t {
+    REGISTER_SYSCALL_IMPL_X32(epoll_ctl, [](FEXCore::Core::CpuStateFrame *Frame, int epfd, int op, int fd, compat_ptr<epoll_event_x86> event) -> uint64_t {
       struct epoll_event Event = *event;
       uint64_t Result = ::syscall(SYS_epoll_ctl, epfd, op, fd, &Event);
       if (Result != -1) {
@@ -64,7 +65,7 @@ namespace FEX::HLE::x32 {
       SYSCALL_ERRNO();
     });
 
-    if (Handler->GetHostKernelVersion() >= FEX::HLE::SyscallHandler::KernelVersion(5, 11, 0)) {
+    if (Handler->IsHostKernelVersionAtLeast(5, 11, 0)) {
 #ifndef SYS_epoll_pwait2
 #define SYS_epoll_pwait2 354
 #endif

Source/Tests/LinuxSyscalls/x32/FD.cpp

@@ -119,6 +119,21 @@ namespace FEX::HLE::x32 {
       SYSCALL_ERRNO();
     });
 
+    REGISTER_SYSCALL_IMPL_X32(chown32, [](FEXCore::Core::CpuStateFrame *Frame, const char *pathname, uid_t owner, gid_t group) -> uint64_t {
+      uint64_t Result = ::chown(pathname, owner, group);
+      SYSCALL_ERRNO();
+    });
+
+    REGISTER_SYSCALL_IMPL_X32(fchown32, [](FEXCore::Core::CpuStateFrame *Frame, int fd, uid_t owner, gid_t group) -> uint64_t {
+      uint64_t Result = ::fchown(fd, owner, group);
+      SYSCALL_ERRNO();
+    });
+
+    REGISTER_SYSCALL_IMPL_X32(lchown32, [](FEXCore::Core::CpuStateFrame *Frame, const char *pathname, uid_t owner, gid_t group) -> uint64_t {
+      uint64_t Result = ::lchown(pathname, owner, group);
+      SYSCALL_ERRNO();
+    });
+
     REGISTER_SYSCALL_IMPL_X32(stat, [](FEXCore::Core::CpuStateFrame *Frame, const char *pathname, stat32 *buf) -> uint64_t {
       struct stat host_stat;
       uint64_t Result = FEX::HLE::_SyscallHandler->FM.Stat(pathname, &host_stat);
@@ -174,6 +189,24 @@ namespace FEX::HLE::x32 {
       SYSCALL_ERRNO();
     });
 
+    REGISTER_SYSCALL_IMPL_X32(statfs, [](FEXCore::Core::CpuStateFrame *Frame, const char *path, statfs32_32 *buf) -> uint64_t {
+      struct statfs host_stat;
+      uint64_t Result = FEX::HLE::_SyscallHandler->FM.Statfs(path, &host_stat);
+      if (Result != -1) {
+        *buf = host_stat;
+      }
+      SYSCALL_ERRNO();
+    });
+
+    REGISTER_SYSCALL_IMPL_X32(fstatfs, [](FEXCore::Core::CpuStateFrame *Frame, int fd, statfs32_32 *buf) -> uint64_t {
+      struct statfs host_stat;
+      uint64_t Result = ::fstatfs(fd, &host_stat);
+      if (Result != -1) {
+        *buf = host_stat;
+      }
+      SYSCALL_ERRNO();
+    });
+
     REGISTER_SYSCALL_IMPL_X32(fstatfs64, [](FEXCore::Core::CpuStateFrame *Frame, int fd, size_t sz, struct statfs64_32 *buf) -> uint64_t {
       LOGMAN_THROW_A(sz == sizeof(struct statfs64_32), "This needs to match");
 
@@ -207,8 +240,9 @@ namespace FEX::HLE::x32 {
 
       void *lock_arg = (void*)arg;
       struct flock tmp{};
+      int old_cmd = cmd;
 
-      switch (cmd) {
+      switch (old_cmd) {
         case OP_GETLK64_32: {
           cmd = F_GETLK;
           lock_arg = (void*)&tmp;
@@ -243,7 +277,7 @@ namespace FEX::HLE::x32 {
         }
 
         case F_SETFL:
-          lock_arg = (void*)FEX::HLE::RemapFromX86Flags(arg);
+          lock_arg = reinterpret_cast<void*>(FEX::HLE::RemapFromX86Flags(arg));
           break;
         // Maps directly
         case F_DUPFD:
@@ -259,7 +293,7 @@ namespace FEX::HLE::x32 {
       uint64_t Result = ::fcntl(fd, cmd, lock_arg);
 
       if (Result != -1) {
-        switch (cmd) {
+        switch (old_cmd) {
           case OP_GETLK64_32: {
             *reinterpret_cast<flock64_32*>(arg) = tmp;
             break;

Source/Tests/LinuxSyscalls/x32/FS.cpp

@@ -17,6 +17,14 @@ namespace FEX::HLE::x32 {
       SYSCALL_ERRNO();
     });
 
+    REGISTER_SYSCALL_IMPL_X32(truncate64, [](FEXCore::Core::CpuStateFrame *Frame, const char *path, uint32_t offset_low, uint32_t offset_high) -> uint64_t {
+      uint64_t Offset = offset_high;
+      Offset <<= 32;
+      Offset |= offset_low;
+      uint64_t Result = ::truncate(path, Offset);
+      SYSCALL_ERRNO();
+    });
+
     REGISTER_SYSCALL_IMPL_X32(ftruncate64, [](FEXCore::Core::CpuStateFrame *Frame, int fd, uint32_t offset_low, uint32_t offset_high) -> uint64_t {
       uint64_t Offset = offset_high;
       Offset <<= 32;

Source/Tests/LinuxSyscalls/x32/Info.cpp

@@ -28,8 +28,9 @@ namespace FEX::HLE::x32 {
     uint32_t freeswap;
     uint16_t procs;
     uint32_t totalhigh;
+    uint32_t freehigh;
     uint32_t mem_unit;
-    char _pad[12];
+    char _pad[8];
   };
 
   static_assert(sizeof(sysinfo32) == 64, "Needs to be 64bytes");
@@ -49,16 +50,32 @@ namespace FEX::HLE::x32 {
       if (Result != -1) {
 #define Copy(x) info->x = static_cast<decltype(info->x)>(std::min(Host.x, static_cast<decltype(Host.x)>(std::numeric_limits<decltype(info->x)>::max())));
         Copy(uptime);
+        Copy(procs);
+#define CopyShift(x) info->x = static_cast<decltype(info->x)>(Host.x >> ShiftAmount);
+
         info->loads[0] = std::min(Host.loads[0], static_cast<unsigned long>(std::numeric_limits<uint32_t>::max()));
         info->loads[1] = std::min(Host.loads[1], static_cast<unsigned long>(std::numeric_limits<uint32_t>::max()));
         info->loads[2] = std::min(Host.loads[2], static_cast<unsigned long>(std::numeric_limits<uint32_t>::max()));
-        Copy(totalram);
-        Copy(sharedram);
-        Copy(bufferram);
-        Copy(totalswap);
-        Copy(freeswap);
-        Copy(procs);
-        Copy(totalhigh);
+
+        // If any result can't fit in to a uint32_t then we need to shift the mem_unit and all the members
+        // Set the mem_unit to the pagesize
+        uint32_t ShiftAmount{};
+        if ((Host.totalram >> 32) != 0 ||
+            (Host.totalswap >> 32) != 0) {
+
+          while (Host.mem_unit < 4096) {
+            Host.mem_unit <<= 1;
+            ++ShiftAmount;
+          }
+        }
+
+        CopyShift(totalram);
+        CopyShift(sharedram);
+        CopyShift(bufferram);
+        CopyShift(totalswap);
+        CopyShift(freeswap);
+        CopyShift(totalhigh);
+        CopyShift(freehigh);
         Copy(mem_unit);
       }
       SYSCALL_ERRNO();

Source/Tests/LinuxSyscalls/x32/Memory.cpp

@@ -31,7 +31,7 @@ namespace FEX::HLE::x32 {
         mmap(reinterpret_cast<void*>(addr), length, prot,flags, fd, offset);
 
       auto Thread = Frame->Thread;
-      if (Result != -1) {
+      if (Result < -4096) {
         if (!(flags & MAP_ANONYMOUS)) {
           auto filename = get_fdpath(fd);
 
@@ -47,7 +47,7 @@ namespace FEX::HLE::x32 {
         mmap(reinterpret_cast<void*>(addr), length, prot,flags, fd, (uint64_t)pgoffset * 0x1000);
 
       auto Thread = Frame->Thread;
-      if (Result != -1) {
+      if (Result < -4096) {
         if (!(flags & MAP_ANONYMOUS)) {
           auto filename = get_fdpath(fd);
 
@@ -63,7 +63,7 @@ namespace FEX::HLE::x32 {
       auto Result = static_cast<FEX::HLE::x32::x32SyscallHandler*>(FEX::HLE::_SyscallHandler)->GetAllocator()->
         munmap(addr, length);
 
-      if (Result != -1) {
+      if (Result == 0) {
         FEXCore::Context::RemoveNamedRegion(Frame->Thread->CTX, (uintptr_t)addr, length);
         FEXCore::Context::FlushCodeRange(Frame->Thread, (uintptr_t)addr, length);
       }
@@ -105,6 +105,12 @@ namespace FEX::HLE::x32 {
         return Result;
       }
     });
+
+    REGISTER_SYSCALL_IMPL_X32(shmdt, [](FEXCore::Core::CpuStateFrame *Frame, const void *shmaddr) -> uint64_t {
+      uint64_t Result = static_cast<FEX::HLE::x32::x32SyscallHandler*>(FEX::HLE::_SyscallHandler)->GetAllocator()->
+        shmdt(shmaddr);
+      SYSCALL_ERRNO();
+    });
   }
 
 }

Source/Tests/LinuxSyscalls/x32/Signals.cpp

@@ -21,6 +21,16 @@ namespace SignalDelegator {
 
 namespace FEX::HLE::x32 {
   void RegisterSignals() {
+    REGISTER_SYSCALL_IMPL_X32(sigpending, [](FEXCore::Core::CpuStateFrame *Frame, compat_old_sigset_t *set) -> uint64_t {
+      uint64_t HostSet{};
+      uint64_t Result = FEX::HLE::_SyscallHandler->GetSignalDelegator()->GuestSigPending(&HostSet, 8);
+      if (Result == 0) {
+        // This old interface only returns the lower signals
+        *set = HostSet & ~0U;
+      }
+      return Result;
+    });
+
     REGISTER_SYSCALL_IMPL_X32(signal, [](FEXCore::Core::CpuStateFrame *Frame, int signum, uint32_t handler) -> uint64_t {
       FEXCore::GuestSigAction newact{};
       FEXCore::GuestSigAction oldact{};

Source/Tests/LinuxSyscalls/x32/Socket.cpp

@@ -40,6 +40,123 @@ namespace FEX::HLE::x32 {
     OP_SENDMMSG = 20,
   };
 
+  static uint64_t SendMsg(int sockfd, const struct msghdr32 *msg, int flags) {
+    struct msghdr HostHeader{};
+    std::vector<iovec> Host_iovec(msg->msg_iovlen);
+    for (int i = 0; i < msg->msg_iovlen; ++i) {
+      Host_iovec[i] = msg->msg_iov[i];
+    }
+
+    HostHeader.msg_name = msg->msg_name;
+    HostHeader.msg_namelen = msg->msg_namelen;
+
+    HostHeader.msg_iov = &Host_iovec.at(0);
+    HostHeader.msg_iovlen = msg->msg_iovlen;
+
+    HostHeader.msg_control = alloca(msg->msg_controllen * 2);
+    HostHeader.msg_controllen = msg->msg_controllen;
+
+    HostHeader.msg_flags = msg->msg_flags;
+    if (HostHeader.msg_controllen) {
+      void *CurrentGuestPtr = msg->msg_control;
+      struct cmsghdr *CurrentHost = reinterpret_cast<struct cmsghdr*>(HostHeader.msg_control);
+
+      for (cmsghdr32 *msghdr_guest = reinterpret_cast<cmsghdr32*>(CurrentGuestPtr);
+          CurrentGuestPtr != 0;
+          msghdr_guest = reinterpret_cast<cmsghdr32*>(CurrentGuestPtr)) {
+
+        CurrentHost->cmsg_level = msghdr_guest->cmsg_level;
+        CurrentHost->cmsg_type = msghdr_guest->cmsg_type;
+
+        if (msghdr_guest->cmsg_len) {
+          size_t SizeIncrease = (CMSG_LEN(0) - sizeof(cmsghdr32));
+          CurrentHost->cmsg_len = msghdr_guest->cmsg_len + SizeIncrease;
+          HostHeader.msg_controllen += SizeIncrease;
+          memcpy(CMSG_DATA(CurrentHost), msghdr_guest->cmsg_data, msghdr_guest->cmsg_len - sizeof(cmsghdr32));
+        }
+
+        // Go to next host
+        CurrentHost = CMSG_NXTHDR(&HostHeader, CurrentHost);
+
+        // Go to next msg
+        if (msghdr_guest->cmsg_len < sizeof(cmsghdr32)) {
+          CurrentGuestPtr = nullptr;
+        }
+        else {
+          CurrentGuestPtr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(CurrentGuestPtr) + msghdr_guest->cmsg_len);
+          CurrentGuestPtr = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(CurrentGuestPtr) + 3) & ~3ULL);
+          if (CurrentGuestPtr >= reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(static_cast<void*>(msg->msg_control)) + msg->msg_controllen)) {
+            CurrentGuestPtr = nullptr;
+          }
+        }
+      }
+    }
+
+    uint64_t Result = ::sendmsg(sockfd, &HostHeader, flags);
+    SYSCALL_ERRNO();
+  }
+
+  static uint64_t RecvMsg(int sockfd, struct msghdr32 *msg, int flags) {
+    struct msghdr HostHeader{};
+    std::vector<iovec> Host_iovec(msg->msg_iovlen);
+    for (int i = 0; i < msg->msg_iovlen; ++i) {
+      Host_iovec[i] = msg->msg_iov[i];
+    }
+
+    HostHeader.msg_name = msg->msg_name;
+    HostHeader.msg_namelen = msg->msg_namelen;
+
+    HostHeader.msg_iov = &Host_iovec.at(0);
+    HostHeader.msg_iovlen = msg->msg_iovlen;
+
+    HostHeader.msg_control = alloca(msg->msg_controllen*2);
+    HostHeader.msg_controllen = msg->msg_controllen*2;
+
+    HostHeader.msg_flags = msg->msg_flags;
+
+    uint64_t Result = ::recvmsg(sockfd, &HostHeader, flags);
+    if (Result != -1) {
+      for (int i = 0; i < msg->msg_iovlen; ++i) {
+        msg->msg_iov[i] = Host_iovec[i];
+      }
+
+      msg->msg_namelen = HostHeader.msg_namelen;
+      msg->msg_controllen = HostHeader.msg_controllen;
+      msg->msg_flags = HostHeader.msg_flags;
+      if (HostHeader.msg_controllen) {
+        // Host and guest cmsg data structures aren't compatible.
+        // Copy them over now
+        void *CurrentGuestPtr = msg->msg_control;
+        for (struct cmsghdr *cmsg = CMSG_FIRSTHDR(&HostHeader);
+            cmsg != nullptr;
+            cmsg = CMSG_NXTHDR(&HostHeader, cmsg)) {
+          cmsghdr32 *CurrentGuest = reinterpret_cast<cmsghdr32*>(CurrentGuestPtr);
+
+          // Copy over the header first
+          // cmsg_len needs to be adjusted by the size of the header between host and guest
+          // Host is 16 bytes, guest is 12 bytes
+          CurrentGuest->cmsg_level = cmsg->cmsg_level;
+          CurrentGuest->cmsg_type = cmsg->cmsg_type;
+
+          // Now copy over the data
+          if (cmsg->cmsg_len) {
+            size_t SizeIncrease = (CMSG_LEN(0) - sizeof(cmsghdr32));
+            CurrentGuest->cmsg_len = cmsg->cmsg_len - SizeIncrease;
+
+            // Controllen size also changes
+            msg->msg_controllen -= SizeIncrease;
+
+            memcpy(CurrentGuest->cmsg_data, CMSG_DATA(cmsg), cmsg->cmsg_len - sizeof(struct cmsghdr));
+            CurrentGuestPtr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(CurrentGuestPtr) + CurrentGuest->cmsg_len);
+            CurrentGuestPtr = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(CurrentGuestPtr) + 3) & ~3ULL);
+
+          }
+        }
+      }
+    }
+    SYSCALL_ERRNO();
+  }
+
   void RegisterSocket() {
     REGISTER_SYSCALL_IMPL_X32(socketcall, [](FEXCore::Core::CpuStateFrame *Frame, uint32_t call, uint32_t *Arguments) -> uint64_t {
       uint64_t Result{};
@@ -130,122 +247,11 @@ namespace FEX::HLE::x32 {
           break;
         }
         case OP_SENDMSG: {
-          const struct msghdr32 *guest_msg = reinterpret_cast<const struct msghdr32*>(Arguments[1]);
-
-          struct msghdr HostHeader{};
-          std::vector<iovec> Host_iovec(guest_msg->msg_iovlen);
-          for (int i = 0; i < guest_msg->msg_iovlen; ++i) {
-            Host_iovec[i] = guest_msg->msg_iov[i];
-          }
-
-          HostHeader.msg_name = guest_msg->msg_name;
-          HostHeader.msg_namelen = guest_msg->msg_namelen;
-
-          HostHeader.msg_iov = &Host_iovec.at(0);
-          HostHeader.msg_iovlen = guest_msg->msg_iovlen;
-
-          HostHeader.msg_control = alloca(guest_msg->msg_controllen * 2);
-          HostHeader.msg_controllen = guest_msg->msg_controllen;
-
-          HostHeader.msg_flags = guest_msg->msg_flags;
-          if (HostHeader.msg_controllen) {
-            void *CurrentGuestPtr = guest_msg->msg_control;
-            struct cmsghdr *CurrentHost = reinterpret_cast<struct cmsghdr*>(HostHeader.msg_control);
-
-            for (cmsghdr32 *msghdr_guest = reinterpret_cast<cmsghdr32*>(CurrentGuestPtr);
-                CurrentGuestPtr != 0;
-                msghdr_guest = reinterpret_cast<cmsghdr32*>(CurrentGuestPtr)) {
-
-              CurrentHost->cmsg_level = msghdr_guest->cmsg_level;
-              CurrentHost->cmsg_type = msghdr_guest->cmsg_type;
-
-              if (msghdr_guest->cmsg_len) {
-                size_t SizeIncrease = (CMSG_LEN(0) - sizeof(cmsghdr32));
-                CurrentHost->cmsg_len = msghdr_guest->cmsg_len + SizeIncrease;
-                HostHeader.msg_controllen += SizeIncrease;
-                memcpy(CMSG_DATA(CurrentHost), msghdr_guest->cmsg_data, msghdr_guest->cmsg_len - sizeof(cmsghdr32));
-              }
-
-              // Go to next host
-              CurrentHost = CMSG_NXTHDR(&HostHeader, CurrentHost);
-
-              // Go to next msg
-              if (msghdr_guest->cmsg_len < sizeof(cmsghdr32)) {
-                CurrentGuestPtr = nullptr;
-              }
-              else {
-                CurrentGuestPtr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(CurrentGuestPtr) + msghdr_guest->cmsg_len);
-                CurrentGuestPtr = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(CurrentGuestPtr) + 3) & ~3ULL);
-                if (CurrentGuestPtr >= reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(static_cast<void*>(guest_msg->msg_control)) + guest_msg->msg_controllen)) {
-                  CurrentGuestPtr = nullptr;
-                }
-              }
-            }
-          }
-
-          Result = ::sendmsg(Arguments[0], &HostHeader, Arguments[2]);
+          return SendMsg(Arguments[0], reinterpret_cast<const struct msghdr32*>(Arguments[1]), Arguments[2]);
           break;
         }
         case OP_RECVMSG: {
-          struct msghdr32 *guest_msg = reinterpret_cast<struct msghdr32*>(Arguments[1]);
-
-          struct msghdr HostHeader{};
-          std::vector<iovec> Host_iovec(guest_msg->msg_iovlen);
-          for (int i = 0; i < guest_msg->msg_iovlen; ++i) {
-            Host_iovec[i] = guest_msg->msg_iov[i];
-          }
-
-          HostHeader.msg_name = guest_msg->msg_name;
-          HostHeader.msg_namelen = guest_msg->msg_namelen;
-
-          HostHeader.msg_iov = &Host_iovec.at(0);
-          HostHeader.msg_iovlen = guest_msg->msg_iovlen;
-
-          HostHeader.msg_control = alloca(guest_msg->msg_controllen*2);
-          HostHeader.msg_controllen = guest_msg->msg_controllen*2;
-
-          HostHeader.msg_flags = guest_msg->msg_flags;
-
-          Result = ::recvmsg(Arguments[0], &HostHeader, Arguments[2]);
-          if (Result != -1) {
-            for (int i = 0; i < guest_msg->msg_iovlen; ++i) {
-              guest_msg->msg_iov[i] = Host_iovec[i];
-            }
-
-            guest_msg->msg_namelen = HostHeader.msg_namelen;
-            guest_msg->msg_controllen = HostHeader.msg_controllen;
-            guest_msg->msg_flags = HostHeader.msg_flags;
-            if (HostHeader.msg_controllen) {
-              // Host and guest cmsg data structures aren't compatible.
-              // Copy them over now
-              void *CurrentGuestPtr = guest_msg->msg_control;
-              for (struct cmsghdr *cmsg = CMSG_FIRSTHDR(&HostHeader);
-                  cmsg != nullptr;
-                  cmsg = CMSG_NXTHDR(&HostHeader, cmsg)) {
-                cmsghdr32 *CurrentGuest = reinterpret_cast<cmsghdr32*>(CurrentGuestPtr);
-
-                // Copy over the header first
-                // cmsg_len needs to be adjusted by the size of the header between host and guest
-                // Host is 16 bytes, guest is 12 bytes
-                CurrentGuest->cmsg_level = cmsg->cmsg_level;
-                CurrentGuest->cmsg_type = cmsg->cmsg_type;
-
-                // Now copy over the data
-                if (cmsg->cmsg_len) {
-                  size_t SizeIncrease = (CMSG_LEN(0) - sizeof(cmsghdr32));
-                  CurrentGuest->cmsg_len = cmsg->cmsg_len - SizeIncrease;
-
-                  // Controllen size also changes
-                  guest_msg->msg_controllen -= SizeIncrease;
-
-                  memcpy(CurrentGuest->cmsg_data, CMSG_DATA(cmsg), cmsg->cmsg_len - sizeof(struct cmsghdr));
-                  CurrentGuestPtr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(CurrentGuestPtr) + CurrentGuest->cmsg_len);
-                  CurrentGuestPtr = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(CurrentGuestPtr) + 3) & ~3ULL);
-
-                }
-              }
-            }
-          }
+          return RecvMsg(Arguments[0], reinterpret_cast<struct msghdr32*>(Arguments[1]), Arguments[2]);
           break;
         }
         default:
@@ -255,11 +261,13 @@ namespace FEX::HLE::x32 {
       SYSCALL_ERRNO();
     });
 
+    REGISTER_SYSCALL_IMPL_X32(sendmsg, [](FEXCore::Core::CpuStateFrame *Frame, int sockfd, const struct msghdr32 *msg, int flags) -> uint64_t {
+      return SendMsg(sockfd, msg, flags);
+    });
+
     REGISTER_SYSCALL_IMPL_X32(sendmmsg, [](FEXCore::Core::CpuStateFrame *Frame, int sockfd, compat_ptr<mmsghdr_32> msgvec, uint32_t vlen, int flags) -> uint64_t {
       std::vector<iovec> Host_iovec;
-      std::vector<uint8_t> Controllen;
-      std::vector<struct msghdr> Messages{vlen};
-      std::vector<struct mmsghdr> HostMmsg{vlen};
+      std::vector<struct mmsghdr> HostMmsg(vlen);
 
       // Walk the iovec and convert them
       // Calculate controllen at the same time
@@ -274,7 +282,7 @@ namespace FEX::HLE::x32 {
         }
       }
 
-      Controllen.resize(Controllen_size);
+      std::vector<uint8_t> Controllen(Controllen_size);
 
       size_t current_iov{};
       size_t current_controllen_offset{};
@@ -323,6 +331,7 @@ namespace FEX::HLE::x32 {
             }
             else {
               CurrentGuestPtr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(CurrentGuestPtr) + msghdr_guest->cmsg_len);
+              CurrentGuestPtr = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(CurrentGuestPtr) + 3) & ~3ULL);
               if (CurrentGuestPtr >= reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(static_cast<void*>(guest.msg_control)) + guest.msg_controllen)) {
                 CurrentGuestPtr = nullptr;
               }
@@ -333,7 +342,7 @@ namespace FEX::HLE::x32 {
         HostMmsg[i].msg_len = msgvec[i].msg_len;
       }
 
-      uint64_t Result = ::sendmmsg(sockfd, &HostMmsg.at(0), vlen, flags);
+      uint64_t Result = ::sendmmsg(sockfd, HostMmsg.data(), vlen, flags);
 
       if (Result != -1) {
         // Update guest msglen
@@ -343,5 +352,9 @@ namespace FEX::HLE::x32 {
       }
       SYSCALL_ERRNO();
     });
+
+    REGISTER_SYSCALL_IMPL_X32(recvmsg, [](FEXCore::Core::CpuStateFrame *Frame, int sockfd, struct msghdr32 *msg, int flags) -> uint64_t {
+      return RecvMsg(sockfd, msg, flags);
+    });
   }
 }