Docs: Update for release FEX-2109

Fix unaligned CASPair on ARMv8.0

CMakeLists.txt

@@ -16,6 +16,7 @@ option(ENABLE_STRICT_WERROR "Enables stricter -Werror for CI" FALSE)
 option(ENABLE_WERROR "Enables -Werror" FALSE)
 option(ENABLE_STATIC_PIE "Enables static-pie build" FALSE)
 option(ENABLE_JEMALLOC "Enables jemalloc allocator" TRUE)
+option(ENABLE_OFFLINE_TELEMETRY "Enables FEX offline telemetry" TRUE)
 
 set (X86_C_COMPILER "x86_64-linux-gnu-gcc" CACHE STRING "c compiler for compiling x86 guest libs")
 set (X86_CXX_COMPILER "x86_64-linux-gnu-g++" CACHE STRING "c++ compiler for compiling x86 guest libs")
@@ -82,6 +83,11 @@ if (ENABLE_LLD)
   link_libraries(${LD_OVERRIDE})
 endif()
 
+if (NOT ENABLE_OFFLINE_TELEMETRY)
+  # Disable FEX offline telemetry entirely if asked
+  add_definitions(-DFEX_DISABLE_TELEMETRY=1)
+endif()
+
 if (ENABLE_STATIC_PIE)
   if (_M_ARM_64 AND ENABLE_LLD)
     message (FATAL_ERROR "Static linking does not currently work with AArch64+LLD. Use GNU ld for now.")
@@ -219,6 +225,8 @@ endif()
 
 if (ENABLE_JEMALLOC)
   add_definitions(-DENABLE_JEMALLOC=1)
+  add_subdirectory(External/jemalloc/)
+  include_directories(External/jemalloc/pregen/include/)
 else()
   message (STATUS
     " jemalloc disabled!\n"
@@ -254,9 +262,6 @@ endif()
 add_definitions(-Wno-trigraphs)
 add_definitions(-DGLOBAL_DATA_DIRECTORY="${DATA_DIRECTORY}/")
 
-add_subdirectory(External/jemalloc/)
-include_directories(External/jemalloc/pregen/include/)
-
 add_subdirectory(External/cpp-optparse/)
 include_directories(External/cpp-optparse/)
 

External/FEXCore/Source/CMakeLists.txt

@@ -86,6 +86,7 @@ set (SRCS
   Interface/Core/OpcodeDispatcher/Vector.cpp
   Interface/Core/OpcodeDispatcher/X87.cpp
   Interface/Core/OpcodeDispatcher.cpp
+  Interface/Core/SignalDelegator.cpp
   Interface/Core/X86Tables.cpp
   Interface/Core/X86DebugInfo.cpp
   Interface/Core/X86HelperGen.cpp
@@ -129,6 +130,7 @@ set (SRCS
   Utils/Allocator.cpp
   Utils/Allocator/64BitAllocator.cpp
   Utils/LogManager.cpp
+  Utils/Telemetry.cpp
   Utils/Threads.cpp
   )
 
@@ -183,6 +185,11 @@ if (ENABLE_JITSYMBOLS)
   list(APPEND DEFINES -DENABLE_JITSYMBOLS=1)
 endif()
 
+set (LIBS vixl dl fmt::fmt xxhash tiny-json)
+if (ENABLE_JEMALLOC)
+  list (APPEND LIBS FEX_jemalloc)
+endif()
+
 # Generate IR include file
 set(OUTPUT_IR_FOLDER "${CMAKE_BINARY_DIR}/include/FEXCore/IR")
 set(OUTPUT_NAME "${OUTPUT_IR_FOLDER}/IRDefines.inc")
@@ -269,7 +276,7 @@ function(AddObject Name Type)
   add_dependencies(${Name} IR_INC)
   add_dependencies(${Name} CONFIG_INC)
 
-  target_link_libraries(${Name} vixl dl fmt::fmt xxhash)
+  target_link_libraries(${Name} ${LIBS})
   set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)
   set_target_properties(${Name} PROPERTIES C_VISIBILITY_PRESET hidden)
   set_target_properties(${Name} PROPERTIES CXX_VISIBILITY_PRESET hidden)
@@ -310,7 +317,7 @@ endfunction()
 
 function(AddLibrary Name Type)
   add_library(${Name} ${Type} $<TARGET_OBJECTS:${PROJECT_NAME}_object>)
-  target_link_libraries(${Name} vixl dl fmt::fmt xxhash)
+  target_link_libraries(${Name} ${LIBS})
   set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)
   set_target_properties(${Name} PROPERTIES C_VISIBILITY_PRESET hidden)
   set_target_properties(${Name} PROPERTIES CXX_VISIBILITY_PRESET hidden)

External/FEXCore/Source/Common/NetStream.cpp

@@ -1,11 +1,7 @@
 #include "NetStream.h"
 
-#include <cstring>
-
-#include <sys/types.h>
 #include <sys/socket.h>
 
-#include <stdio.h>
 #include <unistd.h>
 
 int NetStream::NetBuf::flushBuffer(const char *buffer, size_t size) {
@@ -29,7 +25,7 @@ std::streamsize NetStream::NetBuf::xsputn(const char* buffer, std::streamsize si
 
     // Check if the string fits neatly in our buffer
     if (size <= buf_remaining) {
-        std::memcpy(pptr(), buffer, size);
+        ::memcpy(pptr(), buffer, size);
         pbump(size);
         return size;
     }

External/FEXCore/Source/Common/NetStream.h

@@ -2,6 +2,7 @@
 
 #include <array>
 #include <iostream>
+#include <iterator>
 #include <string.h>
 
 class NetStream : public std::iostream {

External/FEXCore/Source/Common/Paths.cpp

@@ -3,12 +3,44 @@
 
 #include <cstdlib>
 #include <filesystem>
-#include <sys/stat.h>
+#include <memory>
+#include <pwd.h>
+#include <system_error>
+#include <unistd.h>
 
 namespace FEXCore::Paths {
   std::unique_ptr<std::string> CachePath;
   std::unique_ptr<std::string> EntryCache;
 
+  char const* FindUserHomeThroughUID() {
+    auto passwd = getpwuid(geteuid());
+    if (passwd) {
+      return passwd->pw_dir;
+    }
+    return nullptr;
+  }
+
+  const char *GetHomeDirectory() {
+    char const *HomeDir = getenv("HOME");
+
+    // Try to get home directory from uid
+    if (!HomeDir) {
+      HomeDir = FindUserHomeThroughUID();
+    }
+
+    // try the PWD
+    if (!HomeDir) {
+      HomeDir = getenv("PWD");
+    }
+
+    // Still doesn't exit? You get local
+    if (!HomeDir) {
+      HomeDir = ".";
+    }
+
+    return HomeDir;
+  }
+
   void InitializePaths() {
     CachePath = std::make_unique<std::string>();
     EntryCache = std::make_unique<std::string>();

External/FEXCore/Source/Common/Paths.h

@@ -4,6 +4,9 @@
 namespace FEXCore::Paths {
   void InitializePaths();
   void ShutdownPaths();
+
+  const char *GetHomeDirectory();
+
   std::string GetCachePath();
   std::string GetEntryCachePath();
 }

External/FEXCore/Source/Interface/Config/Config.cpp

@@ -1,43 +1,164 @@
 #include "Common/StringConv.h"
-
-#include <FEXCore/Utils/LogManager.h>
-#include "Interface/Context/Context.h"
+#include "Common/Paths.h"
 
 #include <FEXCore/Config/Config.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <array>
+#include <assert.h>
+#include <cstdlib>
 #include <filesystem>
-#include <pwd.h>
+#include <fstream>
+#include <functional>
 #include <map>
+#include <memory>
+#include <list>
+#include <optional>
+#include <stddef.h>
+#include <stdint.h>
+#include <string>
+#include <string_view>
 #include <sys/sysinfo.h>
-#include <unistd.h>
+#include <system_error>
+#include <type_traits>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <tiny-json.h>
+
+namespace FEXCore::Context {
+  struct Context;
+}
 
 namespace FEXCore::Config {
-  char const* FindUserHomeThroughUID() {
-    auto passwd = getpwuid(geteuid());
-    if (passwd) {
-      return passwd->pw_dir;
+namespace DefaultValues {
+#define P(x) x
+#define OPT_BASE(type, group, enum, json, default) const P(type) P(enum) = P(default);
+#define OPT_STR(group, enum, json, default) const std::string_view P(enum) = P(default);
+#define OPT_STRARRAY(group, enum, json, default) OPT_STR(group, enum, json, default)
+#include <FEXCore/Config/ConfigValues.inl>
+}
+
+namespace JSON {
+  static bool LoadConfigFile(std::vector<char> &Data, const std::string &Config) {
+    std::fstream ConfigFile;
+    ConfigFile.open(Config, std::ios::in);
+
+    if (!ConfigFile.is_open()) {
+      return false;
     }
-    return nullptr;
+
+    if (!ConfigFile.seekg(0, std::fstream::end)) {
+      LogMan::Msg::D("Couldn't load configuration file: Seek end");
+      return false;
+    }
+
+    auto FileSize = ConfigFile.tellg();
+    if (ConfigFile.fail()) {
+      LogMan::Msg::D("Couldn't load configuration file: tellg");
+      return false;
+    }
+
+    if (!ConfigFile.seekg(0, std::fstream::beg)) {
+      LogMan::Msg::D("Couldn't load configuration file: Seek beginning");
+      return false;
+    }
+
+    if (FileSize > 0) {
+      Data.resize(FileSize);
+      if (!ConfigFile.read(&Data.at(0), FileSize)) {
+        // Probably means permissions aren't set. Just early exit
+        return false;
+      }
+      ConfigFile.close();
+    }
+    else {
+      return false;
+    }
+
+    return true;
   }
 
-  const char *GetHomeDirectory() {
-    char const *HomeDir = getenv("HOME");
+  struct JsonAllocator {
+    jsonPool_t PoolObject;
+    std::unique_ptr<std::list<json_t>> json_objects;
+  };
+  static_assert(offsetof(JsonAllocator, PoolObject) == 0, "This needs to be at offset zero");
 
-    // Try to get home directory from uid
-    if (!HomeDir) {
-      HomeDir = FindUserHomeThroughUID();
+  json_t* PoolInit(jsonPool_t* Pool) {
+    JsonAllocator* alloc = reinterpret_cast<JsonAllocator*>(Pool);
+    alloc->json_objects = std::make_unique<std::list<json_t>>();
+    return &*alloc->json_objects->emplace(alloc->json_objects->end());
+  }
+
+  json_t* PoolAlloc(jsonPool_t* Pool) {
+    JsonAllocator* alloc = reinterpret_cast<JsonAllocator*>(Pool);
+    return &*alloc->json_objects->emplace(alloc->json_objects->end());
+  }
+
+  static void LoadJSonConfig(const std::string &Config, std::function<void(const char *Name, const char *ConfigSring)> Func) {
+    std::vector<char> Data;
+    if (!LoadConfigFile(Data, Config)) {
+      return;
     }
 
-    // try the PWD
-    if (!HomeDir) {
-      HomeDir = getenv("PWD");
+    JsonAllocator Pool {
+      .PoolObject = {
+        .init = PoolInit,
+        .alloc = PoolAlloc,
+      },
+    };
+
+    json_t const *json = json_createWithPool(&Data.at(0), &Pool.PoolObject);
+    if (!json) {
+      LogMan::Msg::E("Couldn't create json");
+      return;
     }
 
-    // Still doesn't exit? You get local
-    if (!HomeDir) {
-      HomeDir = ".";
+    json_t const* ConfigList = json_getProperty(json, "Config");
+
+    if (!ConfigList) {
+      LogMan::Msg::E("Couldn't get config list");
+      return;
     }
 
-    return HomeDir;
+    for (json_t const* ConfigItem = json_getChild(ConfigList);
+      ConfigItem != nullptr;
+      ConfigItem = json_getSibling(ConfigItem)) {
+      const char* ConfigName = json_getName(ConfigItem);
+      const char* ConfigString = json_getValue(ConfigItem);
+
+      if (!ConfigName) {
+        LogMan::Msg::E("Couldn't get config name");
+        return;
+      }
+
+      if (!ConfigString) {
+        LogMan::Msg::E("Couldn't get ConfigString for '%s'", ConfigName);
+        return;
+      }
+
+      Func(ConfigName, ConfigString);
+    }
+  }
+}
+
+  std::string GetDataDirectory() {
+    std::string DataDir{};
+
+    char const *HomeDir = Paths::GetHomeDirectory();
+    char const *DataXDG = getenv("XDG_DATA_HOME");
+    char const *DataOverride = getenv("FEX_APP_DATA_LOCATION");
+    if (DataOverride) {
+      // Data override will override the complete directory
+      DataDir = DataOverride;
+    }
+    else {
+      DataDir = DataXDG ?: HomeDir;
+      DataDir += "/.fex-emu/";
+    }
+    return DataDir;
   }
 
   std::string GetConfigDirectory(bool Global) {
@@ -46,7 +167,7 @@ namespace FEXCore::Config {
       ConfigDir = GLOBAL_DATA_DIRECTORY;
     }
     else {
-      char const *HomeDir = GetHomeDirectory();
+      char const *HomeDir = Paths::GetHomeDirectory();
       char const *ConfigXDG = getenv("XDG_CONFIG_HOME");
       char const *ConfigOverride = getenv("FEX_APP_CONFIG_LOCATION");
       if (ConfigOverride) {
@@ -109,23 +230,6 @@ namespace FEXCore::Config {
     return ConfigFile;
   }
 
-  std::string GetDataDirectory() {
-    std::string DataDir{};
-
-    char const *HomeDir = GetHomeDirectory();
-    char const *DataXDG = getenv("XDG_DATA_HOME");
-    char const *DataOverride = getenv("FEX_APP_DATA_LOCATION");
-    if (DataOverride) {
-      // Data override will override the complete directory
-      DataDir = DataOverride;
-    }
-    else {
-      DataDir = DataXDG ?: HomeDir;
-      DataDir += "/.fex-emu/";
-    }
-    return DataDir;
-  }
-
   void SetConfig(FEXCore::Context::Context *CTX, ConfigOption Option, uint64_t Config) {
   }
 
@@ -417,6 +521,17 @@ namespace FEXCore::Config {
     }
   }
 
+  template<>
+  std::string Value<std::string>::GetIfExists(FEXCore::Config::ConfigOption Option, std::string_view Default) {
+    auto Value = FEXCore::Config::Get(Option);
+    if (Value) {
+      return **Value;
+    }
+    else {
+      return std::string(Default);
+    }
+  }
+
   template bool     Value<bool>::GetIfExists(FEXCore::Config::ConfigOption Option, bool Default);
   template int8_t   Value<int8_t>::GetIfExists(FEXCore::Config::ConfigOption Option, int8_t Default);
   template uint8_t  Value<uint8_t>::GetIfExists(FEXCore::Config::ConfigOption Option, uint8_t Default);
@@ -442,5 +557,143 @@ namespace FEXCore::Config {
     }
   }
   template void Value<std::string>::GetListIfExists(FEXCore::Config::ConfigOption Option, std::list<std::string> *List);
+
+  // Application loaders
+  class MainLoader final : public FEXCore::Config::OptionMapper {
+  public:
+    explicit MainLoader();
+    explicit MainLoader(std::string ConfigFile);
+    void Load() override;
+
+  private:
+    std::string Config;
+  };
+
+  class AppLoader final : public FEXCore::Config::OptionMapper {
+  public:
+    explicit AppLoader(const std::string& Filename, bool Global);
+    void Load();
+
+  private:
+    std::string Config;
+  };
+
+  class EnvLoader final : public FEXCore::Config::Layer {
+  public:
+    explicit EnvLoader(char *const _envp[]);
+    void Load() override;
+
+  private:
+    char *const *envp;
+  };
+
+  static const std::map<std::string, FEXCore::Config::ConfigOption, std::less<>> ConfigLookup = {{
+#define OPT_BASE(type, group, enum, json, default) {#json, FEXCore::Config::ConfigOption::CONFIG_##enum},
+#include <FEXCore/Config/ConfigValues.inl>
+  }};
+  static const std::vector<std::pair<const char*, FEXCore::Config::ConfigOption>> EnvConfigLookup = {{
+#define OPT_BASE(type, group, enum, json, default) {"FEX_" #enum, FEXCore::Config::ConfigOption::CONFIG_##enum},
+#include <FEXCore/Config/ConfigValues.inl>
+  }};
+
+  OptionMapper::OptionMapper(FEXCore::Config::LayerType Layer)
+    : FEXCore::Config::Layer(Layer) {
+  }
+
+  void OptionMapper::MapNameToOption(const char *ConfigName, const char *ConfigString) {
+    auto it = ConfigLookup.find(ConfigName);
+    if (it != ConfigLookup.end()) {
+      Set(it->second, ConfigString);
+    }
+  }
+
+  MainLoader::MainLoader()
+    : FEXCore::Config::OptionMapper(FEXCore::Config::LayerType::LAYER_MAIN)
+    , Config{FEXCore::Config::GetConfigFileLocation()} {
+  }
+
+  MainLoader::MainLoader(std::string ConfigFile)
+    : FEXCore::Config::OptionMapper(FEXCore::Config::LayerType::LAYER_MAIN)
+    , Config{std::move(ConfigFile)} {
+  }
+
+  void MainLoader::Load() {
+    JSON::LoadJSonConfig(Config, [this](const char *Name, const char *ConfigString) {
+      MapNameToOption(Name, ConfigString);
+    });
+  }
+
+  AppLoader::AppLoader(const std::string& Filename, bool Global)
+    : FEXCore::Config::OptionMapper(Global ? FEXCore::Config::LayerType::LAYER_GLOBAL_APP : FEXCore::Config::LayerType::LAYER_LOCAL_APP) {
+    Config = FEXCore::Config::GetApplicationConfig(Filename, Global);
+
+    // Immediately load so we can reload the meta layer
+    Load();
+  }
+
+  void AppLoader::Load() {
+    JSON::LoadJSonConfig(Config, [this](const char *Name, const char *ConfigString) {
+      MapNameToOption(Name, ConfigString);
+    });
+  }
+
+  EnvLoader::EnvLoader(char *const _envp[])
+    : FEXCore::Config::Layer(FEXCore::Config::LayerType::LAYER_ENVIRONMENT)
+    , envp {_envp} {
+  }
+
+  void EnvLoader::Load() {
+    std::unordered_map<std::string_view, std::string_view> EnvMap;
+
+    for(const char *const *pvar=envp; pvar && *pvar; pvar++) {
+      std::string_view Var(*pvar);
+      size_t pos = Var.rfind('=');
+      if (std::string::npos == pos)
+        continue;
+
+      std::string_view Ident = Var.substr(0,pos);
+      std::string_view Value = Var.substr(pos+1);
+      EnvMap[Ident]=Value;
+    }
+
+    std::function GetVar = [=](const std::string_view id)  -> std::optional<std::string_view> {
+      if (EnvMap.find(id) != EnvMap.end())
+        return EnvMap.at(id);
+
+      // If envp[] was empty, search using std::getenv()
+      const char* vs = std::getenv(id.data());
+      if (vs) {
+        return vs;
+      }
+      else {
+        return std::nullopt;
+      }
+    };
+
+    std::optional<std::string_view> Value;
+
+    for (auto &it : EnvConfigLookup) {
+      if ((Value = GetVar(it.first)).has_value()) {
+        Set(it.second, std::string(*Value));
+      }
+    }
+  }
+
+  std::unique_ptr<FEXCore::Config::Layer> CreateMainLayer(std::string const *File) {
+    if (File) {
+      return std::make_unique<FEXCore::Config::MainLoader>(*File);
+    }
+    else {
+      return std::make_unique<FEXCore::Config::MainLoader>();
+    }
+  }
+
+  std::unique_ptr<FEXCore::Config::Layer> CreateAppLayer(const std::string& Filename, bool Global) {
+    return std::make_unique<FEXCore::Config::AppLoader>(Filename, Global);
+  }
+
+  std::unique_ptr<FEXCore::Config::Layer> CreateEnvironmentLayer(char *const _envp[]) {
+    return std::make_unique<FEXCore::Config::EnvLoader>(_envp);
+  }
 }
 

External/FEXCore/Source/Interface/Config/Config.json

@@ -137,6 +137,13 @@
           "Disables optimizations passes for debugging."
         ]
       },
+      "SRA": {
+        "Type": "bool",
+        "Default": "true",
+        "Desc": [
+          "Set to false to disable Static Register Allocation"
+        ]
+      },
       "Force32BitAllocator": {
         "Type": "bool",
         "Default": "false",

External/FEXCore/Source/Interface/Context/Context.cpp

@@ -4,9 +4,18 @@
 #include "Interface/Core/OpcodeDispatcher.h"
 #include "Interface/Core/X86Tables/X86Tables.h"
 
-#include <FEXCore/Config/Config.h>
 #include <FEXCore/Core/CoreState.h>
-#include <FEXCore/Debug/X86Tables.h>
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Core/CPUID.h>
+#include <FEXCore/Core/SignalDelegator.h>
+#include "FEXCore/Debug/InternalThreadState.h"
+
+#include <string.h>
+#include <utility>
+
+namespace FEXCore::HLE {
+  class SyscallVisitor;
+}
 
 namespace FEXCore::Context {
   void InitializeStaticTables(OperatingMode Mode) {
@@ -34,7 +43,7 @@ namespace FEXCore::Context {
     delete CTX;
   }
 
-  bool InitCore(FEXCore::Context::Context *CTX, FEXCore::CodeLoader *Loader) {
+  FEXCore::Core::InternalThreadState* InitCore(FEXCore::Context::Context *CTX, FEXCore::CodeLoader *Loader) {
     return CTX->InitCore(Loader);
   }
 
@@ -101,8 +110,8 @@ namespace FEXCore::Context {
   void RegisterExternalSyscallVisitor(FEXCore::Context::Context *CTX, [[maybe_unused]] uint64_t Syscall, [[maybe_unused]] FEXCore::HLE::SyscallVisitor *Visitor) {
   }
 
-  void HandleCallback(FEXCore::Context::Context *CTX, uint64_t RIP) {
-    CTX->HandleCallback(RIP);
+  void HandleCallback(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread, uint64_t RIP) {
+    CTX->HandleCallback(Thread, RIP);
   }
 
   void RegisterHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func, bool Required) {
@@ -117,6 +126,10 @@ namespace FEXCore::Context {
     return CTX->CreateThread(NewThreadState, ParentTID);
   }
 
+  void ExecutionThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread) {
+    return CTX->ExecutionThread(Thread);
+  }
+
   void InitializeThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread) {
     return CTX->InitializeThread(Thread);
   }

External/FEXCore/Source/Interface/Context/Context.h

@@ -1,48 +1,47 @@
 #pragma once
 
-#include "Common/JitSymbols.h"
 #include "Interface/Core/CPUID.h"
-#include "Interface/Core/Frontend.h"
 #include "Interface/Core/HostFeatures.h"
-#include "Interface/Core/InternalThreadState.h"
 #include "Interface/Core/X86HelperGen.h"
-#include "Interface/IR/PassManager.h"
-#include "Interface/IR/Passes/RegisterAllocationPass.h"
 #include <FEXCore/Config/Config.h>
-#include <FEXCore/Core/CPUBackend.h>
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/SignalDelegator.h>
+#include <FEXCore/Debug/InternalThreadState.h>
 #include <FEXCore/Utils/CompilerDefs.h>
 #include <FEXCore/Utils/Event.h>
 #include <stdint.h>
 
+#include <atomic>
+#include <condition_variable>
 #include <functional>
 #include <istream>
 #include <map>
 #include <memory>
 #include <mutex>
-#include <optional>
-#include <ostream>
-#include <set>
 #include <shared_mutex>
+#include <stddef.h>
+#include <string>
 #include <unordered_map>
 #include <queue>
+#include <vector>
 
 namespace FEXCore {
+class CodeLoader;
 class ThunkHandler;
-class BlockSamplingData;
 class GdbServer;
-class SiganlDelegator;
 
 namespace CPU {
   class Arm64JITCore;
   class X86JITCore;
 }
 namespace HLE {
+struct SyscallArguments;
 class SyscallHandler;
 }
 }
 
 namespace FEXCore::IR {
-  class RegisterAllocationPass;
   class RegisterAllocationData;
   class IRListView;
 namespace Validation {
@@ -122,6 +121,7 @@ namespace FEXCore::Context {
       FEX_CONFIG_OPT(RootFSPath, ROOTFS);
       FEX_CONFIG_OPT(ThunkHostLibsPath, THUNKHOSTLIBS);
       FEX_CONFIG_OPT(DumpIR, DUMPIR);
+      FEX_CONFIG_OPT(StaticRegisterAllocation, SRA);
     } Config;
 
     using IntCallbackReturn =  FEX_NAKED void(*)(FEXCore::Core::InternalThreadState *Thread, volatile void *Host_RSP);
@@ -183,7 +183,7 @@ namespace FEXCore::Context {
     Context();
     ~Context();
 
-    bool InitCore(FEXCore::CodeLoader *Loader);
+    FEXCore::Core::InternalThreadState* InitCore(FEXCore::CodeLoader *Loader);
     FEXCore::Context::ExitReason RunUntilExit();
     int GetProgramStatus() const;
     bool IsPaused() const { return !Running; }
@@ -199,7 +199,7 @@ namespace FEXCore::Context {
     bool GetGdbServerStatus() const { return DebugServer != nullptr; }
     void StartGdbServer();
     void StopGdbServer();
-    void HandleCallback(uint64_t RIP);
+    void HandleCallback(FEXCore::Core::InternalThreadState *Thread, uint64_t RIP);
     void RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required);
     void RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required);
 

External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64.cpp

@@ -2,6 +2,7 @@
 #include "Interface/Core/ArchHelpers/MContext.h"
 
 #include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/Telemetry.h>
 
 #include <atomic>
 #include <stdint.h>
@@ -9,6 +10,9 @@
 #include <signal.h>
 
 namespace FEXCore::ArchHelpers::Arm64 {
+FEXCORE_TELEMETRY_STATIC_INIT(SplitLock, TYPE_HAS_SPLIT_LOCKS);
+FEXCORE_TELEMETRY_STATIC_INIT(SplitLock16B, TYPE_16BYTE_SPLIT);
+
 static __uint128_t LoadAcquire128(uint64_t Addr) {
   __uint128_t Result{};
   uint64_t Lower;
@@ -60,22 +64,12 @@ static bool StoreCAS8(uint8_t &Expected, uint8_t Val, uint64_t Addr) {
   return Atom->compare_exchange_strong(Expected, Val);
 }
 
-bool HandleCASPAL(void *_ucontext, void *_info, uint32_t Instr) {
+
+
+static bool RunCASPAL(void *_ucontext, void *_info, uint32_t Size, uint32_t DesiredReg1, uint32_t DesiredReg2, uint32_t ExpectedReg1, uint32_t ExpectedReg2, uint32_t AddressReg) {
   mcontext_t* mcontext = &reinterpret_cast<ucontext_t*>(_ucontext)->uc_mcontext;
-  siginfo_t* info = reinterpret_cast<siginfo_t*>(_info);
-
-  if (info->si_code != BUS_ADRALN) {
-    // This only handles alignment problems
-    return false;
-  }
-
-  uint32_t Size = (Instr >> 30) & 1;
-
-  uint32_t DesiredReg1 = Instr & 0b11111;
-  uint32_t DesiredReg2 = DesiredReg1 + 1;
-  uint32_t ExpectedReg1 = (Instr >> 16) & 0b11111;
-  uint32_t ExpectedReg2 = ExpectedReg1 + 1;
-  uint32_t AddressReg = (Instr >> 5) & 0b11111;
+  
+  //Bus_ADRALN check happens in HandleCASPAL and HandleCASPAL_ARMv8
 
   if (Size == 0) {
     // 32bit
@@ -94,8 +88,15 @@ bool HandleCASPAL(void *_ucontext, void *_info, uint32_t Instr) {
     // Both cross-cacheline and cross 16byte both need dual CAS loops that can tear
     // ARMv8.4 LSE2 solves all atomic issues except cross-cacheline
 
+    // Check for Split lock across a cacheline
+    if ((Addr & 63) > 56) {
+      FEXCORE_TELEMETRY_SET(SplitLock, 1);
+    }
+
     uint64_t AlignmentMask = 0b1111;
     if ((Addr & AlignmentMask) > 8) {
+      FEXCORE_TELEMETRY_SET(SplitLock16B, 1);
+
       uint64_t Alignment = Addr & 0b111;
       Addr &= ~0b111ULL;
       uint64_t AddrUpper = Addr + 8;
@@ -248,6 +249,87 @@ bool HandleCASPAL(void *_ucontext, void *_info, uint32_t Instr) {
   return false;
 }
 
+bool HandleCASPAL(void *_ucontext, void *_info, uint32_t Instr) {
+  siginfo_t* info = reinterpret_cast<siginfo_t*>(_info);
+
+  if (info->si_code != BUS_ADRALN) {
+    // This only handles alignment problems
+    return false;
+  }
+
+  uint32_t Size = (Instr >> 30) & 1;
+
+  uint32_t DesiredReg1 = Instr & 0b11111;
+  uint32_t DesiredReg2 = DesiredReg1 + 1;
+  uint32_t ExpectedReg1 = (Instr >> 16) & 0b11111;
+  uint32_t ExpectedReg2 = ExpectedReg1 + 1;
+  uint32_t AddressReg = (Instr >> 5) & 0b11111;
+  
+  return RunCASPAL(_ucontext, _info, Size, DesiredReg1, DesiredReg2, ExpectedReg1, ExpectedReg2, AddressReg);
+}
+
+uint64_t HandleCASPAL_ARMv8(void *_ucontext, void *_info, uint32_t Instr) {
+  mcontext_t* mcontext = &reinterpret_cast<ucontext_t*>(_ucontext)->uc_mcontext;
+  siginfo_t* info = reinterpret_cast<siginfo_t*>(_info);
+
+  if (info->si_code != BUS_ADRALN) {
+    // This only handles alignment problems
+    return 0;
+  }
+  // caspair
+  // [1] ldaxp(TMP2.W(), TMP3.W(), MemOperand(MemSrc)); <-- DataReg & AddrReg
+  // [2] cmp(TMP2.W(), Expected.first.W()); <-- ExpectedReg1
+  // [3] ccmp(TMP3.W(), Expected.second.W(), NoFlag, Condition::eq); <-- ExpectedREg2
+  // [4] b(&LoopNotExpected, Condition::ne);
+  // [5] stlxp(TMP2.W(), Desired.first.W(), Desired.second.W(), MemOperand(MemSrc)); <-- DesiredReg
+  // [6] cbnz(TMP2.W(), &LoopTop);
+  // [7] mov(Dst.first.W(), Expected.first.W());
+  // [8] mov(Dst.second.W(), Expected.second.W());
+  // [9] b(&LoopExpected);
+  // [10] mov(Dst.first.W(), TMP2.W());
+  // [11] mov(Dst.second.W(), TMP3.W());
+  // [12] clrex();
+
+  uint32_t *PC = (uint32_t*)ArchHelpers::Context::GetPc(_ucontext);
+  
+  uint32_t Size = (Instr >> 30) & 1;
+  uint32_t AddrReg = (Instr >> 5) & 0x1F;
+  uint32_t DataReg = Instr & 0x1F;
+  uint32_t DataReg2 = (Instr >> 10) & 0x1F;
+  
+  uint32_t ExpectedReg1{};
+  uint32_t ExpectedReg2{};
+  
+  uint32_t DesiredReg1{};
+  uint32_t DesiredReg2{};
+  
+  if(Size != 0) { //Only 32-bit pairs
+    return 0; 
+  }
+  
+  for(int i = 1; i < 10; i++) {
+    uint32_t NextInstr = PC[i];
+    if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::CMP_INST) {
+       ExpectedReg1 = GetRmReg(NextInstr);
+    } else if ((NextInstr & FEXCore::ArchHelpers::Arm64::CCMP_MASK) == FEXCore::ArchHelpers::Arm64::CCMP_INST) {
+       ExpectedReg2 = GetRmReg(NextInstr);
+    } else if ((NextInstr & FEXCore::ArchHelpers::Arm64::STLXP_MASK) == FEXCore::ArchHelpers::Arm64::STLXP_INST) {
+       DesiredReg1 = (NextInstr & 0x1F);
+       DesiredReg2 = (NextInstr >> 10) & 0x1F;
+    }
+  }
+  
+  //mov expected into the temp registers used by JIT
+  mcontext->regs[DataReg] = mcontext->regs[ExpectedReg1];
+  mcontext->regs[DataReg2] = mcontext->regs[ExpectedReg2];
+  
+  if(RunCASPAL(_ucontext, _info, Size, DesiredReg1, DesiredReg2, DataReg, DataReg2, AddrReg)) {
+    return 9 * sizeof(uint32_t); // skip to mov + clrex
+  } else {
+    return 0;
+  }
+}
+
 uint16_t DoLoad16(uint64_t Addr) {
   uint64_t AlignmentMask = 0b1111;
   if ((Addr & AlignmentMask) == 15) {
@@ -430,9 +512,16 @@ uint16_t DoCAS16(
   uint64_t Addr,
   CASExpectedFn<uint16_t> ExpectedFunction,
   CASDesiredFn<uint16_t> DesiredFunction) {
+
+  if ((Addr & 63) == 63) {
+    FEXCORE_TELEMETRY_SET(SplitLock, 1);
+  }
+
   // 16 bit
   uint64_t AlignmentMask = 0b1111;
   if ((Addr & AlignmentMask) == 15) {
+    FEXCORE_TELEMETRY_SET(SplitLock16B, 1);
+
     // Address crosses over 16byte or 64byte threshold
     // Need a dual 8bit CAS loop
     uint64_t AddrUpper = Addr + 1;
@@ -706,9 +795,16 @@ uint32_t DoCAS32(
   uint64_t Addr,
   CASExpectedFn<uint32_t> ExpectedFunction,
   CASDesiredFn<uint32_t> DesiredFunction) {
+
+  if ((Addr & 63) > 60) {
+    FEXCORE_TELEMETRY_SET(SplitLock, 1);
+  }
+
   // 32 bit
   uint64_t AlignmentMask = 0b1111;
   if ((Addr & AlignmentMask) > 12) {
+    FEXCORE_TELEMETRY_SET(SplitLock16B, 1);
+
     // Address crosses over 16byte threshold
     // Needs dual 4 byte CAS loop
     uint64_t Alignment = Addr & 0b11;
@@ -936,9 +1032,16 @@ uint64_t DoCAS64(
   uint64_t Addr,
   CASExpectedFn<uint64_t> ExpectedFunction,
   CASDesiredFn<uint64_t> DesiredFunction) {
+
+  if ((Addr & 63) > 56) {
+    FEXCORE_TELEMETRY_SET(SplitLock, 1);
+  }
+
   // 64bit
   uint64_t AlignmentMask = 0b1111;
   if ((Addr & AlignmentMask) > 8) {
+    FEXCORE_TELEMETRY_SET(SplitLock16B, 1);
+
     uint64_t Alignment = Addr & 0b111;
     Addr &= ~0b111ULL;
     uint64_t AddrUpper = Addr + 8;
@@ -1091,21 +1194,10 @@ uint64_t DoCAS64(
   }
 }
 
-bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr) {
+static bool RunCASAL(void *_ucontext, void *_info, uint32_t Size, uint32_t DesiredReg, uint32_t ExpectedReg, uint32_t AddressReg) {
   mcontext_t* mcontext = &reinterpret_cast<ucontext_t*>(_ucontext)->uc_mcontext;
   siginfo_t* info = reinterpret_cast<siginfo_t*>(_info);
 
-  if (info->si_code != BUS_ADRALN) {
-    // This only handles alignment problems
-    return false;
-  }
-
-  uint32_t Size = 1 << (Instr >> 30);
-
-  uint32_t DesiredReg = Instr & 0b11111;
-  uint32_t ExpectedReg = (Instr >> 16) & 0b11111;
-  uint32_t AddressReg = (Instr >> 5) & 0b11111;
-
   uint64_t Addr = mcontext->regs[AddressReg];
 
   // Cross-cacheline CAS doesn't work on ARM
@@ -1185,6 +1277,23 @@ bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr) {
   return false;
 }
 
+bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr) {
+  mcontext_t* mcontext = &reinterpret_cast<ucontext_t*>(_ucontext)->uc_mcontext;
+  siginfo_t* info = reinterpret_cast<siginfo_t*>(_info);
+
+  if (info->si_code != BUS_ADRALN) {
+    // This only handles alignment problems
+    return false;
+  }
+
+  uint32_t Size = 1 << (Instr >> 30);
+
+  uint32_t DesiredReg = Instr & 0b11111;
+  uint32_t ExpectedReg = (Instr >> 16) & 0b11111;
+  uint32_t AddressReg = (Instr >> 5) & 0b11111;
+  return RunCASAL(_ucontext, _info, Size, DesiredReg, ExpectedReg, AddressReg);
+}
+
 bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
   mcontext_t* mcontext = &reinterpret_cast<ucontext_t*>(_ucontext)->uc_mcontext;
   siginfo_t* info = reinterpret_cast<siginfo_t*>(_info);
@@ -1527,6 +1636,53 @@ bool HandleAtomicLoad128(void *_ucontext, void *_info, uint32_t Instr) {
   return true;
 }
 
+static uint64_t HandleCAS_NoAtomics(void *_ucontext, void *_info)
+{
+  mcontext_t* mcontext = &reinterpret_cast<ucontext_t*>(_ucontext)->uc_mcontext;
+  siginfo_t* info = reinterpret_cast<siginfo_t*>(_info);
+  
+  // ARMv8.0 CAS
+  // [1] ldaxrb(TMP2.W(), MemOperand(MemSrc))
+  // [2] cmp (TMP2.W(), Expected.W())
+  // [3] b
+  // [4] stlxrb(TMP3.W(), Desired.W(), MemOperand(MemSrc)
+  // [5] cbnz
+  // [6] mov
+  // [7] b
+  // [8] mov (.., TMP2.W());
+  // [9] clrex
+  
+  uint32_t *PC = (uint32_t*)ArchHelpers::Context::GetPc(_ucontext);
+  uint32_t Instr = PC[0];
+  uint32_t Size = 1 << (Instr >> 30);
+  uint32_t AddressReg = GetRnReg(Instr);
+  uint32_t ResultReg = GetRdReg(Instr); //TMP2 
+  uint32_t DesiredReg = 0;
+  uint32_t ExpectedReg = 0;
+  for (size_t i = 1; i < 6; ++i) {
+     uint32_t NextInstr = PC[i];
+     if ((NextInstr & FEXCore::ArchHelpers::Arm64::STLXR_MASK) == FEXCore::ArchHelpers::Arm64::STLXR_INST) {
+       #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
+       // Just double check that the memory destination matches
+       uint32_t StoreAddressReg = GetRnReg(NextInstr);
+       LOGMAN_THROW_A(StoreAddressReg == AddressReg, "StoreExclusive memory register didn't match the store exclusive register");
+       #endif
+       DesiredReg = GetRdReg(NextInstr);
+     }
+     else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::CMP_INST) {
+       ExpectedReg = GetRmReg(NextInstr);
+     }
+  }
+  //set up CASAL by doing mov(TMP2, Expected)
+  mcontext->regs[ResultReg] = mcontext->regs[ExpectedReg];
+  
+  if(RunCASAL(_ucontext, _info, Size, DesiredReg, ResultReg, AddressReg)) {
+    return 7 * sizeof(uint32_t); //jump to mov to allocated register
+  } else {
+    return 0;
+  }
+}
+
 uint64_t HandleAtomicLoadstoreExclusive(void *_ucontext, void *_info) {
   mcontext_t* mcontext = &reinterpret_cast<ucontext_t*>(_ucontext)->uc_mcontext;
   siginfo_t* info = reinterpret_cast<siginfo_t*>(_info);
@@ -1611,6 +1767,9 @@ uint64_t HandleAtomicLoadstoreExclusive(void *_ucontext, void *_info) {
       }
       DataSourceReg = GetRmReg(NextInstr);
     }
+    else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::CMP_INST) {
+    	return HandleCAS_NoAtomics(_ucontext, _info); //ARMv8.0 CAS
+    }
     else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::AND_INST) {
       AtomicOp = ExclusiveAtomicPairType::TYPE_AND;
       DataSourceReg = GetRmReg(NextInstr);

External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64.h

@@ -30,9 +30,14 @@ namespace FEXCore::ArchHelpers::Arm64 {
   constexpr uint32_t ALU_OP_MASK = 0x7F'00'00'00;
   constexpr uint32_t ADD_INST    = 0x0B'00'00'00;
   constexpr uint32_t SUB_INST    = 0x4B'00'00'00;
+  constexpr uint32_t CMP_INST    = 0x6B'00'00'00;
   constexpr uint32_t AND_INST    = 0x0A'00'00'00;
   constexpr uint32_t OR_INST     = 0x2A'00'00'00;
   constexpr uint32_t EOR_INST    = 0x4A'00'00'00;
+  
+  constexpr uint32_t CCMP_MASK   = 0x7F'E0'0C'10;
+  constexpr uint32_t CCMP_INST   = 0x7A'40'00'00;
+  
   enum ExclusiveAtomicPairType {
     TYPE_SWAP,
     TYPE_ADD,
@@ -77,6 +82,7 @@ namespace FEXCore::ArchHelpers::Arm64 {
   bool HandleAtomicLoad128(void *_ucontext, void *_info, uint32_t Instr);
   uint64_t HandleAtomicLoadstoreExclusive(void *_ucontext, void *_info);
   bool HandleCASPAL(void *_ucontext, void *_info, uint32_t Instr);
+  uint64_t HandleCASPAL_ARMv8(void *_ucontext, void *_info, uint32_t Instr);
   bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr);
   bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr);
 }

External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp

@@ -4,6 +4,11 @@
 #include <FEXCore/Core/CoreState.h>
 
 #include "aarch64/cpu-aarch64.h"
+#include "cpu-features.h"
+#include "aarch64/instructions-aarch64.h"
+#include "utils-vixl.h"
+
+#include <tuple>
 
 namespace FEXCore::CPU {
 #define STATE x28
@@ -143,22 +148,26 @@ void Arm64Emitter::PopCalleeSavedRegisters() {
 
 
 void Arm64Emitter::SpillStaticRegs() {
-  for (size_t i = 0; i < SRA64.size(); i+=2) {
-      stp(SRA64[i], SRA64[i+1], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])));
-  }
+  if (StaticRegisterAllocation()) {
+    for (size_t i = 0; i < SRA64.size(); i+=2) {
+        stp(SRA64[i], SRA64[i+1], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])));
+    }
 
-  for (size_t i = 0; i < SRAFPR.size(); i+=2) {
-    stp(SRAFPR[i].Q(), SRAFPR[i+1].Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm[i][0])));
+    for (size_t i = 0; i < SRAFPR.size(); i+=2) {
+      stp(SRAFPR[i].Q(), SRAFPR[i+1].Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm[i][0])));
+    }
   }
 }
 
 void Arm64Emitter::FillStaticRegs() {
-  for (size_t i = 0; i < SRA64.size(); i+=2) {
-    ldp(SRA64[i], SRA64[i+1], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])));
-  }
+  if (StaticRegisterAllocation()) {
+    for (size_t i = 0; i < SRA64.size(); i+=2) {
+      ldp(SRA64[i], SRA64[i+1], MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])));
+    }
 
-  for (size_t i = 0; i < SRAFPR.size(); i+=2) {
-    ldp(SRAFPR[i].Q(), SRAFPR[i+1].Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm[i][0])));
+    for (size_t i = 0; i < SRAFPR.size(); i+=2) {
+      ldp(SRAFPR[i].Q(), SRAFPR[i+1].Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm[i][0])));
+    }
   }
 }
 
@@ -222,7 +231,7 @@ void Arm64Emitter::ResetStack() {
 }
 
 void Arm64Emitter::Align16B() {
-  uint64_t CurrentOffset = GetBuffer()->GetOffsetAddress<uint64_t>(GetCursorOffset());
+  uint64_t CurrentOffset = GetCursorAddress<uint64_t>();
   for (uint64_t i = (16 - (CurrentOffset & 0xF)); i != 0; i -= 4) {
       nop();
   }

External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h

@@ -1,7 +1,16 @@
 #pragma once
 
 #include "aarch64/assembler-aarch64.h"
+#include "aarch64/constants-aarch64.h"
 #include "aarch64/cpu-aarch64.h"
+#include "aarch64/operands-aarch64.h"
+#include "platform-vixl.h"
+#include "FEXCore/Config/Config.h"
+
+#include <array>
+#include <stddef.h>
+#include <stdint.h>
+#include <utility>
 
 namespace FEXCore::CPU {
 using namespace vixl;
@@ -72,6 +81,8 @@ protected:
 
   uint32_t DCacheLineSize{};
   uint32_t ICacheLineSize{};
+
+  FEX_CONFIG_OPT(StaticRegisterAllocation, SRA);
 };
 
 }

External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64_stubs.cpp

@@ -1,6 +1,7 @@
 #include "Interface/Core/ArchHelpers/Arm64.h"
 
 #include <FEXCore/Utils/LogManager.h>
+#include <stdint.h>
 
 namespace FEXCore::ArchHelpers::Arm64 {
 
@@ -23,4 +24,4 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
 }
 #endif
 
-}
+}

External/FEXCore/Source/Interface/Core/ArchHelpers/MContext.h

@@ -18,6 +18,7 @@ struct X86ContextBackup {
   // RIP and RSP is stored in GPRs here
   uint64_t GPRs[23];
   FEXCore::x86_64::_libc_fpstate FPRState;
+  uint64_t sa_mask;
 
   // Guest state
   int Signal;
@@ -35,6 +36,7 @@ struct ArmContextBackup {
   uint32_t FPSR;
   uint32_t FPCR;
   __uint128_t FPRs[32];
+  uint64_t sa_mask;
 
   // Guest state
   int Signal;
@@ -44,6 +46,11 @@ struct ArmContextBackup {
   static constexpr int RedZoneSize = 0;
 };
 
+static inline ucontext_t* GetUContext(void* ucontext) {
+  ucontext_t* _context = (ucontext_t*)ucontext;
+  return _context;
+}
+
 static inline mcontext_t* GetMContext(void* ucontext) {
   ucontext_t* _context = (ucontext_t*)ucontext;
   return &_context->uc_mcontext;
@@ -102,6 +109,7 @@ using ContextBackup = ArmContextBackup;
 template <typename T>
 static inline void BackupContext(void* ucontext, T *Backup) {
   if constexpr (std::is_same<T, ArmContextBackup>::value) {
+    auto _ucontext = GetUContext(ucontext);
     auto _mcontext = GetMContext(ucontext);
 
     memcpy(&Backup->GPRs[0], &_mcontext->regs[0], 31 * sizeof(uint64_t));
@@ -115,6 +123,9 @@ static inline void BackupContext(void* ucontext, T *Backup) {
     Backup->FPSR = HostState->FPSR;
     Backup->FPCR = HostState->FPCR;
     memcpy(&Backup->FPRs[0], &HostState->FPRs[0], 32 * sizeof(__uint128_t));
+
+    // Save the signal mask so we can restore it
+    memcpy(&Backup->sa_mask, &_ucontext->uc_sigmask, sizeof(uint64_t));
   } else {
       ERROR_AND_DIE("Wrong context type"); // This must be a runtime error
   }
@@ -123,6 +134,7 @@ static inline void BackupContext(void* ucontext, T *Backup) {
 template <typename T>
 static inline void RestoreContext(void* ucontext, T *Backup) {
   if constexpr (std::is_same<T, ArmContextBackup>::value) {
+    auto _ucontext = GetUContext(ucontext);
    auto _mcontext = GetMContext(ucontext);
 
     HostFPRState *HostState = reinterpret_cast<HostFPRState*>(&_mcontext->__reserved[0]);
@@ -136,6 +148,9 @@ static inline void RestoreContext(void* ucontext, T *Backup) {
     ArchHelpers::Context::SetPc(ucontext, Backup->PrevPC);
     ArchHelpers::Context::SetSp(ucontext, Backup->PrevSP);
     memcpy(&_mcontext->regs[0], &Backup->GPRs[0], 31 * sizeof(uint64_t));
+
+    // Restore the signal mask now
+    memcpy(&_ucontext->uc_sigmask, &Backup->sa_mask, sizeof(uint64_t));
   } else {
       ERROR_AND_DIE("Wrong context type"); // This must be a runtime error
   }
@@ -181,6 +196,7 @@ using ContextBackup = X86ContextBackup;
 template <typename T>
 static inline void BackupContext(void* ucontext, T *Backup) {
   if constexpr (std::is_same<T, X86ContextBackup>::value) {
+    auto _ucontext = GetUContext(ucontext);
     auto _mcontext = GetMContext(ucontext);
 
     // Copy the GPRs
@@ -188,6 +204,9 @@ static inline void BackupContext(void* ucontext, T *Backup) {
     // Copy the FPRState
     memcpy(&Backup->FPRState, _mcontext->fpregs, sizeof(X86ContextBackup::FPRState));
     // XXX: Save 256bit and 512bit AVX register state
+
+    // Save the signal mask so we can restore it
+    memcpy(&Backup->sa_mask, &_ucontext->uc_sigmask, sizeof(uint64_t));
   } else {
     ERROR_AND_DIE("Wrong context type"); // This must be a runtime error
   }
@@ -196,12 +215,16 @@ static inline void BackupContext(void* ucontext, T *Backup) {
 template <typename T>
 static inline void RestoreContext(void* ucontext, T *Backup) {
   if constexpr (std::is_same<T, X86ContextBackup>::value) {
+    auto _ucontext = GetUContext(ucontext);
     auto _mcontext = GetMContext(ucontext);
 
     // Copy the GPRs
     memcpy(&_mcontext->gregs[0], &Backup->GPRs[0], sizeof(X86ContextBackup::GPRs));
     // Copy the FPRState
     memcpy(_mcontext->fpregs, &Backup->FPRState, sizeof(X86ContextBackup::FPRState));
+
+    // Restore the signal mask now
+    memcpy(&_ucontext->uc_sigmask, &Backup->sa_mask, sizeof(uint64_t));
   } else {
     ERROR_AND_DIE("Wrong context type"); // This must be a runtime error
   }
@@ -209,4 +232,4 @@ static inline void RestoreContext(void* ucontext, T *Backup) {
 
 #endif
 
-} // namespace FEXCore::ArchHelpers::Context
+} // namespace FEXCore::ArchHelpers::Context

External/FEXCore/Source/Interface/Core/BlockSamplingData.cpp

@@ -2,6 +2,7 @@
 #include <FEXCore/Utils/LogManager.h>
 #include <cstring>
 #include <fstream>
+#include <utility>
 
 namespace FEXCore {
   void BlockSamplingData::DumpBlockData() {

External/FEXCore/Source/Interface/Core/BlockSamplingData.h

@@ -1,6 +1,7 @@
 #pragma once
+
+#include <cstdint>
 #include <unordered_map>
-#include <stdint.h>
 
 namespace FEXCore {
 class BlockSamplingData {

External/FEXCore/Source/Interface/Core/CPUID.cpp

@@ -5,8 +5,11 @@ desc: Handles presented capability bits for guest cpu
 $end_info$
 */
 
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/CPUID.h>
 #include "Interface/Context/Context.h"
 #include "Interface/Core/CPUID.h"
+#include "Interface/Core/HostFeatures.h"
 #include "git_version.h"
 
 #include <cstring>

External/FEXCore/Source/Interface/Core/CPUID.h

@@ -4,7 +4,9 @@
 
 #include <FEXCore/Core/CPUID.h>
 #include <FEXCore/Config/Config.h>
-#include <FEXCore/Utils/LogManager.h>
+
+#include <cstdint>
+#include <utility>
 
 namespace FEXCore {
 namespace Context {

External/FEXCore/Source/Interface/Core/CompileService.cpp

@@ -1,8 +1,21 @@
 #include "Interface/Context/Context.h"
 #include "Interface/Core/LookupCache.h"
 #include "Interface/Core/CompileService.h"
-#include "Interface/Core/InternalThreadState.h"
 #include "Interface/Core/OpcodeDispatcher.h"
+#include "FEXCore/Debug/InternalThreadState.h"
+#include "FEXCore/HLE/Linux/ThreadManagement.h"
+#include "Interface/IR/PassManager.h"
+
+#include <FEXCore/Core/CPUBackend.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/SignalDelegator.h>
+#include <FEXCore/Utils/Event.h>
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/Threads.h>
+
+#include <memory>
+#include <pthread.h>
+#include <stdio.h>
 
 namespace FEXCore {
   static void* ThreadHandler(void *Arg) {

External/FEXCore/Source/Interface/Core/CompileService.h

@@ -1,23 +1,22 @@
 #pragma once
 
-#include <FEXCore/Core/CPUBackend.h>
+#include <FEXCore/Debug/InternalThreadState.h>
 #include <FEXCore/Utils/Event.h>
 #include <FEXCore/Utils/Threads.h>
 
+#include <atomic>
 #include <memory>
-#include <thread>
-#include <unordered_map>
+#include <mutex>
 #include <queue>
+#include <stdint.h>
+#include <vector>
 
 namespace FEXCore {
 namespace Context {
   struct Context;
 }
-namespace Core {
-  struct InternalThreadState;
-}
-
 namespace IR {
+  class IRListView;
   class RegisterAllocationData;
 };
 class CompileService final {

External/FEXCore/Source/Interface/Core/Core.cpp

@@ -7,42 +7,70 @@ desc: Glues Frontend, OpDispatcher and IR Opts & Compilation, LookupCache, Dispa
 $end_info$
 */
 
-#include "Common/MathUtils.h"
-#include "Common/Paths.h"
-
 #include "Interface/Context/Context.h"
 #include "Interface/Core/LookupCache.h"
-#include "Interface/Core/BlockSamplingData.h"
 #include "Interface/Core/CompileService.h"
 #include "Interface/Core/Core.h"
-#include "Interface/Core/DebugData.h"
+#include "Interface/Core/CPUID.h"
+#include "Interface/Core/Frontend.h"
+#include "Interface/Core/GdbServer.h"
 #include "Interface/Core/OpcodeDispatcher.h"
 #include "Interface/Core/Interpreter/InterpreterCore.h"
 #include "Interface/Core/JIT/JITCore.h"
+#include "Interface/HLE/Thunks/Thunks.h"
 #include "Interface/IR/Passes/RegisterAllocationPass.h"
 #include "Interface/IR/Passes.h"
+#include "Interface/IR/PassManager.h"
 
 #include <FEXCore/Config/Config.h>
 #include <FEXCore/Core/CodeLoader.h>
+#include <FEXCore/Core/Context.h>
 #include <FEXCore/Core/CoreState.h>
 #include <FEXCore/Core/CPUBackend.h>
+#include <FEXCore/Core/SignalDelegator.h>
 #include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/Debug/InternalThreadState.h>
+#include <FEXCore/Debug/X86Tables.h>
 #include <FEXCore/HLE/SyscallHandler.h>
+#include <FEXCore/HLE/Linux/ThreadManagement.h>
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+#include <FEXCore/IR/RegisterAllocationData.h>
 #include <FEXCore/Utils/Allocator.h>
+#include <FEXCore/Utils/Event.h>
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/Threads.h>
 
-#include "Interface/HLE/Thunks/Thunks.h"
-#include "FEXCore/Utils/Allocator.h"
-
-#include <xxhash.h>
-#include <fstream>
-#include <unistd.h>
-#include <filesystem>
 #include <algorithm>
+#include <array>
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdint>
+#include <filesystem>
+#include <functional>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <queue>
+#include <set>
+#include <shared_mutex>
+#include <signal.h>
+#include <stdio.h>
+#include <string.h>
+#include <string>
+#include <string_view>
+#include <sstream>
 #include <sys/mman.h>
-#include <unistd.h>
 #include <sys/stat.h>
-
-#include "Interface/Core/GdbServer.h"
+#include <sys/syscall.h>
+#include <type_traits>
+#include <unistd.h>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+#include <xxhash.h>
 
 namespace FEXCore::CPU {
   bool CreateCPUCore(FEXCore::Context::Context *CTX) {
@@ -195,7 +223,7 @@ namespace FEXCore::Context {
     }
   }
 
-  bool Context::InitCore(FEXCore::CodeLoader *Loader) {
+  FEXCore::Core::InternalThreadState* Context::InitCore(FEXCore::CodeLoader *Loader) {
     ThunkHandler.reset(FEXCore::ThunkHandler::Create());
 
     LocalLoader = Loader;
@@ -228,8 +256,7 @@ namespace FEXCore::Context {
     Thread->CurrentFrame->State.rip = StartingRIP = Loader->DefaultRIP();
 
     InitializeThreadData(Thread);
-
-    return true;
+    return Thread;
   }
 
   void Context::StartGdbServer() {
@@ -243,8 +270,7 @@ namespace FEXCore::Context {
     DebugServer.reset();
   }
 
-  void Context::HandleCallback(uint64_t RIP) {
-    auto Thread = Core::ThreadData.Thread;
+  void Context::HandleCallback(FEXCore::Core::InternalThreadState *Thread, uint64_t RIP) {
     Thread->CPUBackend->CallbackPtr(Thread->CurrentFrame, RIP);
   }
 
@@ -471,8 +497,10 @@ namespace FEXCore::Context {
         Stop(false /* Ignore current thread */);
     });
 
+    State->CTX = this;
+
     #if _M_ARM_64
-    bool DoSRA = true;
+    bool DoSRA = State->CTX->Config.StaticRegisterAllocation;
     #else
     bool DoSRA = false;
     #endif
@@ -482,8 +510,6 @@ namespace FEXCore::Context {
 
     State->PassManager->RegisterSyscallHandler(SyscallHandler);
 
-    State->CTX = this;
-
     // Create CPU backend
     switch (Config.Core) {
     case FEXCore::Config::CONFIG_INTERPRETER:
@@ -1091,7 +1117,9 @@ namespace FEXCore::Context {
 
     if (NewBlock == 0) {
       LogMan::Msg::E("CompileBlockJit: Failed to compile code %lX - aborting process", GuestRIP);
-      abort();
+      // Return similar behaviour of SIGILL abort
+      Frame->Thread->StatusCode = 128 + SIGILL;
+      Stop(false /* Ignore current thread */);
     }
   }
 

External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.cpp

@@ -2,17 +2,28 @@
 
 #include "Interface/Core/ArchHelpers/MContext.h"
 #include "Interface/Core/Dispatcher/Arm64Dispatcher.h"
-
 #include "Interface/Core/Interpreter/InterpreterClass.h"
-#include "Interface/Context/Context.h"
 
+#include "Interface/Context/Context.h"
+#include "Interface/Core/X86HelperGen.h"
+
+#include <FEXCore/Core/CPUBackend.h>
+#include <FEXCore/Core/CoreState.h>
 #include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/Debug/InternalThreadState.h>
+#include <array>
 #include <bit>
 #include <cmath>
+#include <cstdint>
+#include <memory>
+#include <stddef.h>
 
 #include "aarch64/assembler-aarch64.h"
+#include "aarch64/constants-aarch64.h"
+#include "aarch64/operands-aarch64.h"
 #include "aarch64/cpu-aarch64.h"
-#include "aarch64/disasm-aarch64.h"
+#include "code-buffer-vixl.h"
+#include "platform-vixl.h"
 
 namespace FEXCore::CPU {
 
@@ -27,8 +38,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
   SRAEnabled = config.StaticRegisterAssignment;
   SetAllowAssembler(true);
 
-  auto Buffer = GetBuffer();
-  DispatchPtr = Buffer->GetOffsetAddress<CPUBackend::AsmDispatch>(GetCursorOffset());
+  DispatchPtr = GetCursorAddress<CPUBackend::AsmDispatch>();
 
   // while (true) {
   //    Ptr = FindBlock(RIP)
@@ -61,7 +71,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
   add(x0, sp, 0);
   str(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, ReturningStackLocation)));
 
-  AbsoluteLoopTopAddressFillSRA = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
+  AbsoluteLoopTopAddressFillSRA = GetCursorAddress<uint64_t>();
 
   if (SRAEnabled) {
     FillStaticRegs();
@@ -180,11 +190,11 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
 
   {
     bind(&ExitSpillSRA);
-    ThreadStopHandlerAddressSpillSRA = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
+    ThreadStopHandlerAddressSpillSRA = GetCursorAddress<uint64_t>();
     if (SRAEnabled)
       SpillStaticRegs();
 
-    ThreadStopHandlerAddress = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
+    ThreadStopHandlerAddress = GetCursorAddress<uint64_t>();
 
     PopCalleeSavedRegisters();
 
@@ -194,7 +204,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
   }
 
   {
-    ExitFunctionLinkerAddress = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
+    ExitFunctionLinkerAddress = GetCursorAddress<uint64_t>();
     if (SRAEnabled)
       SpillStaticRegs();
 
@@ -231,7 +241,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
   }
 
   {
-    SignalHandlerReturnAddress = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
+    SignalHandlerReturnAddress = GetCursorAddress<uint64_t>();
 
     // Now to get back to our old location we need to do a fault dance
     // We can't use SIGTRAP here since gdb catches it and never gives it to the application!
@@ -239,12 +249,12 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
   }
 
   {
-    ThreadPauseHandlerAddressSpillSRA = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
+    ThreadPauseHandlerAddressSpillSRA = GetCursorAddress<uint64_t>();
     if (SRAEnabled)
       SpillStaticRegs();
 
     bind(&ThreadPauseHandler);
-    ThreadPauseHandlerAddress = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
+    ThreadPauseHandlerAddress = GetCursorAddress<uint64_t>();
     // We are pausing, this means the frontend should be waiting for this thread to idle
     // We will have faulted and jumped to this location at this point
 
@@ -254,7 +264,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
     ldr(x2, &l_Sleep);
     blr(x2);
 
-    PauseReturnInstruction = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
+    PauseReturnInstruction = GetCursorAddress<uint64_t>();
     // Fault to start running again
     hlt(0);
   }
@@ -275,7 +285,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
     // On return to the thunk, the thunk can get whatever its return value is from the thread context depending on ABI handling on its end
     // When the thunk itself returns, it'll do its regular return logic there
     // void ReentrantCallback(FEXCore::Core::InternalThreadState *Thread, uint64_t RIP);
-    CallbackPtr = Buffer->GetOffsetAddress<CPUBackend::JITCallback>(GetCursorOffset());
+    CallbackPtr = GetCursorAddress<CPUBackend::JITCallback>();
 
     // We expect the thunk to have previously pushed the registers it was using
     PushCalleeSavedRegisters();
@@ -324,7 +334,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
 
   FinalizeCode();
   Start = reinterpret_cast<uint64_t>(DispatchPtr);
-  End = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
+  End = GetCursorAddress<uint64_t>();
   vixl::aarch64::CPU::EnsureIAndDCacheCoherency(reinterpret_cast<void*>(DispatchPtr), End - reinterpret_cast<uint64_t>(DispatchPtr));
   GetBuffer()->SetExecutable();
 

External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.h

@@ -3,7 +3,13 @@
 #include "Interface/Core/ArchHelpers/Arm64Emitter.h"
 #include "Interface/Core/Dispatcher/Dispatcher.h"
 
-#include "aarch64/assembler-aarch64.h"
+namespace FEXCore::Context {
+struct Context;
+}
+
+namespace FEXCore::Core {
+struct InternalThreadState;
+}
 
 namespace FEXCore::CPU {
 
@@ -15,4 +21,4 @@ class Arm64Dispatcher final : public Dispatcher, public Arm64Emitter {
     void SpillSRA(void *ucontext) override;
 };
 
-}
+}

External/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp

@@ -1,8 +1,23 @@
-#include "Interface/Core/Dispatcher/Dispatcher.h"
 
 #include "Common/MathUtils.h"
 #include "Interface/Core/ArchHelpers/MContext.h"
+#include "Interface/Core/Dispatcher/Dispatcher.h"
+#include "Interface/Core/X86HelperGen.h"
+
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/SignalDelegator.h>
+#include <FEXCore/Core/UContext.h>
 #include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/Debug/InternalThreadState.h>
+#include <FEXCore/Utils/Event.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <atomic>
+#include <condition_variable>
+#include <bits/types/siginfo_t.h>
+#include <signal.h>
+#include <string.h>
 
 namespace FEXCore::CPU {
 
@@ -65,10 +80,38 @@ void Dispatcher::RestoreThreadState(void *ucontext) {
 
   // Now restore host state
   ArchHelpers::Context::RestoreContext(ucontext, Context);
+}
 
-  // Restore the previous signal state
-  // This allows recursive signals to properly handle signal masking as we are walking back up the list of signals
-  CTX->SignalDelegation->SetCurrentSignal(Context->Signal);
+static uint32_t ConvertSignalToTrapNo(int Signal, siginfo_t *HostSigInfo) {
+  switch (Signal) {
+    case SIGSEGV:
+      if (HostSigInfo->si_code == SEGV_MAPERR ||
+          HostSigInfo->si_code == SEGV_ACCERR) {
+        // Protection fault
+        return X86State::X86_TRAPNO_PF;
+      }
+      break;
+  }
+
+  // Unknown mapping, fall back to old behaviour and just pass signal
+  return Signal;
+}
+
+static uint32_t ConvertSignalToError(int Signal, siginfo_t *HostSigInfo) {
+  switch (Signal) {
+    case SIGSEGV:
+      if (HostSigInfo->si_code == SEGV_MAPERR ||
+          HostSigInfo->si_code == SEGV_ACCERR) {
+        // Protection fault
+        // Always a user fault for us
+        // XXX: PF_PROT and PF_WRITE
+        return X86State::X86_PF_USER;
+      }
+      break;
+  }
+
+  // Not a page fault issue
+  return 0;
 }
 
 bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack) {
@@ -79,70 +122,95 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
   // We use this to track if it is safe to clear cache
   ++SignalHandlerRefCounter;
 
+  uint64_t OldPC = ArchHelpers::Context::GetPc(ucontext);
   // Set the new PC
   ArchHelpers::Context::SetPc(ucontext, AbsoluteLoopTopAddressFillSRA);
   // Set our state register to point to our guest thread data
   ArchHelpers::Context::SetState(ucontext, reinterpret_cast<uint64_t>(Frame));
 
-
   uint64_t OldGuestSP = Frame->State.gregs[X86State::REG_RSP];
   uint64_t NewGuestSP = OldGuestSP;
 
-  if (!(GuestStack->ss_flags & SS_DISABLE)) {
-    // If our guest is already inside of the alternative stack
-    // Then that means we are hitting recursive signals and we need to walk back the stack correctly
-    uint64_t AltStackBase = reinterpret_cast<uint64_t>(GuestStack->ss_sp);
-    uint64_t AltStackEnd = AltStackBase + GuestStack->ss_size;
-    if (OldGuestSP >= AltStackBase &&
-        OldGuestSP <= AltStackEnd) {
-      // We are already in the alt stack, the rest of the code will handle adjusting this
-    }
-    else {
-      NewGuestSP = AltStackEnd;
+  // Pulling from context here
+  bool Is64BitMode = CTX->Config.Is64BitMode;
+  uint64_t SignalReturn = CTX->X86CodeGen.SignalReturn;
+
+  // Spill the SRA regardless of signal handler type
+  // We are going to be returning to the top of the dispatcher which will fill again
+  // Otherwise we might load garbage
+  if (SRAEnabled) {
+    if (IsAddressInJITCode(OldPC, false)) {
+      // We are in jit, SRA must be spilled
+      SpillSRA(ucontext);
+    } else {
+      if (!IsAddressInJITCode(OldPC, true)) {
+        // This is likely to cause issues but in some cases it isn't fatal
+        // This can also happen if we have put a signal on hold, then we just reenabled the signal
+        // So we are in the syscall handler
+        // Only throw a log message in this case
+        LogMan::Msg::E("Signals in dispatcher have unsynchronized context");
+      }
     }
   }
 
-  // Back up past the redzone, which is 128bytes
-  // Don't need this offset if we aren't going to be putting siginfo in to it
-  NewGuestSP -= 128;
+  // altstack is only used if the signal handler was setup with SA_ONSTACK
+  if (GuestAction->sa_flags & SA_ONSTACK) {
+    // Additionally the altstack is only used if the enabled (SS_DISABLE flag is not set)
+    if (!(GuestStack->ss_flags & SS_DISABLE)) {
+      // If our guest is already inside of the alternative stack
+      // Then that means we are hitting recursive signals and we need to walk back the stack correctly
+      uint64_t AltStackBase = reinterpret_cast<uint64_t>(GuestStack->ss_sp);
+      uint64_t AltStackEnd = AltStackBase + GuestStack->ss_size;
+      if (OldGuestSP >= AltStackBase &&
+          OldGuestSP <= AltStackEnd) {
+        // We are already in the alt stack, the rest of the code will handle adjusting this
+      }
+      else {
+        NewGuestSP = AltStackEnd;
+      }
+    }
+  }
+
+  if (Is64BitMode) {
+    // Back up past the redzone, which is 128bytes
+    // 32-bit doesn't have a redzone
+    NewGuestSP -= 128;
+  }
 
   // siginfo_t
   siginfo_t *HostSigInfo = reinterpret_cast<siginfo_t*>(info);
 
-  if (GuestAction->sa_flags & SA_SIGINFO &&
-      !(HostSigInfo->si_code == SI_QUEUE || // If the siginfo comes from sigqueue or user then we don't need to check
-        HostSigInfo->si_code == SI_USER)) {
-    if (SRAEnabled) {
-      if (!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), false)) {
-        LOGMAN_THROW_A(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true), "Signals in dispatcher have unsynchronized context");
-      } else {
-        // We are in jit, SRA must be spilled
-        SpillSRA(ucontext);
-      }
-    }
+  if (GuestAction->sa_flags & SA_SIGINFO) {
 
     // Setup ucontext a bit
-    if (CTX->Config.Is64BitMode) {
+    if (Is64BitMode) {
+      NewGuestSP -= sizeof(FEXCore::x86_64::_libc_fpstate);
+      NewGuestSP = AlignDown(NewGuestSP, alignof(FEXCore::x86_64::_libc_fpstate));
+      uint64_t FPStateLocation = NewGuestSP;
+
       NewGuestSP -= sizeof(FEXCore::x86_64::ucontext_t);
+      NewGuestSP = AlignDown(NewGuestSP, alignof(FEXCore::x86_64::ucontext_t));
       uint64_t UContextLocation = NewGuestSP;
 
       NewGuestSP -= sizeof(siginfo_t);
+      NewGuestSP = AlignDown(NewGuestSP, alignof(siginfo_t));
       uint64_t SigInfoLocation = NewGuestSP;
 
       FEXCore::x86_64::ucontext_t *guest_uctx = reinterpret_cast<FEXCore::x86_64::ucontext_t*>(UContextLocation);
       siginfo_t *guest_siginfo = reinterpret_cast<siginfo_t*>(SigInfoLocation);
 
       // We have extended float information
-      guest_uctx->uc_flags |= FEXCore::x86_64::UC_FP_XSTATE;
+      guest_uctx->uc_flags = FEXCore::x86_64::UC_FP_XSTATE;
 
       // Pointer to where the fpreg memory is
-      guest_uctx->uc_mcontext.fpregs = &guest_uctx->__fpregs_mem;
+      guest_uctx->uc_mcontext.fpregs = reinterpret_cast<FEXCore::x86_64::_libc_fpstate*>(FPStateLocation);
+      FEXCore::x86_64::_libc_fpstate *fpstate = reinterpret_cast<FEXCore::x86_64::_libc_fpstate*>(FPStateLocation);
 
       guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_RIP] = Frame->State.rip;
       guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_EFL] = 0;
       guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_CSGSFS] = 0;
-      guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_ERR] = 0;
-      guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_TRAPNO] = Signal;
+      guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_ERR] = ConvertSignalToError(Signal, HostSigInfo);
+      guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_TRAPNO] = ConvertSignalToTrapNo(Signal, HostSigInfo);
       guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_OLDMASK] = 0;
       guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_CR2] = 0;
 
@@ -167,15 +235,15 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
 #undef COPY_REG
 
       // Copy float registers
-      memcpy(guest_uctx->__fpregs_mem._st, Frame->State.mm, sizeof(Frame->State.mm));
-      memcpy(guest_uctx->__fpregs_mem._xmm, Frame->State.xmm, sizeof(Frame->State.xmm));
+      memcpy(fpstate->_st, Frame->State.mm, sizeof(Frame->State.mm));
+      memcpy(fpstate->_xmm, Frame->State.xmm, sizeof(Frame->State.xmm));
 
       // FCW store default
-      guest_uctx->__fpregs_mem.fcw = Frame->State.FCW;
-      guest_uctx->__fpregs_mem.ftw = Frame->State.FTW;
+      fpstate->fcw = Frame->State.FCW;
+      fpstate->ftw = Frame->State.FTW;
 
       // Reconstruct FSW
-      guest_uctx->__fpregs_mem.fsw =
+      fpstate->fsw =
         (Frame->State.flags[FEXCore::X86State::X87FLAG_TOP_LOC] << 11) |
         (Frame->State.flags[FEXCore::X86State::X87FLAG_C0_LOC] << 8) |
         (Frame->State.flags[FEXCore::X86State::X87FLAG_C1_LOC] << 9) |
@@ -196,27 +264,34 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
       Frame->State.gregs[X86State::REG_RDX] = UContextLocation;
     }
     else {
-      // XXX: 32bit Support
+      NewGuestSP -= sizeof(FEXCore::x86::_libc_fpstate);
+      NewGuestSP = AlignDown(NewGuestSP, alignof(FEXCore::x86::_libc_fpstate));
+      uint64_t FPStateLocation = NewGuestSP;
+
       NewGuestSP -= sizeof(FEXCore::x86::ucontext_t);
+      NewGuestSP = AlignDown(NewGuestSP, alignof(FEXCore::x86::ucontext_t));
       uint64_t UContextLocation = NewGuestSP;
+
       NewGuestSP -= sizeof(FEXCore::x86::siginfo_t);
+      NewGuestSP = AlignDown(NewGuestSP, alignof(FEXCore::x86::siginfo_t));
       uint64_t SigInfoLocation = NewGuestSP;
 
       FEXCore::x86::ucontext_t *guest_uctx = reinterpret_cast<FEXCore::x86::ucontext_t*>(UContextLocation);
       FEXCore::x86::siginfo_t *guest_siginfo = reinterpret_cast<FEXCore::x86::siginfo_t*>(SigInfoLocation);
 
       // We have extended float information
-      guest_uctx->uc_flags |= FEXCore::x86::UC_FP_XSTATE;
+      guest_uctx->uc_flags = FEXCore::x86::UC_FP_XSTATE;
 
       // Pointer to where the fpreg memory is
-      guest_uctx->uc_mcontext.fpregs = static_cast<uint32_t>(reinterpret_cast<uint64_t>(&guest_uctx->__fpregs_mem));
+      guest_uctx->uc_mcontext.fpregs = static_cast<uint32_t>(FPStateLocation);
+      FEXCore::x86::_libc_fpstate *fpstate = reinterpret_cast<FEXCore::x86::_libc_fpstate*>(FPStateLocation);
 
       guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_GS] = Frame->State.gs;
       guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_FS] = Frame->State.fs;
       guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_ES] = Frame->State.es;
       guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_DS] = Frame->State.ds;
-      guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_TRAPNO] = Signal;
-      guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_ERR] = 0;
+      guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_TRAPNO] = ConvertSignalToTrapNo(Signal, HostSigInfo);
+      guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_ERR] = ConvertSignalToError(Signal, HostSigInfo);
       guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_EIP] = Frame->State.rip;
       guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_CS] = Frame->State.cs;
       guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_EFL] = 0;
@@ -236,20 +311,20 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
 #undef COPY_REG
 
       // Copy float registers
-      memcpy(guest_uctx->__fpregs_mem._st, Frame->State.mm, sizeof(Frame->State.mm));
-      if (0) {
-        // XXX: Handle XMM
-        // memcpy(guest_uctx->__fpregs_mem._xmm, Frame->State.xmm, sizeof(Frame->State.xmm));
-        guest_uctx->__fpregs_mem.status = FEXCore::x86::fpstate_magic::MAGIC_XFPSTATE;
-      }
-      else {
-        guest_uctx->__fpregs_mem.status = FEXCore::x86::fpstate_magic::MAGIC_FPU;
+      for (size_t i = 0; i < 8; ++i) {
+        // 32-bit st register size is only 10 bytes. Not padded to 16byte like x86-64
+        memcpy(&fpstate->_st[i], &Frame->State.mm[i], 10);
       }
+
+      // Extended XMM state
+      fpstate->status = FEXCore::x86::fpstate_magic::MAGIC_XFPSTATE;
+      memcpy(fpstate->_xmm, Frame->State.xmm, sizeof(Frame->State.xmm));
+
       // FCW store default
-      guest_uctx->__fpregs_mem.fcw = Frame->State.FCW;
-      guest_uctx->__fpregs_mem.ftw = Frame->State.FTW;
+      fpstate->fcw = Frame->State.FCW;
+      fpstate->ftw = Frame->State.FTW;
       // Reconstruct FSW
-      guest_uctx->__fpregs_mem.fsw =
+      fpstate->fsw =
         (Frame->State.flags[FEXCore::X86State::X87FLAG_TOP_LOC] << 11) |
         (Frame->State.flags[FEXCore::X86State::X87FLAG_C0_LOC] << 8) |
         (Frame->State.flags[FEXCore::X86State::X87FLAG_C1_LOC] << 9) |
@@ -269,6 +344,12 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
       switch (Signal) {
         case SIGSEGV:
         case SIGBUS:
+          // Macro expansion to get the si_addr
+          // This is the address trying to be accessed, not the RIP
+          guest_siginfo->_sifields._sigfault.addr = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(HostSigInfo->si_addr));
+          break;
+        case SIGFPE:
+        case SIGILL:
           // Macro expansion to get the si_addr
           // Can't really give a real result here. Pull from the context for now
           guest_siginfo->_sifields._sigfault.addr = Frame->State.rip;
@@ -280,10 +361,15 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
           guest_siginfo->_sifields._sigchld.utime = HostSigInfo->si_utime;
           guest_siginfo->_sifields._sigchld.stime = HostSigInfo->si_stime;
           break;
-      default:
-        // Hope for the best, most things just copy over
-        memcpy(&guest_siginfo->_sifields, &HostSigInfo->_sifields, sizeof(siginfo_t));
-        break;
+        case SIGALRM:
+        case SIGVTALRM:
+          guest_siginfo->_sifields._timer.tid = HostSigInfo->si_timerid;
+          guest_siginfo->_sifields._timer.overrun = HostSigInfo->si_overrun;
+          guest_siginfo->_sifields._timer.sigval.sival_int = HostSigInfo->si_int;
+          break;
+        default:
+          LogMan::Msg::E("Unhandled siginfo_t for signal: %d\n", Signal);
+          break;
       }
 
       NewGuestSP -= 4;
@@ -297,7 +383,7 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
     Frame->State.rip = reinterpret_cast<uint64_t>(GuestAction->sigaction_handler.sigaction);
   }
   else {
-    if (!CTX->Config.Is64BitMode) {
+    if (!Is64BitMode) {
       NewGuestSP -= 4;
       *(uint32_t*)NewGuestSP = Signal;
     }
@@ -305,21 +391,29 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
     Frame->State.rip = reinterpret_cast<uint64_t>(GuestAction->sigaction_handler.handler);
   }
 
-  if (CTX->Config.Is64BitMode) {
-    Frame->State.gregs[X86State::REG_RDI] = Signal;
+  if (Is64BitMode) {
+    Frame->State.gregs[FEXCore::X86State::REG_RDI] = Signal;
 
     // Set up the new SP for stack handling
     NewGuestSP -= 8;
-    *(uint64_t*)NewGuestSP = CTX->X86CodeGen.SignalReturn;
-    Frame->State.gregs[X86State::REG_RSP] = NewGuestSP;
+    *(uint64_t*)NewGuestSP = SignalReturn;
+    Frame->State.gregs[FEXCore::X86State::REG_RSP] = NewGuestSP;
   }
   else {
     NewGuestSP -= 4;
-    *(uint32_t*)NewGuestSP = CTX->X86CodeGen.SignalReturn;
-    LOGMAN_THROW_A(CTX->X86CodeGen.SignalReturn < 0x1'0000'0000ULL, "This needs to be below 4GB");
-    Frame->State.gregs[X86State::REG_RSP] = NewGuestSP;
+    *(uint32_t*)NewGuestSP = SignalReturn;
+    LOGMAN_THROW_A(SignalReturn < 0x1'0000'0000ULL, "This needs to be below 4GB");
+    Frame->State.gregs[FEXCore::X86State::REG_RSP] = NewGuestSP;
   }
 
+  // The guest starts its signal frame with a zero initialized FPU
+  // Set that up now. Little bit costly but it's a requirement
+  // This state will be restored on rt_sigreturn
+  memset(Frame->State.xmm, 0, sizeof(Frame->State.xmm));
+  memset(Frame->State.mm, 0, sizeof(Frame->State.mm));
+  Frame->State.FCW = 0x37F;
+  Frame->State.FTW = 0xFFFF;
+
   return true;
 }
 
@@ -365,9 +459,6 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
       ArchHelpers::Context::SetPc(ucontext, ThreadPauseHandlerAddress);
     }
 
-    // Set the new PC
-    ArchHelpers::Context::SetPc(ucontext, ThreadPauseHandlerAddress);
-
     // Set our state register to point to our guest thread data
     ArchHelpers::Context::SetState(ucontext, reinterpret_cast<uint64_t>(Frame));
 

External/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.h

@@ -1,11 +1,24 @@
 #pragma once
 
 #include <FEXCore/Core/CPUBackend.h>
-#include <FEXCore/Core/SignalDelegator.h>
 
 #include "Interface/Context/Context.h"
 
+#include <bits/types/stack_t.h>
+#include <cstdint>
+#include <stddef.h>
 #include <stack>
+#include <tuple>
+#include <vector>
+
+namespace FEXCore {
+struct GuestSigAction;
+}
+
+namespace FEXCore::Core {
+struct CpuStateFrame;
+struct InternalThreadState;
+}
 
 namespace FEXCore::CPU {
 

External/FEXCore/Source/Interface/Core/Dispatcher/X86Dispatcher.cpp

@@ -3,10 +3,21 @@
 #include "Interface/Core/Dispatcher/X86Dispatcher.h"
 
 #include "Interface/Core/Interpreter/InterpreterClass.h"
+#include "Interface/Core/X86HelperGen.h"
 #include "Interface/Context/Context.h"
 
 #include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/CPUBackend.h>
+#include <FEXCore/Debug/InternalThreadState.h>
+#include <FEXCore/Utils/Allocator.h>
+
 #include <cmath>
+#include <memory>
+#include <stddef.h>
+#include <stdint.h>
+#include <sys/mman.h>
+#include "xbyak/xbyak.h"
 
 namespace FEXCore::CPU {
 static constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096;

External/FEXCore/Source/Interface/Core/Dispatcher/X86Dispatcher.h

@@ -2,11 +2,17 @@
 
 #include "Interface/Core/Dispatcher/Dispatcher.h"
 
-#include <FEXCore/Utils/Allocator.h>
-
 #define XBYAK64
 #include <xbyak/xbyak.h>
 
+namespace FEXCore::Context {
+  struct Context;
+}
+
+namespace FEXCore::Core {
+  struct InternalThreadState;
+}
+
 namespace FEXCore::CPU {
 
 class X86Dispatcher final : public Dispatcher, public Xbyak::CodeGenerator {

External/FEXCore/Source/Interface/Core/Frontend.cpp

@@ -7,14 +7,18 @@ $end_info$
 
 #include "Interface/Context/Context.h"
 #include "Interface/Core/Frontend.h"
-#include "Interface/Core/InternalThreadState.h"
 
 #include <array>
+#include <assert.h>
 #include <algorithm>
 #include <cstring>
+#include <FEXCore/Config/Config.h>
 #include <FEXCore/Core/X86Enums.h>
 #include <FEXCore/Debug/X86Tables.h>
+#include <FEXCore/HLE/SyscallHandler.h>
+#include <FEXCore/Utils/Allocator.h>
 #include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/Telemetry.h>
 #include <set>
 #include <sys/mman.h>
 
@@ -696,6 +700,7 @@ bool Decoder::NormalOpHeader(FEXCore::X86Tables::X86InstInfo const *Info, uint16
     return NormalOp(&X87Ops[X87Op], X87Op);
   }
   else if (Info->Type == FEXCore::X86Tables::TYPE_VEX_TABLE_PREFIX) {
+    FEXCORE_TELEMETRY_SET(VEXOpTelem, 1);
     uint16_t map_select = 1;
     uint16_t pp = 0;
 
@@ -723,6 +728,7 @@ bool Decoder::NormalOpHeader(FEXCore::X86Tables::X86InstInfo const *Info, uint16
 
     if (LocalInfo->Type >= FEXCore::X86Tables::TYPE_VEX_GROUP_12 &&
         LocalInfo->Type <= FEXCore::X86Tables::TYPE_VEX_GROUP_17) {
+    FEXCORE_TELEMETRY_SET(VEXOpTelem, 1);
     // We have ModRM
     uint8_t ModRMByte = ReadByte();
     DecodeInst->ModRM = ModRMByte;
@@ -740,6 +746,8 @@ bool Decoder::NormalOpHeader(FEXCore::X86Tables::X86InstInfo const *Info, uint16
       return NormalOp(LocalInfo, Op);
   }
   else if (Info->Type == FEXCore::X86Tables::TYPE_GROUP_EVEX) {
+    FEXCORE_TELEMETRY_SET(EVEXOpTelem, 1);
+
     /* uint8_t P1 = */ ReadByte();
     /* uint8_t P2 = */ ReadByte();
     /* uint8_t P3 = */ ReadByte();

External/FEXCore/Source/Interface/Core/Frontend.h

@@ -2,12 +2,12 @@
 
 #include <FEXCore/Debug/X86Tables.h>
 #include <FEXCore/HLE/SyscallHandler.h>
+#include <FEXCore/Utils/Telemetry.h>
 
 #include <array>
 #include <cstdint>
-#include <utility>
 #include <set>
-#include <stack>
+#include <stddef.h>
 #include <vector>
 
 namespace FEXCore::Context {
@@ -89,5 +89,8 @@ private:
   };
 
   const uint8_t *AdjustAddrForSpecialRegion(uint8_t const* _InstStream, uint64_t EntryPoint, uint64_t RIP);
+
+  FEXCORE_TELEMETRY_INIT(VEXOpTelem, TYPE_USES_VEX_OPS);
+  FEXCORE_TELEMETRY_INIT(EVEXOpTelem, TYPE_USES_EVEX_OPS);
 };
 }

External/FEXCore/Source/Interface/Core/GdbServer.cpp

@@ -8,30 +8,41 @@ $end_info$
 #include <cstdlib>
 #include <cstdio>
 #include <iomanip>
-#include <iostream>
 #include <sstream>
 #include <string>
 #include <memory>
 #include <optional>
 #include "Common/NetStream.h"
 #include "Common/SoftFloat.h"
+#include "Interface/Context/Context.h"
 
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/SignalDelegator.h>
+#include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/Debug/InternalThreadState.h>
+#include <FEXCore/HLE/Linux/ThreadManagement.h>
 #include <FEXCore/Utils/CompilerDefs.h>
 #include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/Threads.h>
 
+#include <atomic>
 #include <cstring>
+#include <errno.h>
 #include <fcntl.h>
-#include <fmt/format.h>
 #include <fstream>
+#include <fmt/format.h>
 #include <netdb.h>
+#include <signal.h>
+#include <stddef.h>
+#include <string_view>
 #include <sys/socket.h>
-#include <sys/types.h>
 #include <unistd.h>
-
+#include <utility>
+#include <vector>
 
 #include "GdbServer.h"
-#include <FEXCore/Core/CodeLoader.h>
-#include <FEXCore/Core/X86Enums.h>
 
 namespace FEXCore
 {

External/FEXCore/Source/Interface/Core/GdbServer.h

@@ -5,18 +5,21 @@ $end_info$
 */
 #pragma once
 
-#include <mutex>
-#include <thread>
-
-#include "Interface/Context/Context.h"
-#include "Common/NetStream.h"
-
+#include <FEXCore/Config/Config.h>
 #include <FEXCore/Utils/Threads.h>
 
+#include <istream>
+#include <memory>
 #include <mutex>
+#include <stdint.h>
+#include <string>
 
 namespace FEXCore {
 
+namespace Context {
+  struct Context;
+}
+
 class GdbServer {
 public:
     GdbServer(FEXCore::Context::Context *ctx);

External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp

@@ -1,30 +1,31 @@
-#include "Common/MathUtils.h"
-#include "Common/SoftFloat.h"
 #include "Interface/Context/Context.h"
 
 #include "Interface/Core/ArchHelpers/Arm64.h"
 #include "Interface/Core/ArchHelpers/MContext.h"
-#include "Interface/Core/LookupCache.h"
-#include "Interface/Core/DebugData.h"
-#include "Interface/Core/InternalThreadState.h"
+#include "Interface/Core/Dispatcher/Dispatcher.h"
 #include "Interface/Core/Interpreter/InterpreterClass.h"
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/SignalDelegator.h>
+#include <FEXCore/Debug/InternalThreadState.h>
 #include <FEXCore/Utils/LogManager.h>
 
-#include <FEXCore/Core/CPUBackend.h>
-#include <FEXCore/HLE/SyscallHandler.h>
-#include <FEXCore/IR/IR.h>
-#include <FEXCore/IR/IntrusiveIRList.h>
-
-#include "Interface/HLE/Thunks/Thunks.h"
-
-#include <atomic>
-#include <cmath>
-#include <limits>
-#include <vector>
+#include <memory>
+#include <bits/types/stack_t.h>
+#include <signal.h>
+#include <stdint.h>
+#include <unordered_map>
+#include <utility>
 
 #include "InterpreterOps.h"
 
+namespace FEXCore::IR {
+  class IRListView;
+  class RegisterAllocationData;
+}
+
 namespace FEXCore::CPU {
+class CPUBackend;
 
 static void InterpreterExecution(FEXCore::Core::CpuStateFrame *Frame) {
   auto Thread = Frame->Thread;
@@ -117,7 +118,7 @@ InterpreterCore::InterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::
       return Core->Dispatcher->HandleGuestSignal(Signal, info, ucontext, GuestAction, GuestStack);
     };
 
-    for (uint32_t Signal = 0; Signal < SignalDelegator::MAX_SIGNALS; ++Signal) {
+    for (uint32_t Signal = 0; Signal <= SignalDelegator::MAX_SIGNALS; ++Signal) {
       CTX->SignalDelegation->RegisterHostSignalHandlerForGuest(Signal, GuestSignalHandler);
     }
   }

External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp

@@ -1,17 +1,16 @@
-#include "Common/MathUtils.h"
 #include "Common/SoftFloat.h"
+#include "Common/SoftFloat-3e/softfloat.h"
 #include "Interface/Context/Context.h"
+#include "Interface/Core/CPUID.h"
 #include "InterpreterOps.h"
 
 #ifdef _M_ARM_64
 #include "Interface/Core/ArchHelpers/Arm64.h"
 #endif
-#include "Interface/Core/LookupCache.h"
-#include "Interface/Core/DebugData.h"
-#include "Interface/Core/InternalThreadState.h"
-#include "Interface/Core/Interpreter/InterpreterClass.h"
 
 #include <FEXCore/Core/CPUBackend.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Debug/InternalThreadState.h>
 #include <FEXCore/HLE/SyscallHandler.h>
 #include <FEXCore/IR/IR.h>
 #include <FEXCore/IR/IntrusiveIRList.h>
@@ -21,15 +20,18 @@
 
 #include "Interface/HLE/Thunks/Thunks.h"
 
+#include <alloca.h>
 #include <algorithm>
 #include <atomic>
+#include <bit>
 #include <cmath>
+#include <cstdint>
 #include <limits>
-#include <vector>
-#ifdef _M_X86_64
-#include <xmmintrin.h>
-#endif
-#include <unistd.h>
+#include <memory>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
 
 namespace FEXCore::CPU {
 
@@ -92,6 +94,20 @@ static uint64_t AtomicFetchNeg(uint64_t *Addr) {
 
   return Expected;
 }
+
+template<typename T>
+static T AtomicCompareAndSwap(T expected, T desired, T *addr)
+{
+  std::atomic<T> *Data = reinterpret_cast<std::atomic<T>*>(addr);
+
+  T Src1 = expected;
+  T Src2 = desired;
+
+  T Expected = Src1;
+  bool Result = Data->compare_exchange_strong(Expected, Src2);
+  
+  return Result ? Src1 : Expected;
+}
 #else
 // Needs to match what the AArch64 JIT and unaligned signal handler expects
 uint8_t AtomicFetchNeg(uint8_t *Addr) {
@@ -186,6 +202,141 @@ uint64_t AtomicFetchNeg(uint64_t *Addr) {
   return Result;
 }
 
+template<typename T>
+static T AtomicCompareAndSwap(T expected, T desired, T *addr);
+
+template<>
+uint8_t AtomicCompareAndSwap(uint8_t expected, uint8_t desired, uint8_t *addr) {
+  using Type = uint8_t;
+  //force Result to r9 (scratch register) or clang spills to stack
+  register Type Result asm("r9"){};
+  Type Tmp{};
+  Type Tmp2{};
+  __asm__ volatile(
+  R"(
+  1:
+    ldaxrb %w[Tmp], [%[Memory]];
+    cmp %w[Tmp], %w[Expected], uxtb;
+    b.ne 2f;
+    stlxrb %w[Tmp2], %w[Desired], [%[Memory]];
+    cbnz %w[Tmp2], 1b;
+    mov %w[Result], %w[Expected];
+    b 3f;
+  2:
+    mov %w[Result], %w[Tmp];
+    clrex;
+  3:
+  )"
+  : [Tmp] "=r" (Tmp)
+  , [Tmp2] "=r" (Tmp2)
+  , [Desired] "+r" (desired)
+  , [Expected] "+r" (expected)
+  , [Result] "=r" (Result)
+  , [Memory] "+r" (addr)
+  :: "memory"
+  );
+  return Result;
+}
+
+template<>
+uint16_t AtomicCompareAndSwap(uint16_t expected, uint16_t desired, uint16_t *addr) {
+  using Type = uint16_t;
+  //force Result to r9 (scratch register) or clang spills to stack
+  register Type Result asm("r9"){};
+  Type Tmp{};
+  Type Tmp2{};
+  __asm__ volatile(
+  R"(
+  1:
+    ldaxrh %w[Tmp], [%[Memory]];
+    cmp %w[Tmp], %w[Expected], uxth;
+    b.ne 2f;
+    stlxrh %w[Tmp2], %w[Desired], [%[Memory]];
+    cbnz %w[Tmp2], 1b;
+    mov %w[Result], %w[Expected];
+    b 3f;
+  2:
+    mov %w[Result], %w[Tmp];
+    clrex;
+  3:
+  )"
+  : [Tmp] "=r" (Tmp)
+  , [Tmp2] "=r" (Tmp2)
+  , [Desired] "+r" (desired)
+  , [Expected] "+r" (expected)
+  , [Result] "=r" (Result)
+  , [Memory] "+r" (addr)
+  :: "memory"
+  );
+  return Result;
+}
+
+template<>
+uint32_t AtomicCompareAndSwap(uint32_t expected, uint32_t desired, uint32_t *addr) {
+  using Type = uint32_t;
+  //force Result to r9 (scratch register) or clang spills to stack
+  register Type Result asm("r9"){};
+  Type Tmp{};
+  Type Tmp2{};
+  __asm__ volatile(
+  R"(
+  1:
+    ldaxr %w[Tmp], [%[Memory]];
+    cmp %w[Tmp], %w[Expected];
+    b.ne 2f;
+    stlxr %w[Tmp2], %w[Desired], [%[Memory]];
+    cbnz %w[Tmp2], 1b;
+    mov %w[Result], %w[Expected];
+    b 3f;
+  2:
+    mov %w[Result], %w[Tmp];
+    clrex;
+  3:
+  )"
+  : [Tmp] "=r" (Tmp)
+  , [Tmp2] "=r" (Tmp2)
+  , [Desired] "+r" (desired)
+  , [Expected] "+r" (expected)
+  , [Result] "=r" (Result)
+  , [Memory] "+r" (addr)
+  :: "memory"
+  );
+  return Result;
+}
+
+template<>
+uint64_t AtomicCompareAndSwap(uint64_t expected, uint64_t desired, uint64_t *addr) {
+  using Type = uint64_t;
+  //force Result to r9 (scratch register) or clang spills to stack
+  register Type Result asm("r9"){};
+  Type Tmp{};
+  Type Tmp2{};
+  __asm__ volatile(
+  R"(
+  1:
+    ldaxr %[Tmp], [%[Memory]];
+    cmp %[Tmp], %[Expected];
+    b.ne 2f;
+    stlxr %w[Tmp2], %[Desired], [%[Memory]];
+    cbnz %w[Tmp2], 1b;
+    mov %[Result], %[Expected];
+    b 3f;
+  2:
+    mov %[Result], %[Tmp];
+    clrex;
+  3:
+  )"
+  : [Tmp] "=r" (Tmp)
+  , [Tmp2] "=r" (Tmp2)
+  , [Desired] "+r" (desired)
+  , [Expected] "+r" (expected)
+  , [Result] "=r" (Result)
+  , [Memory] "+r" (addr)
+  :: "memory"
+  );
+  return Result;
+}
+
 #endif
 
 namespace AES {
@@ -1515,14 +1666,11 @@ void InterpreterOps::InterpretIR(FEXCore::Core::InternalThreadState *Thread, uin
             // Size is the size of each pair element
             switch (Size) {
               case 4: {
-                std::atomic<uint64_t> *Data = *GetSrc<std::atomic<uint64_t> **>(SSAData, Op->Header.Args[2]);
-
-                uint64_t Src1 = *GetSrc<uint64_t*>(SSAData, Op->Header.Args[0]);
-                uint64_t Src2 = *GetSrc<uint64_t*>(SSAData, Op->Header.Args[1]);
-
-                uint64_t Expected = Src1;
-                bool Result = Data->compare_exchange_strong(Expected, Src2);
-                GD = Result ? Src1 : Expected;
+                GD = AtomicCompareAndSwap(
+                  *GetSrc<uint64_t*>(SSAData, Op->Header.Args[0]), 
+                  *GetSrc<uint64_t*>(SSAData, Op->Header.Args[1]),
+                  *GetSrc<uint64_t**>(SSAData, Op->Header.Args[2])
+                );
                 break;
               }
               case 8: {
@@ -2198,46 +2346,35 @@ void InterpreterOps::InterpretIR(FEXCore::Core::InternalThreadState *Thread, uin
             auto Size = OpSize;
             switch (Size) {
               case 1: {
-                std::atomic<uint8_t> *Data = *GetSrc<std::atomic<uint8_t> **>(SSAData, Op->Header.Args[2]);
-
-                uint8_t Src1 = *GetSrc<uint8_t*>(SSAData, Op->Header.Args[0]);
-                uint8_t Src2 = *GetSrc<uint8_t*>(SSAData, Op->Header.Args[1]);
-
-                uint8_t Expected = Src1;
-                bool Result = Data->compare_exchange_strong(Expected, Src2);
-                GD = Result ? Src1 : Expected;
+                GD = AtomicCompareAndSwap(
+                  *GetSrc<uint8_t*>(SSAData, Op->Header.Args[0]), 
+                  *GetSrc<uint8_t*>(SSAData, Op->Header.Args[1]),
+                  *GetSrc<uint8_t**>(SSAData, Op->Header.Args[2])
+                );
                 break;
               }
               case 2: {
-                std::atomic<uint16_t> *Data = *GetSrc<std::atomic<uint16_t> **>(SSAData, Op->Header.Args[2]);
-                uint16_t Src1 = *GetSrc<uint16_t*>(SSAData, Op->Header.Args[0]);
-                uint16_t Src2 = *GetSrc<uint16_t*>(SSAData, Op->Header.Args[1]);
-
-                uint16_t Expected = Src1;
-                bool Result = Data->compare_exchange_strong(Expected, Src2);
-                GD = Result ? Src1 : Expected;
+                GD = AtomicCompareAndSwap(
+                  *GetSrc<uint16_t*>(SSAData, Op->Header.Args[0]), 
+                  *GetSrc<uint16_t*>(SSAData, Op->Header.Args[1]),
+                  *GetSrc<uint16_t**>(SSAData, Op->Header.Args[2])
+                );
                 break;
               }
               case 4: {
-                std::atomic<uint32_t> *Data = *GetSrc<std::atomic<uint32_t> **>(SSAData, Op->Header.Args[2]);
-
-                uint32_t Src1 = *GetSrc<uint32_t*>(SSAData, Op->Header.Args[0]);
-                uint32_t Src2 = *GetSrc<uint32_t*>(SSAData, Op->Header.Args[1]);
-
-                uint32_t Expected = Src1;
-                bool Result = Data->compare_exchange_strong(Expected, Src2);
-                GD = Result ? Src1 : Expected;
+                GD = AtomicCompareAndSwap(
+                  *GetSrc<uint32_t*>(SSAData, Op->Header.Args[0]), 
+                  *GetSrc<uint32_t*>(SSAData, Op->Header.Args[1]),
+                  *GetSrc<uint32_t**>(SSAData, Op->Header.Args[2])
+                );
                 break;
               }
               case 8: {
-                std::atomic<uint64_t> *Data = *GetSrc<std::atomic<uint64_t> **>(SSAData, Op->Header.Args[2]);
-
-                uint64_t Src1 = *GetSrc<uint64_t*>(SSAData, Op->Header.Args[0]);
-                uint64_t Src2 = *GetSrc<uint64_t*>(SSAData, Op->Header.Args[1]);
-
-                uint64_t Expected = Src1;
-                bool Result = Data->compare_exchange_strong(Expected, Src2);
-                GD = Result ? Src1 : Expected;
+                GD = AtomicCompareAndSwap(
+                  *GetSrc<uint64_t*>(SSAData, Op->Header.Args[0]), 
+                  *GetSrc<uint64_t*>(SSAData, Op->Header.Args[1]),
+                  *GetSrc<uint64_t**>(SSAData, Op->Header.Args[2])
+                );
                 break;
               }
               default: LOGMAN_MSG_A_FMT("Unknown CAS size: {}", Size); break;

External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h

@@ -1,9 +1,13 @@
+#pragma once
+#include <stdint.h>
+
 namespace FEXCore::Core {
   struct InternalThreadState;
 }
 
 namespace FEXCore::IR {
     class IRListView;
+    struct IROp_Header;
 }
 
 namespace FEXCore::Core{
@@ -32,11 +36,11 @@ namespace FEXCore::CPU {
     FallbackABI ABI;
     void *fn;
   };
-  
+
   class InterpreterOps {
 
     public:
       static void InterpretIR(FEXCore::Core::InternalThreadState *Thread, uint64_t Entry, FEXCore::IR::IRListView *CurrentIR, FEXCore::Core::DebugData *DebugData);
       static bool GetFallbackHandler(IR::IROp_Header *IROp, FallbackInfo *Info);
   };
-};
+};

External/FEXCore/Source/Interface/Core/JIT/Arm64/AtomicOps.cpp

@@ -44,15 +44,12 @@ DEF_OP(CASPair) {
         aarch64::Label LoopNotExpected;
         aarch64::Label LoopExpected;
         bind(&LoopTop);
-        nop();
+
         ldaxp(TMP2.W(), TMP3.W(), MemOperand(MemSrc));
-        nop();
         cmp(TMP2.W(), Expected.first.W());
         ccmp(TMP3.W(), Expected.second.W(), NoFlag, Condition::eq);
         b(&LoopNotExpected, Condition::ne);
-        nop();
         stlxp(TMP2.W(), Desired.first.W(), Desired.second.W(), MemOperand(MemSrc));
-        nop();
         cbnz(TMP2.W(), &LoopTop);
         mov(Dst.first.W(), Expected.first.W());
         mov(Dst.second.W(), Expected.second.W());
@@ -73,15 +70,12 @@ DEF_OP(CASPair) {
         aarch64::Label LoopNotExpected;
         aarch64::Label LoopExpected;
         bind(&LoopTop);
-        nop();
+
         ldaxp(TMP2.X(), TMP3.X(), MemOperand(MemSrc));
-        nop();
         cmp(TMP2.X(), Expected.first.X());
         ccmp(TMP3.X(), Expected.second.X(), NoFlag, Condition::eq);
         b(&LoopNotExpected, Condition::ne);
-        nop();
         stlxp(TMP2.X(), Desired.first.X(), Desired.second.X(), MemOperand(MemSrc));
-        nop();
         cbnz(TMP2.X(), &LoopTop);
         mov(Dst.first.X(), Expected.first.X());
         mov(Dst.second.X(), Expected.second.X());

External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp

@@ -392,32 +392,22 @@ bool Arm64JITCore::HandleSIGBUS(int Signal, void *info, void *ucontext) {
     }
   }
   else if ((Instr & FEXCore::ArchHelpers::Arm64::LDAXP_MASK) == FEXCore::ArchHelpers::Arm64::LDAXP_INST) { // LDAXP
-    uint32_t DataReg2 = (Instr >> 10) & 0x1F;
-    // Convert to LDP
-    uint32_t LDP = 0b0010'1001'0100'0000'0000'0000'0000'0000;
-    LDP |= Size << 31;
-    LDP |= DataReg2 << 10;
-    LDP |= AddrReg << 5;
-    LDP |= DataReg;
-    PC[-1] = DMB;
-    PC[0] = LDP;
-    PC[1] = DMB;
-    // Back up one instruction and have another go
-    ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) - 4);
+    //Should be compare and swap pair only. LDAXP not used elsewhere
+    uint64_t BytesToSkip = FEXCore::ArchHelpers::Arm64::HandleCASPAL_ARMv8(ucontext, info, Instr);
+    if (BytesToSkip) {
+      // Skip this instruction now
+      ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) + BytesToSkip);
+      return true;
+    }
+    else {
+      LogMan::Msg::EFmt("Unhandled JIT SIGBUS LDAXP: PC: {} Instruction: 0x{:08x}\n", fmt::ptr(PC), PC[0]);
+      return false;
+    }
   }
   else if ((Instr & FEXCore::ArchHelpers::Arm64::STLXP_MASK) == FEXCore::ArchHelpers::Arm64::STLXP_INST) { // STLXP
-    uint32_t DataReg2 = (Instr >> 10) & 0x1F;
-    // Convert to STP
-    uint32_t STP = 0b0010'1001'0000'0000'0000'0000'0000'0000;
-    STP |= Size << 31;
-    STP |= DataReg2 << 10;
-    STP |= AddrReg << 5;
-    STP |= DataReg;
-    PC[-1] = DMB;
-    PC[0] = STP;
-    PC[1] = DMB;
-    // Back up one instruction and have another go
-    ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) - 4);
+    //Should not trigger - middle of an LDAXP/STAXP pair.
+    LogMan::Msg::EFmt("Unhandled JIT SIGBUS STLXP: PC: {} Instruction: 0x{:08x}\n", fmt::ptr(PC), PC[0]);
+    return false;
   }
   else if ((Instr & FEXCore::ArchHelpers::Arm64::CASPAL_MASK) == FEXCore::ArchHelpers::Arm64::CASPAL_INST) { // CASPAL
     if (FEXCore::ArchHelpers::Arm64::HandleCASPAL(ucontext, info, Instr)) {
@@ -482,7 +472,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern
     DispatcherConfig config;
     config.ExitFunctionLink = reinterpret_cast<uintptr_t>(&ExitFunctionLink);
     config.ExitFunctionLinkThis = reinterpret_cast<uintptr_t>(this);
-    config.StaticRegisterAssignment = true;
+    config.StaticRegisterAssignment = ctx->Config.StaticRegisterAllocation;
 
     Dispatcher = std::make_unique<Arm64Dispatcher>(CTX, ThreadState, config);
     DispatchPtr = Dispatcher->DispatchPtr;
@@ -561,7 +551,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern
       return Core->Dispatcher->HandleGuestSignal(Signal, info, ucontext, GuestAction, GuestStack);
     };
 
-    for (uint32_t Signal = 0; Signal < SignalDelegator::MAX_SIGNALS; ++Signal) {
+    for (uint32_t Signal = 0; Signal <= SignalDelegator::MAX_SIGNALS; ++Signal) {
       CTX->SignalDelegation->RegisterHostSignalHandlerForGuest(Signal, GuestSignalHandler);
     }
   }
@@ -781,8 +771,7 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
   // X1-X3 = Temp
   // X4-r18 = RA
 
-  auto Buffer = GetBuffer();
-  auto GuestEntry = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
+  auto GuestEntry = GetCursorAddress<uint64_t>();
 
  if (CTX->GetGdbServerStatus()) {
     aarch64::Label RunBlock;
@@ -849,7 +838,7 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
     }
 
     if (DebugData) {
-      DebugData->Subblocks.push_back({Buffer->GetOffsetAddress<uintptr_t>(GetCursorOffset()), 0, IR->GetID(BlockNode)});
+      DebugData->Subblocks.push_back({GetCursorAddress<uintptr_t>(), 0, IR->GetID(BlockNode)});
     }
 
     for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {
@@ -861,7 +850,7 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
     }
 
     if (DebugData) {
-      DebugData->Subblocks.back().HostCodeSize = Buffer->GetOffsetAddress<uintptr_t>(GetCursorOffset()) - DebugData->Subblocks.back().HostCodeStart;
+      DebugData->Subblocks.back().HostCodeSize = GetCursorAddress<uintptr_t>() - DebugData->Subblocks.back().HostCodeStart;
     }
   }
 
@@ -874,7 +863,7 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
 
   FinalizeCode();
 
-  auto CodeEnd = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
+  auto CodeEnd = GetCursorAddress<uint64_t>();
   CPU.EnsureIAndDCacheCoherency(reinterpret_cast<void*>(GuestEntry), CodeEnd - reinterpret_cast<uint64_t>(GuestEntry));
 
   if (DebugData) {

External/FEXCore/Source/Interface/Core/JIT/x86_64/ALUOps.cpp

@@ -5,7 +5,14 @@ $end_info$
 */
 
 #include "Interface/Core/JIT/x86_64/JITClass.h"
-#include "Interface/IR/Passes/RegisterAllocationPass.h"
+
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <array>
+#include <stdint.h>
+#include <utility>
+#include <xbyak/xbyak.h>
 
 namespace FEXCore::CPU {
 #define DEF_OP(x) void X86JITCore::Op_##x(FEXCore::IR::IROp_Header *IROp, uint32_t Node)

External/FEXCore/Source/Interface/Core/JIT/x86_64/AtomicOps.cpp

@@ -5,7 +5,14 @@ $end_info$
 */
 
 #include "Interface/Core/JIT/x86_64/JITClass.h"
-#include "Interface/IR/Passes/RegisterAllocationPass.h"
+
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <array>
+#include <stdint.h>
+#include <utility>
+#include <xbyak/xbyak.h>
 
 namespace FEXCore::CPU {
 #define DEF_OP(x) void X86JITCore::Op_##x(FEXCore::IR::IROp_Header *IROp, uint32_t Node)

External/FEXCore/Source/Interface/Core/JIT/x86_64/BranchOps.cpp

@@ -4,14 +4,28 @@ tags: backend|x86-64
 $end_info$
 */
 
+#include "Interface/Context/Context.h"
+#include "Interface/Core/CPUID.h"
+#include "Interface/Core/Dispatcher/Dispatcher.h"
 #include "Interface/Core/LookupCache.h"
-
 #include "Interface/Core/JIT/x86_64/JITClass.h"
-#include "Interface/IR/Passes/RegisterAllocationPass.h"
+#include "Interface/HLE/Thunks/Thunks.h"
 
+#include <FEXCore/Core/CPUID.h>
+#include <FEXCore/Core/CoreState.h>
 #include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/Debug/InternalThreadState.h>
 #include <FEXCore/HLE/SyscallHandler.h>
-#include <Interface/HLE/Thunks/Thunks.h>
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <array>
+#include <memory>
+#include <stddef.h>
+#include <stdint.h>
+#include <unordered_map>
+#include <utility>
+#include <xbyak/xbyak.h>
 
 namespace FEXCore::CPU {
 #define DEF_OP(x) void X86JITCore::Op_##x(FEXCore::IR::IROp_Header *IROp, uint32_t Node)

External/FEXCore/Source/Interface/Core/JIT/x86_64/ConversionOps.cpp

@@ -5,7 +5,13 @@ $end_info$
 */
 
 #include "Interface/Core/JIT/x86_64/JITClass.h"
-#include "Interface/IR/Passes/RegisterAllocationPass.h"
+
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <array>
+#include <stdint.h>
+#include <xbyak/xbyak.h>
 
 namespace FEXCore::CPU {
 

External/FEXCore/Source/Interface/Core/JIT/x86_64/EncryptionOps.cpp

@@ -5,7 +5,12 @@ $end_info$
 */
 
 #include "Interface/Core/JIT/x86_64/JITClass.h"
-#include "Interface/IR/Passes/RegisterAllocationPass.h"
+
+#include <FEXCore/IR/IR.h>
+
+#include <array>
+#include <stdint.h>
+#include <xbyak/xbyak.h>
 
 namespace FEXCore::CPU {
 #define DEF_OP(x) void X86JITCore::Op_##x(FEXCore::IR::IROp_Header *IROp, uint32_t Node)

External/FEXCore/Source/Interface/Core/JIT/x86_64/FlagOps.cpp

@@ -5,7 +5,12 @@ $end_info$
 */
 
 #include "Interface/Core/JIT/x86_64/JITClass.h"
-#include "Interface/IR/Passes/RegisterAllocationPass.h"
+
+#include <FEXCore/IR/IR.h>
+
+#include <array>
+#include <stdint.h>
+#include <xbyak/xbyak.h>
 
 namespace FEXCore::CPU {
 

External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp

@@ -8,20 +8,36 @@ $end_info$
 #include "Interface/Context/Context.h"
 #include "Interface/Core/LookupCache.h"
 
+#include "Interface/Core/Dispatcher/Dispatcher.h"
 #include "Interface/Core/Dispatcher/X86Dispatcher.h"
+#include "Interface/Core/Interpreter/InterpreterOps.h"
 #include "Interface/Core/JIT/x86_64/JITClass.h"
-#include "Interface/Core/InternalThreadState.h"
-
+#include "Interface/IR/PassManager.h"
 #include "Interface/IR/Passes/RegisterAllocationPass.h"
 
-#include <FEXCore/Core/X86Enums.h>
-#include <FEXCore/Core/UContext.h>
+#include <FEXCore/Core/CPUBackend.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/SignalDelegator.h>
+#include <FEXCore/Debug/InternalThreadState.h>
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+#include <FEXCore/IR/RegisterAllocationData.h>
 #include <FEXCore/Utils/Allocator.h>
+#include <FEXCore/Utils/LogManager.h>
 
-#include <cmath>
+#include <algorithm>
+#include <array>
+#include <bits/types/stack_t.h>
+#include <memory>
+#include <stddef.h>
+#include <stdint.h>
 #include <signal.h>
-
-#include "Interface/Core/Interpreter/InterpreterOps.h"
+#include <sys/mman.h>
+#include <tuple>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+#include <xbyak/xbyak.h>
 
 // #define DEBUG_RA 1
 // #define DEBUG_CYCLES
@@ -360,7 +376,7 @@ X86JITCore::X86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalTh
       return Core->Dispatcher->HandleGuestSignal(Signal, info, ucontext, GuestAction, GuestStack);
     };
 
-    for (uint32_t Signal = 0; Signal < SignalDelegator::MAX_SIGNALS; ++Signal) {
+    for (uint32_t Signal = 0; Signal <= SignalDelegator::MAX_SIGNALS; ++Signal) {
       CTX->SignalDelegation->RegisterHostSignalHandlerForGuest(Signal, GuestSignalHandler);
     }
   }

External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp

@@ -5,9 +5,15 @@ $end_info$
 */
 
 #include "Interface/Core/JIT/x86_64/JITClass.h"
-#include "Interface/IR/Passes/RegisterAllocationPass.h"
 
-#include <cmath>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <array>
+#include <stddef.h>
+#include <stdint.h>
+#include <xbyak/xbyak.h>
 
 namespace FEXCore::CPU {
 

External/FEXCore/Source/Interface/Core/JIT/x86_64/MiscOps.cpp

@@ -4,8 +4,18 @@ tags: backend|x86-64
 $end_info$
 */
 
+#include "Interface/Context/Context.h"
+#include "Interface/Core/Dispatcher/Dispatcher.h"
 #include "Interface/Core/JIT/x86_64/JITClass.h"
-#include "Interface/IR/Passes/RegisterAllocationPass.h"
+
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/IR/IR.h>
+
+#include <array>
+#include <stddef.h>
+#include <stdint.h>
+#include <xbyak/xbyak.h>
 
 namespace FEXCore::CPU {
 static void PrintValue(uint64_t Value) {

External/FEXCore/Source/Interface/Core/JIT/x86_64/MoveOps.cpp

@@ -5,7 +5,13 @@ $end_info$
 */
 
 #include "Interface/Core/JIT/x86_64/JITClass.h"
-#include "Interface/IR/Passes/RegisterAllocationPass.h"
+
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/IR/IR.h>
+
+#include <array>
+#include <stdint.h>
+#include <utility>
 
 namespace FEXCore::CPU {
 

External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp

@@ -5,8 +5,14 @@ $end_info$
 */
 
 #include "Interface/Core/JIT/x86_64/JITClass.h"
-#include "Interface/IR/Passes/RegisterAllocationPass.h"
 
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <array>
+#include <stddef.h>
+#include <stdint.h>
+#include <xbyak/xbyak.h>
 
 namespace FEXCore::CPU {
 

External/FEXCore/Source/Interface/Core/LookupCache.cpp

@@ -5,10 +5,11 @@ desc: Stores information about blocks, and provides C++ implementations to looku
 $end_info$
 */
 
-#include "Interface/Context/Context.h"
-#include "Interface/Core/Core.h"
-#include "Interface/Core/LookupCache.h"
 #include <FEXCore/Utils/Allocator.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include "Interface/Context/Context.h"
+#include "Interface/Core/LookupCache.h"
 
 #include <sys/mman.h>
 

External/FEXCore/Source/Interface/Core/LookupCache.h

@@ -1,10 +1,18 @@
 #pragma once
-#include "Interface/Context/Context.h"
 #include <FEXCore/Utils/LogManager.h>
 
+#include <cstdint>
+#include <functional>
 #include <map>
+#include <stddef.h>
+#include <utility>
+#include <vector>
 
 namespace FEXCore {
+namespace Context {
+  struct Context;
+}
+
 class LookupCache {
 public:
 

External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp

@@ -7,16 +7,22 @@ $end_info$
 
 #include "Interface/Context/Context.h"
 #include "Interface/Core/OpcodeDispatcher.h"
-#include "Interface/HLE/Thunks/Thunks.h"
 
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/Context.h>
 #include <FEXCore/Core/CoreState.h>
-#include <bit>
-#include <climits>
-#include <cstddef>
-#include <cstdint>
-
 #include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/Debug/X86Tables.h>
 #include <FEXCore/HLE/SyscallHandler.h>
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <tuple>
 
 namespace FEXCore::IR {
 
@@ -75,13 +81,25 @@ void OpDispatchBuilder::SyscallOp(OpcodeArgs) {
   };
   static_assert(GPRIndexes_64.size() == GPRIndexes_32.size());
 
+  static std::array<uint64_t, SyscallArgs> GPRIndexes_Hangover = {
+    FEXCore::X86State::REG_RCX,
+  };
+
+  size_t NumArguments{};
+
   const auto OSABI = CTX->SyscallHandler->GetOSABI();
   if (OSABI == FEXCore::HLE::SyscallOSABI::OS_LINUX64) {
+    NumArguments = GPRIndexes_64.size();
     GPRIndexes = &GPRIndexes_64;
   }
   else if (OSABI == FEXCore::HLE::SyscallOSABI::OS_LINUX32) {
+    NumArguments = GPRIndexes_64.size();
     GPRIndexes = &GPRIndexes_32;
   }
+  else if (OSABI == FEXCore::HLE::SyscallOSABI::OS_HANGOVER) {
+    NumArguments = 1;
+    GPRIndexes = &GPRIndexes_Hangover;
+  }
   else {
     LogMan::Msg::D("Unhandled OSABI syscall");
   }
@@ -91,16 +109,34 @@ void OpDispatchBuilder::SyscallOp(OpcodeArgs) {
   _StoreContext(GPRClass, GPRSize, offsetof(FEXCore::Core::CPUState, rip), NewRIP);
 
   const auto& GPRIndicesRef = *GPRIndexes;
-  auto SyscallOp = _Syscall(
-    _LoadContext(GPRSize, offsetof(FEXCore::Core::CPUState, gregs) + GPRIndicesRef[0] * 8, GPRClass),
-    _LoadContext(GPRSize, offsetof(FEXCore::Core::CPUState, gregs) + GPRIndicesRef[1] * 8, GPRClass),
-    _LoadContext(GPRSize, offsetof(FEXCore::Core::CPUState, gregs) + GPRIndicesRef[2] * 8, GPRClass),
-    _LoadContext(GPRSize, offsetof(FEXCore::Core::CPUState, gregs) + GPRIndicesRef[3] * 8, GPRClass),
-    _LoadContext(GPRSize, offsetof(FEXCore::Core::CPUState, gregs) + GPRIndicesRef[4] * 8, GPRClass),
-    _LoadContext(GPRSize, offsetof(FEXCore::Core::CPUState, gregs) + GPRIndicesRef[5] * 8, GPRClass),
-    _LoadContext(GPRSize, offsetof(FEXCore::Core::CPUState, gregs) + GPRIndicesRef[6] * 8, GPRClass));
 
-  _StoreContext(GPRClass, GPRSize, offsetof(FEXCore::Core::CPUState, gregs[FEXCore::X86State::REG_RAX]), SyscallOp);
+  OrderedNode *Arguments[SyscallArgs] {
+    InvalidNode,
+    InvalidNode,
+    InvalidNode,
+    InvalidNode,
+    InvalidNode,
+    InvalidNode,
+    InvalidNode,
+  };
+  for (size_t i = 0; i < NumArguments; ++i) {
+    Arguments[i] = _LoadContext(GPRSize, offsetof(FEXCore::Core::CPUState, gregs) + GPRIndicesRef[i] * 8, GPRClass);
+  }
+
+  auto SyscallOp = _Syscall(
+    Arguments[0],
+    Arguments[1],
+    Arguments[2],
+    Arguments[3],
+    Arguments[4],
+    Arguments[5],
+    Arguments[6]);
+
+  if (OSABI != FEXCore::HLE::SyscallOSABI::OS_HANGOVER) {
+    // Hangover doesn't want us returning a result here
+    // syscall is being abused as a thunk for now.
+    _StoreContext(GPRClass, GPRSize, offsetof(FEXCore::Core::CPUState, gregs[FEXCore::X86State::REG_RAX]), SyscallOp);
+  }
 }
 
 void OpDispatchBuilder::ThunkOp(OpcodeArgs) {
@@ -208,17 +244,21 @@ void OpDispatchBuilder::IRETOp(OpcodeArgs) {
   //eflags (lower 16 used)
   auto eflags = _LoadMem(GPRClass, GPRSize, SP, GPRSize);
   SetPackedRFLAG(false, eflags);
+  SP = _Add(SP, Constant);
 
   if (CTX->Config.Is64BitMode) {
     // RSP and SS only happen in 64-bit mode or if this is a CPL mode jump!
-    SP = _Add(SP, Constant);
-    // RSP
+    // FEX doesn't support a CPL mode switch, so don't need to worry about this on 32-bit
     _StoreContext(GPRClass, GPRSize, offsetof(FEXCore::Core::CPUState, gregs[FEXCore::X86State::REG_RSP]), _LoadMem(GPRClass, GPRSize, SP, GPRSize));
     SP = _Add(SP, Constant);
     //ss
     _StoreContext(GPRClass, 2, offsetof(FEXCore::Core::CPUState, ss), _LoadMem(GPRClass, GPRSize, SP, GPRSize));
     SP = _Add(SP, Constant);
   }
+  else {
+    // Store the stack in 32-bit mode
+    _StoreContext(GPRClass, GPRSize, offsetof(FEXCore::Core::CPUState, gregs[FEXCore::X86State::REG_RSP]), SP);
+  }
 
   _ExitFunction(NewRIP);
   BlockSetRIP = true;
@@ -1545,7 +1585,14 @@ void OpDispatchBuilder::MOVSegOp(OpcodeArgs) {
         DecodeFailure = true;
         return;
     }
-    StoreResult(GPRClass, Op, Segment, -1);
+    if (DestIsMem(Op)) {
+      // If the destination is memory then we always store 16-bits only
+      StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Segment, 2, -1);
+    }
+    else {
+      // If the destination is a GPR then we follow register storing rules
+      StoreResult(GPRClass, Op, Segment, -1);
+    }
   }
 }
 
@@ -3113,11 +3160,6 @@ void OpDispatchBuilder::DECOp(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::STOSOp(OpcodeArgs) {
-  if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX) {
-    LogMan::Msg::E("Invalid REPNE on STOS");
-    DecodeFailure = true;
-    return;
-  }
   if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) {
     LogMan::Msg::E("Can't handle adddress size");
     DecodeFailure = true;
@@ -3126,7 +3168,7 @@ void OpDispatchBuilder::STOSOp(OpcodeArgs) {
 
   const auto GPRSize = CTX->GetGPRSize();
   const auto Size = GetSrcSize(Op);
-  const bool Repeat = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX) != 0;
+  const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) != 0;
 
   if (!Repeat) {
     OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1);
@@ -3221,11 +3263,6 @@ void OpDispatchBuilder::STOSOp(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::MOVSOp(OpcodeArgs) {
-  if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX) {
-    LogMan::Msg::E("Invalid REPNE on MOVS");
-    DecodeFailure = true;
-    return;
-  }
   if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) {
     LogMan::Msg::E("Can't handle adddress size");
     DecodeFailure = true;
@@ -3242,7 +3279,7 @@ void OpDispatchBuilder::MOVSOp(OpcodeArgs) {
   auto DF = GetRFLAG(FEXCore::X86State::RFLAG_DF_LOC);
   auto PtrDir = _Select(FEXCore::IR::COND_EQ, DF,  _Constant(0), SizeConst, NegSizeConst);
 
-  if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX) {
+  if (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) {
     // Create all our blocks
     auto LoopHead = CreateNewCodeBlockAfter(GetCurrentBlock());
     auto LoopTail = CreateNewCodeBlockAfter(LoopHead);
@@ -3437,11 +3474,6 @@ void OpDispatchBuilder::CMPSOp(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::LODSOp(OpcodeArgs) {
-  if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX) {
-    LogMan::Msg::E("Invalid REPNE on LODS");
-    DecodeFailure = true;
-    return;
-  }
   if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) {
     LogMan::Msg::E("Can't handle adddress size");
     DecodeFailure = true;
@@ -3450,7 +3482,7 @@ void OpDispatchBuilder::LODSOp(OpcodeArgs) {
 
   const auto GPRSize = CTX->GetGPRSize();
   const auto Size = GetSrcSize(Op);
-  const bool Repeat = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX) != 0;
+  const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) != 0;
 
   if (!Repeat) {
     OrderedNode *Dest_RSI = _LoadContext(GPRSize, offsetof(FEXCore::Core::CPUState, gregs[FEXCore::X86State::REG_RSI]), GPRClass);

External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h

@@ -3,7 +3,8 @@
 #include "Interface/Core/Frontend.h"
 #include "Interface/Context/Context.h"
 
-#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/Context.h>
 #include <FEXCore/Debug/X86Tables.h>
 #include <FEXCore/IR/IntrusiveIRList.h>
 #include <FEXCore/IR/IR.h>
@@ -12,9 +13,10 @@
 #include <FEXCore/Utils/LogManager.h>
 
 #include <cstdint>
-#include <functional>
 #include <map>
-#include <set>
+#include <stddef.h>
+#include <utility>
+#include <vector>
 
 namespace FEXCore::IR {
 class Pass;

External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp

@@ -5,11 +5,16 @@ desc: Handles x86/64 Crypto instructions to IR
 $end_info$
 */
 
+#include <FEXCore/Debug/X86Tables.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/Utils/LogManager.h>
 #include "Interface/Core/OpcodeDispatcher.h"
 
-#include <FEXCore/Core/X86Enums.h>
+#include <stdint.h>
 
 namespace FEXCore::IR {
+class OrderedNode;
+
 #define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op
 
 void OpDispatchBuilder::AESImcOp(OpcodeArgs) {

External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp

@@ -5,9 +5,17 @@ desc: Handles x86/64 flag generation
 $end_info$
 */
 
+#include "Interface/Context/Context.h"
 #include "Interface/Core/OpcodeDispatcher.h"
 
 #include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Debug/X86Tables.h>
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/IR/IR.h>
+
+#include <array>
+#include <cstdint>
 
 namespace FEXCore::IR {
 constexpr std::array<uint32_t, 17> FlagOffsets = {

External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp

@@ -5,9 +5,20 @@ desc: Handles x86/64 Vector instructions to IR
 $end_info$
 */
 
+#include "Interface/Context/Context.h"
 #include "Interface/Core/OpcodeDispatcher.h"
 
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/CoreState.h>
 #include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/Debug/X86Tables.h>
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <array>
+#include <bit>
+#include <cstdint>
+#include <stddef.h>
 
 namespace FEXCore::IR {
 #define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op
@@ -1278,6 +1289,13 @@ void OpDispatchBuilder::FXSaveOp(OpcodeArgs) {
     _StoreMem(GPRClass, 2, MemLocation, FSW, 2);
   }
 
+  {
+    // FTW
+    OrderedNode *MemLocation = _Add(Mem, _Constant(4));
+    auto FTW = _LoadContext(2, offsetof(FEXCore::Core::CPUState, FTW), GPRClass);
+    _StoreMem(GPRClass, 2, MemLocation, FTW, 2);
+  }
+
   // BYTE | 0 1 | 2 3 | 4   | 5     | 6 7 | 8 9 | a b | c d | e f |
   // ------------------------------------------
   //   32 | ST0/MM0                             | <R>
@@ -1296,14 +1314,14 @@ void OpDispatchBuilder::FXSaveOp(OpcodeArgs) {
   //  240 | XMM5
   //  256 | XMM6
   //  272 | XMM7
-  //  288 | XMM8
-  //  304 | XMM9
-  //  320 | XMM10
-  //  336 | XMM11
-  //  352 | XMM12
-  //  368 | XMM13
-  //  384 | XMM14
-  //  400 | XMM15
+  //  288 | 64BitMode ? <R> : XMM8
+  //  304 | 64BitMode ? <R> : XMM9
+  //  320 | 64BitMode ? <R> : XMM10
+  //  336 | 64BitMode ? <R> : XMM11
+  //  352 | 64BitMode ? <R> : XMM12
+  //  368 | 64BitMode ? <R> : XMM13
+  //  384 | 64BitMode ? <R> : XMM14
+  //  400 | 64BitMode ? <R> : XMM15
   //  416 | <R>
   //  432 | <R>
   //  448 | <R>
@@ -1328,7 +1346,9 @@ void OpDispatchBuilder::FXSaveOp(OpcodeArgs) {
 
     _StoreMem(FPRClass, 16, MemLocation, MMReg, 16);
   }
-  for (unsigned i = 0; i < 16; ++i) {
+  unsigned NumRegs = CTX->Config.Is64BitMode ? 16 : 8;
+
+  for (unsigned i = 0; i < NumRegs; ++i) {
     OrderedNode *XMMReg = _LoadContext(16, offsetof(FEXCore::Core::CPUState, xmm[i]), FPRClass);
     OrderedNode *MemLocation = _Add(Mem, _Constant(i * 16 + 160));
 
@@ -1363,12 +1383,21 @@ void OpDispatchBuilder::FXRStoreOp(OpcodeArgs) {
     SetRFLAG<FEXCore::X86State::X87FLAG_C3_LOC>(C3);
   }
 
+  {
+    // FTW
+    OrderedNode *MemLocation = _Add(Mem, _Constant(4));
+    auto NewFTW = _LoadMem(GPRClass, 2, MemLocation, 2);
+    _StoreContext(GPRClass, 2, offsetof(FEXCore::Core::CPUState, FTW), NewFTW);
+  }
+
   for (unsigned i = 0; i < 8; ++i) {
     OrderedNode *MemLocation = _Add(Mem, _Constant(i * 16 + 32));
     auto MMReg = _LoadMem(FPRClass, 16, MemLocation, 16);
     _StoreContext(FPRClass, 16, offsetof(FEXCore::Core::CPUState, mm[i]), MMReg);
   }
-  for (unsigned i = 0; i < 16; ++i) {
+  unsigned NumRegs = CTX->Config.Is64BitMode ? 16 : 8;
+
+  for (unsigned i = 0; i < NumRegs; ++i) {
     OrderedNode *MemLocation = _Add(Mem, _Constant(i * 16 + 160));
     auto XMMReg = _LoadMem(FPRClass, 16, MemLocation, 16);
     _StoreContext(FPRClass, 16, offsetof(FEXCore::Core::CPUState, xmm[i]), XMMReg);

External/FEXCore/Source/Interface/Core/OpcodeDispatcher/X87.cpp

@@ -7,9 +7,18 @@ $end_info$
 
 #include "Interface/Core/OpcodeDispatcher.h"
 
+#include <FEXCore/Core/CoreState.h>
 #include <FEXCore/Core/X86Enums.h>
+#include <FEXCore/Debug/X86Tables.h>
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/IR/IREmitter.h>
+
+#include <stddef.h>
+#include <stdint.h>
 
 namespace FEXCore::IR {
+class OrderedNode;
+
 #define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op
 
 OrderedNode *OpDispatchBuilder::GetX87Top() {

External/FEXCore/Source/Interface/Core/SignalDelegator.cpp

@@ -0,0 +1,113 @@
+#include <FEXCore/Core/SignalDelegator.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <unistd.h>
+#include <signal.h>
+
+namespace FEXCore {
+  struct ThreadState {
+    FEXCore::Core::InternalThreadState *Thread{};
+  };
+
+  thread_local ThreadState ThreadData{};
+
+  static bool IsSynchronous(int Signal) {
+    switch (Signal) {
+    case SIGBUS:
+    case SIGFPE:
+    case SIGILL:
+    case SIGSEGV:
+    case SIGTRAP:
+      return true;
+    default: break;
+    };
+    return false;
+  }
+
+  /**
+   * @brief Masks signals from the signal mask
+   *
+   * @param how Argument to sigmask. SIG_{BLOCK, SETMASK, UNBLOCK}
+   * @param Signal Which signal to set or -1 to sweep through them all
+   */
+  static void MaskSignals(int how, int Signal = -1) {
+    // If we have a helper thread, we need to mask a significant amount of signals so the an errant thread doesn't receive a signal that it shouldn't
+    sigset_t SignalSet{};
+    sigemptyset(&SignalSet);
+
+    if (Signal == -1) {
+      for (int i = 0; i <= SignalDelegator::MAX_SIGNALS; ++i) {
+        // If it is a synchronous signal then don't ignore it
+        if (IsSynchronous(i)) {
+          continue;
+        }
+
+        // Add this signal to the ignore list
+        sigaddset(&SignalSet, i);
+      }
+    }
+    else {
+      sigaddset(&SignalSet, Signal);
+    }
+
+    // Be warned, a thread will inherit the signal mask if created from this thread
+    int Result = pthread_sigmask(how, &SignalSet, nullptr);
+    if (Result != 0) {
+      LogMan::Msg::E("Couldn't register thread to mask signals");
+    }
+  }
+
+  void SignalDelegator::MaskThreadSignals() {
+    MaskSignals(SIG_BLOCK);
+  }
+
+  FEXCore::Core::InternalThreadState *SignalDelegator::GetTLSThread() {
+    return ThreadData.Thread;
+  }
+
+  void SignalDelegator::RegisterTLSState(FEXCore::Core::InternalThreadState *Thread) {
+    ThreadData.Thread = Thread;
+    RegisterFrontendTLSState(Thread);
+  }
+
+  void SignalDelegator::UninstallTLSState(FEXCore::Core::InternalThreadState *Thread) {
+    UninstallFrontendTLSState(Thread);
+    ThreadData.Thread = nullptr;
+  }
+
+  void SignalDelegator::RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required) {
+    SetHostSignalHandler(Signal, Func, Required);
+    FrontendRegisterHostSignalHandler(Signal, Func, Required);
+  }
+
+  void SignalDelegator::RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required) {
+    SetFrontendHostSignalHandler(Signal, Func, Required);
+    FrontendRegisterFrontendHostSignalHandler(Signal, Func, Required);
+  }
+
+  void SignalDelegator::HandleSignal(int Signal, void *Info, void *UContext) {
+    // Let the host take first stab at handling the signal
+    auto Thread = GetTLSThread();
+    HostSignalHandler &Handler = HostHandlers[Signal];
+
+    if (!Thread) {
+      LogMan::Msg::E("[%d] Thread has received a signal and hasn't registered itself with the delegate! Programming error!", ::gettid());
+    }
+    else {
+      if (Handler.Handler &&
+          Handler.Handler(Thread, Signal, Info, UContext)) {
+        // If the host handler handled the fault then we can continue now
+        return;
+      }
+
+      if (Handler.FrontendHandler &&
+          Handler.FrontendHandler(Thread, Signal, Info, UContext)) {
+        return;
+      }
+
+      // Now let the frontend handle the signal
+      // It's clearly a guest signal and this ends up being an OS specific issue
+      HandleGuestSignal(Thread, Signal, Info, UContext);
+    }
+  }
+}

External/FEXCore/Source/Interface/Core/X86HelperGen.cpp

@@ -6,12 +6,15 @@ $end_info$
 */
 
 #include "Interface/Core/X86HelperGen.h"
+
+#include <FEXCore/Config/Config.h>
 #include <FEXCore/Utils/Allocator.h>
 
+#include <cstdint>
 #include <cstring>
-#include <stdlib.h>
 #include <vector>
 #include <sys/mman.h>
+#include <bits/mman-map-flags-generic.h>
 
 namespace FEXCore {
 constexpr size_t CODE_SIZE = 0x1000;

External/FEXCore/Source/Interface/Core/X86HelperGen.h

@@ -5,8 +5,8 @@ $end_info$
 */
 
 #pragma once
-#include <FEXCore/Config/Config.h>
 
+#include <stddef.h>
 #include <stdint.h>
 
 namespace FEXCore {

External/FEXCore/Source/Interface/Core/X86Tables.cpp

@@ -5,14 +5,8 @@ tags: frontend|x86-tables
 $end_info$
 */
 
-#include <FEXCore/Utils/LogManager.h>
-
 #include <FEXCore/Core/Context.h>
 #include <FEXCore/Debug/X86Tables.h>
-#include <array>
-#include <cstdint>
-#include <tuple>
-#include <vector>
 
 namespace FEXCore::X86Tables {
 

External/FEXCore/Source/Interface/Core/X86Tables/BaseTables.cpp

@@ -7,6 +7,9 @@ $end_info$
 #include "Interface/Core/X86Tables/X86Tables.h"
 
 #include <FEXCore/Core/Context.h>
+#include <FEXCore/Debug/X86Tables.h>
+
+#include <iterator>
 
 namespace FEXCore::X86Tables {
 using namespace InstFlags;

External/FEXCore/Source/Interface/Core/X86Tables/DDDTables.cpp

@@ -6,6 +6,10 @@ $end_info$
 
 #include "Interface/Core/X86Tables/X86Tables.h"
 
+#include <FEXCore/Debug/X86Tables.h>
+
+#include <iterator>
+
 namespace FEXCore::X86Tables {
 using namespace InstFlags;
 

External/FEXCore/Source/Interface/Core/X86Tables/EVEXTables.cpp

@@ -6,6 +6,10 @@ $end_info$
 
 #include "Interface/Core/X86Tables/X86Tables.h"
 
+#include <FEXCore/Debug/X86Tables.h>
+
+#include <iterator>
+
 namespace FEXCore::X86Tables {
 using namespace InstFlags;
 

External/FEXCore/Source/Interface/Core/X86Tables/H0F38Tables.cpp

@@ -4,9 +4,11 @@ tags: frontend|x86-tables
 $end_info$
 */
 
-#include <FEXCore/Core/Context.h>
-
 #include "Interface/Core/X86Tables/X86Tables.h"
+#include <FEXCore/Debug/X86Tables.h>
+
+#include <iterator>
+#include <stdint.h>
 
 namespace FEXCore::X86Tables {
 using namespace InstFlags;

External/FEXCore/Source/Interface/Core/X86Tables/H0F3ATables.cpp

@@ -6,6 +6,12 @@ $end_info$
 
 #include "Interface/Core/X86Tables/X86Tables.h"
 
+#include <FEXCore/Debug/X86Tables.h>
+#include <FEXCore/Core/Context.h>
+
+#include <iterator>
+#include <stdint.h>
+
 namespace FEXCore::X86Tables {
 using namespace InstFlags;
 

External/FEXCore/Source/Interface/Core/X86Tables/PrimaryGroupTables.cpp

@@ -6,6 +6,11 @@ $end_info$
 
 #include "Interface/Core/X86Tables/X86Tables.h"
 
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Debug/X86Tables.h>
+
+#include <iterator>
+
 namespace FEXCore::X86Tables {
 using namespace InstFlags;
 

External/FEXCore/Source/Interface/Core/X86Tables/SecondaryGroupTables.cpp

@@ -6,6 +6,11 @@ $end_info$
 
 #include "Interface/Core/X86Tables/X86Tables.h"
 
+#include <FEXCore/Debug/X86Tables.h>
+
+#include <iterator>
+#include <stdint.h>
+
 namespace FEXCore::X86Tables {
 using namespace InstFlags;
 

External/FEXCore/Source/Interface/Core/X86Tables/SecondaryModRMTables.cpp

@@ -6,6 +6,10 @@ $end_info$
 
 #include "Interface/Core/X86Tables/X86Tables.h"
 
+#include <FEXCore/Debug/X86Tables.h>
+
+#include <iterator>
+
 namespace FEXCore::X86Tables {
 using namespace InstFlags;
 

External/FEXCore/Source/Interface/Core/X86Tables/SecondaryTables.cpp

@@ -4,7 +4,12 @@ tags: frontend|x86-tables
 $end_info$
 */
 
- #include "Interface/Core/X86Tables/X86Tables.h"
+#include "Interface/Core/X86Tables/X86Tables.h"
+
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Debug/X86Tables.h>
+
+#include <iterator>
 
 namespace FEXCore::X86Tables {
 using namespace InstFlags;

External/FEXCore/Source/Interface/Core/X86Tables/VEXTables.cpp

@@ -6,6 +6,9 @@ $end_info$
 
 #include "Interface/Core/X86Tables/X86Tables.h"
 
+#include <FEXCore/Debug/X86Tables.h>
+#include <iterator>
+
 namespace FEXCore::X86Tables {
 using namespace InstFlags;
 

External/FEXCore/Source/Interface/Core/X86Tables/X87Tables.cpp

@@ -6,6 +6,9 @@ $end_info$
 
 #include "Interface/Core/X86Tables/X86Tables.h"
 
+#include <FEXCore/Debug/X86Tables.h>
+#include <iterator>
+
 namespace FEXCore::X86Tables {
 using namespace InstFlags;
 

External/FEXCore/Source/Interface/Core/X86Tables/XOPTables.cpp

@@ -6,6 +6,11 @@ $end_info$
 
 #include "Interface/Core/X86Tables/X86Tables.h"
 
+#include <FEXCore/Debug/X86Tables.h>
+
+#include <iterator>
+#include <stdint.h>
+
 namespace FEXCore::X86Tables {
 using namespace InstFlags;
 

External/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp

@@ -5,21 +5,23 @@ tags: glue|thunks
 $end_info$
 */
 
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Debug/InternalThreadState.h>
 #include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/IR/IR.h>
 #include "Thunks.h"
 
-#include "stdio.h"
 #include <dlfcn.h>
 
-
-#include <string>
-#include <map>
-#include <array>
 #include <Interface/Context/Context.h>
-#include "Interface/Core/InternalThreadState.h"
 #include "FEXCore/Core/X86Enums.h"
-#include <mutex>
+#include <map>
+#include <memory>
 #include <shared_mutex>
+#include <stdint.h>
+#include <string>
+#include <utility>
 
 struct LoadlibArgs {
     const char *Name;
@@ -51,7 +53,7 @@ namespace FEXCore {
             Thread->CurrentFrame->State.gregs[FEXCore::X86State::REG_RDI] = (uintptr_t)arg0;
             Thread->CurrentFrame->State.gregs[FEXCore::X86State::REG_RSI] = (uintptr_t)arg1;
 
-            Thread->CTX->HandleCallback((uintptr_t)callback);
+            Thread->CTX->HandleCallback(Thread, (uintptr_t)callback);
         }
 
         static void LoadLib(void *ArgsV) {

External/FEXCore/Source/Interface/HLE/Thunks/Thunks.h

@@ -5,12 +5,15 @@ $end_info$
 */
 
 #pragma once
-#include <FEXCore/IR/IR.h>
 
 namespace FEXCore::Core {
   struct InternalThreadState;
 }
 
+namespace FEXCore::IR {
+  struct SHA256Sum;
+}
+
 namespace FEXCore {
     typedef void ThunkedFunction(void* ArgsRv);
 
@@ -22,4 +25,4 @@ namespace FEXCore {
 
         static ThunkHandler* Create();
     };
-};
+};

External/FEXCore/Source/Interface/IR/IRDumper.cpp

@@ -7,9 +7,14 @@ $end_info$
 
 #include <FEXCore/IR/IR.h>
 #include <FEXCore/IR/IntrusiveIRList.h>
-#include <FEXCore/Utils/LogManager.h>
-#include "Interface/IR/Passes/RegisterAllocationPass.h"
+#include <FEXCore/IR/RegisterAllocationData.h>
 
+#include <algorithm>
+#include <array>
+#include <ostream>
+#include <stdint.h>
+#include <string>
+#include <string_view>
 #include  <iomanip>
 
 namespace FEXCore::IR {

External/FEXCore/Source/Interface/IR/IREmitter.cpp

@@ -5,7 +5,15 @@ tags: ir|emitter
 $end_info$
 */
 
+#include <FEXCore/IR/IR.h>
 #include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <array>
+#include <stdint.h>
+#include <string.h>
+#include <vector>
 
 namespace FEXCore::IR {
 void IREmitter::ResetWorkingList() {

External/FEXCore/Source/Interface/IR/IRParser.cpp

@@ -5,16 +5,24 @@ tags: ir|parser
 $end_info$
 */
 
-#include <string>
-#include <vector>
-#include <istream>
-#include <unordered_map>
-
 #include <FEXCore/IR/IR.h>
 #include <FEXCore/IR/IntrusiveIRList.h>
 #include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/Utils/LogManager.h>
 
-
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <errno.h>
+#include <memory>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+#include <istream>
+#include <unordered_map>
 
 namespace FEXCore::IR {
 namespace {
@@ -70,8 +78,6 @@ std::string DecodeErrorToString(DecodeFailure Failure) {
   return "Unknown Error";
 }
 
-std::unordered_map<std::string_view, FEXCore::IR::IROps> NameToOpMap;
-
 class IRParser: public FEXCore::IR::IREmitter {
   public:
   template<typename Type>
@@ -299,9 +305,10 @@ class IRParser: public FEXCore::IR::IREmitter {
   std::unordered_map<std::string, OrderedNode*> SSANameMapper;
   std::vector<LineDefinition> Defs;
   LineDefinition *CurrentDef{};
+  std::unordered_map<std::string_view, FEXCore::IR::IROps> NameToOpMap;
 
   IRParser(std::istream *text) {
-    InitializeStaticTables();
+    InitializeNameMap();
 
     std::string TmpLine;
     while (!text->eof()) {
@@ -630,7 +637,7 @@ class IRParser: public FEXCore::IR::IREmitter {
 		return true;
 	}
 
-  void InitializeStaticTables() {
+  void InitializeNameMap() {
     if (NameToOpMap.empty()) {
       for (FEXCore::IR::IROps Op = FEXCore::IR::IROps::OP_DUMMY;
          Op <= FEXCore::IR::IROps::OP_LAST;

External/FEXCore/Source/Interface/IR/PassManager.cpp

@@ -13,6 +13,7 @@ $end_info$
 #include <FEXCore/Config/Config.h>
 
 namespace FEXCore::IR {
+class IREmitter;
 
 void PassManager::AddDefaultPasses(bool InlineConstants, bool StaticRegisterAllocation) {
   FEX_CONFIG_OPT(DisablePasses, O0);

External/FEXCore/Source/Interface/IR/PassManager.h

@@ -7,11 +7,10 @@ $end_info$
 #pragma once
 
 #include <FEXCore/Config/Config.h>
-#include <FEXCore/IR/IntrusiveIRList.h>
-#include <FEXCore/IR/IREmitter.h>
 
 #include <functional>
 #include <memory>
+#include <utility>
 #include <vector>
 
 namespace FEXCore::HLE {
@@ -19,8 +18,9 @@ class SyscallHandler;
 }
 
 namespace FEXCore::IR {
-class OpDispatchBuilder;
-class SyscallOptimization;
+class PassManager;
+class IREmitter;
+class RegisterAllocationPass;
 
 using ShouldExitHandler = std::function<void(void)>;
 

External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp

@@ -15,7 +15,20 @@ $end_info$
 #endif
 
 #include "Interface/IR/PassManager.h"
-#include "Interface/Core/OpcodeDispatcher.h"
+
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <bit>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string.h>
+#include <tuple>
+#include <unordered_map>
+#include <utility>
 
 namespace FEXCore::IR {
 

External/FEXCore/Source/Interface/IR/Passes/DeadCodeElimination.cpp

@@ -5,11 +5,12 @@ $end_info$
 */
 
 #include "Interface/IR/PassManager.h"
-#include "Interface/Core/OpcodeDispatcher.h"
 
 #include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
 
-#include <array>
+#include <memory>
 
 namespace FEXCore::IR {
 

External/FEXCore/Source/Interface/IR/Passes/DeadContextStoreElimination.cpp

@@ -7,9 +7,21 @@ $end_info$
 
 #include "Interface/IR/Passes.h"
 #include "Interface/IR/PassManager.h"
-#include "Interface/Core/OpcodeDispatcher.h"
 #include <FEXCore/Core/CoreState.h>
 
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <array>
+#include <memory>
+#include <stddef.h>
+#include <stdint.h>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
 namespace {
   struct ContextMemberClassification {
     size_t Offset;

External/FEXCore/Source/Interface/IR/Passes/DeadStoreElimination.cpp

@@ -6,7 +6,17 @@ $end_info$
 */
 
 #include "Interface/IR/PassManager.h"
-#include "Interface/Core/OpcodeDispatcher.h"
+
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <memory>
+#include <stddef.h>
+#include <stdint.h>
+#include <unordered_map>
 
 namespace FEXCore::IR {
 

External/FEXCore/Source/Interface/IR/Passes/IRCompaction.cpp

@@ -9,7 +9,16 @@ $end_info$
 #include "Interface/IR/PassManager.h"
 #include "Interface/Core/OpcodeDispatcher.h"
 
-#include <map>
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <algorithm>
+#include <memory>
+#include <stdint.h>
+#include <string.h>
+#include <vector>
 
 namespace FEXCore::IR {
 

External/FEXCore/Source/Interface/IR/Passes/IRValidation.cpp

@@ -7,11 +7,22 @@ $end_info$
 
 #include "Interface/IR/PassManager.h"
 #include "Interface/IR/Passes/RegisterAllocationPass.h"
-#include "Interface/Context/Context.h"
-#include "Interface/Core/OpcodeDispatcher.h"
 #include "Common/BitSet.h"
 
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+#include <FEXCore/IR/RegisterAllocationData.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <cstdint>
+#include <memory>
+#include <stddef.h>
+#include <string>
 #include <sstream>
+#include <unordered_map>
+#include <utility>
+#include <vector>
 
 namespace {
   struct BlockInfo {

External/FEXCore/Source/Interface/IR/Passes/LongDivideRemovalPass.cpp

@@ -6,7 +6,12 @@ $end_info$
 */
 
 #include "Interface/IR/PassManager.h"
-#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+
+#include <memory>
+#include <stdint.h>
 
 namespace FEXCore::IR {
 

External/FEXCore/Source/Interface/IR/Passes/PhiValidation.cpp

@@ -5,10 +5,16 @@ desc: Sanity checking pass
 $end_info$
 */
 
-#include "Interface/IR/PassManager.h"
-#include "Interface/Core/OpcodeDispatcher.h"
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+#include <FEXCore/Utils/LogManager.h>
 
+#include "Interface/IR/PassManager.h"
+
+#include <memory>
 #include <sstream>
+#include <string>
 
 namespace FEXCore::IR::Validation {
 

External/FEXCore/Source/Interface/IR/Passes/RedundantFlagCalculationElimination.cpp

@@ -5,8 +5,13 @@ desc: This is not used right now, possibly broken
 $end_info$
 */
 
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
 #include "Interface/IR/PassManager.h"
-#include "Interface/Core/OpcodeDispatcher.h"
+
+#include <array>
+#include <memory>
 
 namespace FEXCore::IR {
 

External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp

@@ -4,15 +4,28 @@ tags: ir|opts
 $end_info$
 */
 
-#include "Common/BitSet.h"
+#include <FEXCore/Utils/BucketList.h>
+#include "Common/MathUtils.h"
 #include "Interface/IR/Passes/RegisterAllocationPass.h"
 #include "Interface/IR/Passes.h"
-#include "Interface/Core/OpcodeDispatcher.h"
-#include <FEXCore/Utils/Allocator.h>
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/SignalDelegator.h>
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+#include <FEXCore/IR/RegisterAllocationData.h>
+#include <FEXCore/Utils/LogManager.h>
 
-#include <iterator>
+#include <algorithm>
+#include <cstdint>
+#include <set>
+#include <stddef.h>
+#include <string.h>
+#include <strings.h>
+#include <unordered_map>
 #include <unordered_set>
-#include <sys/mman.h>
+#include <utility>
+#include <vector>
 
 #define SRA_DEBUG(...) // printf(__VA_ARGS__)
 
@@ -26,137 +39,6 @@ namespace {
   constexpr uint32_t DEFAULT_INTERFERENCE_SPAN_COUNT = 30;
   constexpr uint32_t DEFAULT_NODE_COUNT = 8192;
 
-  const PhysicalRegister INVALID_REGCLASS = PhysicalRegister::Invalid();
-
-  // BucketList is an optimized container, it includes an inline array of Size
-  // and can overflow to a linked list of further buckets
-  //
-  // To optimize for best performance, Size should be big enough to allocate one or two
-  // buckets for the typical case
-  // Picking a Size so sizeof(Bucket<...>) is a power of two is also a small win
-  template<unsigned _Size, typename T = uint32_t>
-  struct BucketList {
-    static constexpr const unsigned Size = _Size;
-
-    T Items[Size];
-    std::unique_ptr<BucketList<Size>> Next;
-
-    void Clear() {
-      Items[0] = 0;
-      #ifndef NDEBUG
-      for (int i = 1; i < Size; i++)
-        Items[i] = 0xDEADBEEF;
-      #endif
-      Next.reset();
-    }
-
-    BucketList() {
-      Clear();
-    }
-
-    template<typename EnumeratorFn>
-    inline void Iterate(EnumeratorFn Enumerator) const {
-      int i = 0;
-      auto Bucket = this;
-
-      for(;;) {
-        auto Item = Bucket->Items[i];
-        if (Item == 0)
-          break;
-
-        Enumerator(Item);
-
-        if (++i == Bucket->Size) {
-          LOGMAN_THROW_A(Bucket->Next != nullptr, "Interference bug");
-          Bucket = Bucket->Next.get();
-          i = 0;
-        }
-      }
-    }
-
-    template<typename EnumeratorFn>
-    inline bool Find(EnumeratorFn Enumerator) const {
-      int i = 0;
-      auto Bucket = this;
-
-      for(;;) {
-        auto Item = Bucket->Items[i];
-        if (Item == 0)
-          break;
-
-        if (Enumerator(Item))
-          return true;
-
-        if (++i == Bucket->Size) {
-          LOGMAN_THROW_A(Bucket->Next != nullptr, "Bucket in bad state");
-          Bucket = Bucket->Next.get();
-          i = 0;
-        }
-      }
-
-      return false;
-    }
-
-    void Append(uint32_t Val) {
-      auto that = this;
-
-      while (that->Next) {
-        that = that->Next.get();
-      }
-
-      int i;
-      for (i = 0; i < Size; i++) {
-        if (that->Items[i] == 0) {
-          that->Items[i] = Val;
-          break;
-        }
-      }
-
-      if (i < (Size-1)) {
-        that->Items[i+1] = 0;
-      } else {
-        that->Next = std::make_unique<BucketList<Size, T>>();
-      }
-    }
-    void Erase(uint32_t Val) {
-      int i = 0;
-      auto that = this;
-      auto foundThat = this;
-      auto foundI = 0;
-
-      for (;;) {
-        if (that->Items[i] == Val) {
-          foundThat = that;
-          foundI = i;
-          break;
-        }
-        else if (++i == Size) {
-          i = 0;
-          LOGMAN_THROW_A(that->Next != nullptr, "Bucket::Erase but element not contained");
-          that = that->Next.get();
-        }
-      }
-
-      for (;;) {
-        if (that->Items[i] == 0) {
-          foundThat->Items[foundI] = that->Items[i-1];
-          that->Items[i-1] = 0;
-          break;
-        }
-        else if (++i == Size) {
-          if (that->Next->Items[0] == 0) {
-            that->Next.reset();
-            foundThat->Items[foundI] = that->Items[Size-1];
-            that->Items[Size-1] = 0;
-            break;
-          }
-          i = 0;
-          that = that->Next.get();
-        }
-      }
-    }
-  };
-
   struct Register {
     bool Virtual;
     uint64_t Index;
@@ -174,7 +56,7 @@ namespace {
       RegisterNode *PhiPartner;
     } Head { ~0U, ~0U, nullptr };
 
-    BucketList<DEFAULT_INTERFERENCE_LIST_COUNT, uint32_t> Interferences;
+    FEXCore::BucketList<DEFAULT_INTERFERENCE_LIST_COUNT, uint32_t> Interferences;
   };
 
   static_assert(sizeof(RegisterNode) == 128 * 4);
@@ -191,7 +73,7 @@ namespace {
     uint32_t End{~0U};
     uint32_t RematCost{0};
     uint32_t PreWritten{0};
-    PhysicalRegister PrefferedRegister{INVALID_REGCLASS};
+    PhysicalRegister PrefferedRegister{PhysicalRegister::Invalid()};
     bool Written{false};
     bool Global{false};
   };
@@ -269,7 +151,7 @@ namespace {
 
     Graph->VisitedNodePredecessors.clear();
     Graph->AllocData.reset((FEXCore::IR::RegisterAllocationData*)FEXCore::Allocator::malloc(FEXCore::IR::RegisterAllocationData::Size(NodeCount)));
-    memset(&Graph->AllocData->Map[0], INVALID_REGCLASS.Raw, NodeCount);
+    memset(&Graph->AllocData->Map[0], PhysicalRegister::Invalid().Raw, NodeCount);
     Graph->AllocData->MapCount = NodeCount;
     Graph->AllocData->IsShared = false; // not shared by default
     Graph->NodeCount = NodeCount;
@@ -639,7 +521,7 @@ namespace FEXCore::IR {
           return PhysicalRegister(FPRFixedClass, reg);
         } else {
           LOGMAN_THROW_A(false, "Unexpected Offset %d", Offset);
-          return INVALID_REGCLASS;
+          return PhysicalRegister::Invalid();
         }
     };
 
@@ -722,7 +604,7 @@ namespace FEXCore::IR {
           // ACCESSED after write, let's not SRA this one
           if (LiveRanges[ArgNode].Written) {
             SRA_DEBUG("Demoting ssa%d because accessed after write in ssa%d\n", ArgNode, Node);
-            LiveRanges[ArgNode].PrefferedRegister = INVALID_REGCLASS;
+            LiveRanges[ArgNode].PrefferedRegister = PhysicalRegister::Invalid();
             auto ArgNodeNode = IR->GetNode(IROp->Args[i]);
             SetNodeClass(Graph, ArgNode, GetRegClassFromNode(IR, ArgNodeNode->Op(IR->GetData())));
           }
@@ -756,7 +638,7 @@ namespace FEXCore::IR {
               uint32_t ID = (*StaticMap) - &LiveRanges[0];
 
               SRA_DEBUG("ssa%d cannot be a pre-write because ssa%d reads from sra%d before storereg", ID, Node, -1 /*vreg*/);
-              (*StaticMap)->PrefferedRegister = INVALID_REGCLASS;
+              (*StaticMap)->PrefferedRegister = PhysicalRegister::Invalid();
               (*StaticMap)->PreWritten = 0;
               SetNodeClass(Graph, ID, Op->Class);
             }
@@ -912,6 +794,9 @@ namespace FEXCore::IR {
         return (uint32_t)PhyReg.Class;
     };
 
+    // SpanStart/SpanEnd assume SSA id will fit in 24bits
+    LOGMAN_THROW_A(NodeCount <= 0xff'ffff, "Block too large for Spans");
+
     SpanStart.resize(NodeCount);
     SpanEnd.resize(NodeCount);
     for (uint32_t i = 0; i < NodeCount; ++i) {
@@ -953,13 +838,13 @@ namespace FEXCore::IR {
     for (uint32_t i = 0; i < Graph->NodeCount; ++i) {
       RegisterNode *CurrentNode = &Graph->Nodes[i];
       auto &CurrentRegAndClass = Graph->AllocData->Map[i];
-      if (CurrentRegAndClass == INVALID_REGCLASS)
+      if (CurrentRegAndClass == PhysicalRegister::Invalid())
         continue;
 
       auto LiveRange = &LiveRanges[i];
 
       FEXCore::IR::RegisterClassType RegClass = FEXCore::IR::RegisterClassType{CurrentRegAndClass.Class};
-      auto RegAndClass = INVALID_REGCLASS;
+      auto RegAndClass = PhysicalRegister::Invalid();
       RegisterClass *RAClass = &Graph->Set.Classes[RegClass];
 
       if (CurrentNode->Head.PhiPartner) {

External/FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.h

@@ -6,11 +6,14 @@ $end_info$
 
 #pragma once
 #include "Interface/IR/PassManager.h"
-#include <FEXCore/IR/RegisterAllocationData.h>
-#include <vector>
+
+#include <memory>
+#include <stdint.h>
 
 namespace FEXCore::IR {
-class IRListView;
+class RegisterAllocationData;
+struct RegisterAllocationDataDeleter;
+struct RegisterClassType;
 
 class RegisterAllocationPass : public FEXCore::IR::Pass {
   public:

External/FEXCore/Source/Interface/IR/Passes/StaticRegisterAllocationPass.cpp

@@ -6,7 +6,15 @@ $end_info$
 */
 
 #include "Interface/IR/PassManager.h"
-#include "Interface/Core/OpcodeDispatcher.h"
+#include <FEXCore/Core/CoreState.h>
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <memory>
+#include <stddef.h>
+#include <stdint.h>
 
 namespace FEXCore::IR {
 

External/FEXCore/Source/Interface/IR/Passes/SyscallOptimization.cpp

@@ -5,12 +5,15 @@ desc: Removes unused arguments if known syscall number
 $end_info$
 */
 
-#include "Interface/Context/Context.h"
 #include "Interface/IR/PassManager.h"
-#include "Interface/Core/OpcodeDispatcher.h"
 
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
 #include <FEXCore/HLE/SyscallHandler.h>
-#include <FEXCore/Utils/LogManager.h>
+
+#include <memory>
+#include <stdint.h>
 
 namespace FEXCore::IR {
 

External/FEXCore/Source/Interface/IR/Passes/ValueDominanceValidation.cpp

@@ -6,12 +6,21 @@ $end_info$
 */
 
 #include "Interface/IR/PassManager.h"
-#include "Interface/Core/OpcodeDispatcher.h"
 
-#include <map>
-#include <list>
+#include <FEXCore/IR/IR.h>
+#include <FEXCore/IR/IREmitter.h>
+#include <FEXCore/IR/IntrusiveIRList.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#include <functional>
+#include <memory>
+#include <set>
+#include <stdint.h>
+#include <string>
 #include <sstream>
 #include <unordered_map>
+#include <utility>
+#include <vector>
 
 namespace {
   struct BlockInfo {

External/FEXCore/Source/Utils/Allocator.cpp

@@ -4,8 +4,10 @@
 #ifdef ENABLE_JEMALLOC
 #include <jemalloc/jemalloc.h>
 #endif
+#include <errno.h>
 #include <memory>
-#include <malloc.h>
+#include <stddef.h>
+#include <stdint.h>
 
 extern "C" {
   typedef void* (*mmap_hook_type)(
@@ -75,6 +77,11 @@ namespace FEXCore::Allocator {
 #endif
     FEXCore::Allocator::mmap = ::mmap;
     FEXCore::Allocator::munmap = ::munmap;
+
+    // XXX: This is currently a leak.
+    // We can't work around this yet until static initializers that allocate memory are completely removed from our codebase
+    // Luckily we only remove this on process shutdown, so the kernel will do the cleanup for us
+    Alloc64.release();
   }
 #pragma GCC diagnostic pop
 

External/FEXCore/Source/Utils/Allocator/64BitAllocator.cpp

@@ -3,23 +3,22 @@
 #include "Utils/Allocator/IntrusiveArenaAllocator.h"
 #include <FEXCore/Utils/LogManager.h>
 
+#include <algorithm>
 #include <array>
-#include <bit>
-#include <bitset>
 #include <cassert>
-#include <cmath>
 #include <cstdint>
-#include <cstring>
+#include <errno.h>
 #include <list>
-#include <malloc.h>
+#include <memory>
 #include <mutex>
-#include <stdio.h>
-#include <set>
+#include <new>
+#include <stddef.h>
+#include <sstream>
 #include <sys/mman.h>
+#include <bits/mman-map-flags-generic.h>
 #include <sys/utsname.h>
-#include <sys/resource.h>
-#include <syscall.h>
-#include <vector>
+#include <type_traits>
+#include <utility>
 
 static constexpr uint64_t PAGE_SHIFT = 12;
 static constexpr uint64_t PAGE_MASK = (1 << PAGE_SHIFT) - 1;

External/FEXCore/Source/Utils/LogManager.cpp

@@ -4,9 +4,14 @@ tags: glue|log-manager
 $end_info$
 */
 
-
+#include <FEXCore/Utils/CompilerDefs.h>
 #include <FEXCore/Utils/LogManager.h>
-#include <sstream>
+
+#include <alloca.h>
+#include <fmt/format.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <string>
 #include <vector>
 
 namespace LogMan {

External/FEXCore/Source/Utils/Telemetry.cpp

@@ -0,0 +1,62 @@
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/Telemetry.h>
+
+#include <array>
+#include <filesystem>
+#include <fstream>
+#include <stddef.h>
+#include <string>
+#include <string_view>
+#include <system_error>
+
+namespace FEXCore::Telemetry {
+#ifndef FEX_DISABLE_TELEMETRY
+  static std::array<Value, FEXCore::Telemetry::TelemetryType::TYPE_LAST> TelemetryValues = {{ }};
+  const std::array<std::string_view, FEXCore::Telemetry::TelemetryType::TYPE_LAST> TelemetryNames {
+    "64byte Split Locks",
+    "16Byte Split atomics",
+    "VEX instructions (AVX)",
+    "EVEX instructions (AVX512)",
+  };
+  void Initialize() {
+    auto DataDirectory = Config::GetDataDirectory();
+    DataDirectory += "Telemetry/";
+
+    // Ensure the folder structure is created for our configuration
+    std::error_code ec{};
+    if (!std::filesystem::exists(DataDirectory, ec) &&
+        !std::filesystem::create_directories(DataDirectory, ec)) {
+      LogMan::Msg::I("Couldn't create telemetry Folder");
+    }
+  }
+
+  void Shutdown(std::filesystem::path &ApplicationName) {
+    auto DataDirectory = Config::GetDataDirectory();
+    DataDirectory += "Telemetry/" + ApplicationName.string() + ".telem";
+
+    std::error_code ec{};
+    if (std::filesystem::exists(DataDirectory, ec)) {
+      // If the file exists, retain a single backup
+      auto Backup = DataDirectory + ".1";
+      std::filesystem::copy_file(DataDirectory, Backup, std::filesystem::copy_options::overwrite_existing, ec);
+    }
+
+    std::fstream fs(DataDirectory, std::ios_base::out | std::ios_base::trunc);
+    if (fs.is_open()) {
+      for (size_t i = 0; i < TelemetryType::TYPE_LAST; ++i) {
+        auto &Name = TelemetryNames.at(i);
+        auto &Data = TelemetryValues.at(i);
+        fs << Name << ": " << *Data << std::endl;
+      }
+
+      fs.flush();
+      fs.close();
+    }
+  }
+
+  Value &GetObject(TelemetryType Type) {
+    return TelemetryValues.at(Type);
+  }
+#endif
+}

External/FEXCore/Source/Utils/Threads.cpp

@@ -2,10 +2,15 @@
 #include <FEXCore/Utils/LogManager.h>
 #include <FEXCore/Utils/Threads.h>
 
+#include <alloca.h>
 #include <cstring>
+#include <functional>
+#include <memory>
 #include <mutex>
 #include <pthread.h>
+#include <stdint.h>
 #include <sys/mman.h>
+#include <bits/mman-map-flags-generic.h>
 #include <deque>
 
 namespace FEXCore::Threads {
@@ -17,8 +22,8 @@ namespace FEXCore::Threads {
   std::mutex DeadStackPoolMutex{};
   std::mutex LiveStackPoolMutex{};
 
-  std::deque<StackPoolItem> DeadStackPool;
-  std::deque<StackPoolItem> LiveStackPool;
+  static std::deque<StackPoolItem> DeadStackPool{};
+  static std::deque<StackPoolItem> LiveStackPool{};
 
   void *AllocateStackObject(size_t Size) {
     std::lock_guard lk{DeadStackPoolMutex};
@@ -58,6 +63,29 @@ namespace FEXCore::Threads {
     }
   }
 
+  void DeallocateStackObject(void *Ptr, size_t Size) {
+    RemoveStackFromLivePool(Ptr);
+    AddStackToDeadPool(Ptr, Size);
+  }
+
+  void Shutdown() {
+    std::lock_guard lk{DeadStackPoolMutex};
+    std::lock_guard lk2{LiveStackPoolMutex};
+    // Erase all the dead stack pools
+    for (auto &Item : DeadStackPool) {
+      FEXCore::Allocator::munmap(Item.Ptr, Item.Size);
+    }
+
+    // Now clean up any that are considered to still be live
+    // We are in shutdown phase, everything in the process is dead
+    for (auto &Item : LiveStackPool) {
+      FEXCore::Allocator::munmap(Item.Ptr, Item.Size);
+    }
+
+    DeadStackPool.clear();
+    LiveStackPool.clear();
+  }
+
   void *InitializeThread(void *Ptr);
 
   class PThread final : public Thread {
@@ -106,8 +134,7 @@ namespace FEXCore::Threads {
     }
 
     void FreeStack() {
-      RemoveStackFromLivePool(Stack);
-      AddStackToDeadPool(Stack, STACK_SIZE);
+      DeallocateStackObject(Stack, STACK_SIZE);
     }
 
     private:

External/FEXCore/include/FEXCore/Config/Config.h

@@ -39,8 +39,8 @@ namespace FEXCore::Config {
 
 namespace DefaultValues {
 #define P(x) x
-#define OPT_BASE(type, group, enum, json, default) constexpr P(type) P(enum) = P(default);
-#define OPT_STR(group, enum, json, default) const std::string P(enum) = P(default);
+#define OPT_BASE(type, group, enum, json, default) extern const P(type) P(enum);
+#define OPT_STR(group, enum, json, default) extern const std::string_view P(enum);
 #define OPT_STRARRAY(group, enum, json, default) OPT_STR(group, enum, json, default)
 #include <FEXCore/Config/ConfigValues.inl>
 
@@ -135,23 +135,27 @@ namespace Type {
   template<typename T>
   class FEX_DEFAULT_VISIBILITY Value {
   public:
-    template <typename TT = T,
-      typename std::enable_if<!std::is_same<TT, std::string>::value, int>::type = 0>
-    Value(FEXCore::Config::ConfigOption _Option, T Default)
+    template <typename TT = T> requires (!std::is_same_v<TT, std::string>)
+    Value(FEXCore::Config::ConfigOption _Option, TT Default)
       : Option {_Option} {
       ValueData = GetIfExists(Option, Default);
     }
 
-    template <typename TT = T,
-      typename std::enable_if<std::is_same<TT, std::string>::value, int>::type = 0>
-    Value(FEXCore::Config::ConfigOption _Option, T Default)
+    template <typename TT = T> requires (std::is_same_v<TT, std::string>)
+    Value(FEXCore::Config::ConfigOption _Option, TT Default)
       : Option {_Option} {
       ValueData = GetIfExists(Option, Default);
       GetListIfExists(Option, &AppendList);
     }
 
-    template <typename TT = T,
-      typename std::enable_if<!std::is_same<TT, std::string>::value, int>::type = 0>
+    template <typename TT = T> requires (std::is_same_v<TT, std::string>)
+    Value(FEXCore::Config::ConfigOption _Option, std::string_view Default)
+      : Option {_Option} {
+      ValueData = GetIfExists(Option, Default);
+      GetListIfExists(Option, &AppendList);
+    }
+
+    template <typename TT = T> requires (!std::is_same_v<TT, std::string>)
     Value(FEXCore::Config::ConfigOption _Option)
       : Option {_Option} {
       if (!FEXCore::Config::Exists(Option)) {
@@ -161,8 +165,7 @@ namespace Type {
       ValueData = Get(Option);
     }
 
-    template <typename TT = T,
-      typename std::enable_if<std::is_same<TT, std::string>::value, int>::type = 0>
+    template <typename TT = T> requires (std::is_same_v<TT, std::string>)
     Value(FEXCore::Config::ConfigOption _Option)
       : Option {_Option} {
       if (!FEXCore::Config::Exists(Option)) {
@@ -185,6 +188,47 @@ namespace Type {
 
     static T Get(FEXCore::Config::ConfigOption Option);
     static T GetIfExists(FEXCore::Config::ConfigOption Option, T Default);
+    static T GetIfExists(FEXCore::Config::ConfigOption Option, std::string_view Default);
+
     static void GetListIfExists(FEXCore::Config::ConfigOption Option, std::list<std::string> *List);
   };
+
+  // Application loaders
+  class FEX_DEFAULT_VISIBILITY OptionMapper : public FEXCore::Config::Layer {
+  public:
+    explicit OptionMapper(FEXCore::Config::LayerType Layer);
+
+  protected:
+    void MapNameToOption(const char *ConfigName, const char *ConfigString);
+  };
+
+  /**
+   * @brief Loads the main application config
+   *
+   * @param File Optional override to load a specific config file in to the main layer
+   * Shouldn't be commonly used
+   *
+   * @return unique_ptr for that layer
+   */
+  FEX_DEFAULT_VISIBILITY std::unique_ptr<FEXCore::Config::Layer> CreateMainLayer(std::string const *File = nullptr);
+
+  /**
+   * @brief Create an application configuration loader
+   *
+   * @param Filename Application filename component
+   * @param Global Load the global configuration or user accessible file
+   *
+   * @return unique_ptr for that layer
+   */
+  FEX_DEFAULT_VISIBILITY std::unique_ptr<FEXCore::Config::Layer> CreateAppLayer(const std::string& Filename, bool Global);
+
+  /**
+   * @brief iCreate an environment configuration loader
+   *
+   * @param _envp[] The environment array from main
+   *
+   * @return unique_ptr for that layer
+   */
+  FEX_DEFAULT_VISIBILITY std::unique_ptr<FEXCore::Config::Layer> CreateEnvironmentLayer(char *const _envp[]);
+
 }