Docs: Update for release FEX-2210

IR: Handle 256-bit VSMul/VUMul

.github/workflows/ccpp.yml

@@ -40,7 +40,6 @@ jobs:
       # Use a bash shell so we can use the same syntax for environment variable
       # access regardless of the host operating system
       shell: bash
-      working-directory: ${{runner.workspace}}/build
       run: $GITHUB_WORKSPACE/Scripts/CI_FetchRootFS.py
 
     - name : submodule checkout
@@ -65,7 +64,7 @@ jobs:
       # Note the current convention is to use the -S and -B options here to specify source
       # and build directories, but this is only available with CMake 3.13 and higher.
       # The CMake binaries on the Github Actions machines are (as of this writing) 3.12
-      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DENABLE_INTERPRETER=True -DBUILD_FEX_LINUX_TESTS=True
+      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DENABLE_INTERPRETER=True -DBUILD_FEX_LINUX_TESTS=True -DBUILD_THUNKS=True -DCMAKE_INSTALL_PREFIX=${{runner.workspace}}/build/install
 
     - name: Build
       working-directory: ${{runner.workspace}}/build
@@ -178,6 +177,51 @@ jobs:
       working-directory: ${{runner.workspace}}/build
       run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_FEXLinuxTests.log || true
 
+    - name: Thunkgen tests
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      run: cmake --build . --config $BUILD_TYPE --target thunkgen_tests
+
+    - name: Thunkgen Results move
+      if: ${{ always() }}
+      shell: bash
+      working-directory: ${{runner.workspace}}/build
+      run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_ThunkgenTests.log || true
+
+    - name: Install
+      if: matrix.arch[1] == 'x64'
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      run: cmake --build . --config $BUILD_TYPE --target install
+
+    - name: Test GL No-Thunks
+      if: matrix.arch[1] == 'x64'
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      env:
+        DISPLAY: ":0"
+      run: cmake --build . --config $BUILD_TYPE --target thunk_functional_tests_nothunks
+
+    - name: No thunks Results move
+      if: ${{ always() }}
+      shell: bash
+      working-directory: ${{runner.workspace}}/build
+      run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_NoThunkResults.log || true
+
+    - name: Test GL Thunks
+      if: matrix.arch[1] == 'x64'
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      env:
+        DISPLAY: ":0"
+      run: cmake --build . --config $BUILD_TYPE --target thunk_functional_tests_thunks
+
+    - name: Thunks Results move
+      if: ${{ always() }}
+      shell: bash
+      working-directory: ${{runner.workspace}}/build
+      run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_ThunkResults.log || true
+
     - name: Truncate test results
       if: ${{ always() }}
       shell: bash
@@ -197,4 +241,3 @@ jobs:
         name: Results-${{ env.runner_name }}
         path: ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log
         retention-days: 3
-

.github/workflows/vixl_simulator.yml

@@ -0,0 +1,118 @@
+name: Vixl Simulator run
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
+  BUILD_TYPE: Release
+  CC: clang
+  CXX: clang++
+
+jobs:
+  build:
+    runs-on: ${{ matrix.arch }}
+    strategy:
+      matrix:
+        # Only the x86-64 runner is fast enough to run this
+        arch: [[self-hosted, x64], [self-hosted, ARMv8.4]]
+      fail-fast: false
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set runner label
+      run: echo "runner_label=${{ matrix.arch[1] }}" >> $GITHUB_ENV
+
+    - name: Set rootfs paths
+      run: |
+        echo "FEX_ROOTFS_MOUNT=/mnt/AutoNFS/rootfs/" >> $GITHUB_ENV
+        echo "FEX_ROOTFS_PATH=$HOME/Rootfs/" >> $GITHUB_ENV
+        echo "FEX_ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
+        echo "ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
+
+    - name: Update RootFS cache
+      # Use a bash shell so we can use the same syntax for environment variable
+      # access regardless of the host operating system
+      shell: bash
+      run: $GITHUB_WORKSPACE/Scripts/CI_FetchRootFS.py
+
+    - name : submodule checkout
+      # Need to update submodules
+      run: |
+        git submodule sync --recursive
+        git submodule update --init --depth 1
+
+    - name: Clean Build Environment
+      run: rm -Rf ${{runner.workspace}}/build
+
+    - name: Create Build Environment
+      # Some projects don't allow in-source building, so create a separate build directory
+      # We'll use this as our working directory for all subsequent commands
+      run: cmake -E make_directory ${{runner.workspace}}/build
+
+    - name: Configure CMake
+      # Use a bash shell so we can use the same syntax for environment variable
+      # access regardless of the host operating system
+      shell: bash
+      working-directory: ${{runner.workspace}}/build
+      # Note the current convention is to use the -S and -B options here to specify source
+      # and build directories, but this is only available with CMake 3.13 and higher.
+      # The CMake binaries on the Github Actions machines are (as of this writing) 3.12
+      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_VIXL_SIMULATOR=True -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True
+
+    - name: Build
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      # Execute the build.  You can specify a specific target with "--target <NAME>"
+      run: cmake --build . --config $BUILD_TYPE
+
+    - name: ASM Tests
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      # Execute the unit tests
+      run: cmake --build . --config $BUILD_TYPE --target asm_tests
+
+    - name: ASM Test Results move
+      if: ${{ always() }}
+      shell: bash
+      working-directory: ${{runner.workspace}}/build
+      run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_ASM.log || true
+
+    - name: IR Tests
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      # Execute the unit tests
+      run: cmake --build . --config $BUILD_TYPE --target ir_tests
+
+    - name: IR Test Results move
+      if: ${{ always() }}
+      shell: bash
+      working-directory: ${{runner.workspace}}/build
+      run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_IR.log || true
+
+    - name: Truncate test results
+      if: ${{ always() }}
+      shell: bash
+      working-directory: ${{runner.workspace}}/build
+      # Cap out the log files at 20M in case something crash spins and dumps fault text
+      # ASM tests get quite close to 10MB
+      run: truncate --size=<20M ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log || true
+
+    - name: Set runner name
+      if: ${{ always() }}
+      run: echo "runner_name=$(hostname)" >> $GITHUB_ENV
+
+    - name: Upload results
+      if: ${{ always() }}
+      uses: 'actions/upload-artifact@v2'
+      with:
+        name: Results-${{ env.runner_name }}
+        path: ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log
+        retention-days: 3
+

CI/GLThunks.json

@@ -0,0 +1,5 @@
+{
+  "ThunksDB": {
+    "GL": 1
+  }
+}

CI/VulkanThunks.json

@@ -0,0 +1,5 @@
+{
+  "ThunksDB": {
+    "Vulkan": 1
+  }
+}

CMakeFiles/cmake_uninstall.cmake.in

@@ -0,0 +1,21 @@
+if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
+  message(FATAL_ERROR "Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt")
+endif()
+
+file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" files)
+string(REGEX REPLACE "\n" ";" files "${files}")
+foreach(file ${files})
+  message(STATUS "Uninstalling $ENV{DESTDIR}${file}")
+  if(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
+    exec_program(
+      "@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\""
+      OUTPUT_VARIABLE rm_out
+      RETURN_VALUE rm_retval
+      )
+    if(NOT "${rm_retval}" STREQUAL 0)
+      message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}")
+    endif()
+  else(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
+    message(STATUS "File $ENV{DESTDIR}${file} does not exist.")
+  endif()
+endforeach()

CMakeLists.txt

@@ -7,6 +7,7 @@ CHECK_INCLUDE_FILES ("gdb/jit-reader.h" HAVE_GDB_JIT_READER_H)
 option(BUILD_TESTS "Build unit tests to ensure sanity" TRUE)
 option(BUILD_FEX_LINUX_TESTS "Build FEXLinuxTests, requires x86 compiler" FALSE)
 option(BUILD_THUNKS "Build thunks" FALSE)
+option(BUILD_CLANG_THUNKS "Build thunks with clang" FALSE)
 option(ENABLE_CLANG_FORMAT "Run clang format over the source" FALSE)
 option(ENABLE_IWYU "Enables include what you use program" FALSE)
 option(ENABLE_LTO "Enable LTO with compilation" TRUE)
@@ -27,10 +28,23 @@ option(ENABLE_LIBCXX "Enables LLVM libc++" FALSE)
 option(ENABLE_INTERPRETER "Enables FEX's Interpreter" FALSE)
 option(ENABLE_CCACHE "Enables ccache for compile caching" TRUE)
 option(ENABLE_TERMUX_BUILD "Forces building for Termux on a non-Termux build machine" FALSE)
+option(ENABLE_VIXL_SIMULATOR "Forces the FEX JIT to use the VIXL simulator" FALSE)
 
-set (X86_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/toolchain_x86.cmake" CACHE FILEPATH "Toolchain file for the x86 (cross-)compiler")
+set (X86_32_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/toolchain_x86_32.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting i686")
+set (X86_64_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/toolchain_x86_64.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
 set (DATA_DIRECTORY "${CMAKE_INSTALL_PREFIX}/share/fex-emu" CACHE PATH "global data directory")
 
+# uninstall target
+if(NOT TARGET uninstall)
+  configure_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/CMakeFiles/cmake_uninstall.cmake.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/cmake_uninstall.cmake"
+    IMMEDIATE @ONLY)
+
+  add_custom_target(uninstall
+    COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/cmake_uninstall.cmake)
+endif()
+
 # These options are meant for package management
 set (TUNE_CPU "native" CACHE STRING "Override the CPU the build is tuned for")
 set (TUNE_ARCH "generic" CACHE STRING "Override the Arch the build is tuned for")
@@ -84,10 +98,9 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
   set(_M_X86_64 1)
   add_definitions(-D_M_X86_64=1)
   set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcx16")
-  set (X86_TOOLCHAIN_FILE "")
 endif()
 
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64|^arm64|^armv8\.*")
   set(_M_ARM_64 1)
   add_definitions(-D_M_ARM_64=1)
 endif()
@@ -363,8 +376,6 @@ if (BUILD_TESTS)
 endif()
 
 add_subdirectory(FEXHeaderUtils/)
-include_directories(FEXHeaderUtils/)
-
 add_subdirectory(External/FEXCore)
 
 # Binfmt_misc files must be installed prior to Source/ installs
@@ -403,8 +414,28 @@ if (BUILD_THUNKS)
     SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ThunkLibs/GuestLibs"
     BINARY_DIR "Guest"
     CMAKE_ARGS
+      "-DBITNESS=64"
       "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
-      "-DCMAKE_TOOLCHAIN_FILE:FILEPATH=${X86_TOOLCHAIN_FILE}"
+      "-DENABLE_CLANG_THUNKS=${ENABLE_CLANG_THUNKS}"
+      "-DCMAKE_TOOLCHAIN_FILE:FILEPATH=${X86_64_TOOLCHAIN_FILE}"
+      "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
+      "-DSTRUCT_VERIFIER=${CMAKE_SOURCE_DIR}/Scripts/StructPackVerifier.py"
+      "-DFEX_PROJECT_SOURCE_DIR=${FEX_PROJECT_SOURCE_DIR}"
+      "-DGENERATOR_EXE=$<TARGET_FILE:thunkgen>"
+    INSTALL_COMMAND ""
+    BUILD_ALWAYS ON
+    DEPENDS thunkgen
+  )
+
+  ExternalProject_Add(guest-libs-32
+    PREFIX guest-libs-32
+    SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ThunkLibs/GuestLibs"
+    BINARY_DIR "Guest_32"
+    CMAKE_ARGS
+      "-DBITNESS=32"
+      "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+      "-DENABLE_CLANG_THUNKS=${ENABLE_CLANG_THUNKS}"
+      "-DCMAKE_TOOLCHAIN_FILE:FILEPATH=${X86_32_TOOLCHAIN_FILE}"
       "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
       "-DSTRUCT_VERIFIER=${CMAKE_SOURCE_DIR}/Scripts/StructPackVerifier.py"
       "-DFEX_PROJECT_SOURCE_DIR=${FEX_PROJECT_SOURCE_DIR}"
@@ -422,6 +453,28 @@ if (BUILD_THUNKS)
     )"
     DEPENDS guest-libs
   )
+
+  install(
+    CODE "MESSAGE(\"-- Installing: guest-libs-32\")"
+    CODE "
+    EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} --build . --target install
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest_32
+    )"
+    DEPENDS guest-libs-32
+  )
+
+  add_custom_target(uninstall_guest-libs
+    COMMAND ${CMAKE_COMMAND} "--build" "." "--target" "uninstall"
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest
+  )
+
+  add_custom_target(uninstall_guest-libs-32
+    COMMAND ${CMAKE_COMMAND} "--build" "." "--target" "uninstall"
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest_32
+  )
+
+  add_dependencies(uninstall uninstall_guest-libs)
+  add_dependencies(uninstall uninstall_guest-libs-32)
 endif()
 
 set(FEX_VERSION_MAJOR "0")
@@ -476,7 +529,7 @@ endif()
 set (CPACK_GENERATOR "DEB")
 set (CPACK_PACKAGE_NAME fex-emu)
 set (CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${GIT_DESCRIBE_STRING}_${CMAKE_SYSTEM_PROCESSOR}")
-set (CPACK_PACKAGE_CONTACT "FEX-Emu Maintainers <team@fex-emu.org>")
+set (CPACK_PACKAGE_CONTACT "FEX-Emu Maintainers <team@fex-emu.com>")
 set (CPACK_PACKAGE_VERSION_MAJOR "${FEX_VERSION_MAJOR}")
 set (CPACK_PACKAGE_VERSION_MINOR "${FEX_VERSION_MINOR}")
 set (CPACK_PACKAGE_VERSION_PATCH "${FEX_VERSION_PATCH}")

CODE_OF_CONDUCT.md

@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at team@fex-emu.org. All
+reported by contacting the project team at team@fex-emu.com. All
 complaints will be reviewed and investigated and will result in a response that
 is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.

Data/ThunksDB.json

@@ -165,6 +165,14 @@
         "@PREFIX_LIB@/x86_64-linux-gnu/libXfixes.so.3.1.0"
       ]
     },
+    "OpenCL": {
+      "Library" : "libOpenCL-guest.so",
+      "Overlay": [
+        "@PREFIX_LIB@/x86_64-linux-gnu/libOpenCL.so",
+        "@PREFIX_LIB@/x86_64-linux-gnu/libOpenCL.so.1",
+        "@PREFIX_LIB@/x86_64-linux-gnu/libOpenCL.so.1.0.0"
+      ]
+    },
     "":{}
   }
 }

External/FEXCore/CMakeLists.txt

@@ -9,12 +9,19 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcx16")
 endif()
 
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64|^arm64|^armv8\.*")
   set(_M_ARM_64 1)
 endif()
 
-set(ENABLE_JIT_X86_64 ${_M_X86_64} CACHE BOOL "Enable the x86_64 JIT")
-set(ENABLE_JIT_ARM64 ${_M_ARM_64} CACHE BOOL "Enable the ARM64 JIT")
+if (ENABLE_VIXL_SIMULATOR)
+  # If the vixl simulator is enabled then we are using the ARM64 JIT
+  option(ENABLE_JIT_X86_64 "Enable the x86_64 JIT" FALSE)
+  option(ENABLE_JIT_ARM64 "Enable the ARM64 JIT" TRUE)
+else()
+  option(ENABLE_JIT_X86_64 "Enable the x86_64 JIT" ${_M_X86_64})
+  option(ENABLE_JIT_ARM64 "Enable the ARM64 JIT" ${_M_ARM_64})
+endif()
+
 option(ENABLE_CLANG_FORMAT "Run clang format over the source" FALSE)
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -27,7 +34,6 @@ set(CMAKE_INCLUDE_CURRENT_DIR ON)
 include(CheckCXXCompilerFlag)
 include(CheckIncludeFileCXX)
 
-
 if (EXISTS ${CMAKE_CURRENT_DIR}/External/vixl/)
     # Useful to have for freestanding libFEXCore
     add_subdirectory(External/vixl/)
@@ -46,14 +52,14 @@ if (OVERRIDE_VERSION STREQUAL "detect")
 
   if (GIT_FOUND)
     execute_process(
-      COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD
+      COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
       WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
       OUTPUT_VARIABLE GIT_SHORT_HASH
       ERROR_QUIET
       OUTPUT_STRIP_TRAILING_WHITESPACE
     )
     execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe
+      COMMAND ${GIT_EXECUTABLE} describe --abbrev=7
       WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
       OUTPUT_VARIABLE GIT_DESCRIBE_STRING
       ERROR_QUIET

External/FEXCore/Source/CMakeLists.txt

@@ -177,6 +177,11 @@ if (_M_ARM_64)
   list(APPEND DEFINES -D_M_ARM_64=1)
 endif()
 
+if (ENABLE_VIXL_SIMULATOR)
+  # We can run the simulator on both x86-64 or AArch64 hosts
+  list(APPEND DEFINES -DVIXL_SIMULATOR=1 -DVIXL_INCLUDE_SIMULATOR_AARCH64=1)
+endif()
+
 if (ENABLE_JIT_X86_64)
   list(APPEND SRCS
     Interface/Core/JIT/x86_64/JIT.cpp
@@ -213,7 +218,7 @@ if (ENABLE_JIT_ARM64)
     )
 endif()
 
-set (LIBS vixl dl xxhash tiny-json)
+set (LIBS fmt::fmt vixl dl xxhash tiny-json FEXHeaderUtils)
 if (ENABLE_JEMALLOC)
   list (APPEND LIBS FEX_jemalloc)
 endif()
@@ -359,14 +364,14 @@ endfunction()
 
 # Build FEXCore_Config static library
 add_library(FEXCore_Base STATIC ${FEXCORE_BASE_SRCS})
-target_link_libraries(FEXCore_Base fmt::fmt tiny-json)
+target_link_libraries(FEXCore_Base ${LIBS})
 AddDefaultOptionsToTarget(FEXCore_Base)
 
 function(AddObject Name Type)
   add_library(${Name} ${Type} ${SRCS})
   add_dependencies(${Name} IR_INC)
 
-  target_link_libraries(${Name} FEXCore_Base ${LIBS})
+  target_link_libraries(${Name} FEXCore_Base)
   AddDefaultOptionsToTarget(${Name})
 
   set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)
@@ -374,7 +379,7 @@ endfunction()
 
 function(AddLibrary Name Type)
   add_library(${Name} ${Type} $<TARGET_OBJECTS:${PROJECT_NAME}_object>)
-  target_link_libraries(${Name} FEXCore_Base ${LIBS})
+  target_link_libraries(${Name} FEXCore_Base)
   set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)
 
   AddDefaultOptionsToTarget(${Name})

External/FEXCore/Source/Interface/Config/Config.cpp

@@ -82,7 +82,7 @@ namespace JSON {
     json_t const* ConfigList = json_getProperty(json, "Config");
 
     if (!ConfigList) {
-      LogMan::Msg::EFmt("Couldn't get config list");
+      // This is a non-error if the configuration file exists but no Config section
       return;
     }
 

External/FEXCore/Source/Interface/Config/Config.json.in

@@ -90,6 +90,13 @@
           "Folder to find the guest-side thunking libraries."
         ]
       },
+      "ThunkGuestLibs32": {
+        "Type": "str",
+        "Default": "@CMAKE_INSTALL_PREFIX@/share/fex-emu/GuestThunks_32/",
+        "Desc": [
+          "Folder to find the 32-bit guest-side thunking libraries."
+        ]
+      },
       "ThunkConfig": {
         "Type": "str",
         "Default": "",
@@ -351,6 +358,13 @@
         "Desc": [
           "Loads an AOT IR cache for the loaded executable."
         ]
+      },
+      "ServerSocketPath": {
+        "Type": "str",
+        "Default": "",
+        "Desc": [
+          "Override for a FEXServer socket path. Only useful for chroots."
+        ]
       }
     }
   },
@@ -368,6 +382,17 @@
         "Type": "str",
         "Default": ""
       },
+      "APP_CONFIG_NAME": {
+        "Type": "str",
+        "Default": "",
+        "Desc": [
+          "This is the application config name that has been loaded.",
+          "This differs from APP_FILENAME in two ways",
+          "Where APP_FILENAME always points to the executable path that FEX-Emu is executing.",
+          "This matches what is used to load the AppLayer configuration name.",
+          "When running through a compatibility layer like wine, this will only be the exe name, instead of wine full path."
+        ]
+      },
       "IS64BIT_MODE": {
         "Type": "bool",
         "Default": "false"

External/FEXCore/Source/Interface/Context/Context.cpp

@@ -202,6 +202,10 @@ namespace FEXCore::Context {
     return CTX->AddCustomIREntrypoint(Entrypoint, Handler, Creator, Data);
   }
 
+  void AppendThunkDefinitions(FEXCore::Context::Context *CTX, std::vector<FEXCore::IR::ThunkDefinition> const& Definitions) {
+    CTX->AppendThunkDefinitions(Definitions);
+  }
+
 namespace Debug {
   void CompileRIP(FEXCore::Context::Context *CTX, uint64_t RIP) {
     CTX->CompileRIP(CTX->ParentThread, RIP);

External/FEXCore/Source/Interface/Context/Context.h

@@ -15,6 +15,7 @@
 #include <FEXCore/Debug/InternalThreadState.h>
 #include <FEXCore/Utils/CompilerDefs.h>
 #include <FEXCore/Utils/Event.h>
+#include <FEXHeaderUtils/Syscalls.h>
 #include <stdint.h>
 
 
@@ -189,7 +190,7 @@ namespace FEXCore::Context {
     static void ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP) {
       auto Thread = Frame->Thread;
       
-      LogMan::Throw::AFmt(Thread->ThreadManager.GetTID() == gettid(), "Must be called from owning thread {}, not {}", Thread->ThreadManager.GetTID(), gettid());
+      LogMan::Throw::AFmt(Thread->ThreadManager.GetTID() == FHU::Syscalls::gettid(), "Must be called from owning thread {}, not {}", Thread->ThreadManager.GetTID(), FHU::Syscalls::gettid());
 
       FHU::ScopedSignalMaskWithUniqueLock lk(Thread->CTX->CodeInvalidationMutex);
 
@@ -322,6 +323,8 @@ namespace FEXCore::Context {
       IRCaptureCache.SetAOTIRRenamer(CacheRenamer);
     }
 
+    void AppendThunkDefinitions(std::vector<FEXCore::IR::ThunkDefinition> const& Definitions);
+
     FEXCore::Utils::PooledAllocatorMMap OpDispatcherAllocator;
     FEXCore::Utils::PooledAllocatorMMap FrontendAllocator;
 

External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp

@@ -17,7 +17,6 @@
 #include <utility>
 
 namespace FEXCore::CPU {
-#define STATE x28
 
 // We want vixl to not allocate a default buffer. Jit and dispatcher will manually create one.
 Arm64Emitter::Arm64Emitter(FEXCore::Context::Context *ctx, size_t size)
@@ -25,11 +24,15 @@ Arm64Emitter::Arm64Emitter(FEXCore::Context::Context *ctx, size_t size)
   , EmitterCTX {ctx} {
   CPU.SetUp();
 
+#ifdef VIXL_SIMULATOR
+  auto Features = vixl::CPUFeatures::All();
+#else
   auto Features = vixl::CPUFeatures::InferFromOS();
   if (ctx->HostFeatures.SupportsAtomics) {
     // Hypervisor can hide this on the c630?
     Features.Combine(vixl::CPUFeatures::Feature::kLORegions);
   }
+#endif
 
   SetCPUFeatures(Features);
 }
@@ -214,7 +217,8 @@ void Arm64Emitter::SpillStaticRegs(bool FPRs, uint32_t GPRSpillMask, uint32_t FP
           const auto Reg = SRAFPR[i];
 
           if (((1U << Reg.GetCode()) & FPRSpillMask) != 0) {
-            str(Reg.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.avx.data[i][0])));
+            mov(TMP4, offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0]));
+            st1b(Reg.Z().VnB(), PRED_TMP_32B, SVEMemOperand(STATE, TMP4));
           }
         }
       } else {
@@ -257,11 +261,19 @@ void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t GPRFillMask, uint32_t FPRF
 
     if (FPRs) {
       if (EmitterCTX->HostFeatures.SupportsAVX) {
+        // Set up predicate registers.
+        // We don't bother spilling these in SpillStaticRegs,
+        // since all that matters is we restore them on a fill.
+        // It's not a concern if they get trounced by something else.
+        ptrue(PRED_TMP_16B.VnB(), SVE_VL16);
+        ptrue(PRED_TMP_32B.VnB(), SVE_VL32);
+
         for (size_t i = 0; i < SRAFPR.size(); i++) {
           const auto Reg = SRAFPR[i];
 
           if (((1U << Reg.GetCode()) & FPRFillMask) != 0) {
-            ldr(Reg.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm.avx.data[i][0])));
+            mov(TMP4, offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0]));
+            ld1b(Reg.Z().VnB(), PRED_TMP_32B.Zeroing(), SVEMemOperand(STATE, TMP4));
           }
         }
       } else {
@@ -286,20 +298,31 @@ void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t GPRFillMask, uint32_t FPRF
 }
 
 void Arm64Emitter::PushDynamicRegsAndLR() {
-  uint64_t SPOffset = AlignUp((RA64.size() + 1) * 8 + RAFPR.size() * 16, 16);
+  const auto CanUseSVE = EmitterCTX->HostFeatures.SupportsAVX;
+  const auto GPRSize = (RA64.size() + 1) * Core::CPUState::GPR_REG_SIZE;
+  const auto FPRRegSize = CanUseSVE ? Core::CPUState::XMM_AVX_REG_SIZE
+                                    : Core::CPUState::XMM_SSE_REG_SIZE;
+  const auto FPRSize = RAFPR.size() * FPRRegSize;
+  const uint64_t SPOffset = AlignUp(GPRSize + FPRSize, 16);
 
   sub(sp, sp, SPOffset);
   int i = 0;
 
-  for (auto RA : RAFPR)
-  {
-    str(RA.Q(), MemOperand(sp, i * 8));
-    i+=2;
+  if (CanUseSVE) {
+    for (const auto& RA : RAFPR) {
+      mov(TMP4, i * 8);
+      st1b(RA.Z().VnB(), PRED_TMP_32B, SVEMemOperand(sp, TMP4));
+      i += 4;
+    }
+  } else {
+    for (const auto& RA : RAFPR) {
+      str(RA.Q(), MemOperand(sp, i * 8));
+      i += 2;
+    }
   }
 
 #if 0 // All GPRs should be caller saved
-  for (auto RA : RA64)
-  {
+  for (const auto& RA : RA64) {
     str(RA, MemOperand(sp, i * 8));
     i++;
   }
@@ -309,18 +332,29 @@ void Arm64Emitter::PushDynamicRegsAndLR() {
 }
 
 void Arm64Emitter::PopDynamicRegsAndLR() {
-  uint64_t SPOffset = AlignUp((RA64.size() + 1) * 8 + RAFPR.size() * 16, 16);
+  const auto CanUseSVE = EmitterCTX->HostFeatures.SupportsAVX;
+  const auto GPRSize = (RA64.size() + 1) * Core::CPUState::GPR_REG_SIZE;
+  const auto FPRRegSize = CanUseSVE ? Core::CPUState::XMM_AVX_REG_SIZE
+                                    : Core::CPUState::XMM_SSE_REG_SIZE;
+  const auto FPRSize = RAFPR.size() * FPRRegSize;
+  const uint64_t SPOffset = AlignUp(GPRSize + FPRSize, 16);
   int i = 0;
 
-  for (auto RA : RAFPR)
-  {
-    ldr(RA.Q(), MemOperand(sp, i * 8));
-    i+=2;
+  if (CanUseSVE) {
+    for (const auto& RA : RAFPR) {
+      mov(TMP4, i * 8);
+      ld1b(RA.Z().VnB(), PRED_TMP_32B.Zeroing(), SVEMemOperand(sp, TMP4));
+      i += 4;
+    }
+  } else {
+    for (const auto& RA : RAFPR) {
+      ldr(RA.Q(), MemOperand(sp, i * 8));
+      i += 2;
+    }
   }
 
 #if 0 // All GPRs should be caller saved
-  for (auto RA : RA64)
-  {
+  for (const auto& RA : RA64) {
     ldr(RA, MemOperand(sp, i * 8));
     i++;
   }

External/FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h

@@ -8,6 +8,10 @@
 #include <aarch64/cpu-aarch64.h>
 #include <aarch64/operands-aarch64.h>
 #include <platform-vixl.h>
+#ifdef VIXL_SIMULATOR
+#include <aarch64/simulator-aarch64.h>
+#include <aarch64/simulator-constants-aarch64.h>
+#endif
 
 #include <FEXCore/Config/Config.h>
 
@@ -58,6 +62,27 @@ const std::array<aarch64::VRegister, 12> RAFPR = {
   v8,  v9,  v10, v11, v12, v13, v14, v15
 };
 
+// Contains the address to the currently available CPU state
+#define STATE x28
+
+// GPR temporaries. Only x3 can be used across spill boundaries
+// so if these ever need to change, be very careful about that.
+#define TMP1 x0
+#define TMP2 x1
+#define TMP3 x2
+#define TMP4 x3
+
+// Vector temporaries
+#define VTMP1 v1
+#define VTMP2 v2
+#define VTMP3 v3
+
+// Predicate register temporaries (used when AVX support is enabled)
+// PRED_TMP_16B indicates a predicate register that indicates the first 16 bytes set to 1.
+// PRED_TMP_32B indicates a predicate register that indicates the first 32 bytes set to 1.
+#define PRED_TMP_16B p6
+#define PRED_TMP_32B p7
+
 // This class contains common emitter utility functions that can
 // be used by both Arm64 JIT and ARM64 Dispatcher
 class Arm64Emitter : public vixl::aarch64::Assembler {
@@ -67,6 +92,10 @@ protected:
   FEXCore::Context::Context *EmitterCTX;
   vixl::aarch64::CPU CPU;
   void LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant, bool NOPPad = false);
+
+  // NOTE: These functions WILL clobber the register TMP4 if AVX support is enabled
+  //       and FPRs are being spilled or filled. If only GPRs are spilled/filled, then
+  //       TMP4 is left alone.
   void SpillStaticRegs(bool FPRs = true, uint32_t GPRSpillMask = ~0U, uint32_t FPRSpillMask = ~0U);
   void FillStaticRegs(bool FPRs = true, uint32_t GPRFillMask = ~0U, uint32_t FPRFillMask = ~0U);
 
@@ -83,6 +112,71 @@ protected:
   void PopCalleeSavedRegisters();
 
   void Align16B();
+#ifdef VIXL_SIMULATOR
+  // Generates a vixl simulator runtime call.
+  //
+  // This matches behaviour of vixl's macro assembler, but we need to reimplement it since we aren't using the macro assembler.
+  // This isn't too complex with how vixl emits this.
+  //
+  // Emit:
+  // 1) hlt(kRuntimeCallOpcode)
+  // 2) Simulator wrapper handler
+  // 3) Function to call
+  // 4) Style of the function call (Call versus tail-call)
+  template<typename R, typename... P>
+  void GenerateRuntimeCall(R (*Function)(P...)) {
+    uintptr_t SimulatorWrapperAddress = reinterpret_cast<uintptr_t>(
+      &(Simulator::RuntimeCallStructHelper<R, P...>::Wrapper));
+
+    uintptr_t FunctionAddress = reinterpret_cast<uintptr_t>(Function);
+
+    hlt(kRuntimeCallOpcode);
+
+    // Simulator wrapper address pointer.
+    dc(SimulatorWrapperAddress);
+
+    // Runtime function address to call
+    dc(FunctionAddress);
+
+    // Call type
+    dc32(kCallRuntime);
+  }
+
+  template<typename R, typename... P>
+  void GenerateIndirectRuntimeCall(vixl::aarch64::Register Reg) {
+    uintptr_t SimulatorWrapperAddress = reinterpret_cast<uintptr_t>(
+      &(Simulator::RuntimeCallStructHelper<R, P...>::Wrapper));
+
+    hlt(kIndirectRuntimeCallOpcode);
+
+    // Simulator wrapper address pointer.
+    dc(SimulatorWrapperAddress);
+
+    // Register that contains the function to call
+    dc(Reg.GetCode());
+
+    // Call type
+    dc32(kCallRuntime);
+  }
+
+  template<>
+  void GenerateIndirectRuntimeCall<float, __uint128_t>(vixl::aarch64::Register Reg) {
+    uintptr_t SimulatorWrapperAddress = reinterpret_cast<uintptr_t>(
+      &(Simulator::RuntimeCallStructHelper<float, __uint128_t>::Wrapper));
+
+    hlt(kIndirectRuntimeCallOpcode);
+
+    // Simulator wrapper address pointer.
+    dc(SimulatorWrapperAddress);
+
+    // Register that contains the function to call
+    dc(Reg.GetCode());
+
+    // Call type
+    dc32(kCallRuntime);
+  }
+
+#endif
 
   FEX_CONFIG_OPT(StaticRegisterAllocation, SRA);
 };

External/FEXCore/Source/Interface/Core/Core.cpp

@@ -221,7 +221,7 @@ namespace FEXCore::Context {
 #if (_M_X86_64 && JIT_X86_64)
       FEXCore::CPU::InitializeX86JITSignalHandlers(this);
       BackendFeatures = FEXCore::CPU::GetX86JITBackendFeatures();
-#elif (_M_ARM_64 && JIT_ARM64)
+#elif (_M_ARM_64 && JIT_ARM64) || defined(VIXL_SIMULATOR)
       FEXCore::CPU::InitializeArm64JITSignalHandlers(this);
       BackendFeatures = FEXCore::CPU::GetArm64JITBackendFeatures();
 #else
@@ -238,16 +238,16 @@ namespace FEXCore::Context {
 
     DispatcherConfig.StaticRegisterAllocation = Config.StaticRegisterAllocation && BackendFeatures.SupportsStaticRegisterAllocation;
 
-#if (_M_X86_64)
-    Dispatcher = FEXCore::CPU::Dispatcher::CreateX86(this, DispatcherConfig);
-#elif (_M_ARM_64)
+#if JIT_ARM64
     Dispatcher = FEXCore::CPU::Dispatcher::CreateArm64(this, DispatcherConfig);
+#elif JIT_X86_64
+    Dispatcher = FEXCore::CPU::Dispatcher::CreateX86(this, DispatcherConfig);
 #else
     ERROR_AND_DIE_FMT("FEXCore has been compiled with an unknown target");
 #endif
 
     // Initialize common signal handlers
-    
+
     auto PauseHandler = [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
       return Thread->CTX->Dispatcher->HandleSignalPause(Thread, Signal, info, ucontext);
     };
@@ -573,7 +573,7 @@ namespace FEXCore::Context {
 
 #if (_M_X86_64 && JIT_X86_64)
       Thread->CPUBackend = FEXCore::CPU::CreateX86JITCore(this, Thread);
-#elif (_M_ARM_64 && JIT_ARM64)
+#elif (_M_ARM_64 && JIT_ARM64) || defined(VIXL_SIMULATOR)
       Thread->CPUBackend = FEXCore::CPU::CreateArm64JITCore(this, Thread);
 #else
       ERROR_AND_DIE_FMT("FEXCore has been compiled without a viable JIT core");
@@ -740,7 +740,7 @@ namespace FEXCore::Context {
     }
   }
 
-  Context::GenerateIRResult Context::GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP, bool ExtendedDebugInfo) {    
+  Context::GenerateIRResult Context::GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP, bool ExtendedDebugInfo) {
     Thread->OpDispatcher->ReownOrClaimBuffer();
     Thread->OpDispatcher->ResetWorkingList();
 
@@ -749,7 +749,7 @@ namespace FEXCore::Context {
 
 
     std::shared_lock lk(CustomIRMutex);
-    
+
     auto Handler = CustomIRHandlers.find(GuestRIP);
     if (Handler != CustomIRHandlers.end()) {
       TotalInstructions = 1;
@@ -872,7 +872,7 @@ namespace FEXCore::Context {
           }
         }
       }
-      
+
       Thread->OpDispatcher->Finalize();
 
       Thread->FrontendDecoder->DelayedDisownBuffer();
@@ -1182,7 +1182,7 @@ namespace FEXCore::Context {
 
   static void InvalidateGuestCodeRangeInternal(FEXCore::Context::Context *CTX, uint64_t Start, uint64_t Length) {
     std::lock_guard lk(CTX->ThreadCreationMutex);
-    
+
     for (auto &Thread : CTX->Threads) {
       InvalidateGuestThreadCodeRange(Thread, Start, Length);
     }
@@ -1190,7 +1190,7 @@ namespace FEXCore::Context {
 
   void InvalidateGuestCodeRange(FEXCore::Context::Context *CTX, uint64_t Start, uint64_t Length) {
     FHU::ScopedSignalMaskWithUniqueLock CodeInvalidationLock(CTX->CodeInvalidationMutex);
-    
+
     InvalidateGuestCodeRangeInternal(CTX, Start, Length);
   }
 
@@ -1233,9 +1233,9 @@ namespace FEXCore::Context {
     Thread->LookupCache->AddBlockLink(GuestDestination, HostLink, delinker);
   }
 
-  void Context::ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {   
+  void Context::ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
     LogMan::Throw::AFmt(Thread->CTX->CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here");
-    
+
     std::lock_guard<std::recursive_mutex> lk(Thread->LookupCache->WriteLock);
 
     Thread->DebugStore.erase(GuestRIP);
@@ -1332,6 +1332,11 @@ namespace FEXCore::Context {
     }
   }
 
+
+  void Context::AppendThunkDefinitions(std::vector<FEXCore::IR::ThunkDefinition> const& Definitions) {
+    ThunkHandler->AppendThunkDefinitions(Definitions);
+  }
+
   void ConfigureAOTGen(FEXCore::Core::InternalThreadState *Thread, std::set<uint64_t> *ExternalBranches, uint64_t SectionMaxAddress) {
     Thread->FrontendDecoder->SetExternalBranches(ExternalBranches);
     Thread->FrontendDecoder->SetSectionMaxAddress(SectionMaxAddress);

External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.cpp

@@ -38,11 +38,20 @@ namespace FEXCore::CPU {
 using namespace vixl;
 using namespace vixl::aarch64;
 
-static constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096;
-#define STATE x28
+constexpr size_t MAX_DISPATCHER_CODE_SIZE = 8192;
 
 Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const DispatcherConfig &config)
-  : FEXCore::CPU::Dispatcher(ctx, config), Arm64Emitter(ctx, MAX_DISPATCHER_CODE_SIZE) {
+  : FEXCore::CPU::Dispatcher(ctx, config), Arm64Emitter(ctx, MAX_DISPATCHER_CODE_SIZE)
+#ifdef VIXL_SIMULATOR
+  , Simulator {&Decoder}
+#endif
+{
+
+#ifdef VIXL_SIMULATOR
+  // Hardcode a 256-bit vector width if we are running in the simulator.
+  Simulator.SetVectorLengthInBits(256);
+#endif
+
   SetAllowAssembler(true);
 
   DispatchPtr = GetCursorAddress<AsmDispatch>();
@@ -178,7 +187,12 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
     ret();
   }
 
+#ifdef VIXL_SIMULATOR
+  // VIXL simulator can't run syscalls.
+  constexpr bool SignalSafeCompile = false;
+#else
   constexpr bool SignalSafeCompile = true;
+#endif
   {
     ExitFunctionLinkerAddress = GetCursorAddress<uint64_t>();
     if (config.StaticRegisterAllocation)
@@ -206,8 +220,12 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
     mov(x0, STATE);
     mov(x1, lr);
 
-    ldr(x3, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionLink));
-    blr(x3);
+    ldr(x2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionLink));
+#ifdef VIXL_SIMULATOR
+    GenerateIndirectRuntimeCall<uintptr_t, void *, void *>(x2);
+#else
+    blr(x2);
+#endif
 
     if (SignalSafeCompile) {
       // Now restore the signal mask
@@ -266,8 +284,11 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
     ldr(x3, &l_CompileBlock);
 
     // X2 contains our guest RIP
+#ifdef VIXL_SIMULATOR
+    GenerateIndirectRuntimeCall<void, void *, uint64_t, void *>(x3);
+#else
     blr(x3); // { CTX, Frame, RIP}
-
+#endif
 
     if (SignalSafeCompile) {
       // Now restore the signal mask
@@ -349,7 +370,11 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
     ldr(x0, &l_CTX);
     mov(x1, STATE);
     ldr(x2, &l_Sleep);
+#ifdef VIXL_SIMULATOR
+    GenerateIndirectRuntimeCall<void, void *, void *>(x2);
+#else
     blr(x2);
+#endif
 
     PauseReturnInstruction = GetCursorAddress<uint64_t>();
     // Fault to start running again
@@ -412,11 +437,14 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
     LUDIVHandlerAddress = GetCursorAddress<uint64_t>();
 
     PushDynamicRegsAndLR();
+    SpillStaticRegs();
 
     ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LUDIV));
-
-    SpillStaticRegs();
+#ifdef VIXL_SIMULATOR
+    GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t>(x3);
+#else
     blr(x3);
+#endif
     FillStaticRegs();
 
     // Result is now in x0
@@ -431,11 +459,14 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
     LDIVHandlerAddress = GetCursorAddress<uint64_t>();
 
     PushDynamicRegsAndLR();
+    SpillStaticRegs();
 
     ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LDIV));
-
-    SpillStaticRegs();
+#ifdef VIXL_SIMULATOR
+    GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t>(x3);
+#else
     blr(x3);
+#endif
     FillStaticRegs();
 
     // Result is now in x0
@@ -450,11 +481,14 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
     LUREMHandlerAddress = GetCursorAddress<uint64_t>();
 
     PushDynamicRegsAndLR();
+    SpillStaticRegs();
 
     ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LUREM));
-
-    SpillStaticRegs();
+#ifdef VIXL_SIMULATOR
+    GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t>(x3);
+#else
     blr(x3);
+#endif
     FillStaticRegs();
 
     // Result is now in x0
@@ -469,11 +503,14 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
     LREMHandlerAddress = GetCursorAddress<uint64_t>();
 
     PushDynamicRegsAndLR();
+    SpillStaticRegs();
 
     ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LREM));
-
-    SpillStaticRegs();
+#ifdef VIXL_SIMULATOR
+    GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t>(x3);
+#else
     blr(x3);
+#endif
     FillStaticRegs();
 
     // Result is now in x0
@@ -504,13 +541,27 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const Dispatche
   }
 }
 
+#ifdef VIXL_SIMULATOR
+void Arm64Dispatcher::ExecuteDispatch(FEXCore::Core::CpuStateFrame *Frame) {
+  Simulator.WriteXRegister(0, reinterpret_cast<int64_t>(Frame));
+  Simulator.RunFrom(reinterpret_cast<Instruction const*>(DispatchPtr));
+}
+
+void Arm64Dispatcher::ExecuteJITCallback(FEXCore::Core::CpuStateFrame *Frame, uint64_t RIP) {
+  Simulator.WriteXRegister(0, reinterpret_cast<int64_t>(Frame));
+  Simulator.WriteXRegister(1, RIP);
+  Simulator.RunFrom(reinterpret_cast<Instruction const*>(CallbackPtr));
+}
+
+#endif
+
 // Used by GenerateGDBPauseCheck, GenerateInterpreterTrampoline, destination buffer is set before use
 static thread_local vixl::aarch64::Assembler emit((uint8_t*)&emit, 1);
 
 size_t Arm64Dispatcher::GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) {
 
   *emit.GetBuffer() = vixl::CodeBuffer(CodeBuffer, MaxGDBPauseCheckSize);
-  
+
   vixl::CodeBufferCheckScope scope(&emit, MaxGDBPauseCheckSize, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert);
 
   aarch64::Label RunBlock;
@@ -546,7 +597,7 @@ size_t Arm64Dispatcher::GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t Gues
 
 size_t Arm64Dispatcher::GenerateInterpreterTrampoline(uint8_t *CodeBuffer) {
   LOGMAN_THROW_AA_FMT(!config.StaticRegisterAllocation, "GenerateInterpreterTrampoline dispatcher does not support SRA");
-  
+
   *emit.GetBuffer() = vixl::CodeBuffer(CodeBuffer, MaxInterpreterTrampolineSize);
 
   vixl::CodeBufferCheckScope scope(&emit, MaxInterpreterTrampolineSize, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert);
@@ -594,7 +645,7 @@ void Arm64Dispatcher::SpillSRA(FEXCore::Core::InternalThreadState *Thread, void
 }
 
 void Arm64Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState *Thread) {
-// Setup dispatcher specific pointers that need to be accessed from JIT code
+  // Setup dispatcher specific pointers that need to be accessed from JIT code
   {
     auto &Common = Thread->CurrentFrame->Pointers.Common;
 

External/FEXCore/Source/Interface/Core/Dispatcher/Arm64Dispatcher.h

@@ -3,6 +3,10 @@
 #include "Interface/Core/ArchHelpers/Arm64Emitter.h"
 #include "Interface/Core/Dispatcher/Dispatcher.h"
 
+#ifdef VIXL_SIMULATOR
+#include <aarch64/simulator-aarch64.h>
+#endif
+
 namespace FEXCore::Context {
 struct Context;
 }
@@ -20,6 +24,11 @@ class Arm64Dispatcher final : public Dispatcher, public Arm64Emitter {
     size_t GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) override;
     size_t GenerateInterpreterTrampoline(uint8_t *CodeBuffer) override;
 
+#ifdef VIXL_SIMULATOR
+  void ExecuteDispatch(FEXCore::Core::CpuStateFrame *Frame) override;
+  void ExecuteJITCallback(FEXCore::Core::CpuStateFrame *Frame, uint64_t RIP) override;
+#endif
+
   protected:
     void SpillSRA(FEXCore::Core::InternalThreadState *Thread, void *ucontext, uint32_t IgnoreMask) override;
 
@@ -29,6 +38,11 @@ class Arm64Dispatcher final : public Dispatcher, public Arm64Emitter {
     uint64_t LDIVHandlerAddress{};
     uint64_t LUREMHandlerAddress{};
     uint64_t LREMHandlerAddress{};
+
+#ifdef VIXL_SIMULATOR
+    vixl::aarch64::Decoder Decoder;
+    vixl::aarch64::Simulator Simulator;
+#endif
 };
 
 }

External/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.h

@@ -32,7 +32,7 @@ struct DispatcherConfig {
 class Dispatcher {
 public:
   virtual ~Dispatcher() = default;
-  
+
   /**
    * @name Dispatch Helper functions
    * @{ */
@@ -75,12 +75,12 @@ public:
 
   static std::unique_ptr<Dispatcher> CreateX86(FEXCore::Context::Context *CTX, const DispatcherConfig &Config);
   static std::unique_ptr<Dispatcher> CreateArm64(FEXCore::Context::Context *CTX, const DispatcherConfig &Config);
-  
-  void ExecuteDispatch(FEXCore::Core::CpuStateFrame *Frame) {
+
+  virtual void ExecuteDispatch(FEXCore::Core::CpuStateFrame *Frame) {
     DispatchPtr(Frame);
   }
 
-  void ExecuteJITCallback(FEXCore::Core::CpuStateFrame *Frame, uint64_t RIP) {
+  virtual void ExecuteJITCallback(FEXCore::Core::CpuStateFrame *Frame, uint64_t RIP) {
     CallbackPtr(Frame, RIP);
   }
 

External/FEXCore/Source/Interface/Core/HostFeatures.cpp

@@ -1,7 +1,7 @@
 #include "Interface/Core/CPUID.h"
 #include <FEXCore/Core/HostFeatures.h>
 
-#ifdef _M_ARM_64
+#if defined(_M_ARM_64) || defined(VIXL_SIMULATOR)
 #include "aarch64/assembler-aarch64.h"
 #include "aarch64/cpu-aarch64.h"
 #include "aarch64/disasm-aarch64.h"
@@ -50,8 +50,12 @@ static uint32_t GetDCZID() {
 
 
 HostFeatures::HostFeatures() {
-#ifdef _M_ARM_64
+#if defined(_M_ARM_64) || defined(VIXL_SIMULATOR)
+#ifdef VIXL_SIMULATOR
+  auto Features = vixl::CPUFeatures::All();
+#else
   auto Features = vixl::CPUFeatures::InferFromOS();
+#endif
   SupportsAES = Features.Has(vixl::CPUFeatures::Feature::kAES);
   SupportsCRC = Features.Has(vixl::CPUFeatures::Feature::kCRC32);
   SupportsAtomics = Features.Has(vixl::CPUFeatures::Feature::kAtomics);
@@ -62,9 +66,24 @@ HostFeatures::HostFeatures() {
   SupportsRCPC = Features.Has(vixl::CPUFeatures::Feature::kRCpc);
   SupportsTSOImm9 = Features.Has(vixl::CPUFeatures::Feature::kRCpcImm);
 
+  Supports3DNow = true;
+  SupportsSSE4A = true;
+#ifdef VIXL_SIMULATOR
+  // Hardcode enable SVE with 256-bit wide registers.
+  SupportsAVX = true;
+#else
   SupportsAVX = Features.Has(vixl::CPUFeatures::Feature::kSVE2) &&
                 vixl::aarch64::CPU::ReadSVEVectorLengthInBits() >= 256;
+#endif
+  SupportsSHA = true;
+  SupportsBMI1 = true;
+  SupportsBMI2 = true;
 
+  if (!SupportsAtomics) {
+    WARN_ONCE_FMT("Host CPU doesn't support atomics. Expect bad performance");
+  }
+
+#ifdef _M_ARM_64
   // We need to get the CPU's cache line size
   // We expect sane targets that have correct cacheline sizes across clusters
   uint64_t CTR;
@@ -74,32 +93,6 @@ HostFeatures::HostFeatures() {
   DCacheLineSize = 4 << ((CTR >> 16) & 0xF);
   ICacheLineSize = 4 << (CTR & 0xF);
 
-  if (!SupportsAtomics) {
-    WARN_ONCE_FMT("Host CPU doesn't support atomics. Expect bad performance");
-  }
-#endif
-#ifdef _M_X86_64
-  Xbyak::util::Cpu Features{};
-  SupportsAES = Features.has(Xbyak::util::Cpu::tAESNI);
-  SupportsCRC = Features.has(Xbyak::util::Cpu::tSSE42);
-  SupportsRAND = Features.has(Xbyak::util::Cpu::tRDRAND) && Features.has(Xbyak::util::Cpu::tRDSEED);
-  SupportsRCPC = true;
-  SupportsTSOImm9 = true;
-  SupportsAVX = true;
-
-  // xbyak doesn't know how to check for CLZero
-  uint32_t eax, ebx, ecx, edx;
-  // First ensure we support a new enough extended CPUID function range
-  __cpuid(0x8000'0000, eax, ebx, ecx, edx);
-  if (eax >= 0x8000'0008U) {
-    // CLZero defined in 8000_00008_EBX[bit 0]
-    __cpuid(0x8000'0008, eax, ebx, ecx, edx);
-    SupportsCLZERO = ebx & 1;
-  }
-
-  SupportsFlushInputsToZero = true;
-  SupportsFloatExceptions = true;
-#else
   // Test if this CPU supports float exception trapping by attempting to enable
   // On unsupported these bits are architecturally defined as RAZ/WI
   constexpr uint32_t ExceptionEnableTraps =
@@ -120,6 +113,39 @@ HostFeatures::HostFeatures() {
   SetFPCR(OriginalFPCR);
 #endif
 
+#endif
+#if defined(_M_X86_64) && !defined(VIXL_SIMULATOR)
+  Xbyak::util::Cpu Features{};
+  SupportsAES = Features.has(Xbyak::util::Cpu::tAESNI);
+  SupportsCRC = Features.has(Xbyak::util::Cpu::tSSE42);
+  SupportsRAND = Features.has(Xbyak::util::Cpu::tRDRAND) && Features.has(Xbyak::util::Cpu::tRDSEED);
+  SupportsRCPC = true;
+  SupportsTSOImm9 = true;
+  Supports3DNow = Features.has(Xbyak::util::Cpu::t3DN) && Features.has(Xbyak::util::Cpu::tE3DN);
+  SupportsSSE4A = Features.has(Xbyak::util::Cpu::tSSE4a);
+  SupportsAVX = true;
+  SupportsSHA = Features.has(Xbyak::util::Cpu::tSHA);
+  SupportsBMI1 = Features.has(Xbyak::util::Cpu::tBMI1);
+  SupportsBMI2 = Features.has(Xbyak::util::Cpu::tBMI2);
+
+  // xbyak doesn't know how to check for CLZero
+  uint32_t eax, ebx, ecx, edx;
+  // First ensure we support a new enough extended CPUID function range
+  __cpuid(0x8000'0000, eax, ebx, ecx, edx);
+  if (eax >= 0x8000'0008U) {
+    // CLZero defined in 8000_00008_EBX[bit 0]
+    __cpuid(0x8000'0008, eax, ebx, ecx, edx);
+    SupportsCLZERO = ebx & 1;
+  }
+
+  SupportsFlushInputsToZero = true;
+  SupportsFloatExceptions = true;
+#endif
+
+#ifdef VIXL_SIMULATOR
+  // simulator doesn't support dc(ZVA)
+  SupportsCLZERO = false;
+#else
   // Check if we can support cacheline clears
   uint32_t DCZID = GetDCZID();
   if ((DCZID & DCZID_DZP_MASK) == 0) {
@@ -129,5 +155,6 @@ HostFeatures::HostFeatures() {
     // This means we can use the instruction
     SupportsCLZERO = DCZID_Bytes == CPUIDEmu::CACHELINE_SIZE;
   }
+#endif
 }
 }

External/FEXCore/Source/Interface/Core/Interpreter/InterpreterDefines.h

@@ -159,21 +159,26 @@
   break;                                            \
   }
 
+struct InterpVector256 {
+  __uint128_t Lower;
+  __uint128_t Upper;
+};
+
 template<typename Res>
 Res GetDest(void* SSAData, FEXCore::IR::OrderedNodeWrapper Op) {
-  auto DstPtr = &reinterpret_cast<__uint128_t*>(SSAData)[Op.ID().Value];
+  auto DstPtr = &reinterpret_cast<InterpVector256*>(SSAData)[Op.ID().Value];
   return reinterpret_cast<Res>(DstPtr);
 }
 
 template<typename Res>
 Res GetDest(void* SSAData, FEXCore::IR::NodeID Op) {
-  auto DstPtr = &reinterpret_cast<__uint128_t*>(SSAData)[Op.Value];
+  auto DstPtr = &reinterpret_cast<InterpVector256*>(SSAData)[Op.Value];
   return reinterpret_cast<Res>(DstPtr);
 }
 
 
 template<typename Res>
 Res GetSrc(void* SSAData, FEXCore::IR::OrderedNodeWrapper Src) {
-  auto DstPtr = &reinterpret_cast<__uint128_t*>(SSAData)[Src.ID().Value];
+  auto DstPtr = &reinterpret_cast<InterpVector256*>(SSAData)[Src.ID().Value];
   return reinterpret_cast<Res>(DstPtr);
 }

External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp

@@ -1,5 +1,6 @@
 #include "Interface/Context/Context.h"
 #include "Interface/Core/CPUID.h"
+#include "InterpreterDefines.h"
 #include "InterpreterOps.h"
 #include "F80Ops.h"
 
@@ -180,13 +181,10 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
   // Move ops
   REGISTER_OP(EXTRACTELEMENTPAIR,     ExtractElementPair);
   REGISTER_OP(CREATEELEMENTPAIR,      CreateElementPair);
-  REGISTER_OP(MOV,                    Mov);
 
   // Vector ops
   REGISTER_OP(VECTORZERO,             VectorZero);
   REGISTER_OP(VECTORIMM,              VectorImm);
-  REGISTER_OP(SPLATVECTOR2,           SplatVector);
-  REGISTER_OP(SPLATVECTOR4,           SplatVector);
   REGISTER_OP(VMOV,                   VMov);
   REGISTER_OP(VAND,                   VAnd);
   REGISTER_OP(VBIC,                   VBic);
@@ -245,8 +243,6 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
   REGISTER_OP(VUSHRS,                 VUShrS);
   REGISTER_OP(VSSHRS,                 VSShrS);
   REGISTER_OP(VINSELEMENT,            VInsElement);
-  REGISTER_OP(VINSSCALARELEMENT,      VInsScalarElement);
-  REGISTER_OP(VEXTRACTELEMENT,        VExtractElement);
   REGISTER_OP(VDUPELEMENT,            VDupElement);
   REGISTER_OP(VEXTR,                  VExtr);
   REGISTER_OP(VSLI,                   VSLI);
@@ -256,7 +252,6 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
   REGISTER_OP(VSHLI,                  VShlI);
   REGISTER_OP(VUSHRNI,                VUShrNI);
   REGISTER_OP(VUSHRNI2,               VUShrNI2);
-  REGISTER_OP(VBITCAST,               VBitcast);
   REGISTER_OP(VSXTL,                  VSXTL);
   REGISTER_OP(VSXTL2,                 VSXTL2);
   REGISTER_OP(VUXTL,                  VUXTL);
@@ -336,23 +331,28 @@ void InterpreterOps::Op_NoOp(FEXCore::IR::IROp_Header *IROp, IROpData *Data, IR:
 void InterpreterOps::InterpretIR(FEXCore::Core::CpuStateFrame *Frame, FEXCore::IR::IRListView const *CurrentIR) {
   volatile void *StackEntry = alloca(0);
 
-  uintptr_t ListSize = CurrentIR->GetSSACount();
+  const uintptr_t ListSize = CurrentIR->GetSSACount();
 
   static_assert(sizeof(FEXCore::IR::IROp_Header) == 4);
   static_assert(sizeof(FEXCore::IR::OrderedNode) == 16);
 
   auto BlockEnd = CurrentIR->GetBlocks().end();
 
-  InterpreterOps::IROpData OpData{};
-  OpData.State = Frame->Thread;
-  OpData.SSAData = alloca(ListSize * 16);
-  OpData.CurrentEntry = Frame->State.rip;
-  OpData.CurrentIR = CurrentIR;
-  OpData.StackEntry = StackEntry;
-  OpData.BlockIterator = CurrentIR->GetBlocks().begin();
+  constexpr size_t ListEntrySizeInBytes = sizeof(InterpVector256);
+  const size_t SSADataSize = ListSize * ListEntrySizeInBytes;
 
-  // Clear them all to zero. Required for Zero-extend semantics
-  memset(OpData.SSAData, 0, ListSize * 16);
+  InterpreterOps::IROpData OpData{
+    .State = Frame->Thread,
+    .CurrentEntry = Frame->State.rip,
+    .CurrentIR = CurrentIR,
+    .StackEntry = StackEntry,
+    .SSAData = alloca(SSADataSize),
+    .BlockResults = {},
+    .BlockIterator = CurrentIR->GetBlocks().begin(),
+  };
+
+  // Clear all SSAData entries to zero. Required for Zero-extend semantics
+  memset(OpData.SSAData, 0, SSADataSize);
 
   while (1) {
     using namespace FEXCore::IR;

External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h

@@ -207,7 +207,6 @@ namespace FEXCore::CPU {
   ///< Vector ops
   DEF_OP(VectorZero);
   DEF_OP(VectorImm);
-  DEF_OP(SplatVector);
   DEF_OP(VMov);
   DEF_OP(VAnd);
   DEF_OP(VBic);
@@ -264,8 +263,6 @@ namespace FEXCore::CPU {
   DEF_OP(VUShrS);
   DEF_OP(VSShrS);
   DEF_OP(VInsElement);
-  DEF_OP(VInsScalarElement);
-  DEF_OP(VExtractElement);
   DEF_OP(VDupElement);
   DEF_OP(VExtr);
   DEF_OP(VSLI);
@@ -275,7 +272,6 @@ namespace FEXCore::CPU {
   DEF_OP(VShlI);
   DEF_OP(VUShrNI);
   DEF_OP(VUShrNI2);
-  DEF_OP(VBitcast);
   DEF_OP(VSXTL);
   DEF_OP(VSXTL2);
   DEF_OP(VUXTL);

External/FEXCore/Source/Interface/Core/Interpreter/MoveOps.cpp

@@ -30,13 +30,6 @@ DEF_OP(CreateElementPair) {
   memcpy(Dst + IROp->ElementSize, Src_Upper, IROp->ElementSize);
 }
 
-DEF_OP(Mov) {
-  auto Op = IROp->C<IR::IROp_Mov>();
-  const uint8_t OpSize = IROp->Size;
-
-  memcpy(GDP, GetSrc<void*>(Data->SSAData, Op->Value), OpSize);
-}
-
 #undef DEF_OP
 
 } // namespace FEXCore::CPU

External/FEXCore/Source/Interface/Core/JIT/Arm64/BranchOps.cpp

@@ -40,7 +40,7 @@ DEF_OP(CallbackReturn) {
   ResetStack();
 
   // We can now lower the ref counter again
-  
+
   ldr(w2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter)));
   sub(w2, w2, 1);
   str(w2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter)));
@@ -197,7 +197,11 @@ DEF_OP(Syscall) {
   ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerFunc)));
   mov(x1, STATE);
   mov(x2, sp);
+#ifdef VIXL_SIMULATOR
+  GenerateIndirectRuntimeCall<uint64_t, void*, void*, void*>(x3);
+#else
   blr(x3);
+#endif
 
   add(sp, sp, SPOffset);
 
@@ -381,7 +385,11 @@ DEF_OP(Thunk) {
 
   auto thunkFn = ThreadState->CTX->ThunkHandler->LookupThunk(Op->ThunkNameHash);
   LoadConstant(x2, (uintptr_t)thunkFn);
+#ifdef VIXL_SIMULATOR
+  GenerateIndirectRuntimeCall<void, void*, void*>(x2);
+#else
   blr(x2);
+#endif
 
   PopDynamicRegsAndLR();
 
@@ -448,7 +456,11 @@ DEF_OP(ThreadRemoveCodeEntry) {
 
   ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.ThreadRemoveCodeEntryFromJIT)));
   SpillStaticRegs();
+#ifdef VIXL_SIMULATOR
+  GenerateIndirectRuntimeCall<void, void*, void*>(x2);
+#else
   blr(x2);
+#endif
   FillStaticRegs();
 
   // Fix the stack and any values that were stepped on
@@ -459,6 +471,7 @@ DEF_OP(CPUID) {
   auto Op = IROp->C<IR::IROp_CPUID>();
 
   PushDynamicRegsAndLR();
+  SpillStaticRegs();
 
   // x0 = CPUID Handler
   // x1 = CPUID Function
@@ -467,10 +480,13 @@ DEF_OP(CPUID) {
   ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDFunction)));
   mov(x1, GetReg<RA_64>(Op->Function.ID()));
   mov(x2, GetReg<RA_64>(Op->Leaf.ID()));
-  SpillStaticRegs();
+#ifdef VIXL_SIMULATOR
+  GenerateIndirectRuntimeCall<__uint128_t, void*, uint64_t, uint64_t>(x3);
+#else
   blr(x3);
-  FillStaticRegs();
+#endif
 
+  FillStaticRegs();
   PopDynamicRegsAndLR();
 
   // Results are in x0, x1

External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp

@@ -94,7 +94,12 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
 
         uxth(w0, GetReg<RA_32>(IROp->Args[0].ID()));
         ldr(x1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<void, uint16_t>(x1);
+#else
         blr(x1);
+#endif
 
         PopDynamicRegsAndLR();
 
@@ -109,7 +114,11 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
 
         fmov(v0.S(), GetSrc(IROp->Args[0].ID()).S()) ;
         ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<__uint128_t, float>(x0);
+#else
         blr(x0);
+#endif
 
         PopDynamicRegsAndLR();
 
@@ -128,7 +137,11 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
 
         mov(v0.D(), GetSrc(IROp->Args[0].ID()).D());
         ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<__uint128_t, double>(x0);
+#else
         blr(x0);
+#endif
 
         PopDynamicRegsAndLR();
 
@@ -153,7 +166,11 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
           mov(w0, GetReg<RA_32>(IROp->Args[0].ID()));
         }
         ldr(x1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<__uint128_t, uint32_t>(x1);
+#else
         blr(x1);
+#endif
 
         PopDynamicRegsAndLR();
 
@@ -174,7 +191,11 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
         umov(w1, GetSrc(IROp->Args[0].ID()).V8H(), 4);
 
         ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<float, uint64_t, uint64_t>(x2);
+#else
         blr(x2);
+#endif
 
         PopDynamicRegsAndLR();
 
@@ -193,7 +214,11 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
         umov(w1, GetSrc(IROp->Args[0].ID()).V8H(), 4);
 
         ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<double, uint64_t, uint64_t>(x2);
+#else
         blr(x2);
+#endif
 
         PopDynamicRegsAndLR();
 
@@ -210,7 +235,11 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
 
         mov(v0.D(), GetSrc(IROp->Args[0].ID()).D());
         ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<double, double>(x0);
+#else
         blr(x0);
+#endif
 
         PopDynamicRegsAndLR();
 
@@ -229,7 +258,11 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
         mov(v0.D(), GetSrc(IROp->Args[0].ID()).D());
         mov(v1.D(), GetSrc(IROp->Args[1].ID()).D());
         ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<double, double, double>(x0);
+#else
         blr(x0);
+#endif
 
         PopDynamicRegsAndLR();
 
@@ -249,7 +282,11 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
         umov(w1, GetSrc(IROp->Args[0].ID()).V8H(), 4);
 
         ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<uint32_t, uint64_t, uint64_t>(x2);
+#else
         blr(x2);
+#endif
 
         PopDynamicRegsAndLR();
 
@@ -267,7 +304,11 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
         umov(w1, GetSrc(IROp->Args[0].ID()).V8H(), 4);
 
         ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<uint32_t, uint64_t, uint64_t>(x2);
+#else
         blr(x2);
+#endif
 
         PopDynamicRegsAndLR();
 
@@ -285,7 +326,11 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
         umov(w1, GetSrc(IROp->Args[0].ID()).V8H(), 4);
 
         ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t>(x2);
+#else
         blr(x2);
+#endif
 
         PopDynamicRegsAndLR();
 
@@ -306,8 +351,11 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
         umov(w3, GetSrc(IROp->Args[1].ID()).V8H(), 4);
 
         ldr(x4, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t, uint64_t>(x4);
+#else
         blr(x4);
-
+#endif
         PopDynamicRegsAndLR();
 
         FillStaticRegs();
@@ -324,7 +372,11 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
         umov(w1, GetSrc(IROp->Args[0].ID()).V8H(), 4);
 
         ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t>(x2);
+#else
         blr(x2);
+#endif
 
         PopDynamicRegsAndLR();
 
@@ -347,7 +399,11 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
         umov(w3, GetSrc(IROp->Args[1].ID()).V8H(), 4);
 
         ldr(x4, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
+#ifdef VIXL_SIMULATOR
+        GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t, uint64_t, uint64_t>(x4);
+#else
         blr(x4);
+#endif
 
         PopDynamicRegsAndLR();
 
@@ -426,6 +482,7 @@ void Arm64JITCore::Op_NoOp(IR::IROp_Header *IROp, IR::NodeID Node) {
 Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread)
   : CPUBackend(Thread, INITIAL_CODE_SIZE, MAX_CODE_SIZE)
   , Arm64Emitter(ctx, 0)
+  , HostSupportsSVE{ctx->HostFeatures.SupportsAVX}
   , CTX {ctx} {
 
   RAPass = Thread->PassManager->GetPass<IR::RegisterAllocationPass>("RA");
@@ -472,7 +529,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern
 
     // Common
     auto &Common = ThreadState->CurrentFrame->Pointers.Common;
-    
+
     Common.PrintValue = reinterpret_cast<uint64_t>(PrintValue);
     Common.PrintVectorValue = reinterpret_cast<uint64_t>(PrintVectorValue);
     Common.ThreadRemoveCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::Context::ThreadRemoveCodeEntryFromJit);
@@ -493,7 +550,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern
 
     // Platform Specific
     auto &AArch64 = ThreadState->CurrentFrame->Pointers.AArch64;
-    
+
     AArch64.LUDIV = reinterpret_cast<uint64_t>(LUDIV);
     AArch64.LDIV = reinterpret_cast<uint64_t>(LDIV);
     AArch64.LUREM = reinterpret_cast<uint64_t>(LUREM);
@@ -510,6 +567,7 @@ void Arm64JITCore::InitializeSignalHandlers(FEXCore::Context::Context *CTX) {
     return Thread->CTX->Dispatcher->HandleSIGILL(Thread, Signal, info, ucontext);
   }, true);
 
+#ifdef _M_ARM_64
   CTX->SignalDelegation->RegisterHostSignalHandler(SIGBUS, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
     if (!Thread->CPUBackend->IsAddressInCodeBuffer(ArchHelpers::Context::GetPc(ucontext))) {
       // Wasn't a sigbus in JIT code
@@ -518,6 +576,7 @@ void Arm64JITCore::InitializeSignalHandlers(FEXCore::Context::Context *CTX) {
 
     return FEXCore::ArchHelpers::Arm64::HandleSIGBUS(Thread->CTX->Config.ParanoidTSO(), Signal, info, ucontext);
   }, true);
+#endif
 }
 
 void Arm64JITCore::EmitDetectionString() {
@@ -529,7 +588,7 @@ void Arm64JITCore::EmitDetectionString() {
 
 void Arm64JITCore::ClearCache() {
   // Get the backing code buffer
-  
+
   auto CodeBuffer = GetEmptyCodeBuffer();
   *GetBuffer() = vixl::CodeBuffer(CodeBuffer->Ptr, CodeBuffer->Size);
   EmitDetectionString();
@@ -724,10 +783,12 @@ void *Arm64JITCore::CompileCode(uint64_t Entry,
   SpillSlots = RAData->SpillSlots();
 
   if (SpillSlots) {
-    if (IsImmAddSub(SpillSlots * 16)) {
-      sub(sp, sp, SpillSlots * 16);
+    const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize;
+
+    if (IsImmAddSub(TotalSpillSlotsSize)) {
+      sub(sp, sp, TotalSpillSlotsSize);
     } else {
-      LoadConstant(x0, SpillSlots * 16);
+      LoadConstant(x0, TotalSpillSlotsSize);
       sub(sp, sp, x0);
     }
   }
@@ -795,14 +856,17 @@ void *Arm64JITCore::CompileCode(uint64_t Entry,
 }
 
 void Arm64JITCore::ResetStack() {
-  if (SpillSlots == 0)
+  if (SpillSlots == 0) {
     return;
+  }
 
-  if (IsImmAddSub(SpillSlots * 16)) {
-    add(sp, sp, SpillSlots * 16);
+  const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize;
+
+  if (IsImmAddSub(TotalSpillSlotsSize)) {
+    add(sp, sp, TotalSpillSlotsSize);
   } else {
    // Too big to fit in a 12bit immediate
-   LoadConstant(x0, SpillSlots * 16);
+   LoadConstant(x0, TotalSpillSlotsSize);
    add(sp, sp, x0);
   }
 }

External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h

@@ -23,16 +23,6 @@ $end_info$
 #include <utility>
 #include <vector>
 
-#define STATE x28
-#define TMP1 x0
-#define TMP2 x1
-#define TMP3 x2
-#define TMP4 x3
-
-#define VTMP1 v1
-#define VTMP2 v2
-#define VTMP3 v3
-
 namespace FEXCore::Core {
   struct InternalThreadState;
 }
@@ -66,6 +56,7 @@ public:
 
 private:
   FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
+  const bool HostSupportsSVE{};
 
   Label *PendingTargetLabel;
   FEXCore::Context::Context *CTX;
@@ -364,13 +355,10 @@ private:
   ///< Move ops
   DEF_OP(ExtractElementPair);
   DEF_OP(CreateElementPair);
-  DEF_OP(Mov);
 
   ///< Vector ops
   DEF_OP(VectorZero);
   DEF_OP(VectorImm);
-  DEF_OP(SplatVector2);
-  DEF_OP(SplatVector4);
   DEF_OP(VMov);
   DEF_OP(VAnd);
   DEF_OP(VBic);
@@ -429,8 +417,6 @@ private:
   DEF_OP(VUShrS);
   DEF_OP(VSShrS);
   DEF_OP(VInsElement);
-  DEF_OP(VInsScalarElement);
-  DEF_OP(VExtractElement);
   DEF_OP(VDupElement);
   DEF_OP(VExtr);
   DEF_OP(VSLI);
@@ -440,7 +426,6 @@ private:
   DEF_OP(VShlI);
   DEF_OP(VUShrNI);
   DEF_OP(VUShrNI2);
-  DEF_OP(VBitcast);
   DEF_OP(VSXTL);
   DEF_OP(VSXTL2);
   DEF_OP(VUXTL);

External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp

@@ -446,9 +446,9 @@ DEF_OP(StoreContextIndexed) {
 }
 
 DEF_OP(SpillRegister) {
-  auto Op = IROp->C<IR::IROp_SpillRegister>();
+  const auto Op = IROp->C<IR::IROp_SpillRegister>();
   const uint8_t OpSize = IROp->Size;
-  const uint32_t SlotOffset = Op->Slot * 16;
+  const uint32_t SlotOffset = Op->Slot * MaxSpillSlotSize;
 
   if (Op->Class == FEXCore::IR::GPRClass) {
     switch (OpSize) {
@@ -468,23 +468,36 @@ DEF_OP(SpillRegister) {
       str(GetReg<RA_64>(Op->Value.ID()), MemOperand(sp, SlotOffset));
       break;
     }
-    default:  LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize);
+    default:
+      LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize);
+      break;
     }
   } else if (Op->Class == FEXCore::IR::FPRClass) {
+    const auto Src = GetSrc(Op->Value.ID());
+
     switch (OpSize) {
     case 4: {
-      str(GetSrc(Op->Value.ID()).S(), MemOperand(sp, SlotOffset));
+      str(Src.S(), MemOperand(sp, SlotOffset));
       break;
     }
     case 8: {
-      str(GetSrc(Op->Value.ID()).D(), MemOperand(sp, SlotOffset));
+      str(Src.D(), MemOperand(sp, SlotOffset));
       break;
     }
     case 16: {
-      str(GetSrc(Op->Value.ID()), MemOperand(sp, SlotOffset));
+      str(Src, MemOperand(sp, SlotOffset));
       break;
     }
-    default:  LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize);
+    case 32: {
+      // TODO: Eliminate ptrue with statically allocated predicate register.
+      ptrue(p7.VnB(), SVE_VL32);
+      mov(TMP3, SlotOffset);
+      st1b(Src.Z().VnB(), p7, SVEMemOperand(sp, TMP3));
+      break;
+    }
+    default:
+      LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize);
+      break;
     }
   } else {
     LOGMAN_MSG_A_FMT("Unhandled SpillRegister class: {}", Op->Class.Val);
@@ -492,9 +505,9 @@ DEF_OP(SpillRegister) {
 }
 
 DEF_OP(FillRegister) {
-  auto Op = IROp->C<IR::IROp_FillRegister>();
-  uint8_t OpSize = IROp->Size;
-  uint32_t SlotOffset = Op->Slot * 16;
+  const auto Op = IROp->C<IR::IROp_FillRegister>();
+  const uint8_t OpSize = IROp->Size;
+  const uint32_t SlotOffset = Op->Slot * MaxSpillSlotSize;
 
   if (Op->Class == FEXCore::IR::GPRClass) {
     switch (OpSize) {
@@ -514,23 +527,36 @@ DEF_OP(FillRegister) {
       ldr(GetReg<RA_64>(Node), MemOperand(sp, SlotOffset));
       break;
     }
-    default:  LOGMAN_MSG_A_FMT("Unhandled FillRegister size: {}", OpSize);
+    default:
+      LOGMAN_MSG_A_FMT("Unhandled FillRegister size: {}", OpSize);
+      break;
     }
   } else if (Op->Class == FEXCore::IR::FPRClass) {
+    const auto Dst = GetDst(Node);
+
     switch (OpSize) {
     case 4: {
-      ldr(GetDst(Node).S(), MemOperand(sp, SlotOffset));
+      ldr(Dst.S(), MemOperand(sp, SlotOffset));
       break;
     }
     case 8: {
-      ldr(GetDst(Node).D(), MemOperand(sp, SlotOffset));
+      ldr(Dst.D(), MemOperand(sp, SlotOffset));
       break;
     }
     case 16: {
-      ldr(GetDst(Node), MemOperand(sp, SlotOffset));
+      ldr(Dst, MemOperand(sp, SlotOffset));
       break;
     }
-    default:  LOGMAN_MSG_A_FMT("Unhandled FillRegister size: {}", OpSize);
+    case 32: {
+      // TODO: Eliminate ptrue with statically allocated predicate register.
+      ptrue(p7.VnB(), SVE_VL32);
+      mov(TMP3, SlotOffset);
+      ld1b(Dst.Z().VnB(), p7.Zeroing(), SVEMemOperand(sp, TMP3));
+      break;
+    }
+    default:
+      LOGMAN_MSG_A_FMT("Unhandled FillRegister size: {}", OpSize);
+      break;
     }
   } else {
     LOGMAN_MSG_A_FMT("Unhandled FillRegister class: {}", Op->Class.Val);

External/FEXCore/Source/Interface/Core/JIT/Arm64/MiscOps.cpp

@@ -119,8 +119,11 @@ DEF_OP(SetRoundingMode) {
 
   mrs(TMP1, FPCR);
 
+  // vixl simulator doesn't support anything beyond ties-to-even rounding
+#ifndef VIXL_SIMULATOR
   // Insert the rounding flags
   bfi(TMP1, TMP2, 22, 2);
+#endif
 
   // Insert the FTZ flag
   lsr(TMP2, Src, 2);
@@ -134,6 +137,7 @@ DEF_OP(Print) {
   auto Op = IROp->C<IR::IROp_Print>();
 
   PushDynamicRegsAndLR();
+  SpillStaticRegs();
 
   if (IsGPR(Op->Value.ID())) {
     mov(x0, GetReg<RA_64>(Op->Value.ID()));
@@ -145,10 +149,10 @@ DEF_OP(Print) {
     fmov(x1, GetSrc(Op->Value.ID()).V1D(), 1);
     ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintVectorValue)));
   }
-  SpillStaticRegs();
-  blr(x3);
-  FillStaticRegs();
 
+  blr(x3);
+
+  FillStaticRegs();
   PopDynamicRegsAndLR();
 }
 

External/FEXCore/Source/Interface/Core/JIT/Arm64/MoveOps.cpp

@@ -68,17 +68,11 @@ DEF_OP(CreateElementPair) {
   }
 }
 
-DEF_OP(Mov) {
-  auto Op = IROp->C<IR::IROp_Mov>();
-  mov(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Value.ID()));
-}
-
 #undef DEF_OP
 void Arm64JITCore::RegisterMoveHandlers() {
 #define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
   REGISTER_OP(EXTRACTELEMENTPAIR, ExtractElementPair);
   REGISTER_OP(CREATEELEMENTPAIR,  CreateElementPair);
-  REGISTER_OP(MOV,                Mov);
 #undef REGISTER_OP
 }
 }

External/FEXCore/Source/Interface/Core/JIT/x86_64/BranchOps.cpp

@@ -33,7 +33,7 @@ namespace FEXCore::CPU {
 DEF_OP(SignalReturn) {
   // Adjust the stack first for a regular return
   if (SpillSlots) {
-    add(rsp, SpillSlots * 16); // + 8 to consume return address
+    add(rsp, SpillSlots * MaxSpillSlotSize); // + 8 to consume return address
   }
 
   jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SignalReturnHandler)]);
@@ -42,7 +42,7 @@ DEF_OP(SignalReturn) {
 DEF_OP(CallbackReturn) {
   // Adjust the stack first for a regular return
   if (SpillSlots) {
-    add(rsp, SpillSlots * 16); // + 8 to consume return address
+    add(rsp, SpillSlots * MaxSpillSlotSize); // + 8 to consume return address
   }
 
   // Make sure to adjust the refcounter so we don't clear the cache now
@@ -71,7 +71,7 @@ DEF_OP(ExitFunction) {
 
 
   if (SpillSlots) {
-    add(rsp, SpillSlots * 16);
+    add(rsp, SpillSlots * MaxSpillSlotSize);
   }
 
   uint64_t NewRIP;

External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp

@@ -25,6 +25,7 @@ $end_info$
 #include <FEXCore/IR/IntrusiveIRList.h>
 #include <FEXCore/IR/RegisterAllocationData.h>
 #include <FEXCore/Utils/Allocator.h>
+#include <FEXCore/Utils/EnumUtils.h>
 #include <FEXCore/Utils/LogManager.h>
 
 #include <algorithm>
@@ -59,32 +60,42 @@ static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) {
 namespace FEXCore::CPU {
 
 void X86JITCore::PushRegs() {
-  sub(rsp, 16 * RAXMM_x.size());
+  const auto AVXRegSize = Core::CPUState::XMM_AVX_REG_SIZE;
+
+  sub(rsp, AVXRegSize * RAXMM_x.size());
   for (size_t i = 0; i < RAXMM_x.size(); ++i) {
-    movaps(ptr[rsp + i * 16], RAXMM_x[i]);
+    vmovups(ptr[rsp + i * AVXRegSize], ToYMM(RAXMM_x[i]));
   }
 
-  for (auto &Reg : RA64)
+  for (const auto &Reg : RA64) {
     push(Reg);
+  }
 
-  auto NumPush = RA64.size();
-  if (NumPush & 1)
-    sub(rsp, 8); // Align
+  const auto NumPush = RA64.size();
+  if ((NumPush & 1) != 0) {
+    // Align
+    sub(rsp, 8);
+  }
 }
 
 void X86JITCore::PopRegs() {
-  auto NumPush = RA64.size();
+  const auto AVXRegSize = Core::CPUState::XMM_AVX_REG_SIZE;
+  const auto NumPush = RA64.size();
 
-  if (NumPush & 1)
-    add(rsp, 8); // Align
-  for (uint32_t i = RA64.size(); i > 0; --i)
-    pop(RA64[i - 1]);
-
-  for (size_t i = 0; i < RAXMM_x.size(); ++i) {
-    movaps(RAXMM_x[i], ptr[rsp + i * 16]);
+  if ((NumPush & 1) != 0) {
+    // Align
+    add(rsp, 8);
   }
 
-  add(rsp, 16 * RAXMM_x.size());
+  for (uint32_t i = RA64.size(); i > 0; --i) {
+    pop(RA64[i - 1]);
+  }
+
+  for (size_t i = 0; i < RAXMM_x.size(); ++i) {
+    vmovups(ToYMM(RAXMM_x[i]), ptr[rsp + i * AVXRegSize]);
+  }
+
+  add(rsp, AVXRegSize * RAXMM_x.size());
 }
 
 void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
@@ -293,7 +304,8 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
       case FABI_UNKNOWN:
       default:
 #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
-        LOGMAN_MSG_A_FMT("Unhandled IR Fallback ABI: {} {}", FEXCore::IR::GetName(IROp->Op), Info.ABI);
+        LOGMAN_MSG_A_FMT("Unhandled IR Fallback ABI: {} {}",
+                         IR::GetName(IROp->Op), ToUnderlying(Info.ABI));
 #endif
         break;
     }
@@ -597,7 +609,7 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
   SpillSlots = RAData->SpillSlots();
 
   if (SpillSlots) {
-    sub(rsp, SpillSlots * 16);
+    sub(rsp, SpillSlots * MaxSpillSlotSize);
   }
 
 #ifdef BLOCKSTATS

External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h

@@ -181,6 +181,10 @@ private:
   [[nodiscard]] Xbyak::Xmm GetSrc(IR::NodeID Node) const;
   [[nodiscard]] Xbyak::Xmm GetDst(IR::NodeID Node) const;
 
+  [[nodiscard]] static Xbyak::Ymm ToYMM(const Xbyak::Xmm& xmm) {
+    return Xbyak::Ymm{xmm.getIdx()};
+  }
+
   [[nodiscard]] Xbyak::RegExp GenerateModRM(Xbyak::Reg Base, IR::OrderedNodeWrapper Offset,
                                             IR::MemOffsetType OffsetType, uint8_t OffsetScale) const;
 
@@ -205,7 +209,7 @@ private:
   * @brief Current guest RIP entrypoint
   */
   uint8_t *GuestEntry{};
-  
+
   using SetCC = void (X86JITCore::*)(const Operand& op);
   using CMovCC = void (X86JITCore::*)(const Reg& reg, const Operand& op);
   using JCC = void (X86JITCore::*)(const Label& label, LabelType type);
@@ -362,12 +366,10 @@ private:
   ///< Move ops
   DEF_OP(ExtractElementPair);
   DEF_OP(CreateElementPair);
-  DEF_OP(Mov);
 
   ///< Vector ops
   DEF_OP(VectorZero);
   DEF_OP(VectorImm);
-  DEF_OP(SplatVector);
   DEF_OP(VMov);
   DEF_OP(VAnd);
   DEF_OP(VBic);
@@ -426,8 +428,6 @@ private:
   DEF_OP(VUShrS);
   DEF_OP(VSShrS);
   DEF_OP(VInsElement);
-  DEF_OP(VInsScalarElement);
-  DEF_OP(VExtractElement);
   DEF_OP(VDupElement);
   DEF_OP(VExtr);
   DEF_OP(VSLI);
@@ -437,7 +437,6 @@ private:
   DEF_OP(VShlI);
   DEF_OP(VUShrNI);
   DEF_OP(VUShrNI2);
-  DEF_OP(VBitcast);
   DEF_OP(VSXTL);
   DEF_OP(VSXTL2);
   DEF_OP(VUXTL);

External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp

@@ -333,10 +333,10 @@ DEF_OP(StoreContextIndexed) {
 }
 
 DEF_OP(SpillRegister) {
-  auto Op = IROp->C<IR::IROp_SpillRegister>();
-  uint8_t OpSize = IROp->Size;
+  const auto Op = IROp->C<IR::IROp_SpillRegister>();
+  const uint8_t OpSize = IROp->Size;
+  const uint32_t SlotOffset = Op->Slot * MaxSpillSlotSize;
 
-  uint32_t SlotOffset = Op->Slot * 16;
   if (Op->Class == FEXCore::IR::GPRClass) {
     switch (OpSize) {
       case 1: {
@@ -355,36 +355,44 @@ DEF_OP(SpillRegister) {
         mov(qword [rsp + SlotOffset], GetSrc<RA_64>(Op->Value.ID()));
         break;
       }
-      default:  LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize);
+      default:
+        LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize);
+        break;
     }
   } else if (Op->Class == FEXCore::IR::FPRClass) {
+    const auto Src = GetSrc(Op->Value.ID());
+
     switch (OpSize) {
       case 4: {
-        movss(dword [rsp + SlotOffset], GetSrc(Op->Value.ID()));
+        movss(dword [rsp + SlotOffset], Src);
         break;
       }
       case 8: {
-        movsd(qword [rsp + SlotOffset], GetSrc(Op->Value.ID()));
+        movsd(qword [rsp + SlotOffset], Src);
         break;
       }
       case 16: {
-        movaps(xword [rsp + SlotOffset], GetSrc(Op->Value.ID()));
+        movaps(xword [rsp + SlotOffset], Src);
         break;
       }
-      default:  LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize);
+      case 32: {
+        vmovaps(yword [rsp + SlotOffset], ToYMM(Src));
+        break;
+      }
+      default:
+        LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize);
+        break;
     }
   } else {
     LOGMAN_MSG_A_FMT("Unhandled SpillRegister class: {}", Op->Class.Val);
   }
-
-
 }
 
 DEF_OP(FillRegister) {
-  auto Op = IROp->C<IR::IROp_FillRegister>();
-  uint8_t OpSize = IROp->Size;
+  const auto Op = IROp->C<IR::IROp_FillRegister>();
+  const uint8_t OpSize = IROp->Size;
+  const uint32_t SlotOffset = Op->Slot * MaxSpillSlotSize;
 
-  uint32_t SlotOffset = Op->Slot * 16;
   if (Op->Class == FEXCore::IR::GPRClass) {
     switch (OpSize) {
       case 1: {
@@ -403,23 +411,33 @@ DEF_OP(FillRegister) {
         mov(GetDst<RA_64>(Node), qword [rsp + SlotOffset]);
         break;
       }
-      default:  LOGMAN_MSG_A_FMT("Unhandled FillRegister size: {}", OpSize);
+      default:
+        LOGMAN_MSG_A_FMT("Unhandled FillRegister size: {}", OpSize);
+        break;
     }
   } else if (Op->Class == FEXCore::IR::FPRClass) {
+    const auto Dst = GetDst(Node);
+
     switch (OpSize) {
       case 4: {
-        movss(GetDst(Node), dword [rsp + SlotOffset]);
+        movss(Dst, dword [rsp + SlotOffset]);
         break;
       }
       case 8: {
-        movsd(GetDst(Node), qword [rsp + SlotOffset]);
+        movsd(Dst, qword [rsp + SlotOffset]);
         break;
       }
       case 16: {
-        movaps(GetDst(Node), xword [rsp + SlotOffset]);
+        movaps(Dst, xword [rsp + SlotOffset]);
         break;
       }
-      default:  LOGMAN_MSG_A_FMT("Unhandled FillRegister size: {}", OpSize);
+      case 32: {
+        vmovaps(ToYMM(Dst), yword [rsp + SlotOffset]);
+        break;
+      }
+      default:
+        LOGMAN_MSG_A_FMT("Unhandled FillRegister size: {}", OpSize);
+        break;
     }
   } else {
     LOGMAN_MSG_A_FMT("Unhandled FillRegister class: {}", Op->Class.Val);

External/FEXCore/Source/Interface/Core/JIT/x86_64/MiscOps.cpp

@@ -47,7 +47,7 @@ DEF_OP(Break) {
   auto Op = IROp->C<IR::IROp_Break>();
 
   if (SpillSlots) {
-    add(rsp, SpillSlots * 16);
+    add(rsp, SpillSlots * MaxSpillSlotSize);
   }
 
   mov(byte [STATE + offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData.FaultToTopAndGeneratedException)], 1);

External/FEXCore/Source/Interface/Core/JIT/x86_64/MoveOps.cpp

@@ -73,17 +73,11 @@ DEF_OP(CreateElementPair) {
   }
 }
 
-DEF_OP(Mov) {
-  auto Op = IROp->C<IR::IROp_Mov>();
-  mov (GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Value.ID()));
-}
-
 #undef DEF_OP
 void X86JITCore::RegisterMoveHandlers() {
 #define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
   REGISTER_OP(EXTRACTELEMENTPAIR, ExtractElementPair);
   REGISTER_OP(CREATEELEMENTPAIR,  CreateElementPair);
-  REGISTER_OP(MOV,                Mov);
 #undef REGISTER_OP
 }
 }

External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp

@@ -126,10 +126,20 @@ void OpDispatchBuilder::ThunkOp(OpcodeArgs) {
   const uint8_t GPRSize = CTX->GetGPRSize();
   uint8_t *sha256 = (uint8_t *)(Op->PC + 2);
 
-  _Thunk(
-    _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)),
-    *reinterpret_cast<SHA256Sum*>(sha256)
-  );
+  if (CTX->Config.Is64BitMode) {
+    // x86-64 ABI puts the function argument in RDI
+    _Thunk(
+      _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)),
+      *reinterpret_cast<SHA256Sum*>(sha256)
+    );
+  }
+  else {
+    // x86 fastcall ABI puts the function argument in ECX
+    _Thunk(
+      _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RCX)),
+      *reinterpret_cast<SHA256Sum*>(sha256)
+    );
+  }
 
   auto Constant = _Constant(GPRSize);
   auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset);

External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp

@@ -74,13 +74,12 @@ void OpDispatchBuilder::MOVLPOp(OpcodeArgs) {
     // xmm, xmm is movhlps special case
     if (Op->Src[0].IsGPR()) {
       OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, 8, 16);
-      Src = _VExtractElement(16, 8, Src, 1);
-      auto Result = _VInsScalarElement(16, 8, 0, Dest, Src);
+      auto Result = _VInsElement(16, 8, 0, 1, Dest, Src);
       StoreResult_WithOpSize(FPRClass, Op, Op->Dest, Result, 16, 16);
     }
     else {
       OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, 8, 16);
-      auto Result = _VInsScalarElement(16, 8, 0, Dest, Src);
+      auto Result = _VInsElement(16, 8, 0, 0, Dest, Src);
       StoreResult_WithOpSize(FPRClass, Op, Op->Dest, Result, 8, 16);
     }
   }
@@ -112,7 +111,7 @@ void OpDispatchBuilder::MOVSSOp(OpcodeArgs) {
     // MOVSS xmm1, xmm2
     OrderedNode *Dest = LoadSource_WithOpSize(FPRClass, Op, Op->Dest, 16, Op->Flags, -1);
     OrderedNode *Src = LoadSource_WithOpSize(FPRClass, Op, Op->Src[0], 4, Op->Flags, -1);
-    auto Result = _VInsScalarElement(16, 4, 0, Dest, Src);
+    auto Result = _VInsElement(16, 4, 0, 0, Dest, Src);
     StoreResult(FPRClass, Op, Result, -1);
   }
   else if (Op->Dest.IsGPR()) {
@@ -133,7 +132,7 @@ void OpDispatchBuilder::MOVSDOp(OpcodeArgs) {
     // xmm1[63:0] <- xmm2[63:0]
     OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
     OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
-    auto Result = _VInsScalarElement(16, 8, 0, Dest, Src);
+    auto Result = _VInsElement(16, 8, 0, 0, Dest, Src);
     StoreResult(FPRClass, Op, Result, -1);
   }
   else if (Op->Dest.IsGPR()) {
@@ -335,7 +334,7 @@ void OpDispatchBuilder::VectorScalarALUOp(OpcodeArgs) {
 
   if (Size != ElementSize) {
     // Insert the lower bits
-    Result = _VInsScalarElement(Size, ElementSize, 0, Dest, Result);
+    Result = _VInsElement(Size, ElementSize, 0, 0, Dest, Result);
   }
 
   StoreResult(FPRClass, Op, Result, -1);
@@ -381,7 +380,7 @@ void OpDispatchBuilder::VectorUnaryOp(OpcodeArgs) {
 
   if constexpr (Scalar) {
     // Insert the lower bits
-    auto Result = _VInsScalarElement(GetSrcSize(Op), ElementSize, 0, Dest, ALUOp);
+    auto Result = _VInsElement(GetSrcSize(Op), ElementSize, 0, 0, Dest, ALUOp);
     StoreResult(FPRClass, Op, Result, -1);
   }
   else {
@@ -978,7 +977,8 @@ void OpDispatchBuilder::PAVGOp<2>(OpcodeArgs);
 
 void OpDispatchBuilder::MOVDDUPOp(OpcodeArgs) {
   OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
-  OrderedNode *Res =  _SplatVector2(Src);
+  OrderedNode *Res = _VDupElement(16, GetSrcSize(Op), Src, 0);
+
   StoreResult(FPRClass, Op, Res, -1);
 }
 
@@ -992,7 +992,7 @@ void OpDispatchBuilder::CVTGPR_To_FPR(OpcodeArgs) {
 
   OrderedNode *Dest = LoadSource_WithOpSize(FPRClass, Op, Op->Dest, 16, Op->Flags, -1);
 
-  Src = _VInsScalarElement(16, DstElementSize, 0, Dest, Src);
+  Src = _VInsElement(16, DstElementSize, 0, 0, Dest, Src);
 
   StoreResult(FPRClass, Op, Src, -1);
 }
@@ -1091,7 +1091,7 @@ void OpDispatchBuilder::Scalar_CVT_Float_To_Float(OpcodeArgs) {
   OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
 
   Src = _Float_FToF(DstElementSize, SrcElementSize, Src);
-  Src = _VInsScalarElement(16, DstElementSize, 0, Dest, Src);
+  Src = _VInsElement(16, DstElementSize, 0, 0, Dest, Src);
 
   StoreResult(FPRClass, Op, Src, -1);
 }
@@ -1281,7 +1281,7 @@ void OpDispatchBuilder::VFCMPOp(OpcodeArgs) {
 
   if constexpr (Scalar) {
     // Insert the lower bits
-    Result = _VInsScalarElement(GetDstSize(Op), ElementSize, 0, Dest, Result);
+    Result = _VInsElement(GetDstSize(Op), ElementSize, 0, 0, Dest, Result);
   }
 
   StoreResult(FPRClass, Op, Result, -1);
@@ -1714,8 +1714,8 @@ void OpDispatchBuilder::PFNACCOp(OpcodeArgs) {
 
   OrderedNode *ResSubSrc{};
   OrderedNode *ResSubDest{};
-  auto UpperSubDest = _VExtractElement(Size, 4, Dest, 1);
-  auto UpperSubSrc = _VExtractElement(Size, 4, Src, 1);
+  auto UpperSubDest = _VDupElement(Size, 4, Dest, 1);
+  auto UpperSubSrc = _VDupElement(Size, 4, Src, 1);
 
   ResSubDest = _VFSub(4, 4, Dest, UpperSubDest);
   ResSubSrc = _VFSub(4, 4, Src, UpperSubSrc);
@@ -1733,7 +1733,7 @@ void OpDispatchBuilder::PFPNACCOp(OpcodeArgs) {
 
   OrderedNode *ResAdd{};
   OrderedNode *ResSub{};
-  auto UpperSubDest = _VExtractElement(Size, 4, Dest, 1);
+  auto UpperSubDest = _VDupElement(Size, 4, Dest, 1);
 
   ResSub = _VFSub(4, 4, Dest, UpperSubDest);
   ResAdd = _VFAddP(Size, 4, Src, Src);
@@ -1866,8 +1866,6 @@ void OpDispatchBuilder::PMADDWD(OpcodeArgs) {
 
   if (Size == 8) {
     Size <<= 1;
-    Src1 = _VBitcast(Size, 2, Src1);
-    Src2 = _VBitcast(Size, 2, Src2);
   }
 
   auto Src1_L = _VSXTL(Size, 2, Src1);  // [15:0 ], [31:16], [32:47 ], [63:48  ]
@@ -1954,9 +1952,6 @@ void OpDispatchBuilder::PMULHW(OpcodeArgs) {
 
   OrderedNode *Res{};
   if (Size == 8) {
-    Dest = _VBitcast(Size * 2, 2, Dest);
-    Src = _VBitcast(Size * 2, 2, Src);
-
     // Implementation is more efficient for 8byte registers
     if (Signed)
       Res = _VSMull(Size * 2, 2, Dest, Src);
@@ -2333,7 +2328,7 @@ void OpDispatchBuilder::VectorRound(OpcodeArgs) {
   if constexpr (Scalar) {
     // Insert the lower bits
     OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
-    auto Result = _VInsScalarElement(GetDstSize(Op), ElementSize, 0, Dest, Src);
+    auto Result = _VInsElement(GetDstSize(Op), ElementSize, 0, 0, Dest, Src);
     StoreResult(FPRClass, Op, Result, -1);
   }
   else {

External/FEXCore/Source/Interface/Core/OpcodeDispatcher/X87.cpp

@@ -1185,7 +1185,7 @@ void OpDispatchBuilder::X87FNSAVE(OpcodeArgs) {
   // upper 16 bits [79:64]
   _StoreMem(FPRClass, 8, ST0Location, data, 1);
   ST0Location = _Add(ST0Location, _Constant(8));
-  auto topBytes = _VExtractElement(16, 2, data, 4);
+  auto topBytes = _VDupElement(16, 2, data, 4);
   _StoreMem(FPRClass, 2, ST0Location, topBytes, 1);
 
   // reset to default

External/FEXCore/Source/Interface/Core/OpcodeDispatcher/X87F64.cpp

@@ -996,7 +996,7 @@ void OpDispatchBuilder::X87FNSAVEF64(OpcodeArgs) {
   // upper 16 bits [79:64]
   _StoreMem(FPRClass, 8, ST0Location, data, 1);
   ST0Location = _Add(ST0Location, _Constant(8));
-  auto topBytes = _VExtractElement(16, 2, data, 4);
+  auto topBytes = _VDupElement(16, 2, data, 4);
   _StoreMem(FPRClass, 2, ST0Location, topBytes, 1);
 
   // reset to default

External/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp

@@ -28,7 +28,9 @@ $end_info$
 #include <string>
 #include <utility>
 
+#ifdef ENABLE_JEMALLOC
 #include "jemalloc/jemalloc.h"
+#endif
 
 struct LoadlibArgs {
     const char *Name;
@@ -137,7 +139,7 @@ namespace FEXCore {
                 // sha256(fex:allocate_host_trampoline_for_guest_function)
                 { 0x9b, 0xb2, 0xf4, 0xb4, 0x83, 0x7d, 0x28, 0x93, 0x40, 0xcb, 0xf4, 0x7a, 0x0b, 0x47, 0x85, 0x87, 0xf9, 0xbc, 0xb5, 0x27, 0xca, 0xa6, 0x93, 0xa5, 0xc0, 0x73, 0x27, 0x24, 0xae, 0xc8, 0xb8, 0x5a },
                 &AllocateHostTrampolineForGuestFunction
-            }
+            },
         };
 
         // Can't be a string_view. We need to keep a copy of the library name in-case string_view pointer goes away.
@@ -241,12 +243,17 @@ namespace FEXCore {
          * and host heap pointers.
          */
         static void IsHostHeapAllocation(void* ArgsRV) {
+#ifdef ENABLE_JEMALLOC
             struct ArgsRV_t {
                 void* ptr;
                 bool rv;
             } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
 
             args->rv = je_is_known_allocation(args->ptr);
+#else
+            // Thunks usage without jemalloc isn't supported
+            ERROR_AND_DIE_FMT("Unsupported: Thunks querying for host heap allocation information");
+#endif
         }
 
         static void LoadLib(void *ArgsV) {
@@ -313,7 +320,7 @@ namespace FEXCore {
             }
         }
 
-        ThunkedFunction* LookupThunk(const IR::SHA256Sum &sha256) {
+        ThunkedFunction* LookupThunk(const IR::SHA256Sum &sha256) override {
 
             std::shared_lock lk(ThunksMutex);
 
@@ -326,13 +333,19 @@ namespace FEXCore {
             }
         }
 
-        void RegisterTLSState(FEXCore::Core::InternalThreadState *Thread) {
+        void RegisterTLSState(FEXCore::Core::InternalThreadState *Thread) override {
             ::Thread = Thread;
         }
+
+        void AppendThunkDefinitions(std::vector<FEXCore::IR::ThunkDefinition> const& Definitions) override {
+          for (auto & Definition : Definitions) {
+            Thunks.emplace(Definition.Sum, Definition.ThunkFunction);
+          }
+        }
     };
 
     ThunkHandler* ThunkHandler::Create() {
-        return new ThunkHandler_impl();
+      return new ThunkHandler_impl();
     }
 
     /**
@@ -358,7 +371,7 @@ namespace FEXCore {
      * is to allocate the trampoline for a given GuestTarget/GuestUnpacker on the guest-side,
      * and provide the HostPacker host-side.
      */
-    __attribute__((visibility("default")))
+    FEX_DEFAULT_VISIBILITY
     HostToGuestTrampolinePtr* MakeHostTrampolineForGuestFunction(void* HostPacker, uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
       LOGMAN_THROW_AA_FMT(GuestTarget, "Tried to create host-trampoline to null pointer guest function");
 
@@ -416,7 +429,7 @@ namespace FEXCore {
       return HostTrampoline;
     }
 
-    __attribute__((visibility("default")))
+    FEX_DEFAULT_VISIBILITY
     void FinalizeHostTrampolineForGuestFunction(HostToGuestTrampolinePtr* TrampolineAddress, void* HostPacker) {
       auto& Trampoline = GetInstanceInfo(TrampolineAddress);
 

External/FEXCore/Source/Interface/HLE/Thunks/Thunks.h

@@ -6,6 +6,10 @@ $end_info$
 
 #pragma once
 
+#include <FEXCore/IR/IR.h>
+
+#include <vector>
+
 namespace FEXCore::Context {
   struct Context;
 }
@@ -28,5 +32,7 @@ namespace FEXCore {
         virtual ~ThunkHandler() { }
 
         static ThunkHandler* Create();
+
+        virtual void AppendThunkDefinitions(std::vector<FEXCore::IR::ThunkDefinition> const& Definitions) = 0;
     };
 };

External/FEXCore/Source/Interface/IR/IR.json

@@ -302,10 +302,6 @@
       }
     },
     "Moves": {
-      "GPR = Mov GPR:$Value": {
-        "DestSize": "GetOpSize(_Value)"
-      },
-
       "GPR = ExtractElementPair GPRPair:$Pair, u8:$Element": {
         "Desc": ["Extracts a register for the register pair"],
         "DestSize": "GetOpSize(_Pair) >> 1"
@@ -364,11 +360,11 @@
       },
 
       "StoreContext u8:#ByteSize, RegisterClass:$Class, SSA:$Value, u32:$Offset": {
-				"Desc": ["Stores a value to the context with offset",
-								 "Ctx[Offset] = Value",
-								 "Zero Extends if value's type is too small",
-								 "Truncates if value's type is too large"
-								],
+        "Desc": ["Stores a value to the context with offset",
+                 "Ctx[Offset] = Value",
+                 "Zero Extends if value's type is too small",
+                 "Truncates if value's type is too large"
+                ],
         "HasSideEffects": true,
         "DestSize": "ByteSize",
         "EmitValidation": [
@@ -845,27 +841,25 @@
         "DestSize": "std::max<uint8_t>(4, std::max<uint8_t>(GetOpSize(_TrueVal), GetOpSize(_FalseVal)))"
       },
       "GPR = Extr GPR:$Upper, GPR:$Lower, u8:$LSB": {
-				"Desc": ["Concats the two GPRs to create a value that is the size of the full two GPRs",
-								 "It then extracts a bitfield width that size of a GPR from the LSB",
-								 "Valid LSB range is 0-31 for 32bit and 0-63 for 64bit",
-								 "<Size * 2> ConcatValue = $Upper:$Lower",
-								 "Result = ConcatValue<LSB+Size - 1: LSB>"
-								]
+        "Desc": ["Concats the two GPRs to create a value that is the size of the full two GPRs",
+                 "It then extracts a bitfield width that size of a GPR from the LSB",
+                 "Valid LSB range is 0-31 for 32bit and 0-63 for 64bit",
+                 "<Size * 2> ConcatValue = $Upper:$Lower",
+                 "Result = ConcatValue<LSB+Size - 1: LSB>"
+                ]
       },
       "GPR = PDep GPR:$Input, GPR:$Mask": {
-				"Desc": [
-					"Performs a parallel bit deposit.",
-					"Takes the contiguous low-order bits and deposits them into",
-					"the destination at the locations specified by the Mask."
-				]
+        "Desc": ["Performs a parallel bit deposit.",
+                 "Takes the contiguous low-order bits and deposits them into",
+                 "the destination at the locations specified by the Mask."
+                ]
       },
 
       "GPR = PExt GPR:$Input, GPR:$Mask": {
-				"Desc": [
-					"Performs a parallel bit extract.",
-					"Each bit set in the mask will select the corresponding bit in the Input",
-					"and transfers them to the lower contiguous bits in the destination."
-				]
+        "Desc": ["Performs a parallel bit extract.",
+                 "Each bit set in the mask will select the corresponding bit in the Input",
+                 "and transfers them to the lower contiguous bits in the destination."
+                ]
       },
 
       "GPR = LDiv GPR:$Lower, GPR:$Upper, GPR:$Divisor": {
@@ -924,15 +918,6 @@
       }
     },
     "Vector": {
-      "FPR = SplatVector2 FPR:$Scalar": {
-        "NumElements": "2",
-        "DestSize": "GetOpSize(_Scalar) * 2"
-      },
-      "FPR = SplatVector4 FPR:$Scalar": {
-        "NumElements": "4",
-        "DestSize": "GetOpSize(_Scalar) * 4"
-      },
-
       "FPR = VMov u8:#RegisterSize, FPR:$Source": {
         "Desc" : ["Copy vector register",
                   "When Register size is smaller than Source register size,",
@@ -941,12 +926,6 @@
         "DestSize": "RegisterSize"
       },
 
-      "FPR = VBitcast u8:#RegisterSize, u8:#ElementSize, FPR:$Source": {
-        "Desc": ["Workaround for issue with LLVM breaking when loading scalar elements to vectors"],
-        "DestSize": "RegisterSize",
-        "NumElements": "RegisterSize / ElementSize"
-      },
-
       "FPR = VectorZero u8:#RegisterSize": {
         "Desc": ["Generates a vector zero",
                  "Useful to generate a zero vector without any previous dependencies"
@@ -1038,9 +1017,6 @@
         "DestSize": "RegisterSize",
         "NumElements": "RegisterSize / ElementSize"
       },
-      "FPR = VExtractElement u8:#RegisterSize, u8:#ElementSize, FPR:$Vector, u8:$Index": {
-        "DestSize": "ElementSize"
-      },
       "FPR = VDupElement u8:#RegisterSize, u8:#ElementSize, FPR:$Vector, u8:$Index": {
         "Desc": ["Duplicates one element from the source register across the whole register"],
         "DestSize": "RegisterSize",
@@ -1124,9 +1100,9 @@
       },
 
       "FPR = VRev64 u8:#RegisterSize, u8:#ElementSize, FPR:$Vector": {
-				"Desc" : ["Reverses elements in 64-bit halfwords",
-									"Available element size: 1byte, 2 byte, 4 byte"
-								 ],
+        "Desc" : ["Reverses elements in 64-bit halfwords",
+                  "Available element size: 1byte, 2 byte, 4 byte"
+                 ],
         "DestSize": "RegisterSize",
         "NumElements": "RegisterSize / ElementSize"
       },
@@ -1320,10 +1296,6 @@
         "DestSize": "RegisterSize",
         "NumElements": "RegisterSize / ElementSize"
       },
-      "FPR = VInsScalarElement u8:#RegisterSize, u8:#ElementSize, u8:$DestIdx, FPR:$DestVector, FPR:$SrcScalar": {
-        "DestSize": "RegisterSize",
-        "NumElements": "RegisterSize / ElementSize"
-      },
       "FPR = VInsGPR u8:#RegisterSize, u8:#ElementSize, u8:$DestIdx, FPR:$DestVector, GPR:$Src": {
         "DestSize": "RegisterSize",
         "NumElements": "RegisterSize / ElementSize"
@@ -1587,9 +1559,9 @@
         "DestSize": "16"
       },
       "GPR = F80Cmp FPR:$X80Src1, FPR:$X80Src2, u32:$Flags": {
-				"Desc": ["Does a scalar unordered compare and stores the asked for flags in to a GPR",
-								 "Ordering flag result is true if either float input is NaN"
-								],
+        "Desc": ["Does a scalar unordered compare and stores the asked for flags in to a GPR",
+                 "Ordering flag result is true if either float input is NaN"
+                ],
         "DestSize": "4"
       },
       "FPR = F80BCDLoad FPR:$X80Src": {

External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp

@@ -6,7 +6,7 @@ $end_info$
 */
 
 
-#if defined(_M_ARM_64)
+#if JIT_ARM64
 //aarch64 heuristics
 #include "aarch64/assembler-aarch64.h"
 #include "aarch64/cpu-aarch64.h"
@@ -45,20 +45,20 @@ uint64_t getMask(IROp_Header* Op) {
   return (~0ULL) >> (64 - NumBits);
 }
 
-#ifdef _M_X86_64
-// very lazy heuristics
-static bool IsImmLogical(uint64_t imm, unsigned width) { return imm < 0x8000'0000; }
-static bool IsImmAddSub(uint64_t imm) { return imm < 0x8000'0000; }
-static bool IsMemoryScale(uint64_t Scale, uint8_t AccessSize) {
-  return Scale  == 1 || Scale  == 2 || Scale  == 4 || Scale  == 8;
-}
-#elif defined(_M_ARM_64)
+#if JIT_ARM64
 //aarch64 heuristics
 static bool IsImmLogical(uint64_t imm, unsigned width) { if (width < 32) width = 32; return vixl::aarch64::Assembler::IsImmLogical(imm, width); }
 static bool IsImmAddSub(uint64_t imm) { return vixl::aarch64::Assembler::IsImmAddSub(imm); }
 static bool IsMemoryScale(uint64_t Scale, uint8_t AccessSize) {
   return Scale  == AccessSize;
 }
+#elif JIT_X86_64
+// very lazy heuristics
+static bool IsImmLogical(uint64_t imm, unsigned width) { return imm < 0x8000'0000; }
+static bool IsImmAddSub(uint64_t imm) { return imm < 0x8000'0000; }
+static bool IsMemoryScale(uint64_t Scale, uint8_t AccessSize) {
+  return Scale  == 1 || Scale  == 2 || Scale  == 4 || Scale  == 8;
+}
 #else
 #error No inline constant heuristics for this target
 #endif

External/FEXCore/Source/Interface/IR/Passes/DeadContextStoreElimination.cpp

@@ -449,7 +449,6 @@ void RCLSE::CalculateControlFlowInfo(FEXCore::IR::IREmitter *IREmit) {
  *   %ssa26 i128 = LoadMem %ssa25 i64, 0x10
  *   (%%ssa27) StoreContext %ssa26 i128, 0x10, 0xb0
  *   %ssa28 i128 = LoadContext 0x10, 0x90
- *   %ssa29 i128 = VBitcast %ssa26 i128
  *
  * eg.
  * 		%ssa6 i128 = LoadContext 0x10, 0x90
@@ -462,13 +461,11 @@ void RCLSE::CalculateControlFlowInfo(FEXCore::IR::IREmitter *IREmit) {
  * eg.
  *   (%%ssa189) StoreContext %ssa188 i128, 0x10, 0xa0
  *   %ssa190 i128 = LoadContext 0x10, 0x90
- *   %ssa191 i128 = VBitcast %ssa188 i128
- *   %ssa192 i128 = VAdd %ssa191 i128, %ssa190 i128, 0x10, 0x4
+ *   %ssa192 i128 = VAdd %ssa188 i128, %ssa190 i128, 0x10, 0x4
  *   (%%ssa193) StoreContext %ssa192 i128, 0x10, 0xa0
  * Converts to
  *   %ssa173 i128 = LoadContext 0x10, 0x90
- *   %ssa174 i128 = VBitcast %ssa172 i128
- *   %ssa175 i128 = VAdd %ssa174 i128, %ssa173 i128, 0x10, 0x4
+ *   %ssa175 i128 = VAdd %ssa172 i128, %ssa173 i128, 0x10, 0x4
  *   (%%ssa176) StoreContext %ssa175 i128, 0x10, 0xa0
 
  */

External/FEXCore/Source/Utils/Allocator.cpp

@@ -6,6 +6,10 @@
 #include <FEXHeaderUtils/TypeDefines.h>
 
 #include <array>
+#include <asm-generic/errno-base.h>
+#include <cctype>
+#include <cstdio>
+#include <fcntl.h>
 #include <sys/mman.h>
 #include <sys/user.h>
 #ifdef ENABLE_JEMALLOC
@@ -41,6 +45,8 @@ namespace FEXCore::Allocator {
   FREE_Hook free {::free};
 #endif
 
+  uint64_t HostVASize{};
+
   using GLIBC_MALLOC_Hook = void*(*)(size_t, const void *caller);
   using GLIBC_REALLOC_Hook = void*(*)(void*, size_t, const void *caller);
   using GLIBC_FREE_Hook = void(*)(void*, const void *caller);
@@ -93,6 +99,10 @@ namespace FEXCore::Allocator {
 #pragma GCC diagnostic pop
 
   FEX_DEFAULT_VISIBILITY size_t DetermineVASize() {
+    if (HostVASize) {
+      return HostVASize;
+    }
+
     static constexpr std::array<uintptr_t, 7> TLBSizes = {
       57,
       52,
@@ -123,6 +133,7 @@ namespace FEXCore::Allocator {
       };
 
       if (Find(Size)) {
+        HostVASize = Bits;
         return Bits;
       }
     }
@@ -131,91 +142,119 @@ namespace FEXCore::Allocator {
     FEX_UNREACHABLE;
   }
 
-  PtrCache* StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
-    PtrCache *Cache{};
-    uint64_t CacheSize{};
-    uint64_t CurrentCacheOffset = 0;
-    constexpr std::array<size_t, 10> ReservedVMARegionSizes = {{
-      // Anything larger than 64GB fails out
-      64ULL * 1024 * 1024 * 1024,   // 64GB
-      32ULL * 1024 * 1024 * 1024,   // 32GB
-      16ULL * 1024 * 1024 * 1024,   // 16GB
-      4ULL * 1024 * 1024 * 1024,    // 4GB
-      1ULL * 1024 * 1024 * 1024, // 1GB
-      512ULL * 1024 * 1024,      // 512MB
-      128ULL * 1024 * 1024,      // 128MB
-      32ULL * 1024 * 1024,       // 32MB
-      1ULL * 1024 * 1024,        // 1MB
-      4096ULL                    // One page
-    }};
-    constexpr size_t AllocationSizeMaxIndex = ReservedVMARegionSizes.size() - 1;
-    uint64_t CurrentSizeIndex = 0;
+  #define STEAL_LOG(...) // fprintf(stderr, __VA_ARGS__)
 
-    int PROT_FLAGS = PROT_READ | PROT_WRITE;
-    for (size_t MemoryOffset = Begin; MemoryOffset < End;) {
-      size_t AllocationSize = ReservedVMARegionSizes[CurrentSizeIndex];
-      size_t MemoryOffsetUpper = MemoryOffset + AllocationSize;
+  std::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
+    std::vector<MemoryRegion> Regions;
+    
+    int MapsFD = open("/proc/self/maps", O_RDONLY);
+    LogMan::Throw::AFmt(MapsFD != -1, "Failed to open /proc/self/maps");
 
-      // If we would go above the upper bound on size then try the next size
-      if (MemoryOffsetUpper > End) {
-        ++CurrentSizeIndex;
-        continue;
+    enum {ParseBegin, ParseEnd, ScanEnd} State = ParseBegin;
+
+    uintptr_t RegionBegin = 0;
+    uintptr_t RegionEnd = 0;
+
+    char Buffer[2048];
+    const char *Cursor;
+    ssize_t Remaining = 0;
+
+    for(;;) {
+
+      if (Remaining == 0) {
+        do { 
+          Remaining = read(MapsFD, Buffer, sizeof(Buffer));
+        } while ( Remaining == -1 && errno == EAGAIN);
+
+        Cursor = Buffer;
       }
 
-      void *Ptr = ::mmap(reinterpret_cast<void*>(MemoryOffset), AllocationSize, PROT_FLAGS, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_FIXED_NOREPLACE, -1, 0);
+      if (Remaining == 0 && State == ParseBegin) {
+        STEAL_LOG("[%d] EndOfFile; RegionBegin: %016lX RegionEnd: %016lX\n", __LINE__, RegionBegin, RegionEnd);
 
-      // If we managed to allocate and not get the address we want then unmap it
-      // This happens with kernels older than 4.17
-      if (reinterpret_cast<uintptr_t>(Ptr) + AllocationSize > End) {
-        ::munmap(Ptr, AllocationSize);
-        Ptr = reinterpret_cast<void*>(~0ULL);
-      }
+        auto MapBegin = std::max(RegionEnd, Begin);
+        auto MapEnd = End;
 
-      // If we failed to allocate and we are on the smallest allocation size then just continue onward
-      // This page was unmappable
-      if (reinterpret_cast<uintptr_t>(Ptr) == ~0ULL && CurrentSizeIndex == AllocationSizeMaxIndex) {
-        CurrentSizeIndex = 0;
-        MemoryOffset += AllocationSize;
-        continue;
-      }
+        STEAL_LOG("     MapBegin: %016lX MapEnd: %016lX\n", MapBegin, MapEnd);
 
-      // Congratulations we were able to map this bit
-      // Reset and claim it was available
-      if (reinterpret_cast<uintptr_t>(Ptr) != ~0ULL) {
-        if (!Cache) {
-          Cache = reinterpret_cast<PtrCache *>(Ptr);
-          CacheSize = AllocationSize;
-          PROT_FLAGS = PROT_NONE;
-        }
-        else {
-          Cache[CurrentCacheOffset] = {
-            .Ptr = static_cast<uint64_t>(reinterpret_cast<uint64_t>(Ptr)),
-            .Size = static_cast<uint64_t>(AllocationSize)
-          };
-          ++CurrentCacheOffset;
+        if (MapEnd > MapBegin) {
+          STEAL_LOG("     Reserving\n");
+
+          auto MapSize = MapEnd - MapBegin;
+          auto Alloc = mmap((void*)MapBegin, MapSize, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED_NOREPLACE, -1, 0);
+
+          LogMan::Throw::AFmt(Alloc != MAP_FAILED, "mmap({:x},{:x}) failed", MapBegin, MapSize);
+          LogMan::Throw::AFmt(Alloc == (void*)MapBegin, "mmap({},{:x}) returned {} instead of {:x}", Alloc, MapBegin);
+
+          Regions.push_back({(void*)MapBegin, MapSize});
         }
 
-        CurrentSizeIndex = 0;
-        MemoryOffset += AllocationSize;
+        close(MapsFD);
+        return Regions;
+      }
+
+      LogMan::Throw::AFmt(Remaining > 0, "Failed to parse /proc/self/maps");
+
+      auto c = *Cursor++;
+      Remaining--;
+
+      if (State == ScanEnd) {
+        if (c == '\n') {
+          State = ParseBegin;
+        }
         continue;
       }
 
-      // Couldn't allocate at this size
-      // Increase and continue
-      ++CurrentSizeIndex;
+      if (State == ParseBegin) {
+        if (c == '-') {
+          STEAL_LOG("[%d] ParseBegin; RegionBegin: %016lX RegionEnd: %016lX\n", __LINE__, RegionBegin, RegionEnd);
+
+          auto MapBegin = std::max(RegionEnd, Begin);
+          auto MapEnd = std::min(RegionBegin, End);
+          
+          STEAL_LOG("     MapBegin: %016lX MapEnd: %016lX\n", MapBegin, MapEnd);
+
+          if (MapEnd > MapBegin) {
+            STEAL_LOG("     Reserving\n");
+
+            auto MapSize = MapEnd - MapBegin;
+            auto Alloc = mmap((void*)MapBegin, MapSize, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED_NOREPLACE, -1, 0);
+
+            LogMan::Throw::AFmt(Alloc != MAP_FAILED, "mmap({:x},{:x}) failed", MapBegin, MapSize);
+            LogMan::Throw::AFmt(Alloc == (void*)MapBegin, "mmap({},{:x}) returned {} instead of {:x}", Alloc, MapBegin);
+
+            Regions.push_back({(void*)MapBegin, MapSize});
+          }
+          RegionBegin = 0;
+          RegionEnd = 0;
+          State = ParseEnd;
+          continue;
+        } else {
+          LogMan::Throw::AFmt(std::isalpha(c) || std::isdigit(c), "Unexpected char '{}' in ParseBegin", c);
+          RegionBegin = (RegionBegin << 4) | (c <= '9' ? (c - '0') : (c - 'a' + 10));
+        }
+      }
+
+      if (State == ParseEnd) {
+        if (c == ' ') {
+          STEAL_LOG("[%d] ParseEnd; RegionBegin: %016lX RegionEnd: %016lX\n", __LINE__, RegionBegin, RegionEnd);
+
+          State = ScanEnd;
+          continue;
+        } else {
+          LogMan::Throw::AFmt(std::isalpha(c) || std::isdigit(c), "Unexpected char '{}' in ParseEnd", c);
+          RegionEnd = (RegionEnd << 4) | (c <= '9' ? (c - '0') : (c - 'a' + 10));
+        }
+      }
     }
 
-    Cache[CurrentCacheOffset] = {
-      .Ptr = static_cast<uint64_t>(reinterpret_cast<uint64_t>(Cache)),
-      .Size = CacheSize,
-    };
-    return Cache;
+    ERROR_AND_DIE_FMT("unreachable");
   }
 
-  PtrCache* Steal48BitVA() {
+  std::vector<MemoryRegion> Steal48BitVA() {
     size_t Bits = FEXCore::Allocator::DetermineVASize();
     if (Bits < 48) {
-      return nullptr;
+      return {};
     }
 
     uintptr_t Begin48BitVA = 0x0'8000'0000'0000ULL;
@@ -223,18 +262,9 @@ namespace FEXCore::Allocator {
     return StealMemoryRegion(Begin48BitVA, End48BitVA);
   }
 
-  void ReclaimMemoryRegion(PtrCache* Regions) {
-    if (Regions == nullptr) {
-      return;
-    }
-
-    for (size_t i = 0;; ++i) {
-      void *Ptr = reinterpret_cast<void*>(Regions[i].Ptr);
-      size_t Size = Regions[i].Size;
-      ::munmap(Ptr, Size);
-      if (Ptr == Regions) {
-        break;
-      }
+  void ReclaimMemoryRegion(const std::vector<MemoryRegion> &Regions) {
+    for (const auto &Region: Regions) {
+      ::munmap(Region.Ptr, Region.Size);
     }
   }
 }

External/FEXCore/Source/Utils/Allocator/64BitAllocator.cpp

@@ -70,7 +70,10 @@ namespace Alloc::OSAllocator {
         ReservedVMARegion *SlabInfo;
         uint64_t FreeSpace{};
         uint32_t LastPageAllocation{};
-        FEXCore::FlexBitSet<uint64_t> UsedPages;
+
+        // Align UsedPages so it pads to the next page.
+        // Necessary to take advantage of madvise zero page pooling.
+        alignas(4096) FEXCore::FlexBitSet<uint64_t> UsedPages;
 
         // This returns the size of the LiveVMARegion in addition to the flex set that tracks the used data
         // The LiveVMARegion lives at the start of the VMA region which means on initialization we need to set that
@@ -93,16 +96,23 @@ namespace Alloc::OSAllocator {
           Region->FreeSpace = Region->SlabInfo->RegionSize - SizePlusManagedData;
 
           size_t NumPages = SizePlusManagedData >> FHU::FEX_PAGE_SHIFT;
-          // Memset the full tracking to zero to state nothing used
-          Region->UsedPages.MemSet(Region->SlabInfo->RegionSize >> FHU::FEX_PAGE_SHIFT);
+
+          // Use madvise to set the full tracking region to zero.
+          // This ensures unused pages are zero, while not having the backing pages consuming memory.
+          ::madvise(Region->UsedPages.Memory + (NumPages * 4096), (Region->SlabInfo->RegionSize >> FHU::FEX_PAGE_SHIFT) - (NumPages * 4096), MADV_DONTNEED);
+
+          // Use madvise to claim WILLNEED on the beginning pages for initial state tracking.
+          // Improves performance of the following MemClear by not doing a page level fault dance for data necessary to track >170TB of used pages.
+          ::madvise(Region->UsedPages.Memory, NumPages * 4096, MADV_WILLNEED);
+
           // Set our reserved pages
-          for (size_t i = 0; i < NumPages; ++i) {
-            // Set our used pages
-            Region->UsedPages.Set(i);
-          }
+          Region->UsedPages.MemSet(NumPages);
           Region->LastPageAllocation = NumPages;
         }
       };
+
+      static_assert(sizeof(LiveVMARegion) == 4096, "Needs to be the size of a page");
+
       static_assert(std::is_trivially_copyable<LiveVMARegion>::value, "Needs to be trivially copyable");
       static_assert(offsetof(LiveVMARegion, UsedPages) == sizeof(LiveVMARegion), "FlexBitSet needs to be at the end");
 
@@ -131,23 +141,29 @@ namespace Alloc::OSAllocator {
 
         // Copy over the reserved data
         LiveRange->SlabInfo = ReservedRegion;
+
         // Initialize VMA
         LiveVMARegion::InitializeVMARegionUsed(LiveRange, UsedSize);
 
         // Add to our active tracked ranges
         auto LiveIter = LiveRegions->emplace_back(LiveRange);
-
         return LiveIter;
       }
 
       // 32-bit old kernel workarounds
-      FEXCore::Allocator::PtrCache *Steal32BitIfOldKernel();
+      std::vector<FEXCore::Allocator::MemoryRegion> Steal32BitIfOldKernel();
   };
 
 void OSAllocator_64Bit::DetermineVASize() {
   size_t Bits = FEXCore::Allocator::DetermineVASize();
   uintptr_t Size = 1ULL << Bits;
+
   UPPER_BOUND = Size;
+
+  #if _M_X86_64 // Last page cannot be allocated on x86
+    UPPER_BOUND -= FHU::FEX_PAGE_SIZE;
+  #endif
+
   UPPER_BOUND_PAGE = UPPER_BOUND / FHU::FEX_PAGE_SIZE;
 }
 
@@ -490,11 +506,11 @@ int OSAllocator_64Bit::Munmap(void *addr, size_t length) {
   return 0;
 }
 
-FEXCore::Allocator::PtrCache *OSAllocator_64Bit::Steal32BitIfOldKernel() {
+std::vector<FEXCore::Allocator::MemoryRegion> OSAllocator_64Bit::Steal32BitIfOldKernel() {
   // First calculate kernel version
   struct utsname buf{};
   if (uname(&buf) == -1) {
-    return nullptr;
+    return {};
   }
 
   int32_t Major{};
@@ -512,7 +528,7 @@ FEXCore::Allocator::PtrCache *OSAllocator_64Bit::Steal32BitIfOldKernel() {
 
   if (Version >= ((4 << 24) | (17 << 16) | 0)) {
     // If the kernel is >= 4.17 then it supports MAP_FIXED_NOREPLACE
-    return nullptr;
+    return {};
   }
 
   constexpr size_t LOWER_BOUND_32 = 0x1'0000;
@@ -523,101 +539,39 @@ FEXCore::Allocator::PtrCache *OSAllocator_64Bit::Steal32BitIfOldKernel() {
 
 OSAllocator_64Bit::OSAllocator_64Bit() {
   DetermineVASize();
-  auto ArrayPtr = Steal32BitIfOldKernel();
+  auto LowMem = Steal32BitIfOldKernel();
 
-  // On allocation try and steal the entire upper 64bits of address space for mapping
-  constexpr std::array<size_t, 8> ReservedVMARegionSizes = {{
-    // Anything larger than 64GB fails out
-    64ULL * 1024 * 1024 * 1024,   // 64GB
-    32ULL * 1024 * 1024 * 1024,   // 32GB
-    16ULL * 1024 * 1024 * 1024,   // 16GB
-    4ULL * 1024 * 1024 * 1024,    // 4GB
-    1ULL * 1024 * 1024 * 1024,    // 1GB
-    512ULL * 1024 * 1024,         // 512MB
-    128ULL * 1024 * 1024,         // 128MB
-    4096ULL                       // One page
-  }};
+  auto Ranges = FEXCore::Allocator::StealMemoryRegion(LOWER_BOUND, UPPER_BOUND);
 
-  constexpr size_t AllocationSizeMaxIndex = ReservedVMARegionSizes.size() - 1;
+  for (auto [Ptr, AllocationSize]: Ranges) {
+    if (!ObjectAlloc) {
+      auto MaxSize = std::min(size_t(64) * 1024 * 1024, AllocationSize);
 
-  // Have the first region only be 4GB VMA
-  // Avoids conflicts with some tests
-  uint64_t CurrentSizeIndex = 3;
-  ReservedVMARegion *PrevReserved{};
-  for (size_t MemoryOffset = LOWER_BOUND; MemoryOffset < UPPER_BOUND;) {
-    size_t AllocationSize = ReservedVMARegionSizes[CurrentSizeIndex];
-    size_t MemoryOffsetUpper = MemoryOffset + AllocationSize;
+      // Allocate up to 64 MiB the first allocation for an intrusive allocator
+      mprotect(Ptr, MaxSize, PROT_READ | PROT_WRITE);
 
-    // If we would go above the upper bound on size then try the next size
-    if (MemoryOffsetUpper > UPPER_BOUND) {
-      ++CurrentSizeIndex;
-      continue;
-    }
+      // This enables the kernel to use transparent large pages in the allocator which can reduce memory pressure
+      ::madvise(Ptr, MaxSize, MADV_HUGEPAGE);
 
-    void *Ptr = ::mmap(reinterpret_cast<void*>(MemoryOffset), AllocationSize, PROT_NONE, MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+      ObjectAlloc = new (Ptr) Alloc::ForwardOnlyIntrusiveArenaAllocator(Ptr, MaxSize);
+      ReservedRegions = ObjectAlloc->new_construct(ReservedRegions, ObjectAlloc);
+      LiveRegions = ObjectAlloc->new_construct(LiveRegions, ObjectAlloc);
 
-    // If we managed to allocate and not get the address we want then unmap it
-    // This happens with kernels older than 4.17
-    if (reinterpret_cast<uintptr_t>(Ptr) != MemoryOffset &&
-        reinterpret_cast<uintptr_t>(Ptr) < LOWER_BOUND) {
-      ::munmap(Ptr, AllocationSize);
-      Ptr = reinterpret_cast<void*>(~0ULL);
-    }
-
-    // If we failed to allocate and we are on the smallest allocation size then just continue onward
-    // This page was unmappable
-    if (reinterpret_cast<uintptr_t>(Ptr) == ~0ULL && CurrentSizeIndex == AllocationSizeMaxIndex) {
-      CurrentSizeIndex = 0;
-      MemoryOffset += AllocationSize;
-      continue;
-    }
-
-    // Congratulations we were able to map this bit
-    // Reset and claim it was available
-    if (reinterpret_cast<uintptr_t>(Ptr) != ~0ULL) {
-      if (!ObjectAlloc) {
-        // Steal the first allocation for an intrusive allocator
-        // Will be mprotected correctly already
-        mprotect(Ptr, AllocationSize, PROT_READ | PROT_WRITE);
-        ObjectAlloc = new (Ptr) Alloc::ForwardOnlyIntrusiveArenaAllocator(Ptr, AllocationSize);
-        ReservedRegions = ObjectAlloc->new_construct(ReservedRegions, ObjectAlloc);
-        LiveRegions = ObjectAlloc->new_construct(LiveRegions, ObjectAlloc);
+      if (AllocationSize > MaxSize) {
+        AllocationSize -= MaxSize;
+        (uint8_t*&)Ptr += MaxSize;
+      } else {
+        continue;
       }
-      else {
-
-        // If the allocation size is large than a page, then try allowing it to be a huge page
-        // This enables the kernel to use transparent large pages in the allocator which can reduce memory pressure
-        // Considering we are allocating the entire VA space, this is a good thing
-        // If MADV_HUGEPAGE isn't support then this will fail harmlessly
-        if (AllocationSize > 4096) {
-          ::madvise(Ptr, AllocationSize, MADV_HUGEPAGE);
-        }
-
-        bool Merged = false;
-        if (PrevReserved) {
-          Merged = MergeReservedRegionIfPossible(PrevReserved, reinterpret_cast<uint64_t>(Ptr), AllocationSize);
-        }
-
-        if (!Merged) {
-          ReservedVMARegion *Region = ObjectAlloc->new_construct<ReservedVMARegion>();
-          Region->Base = reinterpret_cast<uint64_t>(Ptr);
-          Region->RegionSize = AllocationSize;
-          ReservedRegions->emplace_back(Region);
-          PrevReserved = Region;
-        }
-      }
-
-      CurrentSizeIndex = 0;
-      MemoryOffset += AllocationSize;
-      continue;
     }
-
-    // Couldn't allocate at this size
-    // Increase and continue
-    ++CurrentSizeIndex;
+    
+    ReservedVMARegion *Region = ObjectAlloc->new_construct<ReservedVMARegion>();
+    Region->Base = reinterpret_cast<uint64_t>(Ptr);
+    Region->RegionSize = AllocationSize;
+    ReservedRegions->emplace_back(Region);
   }
 
-  FEXCore::Allocator::ReclaimMemoryRegion(ArrayPtr);
+  FEXCore::Allocator::ReclaimMemoryRegion(LowMem);
 }
 
 OSAllocator_64Bit::~OSAllocator_64Bit() {

External/FEXCore/include/FEXCore/Core/CPUBackend.h

@@ -122,6 +122,10 @@ namespace CPU {
     bool IsAddressInCodeBuffer(uintptr_t Address) const;
 
   protected:
+    // Max spill slot size in bytes. We need at most 32 bytes
+    // to be able to handle a 256-bit vector store to a slot.
+    constexpr static uint32_t MaxSpillSlotSize = 32;
+
     FEXCore::Core::InternalThreadState *ThreadState;
 
     size_t InitialCodeSize, MaxCodeSize;

External/FEXCore/include/FEXCore/Core/Context.h

@@ -5,6 +5,7 @@
 
 #include <FEXCore/Core/SignalDelegator.h>
 #include <FEXCore/Core/CPUID.h>
+#include <FEXCore/IR/IR.h>
 #include <FEXCore/Utils/CompilerDefs.h>
 
 #include <istream>
@@ -280,4 +281,12 @@ namespace FEXCore::Context {
 
   FEX_DEFAULT_VISIBILITY void ConfigureAOTGen(FEXCore::Core::InternalThreadState *Thread, std::set<uint64_t> *ExternalBranches, uint64_t SectionMaxAddress);
   FEX_DEFAULT_VISIBILITY CustomIRResult AddCustomIREntrypoint(FEXCore::Context::Context *CTX, uintptr_t Entrypoint, std::function<void(uintptr_t Entrypoint, FEXCore::IR::IREmitter *)> Handler, void *Creator = nullptr, void *Data = nullptr);
+
+  /**
+   * @brief Allows the frontend to register its own thunk handlers independent of what is controlled in the backend.
+   *
+   * @param CTX A valid non-null context instance.
+   * @param Definitions A vector of thunk definitions that the frontend controls
+   */
+  FEX_DEFAULT_VISIBILITY void AppendThunkDefinitions(FEXCore::Context::Context *CTX, std::vector<FEXCore::IR::ThunkDefinition> const& Definitions);
 }

External/FEXCore/include/FEXCore/Core/HostFeatures.h

@@ -21,7 +21,12 @@ class HostFeatures final {
     bool SupportsRCPC{};
     bool SupportsTSOImm9{};
     bool SupportsRAND{};
+    bool Supports3DNow{};
+    bool SupportsSSE4A{};
     bool SupportsAVX{};
+    bool SupportsSHA{};
+    bool SupportsBMI1{};
+    bool SupportsBMI2{};
 
     // Float exception behaviour
     bool SupportsFlushInputsToZero{};

External/FEXCore/include/FEXCore/Core/UContext.h

@@ -58,6 +58,14 @@ namespace FEXCore {
         FEATURE_XTILE_DATA = 1U << 18,
       };
 
+      bool HasExtendedContext() const {
+        return magic1 == FP_XSTATE_MAGIC;
+      }
+
+      bool HasYMMH() const {
+        return (xfeatures & FEATURE_YMM) != 0;
+      }
+
       // If magic1 is set to FP_XSTATE_MAGIC, then the encompassing
       // frame is an xstate frame. If 0, then it's a legacy frame.
       uint32_t magic1;

External/FEXCore/include/FEXCore/IR/IR.h

@@ -413,6 +413,13 @@ struct SHA256Sum final {
   }
 };
 
+typedef void ThunkedFunction(void* ArgsRv);
+
+struct ThunkDefinition final {
+  SHA256Sum Sum;
+  ThunkedFunction *ThunkFunction;
+};
+
 class NodeIterator;
 
 /* This iterator can be used to step though nodes.

External/FEXCore/include/FEXCore/Utils/Allocator.h

@@ -5,6 +5,7 @@
 #include <cstdint>
 #include <functional>
 #include <sys/types.h>
+#include <vector>
 
 namespace FEXCore::Allocator {
   using MMAP_Hook = void*(*)(void*, size_t, int, int, int, off_t);
@@ -24,19 +25,19 @@ namespace FEXCore::Allocator {
   FEX_DEFAULT_VISIBILITY void ClearHooks();
 
   FEX_DEFAULT_VISIBILITY size_t DetermineVASize();
-  // 48-bit VA handling
-  struct PtrCache {
-    uint64_t Ptr;
-    uint64_t Size;
+
+  struct MemoryRegion {
+    void *Ptr;
+    size_t Size;
   };
 
-  FEX_DEFAULT_VISIBILITY PtrCache* StealMemoryRegion(uintptr_t Begin, uintptr_t End);
-  FEX_DEFAULT_VISIBILITY void ReclaimMemoryRegion(PtrCache* Regions);
+  FEX_DEFAULT_VISIBILITY std::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End);
+  FEX_DEFAULT_VISIBILITY void ReclaimMemoryRegion(const std::vector<MemoryRegion> & Regions);
   // When running a 64-bit executable on ARM then userspace guest only gets 47 bits of VA
   // This is a feature of x86-64 where the kernel gets a full 128TB of VA space
   // x86-64 canonical addresses with bit 48 set will sign extend the address (Ignoring LA57)
   // AArch64 canonical addresses are only up to bits 48/52 with the remainder being other things
   // Use this to reserve the top 128TB of VA so the guest never see it
   // Returns nullptr on host VA < 48bits
-  FEX_DEFAULT_VISIBILITY PtrCache* Steal48BitVA();
+  FEX_DEFAULT_VISIBILITY std::vector<MemoryRegion> Steal48BitVA();
 }

FEXHeaderUtils/CMakeLists.txt

@@ -1,74 +1,64 @@
+add_library(FEXHeaderUtils INTERFACE)
+
 # Check for syscall support here
 check_cxx_source_compiles(
-	"
-	#include <sched.h>
-	int main() {
-	return ::getcpu(nullptr, nullptr);
-	}"
+  "
+  #include <sched.h>
+  int main() {
+  return ::getcpu(nullptr, nullptr);
+  }"
   compiles)
 if (compiles)
   message(STATUS "Has getcpu helper")
-  add_definitions(-DHAS_SYSCALL_GETCPU=1)
+  target_compile_definitions(FEXHeaderUtils INTERFACE HAS_SYSCALL_GETCPU=1)
 endif ()
 
 check_cxx_source_compiles(
-	"
-	#include <unistd.h>
-	int main() {
-	return ::gettid();
-	}"
+  "
+  #include <unistd.h>
+  int main() {
+  return ::gettid();
+  }"
   compiles)
 if (compiles)
   message(STATUS "Has gettid helper")
-  add_definitions(-DHAS_SYSCALL_GETTID=1)
+  target_compile_definitions(FEXHeaderUtils INTERFACE HAS_SYSCALL_GETTID=1)
 endif ()
 
 check_cxx_source_compiles(
-	"
-	#include <signal.h>
-	int main() {
-	return ::tgkill(0, 0, 0);
-	}"
+  "
+  #include <signal.h>
+  int main() {
+  return ::tgkill(0, 0, 0);
+  }"
   compiles)
 if (compiles)
   message(STATUS "Has tgkill helper")
-  add_definitions(-DHAS_SYSCALL_TGKILL=1)
+  target_compile_definitions(FEXHeaderUtils INTERFACE HAS_SYSCALL_TGKILL=1)
 endif ()
 
 check_cxx_source_compiles(
-	"
-	#include <sys/stat.h>
-	int main() {
-	return ::statx(0, nullptr, 0, 0, nullptr);
-	}"
+  "
+  #include <sys/stat.h>
+  int main() {
+  return ::statx(0, nullptr, 0, 0, nullptr);
+  }"
   compiles)
 if (compiles)
   message(STATUS "Has statx helper")
-  add_definitions(-DHAS_SYSCALL_STATX=1)
+  target_compile_definitions(FEXHeaderUtils INTERFACE HAS_SYSCALL_STATX=1)
 endif ()
 
 check_cxx_source_compiles(
-	"
-	#include <stdio.h>
-	int main() {
-	return ::renameat2(0, nullptr, 0, nullptr, 0);
-	}"
+  "
+  #include <stdio.h>
+  int main() {
+  return ::renameat2(0, nullptr, 0, nullptr, 0);
+  }"
   compiles)
 if (compiles)
   message(STATUS "Has renameat2 helper")
-  add_definitions(-DHAS_SYSCALL_RENAMEAT2=1)
-endif ()
-
-check_cxx_source_compiles(
-	"
-	#include <stdio.h>
-	#include <syscall.h>
-	int main() {
-	return ::syscall(SYS_pidfd_open, ::getpid(), 0);
-	}"
-  compiles)
-if (compiles)
-  message(STATUS "Has pidfd_open helper")
-  add_definitions(-DHAS_SYSCALL_PIDFD_OPEN=1)
+  target_compile_definitions(FEXHeaderUtils INTERFACE HAS_SYSCALL_RENAMEAT2=1)
 endif ()
 
+target_include_directories(FEXHeaderUtils INTERFACE .)

FEXHeaderUtils/FEXHeaderUtils/Syscalls.h

@@ -38,10 +38,15 @@ namespace FHU::Syscalls {
 #endif
 #endif
 
+// Common syscall numbers
+#ifndef SYS_pidfd_open
+#define SYS_pidfd_open 434
+#endif
+
 inline int32_t getcpu(uint32_t *cpu, uint32_t *node) {
   // Third argument is unused
 #if defined(HAS_SYSCALL_GETCPU) && HAS_SYSCALL_GETCPU
-  return ::getcpu(cpu, node, nullptr);
+  return ::getcpu(cpu, node);
 #else
   return ::syscall(SYS_getcpu, cpu, node, nullptr);
 #endif
@@ -57,7 +62,7 @@ inline int32_t gettid() {
 
 inline int32_t tgkill(pid_t tgid, pid_t tid, int sig) {
 #if defined(HAS_SYSCALL_GETTID) && HAS_SYSCALL_GETTID
-  return ::tgkill(tggid, tid, sig);
+  return ::tgkill(tgid, tid, sig);
 #else
   return ::syscall(SYS_tgkill, tgid, tid, sig);
 #endif
@@ -65,7 +70,7 @@ inline int32_t tgkill(pid_t tgid, pid_t tid, int sig) {
 
 inline int32_t statx(int dirfd, const char *pathname, int32_t flags, uint32_t mask, void *statxbuf) {
 #if defined(HAS_SYSCALL_STATX) && HAS_SYSCALL_STATX
-  return ::statx(dirfd, pathname, flags, mask, statxbuf);
+  return ::statx(dirfd, pathname, flags, mask, reinterpret_cast<struct statx *__restrict>(statxbuf));
 #else
   return ::syscall(SYS_statx, dirfd, pathname, flags, mask, statxbuf);
 #endif
@@ -80,11 +85,7 @@ inline int32_t renameat2(int olddirfd, const char *oldpath, int newdirfd, const
 }
 
 inline int32_t pidfd_open(pid_t pid, unsigned int flags) {
-#if defined(DHAS_SYSCALL_PIDFD_OPEN) && DHAS_SYSCALL_PIDFD_OPEN
-  return ::syscall(SYS_pidfd_open, pid_t pid, unsigned int flags);
-#else
-  return -1;
-#endif
+  return ::syscall(SYS_pidfd_open, pid, flags);
 }
 
 }

Readme.md

@@ -30,11 +30,11 @@ On AArch64 hosts the user **MUST** have an x86-64 RootFS [Creating a RootFS](#Ro
 See the [Source Outline](docs/SourceOutline.md) for more information.
 
 ### Building FEX
-Follow the guide on the official FEX-Emu Wiki [here](https://wiki.fex-emu.org/index.php/Development:Setting_up_FEX).
+Follow the guide on the official FEX-Emu Wiki [here](https://wiki.fex-emu.com/index.php/Development:Setting_up_FEX).
 
 ### RootFS generation
 AArch64 hosts require a rootfs for running applications.
 Follow the guide on the wiki page for seeing how to set up the rootfs from scratch
-https://wiki.fex-emu.org/index.php/Development:Setting_up_RootFS
+https://wiki.fex-emu.com/index.php/Development:Setting_up_RootFS
 
 ![FEX diagram](docs/Diagram.svg)

Scripts/CI_FetchRootFS.py

@@ -83,6 +83,12 @@ def HashFile(file):
 
     return int.from_bytes(x.digest(), "big")
 
+def RemoveRootFSFolder(RootFSPath):
+    print("Removing previous rootfs extraction before copying")
+    shutil.rmtree(RootFSPath, ignore_errors = True)
+    # Recreate the folder
+    os.makedirs(RootFSPath)
+
 def CheckFilesystemForFS(RootFSMountPath, RootFSPath, DistroFit):
     # Check if rootfs mount path exists
     if (not os.path.exists(RootFSMountPath) or
@@ -105,6 +111,7 @@ def CheckFilesystemForFS(RootFSMountPath, RootFSPath, DistroFit):
     MountRootFSImagePath = RootFSMountPath + DistroFit[3]
     RootFSImagePath = RootFSPath + "/" + os.path.basename(DistroFit[3])
     NeedsExtraction = False
+    PreviouslyExistingRootFS = False
 
     if not os.path.exists(MountRootFSImagePath):
         print("Image {} doesn't exist".format(MountRootFSImagePath))
@@ -113,29 +120,39 @@ def CheckFilesystemForFS(RootFSMountPath, RootFSPath, DistroFit):
     if not os.path.exists(RootFSImagePath):
         # Copy over
         print("RootFS image doesn't exist. Copying")
-        shutil.copyfile(MountRootFSImagePath, RootFSImagePath)
-        NeedsExtraction = True
-
-    # Now hash the image
-    RootFSHash = HashFile(RootFSImagePath)
-    if RootFSHash != DistroFit[4]:
-        print("Hash {} did not match {}, copying new image".format(hex(RootFSHash), hex(DistroFit[4])))
+        RemoveRootFSFolder(RootFSPath)
         shutil.copyfile(MountRootFSImagePath, RootFSImagePath)
         NeedsExtraction = True
 
     # Check if the image needs to be extracted
     if not os.path.exists(RootFSPath + "/usr"):
         NeedsExtraction = True
+    else:
+        PreviouslyExistingRootFS = True
+
+    # Now hash the image
+    RootFSHash = HashFile(RootFSImagePath)
+    if RootFSHash != DistroFit[4]:
+        print("Hash {} did not match {}, copying new image".format(hex(RootFSHash), hex(DistroFit[4])))
+
+        if PreviouslyExistingRootFS:
+            RemoveRootFSFolder(RootFSPath)
+
+        shutil.copyfile(MountRootFSImagePath, RootFSImagePath)
+        NeedsExtraction = True
 
     if NeedsExtraction:
         print("Extracting rootfs")
+
         CmdResult = subprocess.call(["unsquashfs", "-f", "-d", RootFSPath, RootFSImagePath])
         if CmdResult != 0:
-            print("Couldn't extract squashfs")
+            print("Couldn't extract squashfs. Removing image file to be safe")
+            os.remove(RootFSImagePath)
             return False
 
     if not os.path.exists(RootFSPath + "/usr"):
-        print("Couldn't extract squashfs")
+        print("Couldn't extract squashfs. Removing image file to be safe")
+        os.remove(RootFSImagePath)
         return False
 
     print("RootFS successfully checked and extracted")

Scripts/DefinitionExtract.py

@@ -154,6 +154,9 @@ def HandleFunctionDeclCursor(Arch, Cursor):
         elif (Child.kind == CursorKind.ASM_LABEL_ATTR):
             # Whatever you are we don't care about you
             return Arch
+        elif (Child.kind == CursorKind.WARN_UNUSED_RESULT_ATTR):
+            # Whatever you are we don't care about you
+            return Arch
         elif (Child.kind == CursorKind.VISIBILITY_ATTR):
             pass
         else:
@@ -165,7 +168,7 @@ def HandleFunctionDeclCursor(Arch, Cursor):
 
 def PrintFunctionDecls():
     for Decl in FunctionDecls:
-        print("fn(\"{0} {1}({2})\")".format(Decl.Ret, Decl.Name, ", ".join(Decl.Params)))
+        print("template<> struct fex_gen_config<{}> {{}};".format(Decl.Name))
 
 def FindClangArguments(OriginalArguments):
     AddedArguments = ["clang"]

Scripts/GenerateSyscallNumbers.py

@@ -59,7 +59,13 @@ SyscallArm64File = "/include/uapi/asm-generic/unistd.h"
 DefinitionRenameDict = {
     "pread64": "pread_64",
     "pwrite64": "pwrite_64",
-    "prlimit64": "prlimit_64"
+    "prlimit64": "prlimit_64",
+    # Shm symbols conflict with termux defines and FEX's syscall token pasting.
+    # Underscore at the start to avoid name collision
+    "shmget": "_shmget",
+    "shmctl": "_shmctl",
+    "shmat": "_shmat",
+    "shmdt": "_shmdt",
 }
 
 Definitions_x64 = []

Scripts/InstallFEX.py

@@ -306,12 +306,14 @@ def GetRootFSPath():
     return _RootFSPath
 
 def CheckRootFSInstallStatus():
-    # Matches what is available on https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links_XXH3.txt
+    # Matches what is available on https://rootfs.fex-emu.com/file/fex-rootfs/RootFS_links.json
     UbuntuVersionToRootFS = {
         "20.04": "Ubuntu_21_04.sqsh",
         "21.04": "Ubuntu_21_04.sqsh",
         "21.10": "Ubuntu_21_10.sqsh",
+        "21.10": "Ubuntu_21_10.ero",
         "22.04": "Ubuntu_22_04.sqsh",
+        "22.04": "Ubuntu_22_04.ero",
     }
 
     return os.path.exists(GetRootFSPath() + UbuntuVersionToRootFS[GetDistro()[1]])

Scripts/StructPackVerifier.py

@@ -10,6 +10,18 @@ import logging
 logger = logging.getLogger()
 logger.setLevel(logging.WARNING)
 
+# These defines are temporarily defined since python3-clang doesn't yet support these.
+# Once this tool gets switched over to C++ then this won't be an issue.
+
+# Expression that references a C++20 concept.
+CursorKind.CONCEPTSPECIALIZATIONEXPR = CursorKind(153),
+
+# C++2a std::bit_cast expression.
+CursorKind.BUILTINBITCASTEXPR = CursorKind(280)
+
+# a concept declaration.
+CursorKind.CONCEPTDECL = CursorKind(604),
+
 @dataclass
 class TypeDefinition:
     TYPE_UNKNOWN = 0
@@ -268,7 +280,7 @@ def HandleTypeDefDeclCursor(Arch, Cursor):
         if (len(TypeDefName) != 0):
             HandleTypeDefDecl(Arch, Cursor, TypeDefName)
 
-	    # Append namespace
+            # Append namespace
             Arch.NamespaceScope.append(TypeDefName)
             SetNamespace(Arch)
 
@@ -404,19 +416,20 @@ def HandleCursor(Arch, Cursor):
         return
 
     for Child in Cursor.get_children():
-        if (Child.kind == CursorKind.TRANSLATION_UNIT):
+        kind = Child.kind
+        if (kind == CursorKind.TRANSLATION_UNIT):
             Arch = HandleCursor(Arch, Child)
-        elif (Child.kind == CursorKind.FIELD_DECL):
+        elif (kind == CursorKind.FIELD_DECL):
             pass
-        elif (Child.kind == CursorKind.UNION_DECL):
+        elif (kind == CursorKind.UNION_DECL):
             Arch = HandleUnionDeclCursor(Arch, Child)
-        elif (Child.kind == CursorKind.STRUCT_DECL):
+        elif (kind == CursorKind.STRUCT_DECL):
             Arch = HandleStructDeclCursor(Arch, Child)
-        elif (Child.kind == CursorKind.TYPEDEF_DECL):
+        elif (kind == CursorKind.TYPEDEF_DECL):
             Arch = HandleTypeDefDeclCursor(Arch, Child)
-        elif (Child.kind == CursorKind.VAR_DECL):
+        elif (kind == CursorKind.VAR_DECL):
             Arch = HandleVarDeclCursor(Arch, Child)
-        elif (Child.kind == CursorKind.NAMESPACE):
+        elif (kind == CursorKind.NAMESPACE):
             # Append namespace
             Arch.NamespaceScope.append(Child.spelling)
             SetNamespace(Arch)
@@ -427,7 +440,7 @@ def HandleCursor(Arch, Cursor):
             # Pop namespace off
             Arch.NamespaceScope.pop()
             SetNamespace(Arch)
-        elif (Child.kind == CursorKind.TYPE_REF):
+        elif (kind == CursorKind.TYPE_REF):
             # Safe to pass on
             pass
         else:
@@ -638,25 +651,21 @@ def main():
         BaseArgs.append(sys.argv[ArgIndex])
 
     args_x86_32 = [
-        "-I/usr/i686-linux-gnu/include/c++/10/i686-linux-gnu/",
-        "-I/usr/i686-linux-gnu/include/",
+        "-I/usr/i686-linux-gnu/include",
         "-O2",
         "-m32",
         "--target=i686-linux-unknown",
     ]
 
     args_x86_64 = [
-        "-I/usr/include/x86_64-linux-gnu",
-        "-I/usr/x86_64-linux-gnu/include/c++/10/x86_64-linux-gnu/",
-        "-I/usr/x86_64-linux-gnu/include/",
+        "-I/usr/x86_64-linux-gnu/include",
         "-O2",
         "--target=x86_64-linux-unknown",
         "-D_M_X86_64",
     ]
 
     args_aarch64 = [
-        "-I/usr/aarch64-linux-gnu/include/c++/10/aarch64-linux-gnu/",
-        "-I/usr/aarch64-linux-gnu/include/",
+        "-I/usr/aarch64-linux-gnu/include",
         "-O2",
         "--target=aarch64-linux-unknown",
         "-D_M_ARM_64",

Scripts/guest_test_runner.py

@@ -3,44 +3,55 @@ import os
 import sys
 import subprocess
 
-# Args: <Known Failures file> <ExpectedOutputsFile> <DisabledTestsFile> <TestName> <FexExecutable> <FexArgs>...
+def LoadTestsFile(File):
+    Dict = {}
+    if not os.path.exists(File):
+        return Dict
+
+    with open(File) as dtf:
+        for line in dtf:
+            test = line.split("#")[0].strip() # remove comments and empty spaces
+            if len(test) > 0:
+                Dict[test] = 1
+
+    return Dict
+
+def LoadTestsFileResults(File):
+    Dict = {}
+    if not os.path.exists(File):
+        return Dict
+
+    with open(File) as dtf:
+        for line in dtf:
+            test = line.split("#")[0].strip() # remove comments and empty spaces
+            if len(test) > 0:
+                parts = line.split(" ")
+                Dict[parts[0]] = int(parts[1])
+
+    return Dict
+
+
+# Args: <Known Failures file> <ExpectedOutputsFile> <DisabledTestsFile> <FlakeTestsFile> <TestName> <Mode> <FexExecutable> <FexArgs>...
 
 # fexargs should also include the test executable
 
-if (len(sys.argv) < 6):
+if (len(sys.argv) < 7):
     sys.exit()
 
 known_failures_file = sys.argv[1]
 expected_output_file = sys.argv[2]
 disabled_tests_file = sys.argv[3]
-test_name = sys.argv[4]
-mode = sys.argv[5]
-fexecutable = sys.argv[6]
+flake_tests_file = sys.argv[4]
+test_name = sys.argv[5]
+mode = sys.argv[6]
+fexecutable = sys.argv[7]
+StartingFEXArgsOffset = 8
 
-known_failures = { }
-expected_output = { }
-disabled_tests = { }
-
-# Open the known failures file and add it to a dictionary
-with open(known_failures_file) as kff:
-    for line in kff:
-        test = line.split("#")[0].strip() # remove comments and empty spaces
-        if len(test) > 0:
-            known_failures[test] = 1
-
-# Open expected outputs and add it to dictionary
-with open(expected_output_file) as eof:
-    for line in eof:
-        line = test = line.split("#")[0].strip() # remove comments and empty spaces
-        if len(line) > 0:
-            parts = line.split(" ")
-            expected_output[parts[0]] = int(parts[1])
-
-with open(disabled_tests_file) as dtf:
-    for line in dtf:
-        test = line.split("#")[0].strip() # remove comments and empty spaces
-        if len(test) > 0:
-            disabled_tests[test] = 1
+# Open test expected information files and load in to dictionaries.
+known_failures = LoadTestsFile(known_failures_file)
+expected_output = LoadTestsFileResults(expected_output_file)
+disabled_tests = LoadTestsFile(disabled_tests_file)
+flake_tests = LoadTestsFile(flake_tests_file)
 
 # run with timeout to avoid locking up
 RunnerArgs = []
@@ -54,25 +65,37 @@ if (mode == "guest"):
         RunnerArgs.append(ROOTFS_ENV)
 
 # Add the rest of the arguments
-for i in range(len(sys.argv) - 7):
-    RunnerArgs.append(sys.argv[7 + i])
+for i in range(len(sys.argv) - StartingFEXArgsOffset):
+    RunnerArgs.append(sys.argv[StartingFEXArgsOffset + i])
 
 #print(RunnerArgs)
 
 ResultCode = 0
 
+# Handle flakes
+TryCount = 1
+if (flake_tests.get(test_name)):
+    TryCount = 5
+
 if (disabled_tests.get(test_name)):
     ResultCode = -73
-else:
-    # Run the test and wait for it to end to get the result
-    Process = subprocess.Popen(RunnerArgs)
-    Process.wait()
-    ResultCode = Process.returncode
 
 # expect zero by default
 if (not test_name in expected_output):
     expected_output[test_name] = 0
 
+if ResultCode == 0:
+    for Try in range(TryCount):
+        # Run the test and wait for it to end to get the result
+        print(RunnerArgs)
+        Process = subprocess.Popen(RunnerArgs)
+        Process.wait()
+        ResultCode = Process.returncode
+
+        # Break if the expected output is the result code
+        if (expected_output[test_name] == ResultCode):
+            break
+
 if (expected_output[test_name] != ResultCode):
     if (test_name in expected_output):
         print("test failed, expected is", expected_output[test_name], "but got", ResultCode)

Scripts/json_config_parse.py

@@ -63,8 +63,15 @@ class Mode(Flag) :
     MODE_64   = 1
 
 class HostFeatures(Flag) :
-    ANY      = 0
-    AVX_ONLY = 1
+    FEATURE_ANY    = 0
+    FEATURE_3DNOW  = (1 << 0)
+    FEATURE_SSE4A  = (1 << 1)
+    FEATURE_AVX    = (1 << 2)
+    FEATURE_RAND   = (1 << 3)
+    FEATURE_SHA    = (1 << 4)
+    FEATURE_CLZERO = (1 << 5)
+    FEATURE_BMI1   = (1 << 6)
+    FEATURE_BMI2   = (1 << 7)
 
 RegStringLookup = {
     "NONE":  Regs.REG_NONE,
@@ -128,8 +135,14 @@ ModeStringLookup = {
 }
 
 HostFeaturesLookup = {
-    "ANY" : HostFeatures.ANY,
-    "AVX" : HostFeatures.AVX_ONLY,
+    "3DNOW"  : HostFeatures.FEATURE_3DNOW,
+    "SSE4A"  : HostFeatures.FEATURE_SSE4A,
+    "AVX"    : HostFeatures.FEATURE_AVX,
+    "RAND"   : HostFeatures.FEATURE_RAND,
+    "SHA"    : HostFeatures.FEATURE_SHA,
+    "CLZERO" : HostFeatures.FEATURE_CLZERO,
+    "BMI1"   : HostFeatures.FEATURE_BMI1,
+    "BMI2"   : HostFeatures.FEATURE_BMI2,
 }
 
 def parse_hexstring(s):
@@ -152,7 +165,7 @@ def parse_json(json_text, output_file):
     OptionIgnore = Regs.REG_NONE
     OptionABI = ABI.ABI_SYSTEMV
     OptionMode = Mode.MODE_64
-    OptionHostFeatures = HostFeatures.ANY
+    OptionHostFeatures = HostFeatures.FEATURE_ANY
     OptionStackSize = 4096
     OptionEntryPoint = 1
     OptionRegData = {}
@@ -208,10 +221,15 @@ def parse_json(json_text, output_file):
 
     if ("HOSTFEATURES" in json_object):
         data = json_object["HOSTFEATURES"]
-        data = data.upper()
-        if not (data in HostFeaturesLookup):
-            sys.exit("Invalid host feature")
-        OptionHostFeatures = HostFeaturesLookup[data]
+        if not (type(data) is list):
+            sys.exit("HostFeatures value must be list of features")
+
+        for data_key in data:
+            data_key = data_key.upper()
+            if not (data_key in HostFeaturesLookup):
+                sys.exit("Invalid host feature")
+
+            OptionHostFeatures |= HostFeaturesLookup[data_key]
 
     if ("STACKSIZE" in json_object):
         data = json_object["STACKSIZE"]

Source/Common/CMakeLists.txt

@@ -8,6 +8,6 @@ set(SRCS
   StringUtil.cpp)
 
 add_library(${NAME} STATIC ${SRCS})
-target_link_libraries(${NAME} FEXCore_Base cpp-optparse json-maker)
+target_link_libraries(${NAME} FEXCore_Base cpp-optparse json-maker FEXHeaderUtils)
 target_include_directories(${NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/External/cpp-optparse/)
 target_include_directories(${NAME} PRIVATE ${CMAKE_BINARY_DIR}/generated)

Source/Common/FDUtils.h

@@ -5,12 +5,13 @@
 #include <fcntl.h>
 #include <filesystem>
 #include <linux/limits.h>
+#include <optional>
 #include <unistd.h>
 
 namespace FEX {
 [[maybe_unused]]
 static
-std::string get_fdpath(int fd) {
+std::optional<std::string> get_fdpath(int fd) {
   char SymlinkPath[PATH_MAX];
   std::filesystem::path Path = std::filesystem::path("/proc/self/fd") / std::to_string(fd);
   int Result = readlinkat(AT_FDCWD, Path.c_str(), SymlinkPath, sizeof(SymlinkPath));
@@ -18,8 +19,8 @@ std::string get_fdpath(int fd) {
     return std::string(SymlinkPath, Result);
   }
 
-  LOGMAN_MSG_A_FMT("Couldn't get symlink from /proc/self/fd/{}", fd);
-  return {};
+  // Not fatal if an FD doesn't point to a file
+  return std::nullopt;
 }
 
 }

Source/Common/FEXServerClient.cpp

@@ -101,7 +101,12 @@ namespace FEXServerClient {
   }
 
   std::string GetServerSocketFile() {
-    return fmt::format("{}/{}.FEXServer.socket", std::filesystem::temp_directory_path().string(), ::geteuid());
+    FEX_CONFIG_OPT(ServerSocketPath, SERVERSOCKETPATH);
+    if (ServerSocketPath().empty()) {
+      return fmt::format("{}/{}.FEXServer.socket", std::filesystem::temp_directory_path().string(), ::geteuid());
+    }
+
+    return ServerSocketPath;
   }
 
   int GetServerFD() {

Source/Tests/CMakeLists.txt

@@ -2,8 +2,14 @@ add_subdirectory(LinuxSyscalls)
 
 list(APPEND LIBS FEXCore Common)
 
+if (TERMUX_BUILD)
+  # Termux needs android-shmem to get the shm emulation library.
+  list(APPEND LIBS android-shmem)
+endif()
+
 add_executable(FEXLoader
   FEXLoader.cpp
+  VDSO_Emulation.cpp
   AOT/AOTGenerator.cpp)
 
 # Enable FEX APIs to be used by targets that use target_link_libraries on FEXLoader
@@ -45,10 +51,10 @@ if(TERMUX_BUILD)
   )
 
   install(
-    CODE "MESSAGE(\"-- Installing: ${CMAKE_INSTALL_PREFIX}/bin/FEXInterpreter\")"
+    CODE "MESSAGE(\"-- Installing: $ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/bin/FEXInterpreter\")"
     CODE "
     EXECUTE_PROCESS(COMMAND cp FEXLoader FEXInterpreter
-      WORKING_DIRECTORY ${CMAKE_INSTALL_PREFIX}/bin/
+      WORKING_DIRECTORY $ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/bin/
     )"
   )
 else()
@@ -58,12 +64,20 @@ else()
   )
 
   install(
-    CODE "MESSAGE(\"-- Installing: ${CMAKE_INSTALL_PREFIX}/bin/FEXInterpreter\")"
+    CODE "MESSAGE(\"-- Installing: $ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/bin/FEXInterpreter\")"
     CODE "
     EXECUTE_PROCESS(COMMAND ln -f FEXLoader FEXInterpreter
-      WORKING_DIRECTORY ${CMAKE_INSTALL_PREFIX}/bin/
+      WORKING_DIRECTORY $ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/bin/
     )"
   )
+
+  if(TARGET uninstall)
+    add_custom_target(uninstall_FEXInterpreter
+      COMMAND "rm" "$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/bin/FEXInterpreter"
+    )
+
+    add_dependencies(uninstall uninstall_FEXInterpreter)
+  endif()
 endif()
 
 install(PROGRAMS "${PROJECT_SOURCE_DIR}/Scripts/FEXUpdateAOTIRCache.sh" DESTINATION bin RENAME FEXUpdateAOTIRCache)
@@ -95,6 +109,17 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
       COMMAND ${CMAKE_COMMAND} -E
       echo "binfmt_misc FEX-x86_64 installed"
     )
+    if(TARGET uninstall)
+      add_custom_target(uninstall_binfmt_misc_32
+        COMMAND update-binfmts --unimport FEX-x86 || (exit 0)
+      )
+      add_custom_target(uninstall_binfmt_misc_64
+        COMMAND update-binfmts --unimport FEX-x86_64 || (exit 0)
+      )
+
+      add_dependencies(uninstall uninstall_binfmt_misc_32)
+      add_dependencies(uninstall uninstall_binfmt_misc_64)
+    endif()
   else()
     # In the case of update-binfmts not being available (Arch for example) then we need to install manually
     add_custom_target(binfmt_misc_32
@@ -123,6 +148,20 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
       COMMAND ${CMAKE_COMMAND} -E
         echo "binfmt_misc FEX-x86_64 installed"
       )
+
+    if(TARGET uninstall)
+      add_custom_target(uninstall_binfmt_misc_32
+        COMMAND ${CMAKE_COMMAND} -E
+          echo -1 > /proc/sys/fs/binfmt_misc/FEX-x86 || (exit 0)
+      )
+      add_custom_target(uninstall_binfmt_misc_64
+        COMMAND ${CMAKE_COMMAND} -E
+          echo -1 > /proc/sys/fs/binfmt_misc/FEX-x86_64 || (exit 0)
+      )
+
+      add_dependencies(uninstall uninstall_binfmt_misc_32)
+      add_dependencies(uninstall uninstall_binfmt_misc_64)
+    endif()
   endif()
 
   add_custom_target(binfmt_misc
@@ -159,18 +198,20 @@ install(TARGETS FEXBash
     COMPONENT runtime
 )
 
-add_executable(TestHarnessRunner TestHarnessRunner/HostRunner.cpp TestHarnessRunner.cpp)
-target_include_directories(TestHarnessRunner
-  PRIVATE
-    ${CMAKE_CURRENT_SOURCE_DIR}/Source/
-    ${CMAKE_BINARY_DIR}/generated
-)
-target_link_libraries(TestHarnessRunner
-  PRIVATE
-    ${LIBS}
-    LinuxEmulation
-    ${PTHREAD_LIB}
-)
+
+if (BUILD_TESTS)
+  add_executable(TestHarnessRunner TestHarnessRunner/HostRunner.cpp TestHarnessRunner.cpp)
+  target_include_directories(TestHarnessRunner
+    PRIVATE
+      ${CMAKE_CURRENT_SOURCE_DIR}/Source/
+      ${CMAKE_BINARY_DIR}/generated
+  )
+  target_link_libraries(TestHarnessRunner
+    PRIVATE
+      ${LIBS}
+      LinuxEmulation
+      ${PTHREAD_LIB}
+  )
 
 # add_executable(UnitTestGenerator UnitTestGenerator.cpp)
 # target_include_directories(UnitTestGenerator
@@ -184,19 +225,19 @@ target_link_libraries(TestHarnessRunner
 # )
 #
 
-add_executable(IRLoader
-  IRLoader.cpp
-)
-target_include_directories(IRLoader
-  PRIVATE
-    ${CMAKE_CURRENT_SOURCE_DIR}/Source/
-    ${CMAKE_BINARY_DIR}/generated
-)
-target_link_libraries(IRLoader
-  PRIVATE
-    ${LIBS}
-    LinuxEmulation
-    ${PTHREAD_LIB}
-    fmt::fmt
-)
-
+  add_executable(IRLoader
+    IRLoader.cpp
+  )
+  target_include_directories(IRLoader
+    PRIVATE
+      ${CMAKE_CURRENT_SOURCE_DIR}/Source/
+      ${CMAKE_BINARY_DIR}/generated
+  )
+  target_link_libraries(IRLoader
+    PRIVATE
+      ${LIBS}
+      LinuxEmulation
+      ${PTHREAD_LIB}
+      fmt::fmt
+  )
+endif()

Source/Tests/ELFCodeLoader2.h

@@ -3,6 +3,7 @@
 
 #include "Common/Config.h"
 #include "Common/FDUtils.h"
+#include "FEXCore/Utils/Allocator.h"
 #include "Tests/LinuxSyscalls/Syscalls.h"
 #include "Linux/Utils/ELFParser.h"
 #include "Linux/Utils/ELFSymbolDatabase.h"
@@ -13,6 +14,7 @@
 #include <cstring>
 #include <filesystem>
 #include <fstream>
+#include <random>
 #include <string>
 #include <vector>
 
@@ -21,6 +23,7 @@
 #include <FEXCore/Core/X86Enums.h>
 #include <FEXCore/Utils/LogManager.h>
 #include <FEXHeaderUtils/Syscalls.h>
+#include <FEXHeaderUtils/TypeDefines.h>
 
 #include <elf.h>
 #include <fcntl.h>
@@ -79,7 +82,9 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
       return false;
     } else {
       auto Filename = FEX::get_fdpath(file.fd);
-      Sections.push_back({Base, (uintptr_t)rv, size, (off_t)off, Filename, (prot & PROT_EXEC) != 0});
+      if (Filename.has_value()) {
+        Sections.push_back({Base, (uintptr_t)rv, size, (off_t)off, Filename.value(), (prot & PROT_EXEC) != 0});
+      }
 
       return true;
     }
@@ -101,7 +106,7 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
   }
 
   template <typename TMap, typename TUnmap>
-  std::optional<uintptr_t> LoadElfFile(ELFParser& Elf, uintptr_t *BrkBase, TMap Mapper, TUnmap Unmapper) {
+  std::optional<uintptr_t> LoadElfFile(ELFParser& Elf, uintptr_t *BrkBase, TMap Mapper, TUnmap Unmapper, uint64_t LoadHint = 0) {
 
     uintptr_t LoadBase = 0;
 
@@ -112,14 +117,11 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
     if (Elf.ehdr.e_type == ET_DYN) {
       // needs base address
       auto TotalSize = CalculateTotalElfSize(Elf.phdrs) + (BrkBase ? BRK_SIZE : 0);
-      LoadBase = (uintptr_t)Mapper(0, TotalSize, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+      LoadBase = (uintptr_t)Mapper(reinterpret_cast<void*>(LoadHint), TotalSize, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
       if ((void*)LoadBase == MAP_FAILED) {
         return {};
       }
 
-      if (Unmapper((void*)LoadBase, TotalSize) == -1) {
-        return {};
-      }
       //fprintf(stderr, "elf %d: %lx-%lx\n", Elf.fd, LoadBase, LoadBase + TotalSize);
       if (BrkBase) {
         *BrkBase = LoadBase + (TotalSize - BRK_SIZE);
@@ -130,10 +132,10 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
       if (Header.p_type != PT_LOAD)
         continue;
 
-			int MapProt = MapFlags(Header);
-      int MapType = MAP_PRIVATE | MAP_DENYWRITE | MAP_FIXED_NOREPLACE;
+      int MapProt = MapFlags(Header);
+      int MapType = MAP_PRIVATE | MAP_DENYWRITE | MAP_FIXED;
 
-			if (!MapFile(Elf, LoadBase, Header, MapProt, MapType, Mapper)) {
+      if (!MapFile(Elf, LoadBase, Header, MapProt, MapType, Mapper)) {
         return {};
       }
 
@@ -207,6 +209,7 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
   };
 
   std::vector<LoadedSection> Sections;
+
   ELFCodeLoader2(std::string const &Filename, std::string const &RootFS, [[maybe_unused]] std::vector<std::string> const &args, std::vector<std::string> const &ParsedArgs, char **const envp = nullptr, FEXCore::Config::Value<std::string> *AdditionalEnvp = nullptr) :
     Args {args} {
 
@@ -357,33 +360,43 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
       return false;
     }
 
-    // load the main elf
-
-    uintptr_t BrkBase = 0;
-
-    uintptr_t LoadBase = 0;
-
-    if (auto elf = LoadElfFile(MainElf, &BrkBase, Mapper, Unmapper)) {
-      LoadBase = *elf;
-      if (MainElf.ehdr.e_type == ET_DYN) {
-        BaseOffset = LoadBase;
-      }
-    } else {
-      LogMan::Msg::EFmt("Failed to load elf file");
-      return false;
-    }
-
-    // XXX Randomise brk?
-
-    BrkStart = (uint64_t)Mapper((void*)BrkBase, BRK_SIZE, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, -1, 0);
-
-    if ((void*)BrkStart == MAP_FAILED) {
-      LogMan::Msg::EFmt("Failed to allocate BRK @ {:x}, {}\n", BrkBase, errno);
-      return false;
-    }
-
-    MainElfBase = LoadBase + MainElf.phdrs.front().p_vaddr - MainElf.phdrs.front().p_offset;
-    MainElfEntrypoint = LoadBase + MainElf.ehdr.e_entry;
+    // Load the interpreter ELF first.
+    // This allows the top-down allocation of the kernel to put this at the top of the VA space.
+    // This matches behaviour of native execution more closely.
+    //
+    // eg:
+    // 555555554000-555555558000 r--p 00000000 103:0a 1311400                   /usr/bin/ls
+    // 555555558000-55555556c000 r-xp 00004000 103:0a 1311400                   /usr/bin/ls
+    // 55555556c000-555555574000 r--p 00018000 103:0a 1311400                   /usr/bin/ls
+    // 555555575000-555555577000 rw-p 00020000 103:0a 1311400                   /usr/bin/ls
+    // 555555577000-555555578000 rw-p 00000000 00:00 0                          [heap]
+    // 7ffff7fbb000-7ffff7fbd000 rw-p 00000000 00:00 0
+    // 7ffff7fbd000-7ffff7fc1000 r--p 00000000 00:00 0                          [vvar]
+    // 7ffff7fc1000-7ffff7fc3000 r-xp 00000000 00:00 0                          [vdso]
+    // 7ffff7fc3000-7ffff7fc5000 r--p 00000000 103:0a 1316948                   /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
+    // 7ffff7fc5000-7ffff7fef000 r-xp 00002000 103:0a 1316948                   /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
+    // 7ffff7fef000-7ffff7ffa000 r--p 0002c000 103:0a 1316948                   /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
+    // 7ffff7ffb000-7ffff7fff000 rw-p 00037000 103:0a 1316948                   /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
+    // 7ffffffdd000-7ffffffff000 rw-p 00000000 00:00 0                          [stack]
+    // ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0                  [vsyscall]
+    //
+    // ARM:
+    // 55ccaf8b1000-55ccaf8b5000 r--p 00000000 00:2a 4                          /tmp/.FEXMount178532-oiFrTF/usr/bin/ls
+    // 55ccaf8b5000-55ccaf8c9000 r-xp 00004000 00:2a 4                          /tmp/.FEXMount178532-oiFrTF/usr/bin/ls
+    // 55ccaf8c9000-55ccaf8d1000 r--p 00018000 00:2a 4                          /tmp/.FEXMount178532-oiFrTF/usr/bin/ls
+    // 55ccaf8d1000-55ccaf8d2000 ---p 00000000 00:00 0
+    // 55ccaf8d2000-55ccaf8d4000 rw-p 00020000 00:2a 4                          /tmp/.FEXMount178532-oiFrTF/usr/bin/ls
+    // 55ccaf8d4000-55ccb00d5000 rw-p 00000000 00:00 0
+    // <... Snip of misc allocations ...>
+    // 7fffff6c2000-7fffff6c4000 r--p 00000000 00:2a 22                         /tmp/.FEXMount178532-oiFrTF/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
+    // 7fffff6c4000-7fffff6ee000 r-xp 00002000 00:2a 22                         /tmp/.FEXMount178532-oiFrTF/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
+    // 7fffff6ee000-7fffff6f9000 r--p 0002c000 00:2a 22                         /tmp/.FEXMount178532-oiFrTF/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
+    // 7fffff6f9000-7fffff6fa000 ---p 00000000 00:00 0
+    // 7fffff6fa000-7fffff6fe000 rw-p 00037000 00:2a 22                         /tmp/.FEXMount178532-oiFrTF/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
+    // 7fffff7fe000-7fffffffe000 rw-p 00000000 00:00 0
+    // 7fffffffe000-7ffffffff000 r--p 00000000 08:82 7082611                    /usr/share/fex-emu/GuestThunks/libVDSO-guest.so
+    // 7ffffffff000-800000000000 rw-p 00000000 00:00 0
+    uint64_t ELFLoadHint = 0;
 
     if (!MainElf.InterpreterElf.empty()) {
       uint64_t InterpLoadBase = 0;
@@ -396,7 +409,83 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
 
       InterpeterElfBase = InterpLoadBase + InterpElf.phdrs.front().p_vaddr - InterpElf.phdrs.front().p_offset;
       Entrypoint = InterpLoadBase + InterpElf.ehdr.e_entry;
+
+      // If the ELF has an interpreter and is dynamic then we should provide a address hint for loading.
+      // The kernel calculates this `load_bias` by dividing the task size by three then multiplying by two.
+      // It then also offsets by a random number for ASLR purposes.
+      //
+      // Random number that gets added to the base needs to be in the number of bits (multiplied by pages):
+      // 64-bit: [28, 32] bits
+      // 32-bit: [8, 16] bits
+      // By default the /minimum/ number of bits is used here.
+      constexpr uint64_t TASK_SIZE_64 = (1ULL << 47);
+      constexpr uint64_t TASK_SIZE_32 = (1ULL << 32);
+      if (Is64BitMode()) {
+        // Ensure that if we are running on a 36-bit VA system, we don't try hinting that an ELF should
+        // live way outside the VA space.
+        uint64_t HostVASize = 1ULL << FEXCore::Allocator::DetermineVASize();
+        ELFLoadHint = std::min(HostVASize, TASK_SIZE_64) / 3 * 2;
+      }
+      else {
+        ELFLoadHint = TASK_SIZE_32 / 3 * 2;
+      }
+#define ASLR_LOAD
+#ifdef ASLR_LOAD
+      // Only enable ASLR randomization if the personality has it enabled.
+      uint32_t Personality = personality(~0ULL);
+      bool NoRandomize = (Personality & ADDR_NO_RANDOMIZE) == ADDR_NO_RANDOMIZE;
+
+      if (!NoRandomize) {
+        constexpr uint64_t ASLR_BITS_64 = 28;
+        constexpr uint64_t ASLR_BITS_32 = 8;
+        std::random_device rd;
+        std::uniform_int_distribution<uint64_t> d(0);
+        uint64_t ASLR_Offset = d(rd);
+
+        if (Is64BitMode()) {
+          ASLR_Offset &= (1ULL << ASLR_BITS_64) - 1;
+        }
+        else {
+          ASLR_Offset &= (1ULL << ASLR_BITS_32) - 1;
+        }
+
+        ASLR_Offset <<= FHU::FEX_PAGE_SHIFT;
+        ELFLoadHint += ASLR_Offset;
+      }
+#endif
+      // Align the mapping
+      ELFLoadHint &= FHU::FEX_PAGE_MASK;
+    }
+
+    // load the main elf
+
+    uintptr_t BrkBase = 0;
+
+    uintptr_t LoadBase = 0;
+
+    if (auto elf = LoadElfFile(MainElf, &BrkBase, Mapper, Unmapper, ELFLoadHint)) {
+      LoadBase = *elf;
+      if (MainElf.ehdr.e_type == ET_DYN) {
+        BaseOffset = LoadBase;
+      }
     } else {
+      LogMan::Msg::EFmt("Failed to load elf file");
+      return false;
+    }
+
+    // XXX Randomise brk?
+
+    BrkStart = (uint64_t)Mapper((void*)BrkBase, BRK_SIZE, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0);
+
+    if ((void*)BrkStart == MAP_FAILED) {
+      LogMan::Msg::EFmt("Failed to allocate BRK @ {:x}, {}\n", BrkBase, errno);
+      return false;
+    }
+
+    MainElfBase = LoadBase + MainElf.phdrs.front().p_vaddr - MainElf.phdrs.front().p_offset;
+    MainElfEntrypoint = LoadBase + MainElf.ehdr.e_entry;
+
+    if (MainElf.InterpreterElf.empty()) {
       InterpeterElfBase = 0;
       Entrypoint = MainElfEntrypoint;
     }
@@ -424,17 +513,20 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
       // On x86 only allows userspace to check for monitor and fs/gs base writing in CPL3
       //AuxVariables.emplace_back(auxv_t{26, 0}); // AT_HWCAP2
 
-      // we don't support vsyscall or vDSO so we don't set those
+      // we don't support vsyscall so we don't set those
       //AuxVariables.emplace_back(auxv_t{32, 0}); // AT_SYSINFO - Entry point to syscall
-      //AuxVariables.emplace_back(auxv_t{33, 0}); // AT_SYSINFO_EHDR - Address of the start of VDSO
     }
     else {
       AuxVariables.emplace_back(auxv_t{4, 0x20}); // AT_PHENT
 
-      // we don't support vsyscall or vDSO so we don't set those
+      // we don't support vsyscall so we don't set those
       //AuxVariables.emplace_back(auxv_t{32, 0}); // AT_SYSINFO - Entry point to syscall
-      //AuxVariables.emplace_back(auxv_t{33, 0}); // AT_SYSINFO_EHDR - Address of the start of VDSO
     }
+
+    if (VDSOBase) {
+      AuxVariables.emplace_back(auxv_t{33, reinterpret_cast<uint64_t>(VDSOBase)}); // AT_SYSINFO_EHDR - Address of the start of VDSO
+    }
+
     AuxVariables.emplace_back(auxv_t{3, MainElfBase + MainElf.ehdr.e_phoff}); // Program header
     AuxVariables.emplace_back(auxv_t{7, InterpeterElfBase}); // AT_BASE - Interpreter address
     AuxVariables.emplace_back(auxv_t{9, MainElfEntrypoint}); // AT_ENTRY
@@ -638,6 +730,10 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
     return ElfValid;
   }
 
+  void SetVDSOBase(void* Base) {
+    VDSOBase = Base;
+  }
+
   constexpr static uint64_t BRK_SIZE = 8 * 1024 * 1024;
   constexpr static uint64_t STACK_SIZE = 8 * 1024 * 1024;
 
@@ -650,6 +746,7 @@ class ELFCodeLoader2 final : public FEXCore::CodeLoader {
   uint64_t ArgumentBackingSize{};
   uint64_t EnvironmentBackingSize{};
   uint64_t BaseOffset{};
-  FEX_CONFIG_OPT(AdditionalArguments, ADDITIONALARGUMENTS);
+  void* VDSOBase{};
 
+  FEX_CONFIG_OPT(AdditionalArguments, ADDITIONALARGUMENTS);
 };

Source/Tests/FEXLoader.cpp

@@ -9,6 +9,7 @@ $end_info$
 #include "Common/ArgumentLoader.h"
 #include "Common/FEXServerClient.h"
 #include "ELFCodeLoader2.h"
+#include "VDSO_Emulation.h"
 #include "Tests/LinuxSyscalls/LinuxAllocator.h"
 #include "Tests/LinuxSyscalls/Syscalls.h"
 #include "Tests/LinuxSyscalls/x32/Syscalls.h"
@@ -327,11 +328,12 @@ int main(int argc, char **argv, char **const envp) {
     return -ENOEXEC;
   }
 
-  FEXCore::Config::Set(FEXCore::Config::CONFIG_APP_FILENAME, std::filesystem::canonical(Program.first).string());
-  FEXCore::Config::Set(FEXCore::Config::CONFIG_IS64BIT_MODE, Loader.Is64BitMode() ? "1" : "0");
+  FEXCore::Config::EraseSet(FEXCore::Config::CONFIG_APP_FILENAME, std::filesystem::canonical(Program.first).string());
+  FEXCore::Config::EraseSet(FEXCore::Config::CONFIG_APP_CONFIG_NAME, Program.second);
+  FEXCore::Config::EraseSet(FEXCore::Config::CONFIG_IS64BIT_MODE, Loader.Is64BitMode() ? "1" : "0");
 
   std::unique_ptr<FEX::HLE::MemAllocator> Allocator;
-  FEXCore::Allocator::PtrCache *Base48Bit{};
+  std::vector<FEXCore::Allocator::MemoryRegion> Base48Bit;
 
   if (Loader.Is64BitMode()) {
     // Destroy the 48th bit if it exists
@@ -388,6 +390,10 @@ int main(int argc, char **argv, char **const envp) {
   auto Mapper = std::bind_front(&FEX::HLE::SyscallHandler::GuestMmap, SyscallHandler.get());
   auto Unmapper = std::bind_front(&FEX::HLE::SyscallHandler::GuestMunmap, SyscallHandler.get());
 
+  // Load VDSO in to memory prior to mapping our ELFs.
+  void* VDSOBase = FEX::VDSO::LoadVDSOThunks(Loader.Is64BitMode(), Mapper);
+  Loader.SetVDSOBase(VDSOBase);
+
   if (!Loader.MapMemory(Mapper, Unmapper)) {
     // failed to map
     LogMan::Msg::EFmt("Failed to map %d-bit elf file.", Loader.Is64BitMode() ? 64 : 32);
@@ -404,6 +410,9 @@ int main(int argc, char **argv, char **const envp) {
   FEXCore::Context::SetSyscallHandler(CTX, SyscallHandler.get());
   FEXCore::Context::InitCore(CTX, Loader.DefaultRIP(), Loader.GetStackPointer());
 
+  // Pass in our VDSO thunks
+  FEXCore::Context::AppendThunkDefinitions(CTX, FEX::VDSO::GetVDSOThunkDefinitions());
+
   FEXCore::Context::ExitReason ShutdownReason = FEXCore::Context::ExitReason::EXIT_SHUTDOWN;
 
   // There might already be an exit handler, leave it installed

Source/Tests/HarnessHelpers.h

@@ -374,7 +374,27 @@ namespace FEX::HarnessHelper {
     }
 
     bool Is64BitMode() const { return BaseConfig.OptionMode == 1; }
-    bool RequiresAVX() const { return BaseConfig.OptionHostFeatures == 1; }
+
+    enum HostFeatures {
+      FEATURE_ANY    = 0,
+      FEATURE_3DNOW  = (1 << 0),
+      FEATURE_SSE4A  = (1 << 1),
+      FEATURE_AVX    = (1 << 2),
+      FEATURE_RAND   = (1 << 3),
+      FEATURE_SHA    = (1 << 4),
+      FEATURE_CLZERO = (1 << 5),
+      FEATURE_BMI1   = (1 << 6),
+      FEATURE_BMI2   = (1 << 7),
+    };
+
+    bool Requires3DNow()  const { return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_3DNOW; }
+    bool RequiresSSE4A()  const { return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_SSE4A; }
+    bool RequiresAVX()    const { return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_AVX; }
+    bool RequiresRAND()   const { return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_RAND; }
+    bool RequiresSHA()    const { return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_SHA; }
+    bool RequiresCLZERO() const { return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_CLZERO; }
+    bool RequiresBMI1()   const { return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_BMI1; }
+    bool RequiresBMI2()   const { return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_BMI2; }
 
   private:
     FEX_CONFIG_OPT(ConfigDumpGPRs, DUMPGPRS);
@@ -506,7 +526,14 @@ namespace FEX::HarnessHelper {
     }
 
     bool Is64BitMode() const { return Config.Is64BitMode(); }
-    bool RequiresAVX() const { return Config.RequiresAVX(); }
+    bool Requires3DNow()  const { return Config.Requires3DNow(); }
+    bool RequiresSSE4A()  const { return Config.RequiresSSE4A(); }
+    bool RequiresAVX()    const { return Config.RequiresAVX(); }
+    bool RequiresRAND()   const { return Config.RequiresRAND(); }
+    bool RequiresSHA()    const { return Config.RequiresSHA(); }
+    bool RequiresCLZERO() const { return Config.RequiresCLZERO(); }
+    bool RequiresBMI1()   const { return Config.RequiresBMI1(); }
+    bool RequiresBMI2()   const { return Config.RequiresBMI2(); }
 
   private:
     constexpr static uint64_t STACK_SIZE = FHU::FEX_PAGE_SIZE;

Source/Tests/LinuxSyscalls/Arm64/SyscallsEnum.h

@@ -204,10 +204,10 @@ enum Syscalls_Arm64 {
   SYSCALL_Arm64_semctl = 191,
   SYSCALL_Arm64_semtimedop = 192,
   SYSCALL_Arm64_semop = 193,
-  SYSCALL_Arm64_shmget = 194,
-  SYSCALL_Arm64_shmctl = 195,
-  SYSCALL_Arm64_shmat = 196,
-  SYSCALL_Arm64_shmdt = 197,
+  SYSCALL_Arm64__shmget = 194,
+  SYSCALL_Arm64__shmctl = 195,
+  SYSCALL_Arm64__shmat = 196,
+  SYSCALL_Arm64__shmdt = 197,
   SYSCALL_Arm64_socket = 198,
   SYSCALL_Arm64_socketpair = 199,
   SYSCALL_Arm64_bind = 200,

Source/Tests/LinuxSyscalls/EmulatedFiles/EmulatedFiles.cpp

@@ -714,7 +714,7 @@ namespace FEX::EmulatedFile {
         dirfs != AT_FDCWD) {
       // Passed in a dirfd that isn't magic FDCWD
       // We need to get the path from the fd now
-      Path = FEX::get_fdpath(dirfs);
+      Path = FEX::get_fdpath(dirfs).value_or("");
 
       if (pathname) {
         if (!Path.empty()) {

Source/Tests/LinuxSyscalls/FileManagement.cpp

@@ -7,6 +7,7 @@ $end_info$
 
 #include "Common/FDUtils.h"
 
+#include "FEXCore/Config/Config.h"
 #include "Tests/LinuxSyscalls/FileManagement.h"
 #include "Tests/LinuxSyscalls/EmulatedFiles/EmulatedFiles.h"
 #include "Tests/LinuxSyscalls/Syscalls.h"
@@ -222,16 +223,81 @@ void FileManager::LoadThunkDatabase(bool Global) {
 FileManager::FileManager(FEXCore::Context::Context *ctx)
   : EmuFD {ctx} {
 
+  bool LoadedThunkDatabase{};
   auto ThunkConfigFile = ThunkConfig();
+  auto ThunkGuestPath = std::filesystem::path(ThunkGuestLibs());
 
-  if (ThunkConfigFile.size()) {
+  auto LoadThunksDB = [this, ThunkGuestPath](bool *LoadedThunkDatabase, json_t const* ThunksDB) {
+    // If a thunks DB property exists then we pull in data from the thunks database
+    // Load the initial thunks database
+    if (LoadedThunkDatabase) {
+      LoadThunkDatabase(true);
+      LoadThunkDatabase(false);
+      *LoadedThunkDatabase = true;
+    }
 
-    auto ThunkGuestPath = std::filesystem::path(ThunkGuestLibs());
+    // Now load this property
+    for (json_t const* Item = json_getChild(ThunksDB); Item != nullptr; Item = json_getSibling(Item)) {
+      const char *LibraryName = json_getName(Item);
+      int64_t LibraryEnabled = json_getInteger(Item);
+      if (LibraryEnabled != 0) {
+        // If the library is enabled then find it in the DB
+        // Enable the overlay and all the dependencies in one go
+        auto DBObject = ThunkDB.find(LibraryName);
+        if (DBObject != ThunkDB.end() &&
+            DBObject->second.Enabled == false) {
 
+          auto ThunkPath = ThunkGuestPath / DBObject->second.LibraryName;
+          if (std::filesystem::exists(ThunkPath)) {
+            for (auto Overlay : DBObject->second.Overlays) {
+              // Direct full path in guest RootFS to our overlay file
+              ThunkOverlays.emplace(Overlay, ThunkPath);
+            }
+          }
+          DBObject->second.Enabled = true;
+          // Now walk the dependencies and set them up as well
+          // Make sure to enable each one as we go to remove circular dependencies
+          std::function<void(std::unordered_set<std::string> &Depends)> InsertDependencies
+            = [this, &ThunkGuestPath, &InsertDependencies](std::unordered_set<std::string> &Depends) -> void {
+            for (auto &Depend : Depends) {
+              auto DBDepend = ThunkDB.find(Depend);
+              if (DBDepend != ThunkDB.end() &&
+                  DBDepend->second.Enabled == false) {
+
+                auto ThunkPath = ThunkGuestPath / DBDepend->second.LibraryName;
+                if (std::filesystem::exists(ThunkPath)) {
+                  for (auto Overlay : DBDepend->second.Overlays) {
+                    // Direct full path in guest RootFS to our overlay file
+                    ThunkOverlays.emplace(Overlay, ThunkPath);
+                  }
+                }
+
+                // Enabled, now walk this dependencies
+                DBDepend->second.Enabled = true;
+                InsertDependencies(DBDepend->second.Depends);
+              }
+            }
+          };
+          InsertDependencies(DBObject->second.Depends);
+        }
+      }
+    }
+  };
+
+  // We try to load ThunksDB from {FEX global config, FEX user config, AppConfig Global, AppConfig Local, Defined ThunksConfig option}
+  // This doesn't support the classic thunks interface.
+
+  std::vector<std::string> ConfigPaths {
+    FEXCore::Config::GetConfigFileLocation(true),
+    FEXCore::Config::GetConfigFileLocation(false),
+    FEXCore::Config::GetApplicationConfig(AppConfigName(), true),
+    FEXCore::Config::GetApplicationConfig(AppConfigName(), false),
+    ThunkConfigFile,
+  };
+
+  for (const auto &Path : ConfigPaths) {
     std::vector<char> FileData;
-    if (LoadFile(FileData, ThunkConfigFile)) {
-      FileData.push_back(0);
-
+    if (LoadFile(FileData, Path)) {
       JSON::JsonAllocator Pool {
         .PoolObject = {
           .init = JSON::PoolInit,
@@ -240,101 +306,22 @@ FileManager::FileManager(FEXCore::Context::Context *ctx)
       };
 
       json_t const *json = json_createWithPool(&FileData.at(0), &Pool.PoolObject);
-
-      json_t const* thunks = json_getProperty( json, "thunks" );
-      if (thunks && json_getType(thunks) == JSON_OBJ) {
-        json_t const* thunk;
-        for( thunk = json_getChild( thunks ); thunk != 0; thunk = json_getSibling( thunk )) {
-          char const* GuestThunk = json_getName( thunk );
-          jsonType_t propertyType = json_getType( thunk );
-
-          if (propertyType == JSON_TEXT) {
-            char const* RootFSLib = json_getValue( thunk );
-            auto ThunkPath = ThunkGuestPath / GuestThunk;
-            if (std::filesystem::exists(ThunkPath)) {
-              ThunkOverlays.emplace(RootFSLib, ThunkPath);
-            }
-          } else if (propertyType == JSON_ARRAY) {
-            json_t const* child;
-            for( child = json_getChild( thunk ); child != 0; child = json_getSibling( child ) ) {
-              if (json_getType( child ) == JSON_TEXT) {
-                char const* RootFSLib = json_getValue( child );
-                auto ThunkPath = ThunkGuestPath / GuestThunk;
-                if (std::filesystem::exists(ThunkPath)) {
-                  ThunkOverlays.emplace(RootFSLib, ThunkPath);
-                }
-              }
-            }
-          }
-        }
-      }
-
       json_t const* ThunksDB = json_getProperty( json, "ThunksDB" );
       if (ThunksDB) {
-        // If a thunks DB property exists then we pull in data from the thunks database
-        // Load the initial thunks database
-        LoadThunkDatabase(true);
-        LoadThunkDatabase(false);
-
-        // Now load this property
-        for (json_t const* Item = json_getChild(ThunksDB); Item != nullptr; Item = json_getSibling(Item)) {
-          const char *LibraryName = json_getName(Item);
-          int64_t LibraryEnabled = json_getInteger(Item);
-          if (LibraryEnabled != 0) {
-            // If the library is enabled then find it in the DB
-            // Enable the overlay and all the dependencies in one go
-            auto DBObject = ThunkDB.find(LibraryName);
-            if (DBObject != ThunkDB.end() &&
-                DBObject->second.Enabled == false) {
-
-              auto ThunkPath = ThunkGuestPath / DBObject->second.LibraryName;
-              if (std::filesystem::exists(ThunkPath)) {
-                for (auto Overlay : DBObject->second.Overlays) {
-                  // Direct full path in guest RootFS to our overlay file
-                  ThunkOverlays.emplace(Overlay, ThunkPath);
-                }
-              }
-              DBObject->second.Enabled = true;
-              // Now walk the dependencies and set them up as well
-              // Make sure to enable each one as we go to remove circular dependencies
-              std::function<void(std::unordered_set<std::string> &Depends)> InsertDependencies
-                = [this, &ThunkGuestPath, &InsertDependencies](std::unordered_set<std::string> &Depends) -> void {
-                for (auto &Depend : Depends) {
-                  auto DBDepend = ThunkDB.find(Depend);
-                  if (DBDepend != ThunkDB.end() &&
-                      DBDepend->second.Enabled == false) {
-
-                    auto ThunkPath = ThunkGuestPath / DBDepend->second.LibraryName;
-                    if (std::filesystem::exists(ThunkPath)) {
-                      for (auto Overlay : DBDepend->second.Overlays) {
-                        // Direct full path in guest RootFS to our overlay file
-                        ThunkOverlays.emplace(Overlay, ThunkPath);
-                      }
-                    }
-
-                    // Enabled, now walk this dependencies
-                    DBDepend->second.Enabled = true;
-                    InsertDependencies(DBDepend->second.Depends);
-                  }
-                }
-              };
-              InsertDependencies(DBObject->second.Depends);
-            }
-          }
-        }
-
-        // Now clear the thunk database since we're loaded
-        ThunkDB.clear();
+        LoadThunksDB(&LoadedThunkDatabase, ThunksDB);
       }
     }
+  }
 
-    if (false) {
-      // Useful for debugging
-      if (ThunkOverlays.size()) {
-        LogMan::Msg::IFmt("Thunk Overlays:");
-        for (const auto& [Overlay, ThunkPath] : ThunkOverlays) {
-          LogMan::Msg::IFmt("\t{} -> {}", Overlay, ThunkPath);
-        }
+  // Now clear the thunk database since we're loaded
+  ThunkDB.clear();
+
+  if (false) {
+    // Useful for debugging
+    if (ThunkOverlays.size()) {
+      LogMan::Msg::IFmt("Thunk Overlays:");
+      for (const auto& [Overlay, ThunkPath] : ThunkOverlays) {
+        LogMan::Msg::IFmt("\t{} -> {}", Overlay, ThunkPath);
       }
     }
   }
@@ -575,7 +562,7 @@ uint64_t FileManager::Readlinkat(int dirfd, const char *pathname, char *buf, siz
         dirfd != AT_FDCWD) {
     // Passed in a dirfd that isn't magic FDCWD
     // We need to get the path from the fd now
-    Path = FEX::get_fdpath(dirfd);
+    Path = FEX::get_fdpath(dirfd).value_or("");
 
     if (pathname) {
       if (!Path.empty()) {

Source/Tests/LinuxSyscalls/FileManagement.h

@@ -80,6 +80,7 @@ private:
   FEX_CONFIG_OPT(ThunkHostLibs, THUNKHOSTLIBS);
   FEX_CONFIG_OPT(ThunkGuestLibs, THUNKGUESTLIBS);
   FEX_CONFIG_OPT(ThunkConfig, THUNKCONFIG);
+  FEX_CONFIG_OPT(AppConfigName, APP_CONFIG_NAME);
   uint32_t CurrentPID{};
 
   void LoadThunkDatabase(bool Global);

Source/Tests/LinuxSyscalls/LinuxAllocator.cpp

@@ -44,11 +44,11 @@ public:
       FindPageRangePtr = &MemAllocator32Bit::FindPageRange;
     }
   }
-  void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) override;
-  int munmap(void *addr, size_t length) override;
-  void *mremap(void *old_address, size_t old_size, size_t new_size, int flags, void *new_address) override;
-  uint64_t shmat(int shmid, const void* shmaddr, int shmflg, uint32_t *ResultAddress) override;
-  uint64_t shmdt(const void* shmaddr) override;
+  void *Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) override;
+  int Munmap(void *addr, size_t length) override;
+  void *Mremap(void *old_address, size_t old_size, size_t new_size, int flags, void *new_address) override;
+  uint64_t Shmat(int shmid, const void* shmaddr, int shmflg, uint32_t *ResultAddress) override;
+  uint64_t Shmdt(const void* shmaddr) override;
   static constexpr bool SearchDown = true;
 
   // PageAddr is a page already shifted to page index
@@ -131,7 +131,7 @@ uint64_t MemAllocator32Bit::FindPageRange_TopDown(uint64_t Start, size_t Pages)
   return 0;
 }
 
-void *MemAllocator32Bit::mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
+void *MemAllocator32Bit::Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
   std::scoped_lock<std::mutex> lk{AllocMutex};
   size_t PagesLength = FEXCore::AlignUp(length, FHU::FEX_PAGE_SIZE) >> FHU::FEX_PAGE_SHIFT;
 
@@ -282,7 +282,7 @@ restart:
   return 0;
 }
 
-int MemAllocator32Bit::munmap(void *addr, size_t length) {
+int MemAllocator32Bit::Munmap(void *addr, size_t length) {
   std::scoped_lock<std::mutex> lk{AllocMutex};
   size_t PagesLength = FEXCore::AlignUp(length, FHU::FEX_PAGE_SIZE) >> FHU::FEX_PAGE_SHIFT;
 
@@ -327,7 +327,7 @@ int MemAllocator32Bit::munmap(void *addr, size_t length) {
   return 0;
 }
 
-void *MemAllocator32Bit::mremap(void *old_address, size_t old_size, size_t new_size, int flags, void *new_address) {
+void *MemAllocator32Bit::Mremap(void *old_address, size_t old_size, size_t new_size, int flags, void *new_address) {
   size_t OldPagesLength = FEXCore::AlignUp(old_size, FHU::FEX_PAGE_SIZE) >> FHU::FEX_PAGE_SHIFT;
   size_t NewPagesLength = FEXCore::AlignUp(new_size, FHU::FEX_PAGE_SIZE) >> FHU::FEX_PAGE_SHIFT;
 
@@ -404,7 +404,7 @@ void *MemAllocator32Bit::mremap(void *old_address, size_t old_size, size_t new_s
   // New Size is >= old size
 
   // First, try and allocate a region the size of the new size
-  void *MappedPtr = this->mmap(nullptr, new_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  void *MappedPtr = this->Mmap(nullptr, new_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
   std::scoped_lock<std::mutex> lk{AllocMutex};
   if (FEX::HLE::HasSyscallError(MappedPtr)) {
     // Couldn't find a region that fit our space
@@ -435,7 +435,7 @@ void *MemAllocator32Bit::mremap(void *old_address, size_t old_size, size_t new_s
   return reinterpret_cast<void*>(-errno);
 }
 
-uint64_t MemAllocator32Bit::shmat(int shmid, const void* shmaddr, int shmflg, uint32_t *ResultAddress) {
+uint64_t MemAllocator32Bit::Shmat(int shmid, const void* shmaddr, int shmflg, uint32_t *ResultAddress) {
   std::scoped_lock<std::mutex> lk{AllocMutex};
 
   if (shmaddr != nullptr) {
@@ -548,9 +548,9 @@ restart:
     }
   }
 }
-uint64_t MemAllocator32Bit::shmdt(const void* shmaddr) {
+uint64_t MemAllocator32Bit::Shmdt(const void* shmaddr) {
   std::scoped_lock<std::mutex> lk{AllocMutex};
-  
+
   uint32_t AddrPage = reinterpret_cast<uint64_t>(shmaddr) >> FHU::FEX_PAGE_SHIFT;
   auto it = PageToShm.find(AddrPage);
 
@@ -567,7 +567,7 @@ uint64_t MemAllocator32Bit::shmdt(const void* shmaddr) {
 
 class MemAllocatorPassThrough final : public FEX::HLE::MemAllocator {
 public:
-  void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) override {
+  void *Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) override {
     uint64_t Result = (uint64_t)::mmap(addr, length, prot, flags, fd, offset);
     if (Result == ~0ULL) {
       return reinterpret_cast<void*>(-errno);
@@ -575,12 +575,12 @@ public:
     return reinterpret_cast<void*>(Result);
   }
 
-  int munmap(void *addr, size_t length) override {
+  int Munmap(void *addr, size_t length) override {
     uint64_t Result = (uint64_t)::munmap(addr, length);
     SYSCALL_ERRNO();
   }
 
-  void *mremap(void *old_address, size_t old_size, size_t new_size, int flags, void *new_address) override {
+  void *Mremap(void *old_address, size_t old_size, size_t new_size, int flags, void *new_address) override {
     uint64_t Result = (uint64_t)::mremap(old_address, old_size, new_size, flags, new_address);
     if (Result == ~0ULL) {
       return reinterpret_cast<void*>(-errno);
@@ -588,7 +588,7 @@ public:
     return reinterpret_cast<void*>(Result);
   }
 
-  uint64_t shmat(int shmid, const void* shmaddr, int shmflg, uint32_t *ResultAddress) override {
+  uint64_t Shmat(int shmid, const void* shmaddr, int shmflg, uint32_t *ResultAddress) override {
     uint64_t Result = (uint64_t)::shmat(shmid, reinterpret_cast<const void*>(shmaddr), shmflg);
     if (Result != ~0ULL) {
       *ResultAddress = Result;
@@ -597,7 +597,7 @@ public:
     SYSCALL_ERRNO();
   }
 
-  uint64_t shmdt(const void* shmaddr) override {
+  uint64_t Shmdt(const void* shmaddr) override {
     uint64_t Result = ::shmdt(shmaddr);
     SYSCALL_ERRNO();
   }

Source/Tests/LinuxSyscalls/LinuxAllocator.h

@@ -9,11 +9,11 @@ namespace FEX::HLE {
   class MemAllocator {
   public:
     virtual ~MemAllocator() = default;
-    virtual void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) = 0;
-    virtual int munmap(void *addr, size_t length) = 0;
-    virtual void *mremap(void *old_address, size_t old_size, size_t new_size, int flags, void *new_address) = 0;
-    virtual uint64_t shmat(int shmid, const void* shmaddr, int shmflg, uint32_t *ResultAddress) = 0;
-    virtual uint64_t shmdt(const void* shmaddr) = 0;
+    virtual void *Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) = 0;
+    virtual int Munmap(void *addr, size_t length) = 0;
+    virtual void *Mremap(void *old_address, size_t old_size, size_t new_size, int flags, void *new_address) = 0;
+    virtual uint64_t Shmat(int shmid, const void* shmaddr, int shmflg, uint32_t *ResultAddress) = 0;
+    virtual uint64_t Shmdt(const void* shmaddr) = 0;
   };
 
   std::unique_ptr<FEX::HLE::MemAllocator> Create32BitAllocator();

Source/Tests/LinuxSyscalls/Syscalls.cpp

@@ -14,6 +14,8 @@ $end_info$
 #include "Tests/LinuxSyscalls/Syscalls/Thread.h"
 #include "Tests/LinuxSyscalls/x32/Syscalls.h"
 #include "Tests/LinuxSyscalls/x64/Syscalls.h"
+#include "Tests/LinuxSyscalls/x32/Types.h"
+#include "Tests/LinuxSyscalls/x64/Types.h"
 
 #include <FEXCore/Config/Config.h>
 #include <FEXCore/Core/Context.h>
@@ -58,6 +60,112 @@ namespace FEX::HLE {
 class SignalDelegator;
 SyscallHandler *_SyscallHandler{};
 
+
+template<bool IncrementOffset, typename T>
+uint64_t GetDentsEmulation(int fd, T *dirp, uint32_t count) {
+  std::vector<uint8_t> TmpVector(count);
+  void *TmpPtr = reinterpret_cast<void*>(&TmpVector.at(0));
+
+  uint64_t Offset = 0;
+  uint64_t TmpOffset = 0;
+  // Copy the incoming structures to our temporary array
+  while (Offset < count) {
+    T *Incoming = (T*)(reinterpret_cast<uint64_t>(dirp) + Offset);
+    FEX::HLE::x64::linux_dirent_64 *Tmp = (FEX::HLE::x64::linux_dirent_64*)(reinterpret_cast<uint64_t>(TmpPtr) + TmpOffset);
+
+    if (!Incoming->d_reclen ||
+        (Offset + Incoming->d_reclen) > count) {
+      break;
+    }
+
+    size_t NewRecLen = FEXCore::AlignUp(Incoming->d_reclen + (sizeof(std::remove_reference<decltype(*Tmp)>::type) - sizeof(*Incoming)),
+                                        alignof(decltype(Tmp->d_ino)));
+    Tmp->d_ino    = Incoming->d_ino;
+    Tmp->d_off    = Incoming->d_off;
+    Tmp->d_reclen = NewRecLen;
+
+    // d_type is hidden at the very end of reclen
+    Tmp->d_type   = Incoming->d_name[Incoming->d_reclen - offsetof(T, d_name) - 1];
+
+    // This actually copies one more byte than the string of d_name
+    // Copies a null byte for the string
+    size_t CopySize = std::clamp<uint32_t>(Incoming->d_reclen - offsetof(T, d_name) - 1, 0U, count - Offset);
+    memcpy(Tmp->d_name, Incoming->d_name, CopySize);
+
+    // We take up 8 more bytes of space
+    TmpOffset += NewRecLen;
+    Offset += Incoming->d_reclen;
+  }
+
+  uint64_t Result = syscall(SYSCALL_DEF(getdents64),
+    static_cast<uint64_t>(fd),
+    TmpPtr,
+    static_cast<uint64_t>(count));
+
+  // Now copy back in to the array we were given
+  if (Result != -1) {
+    // If the outgoing d_ino is smaller than the incoming d_ino from the kernel
+    // Then we need to check for overflow before writing any of the data back
+    if (sizeof(decltype(FEX::HLE::x64::linux_dirent_64::d_ino)) > sizeof(decltype(T::d_ino))) {
+      uint64_t TmpOffset = 0;
+      while (TmpOffset < Result) {
+        FEX::HLE::x64::linux_dirent_64 *Tmp = (FEX::HLE::x64::linux_dirent_64*)(reinterpret_cast<uint64_t>(TmpPtr) + TmpOffset);
+        decltype(T::d_ino) Result_d_ino = Tmp->d_ino;
+
+        if (Result_d_ino != Tmp->d_ino) {
+          // The resulting d_ino truncated, return error
+          return -EOVERFLOW;
+        }
+        TmpOffset += Tmp->d_reclen;
+      }
+    }
+
+    uint64_t Offset = 0;
+    uint64_t TmpOffset = 0;
+    size_t OffsetIndex = 1;
+    // With how the emulation occurs we will always return a smaller buffer than what was given to us
+    while (TmpOffset < Result) {
+      T *Outgoing = (T*)(reinterpret_cast<uint64_t>(dirp) + Offset);
+      FEX::HLE::x64::linux_dirent_64 *Tmp = (FEX::HLE::x64::linux_dirent_64*)(reinterpret_cast<uint64_t>(TmpPtr) + TmpOffset);
+
+      if (!Tmp->d_reclen) {
+        break;
+      }
+
+      size_t NewRecLen = FEXCore::AlignUp(Tmp->d_reclen - (sizeof(std::remove_reference<decltype(*Tmp)>::type) - sizeof(*Outgoing)),
+                                          alignof(decltype(Tmp->d_ino)));
+      Outgoing->d_ino = Tmp->d_ino;
+
+      // 32-bit getdents can't safely handle d_off
+      // A safe way of emulating this is to just use an incrementing offset from 1
+      Outgoing->d_off = IncrementOffset ? OffsetIndex : Tmp->d_off;
+      size_t OffsetOfName = offsetof(std::remove_reference<decltype(*Tmp)>::type, d_name);
+      Outgoing->d_reclen = NewRecLen;
+
+      // Copies null character as well
+      size_t NameLength = Tmp->d_reclen - OffsetOfName - 1;
+      memcpy(Outgoing->d_name, Tmp->d_name, NameLength);
+
+      // Copy the hidden d_type flag
+      Outgoing->d_name[Outgoing->d_reclen - offsetof(T, d_name) - 1] = Tmp->d_type;
+
+      TmpOffset += Tmp->d_reclen;
+      // Outgoing is 5 bytes smaller
+      Offset += NewRecLen;
+
+      ++OffsetIndex;
+    }
+    Result = Offset;
+  }
+  SYSCALL_ERRNO();
+}
+
+template
+uint64_t GetDentsEmulation<false>(int, FEX::HLE::x64::linux_dirent*, uint32_t);
+
+template
+uint64_t GetDentsEmulation<true>(int, FEX::HLE::x32::linux_dirent_32*, uint32_t);
+
 static bool IsSupportedByInterpreter(std::string const &Filename) {
   // If it is a supported ELF then we can
   if (ELFLoader::ELFContainer::IsSupportedELF(Filename.c_str())) {

Source/Tests/LinuxSyscalls/Syscalls.h

@@ -509,6 +509,8 @@ static bool HasSyscallError(const void* Result) {
   return HasSyscallError(reinterpret_cast<uintptr_t>(Result));
 }
 
+template<bool IncrementOffset, typename T>
+uint64_t GetDentsEmulation(int fd, T *dirp, uint32_t count);
 }
 
 // Registers syscall for both 32bit and 64bit

Source/Tests/LinuxSyscalls/Syscalls/SHM.cpp

@@ -18,14 +18,14 @@ namespace FEX::HLE {
   void RegisterSHM(FEX::HLE::SyscallHandler *Handler) {
     using namespace FEXCore::IR;
 
-    REGISTER_SYSCALL_IMPL_PASS_FLAGS(shmget, SyscallFlags::OPTIMIZETHROUGH | SyscallFlags::NOSYNCSTATEONENTRY,
+    REGISTER_SYSCALL_IMPL_PASS_FLAGS(_shmget, SyscallFlags::OPTIMIZETHROUGH | SyscallFlags::NOSYNCSTATEONENTRY,
       [](FEXCore::Core::CpuStateFrame *Frame, key_t key, size_t size, int shmflg) -> uint64_t {
       uint64_t Result = shmget(key, size, shmflg);
       SYSCALL_ERRNO();
     });
 
     // XXX: shmid_ds is definitely not correct for 32-bit
-    REGISTER_SYSCALL_IMPL_PASS_FLAGS(shmctl, SyscallFlags::OPTIMIZETHROUGH | SyscallFlags::NOSYNCSTATEONENTRY,
+    REGISTER_SYSCALL_IMPL_PASS_FLAGS(_shmctl, SyscallFlags::OPTIMIZETHROUGH | SyscallFlags::NOSYNCSTATEONENTRY,
       [](FEXCore::Core::CpuStateFrame *Frame, int shmid, int cmd, struct shmid_ds *buf) -> uint64_t {
       uint64_t Result = ::shmctl(shmid, cmd, buf);
       SYSCALL_ERRNO();

Source/Tests/LinuxSyscalls/SyscallsSMCTracking.cpp

@@ -199,16 +199,17 @@ void SyscallHandler::TrackMmap(uintptr_t Base, uintptr_t Size, int Prot, int Fla
       fstat64(fd, &buf);
       MRID mrid {buf.st_dev, buf.st_ino};
 
-      auto [Iter, Inserted] = VMATracking.MappedResources.emplace(mrid, MappedResource {nullptr, nullptr, 0});
-      Resource = &Iter->second;
+      auto filename = FEX::get_fdpath(fd);
 
-      if (Inserted) {
-        auto filename = FEX::get_fdpath(fd);
+      if (filename.has_value()) {
+        auto [Iter, Inserted] = VMATracking.MappedResources.emplace(mrid, MappedResource {nullptr, nullptr, 0});
+        Resource = &Iter->second;
 
-        Resource->AOTIRCacheEntry = FEXCore::Context::LoadAOTIRCacheEntry(CTX, filename);
-        Resource->Iterator = Iter;
+        if (Inserted) {
+          Resource->AOTIRCacheEntry = FEXCore::Context::LoadAOTIRCacheEntry(CTX, filename.value());
+          Resource->Iterator = Iter;
+        }
       }
-
     } else if (Flags & MAP_SHARED) {
       MRID mrid{SpecialDev::Anon, AnonSharedId++};
 
@@ -340,9 +341,12 @@ void SyscallHandler::TrackShmat(int shmid, uintptr_t Base, int shmflg) {
 }
 
 void SyscallHandler::TrackShmdt(uintptr_t Base) {
-  FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex);
+  uintptr_t Length = 0;
+  {
+    FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex);
 
-  auto Length = VMATracking.ClearShmUnsafe(CTX, Base);
+    Length = VMATracking.ClearShmUnsafe(CTX, Base);
+  }
 
   if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) {
     // This might over flush if the shm has holes in it

Source/Tests/LinuxSyscalls/x32/FD.cpp

@@ -583,75 +583,7 @@ namespace FEX::HLE::x32 {
     REGISTER_SYSCALL_IMPL_X32(ioctl, ioctl32);
 
     REGISTER_SYSCALL_IMPL_X32(getdents, [](FEXCore::Core::CpuStateFrame *Frame, int fd, void *dirp, uint32_t count) -> uint64_t {
-#ifdef SYS_getdents
-      std::vector<uint8_t> TmpVector(count);
-      void *TmpPtr = reinterpret_cast<void*>(&TmpVector.at(0));
-
-      // Copy the incoming structures to our temporary array
-      for (uint64_t Offset = 0, TmpOffset = 0;
-          Offset < count;) {
-        linux_dirent_32 *Incoming = (linux_dirent_32*)(reinterpret_cast<uint64_t>(dirp) + Offset);
-        linux_dirent *Tmp = (linux_dirent*)(reinterpret_cast<uint64_t>(TmpPtr) + TmpOffset);
-
-        if (!Incoming->d_reclen ||
-            (Offset + Incoming->d_reclen) > count) {
-          break;
-        }
-
-        size_t NewRecLen = Incoming->d_reclen + (sizeof(linux_dirent) - sizeof(linux_dirent_32));
-        Tmp->d_ino    = Incoming->d_ino;
-        Tmp->d_off    = Incoming->d_off;
-        Tmp->d_reclen = NewRecLen;
-
-        // This actually copies two more bytes than the string of d_name
-        // Copies a null byte for the string
-        // Copies a d_type flag that lives after the name
-        size_t CopySize = std::clamp<uint32_t>(Incoming->d_reclen - offsetof(linux_dirent_32, d_name), 0U, count - Offset);
-        memcpy(Tmp->d_name, Incoming->d_name, CopySize);
-
-        // We take up 8 more bytes of space
-        TmpOffset += NewRecLen;
-        Offset += Incoming->d_reclen;
-      }
-
-      uint64_t Result = syscall(SYSCALL_DEF(getdents),
-        static_cast<uint64_t>(fd),
-        TmpPtr,
-        static_cast<uint64_t>(count));
-
-      // Now copy back in to the array we were given
-      if (Result != -1) {
-        uint64_t Offset = 0;
-        // With how the emulation occurs we will always return a smaller buffer than what was given to us
-        for (uint64_t TmpOffset = 0, num = 0; TmpOffset < Result; ++num) {
-          linux_dirent_32 *Outgoing = (linux_dirent_32*)(reinterpret_cast<uint64_t>(dirp) + Offset);
-          linux_dirent *Tmp = (linux_dirent*)(reinterpret_cast<uint64_t>(TmpPtr) + TmpOffset);
-
-          if (!Tmp->d_reclen) {
-            break;
-          }
-
-          size_t NewRecLen = Tmp->d_reclen - (sizeof(std::remove_reference<decltype(*Tmp)>::type) - sizeof(*Outgoing));
-          Outgoing->d_ino = Tmp->d_ino;
-          // If we pass d_off directly then we seem to encounter issues?
-          Outgoing->d_off = num; //Tmp->d_off;
-          size_t OffsetOfName = offsetof(std::remove_reference<decltype(*Tmp)>::type, d_name);
-          Outgoing->d_reclen = NewRecLen;
-
-          // Copies null character and d_type flag as well
-          memcpy(Outgoing->d_name, Tmp->d_name, Tmp->d_reclen - OffsetOfName);
-
-          TmpOffset += Tmp->d_reclen;
-          // Outgoing is 8 bytes smaller
-          Offset += NewRecLen;
-        }
-        Result = Offset;
-      }
-      SYSCALL_ERRNO();
-#else
-      // XXX: Emulate
-      return -ENOSYS;
-#endif
+      return GetDentsEmulation<true>(fd, reinterpret_cast<FEX::HLE::x32::linux_dirent_32*>(dirp), count);
     });
 
     REGISTER_SYSCALL_IMPL_X32(getdents64, [](FEXCore::Core::CpuStateFrame *Frame, int fd, void *dirp, uint32_t count) -> uint64_t {

Source/Tests/LinuxSyscalls/x32/Memory.cpp

@@ -23,7 +23,7 @@ namespace FEX::HLE::x32 {
   void *x32SyscallHandler::GuestMmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
     LOGMAN_THROW_AA_FMT((length >> 32) == 0, "values must fit to 32 bits");
 
-    auto Result = (uint64_t)GetAllocator()->mmap((void*)addr, length, prot, flags, fd, offset);
+    auto Result = (uint64_t)GetAllocator()->Mmap((void*)addr, length, prot, flags, fd, offset);
 
     LOGMAN_THROW_AA_FMT((Result >> 32) == 0|| (Result >> 32) == 0xFFFFFFFF, "values must fit to 32 bits");
 
@@ -40,7 +40,7 @@ namespace FEX::HLE::x32 {
     LOGMAN_THROW_AA_FMT((uintptr_t(addr) >> 32) == 0, "values must fit to 32 bits");
     LOGMAN_THROW_AA_FMT((length >> 32) == 0, "values must fit to 32 bits");
 
-    auto Result = GetAllocator()->munmap(addr, length);
+    auto Result = GetAllocator()->Munmap(addr, length);
 
     if (Result == 0) {
       FEX::HLE::_SyscallHandler->TrackMunmap((uintptr_t)addr, length);
@@ -92,7 +92,7 @@ namespace FEX::HLE::x32 {
 
     REGISTER_SYSCALL_IMPL_X32(mremap, [](FEXCore::Core::CpuStateFrame *Frame, void *old_address, size_t old_size, size_t new_size, int flags, void *new_address) -> uint64_t {
       uint64_t Result = reinterpret_cast<uint64_t>(static_cast<FEX::HLE::x32::x32SyscallHandler*>(FEX::HLE::_SyscallHandler)->GetAllocator()->
-        mremap(old_address, old_size, new_size, flags, new_address));
+        Mremap(old_address, old_size, new_size, flags, new_address));
 
       if (!FEX::HLE::HasSyscallError(Result)) {
         FEX::HLE::_SyscallHandler->TrackMremap((uintptr_t)old_address, old_size, new_size, flags, Result);
@@ -111,11 +111,11 @@ namespace FEX::HLE::x32 {
       SYSCALL_ERRNO();
     });
 
-    REGISTER_SYSCALL_IMPL_X32(shmat, [](FEXCore::Core::CpuStateFrame *Frame, int shmid, const void *shmaddr, int shmflg) -> uint64_t {
+    REGISTER_SYSCALL_IMPL_X32(_shmat, [](FEXCore::Core::CpuStateFrame *Frame, int shmid, const void *shmaddr, int shmflg) -> uint64_t {
       // also implemented in ipc:OP_SHMAT
       uint32_t ResultAddr{};
       uint64_t Result = static_cast<FEX::HLE::x32::x32SyscallHandler*>(FEX::HLE::_SyscallHandler)->GetAllocator()->
-          shmat(shmid, reinterpret_cast<const void*>(shmaddr), shmflg, &ResultAddr);
+          Shmat(shmid, reinterpret_cast<const void*>(shmaddr), shmflg, &ResultAddr);
 
       if (!FEX::HLE::HasSyscallError(Result)) {
         FEX::HLE::_SyscallHandler->TrackShmat(shmid, ResultAddr, shmflg);
@@ -126,11 +126,11 @@ namespace FEX::HLE::x32 {
       }
     });
 
-    REGISTER_SYSCALL_IMPL_X32(shmdt, [](FEXCore::Core::CpuStateFrame *Frame, const void *shmaddr) -> uint64_t {
+    REGISTER_SYSCALL_IMPL_X32(_shmdt, [](FEXCore::Core::CpuStateFrame *Frame, const void *shmaddr) -> uint64_t {
       // also implemented in ipc:OP_SHMDT
       uint64_t Result = static_cast<FEX::HLE::x32::x32SyscallHandler*>(FEX::HLE::_SyscallHandler)->GetAllocator()->
-        shmdt(shmaddr);
-      
+        Shmdt(shmaddr);
+
       if (!FEX::HLE::HasSyscallError(Result)) {
         FEX::HLE::_SyscallHandler->TrackShmdt((uintptr_t)shmaddr);
       }

Source/Tests/LinuxSyscalls/x32/Semaphore.cpp

@@ -281,7 +281,7 @@ namespace FEX::HLE::x32 {
       case OP_SHMAT: {
         // also implemented in memory:shmat
         Result = static_cast<FEX::HLE::x32::x32SyscallHandler*>(FEX::HLE::_SyscallHandler)->GetAllocator()->
-          shmat(first, reinterpret_cast<const void*>(ptr), second, reinterpret_cast<uint32_t*>(third));
+          Shmat(first, reinterpret_cast<const void*>(ptr), second, reinterpret_cast<uint32_t*>(third));
         if (!FEX::HLE::HasSyscallError(Result)) {
           FEX::HLE::_SyscallHandler->TrackShmat(first, *reinterpret_cast<uint32_t*>(third), second);
         }
@@ -290,7 +290,7 @@ namespace FEX::HLE::x32 {
       case OP_SHMDT: {
         // also implemented in memory:shmdt
         Result = static_cast<FEX::HLE::x32::x32SyscallHandler*>(FEX::HLE::_SyscallHandler)->GetAllocator()->
-          shmdt(reinterpret_cast<void*>(ptr));
+          Shmdt(reinterpret_cast<void*>(ptr));
         if (!FEX::HLE::HasSyscallError(Result)) {
           FEX::HLE::_SyscallHandler->TrackShmdt(ptr);
         }
@@ -317,7 +317,7 @@ namespace FEX::HLE::x32 {
             else {
               buf = *shmun.buf32;
             }
-            Result = ::syscall(SYSCALL_DEF(shmctl), shmid, cmd, &buf);
+            Result = ::syscall(SYSCALL_DEF(_shmctl), shmid, cmd, &buf);
             // IPC_SET sets the internal data structure that the kernel uses
             // No need to writeback
             break;
@@ -326,7 +326,7 @@ namespace FEX::HLE::x32 {
           case SHM_STAT_ANY:
           case IPC_STAT: {
             struct shmid64_ds buf{};
-            Result = ::syscall(SYSCALL_DEF(shmctl), shmid, cmd, &buf);
+            Result = ::syscall(SYSCALL_DEF(_shmctl), shmid, cmd, &buf);
             if (Result != -1) {
               if (IPC64) {
                 *shmun.buf64 = buf;
@@ -339,7 +339,7 @@ namespace FEX::HLE::x32 {
           }
           case IPC_INFO: {
             struct shminfo si{};
-            Result = ::syscall(SYSCALL_DEF(shmctl), shmid, cmd, reinterpret_cast<struct shmid_ds*>(&si));
+            Result = ::syscall(SYSCALL_DEF(_shmctl), shmid, cmd, reinterpret_cast<struct shmid_ds*>(&si));
             if (Result != -1) {
               if (IPC64) {
                 *shmun.__buf64 = si;
@@ -352,7 +352,7 @@ namespace FEX::HLE::x32 {
           }
           case SHM_INFO: {
             struct shm_info si{};
-            Result = ::syscall(SYSCALL_DEF(shmctl), shmid, cmd, reinterpret_cast<struct shmid_ds*>(&si));
+            Result = ::syscall(SYSCALL_DEF(_shmctl), shmid, cmd, reinterpret_cast<struct shmid_ds*>(&si));
             if (Result != -1) {
               // SHM_INFO doesn't follow IPC64 behaviour
               *shmun.__buf_info_32 = si;
@@ -360,13 +360,13 @@ namespace FEX::HLE::x32 {
             break;
           }
           case SHM_LOCK:
-            Result = ::syscall(SYSCALL_DEF(shmctl), shmid, cmd, nullptr);
+            Result = ::syscall(SYSCALL_DEF(_shmctl), shmid, cmd, nullptr);
             break;
           case SHM_UNLOCK:
-            Result = ::syscall(SYSCALL_DEF(shmctl), shmid, cmd, nullptr);
+            Result = ::syscall(SYSCALL_DEF(_shmctl), shmid, cmd, nullptr);
             break;
           case IPC_RMID:
-            Result = ::syscall(SYSCALL_DEF(shmctl), shmid, cmd, nullptr);
+            Result = ::syscall(SYSCALL_DEF(_shmctl), shmid, cmd, nullptr);
             break;
 
           default:

Source/Tests/LinuxSyscalls/x32/SyscallsEnum.h

@@ -410,10 +410,10 @@ enum Syscalls_x86 {
   SYSCALL_x86_rseq = 386,
   SYSCALL_x86_semget = 393,
   SYSCALL_x86_semctl = 394,
-  SYSCALL_x86_shmget = 395,
-  SYSCALL_x86_shmctl = 396,
-  SYSCALL_x86_shmat = 397,
-  SYSCALL_x86_shmdt = 398,
+  SYSCALL_x86__shmget = 395,
+  SYSCALL_x86__shmctl = 396,
+  SYSCALL_x86__shmat = 397,
+  SYSCALL_x86__shmdt = 398,
   SYSCALL_x86_msgget = 399,
   SYSCALL_x86_msgsnd = 400,
   SYSCALL_x86_msgrcv = 401,

Source/Tests/LinuxSyscalls/x64/FD.cpp

@@ -10,6 +10,7 @@ $end_info$
 #include "Tests/LinuxSyscalls/x64/Types.h"
 
 #include <FEXCore/Utils/CompilerDefs.h>
+#include <FEXCore/Utils/MathUtils.h>
 
 #include <fcntl.h>
 #include <poll.h>
@@ -196,16 +197,7 @@ namespace FEX::HLE::x64 {
     });
 
     REGISTER_SYSCALL_IMPL_X64(getdents, [](FEXCore::Core::CpuStateFrame *Frame, int fd, void *dirp, uint32_t count) -> uint64_t {
-  #ifdef SYS_getdents
-      uint64_t Result = syscall(SYSCALL_DEF(getdents),
-        static_cast<uint64_t>(fd),
-        reinterpret_cast<uint64_t>(dirp),
-        static_cast<uint64_t>(count));
-      SYSCALL_ERRNO();
-  #else
-      // XXX: Emulate
-      return -EFAULT;
-  #endif
+      return GetDentsEmulation<false>(fd, reinterpret_cast<FEX::HLE::x64::linux_dirent*>(dirp), count);
     });
 
     REGISTER_SYSCALL_IMPL_X64_PASS(getdents64, [](FEXCore::Core::CpuStateFrame *Frame, int fd, void *dirp, uint32_t count) -> uint64_t {

Source/Tests/LinuxSyscalls/x64/Memory.cpp

@@ -30,7 +30,7 @@ namespace FEX::HLE::x64 {
 
     bool Map32Bit = flags & FEX::HLE::X86_64_MAP_32BIT;
     if (Map32Bit) {
-      Result = (uint64_t)Get32BitAllocator()->mmap(addr, length, prot,flags, fd, offset);
+      Result = (uint64_t)Get32BitAllocator()->Mmap(addr, length, prot,flags, fd, offset);
       if (FEX::HLE::HasSyscallError(Result)) {
         errno = -Result;
         Result = -1;
@@ -49,7 +49,7 @@ namespace FEX::HLE::x64 {
   int x64SyscallHandler::GuestMunmap(void *addr, uint64_t length) {
     uint64_t Result{};
     if (reinterpret_cast<uintptr_t>(addr) < 0x1'0000'0000ULL) {
-      Result = Get32BitAllocator()->munmap(addr, length);
+      Result = Get32BitAllocator()->Munmap(addr, length);
 
       if (FEX::HLE::HasSyscallError(Result)) {
         errno = -Result;
@@ -117,7 +117,7 @@ namespace FEX::HLE::x64 {
       SYSCALL_ERRNO();
     });
 
-    REGISTER_SYSCALL_IMPL_X64_FLAGS(shmat, SyscallFlags::OPTIMIZETHROUGH | SyscallFlags::NOSYNCSTATEONENTRY,
+    REGISTER_SYSCALL_IMPL_X64_FLAGS(_shmat, SyscallFlags::OPTIMIZETHROUGH | SyscallFlags::NOSYNCSTATEONENTRY,
       [](FEXCore::Core::CpuStateFrame *Frame, int shmid, const void *shmaddr, int shmflg) -> uint64_t {
       uint64_t Result = reinterpret_cast<uint64_t>(shmat(shmid, shmaddr, shmflg));
 
@@ -127,7 +127,7 @@ namespace FEX::HLE::x64 {
       SYSCALL_ERRNO();
     });
 
-    REGISTER_SYSCALL_IMPL_X64_FLAGS(shmdt, SyscallFlags::OPTIMIZETHROUGH | SyscallFlags::NOSYNCSTATEONENTRY,
+    REGISTER_SYSCALL_IMPL_X64_FLAGS(_shmdt, SyscallFlags::OPTIMIZETHROUGH | SyscallFlags::NOSYNCSTATEONENTRY,
       [](FEXCore::Core::CpuStateFrame *Frame, const void *shmaddr) -> uint64_t {
       uint64_t Result = ::shmdt(shmaddr);
 

Source/Tests/LinuxSyscalls/x64/SyscallsEnum.h

@@ -36,9 +36,9 @@ enum Syscalls_x64 {
   SYSCALL_x64_msync = 26,
   SYSCALL_x64_mincore = 27,
   SYSCALL_x64_madvise = 28,
-  SYSCALL_x64_shmget = 29,
-  SYSCALL_x64_shmat = 30,
-  SYSCALL_x64_shmctl = 31,
+  SYSCALL_x64__shmget = 29,
+  SYSCALL_x64__shmat = 30,
+  SYSCALL_x64__shmctl = 31,
   SYSCALL_x64_dup = 32,
   SYSCALL_x64_dup2 = 33,
   SYSCALL_x64_pause = 34,
@@ -74,7 +74,7 @@ enum Syscalls_x64 {
   SYSCALL_x64_semget = 64,
   SYSCALL_x64_semop = 65,
   SYSCALL_x64_semctl = 66,
-  SYSCALL_x64_shmdt = 67,
+  SYSCALL_x64__shmdt = 67,
   SYSCALL_x64_msgget = 68,
   SYSCALL_x64_msgsnd = 69,
   SYSCALL_x64_msgrcv = 70,

Source/Tests/LinuxSyscalls/x64/Types.h

@@ -194,4 +194,37 @@ using __time_t = time_t;
   // Original definition in `arch/x86/include/uapi/asm/stat.h` for future excavation
   static_assert(std::is_trivial<FEX::HLE::x64::guest_stat>::value, "Needs to be trivial");
   static_assert(sizeof(FEX::HLE::x64::guest_stat) == 144, "Incorrect size");
+
+  // There is no public definition of this struct
+  // Matches the definition of `struct linux_dirent` in fs/readdir.c
+  struct
+  FEX_ANNOTATE("fex-match")
+  linux_dirent {
+    uint64_t d_ino;
+    uint64_t d_off;
+    uint16_t d_reclen;
+    char d_name[1];
+    /* Has hidden null character and d_type */
+  };
+  static_assert(std::is_trivial<linux_dirent>::value, "Needs to be trivial");
+  static_assert(offsetof(linux_dirent, d_ino) == 0, "Incorrect offset");
+  static_assert(offsetof(linux_dirent, d_off) == 8, "Incorrect offset");
+  static_assert(offsetof(linux_dirent, d_reclen) == 16, "Incorrect offset");
+  static_assert(offsetof(linux_dirent, d_name) == 18, "Incorrect offset");
+  static_assert(sizeof(linux_dirent) == 24, "Incorrect size");
+
+  // There is no public definition of this struct
+  // Matches the definition of `struct linux_dirent64` in include/linux/dirent.h
+  struct
+  FEX_ANNOTATE("fex-match")
+  FEX_PACKED
+  linux_dirent_64 {
+    uint64_t d_ino;
+    uint64_t d_off;
+    uint16_t d_reclen;
+    uint8_t  d_type;
+    char d_name[];
+  };
+  static_assert(std::is_trivial<linux_dirent_64>::value, "Needs to be trivial");
+  static_assert(sizeof(linux_dirent_64) == 19, "Incorrect size");
 }

Source/Tests/TestHarnessRunner.cpp

@@ -139,7 +139,7 @@ int main(int argc, char **argv, char **const envp) {
   FEX_CONFIG_OPT(Core, CORE);
 
   std::unique_ptr<FEX::HLE::MemAllocator> Allocator;
-   
+
   if (!Loader.Is64BitMode()) {
     // Setup our userspace allocator
     uint32_t KernelVersion = FEX::HLE::SyscallHandler::CalculateHostKernelVersion();
@@ -159,6 +159,32 @@ int main(int argc, char **argv, char **const envp) {
   bool DidFault = false;
   bool SupportsAVX = false;
   FEXCore::Core::CPUState State;
+
+  FEXCore::Context::InitializeStaticTables(Loader.Is64BitMode() ? FEXCore::Context::MODE_64BIT : FEXCore::Context::MODE_32BIT);
+
+  auto CTX = FEXCore::Context::CreateNewContext();
+
+  FEXCore::Context::InitializeContext(CTX);
+
+  // Skip any tests that the host doesn't support features for
+  auto HostFeatures = FEXCore::Context::GetHostFeatures(CTX);
+  SupportsAVX = HostFeatures.SupportsAVX;
+
+  bool TestUnsupported =
+    (!HostFeatures.Supports3DNow && Loader.Requires3DNow()) ||
+    (!HostFeatures.SupportsSSE4A && Loader.RequiresSSE4A()) ||
+    (!SupportsAVX && Loader.RequiresAVX()) ||
+    (!HostFeatures.SupportsRAND && Loader.RequiresRAND()) ||
+    (!HostFeatures.SupportsSHA && Loader.RequiresSHA()) ||
+    (!HostFeatures.SupportsCLZERO && Loader.RequiresCLZERO()) ||
+    (!HostFeatures.SupportsBMI1 && Loader.RequiresBMI1()) ||
+    (!HostFeatures.SupportsBMI2 && Loader.RequiresBMI2());
+
+  if (TestUnsupported) {
+    FEXCore::Context::DestroyContext(CTX);
+    return 0;
+  }
+
   if (Core != FEXCore::Config::CONFIG_CUSTOM) {
     jmp_buf LongJump{};
     int LongJumpVal{};
@@ -175,12 +201,6 @@ int main(int argc, char **argv, char **const envp) {
     }, true);
 
     // Run through FEX
-    FEXCore::Context::InitializeStaticTables(Loader.Is64BitMode() ? FEXCore::Context::MODE_64BIT : FEXCore::Context::MODE_32BIT);
-
-    auto CTX = FEXCore::Context::CreateNewContext();
-    
-    FEXCore::Context::InitializeContext(CTX);
-
     auto SyscallHandler = Loader.Is64BitMode() ? FEX::HLE::x64::CreateHandler(CTX, SignalDelegation.get())
                                                : FEX::HLE::x32::CreateHandler(CTX, SignalDelegation.get(), std::move(Allocator));
 
@@ -195,14 +215,13 @@ int main(int argc, char **argv, char **const envp) {
 
     FEXCore::Context::SetSignalDelegator(CTX, SignalDelegation.get());
     FEXCore::Context::SetSyscallHandler(CTX, SyscallHandler.get());
+
     bool Result1 = FEXCore::Context::InitCore(CTX, Loader.DefaultRIP(), Loader.GetStackPointer());
 
     if (!Result1) {
       return 1;
     }
 
-    SupportsAVX = FEXCore::Context::GetHostFeatures(CTX).SupportsAVX;
-
     LongJumpVal = setjmp(LongJump);
     if (!LongJumpVal) {
       FEXCore::Context::RunUntilExit(CTX);
@@ -212,8 +231,6 @@ int main(int argc, char **argv, char **const envp) {
     FEXCore::Context::GetCPUState(CTX, &State);
 
     SyscallHandler.reset();
-    FEXCore::Context::DestroyContext(CTX);
-    FEXCore::Context::ShutdownStaticTables();
   } else {
     // Run as host
     SupportsAVX = true;
@@ -227,12 +244,15 @@ int main(int argc, char **argv, char **const envp) {
     RunAsHost(SignalDelegation, Loader.DefaultRIP(), Loader.GetStackPointer(), &State);
   }
 
+  FEXCore::Context::DestroyContext(CTX);
+  FEXCore::Context::ShutdownStaticTables();
+
   bool Passed = !DidFault && Loader.CompareStates(&State, nullptr, SupportsAVX);
 
   LogMan::Msg::IFmt("Faulted? {}", DidFault ? "Yes" : "No");
   LogMan::Msg::IFmt("Passed? {}", Passed ? "Yes" : "No");
 
-  
+
   SignalDelegation.reset();
 
   FEXCore::Config::Shutdown();

Source/Tests/TestHarnessRunner/HostRunner.cpp

@@ -3,6 +3,7 @@
 #include <FEXCore/Core/CPUBackend.h>
 #include <FEXCore/Core/Context.h>
 #include <FEXCore/Core/CoreState.h>
+#include <FEXCore/Core/UContext.h>
 #include <FEXCore/Core/X86Enums.h>
 #include <FEXCore/Debug/InternalThreadState.h>
 #include <FEXCore/Utils/LogManager.h>
@@ -152,9 +153,16 @@ public:
     for (size_t i = 0; i < FEXCore::Core::CPUState::NUM_XMMS; ++i) {
       memcpy(&OutState->xmm.avx.data[i], &_mcontext->fpregs->_xmm[i], sizeof(_mcontext->fpregs->_xmm[0]));
     }
+    const auto* xstate = reinterpret_cast<FEXCore::x86_64::xstate*>(_mcontext->fpregs);
+    const auto* reserved = &xstate->fpstate.sw_reserved;
+    if (reserved->HasExtendedContext() && reserved->HasYMMH()) {
+      for (size_t i = 0; i < FEXCore::Core::CPUState::NUM_XMMS; i++) {
+        memcpy(&OutState->xmm.avx.data[i][2], &xstate->ymmh.ymmh_space[i],
+               sizeof(xstate->ymmh.ymmh_space[0]));
+      }
+    }
 
-    uint16_t CurrentOffset = (_mcontext->fpregs->swd >> 11) & 7;
-
+    const uint16_t CurrentOffset = (_mcontext->fpregs->swd >> 11) & 7;
     for (size_t i = 0; i < FEXCore::Core::CPUState::NUM_MMS; ++i) {
       memcpy(&OutState->mm[(i + CurrentOffset) % 8], &_mcontext->fpregs->_st[i], sizeof(_mcontext->fpregs->_st[0]));
     }

Source/Tests/VDSO_Emulation.cpp

@@ -0,0 +1,297 @@
+#include "VDSO_Emulation.h"
+#include "FEXCore/IR/IR.h"
+#include "Tests/LinuxSyscalls/x32/Types.h"
+
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Utils/MathUtils.h>
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXHeaderUtils/Syscalls.h>
+
+#include <dlfcn.h>
+#include <fcntl.h>
+#include <filesystem>
+#include <sys/mman.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+namespace FEX::VDSO {
+  using TimeType = decltype(::time)*;
+  using GetTimeOfDayType = decltype(::gettimeofday)*;
+  using ClockGetTimeType = decltype(::clock_gettime)*;
+  using ClockGetResType = decltype(::clock_getres)*;
+  using GetCPUType = decltype(FHU::Syscalls::getcpu)*;
+
+  TimeType TimePtr = ::time;
+  GetTimeOfDayType GetTimeOfDayPtr = ::gettimeofday;
+  ClockGetTimeType ClockGetTimePtr = ::clock_gettime;
+  ClockGetResType ClockGetResPtr = ::clock_getres;
+  GetCPUType GetCPUPtr = FHU::Syscalls::getcpu;
+
+  static void time(void* ArgsRV) {
+    struct ArgsRV_t {
+      time_t *a_0;
+      uint64_t rv;
+    } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
+
+    args->rv = TimePtr(args->a_0);
+  }
+
+  static void gettimeofday(void* ArgsRV) {
+    struct ArgsRV_t {
+      struct timeval *tv;
+      struct timezone *tz;
+      uint64_t rv;
+    } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
+
+    args->rv = GetTimeOfDayPtr(args->tv, args->tz);
+  }
+
+  static void clock_gettime(void* ArgsRV) {
+    struct ArgsRV_t {
+      clockid_t clk_id;
+      struct timespec *tp;
+      uint64_t rv;
+    } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
+
+    args->rv = ClockGetTimePtr(args->clk_id, args->tp);
+  }
+
+  static void clock_getres(void* ArgsRV) {
+    struct ArgsRV_t {
+      clockid_t clk_id;
+      struct timespec *tp;
+      uint64_t rv;
+    } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
+
+    args->rv = ClockGetResPtr(args->clk_id, args->tp);
+  }
+
+  static void getcpu(void* ArgsRV) {
+    struct ArgsRV_t {
+      uint32_t *cpu;
+      uint32_t *node;
+      uint64_t rv;
+    } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
+
+    args->rv = GetCPUPtr(args->cpu, args->node);
+  }
+
+  namespace x32 {
+    static void time(void* ArgsRV) {
+      struct ArgsRV_t {
+        HLE::x32::compat_ptr<FEX::HLE::x32::old_time32_t> a_0;
+        int rv;
+      } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
+
+      time_t Host{};
+      args->rv = TimePtr(&Host);
+      if (args->a_0) {
+        *args->a_0 = Host;
+      }
+    }
+
+    static void gettimeofday(void* ArgsRV) {
+      struct ArgsRV_t {
+        HLE::x32::compat_ptr<FEX::HLE::x32::timeval32> tv;
+        HLE::x32::compat_ptr<struct timezone> tz;
+        int rv;
+      } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
+
+      struct timeval tv64{};
+      struct timeval *tv_ptr{};
+      if (args->tv) {
+        tv_ptr = &tv64;
+      }
+
+      args->rv = GetTimeOfDayPtr(tv_ptr, args->tz);
+
+      if (args->tv) {
+        *args->tv = tv64;
+      }
+    }
+
+    static void clock_gettime(void* ArgsRV) {
+      struct ArgsRV_t {
+        clockid_t clk_id;
+        HLE::x32::compat_ptr<HLE::x32::timespec32> tp;
+        int rv;
+      } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
+
+      struct timespec tp64{};
+      args->rv = ClockGetTimePtr(args->clk_id, &tp64);
+
+      if (args->tp) {
+        *args->tp = tp64;
+      }
+    }
+
+    static void clock_gettime64(void* ArgsRV) {
+      struct ArgsRV_t {
+        clockid_t clk_id;
+        HLE::x32::compat_ptr<struct timespec> tp;
+        int rv;
+      } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
+
+      args->rv = ClockGetTimePtr(args->clk_id, args->tp);
+    }
+
+    static void clock_getres(void* ArgsRV) {
+      struct ArgsRV_t {
+        clockid_t clk_id;
+        HLE::x32::compat_ptr<HLE::x32::timespec32> tp;
+        int rv;
+      } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
+
+      struct timespec tp64{};
+
+      args->rv = ClockGetResPtr(args->clk_id, &tp64);
+
+      if (args->tp) {
+        *args->tp = tp64;
+      }
+    }
+
+    static void getcpu(void* ArgsRV) {
+      struct ArgsRV_t {
+        HLE::x32::compat_ptr<uint32_t> cpu;
+        HLE::x32::compat_ptr<uint32_t> node;
+        int rv;
+      } *args = reinterpret_cast<ArgsRV_t*>(ArgsRV);
+
+      args->rv = GetCPUPtr(args->cpu, args->node);
+    }
+  }
+
+  void LoadHostVDSO() {
+
+    void *vdso = dlopen("linux-vdso.so.1", RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);
+    if (!vdso) {
+      vdso = dlopen("linux-gate.so.1", RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);
+    }
+
+    if (!vdso) {
+      // We couldn't load VDSO, fallback to C implementations. Which will still be faster than emulated libc versions.
+      LogMan::Msg::IFmt("linux-vdso implementation falling back to libc. Consider enabling VDSO in your kernel.");
+      return;
+    }
+
+    auto SymbolPtr = dlsym(vdso, "__vdso_time");
+    if (SymbolPtr) {
+      TimePtr = reinterpret_cast<TimeType>(SymbolPtr);
+    }
+
+    SymbolPtr = dlsym(vdso, "__vdso_gettimeofday");
+    if (SymbolPtr) {
+      GetTimeOfDayPtr = reinterpret_cast<GetTimeOfDayType>(SymbolPtr);
+    }
+
+    SymbolPtr = dlsym(vdso, "__vdso_clock_gettime");
+    if (SymbolPtr) {
+      ClockGetTimePtr = reinterpret_cast<ClockGetTimeType>(SymbolPtr);
+    }
+
+    SymbolPtr = dlsym(vdso, "__vdso_clock_getres");
+    if (SymbolPtr) {
+      ClockGetResPtr = reinterpret_cast<ClockGetResType>(SymbolPtr);
+    }
+
+    SymbolPtr = dlsym(vdso, "__vdso_getcpu");
+    if (SymbolPtr) {
+      GetCPUPtr = reinterpret_cast<GetCPUType>(SymbolPtr);
+    }
+    dlclose(vdso);
+  }
+
+  static std::vector<FEXCore::IR::ThunkDefinition> VDSODefinitions = {
+    {
+        // sha256(libVDSO:time)
+        { 0x37, 0x63, 0x46, 0xb0, 0x79, 0x06, 0x5f, 0x9d, 0x00, 0xb6, 0x8d, 0xfd, 0x9e, 0x4a, 0x62, 0xcd, 0x1e, 0x6c, 0xcc, 0x22, 0xcd, 0xb2, 0xc0, 0x17, 0x7d, 0x42, 0x6a, 0x40, 0xd1, 0xeb, 0xfa, 0xe0 },
+        nullptr,
+    },
+    {
+        // sha256(libVDSO:gettimeofday)
+        { 0x77, 0x2a, 0xde, 0x1c, 0x13, 0x2d, 0xe9, 0x48, 0xaf, 0xe0, 0xba, 0xcc, 0x6a, 0x89, 0xff, 0xca, 0x4a, 0xdc, 0xd5, 0x63, 0x2c, 0xc5, 0x62, 0x8b, 0x5d, 0xde, 0x0b, 0x15, 0x35, 0xc6, 0xc7, 0x14 },
+        nullptr,
+    },
+    {
+        // sha256(libVDSO:clock_gettime)
+        { 0x3c, 0x96, 0x9b, 0x2d, 0xc3, 0xad, 0x2b, 0x3b, 0x9c, 0x4e, 0x4d, 0xca, 0x1c, 0xe8, 0x18, 0x4a, 0x12, 0x8a, 0xe4, 0xc1, 0x56, 0x92, 0x73, 0xce, 0x65, 0x85, 0x5f, 0x65, 0x7e, 0x94, 0x26, 0xbe },
+        nullptr,
+    },
+
+    {
+        // sha256(libVDSO:clock_gettime64)
+        { 0xba, 0xe9, 0x6d, 0x30, 0xc0, 0x68, 0xc6, 0xd7, 0x59, 0x04, 0xf7, 0x10, 0x06, 0x72, 0x88, 0xfd, 0x4c, 0x57, 0x0f, 0x31, 0xa5, 0xea, 0xa9, 0xb9, 0xd3, 0x8d, 0x03, 0x81, 0x50, 0x16, 0x22, 0x71 },
+        nullptr,
+    },
+
+    {
+        // sha256(libVDSO:clock_getres)
+        { 0xe4, 0xa1, 0xf6, 0x23, 0x35, 0xae, 0xb7, 0xb6, 0xb0, 0x37, 0xc5, 0xc3, 0xa3, 0xfd, 0xbf, 0xa2, 0xa1, 0xc8, 0x95, 0x78, 0xe5, 0x76, 0x86, 0xdb, 0x3e, 0x6c, 0x54, 0xd5, 0x02, 0x60, 0xd8, 0x6d },
+        nullptr,
+    },
+    {
+        // sha256(libVDSO:getcpu)
+        { 0x39, 0x83, 0x39, 0x36, 0x0f, 0x68, 0xd6, 0xfc, 0xc2, 0x3a, 0x97, 0x11, 0x85, 0x09, 0xc7, 0x25, 0xbb, 0x50, 0x49, 0x55, 0x6b, 0x0c, 0x9f, 0x50, 0x37, 0xf5, 0x9d, 0xb0, 0x38, 0x58, 0x57, 0x12 },
+        nullptr,
+    },
+  };
+
+  void* LoadVDSOThunks(bool Is64Bit, MapperFn Mapper) {
+    void* VDSOBase{};
+    FEX_CONFIG_OPT(ThunkGuestLibs, THUNKGUESTLIBS);
+    FEX_CONFIG_OPT(ThunkGuestLibs32, THUNKGUESTLIBS32);
+
+    std::filesystem::path ThunkGuestPath{};
+    if (Is64Bit) {
+      ThunkGuestPath = std::filesystem::path(ThunkGuestLibs()) / "libVDSO-guest.so";
+
+      // Set the Thunk definition pointers for x86-64
+      VDSODefinitions[0].ThunkFunction = &FEX::VDSO::time;
+      VDSODefinitions[1].ThunkFunction = &FEX::VDSO::gettimeofday;
+      VDSODefinitions[2].ThunkFunction = &FEX::VDSO::clock_gettime;
+      VDSODefinitions[3].ThunkFunction = &FEX::VDSO::clock_gettime;
+      VDSODefinitions[4].ThunkFunction = &FEX::VDSO::clock_getres;
+      VDSODefinitions[5].ThunkFunction = &FEX::VDSO::getcpu;
+    }
+    else {
+      ThunkGuestPath = std::filesystem::path(ThunkGuestLibs32()) / "libVDSO-guest.so";
+
+      // Set the Thunk definition pointers for x86
+      VDSODefinitions[0].ThunkFunction = &FEX::VDSO::x32::time;
+      VDSODefinitions[1].ThunkFunction = &FEX::VDSO::x32::gettimeofday;
+      VDSODefinitions[2].ThunkFunction = &FEX::VDSO::x32::clock_gettime;
+      VDSODefinitions[3].ThunkFunction = &FEX::VDSO::x32::clock_gettime64;
+      VDSODefinitions[4].ThunkFunction = &FEX::VDSO::x32::clock_getres;
+      VDSODefinitions[5].ThunkFunction = &FEX::VDSO::x32::getcpu;
+    }
+
+    // Load VDSO if we can
+    int VDSOFD = ::open(ThunkGuestPath.string().c_str(), O_RDONLY);
+
+    if (VDSOFD != -1) {
+      // Get file size
+      size_t VDSOSize = lseek(VDSOFD, 0, SEEK_END);
+
+      if (VDSOSize >= 4) {
+        // Reset to beginning
+        lseek(VDSOFD, 0, SEEK_SET);
+        VDSOSize = FEXCore::AlignUp(VDSOSize, 4096);
+
+        // Map the VDSO file to memory
+        VDSOBase = Mapper(nullptr, VDSOSize, PROT_READ, MAP_PRIVATE, VDSOFD, 0);
+
+        // Since we found our VDSO thunk library, find our host VDSO function implementations.
+        LoadHostVDSO();
+
+      }
+      close(VDSOFD);
+    }
+
+    return VDSOBase;
+  }
+
+  std::vector<FEXCore::IR::ThunkDefinition> const& GetVDSOThunkDefinitions() {
+    return VDSODefinitions;
+  }
+}

Source/Tests/VDSO_Emulation.h

@@ -0,0 +1,9 @@
+#pragma once
+#include <FEXCore/IR/IR.h>
+
+namespace FEX::VDSO {
+  using MapperFn = std::function<void *(void *addr, size_t length, int prot, int flags, int fd, off_t offset)>;
+  void* LoadVDSOThunks(bool Is64Bit, MapperFn Mapper);
+
+  std::vector<FEXCore::IR::ThunkDefinition> const& GetVDSOThunkDefinitions();
+}

Source/Tools/FEXConfig/Main.cpp

@@ -74,6 +74,7 @@ namespace {
     LoadedConfig->Erase(FEXCore::Config::ConfigOption::CONFIG_IS_INTERPRETER);
     LoadedConfig->Erase(FEXCore::Config::ConfigOption::CONFIG_INTERPRETER_INSTALLED);
     LoadedConfig->Erase(FEXCore::Config::ConfigOption::CONFIG_APP_FILENAME);
+    LoadedConfig->Erase(FEXCore::Config::ConfigOption::CONFIG_APP_CONFIG_NAME);
     LoadedConfig->Erase(FEXCore::Config::ConfigOption::CONFIG_IS64BIT_MODE);
   }
 
@@ -106,6 +107,7 @@ namespace {
     LoadedConfig->Erase(FEXCore::Config::ConfigOption::CONFIG_IS_INTERPRETER);
     LoadedConfig->Erase(FEXCore::Config::ConfigOption::CONFIG_INTERPRETER_INSTALLED);
     LoadedConfig->Erase(FEXCore::Config::ConfigOption::CONFIG_APP_FILENAME);
+    LoadedConfig->Erase(FEXCore::Config::ConfigOption::CONFIG_APP_CONFIG_NAME);
     LoadedConfig->Erase(FEXCore::Config::ConfigOption::CONFIG_IS64BIT_MODE);
 
     return true;

Source/Tools/FEXRootFSFetcher/Main.cpp

@@ -8,6 +8,7 @@
 #include <fstream>
 #include <iostream>
 #include <unistd.h>
+#include <optional>
 #include <sstream>
 #include <sys/mman.h>
 #include <sys/syscall.h>
@@ -441,7 +442,7 @@ namespace WebFileFetcher {
     FileType Type;
   };
 
-  const static std::string DownloadURL = "https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links.json";
+  const static std::string DownloadURL = "https://rootfs.fex-emu.com/file/fex-rootfs/RootFS_links.json";
 
   std::string DownloadToString(const std::string &URL) {
     std::string BigArgs =
@@ -514,10 +515,14 @@ namespace WebFileFetcher {
     return &*alloc->json_objects->emplace(alloc->json_objects->end());
   }
 
-  std::vector<FileTargets> GetRootFSLinks() {
+  std::optional<std::vector<FileTargets>> GetRootFSLinks() {
     // Decode the filetargets
     std::string Data = DownloadToString(DownloadURL);
 
+    if (Data.empty()) {
+      return std::nullopt;
+    }
+
     JsonAllocator Pool {
       .PoolObject = {
         .init = PoolInit,
@@ -1121,7 +1126,14 @@ int main(int argc, char **argv, char **const envp) {
   }
 
   if (AskForConfirmation(Question)) {
-    auto Targets = WebFileFetcher::GetRootFSLinks();
+    auto TargetReturn = WebFileFetcher::GetRootFSLinks();
+    if (!TargetReturn.has_value()) {
+      ExecWithInfo("Couldn't download rootfs list from the server. Try again in a minute or report on the fex-emu issue tracker.");
+      return -1;
+    }
+
+    auto Targets = TargetReturn.value();
+
     if (Targets.empty()) {
       ExecWithInfo("Couldn't parse rootfs definition URL.");
       return -1;

Source/Tools/FEXServer/ProcessPipe.cpp

@@ -401,10 +401,29 @@ namespace ProcessPipe {
         case FEXServerClient::PacketType::TYPE_GET_PID_FD: {
           int FD = FHU::Syscalls::pidfd_open(::getpid(), 0);
 
-          SendFDSuccessPacket(Socket, FD);
+          if (FD < 0) {
+            // Couldn't get PIDFD due to too old of kernel.
+            // Return a pipe to track the same information.
+            //
+            int fds[2];
+            pipe2(fds, O_CLOEXEC);
+            SendFDSuccessPacket(Socket, fds[0]);
 
-          // Close the FD now since we've sent it
-          close(FD);
+            // Close the read side now, doesn't matter to us
+            close(fds[0]);
+
+            // Check if we need to increase the FD limit.
+            ++NumFilesOpened;
+            CheckRaiseFDLimit();
+
+            // Write side will naturally close on process exit, letting the other process know we have exited.
+          }
+          else {
+            SendFDSuccessPacket(Socket, FD);
+
+            // Close the FD now since we've sent it
+            close(FD);
+          }
 
           CurrentOffset += sizeof(FEXServerClient::FEXServerRequestPacket::Header);
           break;