Docs: Update for release FEX-2509.1

This takes an f80 as input and returns an f32. A copy-paste error had
this truncating the input float if !TMP_ABIARGS.

.clang-format

@@ -32,7 +32,7 @@ AttributeMacros:
 BinPackArguments: true
 BinPackParameters: true
 BitFieldColonSpacing: Both
-BreakAfterAttributes: Always # clang 16 required
+BreakAfterAttributes: Leave
 BreakBeforeBraces: Attach
 BreakBeforeBinaryOperators: None
 BreakBeforeInlineASMColon: OnlyMultiline # clang 16 required
@@ -60,7 +60,7 @@ IndentRequires: false
 IndentWidth: 2
 InsertBraces: true
 KeepEmptyLinesAtTheStartOfBlocks: true
-LambdaBodyIndentation: OuterScope
+LambdaBodyIndentation: Signature
 LineEnding: LF # clang 16 required
 MaxEmptyLinesToKeep: 2
 NamespaceIndentation: Inner

.clang-format-ignore

@@ -1,8 +1,4 @@
 # This file is used to ignore files and directories from clang-format
-
-# Ignore all files in the External directory
-External/*
-
 Source/Common/cpp-optparse/*
 
 # Files with human-indented tables for readability - don't mess with these

.git-blame-ignore-revs

@@ -16,3 +16,7 @@
 
 # Reformat of CodeEmitter inl files
 8760c593ece92d7e9fa94c40da0368fd367c9cad
+
+# Whole-tree reformat with clang-format-19
+5267cde60e7642852d18f20ae8568643bb5293d5
+

.github/workflows/ccpp.yml

@@ -13,6 +13,7 @@ env:
   BUILD_TYPE: Release
   CC: clang
   CXX: clang++
+  FEX_PORTABLE: 1
 
 jobs:
   build_plus_test:
@@ -136,6 +137,9 @@ jobs:
     - name: FEXLinuxTests
       working-directory: ${{runner.workspace}}/build
       shell: bash
+      env:
+        # These tests require non-portable install due to thunks.
+        FEX_PORTABLE: 0
       run: cmake --build . --config $BUILD_TYPE --target fex_linux_tests_all
 
     - name: FEXLinuxTests Results move

.github/workflows/glibc_fault.yml

@@ -20,6 +20,7 @@ env:
   BUILD_TYPE: Release
   CC: clang
   CXX: clang++
+  FEX_PORTABLE: 1
 
 jobs:
   glibc_fault_test:

.github/workflows/hostrunner.yml

@@ -13,6 +13,7 @@ env:
   BUILD_TYPE: Release
   CC: clang
   CXX: clang++
+  FEX_PORTABLE: 1
 
 jobs:
   hostrunner_tests:

.github/workflows/mingw_build.yml

@@ -78,7 +78,7 @@ jobs:
       # Note the current convention is to use the -S and -B options here to specify source
       # and build directories, but this is only available with CMake 3.13 and higher.
       # The CMake binaries on the Github Actions machines are (as of this writing) 3.12
-      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/toolchain_mingw.cmake -DMINGW_TRIPLE=$MINGW_TRIPLE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DBUILD_TESTS=False -DCMAKE_INSTALL_PREFIX=${{runner.workspace}}/build/install
+      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/Data/CMake/toolchain_mingw.cmake -DMINGW_TRIPLE=$MINGW_TRIPLE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DBUILD_TESTS=False -DCMAKE_INSTALL_PREFIX=${{runner.workspace}}/build/install
 
     - name: Build
       working-directory: ${{runner.workspace}}/build

.github/workflows/pr-code-format.yml

@@ -40,11 +40,8 @@ jobs:
           echo "Formatting files:"
           echo "$CHANGED_FILES"
 
-      - name: Check for correct clang-format version
-        run: clang-format --version | grep -qF '16.0.6'
-
-      - name: Check git-clang-format-16 exists
-        run: which git-clang-format-16
+      - name: Check git-clang-format-19 exists
+        run: which git-clang-format-19
 
       - name: Setup Python env
         uses: actions/setup-python@v4
@@ -58,19 +55,15 @@ jobs:
 
       - name: Run code formatter
         env:
-          CLANG_FORMAT_PATH: 'git-clang-format-16'
+          CLANG_FORMAT_PATH: 'git-clang-format-19'
           GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
           START_REV: ${{ github.event.pull_request.base.sha }}
           END_REV: ${{ github.event.pull_request.head.sha }}
           CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
-        # TODO(pmatos): Once we adopt v18, we should be able
-        # to take advantage of the new --diff_from_common_commit option
-        # explicitly in code-format-helper.py and not have to diff starting at
-        # the merge base.
         run: |
           python ./External/code-format-helper/code-format-helper.py \
             --repo "FEX-emu/FEX" \
             --issue-number $GITHUB_PR_NUMBER \
-            --start-rev $(git merge-base $START_REV $END_REV) \
+            --start-rev $START_REV \
             --end-rev $END_REV \
             --changed-files "$CHANGED_FILES"

.github/workflows/vixl_simulator.yml

@@ -13,6 +13,7 @@ env:
   BUILD_TYPE: Release
   CC: clang
   CXX: clang++
+  FEX_PORTABLE: 1
 
 jobs:
   vixl_simulator:

.github/workflows/wine_dll_artifacts.yml

@@ -0,0 +1,88 @@
+name: Wine DLL artifacts
+
+on:
+  push:
+    branches:
+      - main
+
+env:
+  BUILD_TYPE: Release
+
+jobs:
+  wine_dll_artifacts:
+    runs-on: ${{ matrix.arch }}
+    strategy:
+      matrix:
+        arch: [[self-hosted, ARM64, mingw]]
+      fail-fast: false
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Add MingGW to PATH
+      run: echo "$HOME/llvm-mingw/build/bin/" >> $GITHUB_PATH
+
+    - name : submodule checkout
+      # Need to update submodules
+      run: |
+        git submodule sync --recursive
+        git submodule update --init --depth 1
+
+    - name: Clean install directory
+      run: |
+        rm -Rf ${{runner.workspace}}/build_install
+        mkdir ${{runner.workspace}}/build_install
+
+    - name: Clean Build Environment
+      run: |
+        rm -Rf ${{runner.workspace}}/build_arm64ec
+        rm -Rf ${{runner.workspace}}/build_wow64
+
+    - name: Create Build Environment arm64ec
+      run: |
+        cmake -E make_directory ${{runner.workspace}}/build_arm64ec
+        cmake -E make_directory ${{runner.workspace}}/build_wow64
+
+    - name: Configure CMake arm64ec
+      shell: bash
+      working-directory: ${{runner.workspace}}/build_arm64ec
+      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/Data/CMake/toolchain_mingw.cmake -DMINGW_TRIPLE=arm64ec-w64-mingw32 -DCMAKE_INSTALL_LIBDIR=/usr/lib/wine/aarch64-windows -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=False -DENABLE_JEMALLOC_GLIBC_ALLOC=False -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_TESTS=False -DCMAKE_INSTALL_PREFIX=/usr
+
+    - name: Configure CMake wow64
+      shell: bash
+      working-directory: ${{runner.workspace}}/build_wow64
+      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/Data/CMake/toolchain_mingw.cmake -DMINGW_TRIPLE=aarch64-w64-mingw32 -DCMAKE_INSTALL_LIBDIR=/usr/lib/wine/aarch64-windows -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=False -DENABLE_JEMALLOC_GLIBC_ALLOC=False -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_TESTS=False -DCMAKE_INSTALL_PREFIX=/usr
+
+    - name: Build arm64ec
+      working-directory: ${{runner.workspace}}/build_arm64ec
+      shell: bash
+      run: cmake --build . --config $BUILD_TYPE
+
+    - name: install arm64ec
+      working-directory: ${{runner.workspace}}/build_arm64ec
+      shell: bash
+      env:
+        DESTDIR: ${{runner.workspace}}/build_install
+      run: cmake --build . --config $BUILD_TYPE -t install
+
+    - name: Build wow64
+      working-directory: ${{runner.workspace}}/build_wow64
+      shell: bash
+      run: cmake --build . --config $BUILD_TYPE
+
+    - name: install wow64
+      working-directory: ${{runner.workspace}}/build_wow64
+      shell: bash
+      env:
+        DESTDIR: ${{runner.workspace}}/build_install
+      run: cmake --build . --config $BUILD_TYPE -t install
+
+    - name: Upload libraries
+      uses: 'actions/upload-artifact@v4'
+      timeout-minutes: 1
+      with:
+        overwrite: true
+        name: wine_dll_artifacts
+        path: ${{runner.workspace}}/build_install/usr/lib/wine/aarch64-windows/lib*.dll
+        retention-days: 60
+        compression-level: 9

.gitmodules

@@ -46,3 +46,6 @@
 [submodule "External/tracy"]
 	path = External/tracy
 	url = https://github.com/wolfpld/tracy
+[submodule "External/range-v3"]
+	path = External/range-v3
+	url = https://github.com/ericniebler/range-v3.git

CMakeLists.txt

@@ -35,10 +35,19 @@ set (FEXCORE_PROFILER_BACKEND "gpuvis" CACHE STRING "Set which backend to use fo
 option(ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT "Enables glibc memory allocation hooking with fault for CI testing")
 option(USE_PDB_DEBUGINFO "Builds debug info in PDB format" FALSE)
 
-set (X86_32_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/toolchain_x86_32.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting i686")
-set (X86_64_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/toolchain_x86_64.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
+set (X86_32_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_32.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting i686")
+set (X86_64_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_64.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
 set (X86_DEV_ROOTFS "/" CACHE FILEPATH "Path to the sysroot used for cross-compiling for i686 and x86_64")
-set (DATA_DIRECTORY "${CMAKE_INSTALL_PREFIX}/share/fex-emu" CACHE PATH "global data directory")
+set (DATA_DIRECTORY "" CACHE PATH "Global data directory (override)")
+set (HOSTLIBS_DATA_DIRECTORY "" CACHE PATH "Global data directory (override)")
+if (NOT DATA_DIRECTORY)
+  set (DATA_DIRECTORY "${CMAKE_INSTALL_PREFIX}/share/fex-emu")
+endif()
+
+include(GNUInstallDirs)
+if (NOT HOSTLIBS_DATA_DIRECTORY)
+  set(HOSTLIBS_DATA_DIRECTORY "${CMAKE_INSTALL_FULL_LIBDIR}/fex-emu")
+endif()
 
 string(FIND ${CMAKE_BASE_NAME} mingw CONTAINS_MINGW)
 if (NOT CONTAINS_MINGW EQUAL -1)
@@ -94,7 +103,7 @@ endif()
 # uninstall target
 if(NOT TARGET uninstall)
   configure_file(
-    "${CMAKE_CURRENT_SOURCE_DIR}/CMakeFiles/cmake_uninstall.cmake.in"
+    "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/cmake_uninstall.cmake.in"
     "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/cmake_uninstall.cmake"
     IMMEDIATE @ONLY)
 
@@ -345,6 +354,12 @@ if (NOT fmt_FOUND)
   add_subdirectory(External/fmt/)
 endif()
 
+find_package(range-v3 QUIET)
+if (NOT range-v3_FOUND)
+  add_subdirectory(External/range-v3/)
+  target_compile_definitions(range-v3 INTERFACE RANGES_DISABLE_DEPRECATED_WARNINGS)
+endif()
+
 add_subdirectory(External/tiny-json/)
 include_directories(External/tiny-json/)
 
@@ -403,6 +418,13 @@ if (TUNE_CPU STREQUAL "native")
 
       string(STRIP ${AARCH64_CPU} AARCH64_CPU)
 
+      execute_process(COMMAND python3 "${PROJECT_SOURCE_DIR}/Scripts/NeedDisabledSVE.py"
+        RESULT_VARIABLE NEEDS_SVE_DISABLED)
+      if (NEEDS_SVE_DISABLED)
+        message(STATUS "Platform has bugged SVE. Disabling")
+        set(AARCH64_CPU "cortex-a78")
+      endif()
+
       check_cxx_compiler_flag("-mcpu=${AARCH64_CPU}" COMPILER_SUPPORTS_CPU_TYPE)
       if(COMPILER_SUPPORTS_CPU_TYPE)
         list(APPEND FEX_TUNE_COMPILE_FLAGS "-mcpu=${AARCH64_CPU}")
@@ -414,7 +436,7 @@ if (TUNE_CPU STREQUAL "native")
       list(APPEND FEX_TUNE_COMPILE_FLAGS "-march=native")
     endif()
   endif()
-else()
+elseif (NOT TUNE_CPU STREQUAL "none")
   check_cxx_compiler_flag("-mcpu=${TUNE_CPU}" COMPILER_SUPPORTS_CPU_TYPE)
   if(COMPILER_SUPPORTS_CPU_TYPE)
     list(APPEND FEX_TUNE_COMPILE_FLAGS "-mcpu=${TUNE_CPU}")
@@ -433,10 +455,6 @@ endif()
 
 add_compile_options(-Wall)
 
-configure_file(
-  ${CMAKE_CURRENT_SOURCE_DIR}/include/Config.h.in
-  ${CMAKE_BINARY_DIR}/generated/ConfigDefines.h)
-
 include(CTest)
 if (BUILD_TESTS)
   message(STATUS "Unit tests are enabled")
@@ -619,12 +637,12 @@ set (CPACK_PACKAGE_CONTACT "FEX-Emu Maintainers <team@fex-emu.com>")
 set (CPACK_PACKAGE_VERSION_MAJOR "${FEX_VERSION_MAJOR}")
 set (CPACK_PACKAGE_VERSION_MINOR "${FEX_VERSION_MINOR}")
 set (CPACK_PACKAGE_VERSION_PATCH "${FEX_VERSION_PATCH}")
-set (CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/CPack/Description.txt")
+set (CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/CPack/Description.txt")
 
 # Debian defines
 set (CPACK_DEBIAN_PACKAGE_DEPENDS "libc6, libstdc++6, libepoxy0, libsdl2-2.0-0, libegl1, libx11-6, squashfuse")
 set (CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA
-  "${CMAKE_CURRENT_SOURCE_DIR}/CPack/postinst;${CMAKE_CURRENT_SOURCE_DIR}/CPack/prerm;${CMAKE_CURRENT_SOURCE_DIR}/CPack/triggers")
+  "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/CPack/postinst;${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/CPack/prerm;${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/CPack/triggers")
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
   # binfmt_misc conflicts with qemu-user-static
   # We also only install binfmt_misc on aarch64 hosts

CPack/Description.txt

@@ -1,3 +0,0 @@
-x86 and x86-64 Linux emulator
-
-FEX is very much work in progress, so expect things to change.

CodeEmitter/CodeEmitter/ALUOps.inl

@@ -174,7 +174,7 @@ public:
   // Logical immediate
   void and_(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {
     uint32_t n, immr, imms;
-    [[maybe_unused]] const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
+    const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
     LOGMAN_THROW_A_FMT(IsImm, "Couldn't encode immediate to logical op");
     and_(s, rd, rn, n, immr, imms);
   }
@@ -185,7 +185,7 @@ public:
 
   void ands(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {
     uint32_t n, immr, imms;
-    [[maybe_unused]] const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
+    const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
     LOGMAN_THROW_A_FMT(IsImm, "Couldn't encode immediate to logical op");
     ands(s, rd, rn, n, immr, imms);
   }
@@ -196,14 +196,14 @@ public:
 
   void orr(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {
     uint32_t n, immr, imms;
-    [[maybe_unused]] const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
+    const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
     LOGMAN_THROW_A_FMT(IsImm, "Couldn't encode immediate to logical op");
     orr(s, rd, rn, n, immr, imms);
   }
 
   void eor(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {
     uint32_t n, immr, imms;
-    [[maybe_unused]] const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
+    const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
     LOGMAN_THROW_A_FMT(IsImm, "Couldn't encode immediate to logical op");
     eor(s, rd, rn, n, immr, imms);
   }
@@ -333,7 +333,7 @@ public:
     bfi(s, rd, Reg::zr, lsb, width);
   }
   void bfxil(ARMEmitter::Size s, Register rd, Register rn, uint32_t lsb, uint32_t width) {
-    [[maybe_unused]] const auto reg_size_bits = RegSizeInBits(s);
+    const auto reg_size_bits = RegSizeInBits(s);
     const auto lsb_p_width = lsb + width;
 
     LOGMAN_THROW_A_FMT(width >= 1, "bfxil needs width >= 1");
@@ -977,7 +977,7 @@ private:
   }
 
   void xbfiz_helper(bool is_signed, ARMEmitter::Size s, Register rd, Register rn, uint32_t lsb, uint32_t width) {
-    [[maybe_unused]] const auto lsb_p_width = lsb + width;
+    const auto lsb_p_width = lsb + width;
     const auto reg_size_bits = RegSizeInBits(s);
 
     LOGMAN_THROW_A_FMT(lsb_p_width <= reg_size_bits, "lsb + width ({}) must be <= {}. lsb={}, width={}", lsb_p_width, reg_size_bits, lsb, width);

CodeEmitter/CodeEmitter/Emitter.h

@@ -12,6 +12,7 @@
 #include <CodeEmitter/Registers.h>
 
 #include <array>
+#include <bit>
 #include <cstdint>
 #include <utility>
 #include <type_traits>
@@ -86,6 +87,14 @@ constexpr size_t SubRegSizeInBits(SubRegSize size) {
   return size_t {8} << FEXCore::ToUnderlying(size);
 }
 
+// Many floating point operations constrain their element sizes to the
+// main three float sizes half, single, and double precision. This just
+// combines all the checks together for brevity.
+[[nodiscard]]
+constexpr bool IsStandardFloatSize(SubRegSize size) {
+  return size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit;
+}
+
 /* This `ScalarRegSize` enum is used for most scalar float
  * operations.
  *

CodeEmitter/CodeEmitter/LoadstoreOps.inl

@@ -1648,8 +1648,8 @@ public:
   template<typename T>
   void ASIMDLoadStoreSinglePost(uint32_t Op, uint32_t Q, uint32_t L, uint32_t R, uint32_t opcode, uint32_t S, uint32_t size,
                                 ARMEmitter::Register rm, ARMEmitter::Register rn, T rt) {
-    LOGMAN_THROW_A_FMT(std::is_same_v<ARMEmitter::QRegister, T> || std::is_same_v<ARMEmitter::DRegister, T>, "Only supports 128-bit and "
-                                                                                                             "64-bit vector registers.");
+    LOGMAN_THROW_A_FMT((std::is_same_v<ARMEmitter::QRegister, T> || std::is_same_v<ARMEmitter::DRegister, T>), "Only supports 128-bit and "
+                                                                                                               "64-bit vector registers.");
     uint32_t Instr = Op;
 
     Instr |= Q << 30;

CodeEmitter/CodeEmitter/SVEOps.inl

@@ -60,8 +60,7 @@ public:
   }
 
   void fcmla(SubRegSize size, ZRegister zda, PRegisterMerge pv, ZRegister zn, ZRegister zm, Rotation rot) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
     LOGMAN_THROW_A_FMT(pv <= PReg::p7.Merging(), "fcmla can only use p0 to p7");
 
     uint32_t Op = 0b0110'0100'0000'0000'0000'0000'0000'0000;
@@ -76,8 +75,7 @@ public:
   }
 
   void fcadd(SubRegSize size, ZRegister zd, PRegisterMerge pv, ZRegister zn, ZRegister zm, Rotation rot) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
     LOGMAN_THROW_A_FMT(pv <= PReg::p7.Merging(), "fcadd can only use p0 to p7");
     LOGMAN_THROW_A_FMT(rot == Rotation::ROTATE_90 || rot == Rotation::ROTATE_270, "fcadd rotation may only be 90 or 270 degrees");
     LOGMAN_THROW_A_FMT(zd == zn, "fcadd zd and zn must be the same register");
@@ -815,16 +813,12 @@ public:
   // SVE Integer Misc - Unpredicated
   // SVE floating-point trig select coefficient
   void ftssel(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "ftssel may only have "
-                                                                                                               "16-bit, 32-bit, or 64-bit "
-                                                                                                               "element sizes");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "ftssel may only use 16/32/64-bit element sizes");
     SVEIntegerMiscUnpredicated(0b00, zm.Idx(), FEXCore::ToUnderlying(size), zd, zn);
   }
   // SVE floating-point exponential accelerator
   void fexpa(SubRegSize size, ZRegister zd, ZRegister zn) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "fexpa may only have "
-                                                                                                               "16-bit, 32-bit, or 64-bit "
-                                                                                                               "element sizes");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "fexpa may only use 16/32/64-bit element sizes");
     SVEIntegerMiscUnpredicated(0b10, 0b00000, FEXCore::ToUnderlying(size), zd, zn);
   }
   // SVE constructive prefix (unpredicated)
@@ -1503,9 +1497,9 @@ public:
   }
 
   // SVE broadcast floating-point immediate (unpredicated)
-  void fdup(ARMEmitter::SubRegSize size, ARMEmitter::ZRegister zd, float Value) {
-    LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i16Bit || size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit,
-                       "Unsupported fmov size");
+  void fdup(SubRegSize size, ZRegister zd, float Value) {
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Unsupported fmov size");
+
     uint32_t Imm {};
     if (size == SubRegSize::i16Bit) {
       LOGMAN_MSG_A_FMT("Unsupported");
@@ -1518,7 +1512,7 @@ public:
 
     SVEBroadcastFloatImmUnpredicated(0b00, 0, Imm, size, zd);
   }
-  void fmov(ARMEmitter::SubRegSize size, ARMEmitter::ZRegister zd, float Value) {
+  void fmov(SubRegSize size, ZRegister zd, float Value) {
     fdup(size, zd, Value);
   }
 
@@ -1547,7 +1541,7 @@ public:
   void sqincp(SubRegSize size, XRegister rdn, PRegister pm) {
     SVEIncDecPredicateCountScalar(0, 1, 0b10, 0b00, size, rdn, pm);
   }
-  void sqincp(SubRegSize size, XRegister rdn, PRegister pm, [[maybe_unused]] WRegister wn) {
+  void sqincp(SubRegSize size, XRegister rdn, PRegister pm, WRegister wn) {
     LOGMAN_THROW_A_FMT(rdn.Idx() == wn.Idx(), "rdn and wn must be the same");
     SVEIncDecPredicateCountScalar(0, 1, 0b00, 0b00, size, rdn, pm);
   }
@@ -1560,7 +1554,7 @@ public:
   void sqdecp(SubRegSize size, XRegister rdn, PRegister pm) {
     SVEIncDecPredicateCountScalar(0, 1, 0b10, 0b10, size, rdn, pm);
   }
-  void sqdecp(SubRegSize size, XRegister rdn, PRegister pm, [[maybe_unused]] WRegister wn) {
+  void sqdecp(SubRegSize size, XRegister rdn, PRegister pm, WRegister wn) {
     LOGMAN_THROW_A_FMT(rdn.Idx() == wn.Idx(), "rdn and wn must be the same");
     SVEIncDecPredicateCountScalar(0, 1, 0b00, 0b10, size, rdn, pm);
   }
@@ -3302,7 +3296,7 @@ private:
     const auto log2_size_bytes = FEXCore::ilog2(size_bytes);
 
     // We can index up to 512-bit registers with dup
-    [[maybe_unused]] const auto max_index = (64U >> log2_size_bytes) - 1;
+    const auto max_index = (64U >> log2_size_bytes) - 1;
     LOGMAN_THROW_A_FMT(Index <= max_index, "dup index ({}) too large. Must be within [0, {}].", Index, max_index);
 
     // imm2:tsz make up a 7 bit wide field, with each increasing element size
@@ -3332,7 +3326,7 @@ private:
 
     uint32_t shift = 0;
     if (!is_uint8_imm) {
-      [[maybe_unused]] const bool is_uint16_imm = (imm >> 16) == 0;
+      const bool is_uint16_imm = (imm >> 16) == 0;
 
       LOGMAN_THROW_A_FMT(is_uint16_imm, "Immediate ({}) must be a 16-bit value within [256, 65280]", imm);
       LOGMAN_THROW_A_FMT((imm % 256) == 0, "Immediate ({}) must be a multiple of 256", imm);
@@ -3401,8 +3395,8 @@ private:
   }
 
   void SVEBroadcastFloatImmPredicated(SubRegSize size, ZRegister zd, PRegister pg, float value) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Unsupported fcpy/fmov "
-                                                                                                               "size");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Unsupported fcpy/fmov size");
+
     uint32_t imm {};
     if (size == SubRegSize::i16Bit) {
       LOGMAN_MSG_A_FMT("Unsupported");
@@ -3578,7 +3572,7 @@ private:
   // SVE2 floating-point pairwise operations
   void SVEFloatPairwiseArithmetic(uint32_t opc, SubRegSize size, PRegister pg, ZRegister zd, ZRegister zn, ZRegister zm) {
     LOGMAN_THROW_A_FMT(zd == zn, "zd needs to equal zn");
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Invalid float size");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Invalid float size");
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
 
     uint32_t Instr = 0b0110'0100'0001'0000'1000'0000'0000'0000;
@@ -3592,7 +3586,7 @@ private:
 
   // SVE floating-point arithmetic (unpredicated)
   void SVEFloatArithmeticUnpredicated(uint32_t opc, SubRegSize size, ZRegister zm, ZRegister zn, ZRegister zd) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Invalid float size");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Invalid float size");
 
     uint32_t Instr = 0b0110'0101'0000'0000'0000'0000'0000'0000;
     Instr |= FEXCore::ToUnderlying(size) << 22;
@@ -3700,7 +3694,7 @@ private:
   // SVE floating-point arithmetic (predicated)
   void SVEFloatArithmeticPredicated(uint32_t opc, SubRegSize size, PRegister pg, ZRegister zd, ZRegister zn, ZRegister zm) {
     LOGMAN_THROW_A_FMT(zd == zn, "zn needs to equal zd");
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Invalid float size");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Invalid float size");
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
 
     uint32_t Instr = 0b0110'0101'0000'0000'1000'0000'0000'0000;
@@ -3728,9 +3722,7 @@ private:
   }
 
   void SVEFPRecursiveReduction(uint32_t opc, SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "FP reduction operation can "
-                                                                                                               "only use 16-bit, 32-bit, "
-                                                                                                               "or 64-bit element sizes");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "FP reduction operation can only use 16/32/64-bit element sizes");
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "FP reduction operation can only use p0-p7 as a governing predicate");
 
     uint32_t Instr = 0b0110'0101'0000'0000'0010'0000'0000'0000;
@@ -4112,7 +4104,7 @@ private:
     // 0b111 - I - Current
 
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Unsupported size in {}", __func__);
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Unsupported size in {}", __func__);
 
     uint32_t Instr = 0b0110'0101'0000'0000'1010'0000'0000'0000;
     Instr |= FEXCore::ToUnderlying(size) << 22;
@@ -4160,7 +4152,7 @@ private:
 
     const auto& op_data = mem_op.MetaType.ScalarVectorType;
     const bool is_scaled = op_data.scale != 0;
-    [[maybe_unused]] const auto msize_value = FEXCore::ToUnderlying(msize);
+    const auto msize_value = FEXCore::ToUnderlying(msize);
 
     LOGMAN_THROW_A_FMT(op_data.scale == 0 || op_data.scale == msize_value, "scale may only be 0 or {}", msize_value);
 
@@ -4274,7 +4266,7 @@ private:
     const auto msize_value = FEXCore::ToUnderlying(msize);
     const auto msize_bytes = 1U << msize_value;
 
-    [[maybe_unused]] const auto imm_limit = (32U << msize_value) - msize_bytes;
+    const auto imm_limit = (32U << msize_value) - msize_bytes;
     const auto imm = mem_op.MetaType.VectorImmType.Imm;
     const auto imm_to_encode = imm >> msize_value;
 
@@ -4340,8 +4332,8 @@ private:
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
     LOGMAN_THROW_A_FMT((imm % num_regs) == 0, "Offset must be a multiple of {}", num_regs);
 
-    [[maybe_unused]] const auto min_offset = -8 * num_regs;
-    [[maybe_unused]] const auto max_offset = 7 * num_regs;
+    const auto min_offset = -8 * num_regs;
+    const auto max_offset = 7 * num_regs;
     LOGMAN_THROW_A_FMT(imm >= min_offset && imm <= max_offset,
                        "Invalid load/store offset ({}). Offset must be a multiple of {} and be within [{}, {}]", imm, num_regs, min_offset,
                        max_offset);
@@ -4448,8 +4440,8 @@ private:
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
 
     const auto esize = static_cast<int>(16 << ssz);
-    [[maybe_unused]] const auto max_imm = (esize << 3) - esize;
-    [[maybe_unused]] const auto min_imm = -(max_imm + esize);
+    const auto max_imm = (esize << 3) - esize;
+    const auto min_imm = -(max_imm + esize);
 
     LOGMAN_THROW_A_FMT((imm % esize) == 0, "imm ({}) must be a multiple of {}", imm, esize);
     LOGMAN_THROW_A_FMT(imm >= min_imm && imm <= max_imm, "imm ({}) must be within [{}, {}]", imm, min_imm, max_imm);
@@ -4493,7 +4485,7 @@ private:
     const auto msize_value = FEXCore::ToUnderlying(msize);
 
     const auto data_size_bytes = 1U << msize_value;
-    [[maybe_unused]] const auto max_imm = (64U << msize_value) - data_size_bytes;
+    const auto max_imm = (64U << msize_value) - data_size_bytes;
     LOGMAN_THROW_A_FMT((imm % data_size_bytes) == 0 && imm <= max_imm, "imm must be a multiple of {} and be within [0, {}]",
                        data_size_bytes, max_imm);
 
@@ -4721,7 +4713,7 @@ private:
 
   void SVEFloatUnary(uint32_t opc, SubRegSize size, PRegister pg, ZRegister zn, ZRegister zd) {
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Unsupported size in {}", __func__);
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Unsupported size in {}", __func__);
 
     uint32_t Instr = 0b0110'0101'0000'1100'1010'0000'0000'0000;
     Instr |= FEXCore::ToUnderlying(size) << 22;
@@ -4809,8 +4801,7 @@ private:
   }
 
   void SVEFPUnaryOpsUnpredicated(uint32_t opc, SubRegSize size, ZRegister zd, ZRegister zn) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
 
     uint32_t Instr = 0b0110'0101'0000'1000'0011'0000'0000'0000;
     Instr |= FEXCore::ToUnderlying(size) << 22;
@@ -4821,8 +4812,7 @@ private:
   }
 
   void SVEFPSerialReductionPredicated(uint32_t opc, SubRegSize size, VRegister vd, PRegister pg, VRegister vn, ZRegister zm) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
     LOGMAN_THROW_A_FMT(vd == vn, "vn must be the same as vd");
 
@@ -4836,8 +4826,7 @@ private:
   }
 
   void SVEFPCompareWithZero(uint32_t eqlt, uint32_t ne, SubRegSize size, PRegister pd, PRegister pg, ZRegister zn) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
 
     uint32_t Instr = 0b0110'0101'0001'0000'0010'0000'0000'0000;
@@ -4852,8 +4841,7 @@ private:
 
   void SVEFPMultiplyAdd(uint32_t opc, SubRegSize size, ZRegister zd, PRegister pg, ZRegister zn, ZRegister zm) {
     // NOTE: opc also includes the op0 bit (bit 15) like op0:opc, since the fields are adjacent
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
 
     uint32_t Instr = 0b0110'0101'0010'0000'0000'0000'0000'0000;
@@ -4867,14 +4855,13 @@ private:
   }
 
   void SVEFPMultiplyAddIndexed(uint32_t op, SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
     LOGMAN_THROW_A_FMT((size <= SubRegSize::i32Bit && zm <= ZReg::z7) || (size == SubRegSize::i64Bit && zm <= ZReg::z15),
                        "16-bit and 32-bit indexed variants may only use Zm between z0-z7\n"
                        "64-bit variants may only use Zm between z0-z15");
 
     const auto Underlying = FEXCore::ToUnderlying(size);
-    [[maybe_unused]] const uint32_t IndexMax = (16 / (1U << Underlying)) - 1;
+    const uint32_t IndexMax = (16 / (1U << Underlying)) - 1;
     LOGMAN_THROW_A_FMT(index <= IndexMax, "Index must be within 0-{}", IndexMax);
 
     // Can be bit 20 or 19 depending on whether or not the element size is 64-bit.
@@ -5130,12 +5117,13 @@ private:
   requires (std::is_same_v<T, float> || std::is_same_v<T, double>)
   using FloatToEquivalentUInt = std::conditional_t<std::is_same_v<T, float>, uint32_t, uint64_t>;
 
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
   // Determines if a floating-point value is capable of being converted
   // into an 8-bit immediate. See pseudocode definition of VFPExpandImm
   // in ARM A-profile reference manual for a general overview of how this was derived.
   template<typename T>
   requires (std::is_same_v<T, float> || std::is_same_v<T, double>)
-  [[nodiscard, maybe_unused]]
+  [[nodiscard]]
   static bool IsValidFPValueForImm8(T value) {
     const uint64_t bits = FEXCore::BitCast<FloatToEquivalentUInt<T>>(value);
     const uint64_t datasize_idx = FEXCore::ilog2(sizeof(T)) - 1;
@@ -5175,10 +5163,13 @@ private:
 
     return true;
   }
+#endif
 
 protected:
   static uint32_t FP32ToImm8(float value) {
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
     LOGMAN_THROW_A_FMT(IsValidFPValueForImm8(value), "Value ({}) cannot be encoded into an 8-bit immediate", value);
+#endif
 
     const auto bits = FEXCore::BitCast<uint32_t>(value);
     const auto sign = (bits & 0x80000000) >> 24;
@@ -5189,7 +5180,9 @@ protected:
   }
 
   static uint32_t FP64ToImm8(double value) {
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
     LOGMAN_THROW_A_FMT(IsValidFPValueForImm8(value), "Value ({}) cannot be encoded into an 8-bit immediate", value);
+#endif
 
     const auto bits = FEXCore::BitCast<uint64_t>(value);
     const auto sign = (bits & 0x80000000'00000000) >> 56;
@@ -5215,7 +5208,7 @@ private:
     uint32_t shift = 0;
     if (!is_int8_imm) {
       const int32_t imm16_limit = 32768;
-      [[maybe_unused]] const bool is_int16_imm = -imm16_limit <= imm && imm < imm16_limit;
+      const bool is_int16_imm = -imm16_limit <= imm && imm < imm16_limit;
 
       LOGMAN_THROW_A_FMT(is_int16_imm, "Immediate ({}) must be a 16-bit value within [-32768, 32512]", imm);
       LOGMAN_THROW_A_FMT((imm % 256) == 0, "Immediate ({}) must be a multiple of 256", imm);

CodeEmitter/CodeEmitter/ScalarOps.inl

@@ -27,21 +27,19 @@ struct EmitterOps : Emitter {
 public:
   // Advanced SIMD scalar copy
   void dup(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Index) {
-    constexpr uint32_t Op = 0b0101'1110'0000'0000'0000'01 << 10;
-
     const uint32_t SizeImm = FEXCore::ToUnderlying(size);
     const uint32_t IndexShift = SizeImm + 1;
     const uint32_t ElementSize = 1U << SizeImm;
-    [[maybe_unused]] const uint32_t MaxIndex = 128U / (ElementSize * 8);
+    const uint32_t MaxIndex = 128U / (ElementSize * 8);
 
     LOGMAN_THROW_A_FMT(Index < MaxIndex, "Index too large. Index={}, Max Index: {}", Index, MaxIndex);
 
     const uint32_t imm5 = (Index << IndexShift) | ElementSize;
 
-    ASIMDScalarCopy(Op, 1, imm5, 0b0000, rd, rn);
+    ASIMDScalarCopy(1, 1, imm5, 0b0000, rd, rn);
   }
 
-  void mov(ARMEmitter::ScalarRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, uint32_t Index) {
+  void mov(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Index) {
     dup(size, rd, rn, Index);
   }
 
@@ -1282,10 +1280,10 @@ public:
 
 private:
   // Advanced SIMD scalar copy
-  void ASIMDScalarCopy(uint32_t Op, uint32_t Q, uint32_t imm5, uint32_t imm4, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn) {
-    uint32_t Instr = Op;
-
+  void ASIMDScalarCopy(uint32_t Q, uint32_t b28, uint32_t imm5, uint32_t imm4, VRegister rd, VRegister rn) {
+    uint32_t Instr = 0b0000'1110'0000'0000'0000'01U << 10;
     Instr |= Q << 30;
+    Instr |= b28 << 28;
     Instr |= imm5 << 16;
     Instr |= imm4 << 11;
     Instr |= Encode_rn(rn);
@@ -1383,7 +1381,7 @@ private:
   void ASIMDScalarXIndexedElement(uint32_t U, ScalarRegSize size, uint32_t opcode, VRegister rm, VRegister rn, VRegister rd, uint32_t index) {
     LOGMAN_THROW_A_FMT(size != ScalarRegSize::i8Bit, "Scalar size must not be 8-bit");
 
-    [[maybe_unused]] const auto invalid_bound = 16U >> FEXCore::ToUnderlying(size);
+    const auto invalid_bound = 16U >> FEXCore::ToUnderlying(size);
     LOGMAN_THROW_A_FMT(index < invalid_bound, "Index ({}) must be within [0-{}]", index, invalid_bound - 1);
 
     uint32_t Instr = 0b0101'1111'0000'0000'0000'0000'0000'0000;

CodeEmitter/CodeEmitter/VixlUtils.inl

@@ -41,7 +41,6 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
   [[maybe_unused]] constexpr auto kDRegSize = 64;
 
   constexpr auto kWRegSize = 32;
-  constexpr auto kXRegSize = 64;
 
   LOGMAN_THROW_A_FMT((width == kBRegSize) || (width == kHRegSize) || (width == kSRegSize) || (width == kDRegSize), "Unexpected imm size");
 
@@ -129,8 +128,8 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
     // Compute the repeat distance d, and set up a bitmask covering the basic
     // unit of repetition (i.e. a word with the bottom d bits set). Also, in all
     // of these cases the N bit of the output will be zero.
-    clz_a = CountLeadingZeros(a, kXRegSize);
-    int clz_c = CountLeadingZeros(c, kXRegSize);
+    clz_a = std::countl_zero(a);
+    int clz_c = std::countl_zero(c);
     d = clz_a - clz_c;
     mask = ((UINT64_C(1) << d) - 1);
     out_n = 0;
@@ -151,7 +150,7 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
       // of set bits in our word, meaning that we have the trivial case of
       // d == 64 and only one 'repetition'. Set up all the same variables as in
       // the general case above, and set the N bit in the output.
-      clz_a = CountLeadingZeros(a, kXRegSize);
+      clz_a = std::countl_zero(a);
       d = 64;
       mask = ~UINT64_C(0);
       out_n = 1;
@@ -159,7 +158,7 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
   }
 
   // If the repeat period d is not a power of two, it can't be encoded.
-  if (!IsPowerOf2(d)) {
+  if (!std::has_single_bit(uint32_t(d))) {
     return false;
   }
 
@@ -179,7 +178,7 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
   static const uint64_t multipliers[] = {
     0x0000000000000001UL, 0x0000000100000001UL, 0x0001000100010001UL, 0x0101010101010101UL, 0x1111111111111111UL, 0x5555555555555555UL,
   };
-  uint64_t multiplier = multipliers[CountLeadingZeros(d, kXRegSize) - 57];
+  uint64_t multiplier = multipliers[std::countl_zero(uint64_t(d)) - 57];
   uint64_t candidate = (b - a) * multiplier;
 
   if (value != candidate) {
@@ -194,7 +193,7 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
   // Count the set bits in our basic stretch. The special case of clz(0) == -1
   // makes the answer come out right for stretches that reach the very top of
   // the word (e.g. numbers like 0xffffc00000000000).
-  int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSize);
+  int clz_b = (b == 0) ? -1 : std::countl_zero(b);
   int s = clz_a - clz_b;
 
   // Decide how many bits to rotate right by, to put the low bit of that basic
@@ -224,9 +223,13 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
   //    11110s     2    UInt(s)
   //
   // So we 'or' (2 * -d) with our computed s to form imms.
-  if ((n != NULL) || (imm_s != NULL) || (imm_r != NULL)) {
+  if (n != nullptr) {
     *n = out_n;
+  }
+  if (imm_s != nullptr) {
     *imm_s = ((2 * -d) | (s - 1)) & 0x3f;
+  }
+  if (imm_r != nullptr) {
     *imm_r = r;
   }
 
@@ -281,11 +284,6 @@ INT_1_TO_63_LIST(DECLARE_IS_UINT_N)
 
 private:
 
-template<typename V>
-static inline bool IsPowerOf2(V value) {
-  return (value != 0) && ((value & (value - 1)) == 0);
-}
-
 // Some compilers dislike negating unsigned integers,
 // so we provide an equivalent.
 template<typename T>
@@ -298,50 +296,4 @@ static inline uint64_t LowestSetBit(uint64_t value) {
   return value & UnsignedNegate(value);
 }
 
-template<typename V>
-static inline int CountLeadingZeros(V value, int width = (sizeof(V) * 8)) {
-#if COMPILER_HAS_BUILTIN_CLZ
-  if (width == 32) {
-    return (value == 0) ? 32 : __builtin_clz(static_cast<unsigned>(value));
-  } else if (width == 64) {
-    return (value == 0) ? 64 : __builtin_clzll(value);
-  }
-#endif
-  return CountLeadingZerosFallBack(value, width);
-}
-
-static inline int CountLeadingZerosFallBack(uint64_t value, int width) {
-  LOGMAN_THROW_A_FMT(IsPowerOf2(width) && (width <= 64), "Invalid width");
-  if (value == 0) {
-    return width;
-  }
-  int count = 0;
-  value = value << (64 - width);
-  if ((value & UINT64_C(0xffffffff00000000)) == 0) {
-    count += 32;
-    value = value << 32;
-  }
-  if ((value & UINT64_C(0xffff000000000000)) == 0) {
-    count += 16;
-    value = value << 16;
-  }
-  if ((value & UINT64_C(0xff00000000000000)) == 0) {
-    count += 8;
-    value = value << 8;
-  }
-  if ((value & UINT64_C(0xf000000000000000)) == 0) {
-    count += 4;
-    value = value << 4;
-  }
-  if ((value & UINT64_C(0xc000000000000000)) == 0) {
-    count += 2;
-    value = value << 2;
-  }
-  if ((value & UINT64_C(0x8000000000000000)) == 0) {
-    count += 1;
-  }
-  count += (value == 0);
-  return count;
-}
-
 public:

Data/CMake/CPack/Description.txt

@@ -0,0 +1,3 @@
+x86 and x86-64 Linux emulator
+
+FEX allows you to run x86 applications on ARM64 Linux devices. It offers broad compatibility with both 32-bit and 64-bit binaries, and it can be used alongside Wine/Proton to play Windows games.

Data/CMake/toolchain_aarch64.cmake

@@ -1,6 +1,6 @@
 # This is a reference AArch64 cross compile script
 # Pass in to cmake when building:
-# eg: cmake -DCMAKE_TOOLCHAIN_FILE=../CMakeToolchains/AArch64.cmake ..
+# eg: cmake --toolchain ../Data/CMake/toolchain_aarch64.cmake ..
 if (NOT DEFINED ENV{SYSROOT})
   message(FATAL_ERROR "Need to have SYSROOT environment variable set")
 endif()

Data/CMake/toolchain_mingw.cmake

@@ -4,6 +4,7 @@ set(CMAKE_RC_COMPILER ${MINGW_TRIPLE}-windres)
 set(CMAKE_C_COMPILER ${MINGW_TRIPLE}-clang)
 set(CMAKE_CXX_COMPILER ${MINGW_TRIPLE}-clang++)
 set(CMAKE_DLLTOOL ${MINGW_TRIPLE}-dlltool)
+set(CMAKE_AR ${MINGW_TRIPLE}-ar)
 
 # Compile everything as static to avoid requiring the MinGW runtime libraries, force page aligned sections so that
 # debug symbols work correctly, and disable loop alignment to workaround an LLVM bug

Data/nix/FEXLinuxTests/shell.nix

@@ -0,0 +1,35 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  pkgsCross32 = pkgs.pkgsCross.gnu32;
+  pkgsCross64 = pkgs.pkgsCross.gnu64;
+
+  gcc32 = pkgs.writeText "toolchain_nix_gcc_x86_32.txt" ''
+    set(CMAKE_SYSTEM_PROCESSOR i686)
+    set(CMAKE_C_COMPILER ${pkgsCross32.buildPackages.gcc}/bin/i686-unknown-linux-gnu-gcc)
+    set(CMAKE_CXX_COMPILER ${pkgsCross32.buildPackages.gcc}/bin/i686-unknown-linux-gnu-g++)
+  '';
+
+  gcc64 = pkgs.writeText "toolchain_nix_gcc_x86_64.txt" ''
+    set(CMAKE_SYSTEM_PROCESSOR x86_64)
+    set(CMAKE_C_COMPILER ${pkgsCross64.buildPackages.gcc}/bin/x86_64-unknown-linux-gnu-gcc)
+    set(CMAKE_CXX_COMPILER ${pkgsCross64.buildPackages.gcc}/bin/x86_64-unknown-linux-gnu-g++)
+  '';
+in
+pkgs.mkShell {
+  buildInputs = [
+    pkgsCross64.buildPackages.clang
+    pkgsCross32.buildPackages.clang
+  ];
+
+  shellHook = ''
+    if [[ $- == *i* ]]; then
+      echo "toolchain32: ${gcc32}"
+      echo "toolchain64: ${gcc64}"
+      echo ""
+      echo "Use \$FEX_CMAKE_TOOLCHAINS to configure CMake."
+    fi
+  '';
+
+  FEX_CMAKE_TOOLCHAINS = "-DX86_32_TOOLCHAIN_FILE=${gcc32} -DX86_64_TOOLCHAIN_FILE=${gcc64}";
+}

Data/nix/LibraryForwarding/shell.nix

@@ -0,0 +1,83 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  pkgsCross32 = pkgs.pkgsCross.gnu32;
+  pkgsCross64 = pkgs.pkgsCross.gnu64;
+
+  devRootFS = pkgs.buildEnv {
+    name = "fex-dev-rootfs";
+    paths = [
+      pkgsCross64.stdenv.cc.libc_dev
+      pkgsCross32.stdenv.cc.libc_dev
+      pkgsCross64.stdenv.cc.cc
+      pkgsCross32.stdenv.cc.cc
+
+      pkgs.alsa-lib.dev
+      pkgs.libdrm.dev
+      pkgs.libGL.dev
+      pkgs.wayland.dev
+      pkgs.xorg.libX11.dev
+      pkgs.xorg.libxcb.dev
+      pkgs.xorg.libXrandr.dev
+      pkgs.xorg.libXrender.dev
+      pkgs.xorg.xorgproto
+    ];
+    ignoreCollisions = true;
+    pathsToLink = [
+      "/include"
+      "/lib"
+    ];
+
+    postBuild = ''
+      mkdir -p $out/usr
+      ln -s $out/include $out/usr/
+    '';
+  };
+
+  toolchain32 = pkgs.writeText "toolchain_nix_x86_32.txt" ''
+    set(CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
+    set(CMAKE_MODULE_LINKER_FLAGS_INIT "-fuse-ld=lld")
+    set(CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
+    set(CMAKE_SYSTEM_PROCESSOR i686)
+    set(CMAKE_C_COMPILER clang)
+    set(CMAKE_CXX_COMPILER clang++)
+    set(CMAKE_C_COMPILER ${pkgsCross32.buildPackages.clang}/bin/i686-unknown-linux-gnu-clang)
+    set(CMAKE_CXX_COMPILER ${pkgsCross32.buildPackages.clang}/bin/i686-unknown-linux-gnu-clang++)
+    set(CLANG_FLAGS "-nodefaultlibs -nostartfiles -lstdc++ -target i686-linux-gnu -msse2 -mfpmath=sse --sysroot=${devRootFS} -iwithsysroot/include")
+    set(CMAKE_C_FLAGS "''${CMAKE_C_FLAGS} ''${CLANG_FLAGS}")
+    set(CMAKE_CXX_FLAGS "''${CMAKE_CXX_FLAGS} ''${CLANG_FLAGS}")
+  '';
+
+  toolchain64 = pkgs.writeText "toolchain_nix_x86_64.txt" ''
+    set(CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
+    set(CMAKE_MODULE_LINKER_FLAGS_INIT "-fuse-ld=lld")
+    set(CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
+    set(CMAKE_SYSTEM_PROCESSOR x86_64)
+    set(CMAKE_C_COMPILER clang)
+    set(CMAKE_CXX_COMPILER clang++)
+    set(CMAKE_C_COMPILER ${pkgsCross64.buildPackages.clang}/bin/x86_64-unknown-linux-gnu-clang)
+    set(CMAKE_CXX_COMPILER ${pkgsCross64.buildPackages.clang}/bin/x86_64-unknown-linux-gnu-clang++)
+    set(CLANG_FLAGS "-nodefaultlibs -nostartfiles -lstdc++ -target x86_64-linux-gnu --sysroot=${devRootFS} -iwithsysroot/usr/include")
+    set(CMAKE_C_FLAGS "''${CMAKE_C_FLAGS} ''${CLANG_FLAGS}")
+    set(CMAKE_CXX_FLAGS "''${CMAKE_CXX_FLAGS} ''${CLANG_FLAGS}")
+  '';
+in
+pkgs.mkShell {
+  buildInputs = [
+    pkgsCross64.buildPackages.clang
+    pkgsCross32.buildPackages.clang
+  ];
+
+  shellHook = ''
+    if [[ $- == *i* ]]; then
+      echo "Set up dev RootFS at ${devRootFS}"
+      echo "toolchain32: ${toolchain32}"
+      echo "toolchain64: ${toolchain64}"
+      echo ""
+      echo "Use \$FEX_CMAKE_TOOLCHAINS to configure CMake."
+    fi
+  '';
+
+  FEX_CMAKE_TOOLCHAINS = "-DX86_32_TOOLCHAIN_FILE=${toolchain32} -DX86_64_TOOLCHAIN_FILE=${toolchain64} -DX86_DEV_ROOTFS=${devRootFS}";
+  ROOTFS = "${devRootFS}";
+}

Data/nix/WineOnArm/shell.nix

@@ -0,0 +1,52 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  toolchain = pkgs.fetchzip {
+    url = "https://github.com/bylaws/llvm-mingw/releases/download/20250305/llvm-mingw-20250305-ucrt-ubuntu-20.04-aarch64.tar.xz";
+    sha256 = "sha256-cA03/ab9O61eO9+S2JzIXD4V0HzTXK5/AYyxW2d73Po=";
+  };
+
+  cmakeToolchainFile = pkgs.substitute {
+    # Use absolute paths that are discoverable outside of the nix shell
+    src = ../../CMake/toolchain_mingw.cmake;
+    substitutions = ["--replace-fail" "\${MINGW_TRIPLE}-" "${toolchain}/bin/\${MINGW_TRIPLE}-"];
+  };
+
+  mesonCrossFile = pkgs.writeText "crossfile_llvm_mingw.txt" ''
+    [binaries]
+    ar = '${toolchain}/bin/arm64ec-w64-mingw32-ar'
+    c = '${toolchain}/bin/arm64ec-w64-mingw32-gcc'
+    cpp = '${toolchain}/bin/arm64ec-w64-mingw32-g++'
+    ld = '${toolchain}/bin/arm64ec-w64-mingw32-ld'
+    windres = '${toolchain}/bin/arm64ec-w64-mingw32-windres'
+    strip = '${toolchain}/bin/strip'
+    widl = '${toolchain}/bin/arm64ec-w64-mingw32-widl'
+    pkgconfig = 'aarch64-linux-gnu-pkg-config'
+    [host_machine]
+    system = 'windows'
+    cpu_family = 'aarch64'
+    cpu = 'aarch64'
+    endian = 'little'
+  '';
+in
+pkgs.mkShell {
+  buildInputs = [
+    toolchain
+  ];
+
+  shellHook = ''
+    if [[ $- == *i* ]]; then
+      echo "llvm-mingw set up at ${toolchain}."
+      echo ""
+      echo "To configure DXVK/vkd3d-proton: meson setup \$FEX_MESON_CROSSFILE"
+      echo ""
+      echo "To configure 32-bit FEX build: cmake \$FEX_CMAKE_TOOLCHAIN_WOW64"
+      echo "To configure 64-bit FEX build: cmake \$FEX_CMAKE_TOOLCHAIN_ARM64EC"
+    fi
+  '';
+
+  # E.g. cmake $FEX_CMAKE_TOOLCHAIN_ARM64EC -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_LTO=False -DBUILD_TESTS=False
+  FEX_CMAKE_TOOLCHAIN_ARM64EC = "--toolchain ${cmakeToolchainFile} -DMINGW_TRIPLE=arm64ec-w64-mingw32 -DCMAKE_INSTALL_LIBDIR=/usr/lib/wine/aarch64-windows";
+  FEX_CMAKE_TOOLCHAIN_WOW64 = "--toolchain ${cmakeToolchainFile} -DMINGW_TRIPLE=aarch64-w64-mingw32 -DCMAKE_INSTALL_LIBDIR=/usr/lib/wine/aarch64-windows";
+  FEX_MESON_CROSSFILE = "--cross-file ${mesonCrossFile}";
+}

Data/nix/cmake_configure_woa32.sh

@@ -0,0 +1,21 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash WineOnArm/shell.nix
+
+# Helper script to configure CMake for building FEX as library for emulation
+# of 32-bit applications in Wine/Proton.
+# The required cross-toolchains will be set up and managed by nix.
+
+if [ $# -eq 0 ]
+then
+  echo "Expected CMake argument list"
+  exit 1
+fi
+
+if [ -f CMakeCache.txt ]
+then
+  echo "Expected empty build folder"
+  exit 1
+fi
+
+set -o xtrace
+cmake $FEX_CMAKE_TOOLCHAIN_WOW64 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_LTO=False -DBUILD_TESTS=False $@

Data/nix/cmake_configure_woa64.sh

@@ -0,0 +1,21 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash WineOnArm/shell.nix
+
+# Helper script to configure CMake for building FEX as library for emulation
+# of 64-bit applications in Wine/Proton
+# Nix is used to install and manage the required cross-toolchains.
+
+if [ $# -eq 0 ]
+then
+  echo "Expected CMake argument list"
+  exit 1
+fi
+
+if [ -f CMakeCache.txt ]
+then
+  echo "Expected empty build folder"
+  exit 1
+fi
+
+set -o xtrace
+cmake $FEX_CMAKE_TOOLCHAIN_ARM64EC -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_LTO=False -DBUILD_TESTS=False $@

Data/nix/cmake_enable_flt.sh

@@ -0,0 +1,17 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash FEXLinuxTests/shell.nix
+
+# Helper script to configure CMake for building FEXLinuxTests.
+# Nix is used to install and manage the required cross-toolchains.
+
+if [ ! -f CMakeCache.txt ]
+then
+  echo "Must be run from a pre-configured CMake build folder"
+  exit 1
+fi
+
+# Remove previous build to ensure the new toolchain is applied
+rm -rf unittests/FEXLinuxTests
+
+set -o xtrace
+cmake . $FEX_CMAKE_TOOLCHAINS -DBUILD_TESTS=ON -DBUILD_FEX_LINUX_TESTS=ON

Data/nix/cmake_enable_libfwd.sh

@@ -0,0 +1,22 @@
+# Helper script to configure CMake for library forwarding in FEX.
+# Nix is used to install and manage the required cross-toolchains.
+
+if [ ! -f CMakeCache.txt ]
+then
+  echo "Must be run from a pre-configured CMake build folder"
+  exit 1
+fi
+
+# Remove previous build to ensure the new toolchain is applied
+rm -rf guest-libs guest-libs-32 Guest Guest_32
+
+# Set clang executable path manually since the one from the nix store
+# will be picked up otherwise
+CLANG_EXEC_PATH=""
+if ! grep -q CLANG_EXEC_PATH CMakeCache.txt
+then
+  CLANG_EXEC_PATH="-DCLANG_EXEC_PATH=`which clang`"
+fi
+
+nix-shell `dirname -- "$0"`/LibraryForwarding/shell.nix \
+  --run "set -o xtrace; cmake . \$FEX_CMAKE_TOOLCHAINS -DBUILD_THUNKS=ON $CLANG_EXEC_PATH; set +o xtrace"

External/.clang-format

@@ -0,0 +1 @@
+DisableFormat: true

External/code-format-helper/code-format-helper.py

@@ -169,14 +169,9 @@ View the diff from {self.name} here.
 
 
 class ClangFormatHelper(FormatHelper):
-    name = "clang-format"
+    name = "git-clang-format"
     friendly_name = "C/C++ code formatter"
 
-    @property
-    def cformat_wrapper_path(self) -> str:
-        relpath = "../../Scripts/clang-format.py"
-        curpath = os.path.dirname(os.path.abspath(__file__))
-        return os.path.abspath(os.path.normpath(os.path.join(curpath, relpath)))
 
     @property
     def instructions(self) -> str:
@@ -199,7 +194,7 @@ class ClangFormatHelper(FormatHelper):
     def clang_fmt_path(self) -> str:
         if "CLANG_FORMAT_PATH" in os.environ:
             return os.environ["CLANG_FORMAT_PATH"]
-        return "git-clang-format"
+        return "git-clang-format-19"
 
     def has_tool(self) -> bool:
         cmd = [self.clang_fmt_path, "-h"]
@@ -217,7 +212,7 @@ class ClangFormatHelper(FormatHelper):
 
         cf_cmd = [
             self.clang_fmt_path,
-            f"--binary={self.cformat_wrapper_path}",
+            "--binary=clang-format-19",
             "--diff",
         ]
 

External/code-format-helper/requirements_formatting.txt

@@ -1,52 +1,392 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.13
 # by the following command:
 #
-#    pip-compile --output-file=llvm/utils/git/requirements_formatting.txt llvm/utils/git/requirements_formatting.txt.in
+#    pip-compile --generate-hashes --output-file=requirements_formatting.txt --strip-extras requirements_formatting.txt.in
 #
-black==23.9.1
+black==25.1.0 \
+    --hash=sha256:030b9759066a4ee5e5aca28c3c77f9c64789cdd4de8ac1df642c40b708be6171 \
+    --hash=sha256:055e59b198df7ac0b7efca5ad7ff2516bca343276c466be72eb04a3bcc1f82d7 \
+    --hash=sha256:0e519ecf93120f34243e6b0054db49c00a35f84f195d5bce7e9f5cfc578fc2da \
+    --hash=sha256:172b1dbff09f86ce6f4eb8edf9dede08b1fce58ba194c87d7a4f1a5aa2f5b3c2 \
+    --hash=sha256:1e2978f6df243b155ef5fa7e558a43037c3079093ed5d10fd84c43900f2d8ecc \
+    --hash=sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666 \
+    --hash=sha256:3b48735872ec535027d979e8dcb20bf4f70b5ac75a8ea99f127c106a7d7aba9f \
+    --hash=sha256:4b60580e829091e6f9238c848ea6750efed72140b91b048770b64e74fe04908b \
+    --hash=sha256:759e7ec1e050a15f89b770cefbf91ebee8917aac5c20483bc2d80a6c3a04df32 \
+    --hash=sha256:8f0b18a02996a836cc9c9c78e5babec10930862827b1b724ddfe98ccf2f2fe4f \
+    --hash=sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717 \
+    --hash=sha256:96c1c7cd856bba8e20094e36e0f948718dc688dba4a9d78c3adde52b9e6c2299 \
+    --hash=sha256:a1ee0a0c330f7b5130ce0caed9936a904793576ef4d2b98c40835d6a65afa6a0 \
+    --hash=sha256:a22f402b410566e2d1c950708c77ebf5ebd5d0d88a6a2e87c86d9fb48afa0d18 \
+    --hash=sha256:a39337598244de4bae26475f77dda852ea00a93bd4c728e09eacd827ec929df0 \
+    --hash=sha256:afebb7098bfbc70037a053b91ae8437c3857482d3a690fefc03e9ff7aa9a5fd3 \
+    --hash=sha256:bacabb307dca5ebaf9c118d2d2f6903da0d62c9faa82bd21a33eecc319559355 \
+    --hash=sha256:bce2e264d59c91e52d8000d507eb20a9aca4a778731a08cfff7e5ac4a4bb7096 \
+    --hash=sha256:d9e6827d563a2c820772b32ce8a42828dc6790f095f441beef18f96aa6f8294e \
+    --hash=sha256:db8ea9917d6f8fc62abd90d944920d95e73c83a5ee3383493e35d271aca872e9 \
+    --hash=sha256:ea0213189960bda9cf99be5b8c8ce66bb054af5e9e861249cd23471bd7b0b3ba \
+    --hash=sha256:f3df5f1bf91d36002b0a75389ca8663510cf0531cca8aa5c1ef695b46d98655f
     # via
-    #   -r llvm/utils/git/requirements_formatting.txt.in
+    #   -r requirements_formatting.txt.in
     #   darker
-certifi==2023.7.22
-    # via requests
-cffi==1.15.1
+certifi==2025.7.14 \
+    --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \
+    --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995
+    # via
+    #   -r requirements_formatting.txt.in
+    #   requests
+cffi==1.15.1 \
+    --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \
+    --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \
+    --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \
+    --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \
+    --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \
+    --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \
+    --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \
+    --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \
+    --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \
+    --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \
+    --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \
+    --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \
+    --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \
+    --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \
+    --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \
+    --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \
+    --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \
+    --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \
+    --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \
+    --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \
+    --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \
+    --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \
+    --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \
+    --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \
+    --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \
+    --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \
+    --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \
+    --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \
+    --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \
+    --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \
+    --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \
+    --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \
+    --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \
+    --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \
+    --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \
+    --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \
+    --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \
+    --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \
+    --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \
+    --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \
+    --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \
+    --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \
+    --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \
+    --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \
+    --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \
+    --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \
+    --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \
+    --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \
+    --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \
+    --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \
+    --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \
+    --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \
+    --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \
+    --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \
+    --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \
+    --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \
+    --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \
+    --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \
+    --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \
+    --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \
+    --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \
+    --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \
+    --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \
+    --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0
     # via
     #   cryptography
     #   pynacl
-charset-normalizer==3.2.0
+charset-normalizer==3.2.0 \
+    --hash=sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96 \
+    --hash=sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c \
+    --hash=sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710 \
+    --hash=sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706 \
+    --hash=sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020 \
+    --hash=sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252 \
+    --hash=sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad \
+    --hash=sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329 \
+    --hash=sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a \
+    --hash=sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f \
+    --hash=sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6 \
+    --hash=sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4 \
+    --hash=sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a \
+    --hash=sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46 \
+    --hash=sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2 \
+    --hash=sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23 \
+    --hash=sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace \
+    --hash=sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd \
+    --hash=sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982 \
+    --hash=sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10 \
+    --hash=sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2 \
+    --hash=sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea \
+    --hash=sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09 \
+    --hash=sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5 \
+    --hash=sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149 \
+    --hash=sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489 \
+    --hash=sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9 \
+    --hash=sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80 \
+    --hash=sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592 \
+    --hash=sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3 \
+    --hash=sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6 \
+    --hash=sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed \
+    --hash=sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c \
+    --hash=sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200 \
+    --hash=sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a \
+    --hash=sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e \
+    --hash=sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d \
+    --hash=sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6 \
+    --hash=sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623 \
+    --hash=sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669 \
+    --hash=sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3 \
+    --hash=sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa \
+    --hash=sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9 \
+    --hash=sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2 \
+    --hash=sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f \
+    --hash=sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1 \
+    --hash=sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4 \
+    --hash=sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a \
+    --hash=sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8 \
+    --hash=sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3 \
+    --hash=sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029 \
+    --hash=sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f \
+    --hash=sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959 \
+    --hash=sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22 \
+    --hash=sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7 \
+    --hash=sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952 \
+    --hash=sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346 \
+    --hash=sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e \
+    --hash=sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d \
+    --hash=sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299 \
+    --hash=sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd \
+    --hash=sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a \
+    --hash=sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3 \
+    --hash=sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037 \
+    --hash=sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94 \
+    --hash=sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c \
+    --hash=sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858 \
+    --hash=sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a \
+    --hash=sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449 \
+    --hash=sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c \
+    --hash=sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918 \
+    --hash=sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1 \
+    --hash=sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c \
+    --hash=sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac \
+    --hash=sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa
     # via requests
-click==8.1.7
+click==8.1.7 \
+    --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \
+    --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de
     # via black
-cryptography==41.0.3
-    # via pyjwt
-darker==1.7.2
-    # via -r llvm/utils/git/requirements_formatting.txt.in
-deprecated==1.2.14
+cryptography==45.0.5 \
+    --hash=sha256:0027d566d65a38497bc37e0dd7c2f8ceda73597d2ac9ba93810204f56f52ebc7 \
+    --hash=sha256:101ee65078f6dd3e5a028d4f19c07ffa4dd22cce6a20eaa160f8b5219911e7d8 \
+    --hash=sha256:12e55281d993a793b0e883066f590c1ae1e802e3acb67f8b442e721e475e6463 \
+    --hash=sha256:14d96584701a887763384f3c47f0ca7c1cce322aa1c31172680eb596b890ec30 \
+    --hash=sha256:1e1da5accc0c750056c556a93c3e9cb828970206c68867712ca5805e46dc806f \
+    --hash=sha256:206210d03c1193f4e1ff681d22885181d47efa1ab3018766a7b32a7b3d6e6afd \
+    --hash=sha256:2089cc8f70a6e454601525e5bf2779e665d7865af002a5dec8d14e561002e135 \
+    --hash=sha256:3a264aae5f7fbb089dbc01e0242d3b67dffe3e6292e1f5182122bdf58e65215d \
+    --hash=sha256:3af26738f2db354aafe492fb3869e955b12b2ef2e16908c8b9cb928128d42c57 \
+    --hash=sha256:3fcfbefc4a7f332dece7272a88e410f611e79458fab97b5efe14e54fe476f4fd \
+    --hash=sha256:460f8c39ba66af7db0545a8c6f2eabcbc5a5528fc1cf6c3fa9a1e44cec33385e \
+    --hash=sha256:57c816dfbd1659a367831baca4b775b2a5b43c003daf52e9d57e1d30bc2e1b0e \
+    --hash=sha256:5aa1e32983d4443e310f726ee4b071ab7569f58eedfdd65e9675484a4eb67bd1 \
+    --hash=sha256:6ff8728d8d890b3dda5765276d1bc6fb099252915a2cd3aff960c4c195745dd0 \
+    --hash=sha256:7259038202a47fdecee7e62e0fd0b0738b6daa335354396c6ddebdbe1206af2a \
+    --hash=sha256:72e76caa004ab63accdf26023fccd1d087f6d90ec6048ff33ad0445abf7f605a \
+    --hash=sha256:7760c1c2e1a7084153a0f68fab76e754083b126a47d0117c9ed15e69e2103492 \
+    --hash=sha256:8c4a6ff8a30e9e3d38ac0539e9a9e02540ab3f827a3394f8852432f6b0ea152e \
+    --hash=sha256:9024beb59aca9d31d36fcdc1604dd9bbeed0a55bface9f1908df19178e2f116e \
+    --hash=sha256:90cb0a7bb35959f37e23303b7eed0a32280510030daba3f7fdfbb65defde6a97 \
+    --hash=sha256:91098f02ca81579c85f66df8a588c78f331ca19089763d733e34ad359f474174 \
+    --hash=sha256:926c3ea71a6043921050eaa639137e13dbe7b4ab25800932a8498364fc1abec9 \
+    --hash=sha256:982518cd64c54fcada9d7e5cf28eabd3ee76bd03ab18e08a48cad7e8b6f31b18 \
+    --hash=sha256:9b4cf6318915dccfe218e69bbec417fdd7c7185aa7aab139a2c0beb7468c89f0 \
+    --hash=sha256:ad0caded895a00261a5b4aa9af828baede54638754b51955a0ac75576b831b27 \
+    --hash=sha256:b85980d1e345fe769cfc57c57db2b59cff5464ee0c045d52c0df087e926fbe63 \
+    --hash=sha256:b8fa8b0a35a9982a3c60ec79905ba5bb090fc0b9addcfd3dc2dd04267e45f25e \
+    --hash=sha256:b9e38e0a83cd51e07f5a48ff9691cae95a79bea28fe4ded168a8e5c6c77e819d \
+    --hash=sha256:bd4c45986472694e5121084c6ebbd112aa919a25e783b87eb95953c9573906d6 \
+    --hash=sha256:be97d3a19c16a9be00edf79dca949c8fa7eff621763666a145f9f9535a5d7f42 \
+    --hash=sha256:c648025b6840fe62e57107e0a25f604db740e728bd67da4f6f060f03017d5097 \
+    --hash=sha256:d05a38884db2ba215218745f0781775806bde4f32e07b135348355fe8e4991d9 \
+    --hash=sha256:dd420e577921c8c2d31289536c386aaa30140b473835e97f83bc71ea9d2baf2d \
+    --hash=sha256:e357286c1b76403dd384d938f93c46b2b058ed4dfcdce64a770f0537ed3feb6f \
+    --hash=sha256:e6c00130ed423201c5bc5544c23359141660b07999ad82e34e7bb8f882bb78e0 \
+    --hash=sha256:e74d30ec9c7cb2f404af331d5b4099a9b322a8a6b25c4632755c8757345baac5 \
+    --hash=sha256:f3562c2f23c612f2e4a6964a61d942f891d29ee320edb62ff48ffb99f3de9ae8
+    # via
+    #   -r requirements_formatting.txt.in
+    #   pyjwt
+darker==2.1.1 \
+    --hash=sha256:a6e6a682c0604e76fe9aec7650e96a944f517563c69b28fcc076db9d957d98ea \
+    --hash=sha256:ead701414c45359fc0312bc285614d3285fc135476d43f3bc08d989ee19d9020
+    # via -r requirements_formatting.txt.in
+darkgraylib==1.2.1 \
+    --hash=sha256:60c59de69842367ce0c78c32c451fa8e9d29500e681312d9864a7416bcdb7792 \
+    --hash=sha256:a5dd6a2015a470d9047278cdd01a91ccb1d746675f8fd4562b3b5f6b8cbda930
+    # via
+    #   darker
+    #   graylint
+deprecated==1.2.14 \
+    --hash=sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c \
+    --hash=sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3
     # via pygithub
-idna==3.4
-    # via requests
-mypy-extensions==1.0.0
-    # via black
-packaging==23.1
-    # via black
-pathspec==0.11.2
-    # via black
-platformdirs==3.10.0
-    # via black
-pycparser==2.21
-    # via cffi
-pygithub==1.59.1
-    # via -r llvm/utils/git/requirements_formatting.txt.in
-pyjwt[crypto]==2.8.0
-    # via pygithub
-pynacl==1.5.0
-    # via pygithub
-requests==2.31.0
-    # via pygithub
-toml==0.10.2
+graylint==1.1.1 \
+    --hash=sha256:0fd8e02972ca03d0ef2bf0adea76b5343efcd492d7afb5f658f3e3a724f55a36 \
+    --hash=sha256:b7e0eab6c159684dbf5ef84e942c3340f6a6549b02a3d11b1a1763cc4f8f0593
     # via darker
-urllib3==2.0.4
-    # via requests
-wrapt==1.15.0
+idna==3.10 \
+    --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
+    --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
+    # via
+    #   -r requirements_formatting.txt.in
+    #   requests
+mypy-extensions==1.0.0 \
+    --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \
+    --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782
+    # via black
+packaging==23.1 \
+    --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
+    --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
+    # via black
+pathspec==0.11.2 \
+    --hash=sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20 \
+    --hash=sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3
+    # via black
+platformdirs==3.10.0 \
+    --hash=sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d \
+    --hash=sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d
+    # via black
+pycparser==2.21 \
+    --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \
+    --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206
+    # via cffi
+pygithub==2.6.1 \
+    --hash=sha256:6f2fa6d076ccae475f9fc392cc6cdbd54db985d4f69b8833a28397de75ed6ca3 \
+    --hash=sha256:b5c035392991cca63959e9453286b41b54d83bf2de2daa7d7ff7e4312cebf3bf
+    # via -r requirements_formatting.txt.in
+pyjwt==2.8.0 \
+    --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \
+    --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320
+    # via pygithub
+pynacl==1.5.0 \
+    --hash=sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858 \
+    --hash=sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d \
+    --hash=sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93 \
+    --hash=sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1 \
+    --hash=sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92 \
+    --hash=sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff \
+    --hash=sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba \
+    --hash=sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394 \
+    --hash=sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b \
+    --hash=sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543
+    # via pygithub
+requests==2.32.4 \
+    --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \
+    --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422
+    # via
+    #   -r requirements_formatting.txt.in
+    #   pygithub
+toml==0.10.2 \
+    --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
+    --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
+    # via
+    #   darker
+    #   darkgraylib
+typing-extensions==4.14.1 \
+    --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \
+    --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76
+    # via pygithub
+urllib3==2.5.0 \
+    --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \
+    --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc
+    # via
+    #   -r requirements_formatting.txt.in
+    #   pygithub
+    #   requests
+wrapt==1.15.0 \
+    --hash=sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0 \
+    --hash=sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420 \
+    --hash=sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a \
+    --hash=sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c \
+    --hash=sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079 \
+    --hash=sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923 \
+    --hash=sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f \
+    --hash=sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1 \
+    --hash=sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8 \
+    --hash=sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86 \
+    --hash=sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0 \
+    --hash=sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364 \
+    --hash=sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e \
+    --hash=sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c \
+    --hash=sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e \
+    --hash=sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c \
+    --hash=sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727 \
+    --hash=sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff \
+    --hash=sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e \
+    --hash=sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29 \
+    --hash=sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7 \
+    --hash=sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72 \
+    --hash=sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475 \
+    --hash=sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a \
+    --hash=sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317 \
+    --hash=sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2 \
+    --hash=sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd \
+    --hash=sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640 \
+    --hash=sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98 \
+    --hash=sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248 \
+    --hash=sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e \
+    --hash=sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d \
+    --hash=sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec \
+    --hash=sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1 \
+    --hash=sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e \
+    --hash=sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9 \
+    --hash=sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92 \
+    --hash=sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb \
+    --hash=sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094 \
+    --hash=sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46 \
+    --hash=sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29 \
+    --hash=sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd \
+    --hash=sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705 \
+    --hash=sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8 \
+    --hash=sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975 \
+    --hash=sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb \
+    --hash=sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e \
+    --hash=sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b \
+    --hash=sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418 \
+    --hash=sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019 \
+    --hash=sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1 \
+    --hash=sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba \
+    --hash=sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6 \
+    --hash=sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2 \
+    --hash=sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3 \
+    --hash=sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7 \
+    --hash=sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752 \
+    --hash=sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416 \
+    --hash=sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f \
+    --hash=sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1 \
+    --hash=sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc \
+    --hash=sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145 \
+    --hash=sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee \
+    --hash=sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a \
+    --hash=sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7 \
+    --hash=sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b \
+    --hash=sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653 \
+    --hash=sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0 \
+    --hash=sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90 \
+    --hash=sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29 \
+    --hash=sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6 \
+    --hash=sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034 \
+    --hash=sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09 \
+    --hash=sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559 \
+    --hash=sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639
     # via deprecated

External/code-format-helper/requirements_formatting.txt.in

@@ -0,0 +1,8 @@
+black~=25.1
+darker==2.1.1
+PyGithub==2.6.1
+cryptography>=43.0.1
+urllib3>=2.5.0
+requests>=2.32.4
+idna>=3.7
+certifi>=2024.7.4

FEXCore/Scripts/config_generator.py

@@ -418,7 +418,7 @@ def print_parse_argloader_options(options):
 
             if (value_type == "strenum"):
                 output_argloader.write("\tfextl::string UserValue = Options[\"{0}\"];\n".format(op_key))
-                output_argloader.write("\tSet(FEXCore::Config::ConfigOption::CONFIG_{}, FEXCore::Config::EnumParser<FEXCore::Config::{}ConfigPair>(FEXCore::Config::{}_EnumPairs, UserValue));\n".format(op_key.upper(), op_key, op_key, op_key))
+                output_argloader.write("\tSet(FEXCore::Config::ConfigOption::CONFIG_{}, FEXCore::Config::EnumParser<FEXCore::Config::{}ConfigPair>(FEXCore::Config::{}_EnumPairs, UserValue));\n".format(op_key.upper(), op_key, op_key))
             elif (value_type == "strarray"):
                 # these need a bit more help
                 output_argloader.write("\tauto Array = Options.all(\"{0}\");\n".format(op_key))
@@ -447,13 +447,13 @@ def print_parse_envloader_options(options):
             value_type = op_vals["Type"]
             if (value_type == "strenum"):
                 output_argloader.write("else if (Key == \"FEX_{0}\") {{\n".format(op_key.upper()))
-                output_argloader.write("Value = FEXCore::Config::EnumParser<FEXCore::Config::{}ConfigPair>(FEXCore::Config::{}_EnumPairs, Value_View);\n".format(op_key, op_key, op_key))
+                output_argloader.write("\tValue = FEXCore::Config::EnumParser<FEXCore::Config::{}ConfigPair>(FEXCore::Config::{}_EnumPairs, Value_View);\n".format(op_key, op_key))
                 output_argloader.write("}\n")
 
             if ("ArgumentHandler" in op_vals):
                 conversion_func = "FEXCore::Config::Handler::{0}".format(op_vals["ArgumentHandler"])
                 output_argloader.write("else if (Key == \"FEX_{0}\") {{\n".format(op_key.upper()))
-                output_argloader.write("Value = {0}(Value_View);\n".format(conversion_func))
+                output_argloader.write("\tValue = {0}(Value_View);\n".format(conversion_func))
                 output_argloader.write("}\n")
     output_argloader.write("#endif\n")
 
@@ -467,15 +467,15 @@ def print_parse_jsonloader_options(options):
             value_type = op_vals["Type"]
             if (value_type == "strenum"):
                 output_argloader.write("else if (KeyName == \"{0}\") {{\n".format(op_key))
-                output_argloader.write("\tSet(KeyOption, FEXCore::Config::EnumParser<FEXCore::Config::{}ConfigPair>(FEXCore::Config::{}_EnumPairs, Value_View));\n".format(op_key, op_key, op_key))
+                output_argloader.write("\tSet(KeyOption, FEXCore::Config::EnumParser<FEXCore::Config::{}ConfigPair>(FEXCore::Config::{}_EnumPairs, Value_View));\n".format(op_key, op_key))
                 output_argloader.write("}\n")
             elif (value_type == "strarray"):
                 output_argloader.write("else if (KeyName == \"{0}\") {{\n".format(op_key))
                 output_argloader.write("\tAppendStrArrayValue(KeyOption, ConfigString);\n")
                 output_argloader.write("}\n")
     assert op_key is not None, "No options found in JSONLOADER"
-    output_argloader.write("else {{\n".format(op_key))
-    output_argloader.write("Set(KeyOption, ConfigString);\n")
+    output_argloader.write("else {\n")
+    output_argloader.write("\tSet(KeyOption, ConfigString);\n")
     output_argloader.write("}\n")
 
     output_argloader.write("#endif\n")

FEXCore/Scripts/json_ir_generator.py

@@ -58,6 +58,7 @@ class OpDefinition:
     JITDispatch: bool
     JITDispatchOverride: str
     TiedSource: int
+    Inline: list
     Arguments: list
     EmitValidation: list
     Desc: list
@@ -278,6 +279,12 @@ def parse_ops(ops):
             if "TiedSource" in op_val:
                 OpDef.TiedSource = op_val["TiedSource"]
 
+            # Pad Inline out to the argument count
+            OpDef.Inline = [''] * len(OpDef.Arguments)
+            if "Inline" in op_val:
+                Value = op_val["Inline"]
+                OpDef.Inline[0:len(Value)] = Value
+
             # Do some fixups of the data here
             if len(OpDef.EmitValidation) != 0:
                 for i in range(len(OpDef.EmitValidation)):
@@ -397,16 +404,16 @@ def print_ir_sizes():
     // Make sure our array maps directly to the IROps enum
     static_assert(IRSizes[IROps::OP_LAST] == -1ULL);
 
-    [[maybe_unused, nodiscard]] static size_t GetSize(IROps Op) { return IRSizes[Op]; }
-    [[nodiscard, gnu::const, gnu::visibility("default")]] std::string_view const& GetName(IROps Op);
-    [[nodiscard, gnu::const, gnu::visibility("default")]] uint8_t GetArgs(IROps Op);
-    [[nodiscard, gnu::const, gnu::visibility("default")]] uint8_t GetRAArgs(IROps Op);
-    [[nodiscard, gnu::const, gnu::visibility("default")]] FEXCore::IR::RegisterClassType GetRegClass(IROps Op);
-    [[nodiscard, gnu::const, gnu::visibility("default")]] bool HasSideEffects(IROps Op);
-    [[nodiscard, gnu::const, gnu::visibility("default")]] bool ImplicitFlagClobber(IROps Op);
-    [[nodiscard, gnu::const, gnu::visibility("default")]] bool GetHasDest(IROps Op);
-    [[nodiscard, gnu::const, gnu::visibility("default")]] bool LoweredX87(IROps Op);
-    [[nodiscard, gnu::const, gnu::visibility("default")]] int8_t TiedSource(IROps Op);
+    [[nodiscard]] inline size_t GetSize(IROps Op) { return IRSizes[Op]; }
+    [[nodiscard, gnu::const]] std::string_view const& GetName(IROps Op);
+    [[nodiscard, gnu::const]] uint8_t GetArgs(IROps Op);
+    [[nodiscard, gnu::const]] uint8_t GetRAArgs(IROps Op);
+    [[nodiscard, gnu::const]] FEXCore::IR::RegisterClassType GetRegClass(IROps Op);
+    [[nodiscard, gnu::const]] bool HasSideEffects(IROps Op);
+    [[nodiscard, gnu::const]] bool ImplicitFlagClobber(IROps Op);
+    [[nodiscard, gnu::const]] bool GetHasDest(IROps Op);
+    [[nodiscard, gnu::const]] bool LoweredX87(IROps Op);
+    [[nodiscard, gnu::const]] int8_t TiedSource(IROps Op);
 
     #undef IROP_SIZES
     #endif
@@ -575,7 +582,7 @@ def print_ir_arg_printer():
 
                 if arg.IsSSA:
                     # SSA value
-                    output_file.write("\tPrintArg(out, IR, Op->Header.Args[{}], RAData);\n".format(SSAArgNum))
+                    output_file.write("\tPrintArg(out, IR, Op->Header.Args[{}]);\n".format(SSAArgNum))
                     SSAArgNum = SSAArgNum + 1
                 else:
                     # User defined op that is stored
@@ -587,6 +594,15 @@ def print_ir_arg_printer():
     output_file.write("#undef IROP_ARGPRINTER_HELPER\n")
     output_file.write("#endif\n")
 
+def print_validation(op):
+    if len(op.EmitValidation) != 0:
+        output_file.write("#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n")
+
+        for Validation in op.EmitValidation:
+            Sanitized = Validation.replace("\"", "\\\"")
+            output_file.write("\t\tLOGMAN_THROW_A_FMT({}, \"{}\");\n".format(Validation, Sanitized))
+        output_file.write("#endif\n")
+
 # Print out IR allocator helpers
 def print_ir_allocator_helpers():
     output_file.write("#ifdef IROP_ALLOCATE_HELPERS\n")
@@ -678,7 +694,7 @@ def print_ir_allocator_helpers():
                     output_file.write("{} {}".format(CType, arg.Name));
                 elif arg.IsSSA:
                     # SSA value
-                    output_file.write("OrderedNode *{}".format(arg.Name))
+                    output_file.write("OrderedNodeWrapper {}".format(arg.Name))
                 else:
                     # User defined op that is stored
                     CType = IRTypesToCXX[arg.Type].CXXName
@@ -708,35 +724,16 @@ def print_ir_allocator_helpers():
             output_file.write("\t\tauto _Op = AllocateOp<IROp_{}, IROps::OP_{}>();\n".format(op.Name, op.Name.upper()))
 
             if op.SSAArgNum != 0:
-                output_file.write("\t\tauto ListDataBegin = DualListData.ListBegin();\n")
                 for arg in op.Arguments:
                     if arg.IsSSA:
-                        output_file.write("\t\t_Op.first->{} = {}->Wrapped(ListDataBegin);\n".format(arg.Name, arg.Name))
-
-            if op.SSAArgNum != 0:
-                for arg in op.Arguments:
-                    if arg.IsSSA:
-                        output_file.write("\t\t{}->AddUse();\n".format(arg.Name))
+                        output_file.write("\t\t_Op.first->{} = {};\n".format(arg.Name, arg.Name))
 
             if len(op.Arguments) != 0:
                 for arg in op.Arguments:
                     if not arg.Temporary and not arg.IsSSA:
                         output_file.write("\t\t_Op.first->{} = {};\n".format(arg.Name, arg.Name))
 
-            if (op.HasDest):
-                # We can only infer a size if we have arguments
-                if op.DestSize == None:
-                    # We need to infer destination size
-                    output_file.write("\t\tIR::OpSize InferSize = OpSize::iUnsized;\n")
-                    if len(op.Arguments) != 0:
-                        for arg in op.Arguments:
-                            if arg.IsSSA:
-                                output_file.write("\t\tauto Size{} = GetOpSize({});\n".format(arg.Name, arg.Name))
-                        for arg in op.Arguments:
-                            if arg.IsSSA:
-                                output_file.write("\t\tInferSize = std::max(InferSize, Size{});\n".format(arg.Name))
-
-                    output_file.write("\t\t_Op.first->Header.Size = InferSize;\n")
+            assert not (op.HasDest and op.DestSize is None)
 
             # Some ops without a destination still need an operating size
             # Effectively reusing the destination size value for operation size
@@ -748,18 +745,84 @@ def print_ir_allocator_helpers():
             else:
                 output_file.write("\t\t_Op.first->Header.ElementSize = {};\n".format(op.ElementSize))
 
-            # Insert validation here
-            if op.EmitValidation != None:
-                output_file.write("\t\t#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n")
 
-                for Validation in op.EmitValidation:
-                    Sanitized = Validation.replace("\"", "\\\"")
-                    output_file.write("\tLOGMAN_THROW_A_FMT({}, \"{}\");\n".format(Validation, Sanitized))
-                output_file.write("\t\t#endif\n")
+            # Only validate here if there's no OrderedNode * version. Else
+            # validation is in that version, see the comment below.
+            if op.SSAArgNum == 0:
+                print_validation(op)
 
             output_file.write("\t\treturn _Op;\n")
             output_file.write("\t}\n\n")
 
+            # Now do the OrderedNode * version if necessary
+            if op.SSAArgNum:
+                output_file.write("\tIRPair<IROp_{}> _{}(" .format(op.Name, op.Name))
+
+                for i in range(0, len(op.Arguments)):
+                    arg = op.Arguments[i]
+                    LastArg = len(op.Arguments) - i - 1 == 0
+
+                    if arg.Temporary:
+                        CType = IRTypesToCXX[arg.Type].CXXName
+                        output_file.write("{} {}".format(CType, arg.Name));
+                    elif arg.IsSSA:
+                        output_file.write("OrderedNode *{}".format(arg.Name))
+                    else:
+                        CType = IRTypesToCXX[arg.Type].CXXName
+                        output_file.write("{} {}".format(CType, arg.Name));
+
+                    if arg.DefaultInitializer != None:
+                        output_file.write(" = {}".format(arg.DefaultInitializer))
+
+                    if not LastArg:
+                        output_file.write(", ")
+
+                output_file.write(") {\n")
+                output_file.write("\t\tauto ListDataBegin = DualListData.ListBegin();\n")
+
+                idx = 0
+                for arg in op.Arguments:
+                    if arg.IsSSA:
+                        # Inline an immediate if we can
+                        inline = op.Inline[idx]
+                        idx += 1
+
+                        if inline != '':
+                            Sized = "Size" in [x.Name for x in op.Arguments]
+                            P = ["Size" if Sized else "OpSize::i64Bit", arg.Name]
+
+                            # A few cases need extra info plumbed.
+                            if inline == "SubtractZero":
+                                P += ["Src2"]
+                            elif inline == "Mem":
+                                P += ["OffsetType", "OffsetScale"]
+                            elif inline == "Memtso":
+                                P += ["OffsetType", "OffsetScale", "true /* TSO */"]
+                                inline = "Mem"
+
+                            output_file.write(f"\t\t{arg.Name} = Inline{inline}({', '.join(P)});\n")
+
+                        output_file.write(f"\t\t{arg.Name}->AddUse();\n")
+
+                # Insert validation here. This is skipped for the
+                # OrderedNodeWrapper version because validation can depend on
+                # the OrderedNode, but that's ok in practice. Everything pre-RA
+                # uses the OrderedNode version, and anything RA-onwards is
+                # dubious to validate.
+                print_validation(op)
+
+                output_file.write(f"\t\treturn _{op.Name}(")
+                for i in range(0, len(op.Arguments)):
+                    arg = op.Arguments[i]
+                    LastArg = len(op.Arguments) - i - 1 == 0
+                    output_file.write(arg.Name)
+                    if arg.IsSSA:
+                        output_file.write("->Wrapped(ListDataBegin)")
+                    if not LastArg:
+                        output_file.write(", ")
+                output_file.write(");\n");
+                output_file.write("\t}\n\n");
+
     output_file.write("#undef IROP_ALLOCATE_HELPERS\n")
     output_file.write("#endif\n")
 

FEXCore/Source/CMakeLists.txt

@@ -1,4 +1,3 @@
-include(GNUInstallDirs)
 set (MAN_DIR share/man CACHE PATH "MAN_DIR")
 
 set (FEXCORE_BASE_SRCS
@@ -24,9 +23,6 @@ set (SRCS
   Interface/Core/Addressing.cpp
   Interface/Core/CPUID.cpp
   Interface/Core/Frontend.cpp
-  Interface/Core/ObjectCache/JobHandling.cpp
-  Interface/Core/ObjectCache/NamedRegionObjectHandler.cpp
-  Interface/Core/ObjectCache/ObjectCacheService.cpp
   Interface/Core/OpcodeDispatcher/AVX_128.cpp
   Interface/Core/OpcodeDispatcher/Crypto.cpp
   Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -34,7 +30,6 @@ set (SRCS
   Interface/Core/OpcodeDispatcher/X87.cpp
   Interface/Core/OpcodeDispatcher/X87F64.cpp
   Interface/Core/OpcodeDispatcher.cpp
-  Interface/Core/X86Tables.cpp
   Interface/Core/X86HelperGen.cpp
   Interface/Core/ArchHelpers/Arm64Emitter.cpp
   Interface/Core/Dispatcher/Dispatcher.cpp
@@ -66,10 +61,8 @@ set (SRCS
   Interface/IR/IRDumper.cpp
   Interface/IR/IREmitter.cpp
   Interface/IR/PassManager.cpp
-  Interface/IR/Passes/ConstProp.cpp
   Interface/IR/Passes/IRDumperPass.cpp
   Interface/IR/Passes/IRValidation.cpp
-  Interface/IR/Passes/RAValidation.cpp
   Interface/IR/Passes/RedundantFlagCalculationElimination.cpp
   Interface/IR/Passes/RegisterAllocationPass.cpp
   Interface/IR/Passes/x87StackOptimizationPass.cpp

FEXCore/Source/Common/JitSymbols.h

@@ -2,12 +2,10 @@
 #pragma once
 
 #include <FEXCore/fextl/memory.h>
-#include <FEXCore/Debug/InternalThreadState.h>
 
 #include <chrono>
+#include <cstddef>
 #include <cstdint>
-#include <cstdio>
-#include <memory>
 #include <string_view>
 
 namespace FEXCore {

FEXCore/Source/Common/SoftFloat.h

@@ -157,7 +157,30 @@ struct FEX_PACKED X80SoftFloat {
     return Result;
 #else
     /*
-     * FPREM is not an IEEE-754 remainder.  From the spec:
+     * Check for invalid operation cases first - Intel FPREM sets Invalid Operation
+     * for several cases including infinity dividend and zero divisor.
+     */
+    X80SoftFloat result = 0;
+    if (HandleInfinityOp(state, lhs, result)) {
+      return result;
+    } else if (lhs.Exponent == 0x7FFF && (lhs.Significand & 0x7FFFFFFFFFFFFFFFULL)) { // NaN
+      // propagate NaN
+      state->exceptionFlags |= softfloat_flag_invalid;
+      return lhs;
+    }
+
+    // Check for zero divisor - fprem(x, 0) is invalid operation
+    if (rhs.Exponent == 0 && rhs.Significand == 0) {
+      state->exceptionFlags |= softfloat_flag_invalid;
+      // Return QNaN
+      result.Sign = 0;
+      result.Exponent = 0x7FFF;
+      result.Significand = 0xC000000000000000ULL;
+      return result;
+    }
+
+    /*
+     * FPREM is not an IEEE-754 remainder.  From the Intel spec:
      *
      *    Computes the remainder obtained from dividing the value in the ST(0)
      *    register (the dividend) by the value in the ST(1) register (the divisor
@@ -271,7 +294,11 @@ struct FEX_PACKED X80SoftFloat {
   FCMP(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs, bool* eq, bool* lt, bool* nan) {
     *eq = extF80_eq(state, lhs, rhs);
     *lt = extF80_lt(state, lhs, rhs);
-    *nan = IsNan(lhs) || IsNan(rhs);
+
+    // Use IEEE 754 semantics: unordered if neither <, =, nor > is true
+    // This is more reliable than custom NaN detection
+    bool gt = !(*eq) && !(*lt) && extF80_le(state, rhs, lhs);
+    *nan = !(*eq) && !(*lt) && !gt;
   }
 
   FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FSCALE(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs) {
@@ -390,6 +417,11 @@ struct FEX_PACKED X80SoftFloat {
 
     return Result;
 #else
+    X80SoftFloat result;
+    if (HandleInfinityOp(state, lhs, result)) {
+      return result;
+    }
+
     BIGFLOAT Src_d = lhs.ToFMax(state);
     Src_d = FEXCore::cephes_128bit::tanl(Src_d);
     return X80SoftFloat(state, Src_d);
@@ -411,6 +443,11 @@ struct FEX_PACKED X80SoftFloat {
 
     return Result;
 #else
+    X80SoftFloat result;
+    if (HandleInfinityOp(state, lhs, result)) {
+      return result;
+    }
+
     BIGFLOAT Src_d = lhs.ToFMax(state);
     Src_d = FEXCore::cephes_128bit::sinl(Src_d);
     return X80SoftFloat(state, Src_d);
@@ -432,6 +469,11 @@ struct FEX_PACKED X80SoftFloat {
 
     return Result;
 #else
+    X80SoftFloat result;
+    if (HandleInfinityOp(state, lhs, result)) {
+      return result;
+    }
+
     BIGFLOAT Src_d = lhs.ToFMax(state);
     Src_d = FEXCore::cephes_128bit::cosl(Src_d);
     return X80SoftFloat(state, Src_d);
@@ -591,6 +633,20 @@ private:
   static constexpr uint64_t IntegerBit = (1ULL << 63);
   static constexpr uint64_t Bottom62Significand = ((1ULL << 62) - 1);
   static constexpr uint32_t ExponentBias = 16383;
+
+  // Helper function to check for infinity and set invalid operation flag.
+  // Returns true if infinity is dealt with, false otherwise.
+  FEXCORE_PRESERVE_ALL_ATTR static bool HandleInfinityOp(softfloat_state* state, const X80SoftFloat& arg, X80SoftFloat& result) {
+    if (arg.Exponent == 0x7FFF && arg.Significand == 0x8000000000000000ULL) {
+      state->exceptionFlags |= softfloat_flag_invalid;
+      // Return QNaN.
+      result.Sign = 0;
+      result.Exponent = 0x7FFF;
+      result.Significand = 0xC000000000000000ULL;
+      return true;
+    }
+    return false;
+  }
 };
 
 #ifndef _WIN32

FEXCore/Source/Common/StringConv.h

@@ -7,69 +7,58 @@
 #include <optional>
 
 namespace FEXCore::StrConv {
-[[maybe_unused]]
-static bool Conv(std::string_view Value, bool* Result) {
+inline bool Conv(std::string_view Value, bool* Result) {
   *Result = std::strtoull(Value.data(), nullptr, 0);
   return true;
 }
 
-[[maybe_unused]]
-static bool Conv(std::string_view Value, uint8_t* Result) {
+inline bool Conv(std::string_view Value, uint8_t* Result) {
   *Result = std::strtoul(Value.data(), nullptr, 0);
   return true;
 }
 
-[[maybe_unused]]
-static bool Conv(std::string_view Value, int8_t* Result) {
+inline bool Conv(std::string_view Value, int8_t* Result) {
   *Result = std::strtol(Value.data(), nullptr, 0);
   return true;
 }
 
-[[maybe_unused]]
-static bool Conv(std::string_view Value, uint16_t* Result) {
+inline bool Conv(std::string_view Value, uint16_t* Result) {
   *Result = std::strtoul(Value.data(), nullptr, 0);
   return true;
 }
 
-[[maybe_unused]]
-static bool Conv(std::string_view Value, int16_t* Result) {
+inline bool Conv(std::string_view Value, int16_t* Result) {
   *Result = std::strtol(Value.data(), nullptr, 0);
   return true;
 }
 
-[[maybe_unused]]
-static bool Conv(std::string_view Value, uint32_t* Result) {
+inline bool Conv(std::string_view Value, uint32_t* Result) {
   *Result = std::strtoul(Value.data(), nullptr, 0);
   return true;
 }
 
-[[maybe_unused]]
-static bool Conv(std::string_view Value, int32_t* Result) {
+inline bool Conv(std::string_view Value, int32_t* Result) {
   *Result = std::strtol(Value.data(), nullptr, 0);
   return true;
 }
 
-[[maybe_unused]]
-static bool Conv(std::string_view Value, uint64_t* Result) {
+inline bool Conv(std::string_view Value, uint64_t* Result) {
   *Result = std::strtoull(Value.data(), nullptr, 0);
   return true;
 }
 
-[[maybe_unused]]
-static bool Conv(std::string_view Value, int64_t* Result) {
+inline bool Conv(std::string_view Value, int64_t* Result) {
   *Result = std::strtoll(Value.data(), nullptr, 0);
   return true;
 }
 
 template<typename T, typename = std::enable_if<std::is_enum<T>::value, T>>
-[[maybe_unused]]
-static bool Conv(std::string_view Value, T* Result) {
+inline bool Conv(std::string_view Value, T* Result) {
   *Result = static_cast<T>(std::stoull(Value.data(), nullptr, 0));
   return true;
 }
 
-[[maybe_unused]]
-static bool Conv(std::string_view Value, fextl::string* Result) {
+inline bool Conv(std::string_view Value, fextl::string* Result) {
   *Result = Value;
   return true;
 }

FEXCore/Source/Common/VectorRegType.h

@@ -3,14 +3,32 @@
 
 #ifdef _M_X86_64
 #include <xmmintrin.h>
+#include <immintrin.h>
 #endif
 
 namespace FEXCore {
+struct VectorScalarF64Pair {
+  double val[2];
+};
+
 #ifdef _M_ARM_64
 // Can't use uint8x16_t directly from arm_neon.h here.
 // Overrides softfloat-3e's defines which causes problems.
 using VectorRegType = __attribute__((neon_vector_type(16))) uint8_t;
+struct VectorRegPairType {
+  VectorRegType val[2];
+};
+
+static inline VectorRegPairType MakeVectorRegPair(VectorRegType low, VectorRegType high) {
+  return VectorRegPairType {low, high};
+}
+
 #elif defined(_M_X86_64)
 using VectorRegType = __m128i;
+using VectorRegPairType = __m256i;
+
+static inline VectorRegPairType MakeVectorRegPair(VectorRegType low, VectorRegType high) {
+  return _mm256_set_m128i(high, low);
+}
 #endif
 } // namespace FEXCore

FEXCore/Source/Interface/Config/Config.json.in

@@ -18,17 +18,6 @@
           "Maximum number of instruction to store in a block"
         ]
       },
-      "CacheObjectCodeCompilation": {
-        "Type": "uint32",
-        "Default": "FEXCore::Config::ConfigObjectCodeHandler::CONFIG_NONE",
-        "TextDefault": "none",
-        "Choices": [ "none", "read", "readwrite" ],
-        "ArgumentHandler": "CacheObjectCodeHandler",
-        "Desc": [
-          "Cache JIT object code to drive.",
-          "Allows JIT code to be shared between applications"
-        ]
-      },
       "HostFeatures": {
         "Type": "strenum",
         "Default": "FEXCore::Config::HostFeatures::OFF",
@@ -68,7 +57,11 @@
           "ENABLESVEBITPERM": "enablesvebitperm",
           "DISABLESVEBITPERM": "disablesvebitperm",
           "ENABLEPRESERVEALLABI": "enablepreserveallabi",
-          "DISABLEPRESERVEALLABI": "disablepreserveallabi"
+          "DISABLEPRESERVEALLABI": "disablepreserveallabi",
+          "ENABLEWFXT": "enablewfxt",
+          "DISABLEWFXT": "disablewfxt",
+          "ENABLE3DNOW": "enable3dnow",
+          "DISABLE3DNOW": "disable3dnow"
         },
         "Desc": [
           "Allows controlling of the CPU features in the JIT.",
@@ -89,7 +82,9 @@
           "\t{enable,disable}crypto: Will force enable or disable crypto extensions even if the host doesn't support it",
           "\t{enable,disable}rpres: Will force enable or disable rpres even if the host doesn't support it",
           "\t{enable,disable}svebitperm: Will force enable or disable svebitperm even if the host doesn't support it",
-          "\t{enable,disable}preserveallabi: Will force enable or disable preserve_all abi even if the host doesn't support it"
+          "\t{enable,disable}preserveallabi: Will force enable or disable preserve_all abi even if the host doesn't support it",
+          "\t{enable,disable}wfxt: Will force enable or disable wfxt even if the host doesn't support it",
+          "\t{enable,disable}3dnow: Will force enable or disable 3DNow even if the host doesn't support it"
         ]
       },
       "SmallTSCScale": {
@@ -118,7 +113,7 @@
       },
       "ThunkHostLibs": {
         "Type": "str",
-        "Default": "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/fex-emu/HostThunks/",
+        "Default": "@CMAKE_INSTALL_FULL_LIBDIR@/fex-emu/HostThunks",
         "ShortArg": "t",
         "Desc": [
           "Folder to find the host-side thunking libraries."
@@ -126,26 +121,12 @@
       },
       "ThunkGuestLibs": {
         "Type": "str",
-        "Default": "@CMAKE_INSTALL_PREFIX@/share/fex-emu/GuestThunks/",
+        "Default": "@CMAKE_INSTALL_PREFIX@/share/fex-emu/GuestThunks",
         "ShortArg": "j",
         "Desc": [
           "Folder to find the guest-side thunking libraries."
         ]
       },
-      "ThunkHostLibs32": {
-        "Type": "str",
-        "Default": "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/fex-emu/HostThunks_32/",
-        "Desc": [
-          "Folder to find the 32-bit host-side thunking libraries."
-        ]
-      },
-      "ThunkGuestLibs32": {
-        "Type": "str",
-        "Default": "@CMAKE_INSTALL_PREFIX@/share/fex-emu/GuestThunks_32/",
-        "Desc": [
-          "Folder to find the 32-bit guest-side thunking libraries."
-        ]
-      },
       "ThunkConfig": {
         "Type": "str",
         "Default": "",
@@ -484,32 +465,16 @@
         "Desc": [
           "Contrains the startup sleep to only apply to processes that match this name."
         ]
+      },
+      "MonoHacks": {
+        "Type": "bool",
+        "Default": "true",
+        "Desc": [
+          "Permits a hook-based SMC approach and smaller JIT blocks when mono is detected."
+        ]
       }
     },
     "Misc": {
-      "AOTIRCapture": {
-        "Type": "bool",
-        "Default": "false",
-        "Desc": [
-          "Captures IR and generates an AOT IR cache.",
-          "Captures both the loaded executable and libraries it loads."
-        ]
-      },
-      "AOTIRGenerate": {
-        "Type": "bool",
-        "Default": "false",
-        "Desc": [
-          "Scans file for executable code and generates an AOT IR cache.",
-          "Does not run the executable."
-        ]
-      },
-      "AOTIRLoad": {
-        "Type": "bool",
-        "Default": "false",
-        "Desc": [
-          "Loads an AOT IR cache for the loaded executable."
-        ]
-      },
       "ServerSocketPath": {
         "Type": "str",
         "Default": "",
@@ -523,6 +488,27 @@
         "Desc": [
           "Disables inline syscalls in order to support seccomp handling"
         ]
+      },
+      "ExtendedVolatileMetadata": {
+        "Type": "str",
+        "Default": "",
+        "Desc": [
+          "Configuration provided volatile metadata. Only implemented for WoW64/arm64ec.",
+          "Limited in its use but can be handy.",
+          "Extends on top of what Microsoft has for volatile metadata, but also supported for WoW64.",
+          "Colon delimited modules, then semi-colon delimited instructions, then comma delimited ranges",
+          "Default disables TSO in the module, unless instructions overlap the range",
+          "<module>;<offset begin>-<offset-end>,...;<instruction offset to force TSO>,...:<another>",
+          "examples:",
+          "  * Disable TSO for a full module: Just provide the module name:",
+          "      `hl2_linux`",
+          "  * Disable TSO for a part of the module:",
+          "      `hl2_linux;<offset begin>-<offset-end>`",
+          "  * Disable TSO for a part of the module, but enable TSO for some instructions within the module",
+          "      `hl2_linux;<offset begin>-<offset-end>;<instruction offset>,<instruction offset>`",
+          "  * Disable TSO for multiple modules",
+          "      `hl2_linux:libsdl2.so`"
+        ]
       }
     }
   },

FEXCore/Source/Interface/Context/Context.cpp

@@ -8,18 +8,12 @@
 #include <FEXCore/Core/CPUID.h>
 #include <FEXCore/Core/HostFeatures.h>
 #include <FEXCore/Core/SignalDelegator.h>
+#include <FEXCore/HLE/SyscallHandler.h>
+
 #include <FEXCore/Core/Thunks.h>
 #include "FEXCore/Debug/InternalThreadState.h"
 
-#include <string.h>
-#include <utility>
-
 namespace FEXCore::Context {
-void InitializeStaticTables(OperatingMode Mode) {
-  X86Tables::InitializeInfoTables(Mode);
-  IR::InstallOpcodeHandlers(Mode);
-}
-
 fextl::unique_ptr<FEXCore::Context::Context> FEXCore::Context::Context::CreateNewContext(const FEXCore::HostFeatures& Features) {
   return fextl::make_unique<FEXCore::Context::ContextImpl>(Features);
 }

FEXCore/Source/Interface/Context/Context.h

@@ -2,11 +2,12 @@
 #pragma once
 
 #include "Common/JitSymbols.h"
+#include "Interface/Core/CPUBackend.h"
 #include "Interface/Core/CPUID.h"
 #include "Interface/Core/X86HelperGen.h"
-#include "Interface/Core/ObjectCache/ObjectCacheService.h"
 #include "Interface/Core/Dispatcher/Dispatcher.h"
 #include "Interface/IR/AOTIR.h"
+#include <Interface/IR/IntrusiveIRList.h>
 #include <FEXCore/Config/Config.h>
 #include <FEXCore/Core/Context.h>
 #include <FEXCore/Core/CoreState.h>
@@ -33,10 +34,6 @@ namespace FEXCore {
 class CodeLoader;
 class ThunkHandler;
 
-namespace CodeSerialize {
-  class CodeObjectSerializeService;
-}
-
 namespace CPU {
   class Arm64JITCore;
   class Dispatcher;
@@ -50,9 +47,6 @@ namespace HLE {
 } // namespace FEXCore
 
 namespace FEXCore::IR {
-class RegisterAllocationData;
-struct IRListCopy;
-class IRListView;
 namespace Validation {
   class IRValidation;
 }
@@ -60,8 +54,9 @@ namespace Validation {
 
 namespace FEXCore::Context {
 struct FEX_PACKED ExitFunctionLinkData {
-  uint64_t HostBranch;
+  uint64_t HostCode;
   uint64_t GuestRIP;
+  int64_t CallerOffset;
 };
 
 struct CustomIRResult {
@@ -76,7 +71,7 @@ struct CustomIRResult {
 using BlockDelinkerFunc = void (*)(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record);
 constexpr uint32_t TSC_SCALE_MAXIMUM = 1'000'000'000; ///< 1Ghz
 
-class ContextImpl final : public FEXCore::Context::Context {
+class ContextImpl final : public FEXCore::Context::Context, CPU::CodeBufferManager {
 public:
   // Context base class implementation.
   bool InitCore() override;
@@ -90,6 +85,7 @@ public:
 
   bool IsAddressInCurrentBlock(FEXCore::Core::InternalThreadState* Thread, uint64_t Address, uint64_t Size) override;
   bool IsCurrentBlockSingleInst(FEXCore::Core::InternalThreadState* Thread) override;
+  uint64_t GetGuestBlockEntry(FEXCore::Core::InternalThreadState* Thread) override;
 
   uint64_t RestoreRIPFromHostPC(FEXCore::Core::InternalThreadState* Thread, uint64_t HostPC) override;
   uint32_t ReconstructCompactedEFLAGS(FEXCore::Core::InternalThreadState* Thread, bool WasInJIT, const uint64_t* HostGPRs, uint64_t PSTATE) override;
@@ -148,25 +144,12 @@ public:
   FEXCore::IR::AOTIRCacheEntry* LoadAOTIRCacheEntry(const fextl::string& Name) override;
   void UnloadAOTIRCacheEntry(FEXCore::IR::AOTIRCacheEntry* Entry) override;
 
-  void SetAOTIRLoader(AOTIRLoaderCBFn CacheReader) override {
-    IRCaptureCache.SetAOTIRLoader(std::move(CacheReader));
-  }
-  void SetAOTIRWriter(AOTIRWriterCBFn CacheWriter) override {
-    IRCaptureCache.SetAOTIRWriter(std::move(CacheWriter));
-  }
-  void SetAOTIRRenamer(AOTIRRenamerCBFn CacheRenamer) override {
-    IRCaptureCache.SetAOTIRRenamer(std::move(CacheRenamer));
-  }
+  void FinalizeAOTIRCache() override {}
 
-  void FinalizeAOTIRCache() override {
-    IRCaptureCache.FinalizeAOTIRCache();
-  }
-  void WriteFilesWithCode(AOTIRCodeFileWriterFn Writer) override {
-    IRCaptureCache.WriteFilesWithCode(Writer);
-  }
-
-  void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread) override;
-  void InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) override;
+  void OnCodeBufferAllocated(CPU::CodeBuffer&) override;
+  void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) override;
+  void InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, uint64_t Start,
+                                uint64_t Length) override;
   FEXCore::ForkableSharedMutex& GetCodeInvalidationMutex() override {
     return CodeInvalidationMutex;
   }
@@ -187,6 +170,12 @@ public:
 
   void RemoveForceTSOInformation(uint64_t Address, uint64_t Size) override;
 
+  void MarkMonoDetected() override {
+    MonoDetected = true;
+  }
+
+  void MarkMonoBackpatcherBlock(uint64_t BlockEntry) override;
+
 public:
   friend class FEXCore::HLE::SyscallHandler;
 #ifdef JIT_ARM64
@@ -211,9 +200,6 @@ public:
     FEX_CONFIG_OPT(VectorTSOEnabled, VECTORTSOENABLED);
     FEX_CONFIG_OPT(MemcpySetTSOEnabled, MEMCPYSETTSOENABLED);
     FEX_CONFIG_OPT(ABILocalFlags, ABILOCALFLAGS);
-    FEX_CONFIG_OPT(AOTIRCapture, AOTIRCAPTURE);
-    FEX_CONFIG_OPT(AOTIRGenerate, AOTIRGENERATE);
-    FEX_CONFIG_OPT(AOTIRLoad, AOTIRLOAD);
     FEX_CONFIG_OPT(SMCChecks, SMCCHECKS);
     FEX_CONFIG_OPT(MaxInstPerBlock, MAXINST);
     FEX_CONFIG_OPT(RootFSPath, ROOTFS);
@@ -222,12 +208,12 @@ public:
     FEX_CONFIG_OPT(BlockJITNaming, BLOCKJITNAMING);
     FEX_CONFIG_OPT(GDBSymbols, GDBSYMBOLS);
     FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
-    FEX_CONFIG_OPT(CacheObjectCodeCompilation, CACHEOBJECTCODECOMPILATION);
     FEX_CONFIG_OPT(x87ReducedPrecision, X87REDUCEDPRECISION);
     FEX_CONFIG_OPT(DisableTelemetry, DISABLETELEMETRY);
     FEX_CONFIG_OPT(DisableVixlIndirectCalls, DISABLE_VIXL_INDIRECT_RUNTIME_CALLS);
     FEX_CONFIG_OPT(SmallTSCScale, SMALLTSCSCALE);
     FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS);
+    FEX_CONFIG_OPT(MonoHacks, MONOHACKS);
   } Config;
 
   FEXCore::ForkableSharedMutex CodeInvalidationMutex;
@@ -246,59 +232,46 @@ public:
   X86GeneratedCode X86CodeGen;
 
   ContextImpl(const FEXCore::HostFeatures& Features);
-  ~ContextImpl();
 
-  static void ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP);
+  static bool ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP);
 
-  template<auto Fn>
-  static uint64_t ThreadExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, ExitFunctionLinkData* Record) {
-    auto Thread = Frame->Thread;
-    auto lk = GuardSignalDeferringSection<std::shared_lock>(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
+  static void ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP);
 
-    return Fn(Frame, Record);
-  }
+  // This is used as a replacement for the SMC writes in the mono callsite backpatcher that avoids atomic operations
+  // (safe as the invalidation mutex is locked) and manually invalidates the modified range. Allowing SMC to be detected
+  // even if faulting is disabled.
+  static void MonoBackpatcherWrite(FEXCore::Core::CpuStateFrame* Frame, uint8_t Size, uint64_t Address, uint64_t Value);
 
-  // Wrapper which takes CpuStateFrame instead of InternalThreadState and unique_locks CodeInvalidationMutex
-  // Must be called from owning thread
-  static void ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP) {
-    auto Thread = Frame->Thread;
-    auto lk = GuardSignalDeferringSection(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
-
-    ThreadRemoveCodeEntry(Thread, GuestRIP);
-  }
-
-  void RemoveCustomIREntrypoint(uintptr_t Entrypoint);
+  void RemoveCustomIREntrypoint(FEXCore::Core::InternalThreadState* Thread, uintptr_t Entrypoint);
 
   struct GenerateIRResult {
     std::optional<IR::IRListView> IRView;
-    IR::RegisterAllocationData* RAData;
     uint64_t TotalInstructions;
     uint64_t TotalInstructionsLength;
     uint64_t StartAddr;
     uint64_t Length;
+    bool NeedsAddGuestCodeRanges;
   };
   [[nodiscard]]
   GenerateIRResult GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, bool ExtendedDebugInfo, uint64_t MaxInst);
 
   struct CompileCodeResult {
-    void* CompiledCode;
+    CPU::CPUBackend::CompiledCode CompiledCode;
     fextl::unique_ptr<FEXCore::Core::DebugData> DebugData;
     uint64_t StartAddr;
     uint64_t Length;
+    bool NeedsAddGuestCodeRanges;
   };
   [[nodiscard]]
   CompileCodeResult CompileCode(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, uint64_t MaxInst = 0);
   uintptr_t CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP, uint64_t MaxInst = 0);
   uintptr_t CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP);
 
-  IR::OpSize GetGPROpSize() const {
-    return Config.Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;
-  }
-
   FEXCore::JITSymbols Symbols;
 
   FEXCore::Utils::PooledAllocatorVirtual OpDispatcherAllocator;
   FEXCore::Utils::PooledAllocatorVirtual FrontendAllocator;
+  FEXCore::Utils::PooledAllocatorVirtual CPUBackendAllocator;
 
   // If Atomic-based TSO emulation is enabled or not.
   bool IsAtomicTSOEnabled() const {
@@ -328,6 +301,10 @@ public:
     return ExitOnHLT;
   }
 
+  bool AreMonoHacksActive() const {
+    return Config.MonoHacks && MonoDetected;
+  }
+
 protected:
   void UpdateAtomicTSOEmulationConfig() {
     if (SupportsHardwareTSO) {
@@ -360,7 +337,6 @@ private:
   void InitializeCompiler(FEXCore::Core::InternalThreadState* Thread);
 
   IR::AOTIRCaptureCache IRCaptureCache;
-  fextl::unique_ptr<FEXCore::CodeSerialize::CodeObjectSerializeService> CodeObjectCacheService;
 
   bool IsMemoryShared = false;
   bool SupportsHardwareTSO = false;
@@ -373,8 +349,16 @@ private:
 
   std::shared_mutex CustomIRMutex;
   std::atomic<bool> HasCustomIRHandlers {};
-  fextl::unordered_map<uint64_t, std::tuple<CustomIREntrypointHandler, void*, void*>> CustomIRHandlers;
+  struct CustomIRHandlerEntry final {
+    CustomIREntrypointHandler Handler;
+    void *Creator;
+    void *Data;
+  };
+  fextl::unordered_map<uint64_t, CustomIRHandlerEntry> CustomIRHandlers;
   IntervalList<uint64_t> ForceTSOValidRanges; // The ranges for which ForceTSOInstructions has populated data
   fextl::set<uint64_t> ForceTSOInstructions;
+
+  bool MonoDetected = false;
+  std::atomic<uint64_t> MonoBackpatcherBlock;
 };
 } // namespace FEXCore::Context

FEXCore/Source/Interface/Core/Addressing.cpp

@@ -11,8 +11,7 @@ Ref LoadEffectiveAddress(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSize, b
   Ref Tmp = A.Base;
 
   if (A.Offset) {
-    Ref Offset = IREmit->_Constant(A.Offset);
-    Tmp = Tmp ? IREmit->_Add(GPRSize, Tmp, Offset) : Offset;
+    Tmp = Tmp ? IREmit->Add(GPRSize, Tmp, A.Offset) : IREmit->Constant(A.Offset);
   }
 
   if (A.Index) {
@@ -22,10 +21,10 @@ Ref LoadEffectiveAddress(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSize, b
       if (Tmp) {
         Tmp = IREmit->_AddShift(GPRSize, Tmp, A.Index, ShiftType::LSL, Log2);
       } else {
-        Tmp = IREmit->_Lshl(GPRSize, A.Index, IREmit->_Constant(Log2));
+        Tmp = IREmit->_Lshl(GPRSize, A.Index, IREmit->Constant(Log2));
       }
     } else {
-      Tmp = Tmp ? IREmit->_Add(GPRSize, Tmp, A.Index) : A.Index;
+      Tmp = Tmp ? IREmit->Add(GPRSize, Tmp, A.Index) : A.Index;
     }
   }
 
@@ -34,31 +33,35 @@ Ref LoadEffectiveAddress(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSize, b
   //
   // If the AddrSize is not the GPRSize then we need to clear the upper bits.
   if ((A.AddrSize < GPRSize) && !AllowUpperGarbage && Tmp) {
-    Tmp = IREmit->_Bfe(GPRSize, IR::OpSizeAsBits(A.AddrSize), 0, Tmp);
+    uint32_t Bits = IR::OpSizeAsBits(A.AddrSize);
+
+    if (A.Base || A.Index) {
+      Tmp = IREmit->_Bfe(GPRSize, Bits, 0, Tmp);
+    } else if (A.Offset) {
+      uint64_t X = A.Offset;
+      X &= (1ull << Bits) - 1;
+      Tmp = IREmit->Constant(X);
+    }
   }
 
   if (A.Segment && AddSegmentBase) {
-    Tmp = Tmp ? IREmit->_Add(GPRSize, Tmp, A.Segment) : A.Segment;
+    Tmp = Tmp ? IREmit->Add(GPRSize, Tmp, A.Segment) : A.Segment;
   }
 
-  return Tmp ?: IREmit->_Constant(0);
+  return Tmp ?: IREmit->Constant(0);
 }
 
 AddressMode SelectAddressMode(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSize, bool HostSupportsTSOImm9, bool AtomicTSO, bool Vector,
                               IR::OpSize AccessSize) {
-  auto SoftwareAddressCalculation = [IREmit, &A, GPRSize]() -> AddressMode {
-    return {
-      .Base = LoadEffectiveAddress(IREmit, A, GPRSize, true),
-      .Index = IREmit->Invalid(),
-    };
-  };
-
   const auto Is32Bit = GPRSize == OpSize::i32Bit;
   const auto GPRSizeMatchesAddrSize = A.AddrSize == GPRSize;
   const auto OffsetIndexToLargeFor32Bit = Is32Bit && (A.Offset <= -16384 || A.Offset >= 16384);
   if (!GPRSizeMatchesAddrSize || OffsetIndexToLargeFor32Bit) {
     // If address size doesn't match GPR size then no optimizations can occur.
-    return SoftwareAddressCalculation();
+    return {
+      .Base = LoadEffectiveAddress(IREmit, A, GPRSize, true),
+      .Index = IREmit->Invalid(),
+    };
   }
 
   // Loadstore rules:
@@ -92,43 +95,31 @@ AddressMode SelectAddressMode(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSi
   const bool OffsetIsSIMM9 = A.Offset && A.Offset >= -256 && A.Offset <= 255;
   const bool OffsetIsUnsignedScaled = A.Offset > 0 && (A.Offset & (AccessSizeAsImm - 1)) == 0 && (A.Offset / AccessSizeAsImm) <= 4095;
 
-  auto InlineImmOffsetLoadstore = [IREmit, &GPRSize](AddressMode A) -> AddressMode {
+  if ((AtomicTSO && !Vector && HostSupportsTSOImm9 && OffsetIsSIMM9) || (!AtomicTSO && (OffsetIsSIMM9 || OffsetIsUnsignedScaled))) {
     // Peel off the offset
     AddressMode B = A;
     B.Offset = 0;
 
     return {
       .Base = LoadEffectiveAddress(IREmit, B, GPRSize, true /* AddSegmentBase */, false),
-      .Index = IREmit->_Constant(A.Offset),
+      .Index = IREmit->Constant(A.Offset),
       .IndexType = MEM_OFFSET_SXTX,
       .IndexScale = 1,
     };
-  };
+  }
 
-  auto ScaledRegisterLoadstore = [IREmit, GPRSize](AddressMode A) -> AddressMode {
+  if (AtomicTSO) {
+    // TODO: LRCPC3 support for vector Imm9.
+  } else if (!Is32Bit && A.Base && (A.Index || A.Segment) && !A.Offset && (A.IndexScale == 1 || A.IndexScale == AccessSizeAsImm)) {
+    // ScaledRegisterLoadstore
     if (A.Index && A.Segment) {
-      A.Base = IREmit->_Add(GPRSize, A.Base, A.Segment);
+      A.Base = IREmit->Add(GPRSize, A.Base, A.Segment);
     } else if (A.Segment) {
       A.Index = A.Segment;
       A.IndexScale = 1;
     }
-    return A;
-  };
 
-  if (AtomicTSO) {
-    if (!Vector) {
-      if (HostSupportsTSOImm9 && OffsetIsSIMM9) {
-        return InlineImmOffsetLoadstore(A);
-      }
-    } else {
-      // TODO: LRCPC3 support for vector Imm9.
-    }
-  } else {
-    if (OffsetIsSIMM9 || OffsetIsUnsignedScaled) {
-      return InlineImmOffsetLoadstore(A);
-    } else if (!Is32Bit && A.Base && (A.Index || A.Segment) && !A.Offset && (A.IndexScale == 1 || A.IndexScale == AccessSizeAsImm)) {
-      return ScaledRegisterLoadstore(A);
-    }
+    return A;
   }
 
   if (Vector || !AtomicTSO) {
@@ -142,7 +133,7 @@ AddressMode SelectAddressMode(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSi
 
         return {
           .Base = LoadEffectiveAddress(IREmit, B, GPRSize, true /* AddSegmentBase */, false),
-          .Index = IREmit->_Constant(A.Offset),
+          .Index = IREmit->Constant(A.Offset),
           .IndexType = MEM_OFFSET_SXTX,
           .IndexScale = 1,
         };
@@ -151,8 +142,11 @@ AddressMode SelectAddressMode(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSi
   }
 
   // Fallback on software address calculation
-  return SoftwareAddressCalculation();
+  return {
+    .Base = LoadEffectiveAddress(IREmit, A, GPRSize, true),
+    .Index = IREmit->Invalid(),
+  };
 }
 
 
-}; // namespace FEXCore::IR
+}; // namespace FEXCore::IR

FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp

@@ -73,13 +73,13 @@ namespace x64 {
     ARMEmitter::Reg::r8, ARMEmitter::Reg::r16, ARMEmitter::Reg::r17,
   };
 
-  constexpr std::array<ARMEmitter::Register, 8> RA = {
+  constexpr std::array<ARMEmitter::Register, 7> RA = {
     // All these callee saved
     ARMEmitter::Reg::r20, ARMEmitter::Reg::r21, ARMEmitter::Reg::r22, ARMEmitter::Reg::r23,
-    ARMEmitter::Reg::r24, ARMEmitter::Reg::r25, ARMEmitter::Reg::r30, ARMEmitter::Reg::r18,
+    ARMEmitter::Reg::r24, ARMEmitter::Reg::r30, ARMEmitter::Reg::r18,
   };
 
-  constexpr unsigned RAPairs = 6;
+  constexpr unsigned RAPairs = 4;
 
   // Dynamic GPRs
   constexpr std::array<ARMEmitter::Register, 2> PreserveAll_Dynamic = {
@@ -143,18 +143,18 @@ namespace x64 {
     ARMEmitter::Reg::r4, ARMEmitter::Reg::r5, ARMEmitter::Reg::r8,
   };
 
-  constexpr std::array<ARMEmitter::Register, 7> RA = {
-    ARMEmitter::Reg::r6,  ARMEmitter::Reg::r7,  ARMEmitter::Reg::r14, ARMEmitter::Reg::r15,
-    ARMEmitter::Reg::r16, ARMEmitter::Reg::r17, ARMEmitter::Reg::r30,
+  constexpr std::array<ARMEmitter::Register, 6> RA = {
+    ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r14, ARMEmitter::Reg::r15, ARMEmitter::Reg::r16, ARMEmitter::Reg::r30,
   };
 
-  constexpr std::array<ARMEmitter::Register, 5> PreserveAll_Dynamic = {
-    ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r16, ARMEmitter::Reg::r17, ARMEmitter::Reg::r30,
-  };
+  constexpr std::array<ARMEmitter::Register, 5> PreserveAll_Dynamic = {ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r16,
+                                                                       ARMEmitter::Reg::r17, ARMEmitter::Reg::r30};
 
-  constexpr std::array<ARMEmitter::Register, 7> NotPreserved_Dynamic = RA;
+  constexpr std::array<ARMEmitter::Register, 7> NotPreserved_Dynamic = {ARMEmitter::Reg::r6,  ARMEmitter::Reg::r7,  ARMEmitter::Reg::r14,
+                                                                        ARMEmitter::Reg::r15, ARMEmitter::Reg::r16, ARMEmitter::Reg::r17,
+                                                                        ARMEmitter::Reg::r30};
 
-  constexpr unsigned RAPairs = 6;
+  constexpr unsigned RAPairs = 4;
 
   constexpr std::array<ARMEmitter::VRegister, 16> SRAFPR = {
     ARMEmitter::VReg::v0,  ARMEmitter::VReg::v1,  ARMEmitter::VReg::v2,  ARMEmitter::VReg::v3,
@@ -245,14 +245,12 @@ namespace x32 {
     REG_AF,
   };
 
-  constexpr std::array<ARMEmitter::Register, 15> RA = {
+  constexpr std::array<ARMEmitter::Register, 14> RA = {
     // All these callee saved
     ARMEmitter::Reg::r20,
     ARMEmitter::Reg::r21,
     ARMEmitter::Reg::r22,
     ARMEmitter::Reg::r23,
-    ARMEmitter::Reg::r24,
-    ARMEmitter::Reg::r25,
 
     // Registers only available on 32-bit
     // All these are caller saved (except for r19).
@@ -265,6 +263,7 @@ namespace x32 {
     ARMEmitter::Reg::r29,
     ARMEmitter::Reg::r30,
 
+    ARMEmitter::Reg::r24,
     ARMEmitter::Reg::r19,
   };
 
@@ -273,7 +272,7 @@ namespace x32 {
     ARMEmitter::Reg::r16, ARMEmitter::Reg::r17, ARMEmitter::Reg::r30,
   };
 
-  constexpr unsigned RAPairs = 12;
+  constexpr unsigned RAPairs = 10;
 
   // All are caller saved
   constexpr std::array<ARMEmitter::VRegister, 8> SRAFPR = {
@@ -370,6 +369,8 @@ Arm64Emitter::Arm64Emitter(FEXCore::Context::ContextImpl* ctx, void* EmissionPtr
   // Hardcode a 256-bit vector width if we are running in the simulator.
   // Allow the user to override this.
   Simulator.SetVectorLengthInBits(ForceSVEWidth() ? ForceSVEWidth() : 256);
+  // FEX doesn't support GCS.
+  Simulator.DisableGCSCheck();
 #endif
 #ifdef VIXL_DISASSEMBLER
   // Only setup the disassembler if enabled.
@@ -495,20 +496,23 @@ void Arm64Emitter::LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, ui
   uint64_t AlignedPC = PC & ~0xFFFULL;
 
   // Offset from aligned PC
-  int64_t AlignedOffset = static_cast<int64_t>(Constant) - static_cast<int64_t>(AlignedPC);
+  auto AlignedOffset = std::bit_cast<int64_t>(Constant - AlignedPC);
 
   int NumMoves = 0;
 
   // If the aligned offset is within the 4GB window then we can use ADRP+ADD
   // and the number of move segments more than 1
-  if (RequiredMoveSegments > 1 && ARMEmitter::Emitter::IsInt32(AlignedOffset)) {
+  // NOTE: JIT output is moved to a different buffer after compilation, so the
+  //       current cursor address doesn't match the runtime instruction address.
+  //       Hence this optimization is disabled until we enable code relocation patches.
+  if (RequiredMoveSegments > 1 && ARMEmitter::Emitter::IsInt32(AlignedOffset) && false) {
     // If this is 4k page aligned then we only need ADRP
     if ((AlignedOffset & 0xFFF) == 0) {
       adrp(Reg, AlignedOffset >> 12);
     } else {
       // If the constant is within 1MB of PC then we can still use ADR to load in a single instruction
       // 21-bit signed integer here
-      int64_t SmallOffset = static_cast<int64_t>(Constant) - static_cast<int64_t>(PC);
+      auto SmallOffset = std::bit_cast<int64_t>(Constant - PC);
       if (ARMEmitter::Emitter::IsInt21(SmallOffset)) {
         adr(Reg, SmallOffset);
       } else {
@@ -590,8 +594,8 @@ void Arm64Emitter::PushCalleeSavedRegisters() {
 
 void Arm64Emitter::PopCalleeSavedRegisters() {
   constexpr static std::array< std::tuple<ARMEmitter::DRegister, ARMEmitter::DRegister, ARMEmitter::DRegister, ARMEmitter::DRegister>, 2> FPRs = {{
-    {ARMEmitter::DReg::d12, ARMEmitter::DReg::d13, ARMEmitter::DReg::d14, ARMEmitter::DReg::d15},
     {ARMEmitter::DReg::d8, ARMEmitter::DReg::d9, ARMEmitter::DReg::d10, ARMEmitter::DReg::d11},
+    {ARMEmitter::DReg::d12, ARMEmitter::DReg::d13, ARMEmitter::DReg::d14, ARMEmitter::DReg::d15},
   }};
 
   for (auto& RegQuad : FPRs) {
@@ -691,6 +695,8 @@ void Arm64Emitter::SpillStaticRegs(ARMEmitter::Register TmpReg, bool FPRs, uint3
   unsigned PFAFSpillMask = GPRSpillMask & PFAFMask;
   GPRSpillMask &= ~PFAFSpillMask;
 
+  str(REG_CALLRET_SP, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.callret_sp));
+
   for (size_t i = 0; i < StaticRegisters.size(); i += 2) {
     auto Reg1 = StaticRegisters[i];
     auto Reg2 = StaticRegisters[i + 1];
@@ -706,7 +712,7 @@ void Arm64Emitter::SpillStaticRegs(ARMEmitter::Register TmpReg, bool FPRs, uint3
   // Now handle PF/AF
   if (PFAFSpillMask) {
     auto PFOffset = offsetof(FEXCore::Core::CpuStateFrame, State.pf_raw);
-    [[maybe_unused]] auto AFOffset = offsetof(FEXCore::Core::CpuStateFrame, State.af_raw);
+    auto AFOffset = offsetof(FEXCore::Core::CpuStateFrame, State.af_raw);
     LOGMAN_THROW_A_FMT(PFAFSpillMask == PFAFMask, "PF/AF not spilled together");
     LOGMAN_THROW_A_FMT(AFOffset == PFOffset + 4, "PF/AF are together");
 
@@ -786,6 +792,8 @@ void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t GPRFillMask, uint32_t FPRF
   ldr(STATE, TmpReg, CPU_AREA_EMULATOR_DATA_OFFSET);
 #endif
 
+  ldr(REG_CALLRET_SP, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.callret_sp));
+
   // Regardless of what GPRs/FPRs we're filling, we need to fill NZCV since it
   // is always static and was almost certainly clobbered.
   //

FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h

@@ -2,7 +2,7 @@
 #pragma once
 
 #include "FEXCore/Utils/EnumUtils.h"
-#include "Interface/Core/ObjectCache/Relocations.h"
+#include "Interface/Core/JIT/Relocations.h"
 
 #ifdef VIXL_DISASSEMBLER
 #include <aarch64/disasm-aarch64.h>
@@ -43,6 +43,8 @@ constexpr bool TMP_ABIARGS = true;
 constexpr auto REG_PF = ARMEmitter::Reg::r26;
 constexpr auto REG_AF = ARMEmitter::Reg::r27;
 
+constexpr auto REG_CALLRET_SP = ARMEmitter::XReg::x25;
+
 // Vector temporaries
 constexpr auto VTMP1 = ARMEmitter::VReg::v0;
 constexpr auto VTMP2 = ARMEmitter::VReg::v1;
@@ -61,6 +63,8 @@ constexpr bool TMP_ABIARGS = false;
 constexpr auto REG_PF = ARMEmitter::Reg::r9;
 constexpr auto REG_AF = ARMEmitter::Reg::r24;
 
+constexpr auto REG_CALLRET_SP = ARMEmitter::XReg::x17;
+
 // Vector temporaries
 constexpr auto VTMP1 = ARMEmitter::VReg::v16;
 constexpr auto VTMP2 = ARMEmitter::VReg::v17;
@@ -84,7 +88,8 @@ constexpr uint64_t EC_CODE_BITMAP_MAX_ADDRESS = 1ULL << 47;
 #endif
 
 // Will force one single instruction block to be generated first if set when entering the JIT filling SRA.
-constexpr auto ENTRY_FILL_SRA_SINGLE_INST_REG = TMP1;
+// FillStaticRegs must preserve this
+constexpr auto ENTRY_FILL_SRA_SINGLE_INST_REG = TMP2;
 
 // Predicate to use in the X87 SVE optimization
 constexpr ARMEmitter::PRegister PRED_X87_SVEOPT = ARMEmitter::PReg::p2;

FEXCore/Source/Interface/Core/CPUBackend.cpp

@@ -6,6 +6,8 @@
 #include "Interface/Core/Dispatcher/Dispatcher.h"
 #include <cstdint>
 
+#include "LookupCache.h"
+
 #ifndef _WIN32
 #include <sys/prctl.h>
 #endif
@@ -13,6 +15,10 @@
 namespace FEXCore {
 namespace CPU {
 
+  static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16;
+  // We don't want to move above 128MB atm because that means we will have to encode longer jumps
+  static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 128;
+
   constexpr static uint64_t NamedVectorConstants[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX][2] = {
     {0x0003'0002'0001'0000ULL, 0x0007'0006'0005'0004ULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX
     {0x000B'000A'0009'0008ULL, 0x000F'000E'000D'000CULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX_UPPER
@@ -264,10 +270,9 @@ namespace CPU {
     return TotalLUT;
   }()};
 
-  CPUBackend::CPUBackend(FEXCore::Core::InternalThreadState* ThreadState, size_t InitialCodeSize, size_t MaxCodeSize)
+  CPUBackend::CPUBackend(CodeBufferManager& CodeBuffers, FEXCore::Core::InternalThreadState* ThreadState)
     : ThreadState(ThreadState)
-    , InitialCodeSize(InitialCodeSize)
-    , MaxCodeSize(MaxCodeSize) {
+    , CodeBuffers(CodeBuffers) {
 
     auto& Common = ThreadState->CurrentFrame->Pointers.Common;
 
@@ -304,52 +309,62 @@ namespace CPU {
 #endif
   }
 
-  CPUBackend::~CPUBackend() {
-    for (auto CodeBuffer : CodeBuffers) {
-      FreeCodeBuffer(CodeBuffer);
-    }
-    CodeBuffers.clear();
-  }
+  CPUBackend::~CPUBackend() = default;
 
   auto CPUBackend::GetEmptyCodeBuffer() -> CodeBuffer* {
-    if (ThreadState->CurrentFrame->SignalHandlerRefCounter == 0) {
-      if (CodeBuffers.empty()) {
-        auto NewCodeBuffer = AllocateNewCodeBuffer(InitialCodeSize);
-        EmplaceNewCodeBuffer(NewCodeBuffer);
-      } else {
-        if (CodeBuffers.size() > 1) {
-          // If we have more than one code buffer we are tracking then walk them and delete
-          // This is a cleanup step
-          for (size_t i = 1; i < CodeBuffers.size(); i++) {
-            FreeCodeBuffer(CodeBuffers[i]);
-          }
-          CodeBuffers.resize(1);
-        }
-        // Set the current code buffer to the initial
-        CurrentCodeBuffer = CodeBuffers.data();
+    auto PrevCodeBuffer = CurrentCodeBuffer;
 
-        if (CurrentCodeBuffer->Size != MaxCodeSize) {
-          FreeCodeBuffer(*CurrentCodeBuffer);
+    // Resize the code buffer and reallocate our code size
+    CurrentCodeBuffer = CodeBuffers.StartLargerCodeBuffer();
 
-          // Resize the code buffer and reallocate our code size
-          CurrentCodeBuffer->Size *= 1.5;
-          CurrentCodeBuffer->Size = std::min(CurrentCodeBuffer->Size, MaxCodeSize);
-
-          *CurrentCodeBuffer = AllocateNewCodeBuffer(CurrentCodeBuffer->Size);
-        }
-      }
-    } else {
-      // We have signal handlers that have generated code
-      // This means that we can not safely clear the code at this point in time
-      // Allocate some new code buffers that we can switch over to instead
-      auto NewCodeBuffer = AllocateNewCodeBuffer(InitialCodeSize);
-      EmplaceNewCodeBuffer(NewCodeBuffer);
-    }
-
-    return CurrentCodeBuffer;
+    RegisterForSignalHandler(std::move(PrevCodeBuffer));
+    return CurrentCodeBuffer.get();
   }
 
-  auto CPUBackend::AllocateNewCodeBuffer(size_t Size) -> CodeBuffer {
+  void CPUBackend::RegisterForSignalHandler(fextl::shared_ptr<CodeBuffer> CodeBuffer) {
+    if (ThreadState->CurrentFrame->SignalHandlerRefCounter != 0) {
+      // We have signal handlers that have generated code
+      // This means that we can not safely clear the code at this point in time
+      // Keep a reference to the old code buffer to delay deallocation
+      SignalHandlerCodeBuffers.push_back(std::move(CodeBuffer));
+    } else {
+      SignalHandlerCodeBuffers.clear();
+    }
+  }
+
+  fextl::shared_ptr<CodeBuffer> CPUBackend::CheckCodeBufferUpdate() {
+    auto NewCodeBuffer = CodeBuffers.GetLatest();
+    if (CurrentCodeBuffer != NewCodeBuffer) {
+      RegisterForSignalHandler(CurrentCodeBuffer);
+      return std::exchange(CurrentCodeBuffer, NewCodeBuffer);
+    }
+    return nullptr;
+  }
+
+  GuestToHostMap& GetLookupCache(const CodeBuffer& Buffer) {
+    return *Buffer.LookupCache;
+  }
+
+  CodeBuffer::CodeBuffer(size_t Size)
+    : Size(Size) {
+    Ptr = static_cast<uint8_t*>(FEXCore::Allocator::VirtualAlloc(Size, true));
+    LOGMAN_THROW_A_FMT(!!Ptr, "Couldn't allocate code buffer");
+
+    // Protect the last page of the allocated buffer to trigger SIGSEGV on write access
+    uintptr_t LastPageAddr = AlignDown(reinterpret_cast<uintptr_t>(Ptr) + Size - 1, FEXCore::Utils::FEX_PAGE_SIZE);
+    if (!FEXCore::Allocator::VirtualProtect(reinterpret_cast<void*>(LastPageAddr), FEXCore::Utils::FEX_PAGE_SIZE,
+                                            FEXCore::Allocator::ProtectOptions::None)) {
+      LogMan::Msg::EFmt("Failed to mprotect last page of code buffer.");
+    }
+
+    LookupCache = fextl::make_unique<GuestToHostMap>();
+  }
+
+  CodeBuffer::~CodeBuffer() {
+    FEXCore::Allocator::VirtualFree(Ptr, Size);
+  }
+
+  auto CodeBufferManager::AllocateNew(size_t Size) -> fextl::shared_ptr<CodeBuffer> {
 #ifndef _WIN32
 // MDWE (Memory-Deny-Write-Execute) is a new Linux 6.3 feature.
 // It's equivalent to systemd's `MemoryDenyWriteExecute` but implemented entirely in the kernel.
@@ -375,39 +390,51 @@ namespace CPU {
     }
 #endif
 
-    CodeBuffer Buffer;
-    Buffer.Size = Size;
-    Buffer.Ptr = static_cast<uint8_t*>(FEXCore::Allocator::VirtualAlloc(Buffer.Size, true));
-    LOGMAN_THROW_A_FMT(!!Buffer.Ptr, "Couldn't allocate code buffer");
+    auto Buffer = fextl::make_shared<CodeBuffer>(Size);
 
-    if (static_cast<Context::ContextImpl*>(ThreadState->CTX)->Config.GlobalJITNaming()) {
-      static_cast<Context::ContextImpl*>(ThreadState->CTX)->Symbols.RegisterJITSpace(Buffer.Ptr, Buffer.Size);
-    }
+    Latest = Buffer;
+    LatestOffset = 0;
 
-    // Protect the last page of the allocated buffer to trigger SIGSEGV on write access
-    uintptr_t LastPageAddr = AlignDown(reinterpret_cast<uintptr_t>(Buffer.Ptr) + Buffer.Size - 1, FEXCore::Utils::FEX_PAGE_SIZE);
-    if (!FEXCore::Allocator::VirtualProtect(reinterpret_cast<void*>(LastPageAddr), FEXCore::Utils::FEX_PAGE_SIZE,
-                                            FEXCore::Allocator::ProtectOptions::None)) {
-      LogMan::Msg::EFmt("Failed to mprotect last page of code buffer.");
-    }
+    OnCodeBufferAllocated(*Buffer);
 
     return Buffer;
   }
 
-  void CPUBackend::FreeCodeBuffer(CodeBuffer Buffer) {
-    FEXCore::Allocator::VirtualFree(Buffer.Ptr, Buffer.Size);
+  fextl::shared_ptr<CodeBuffer> CodeBufferManager::GetLatest() {
+    if (!Latest) {
+      AllocateNew(INITIAL_CODE_SIZE);
+    }
+    return Latest;
   }
 
+  fextl::shared_ptr<CodeBuffer> CodeBufferManager::StartLargerCodeBuffer() {
+    if (!Latest) {
+      // Allocate initial CodeBuffer and return it
+      return GetLatest();
+    }
+
+    auto NewCodeBufferSize = GetLatest()->Size;
+    NewCodeBufferSize = std::min<size_t>(NewCodeBufferSize * 2, MAX_CODE_SIZE);
+    return AllocateNew(NewCodeBufferSize);
+  }
+
+
   bool CPUBackend::IsAddressInCodeBuffer(uintptr_t Address) const {
-    // The last page of the code buffer is protected, so we need to exclude it from the valid range
-    // when checking if the address is in the code buffer.
-    for (auto& Buffer : CodeBuffers) {
+    auto CheckCodeBuffer = [](CodeBuffer& Buffer, uintptr_t Address) {
+      // The last page of the code buffer is protected, so we need to exclude it from the valid range
+      // when checking if the address is in the code buffer.
       uintptr_t LastPageAddr = AlignDown(reinterpret_cast<uintptr_t>(Buffer.Ptr) + Buffer.Size - 1, FEXCore::Utils::FEX_PAGE_SIZE);
-      if (Address >= reinterpret_cast<uintptr_t>(Buffer.Ptr) && Address < LastPageAddr) {
+      return (Address >= reinterpret_cast<uintptr_t>(Buffer.Ptr) && Address < LastPageAddr);
+    };
+
+    if (CheckCodeBuffer(*CurrentCodeBuffer, Address)) {
+      return true;
+    }
+    for (auto& Buffer : SignalHandlerCodeBuffers) {
+      if (CheckCodeBuffer(*Buffer, Address)) {
         return true;
       }
     }
-
     return false;
   }
 

FEXCore/Source/Interface/Core/CPUBackend.h

@@ -9,8 +9,11 @@ $end_info$
 #pragma once
 
 #include <FEXCore/Utils/CompilerDefs.h>
+#include <FEXCore/Utils/SignalScopeGuards.h>
+#include <FEXCore/fextl/memory.h>
 #include <FEXCore/fextl/string.h>
 #include <FEXCore/fextl/vector.h>
+#include <FEXCore/fextl/map.h>
 
 #include <cstdint>
 
@@ -18,7 +21,6 @@ namespace FEXCore {
 
 namespace IR {
   class IRListView;
-  class RegisterAllocationData;
 } // namespace IR
 
 namespace Core {
@@ -32,34 +34,68 @@ namespace CodeSerialize {
   struct CodeObjectFileSection;
 }
 
+struct GuestToHostMap;
+
 namespace CPU {
+  struct CodeBuffer {
+    uint8_t* Ptr;
+    size_t Size;
+
+    fextl::unique_ptr<GuestToHostMap> LookupCache;
+
+    CodeBuffer(size_t Size);
+    CodeBuffer(const CodeBuffer&) = delete;
+    CodeBuffer& operator=(const CodeBuffer&) = delete;
+    CodeBuffer(CodeBuffer&& oth) = delete;
+    CodeBuffer& operator=(CodeBuffer&&) = delete;
+
+    ~CodeBuffer();
+  };
+
+  /**
+   * A manager that coordinates access to the CodeBuffer used for compiling new code across threads.
+   *
+   * The CodeBuffer is managed as a partially persistent data structure:
+   * - Exactly one CodeBuffer is now designated as "active", which means data can be appended to it
+   * - Lossy modifications to the active CodeBuffer will not invalidate any data in use by other threads (which is what enables save CodeBuffer sharing across threads)
+   * - Instead, such lossy modifications trigger a new "version" of the data in the modifying thread. Old versions of the CodeBuffer persist as read-only data for use by the other threads.
+   * - The other threads can update their version of the CodeBuffer. This will decrease the reference count and eventually trigger deallocation of the old version
+   */
+  class CodeBufferManager {
+  public:
+    // Get the CodeBuffer that was most recently allocated.
+    // This is the only CodeBuffer that data may be written to.
+    fextl::shared_ptr<CodeBuffer> GetLatest();
+
+    // Allocate a new CodeBuffer with geometric growth up to an internal maximum.
+    // Subsequent calls to GetLatest will point to the returned buffer.
+    fextl::shared_ptr<CodeBuffer> StartLargerCodeBuffer();
+
+    // Write offset into the latest CodeBuffer
+    std::size_t LatestOffset {};
+
+    // Protects writes to the latest CodeBuffer and changes to LatestOffset
+    FEXCore::ForkableUniqueMutex CodeBufferWriteMutex;
+
+    virtual void OnCodeBufferAllocated(CodeBuffer&) {};
+
+  private:
+    fextl::shared_ptr<CodeBuffer> Latest;
+
+    fextl::shared_ptr<CodeBuffer> AllocateNew(size_t Size);
+  };
+
   class CPUBackend {
   public:
-    struct CodeBuffer {
-      uint8_t* Ptr;
-      size_t Size;
-    };
 
-    /**
-     * @param InitialCodeSize - Initial size for the code buffers
-     * @param MaxCodeSize - Max size for the code buffers
-     */
-    CPUBackend(FEXCore::Core::InternalThreadState* ThreadState, size_t InitialCodeSize, size_t MaxCodeSize);
+    CPUBackend(CodeBufferManager&, FEXCore::Core::InternalThreadState*);
 
     virtual ~CPUBackend();
 
     struct CompiledCode {
       // Where this code block begins.
       uint8_t* BlockBegin;
-      /**
-       * The function entrypoint to this codeblock.
-       *
-       * This may or may not equal `BlockBegin` above. Depending on the CPU backend, it may stick data
-       * prior to the BlockEntry.
-       *
-       * Is actually a function pointer of type `void (FEXCore::Core::ThreadState *Thread)`
-       */
-      uint8_t* BlockEntry;
+      fextl::map<uint64_t, uint8_t*> EntryPoints;
       // The total size of the codeblock from [BlockBegin, BlockBegin+Size).
       size_t Size;
     };
@@ -119,7 +155,7 @@ namespace CPU {
      */
     [[nodiscard]]
     virtual CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
-                                     FEXCore::Core::DebugData* DebugData, const FEXCore::IR::RegisterAllocationData* RAData, bool CheckTF) = 0;
+                                     FEXCore::Core::DebugData* DebugData, bool CheckTF) = 0;
 
     /**
      * @brief Relocates a block of code from the JIT code object cache
@@ -143,6 +179,11 @@ namespace CPU {
 
     bool IsAddressInCodeBuffer(uintptr_t Address) const;
 
+    // Updates the CodeBuffer if needed and returns a reference to the old one.
+    // The returned reference should be kept alive carefully to avoid early deletion of resources.
+    [[nodiscard]]
+    fextl::shared_ptr<CodeBuffer> CheckCodeBufferUpdate();
+
   protected:
     // Max spill slot size in bytes. We need at most 32 bytes
     // to be able to handle a 256-bit vector store to a slot.
@@ -150,24 +191,20 @@ namespace CPU {
 
     FEXCore::Core::InternalThreadState* ThreadState;
 
-    size_t InitialCodeSize, MaxCodeSize;
     [[nodiscard]]
     CodeBuffer* GetEmptyCodeBuffer();
 
-    // This is the current code buffer that we are tracking
-    CodeBuffer* CurrentCodeBuffer {};
+    // This is the code buffer containing the main code under execution by this thread.
+    // CheckCodeBufferUpdate must be used before compiling new code.
+    fextl::shared_ptr<CodeBuffer> CurrentCodeBuffer;
+
+    // Old CodeBuffer generations required to be valid until returning from signal handlers
+    fextl::vector<fextl::shared_ptr<CodeBuffer>> SignalHandlerCodeBuffers;
+
+    CodeBufferManager& CodeBuffers;
 
   private:
-    CodeBuffer AllocateNewCodeBuffer(size_t Size);
-    void FreeCodeBuffer(CodeBuffer Buffer);
-
-    void EmplaceNewCodeBuffer(CodeBuffer Buffer) {
-      CurrentCodeBuffer = &CodeBuffers.emplace_back(Buffer);
-    }
-
-    // This is the array of code buffers. Unless signals force us to keep more than
-    // buffer, there will be only one entry here
-    fextl::vector<CodeBuffer> CodeBuffers {};
+    void RegisterForSignalHandler(fextl::shared_ptr<CodeBuffer>);
   };
 
 } // namespace CPU

FEXCore/Source/Interface/Core/CPUID.cpp

@@ -626,6 +626,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
     // Only enable EnhancedREPMOVS if atomic memcpy tso emulation isn't enabled.
     const uint32_t SupportsEnhancedREPMOVS = CTX->IsMemcpyAtomicTSOEnabled() == false;
     const uint32_t SupportsVPCLMULQDQ = CTX->HostFeatures.SupportsPMULL_128Bit && SupportsAVX();
+    const uint32_t SupportsWFXT = CTX->HostFeatures.SupportsWFXT;
 
     // Number of subfunctions
     Res.eax = 0x0;
@@ -645,39 +646,39 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (1 << 13) |                              // Deprecates FPU CS and DS
               (0 << 14) |                              // Intel MPX
               (0 << 15) |                              // Intel Resource Directory Technology Allocation
-              (0 << 16) |                              // Reserved
-              (0 << 17) |                              // Reserved
+              (0 << 16) |                              // AVX512-F
+              (0 << 17) |                              // AVX512-DQ
               (CTX->HostFeatures.SupportsRAND << 18) | // RDSEED
               (1 << 19) |                              // ADCX and ADOX instructions
               (0 << 20) |                              // SMAP Supervisor mode access prevention and CLAC/STAC instructions
-              (0 << 21) |                              // Reserved
-              (0 << 22) |                              // Reserved
+              (0 << 21) |                              // AVX512-IFMA
+              (0 << 22) |                              // PCOMMIT (deprecated?)
               (1 << 23) |                              // CLFLUSHOPT instruction
               (1 << 24) |                              // CLWB instruction
               (0 << 25) |                              // Intel processor trace
-              (0 << 26) |                              // Reserved
-              (0 << 27) |                              // Reserved
-              (0 << 28) |                              // Reserved
+              (0 << 26) |                              // AVX512-PF
+              (0 << 27) |                              // AVX512-ER
+              (0 << 28) |                              // AVX512-CD
               (Features.SHA << 29) |                   // SHA instructions
-              (0 << 30) |                              // Reserved
-              (0 << 31);                               // Reserved
+              (0 << 30) |                              // AVX512-BW
+              (0 << 31);                               // AVX512-VL
 
     Res.ecx = (1 << 0) |                                // PREFETCHWT1
               (0 << 1) |                                // AVX512VBMI
               (0 << 2) |                                // Usermode instruction prevention
               (0 << 3) |                                // Protection keys for user mode pages
               (0 << 4) |                                // OS protection keys
-              (0 << 5) |                                // waitpkg
-              (0 << 6) |                                // AVX512_VBMI2
+              (SupportsWFXT << 5) |                     // waitpkg
+              (0 << 6) |                                // AVX512-VBMI2
               (0 << 7) |                                // CET shadow stack
               (0 << 8) |                                // GFNI
               (CTX->HostFeatures.SupportsAES256 << 9) | // VAES
               (SupportsVPCLMULQDQ << 10) |              // VPCLMULQDQ
-              (0 << 11) |                               // AVX512_VNNI
-              (0 << 12) |                               // AVX512_BITALG
+              (0 << 11) |                               // AVX512-VNNI
+              (0 << 12) |                               // AVX512-BITALG
               (0 << 13) |                               // Intel Total Memory Encryption
-              (0 << 14) |                               // AVX512_VPOPCNTDQ
-              (0 << 15) |                               // Reserved
+              (0 << 14) |                               // AVX512-VPOPCNTDQ
+              (0 << 15) |                               // FZM (TDX)
               (0 << 16) |                               // 5 Level page tables
               (0 << 17) |                               // MPX MAWAU
               (0 << 18) |                               // MPX MAWAU
@@ -685,28 +686,28 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (0 << 20) |                               // MPX MAWAU
               (0 << 21) |                               // MPX MAWAU
               (1 << 22) |                               // RDPID Read Processor ID
-              (0 << 23) |                               // Reserved
-              (0 << 24) |                               // Reserved
+              (0 << 23) |                               // AES Key Locker
+              (1 << 24) |                               // bus-lock-detect
               (0 << 25) |                               // CLDEMOTE
-              (0 << 26) |                               // Reserved
+              (0 << 26) |                               // MPRR (TDX)
               (0 << 27) |                               // MOVDIRI
               (0 << 28) |                               // MOVDIR64B
-              (0 << 29) |                               // Reserved
+              (0 << 29) |                               // ENQCMD
               (0 << 30) |                               // SGX Launch configuration
-              (0 << 31);                                // Reserved
+              (0 << 31);                                // PKS
 
-    Res.edx = (0 << 0) |                   // Reserved
-              (0 << 1) |                   // Reserved
-              (0 << 2) |                   // AVX512_4VNNIW
-              (0 << 3) |                   // AVX512_4FMAPS
+    Res.edx = (0 << 0) |                   // SGX-TEM (TDX)
+              (0 << 1) |                   // SGX-KEYS
+              (0 << 2) |                   // AVX512-4VNNIW
+              (0 << 3) |                   // AVX512-4FMAPS
               (1 << 4) |                   // Fast Short Rep Mov
-              (0 << 5) |                   // Reserved
+              (0 << 5) |                   // UINTR
               (0 << 6) |                   // Reserved
               (0 << 7) |                   // Reserved
-              (0 << 8) |                   // AVX512_VP2INTERSECT
+              (0 << 8) |                   // AVX512-VP2INTERSECT
               (0 << 9) |                   // SRBDS_CTRL (Special Register Buffer Data Sampling Mitigations)
               (0 << 10) |                  // VERW clears CPU buffers
-              (0 << 11) |                  // Reserved
+              (0 << 11) |                  // rtm-always-abort
               (0 << 12) |                  // Reserved
               (0 << 13) |                  // TSX Force Abort (TSX will force abort if attempted)
               (0 << 14) |                  // SERIALIZE instruction
@@ -718,7 +719,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (0 << 20) |                  // Intel CET
               (0 << 21) |                  // Reserved
               (0 << 22) |                  // AMX-BF16 - Tile computation on bfloat16
-              (0 << 23) |                  // AVX512_FP16 - FP16 AVX512 instructions
+              (0 << 23) |                  // AVX512-FP16 - FP16 AVX512 instructions
               (0 << 24) |                  // AMX-tile - If AMX is implemented
               (0 << 25) |                  // AMX-int8 - AMX on 8-bit integers
               (0 << 26) |                  // IBRS_IBPB - Speculation control
@@ -754,7 +755,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_0Dh(uint32_t Leaf) const {
     // XFeatureSupportedMask[63:32]
     Res.edx = 0; // Upper 32-bits of XFeatureSupportedMask
   } else if (Leaf == 1) {
-    Res.eax = (0 << 0) | // XSAVEOPT
+    Res.eax = (1 << 0) | // XSAVEOPT
               (0 << 1) | // XSAVEC (and XRSTOR)
               (0 << 2) | // XGETBV - XGETBV with ECX=1 supported
               (0 << 3);  // XSAVES - XSAVES, XRSTORS, and IA32_XSS supported
@@ -924,38 +925,38 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0001h(uint32_t Leaf) con
             (0 << 30) | // Reserved
             (0 << 31);  // Reserved
 
-  Res.edx = (1 << 0) |                // FPU
-            (1 << 1) |                // Virtual mode extensions
-            (1 << 2) |                // Debugging extensions
-            (1 << 3) |                // Page size extensions
-            (1 << 4) |                // TSC
-            (1 << 5) |                // MSR support
-            (1 << 6) |                // PAE
-            (1 << 7) |                // Machine Check Exception
-            (1 << 8) |                // CMPXCHG8B
-            (1 << 9) |                // APIC
-            (0 << 10) |               // Reserved
-            (1 << 11) |               // SYSCALL/SYSRET
-            (1 << 12) |               // MTRR
-            (1 << 13) |               // Page global extension
-            (1 << 14) |               // Machine Check architecture
-            (1 << 15) |               // CMOV
-            (1 << 16) |               // Page attribute table
-            (1 << 17) |               // Page-size extensions
-            (0 << 18) |               // Reserved
-            (0 << 19) |               // Reserved
-            (1 << 20) |               // NX
-            (0 << 21) |               // Reserved
-            (1 << 22) |               // MMXExt
-            (1 << 23) |               // MMX
-            (1 << 24) |               // FXSAVE/FXRSTOR
-            (1 << 25) |               // FXSAVE/FXRSTOR Optimizations
-            (0 << 26) |               // 1 gigabit pages
-            (SUPPORTS_RDTSCP << 27) | // RDTSCP
-            (0 << 28) |               // Reserved
-            (1 << 29) |               // Long Mode
-            (1 << 30) |               // 3DNow! Extensions
-            (1 << 31);                // 3DNow!
+  Res.edx = (1 << 0) |                                // FPU
+            (1 << 1) |                                // Virtual mode extensions
+            (1 << 2) |                                // Debugging extensions
+            (1 << 3) |                                // Page size extensions
+            (1 << 4) |                                // TSC
+            (1 << 5) |                                // MSR support
+            (1 << 6) |                                // PAE
+            (1 << 7) |                                // Machine Check Exception
+            (1 << 8) |                                // CMPXCHG8B
+            (1 << 9) |                                // APIC
+            (0 << 10) |                               // Reserved
+            (1 << 11) |                               // SYSCALL/SYSRET
+            (1 << 12) |                               // MTRR
+            (1 << 13) |                               // Page global extension
+            (1 << 14) |                               // Machine Check architecture
+            (1 << 15) |                               // CMOV
+            (1 << 16) |                               // Page attribute table
+            (1 << 17) |                               // Page-size extensions
+            (0 << 18) |                               // Reserved
+            (0 << 19) |                               // Reserved
+            (1 << 20) |                               // NX
+            (0 << 21) |                               // Reserved
+            (1 << 22) |                               // MMXExt
+            (1 << 23) |                               // MMX
+            (1 << 24) |                               // FXSAVE/FXRSTOR
+            (1 << 25) |                               // FXSAVE/FXRSTOR Optimizations
+            (0 << 26) |                               // 1 gigabit pages
+            (SUPPORTS_RDTSCP << 27) |                 // RDTSCP
+            (0 << 28) |                               // Reserved
+            (1 << 29) |                               // Long Mode
+            (CTX->HostFeatures.Supports3DNow << 30) | // 3DNow! Extensions
+            (CTX->HostFeatures.Supports3DNow << 31);  // 3DNow!
   return Res;
 }
 

FEXCore/Source/Interface/Core/CPUID.h

@@ -115,7 +115,7 @@ public:
 
 private:
   const FEXCore::Context::ContextImpl* CTX;
-  bool SupportsCPUIndexInTPIDRRO {};
+  [[maybe_unused]] bool SupportsCPUIndexInTPIDRRO {};
   bool Hybrid {};
   uint32_t Cores {};
   FEX_CONFIG_OPT(HideHypervisorBit, HIDEHYPERVISORBIT);

FEXCore/Source/Interface/Core/Core.cpp

@@ -14,7 +14,6 @@ $end_info$
 #include "Interface/Core/CPUBackend.h"
 #include "Interface/Core/CPUID.h"
 #include "Interface/Core/Frontend.h"
-#include "Interface/Core/ObjectCache/ObjectCacheService.h"
 #include "Interface/Core/OpcodeDispatcher.h"
 #include "Interface/Core/JIT/JITClass.h"
 #include "Interface/Core/Dispatcher/Dispatcher.h"
@@ -46,13 +45,13 @@ $end_info$
 #include "FEXCore/Utils/SignalScopeGuards.h"
 #include <FEXCore/Utils/Threads.h>
 #include <FEXCore/Utils/Profiler.h>
+#include <FEXCore/Utils/SHMStats.h>
 #include <FEXCore/fextl/fmt.h>
 #include <FEXCore/fextl/memory.h>
 #include <FEXCore/fextl/set.h>
 #include <FEXCore/fextl/sstream.h>
 #include <FEXCore/fextl/vector.h>
 #include <FEXHeaderUtils/Syscalls.h>
-#include <FEXHeaderUtils/TodoDefines.h>
 
 #include <algorithm>
 #include <array>
@@ -79,9 +78,6 @@ ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features)
   : HostFeatures {Features}
   , CPUID {this}
   , IRCaptureCache {this} {
-  if (Config.CacheObjectCodeCompilation() != FEXCore::Config::ConfigObjectCodeHandler::CONFIG_NONE) {
-    CodeObjectCacheService = fextl::make_unique<FEXCore::CodeSerialize::CodeObjectSerializeService>(this);
-  }
   if (!Config.Is64BitMode()) {
     // When operating in 32-bit mode, the virtual memory we care about is only the lower 32-bits.
     Config.VirtualMemSize = 1ULL << 32;
@@ -105,14 +101,6 @@ ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features)
   UpdateAtomicTSOEmulationConfig();
 }
 
-ContextImpl::~ContextImpl() {
-  {
-    if (CodeObjectCacheService) {
-      CodeObjectCacheService->Shutdown();
-    }
-  }
-}
-
 struct GetFrameBlockInfoResult {
   const CPU::CPUBackend::JITCodeHeader* InlineHeader;
   const CPU::CPUBackend::JITCodeTail* InlineTail;
@@ -139,6 +127,11 @@ bool ContextImpl::IsCurrentBlockSingleInst(FEXCore::Core::InternalThreadState* T
   return InlineTail && InlineTail->SingleInst;
 }
 
+uint64_t ContextImpl::GetGuestBlockEntry(FEXCore::Core::InternalThreadState* Thread) {
+  auto [_, InlineTail] = GetFrameBlockInfo(Thread->CurrentFrame);
+  return InlineTail ? InlineTail->RIP : 0;
+}
+
 uint64_t ContextImpl::RestoreRIPFromHostPC(FEXCore::Core::InternalThreadState* Thread, uint64_t HostPC) {
   const auto Frame = Thread->CurrentFrame;
   const uint64_t BlockBegin = Frame->State.InlineJITBlockHeader;
@@ -349,36 +342,9 @@ bool ContextImpl::InitCore() {
   Dispatcher = FEXCore::CPU::Dispatcher::Create(this);
 
   // Set up the SignalDelegator config since core is initialized.
-  FEXCore::SignalDelegator::SignalDelegatorConfig SignalConfig {
-    .DispatcherBegin = Dispatcher->Start,
-    .DispatcherEnd = Dispatcher->End,
+  SignalDelegation->SetConfig(Dispatcher->MakeSignalDelegatorConfig());
 
-    .AbsoluteLoopTopAddress = Dispatcher->AbsoluteLoopTopAddress,
-    .AbsoluteLoopTopAddressFillSRA = Dispatcher->AbsoluteLoopTopAddressFillSRA,
-    .SignalHandlerReturnAddress = Dispatcher->SignalHandlerReturnAddress,
-    .SignalHandlerReturnAddressRT = Dispatcher->SignalHandlerReturnAddressRT,
-
-    .PauseReturnInstruction = Dispatcher->PauseReturnInstruction,
-    .ThreadPauseHandlerAddressSpillSRA = Dispatcher->ThreadPauseHandlerAddressSpillSRA,
-    .ThreadPauseHandlerAddress = Dispatcher->ThreadPauseHandlerAddress,
-
-    // Stop handlers.
-    .ThreadStopHandlerAddressSpillSRA = Dispatcher->ThreadStopHandlerAddressSpillSRA,
-    .ThreadStopHandlerAddress = Dispatcher->ThreadStopHandlerAddress,
-
-    // SRA information.
-    .SRAGPRCount = Dispatcher->GetSRAGPRCount(),
-    .SRAFPRCount = Dispatcher->GetSRAFPRCount(),
-  };
-
-  Dispatcher->GetSRAGPRMapping(SignalConfig.SRAGPRMapping);
-  Dispatcher->GetSRAFPRMapping(SignalConfig.SRAFPRMapping);
-
-  // Give this configuration to the SignalDelegator.
-  SignalDelegation->SetConfig(SignalConfig);
-
-#ifndef _WIN32
-#elif !defined(_M_ARM_64EC)
+#if defined(_WIN32) && !defined(_M_ARM_64EC)
   // WOW64 always needs the interrupt fault check to be enabled.
   Config.NeedsPendingInterruptFaultCheck = true;
 #endif
@@ -398,21 +364,15 @@ void ContextImpl::HandleCallback(FEXCore::Core::InternalThreadState* Thread, uin
 void ContextImpl::ExecuteThread(FEXCore::Core::InternalThreadState* Thread) {
   Dispatcher->ExecuteDispatch(Thread->CurrentFrame);
 
-  if (CodeObjectCacheService) {
-    // Ensure the Code Object Serialization service has fully serialized this thread's data before clearing the cache
-    // Use the thread's object cache ref counter for this
-    CodeSerialize::CodeObjectSerializeService::WaitForEmptyJobQueue(&Thread->ObjectCacheRefCounter);
-  }
-
   // If it is the parent thread that died then just leave
-  FEX_TODO("This doesn't make sense when the parent thread doesn't outlive its children");
+  // TODO: This doesn't make sense when the parent thread doesn't outlive its children
 }
 
 void ContextImpl::InitializeCompiler(FEXCore::Core::InternalThreadState* Thread) {
   Thread->OpDispatcher = fextl::make_unique<FEXCore::IR::OpDispatchBuilder>(this);
   Thread->OpDispatcher->SetMultiblock(Config.Multiblock);
   Thread->LookupCache = fextl::make_unique<FEXCore::LookupCache>(this);
-  Thread->FrontendDecoder = fextl::make_unique<FEXCore::Frontend::Decoder>(this);
+  Thread->FrontendDecoder = fextl::make_unique<FEXCore::Frontend::Decoder>(Thread);
   Thread->PassManager = fextl::make_unique<FEXCore::IR::PassManager>();
 
   Thread->CurrentFrame->Pointers.Common.L1Pointer = Thread->LookupCache->GetL1Pointer();
@@ -426,7 +386,7 @@ void ContextImpl::InitializeCompiler(FEXCore::Core::InternalThreadState* Thread)
   Thread->PassManager->RegisterSyscallHandler(SyscallHandler);
 
   // Create CPU backend
-  Thread->PassManager->InsertRegisterAllocationPass();
+  Thread->PassManager->InsertRegisterAllocationPass(this);
   Thread->CPUBackend = FEXCore::CPU::CreateArm64JITCore(this, Thread);
 
   Thread->PassManager->Finalize();
@@ -495,26 +455,31 @@ void ContextImpl::LockBeforeFork(FEXCore::Core::InternalThreadState* Thread) {
 }
 #endif
 
-void ContextImpl::ClearCodeCache(FEXCore::Core::InternalThreadState* Thread) {
-  FEXCORE_PROFILE_INSTANT("ClearCodeCache");
-
-  if (CodeObjectCacheService) {
-    // Ensure the Code Object Serialization service has fully serialized this thread's data before clearing the cache
-    // Use the thread's object cache ref counter for this
-    CodeSerialize::CodeObjectSerializeService::WaitForEmptyJobQueue(&Thread->ObjectCacheRefCounter);
+void ContextImpl::OnCodeBufferAllocated(CPU::CodeBuffer& Buffer) {
+  if (Config.GlobalJITNaming()) {
+    Symbols.RegisterJITSpace(Buffer.Ptr, Buffer.Size);
   }
-  std::lock_guard<std::recursive_mutex> lk(Thread->LookupCache->WriteLock);
-
-  Thread->LookupCache->ClearCache();
-  Thread->CPUBackend->ClearCache();
 }
 
-static void IRDumper(FEXCore::Core::InternalThreadState* Thread, IR::IREmitter* IREmitter, uint64_t GuestRIP, IR::RegisterAllocationData* RA) {
+void ContextImpl::ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer) {
+  FEXCORE_PROFILE_INSTANT("ClearCodeCache");
+
+  if (NewCodeBuffer) {
+    // Allocate new CodeBuffer + L3 LookupCache and clear L1+L2 caches
+    Thread->CPUBackend->ClearCache();
+  } else {
+    // Clear L1+L2 cache of this thread, and clear L3 cache across any threads using it
+    Thread->LookupCache->ClearCache();
+  }
+  Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);
+}
+
+static void IRDumper(FEXCore::Core::InternalThreadState* Thread, IR::IREmitter* IREmitter, uint64_t GuestRIP) {
   FEXCore::File::File FD = FEXCore::File::File::GetStdERR();
   fextl::stringstream out;
   auto NewIR = IREmitter->ViewIR();
-  FEXCore::IR::Dump(&out, &NewIR, RA);
-  fextl::fmt::print(FD, "IR-ShouldDump-{} 0x{:x}:\n{}\n@@@@@\n", RA ? "post" : "pre", GuestRIP, out.str());
+  FEXCore::IR::Dump(&out, &NewIR);
+  fextl::fmt::print(FD, "IR-ShouldDump-{} 0x{:x}:\n{}\n@@@@@\n", NewIR.PostRA() ? "post" : "pre", GuestRIP, out.str());
 };
 
 ContextImpl::GenerateIRResult
@@ -535,7 +500,7 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
     if (Handler != CustomIRHandlers.end()) {
       TotalInstructions = 1;
       TotalInstructionsLength = 1;
-      std::get<0>(Handler->second)(GuestRIP, Thread->OpDispatcher.get());
+      Handler->second.Handler(GuestRIP, Thread->OpDispatcher.get());
       HasCustomIR = true;
     }
   }
@@ -547,19 +512,15 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
     bool HadDispatchError {false};
     bool HadInvalidInst {false};
 
-    Thread->FrontendDecoder->DecodeInstructionsAtEntry(GuestCode, GuestRIP, MaxInst,
-                                                       [Thread](uint64_t BlockEntry, uint64_t Start, uint64_t Length) {
-      if (Thread->LookupCache->AddBlockExecutableRange(BlockEntry, Start, Length)) {
-        static_cast<ContextImpl*>(Thread->CTX)->SyscallHandler->MarkGuestExecutableRange(Thread, Start, Length);
-      }
-    });
+    Thread->FrontendDecoder->DecodeInstructionsAtEntry(Thread, GuestCode, GuestRIP, MaxInst);
 
     auto BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo();
     auto CodeBlocks = &BlockInfo->Blocks;
 
-    Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks, BlockInfo->TotalInstructionCount);
+    Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks, BlockInfo->TotalInstructionCount, BlockInfo->Is64BitMode,
+                                        AreMonoHacksActive() && MonoBackpatcherBlock.load(std::memory_order_relaxed) == GuestRIP);
 
-    const auto GPRSize = GetGPROpSize();
+    const auto GPRSize = Thread->OpDispatcher->GetGPROpSize();
 
     for (size_t j = 0; j < CodeBlocks->size(); ++j) {
       const FEXCore::Frontend::Decoder::DecodedBlocks& Block = CodeBlocks->at(j);
@@ -585,7 +546,7 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
 
       if (InstsInBlock == 0) {
         // Special case for an empty instruction block.
-        Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_EntrypointOffset(GPRSize, Block.Entry - GuestRIP));
+        Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_InlineEntrypointOffset(GPRSize, Block.Entry - GuestRIP));
       }
 
       for (size_t i = 0; i < InstsInBlock; ++i) {
@@ -615,11 +576,12 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
           Thread->OpDispatcher->_GuestOpcode(InstAddress - GuestRIP);
         }
 
-        if (Config.SMCChecks == FEXCore::Config::CONFIG_SMC_FULL) {
-          auto ExistingCodePtr = reinterpret_cast<uint64_t*>(Block.Entry + BlockInstructionsLength);
-
-          auto CodeChanged = Thread->OpDispatcher->_ValidateCode(ExistingCodePtr[0], ExistingCodePtr[1],
-                                                                 (uintptr_t)ExistingCodePtr - GuestRIP, DecodedInfo->InstSize);
+        if (Config.SMCChecks == FEXCore::Config::CONFIG_SMC_FULL || Block.ForceFullSMCDetection) {
+          auto ExistingCodePtr = reinterpret_cast<uint8_t*>(Block.Entry + BlockInstructionsLength);
+          auto InstAddressReg = Thread->OpDispatcher->_EntrypointOffset(GPRSize, InstAddress - GuestRIP);
+          std::array<uint8_t, 0x10> CodeOriginal;
+          memcpy(CodeOriginal.data(), ExistingCodePtr, DecodedInfo->InstSize);
+          auto CodeChanged = Thread->OpDispatcher->_ValidateCode(CodeOriginal, InstAddressReg, DecodedInfo->InstSize);
 
           auto InvalidateCodeCond = Thread->OpDispatcher->CondJump(CodeChanged);
 
@@ -629,7 +591,7 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
 
           Thread->OpDispatcher->SetCurrentCodeBlock(CodeWasChangedBlock);
           Thread->OpDispatcher->_ThreadRemoveCodeEntry();
-          Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_EntrypointOffset(GPRSize, InstAddress - GuestRIP));
+          Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_InlineEntrypointOffset(GPRSize, InstAddress - GuestRIP));
 
           auto NextOpBlock = Thread->OpDispatcher->CreateNewCodeBlockAfter(CurrentBlock);
 
@@ -637,15 +599,21 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
           Thread->OpDispatcher->SetCurrentCodeBlock(NextOpBlock);
         }
 
-        if (TableInfo && TableInfo->OpcodeDispatcher) {
-          auto Fn = TableInfo->OpcodeDispatcher;
+        if (TableInfo && TableInfo->OpcodeDispatcher.OpDispatch) {
+          auto Fn = TableInfo->OpcodeDispatcher.OpDispatch;
           Thread->OpDispatcher->ResetHandledLock();
           Thread->OpDispatcher->ResetDecodeFailure();
-          IR::ForceTSOMode ForceTSO =
-            BlockInForceTSOValidRange ?
-              (InstForceTSOIt != ForceTSOInstructions.end() && *InstForceTSOIt == InstAddress ? IR::ForceTSOMode::ForceEnabled :
-                                                                                                IR::ForceTSOMode::ForceDisabled) :
-              IR::ForceTSOMode::NoOverride;
+          IR::ForceTSOMode ForceTSO = IR::ForceTSOMode::NoOverride;
+          if (BlockInForceTSOValidRange) {
+            if (InstForceTSOIt != ForceTSOInstructions.end() && *InstForceTSOIt == InstAddress) {
+              ForceTSO = IR::ForceTSOMode::ForceEnabled;
+            } else {
+              ForceTSO = IR::ForceTSOMode::ForceDisabled;
+            }
+          } else if (DecodedInfo->ForceTSO) {
+            ForceTSO = IR::ForceTSOMode::ForceEnabled;
+          }
+
           Thread->OpDispatcher->SetForceTSO(ForceTSO);
           std::invoke(Fn, Thread->OpDispatcher, DecodedInfo);
           if (Thread->OpDispatcher->HadDecodeFailure()) {
@@ -673,7 +641,11 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
               LogMan::Msg::EFmt("Invalid or Unknown instruction: {} 0x{:x}", TableInfo->Name ?: "UND", Block.Entry - GuestRIP);
             }
 
-            Thread->OpDispatcher->InvalidOp(DecodedInfo);
+            if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::NOEXEC_INST) {
+              Thread->OpDispatcher->NoExecOp(DecodedInfo);
+            } else {
+              Thread->OpDispatcher->InvalidOp(DecodedInfo);
+            }
           }
 
           HadInvalidInst = true;
@@ -686,12 +658,13 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
         if (HadDispatchError && TotalInstructions == 0) {
           // Couldn't handle any instruction in op dispatcher
           Thread->OpDispatcher->ResetWorkingList();
-          return {{}, nullptr, 0, 0, 0, 0};
+          return {{}, 0, 0, 0, 0};
         }
 
         if (NeedsBlockEnd) {
           // We had some instructions. Early exit
-          Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_EntrypointOffset(GPRSize, Block.Entry + BlockInstructionsLength - GuestRIP));
+          Thread->OpDispatcher->ExitFunction(
+            Thread->OpDispatcher->_InlineEntrypointOffset(GPRSize, Block.Entry + BlockInstructionsLength - GuestRIP));
           break;
         }
 
@@ -712,79 +685,74 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
   auto ShouldDump = Thread->OpDispatcher->ShouldDumpIR();
   // Debug
   if (ShouldDump) {
-    IRDumper(Thread, IREmitter, GuestRIP, nullptr);
+    IRDumper(Thread, IREmitter, GuestRIP);
   }
 
   // Run the passmanager over the IR from the dispatcher
   Thread->PassManager->Run(IREmitter);
 
-  auto RAData = Thread->PassManager->HasPass("RA") ? Thread->PassManager->GetPass<IR::RegisterAllocationPass>("RA")->GetAllocationData() : nullptr;
-
   // Debug
   if (ShouldDump) {
-    IRDumper(Thread, IREmitter, GuestRIP, RAData);
+    IRDumper(Thread, IREmitter, GuestRIP);
   }
 
   return {
     .IRView = IREmitter->ViewIR(),
-    .RAData = RAData,
     .TotalInstructions = TotalInstructions,
     .TotalInstructionsLength = TotalInstructionsLength,
     .StartAddr = Thread->FrontendDecoder->DecodedMinAddress,
     .Length = Thread->FrontendDecoder->DecodedMaxAddress - Thread->FrontendDecoder->DecodedMinAddress,
+    .NeedsAddGuestCodeRanges = !HasCustomIR,
   };
 }
 
 ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, uint64_t MaxInst) {
-  // JIT Code object cache lookup
-  if (CodeObjectCacheService) {
-    auto CodeCacheEntry = CodeObjectCacheService->FetchCodeObjectFromCache(GuestRIP);
-    if (CodeCacheEntry) {
-      auto CompiledCode = Thread->CPUBackend->RelocateJITObjectCode(GuestRIP, CodeCacheEntry);
-      if (CompiledCode) {
-        return {
-          .CompiledCode = CompiledCode,
-          .DebugData = nullptr, // nullptr here ensures that code serialization doesn't occur on from cache read
-          .StartAddr = 0,       // Unused
-          .Length = 0,          // Unused
-        };
-      }
-    }
-  }
-
   if (SourcecodeResolver && Config.GDBSymbols()) {
     auto AOTIRCacheEntry = SyscallHandler->LookupAOTIRCacheEntry(Thread, GuestRIP);
-    if (AOTIRCacheEntry.Entry && !AOTIRCacheEntry.Entry->ContainsCode) {
+    if (AOTIRCacheEntry.Entry) {
       AOTIRCacheEntry.Entry->SourcecodeMap = SourcecodeResolver->GenerateMap(AOTIRCacheEntry.Entry->Filename, AOTIRCacheEntry.Entry->FileId);
     }
   }
 
   // Generate IR + Meta Info
-  auto [IRView, RAData, TotalInstructions, TotalInstructionsLength, StartAddr, Length] =
+  auto [IRView, TotalInstructions, TotalInstructionsLength, StartAddr, Length, NeedsAddGuestCodeRanges] =
     GenerateIR(Thread, GuestRIP, Config.GDBSymbols(), MaxInst);
   if (!IRView) {
-    return {nullptr, nullptr, 0, 0};
+    return {{}, nullptr, 0, 0, false};
   }
+
+  // Attempt to get the CPU backend to compile this code
+  // Re-check if another thread raced us in compiling this block.
+  // We could lock CodeBufferWriteMutex earlier to prevent this from happening,
+  // but this would increase lock contention. Redundant frontend runs aren't
+  // as expensive and are easily reverted.
+  if (MaxInst != 1) {
+    if (auto Block = Thread->LookupCache->FindBlock(GuestRIP)) {
+      Thread->OpDispatcher->DelayedDisownBuffer();
+      return {.CompiledCode = {.BlockBegin = reinterpret_cast<uint8_t*>(Block), .EntryPoints = {{GuestRIP, reinterpret_cast<uint8_t*>(Block)}}},
+              .DebugData = nullptr,
+              .StartAddr = 0,
+              .Length = 0,
+              .NeedsAddGuestCodeRanges = false};
+    }
+  }
+
   auto DebugData = fextl::make_unique<FEXCore::Core::DebugData>();
 
   // If the trap flag is set we generate single instruction blocks that each check to generate a single step exception.
   bool TFSet = Thread->CurrentFrame->State.flags[X86State::RFLAG_TF_RAW_LOC];
 
-  // Attempt to get the CPU backend to compile this code
-
-  auto CompiledCode = Thread->CPUBackend->CompileCode(GuestRIP, Length, TotalInstructions == 1, &*IRView, DebugData.get(), RAData, TFSet);
+  auto CompiledCode = Thread->CPUBackend->CompileCode(GuestRIP, Length, TotalInstructions == 1, &*IRView, DebugData.get(), TFSet);
 
   // Release the IR
   Thread->OpDispatcher->DelayedDisownBuffer();
 
   return {
-    // FEX currently throws away the CPUBackend::CompiledCode object other than the entrypoint
-    // In the future with code caching getting wired up, we will pass the rest of the data forward.
-    // TODO: Pass the data forward when code caching is wired up to this.
-    .CompiledCode = CompiledCode.BlockEntry,
+    .CompiledCode = std::move(CompiledCode),
     .DebugData = std::move(DebugData),
     .StartAddr = StartAddr,
     .Length = Length,
+    .NeedsAddGuestCodeRanges = NeedsAddGuestCodeRanges,
   };
 }
 
@@ -804,14 +772,18 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
     return HostCode;
   }
 
-  auto [CodePtr, DebugData, StartAddr, Length] = CompileCode(Thread, GuestRIP, MaxInst);
+  auto [CompiledCode, DebugData, StartAddr, Length, NeedsAddGuestCodeRanges] = CompileCode(Thread, GuestRIP, MaxInst);
+  auto CodePtr = CompiledCode.EntryPoints[GuestRIP];
   if (CodePtr == nullptr) {
     return 0;
+  } else if (!DebugData) {
+    // DebugData wasn't populated, indicating another thread raced us for compiling this block
+    return reinterpret_cast<uintptr_t>(CodePtr);
   }
 
   // The core managed to compile the code.
   if (Config.BlockJITNaming()) {
-    auto FragmentBasePtr = reinterpret_cast<uint8_t*>(CodePtr);
+    auto FragmentBasePtr = CompiledCode.BlockBegin;
 
     if (DebugData) {
       auto GuestRIPLookup = SyscallHandler->LookupAOTIRCacheEntry(Thread, GuestRIP);
@@ -820,7 +792,7 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
         for (auto& Subblock : DebugData->Subblocks) {
           auto BlockBasePtr = FragmentBasePtr + Subblock.HostCodeOffset;
           if (GuestRIPLookup.Entry) {
-            Symbols.Register(Thread->SymbolBuffer.get(), BlockBasePtr, DebugData->HostCodeSize, GuestRIPLookup.Entry->Filename,
+            Symbols.Register(Thread->SymbolBuffer.get(), BlockBasePtr, CompiledCode.Size, GuestRIPLookup.Entry->Filename,
                              GuestRIP - GuestRIPLookup.VAFileStart);
           } else {
             Symbols.Register(Thread->SymbolBuffer.get(), BlockBasePtr, GuestRIP, Subblock.HostCodeSize);
@@ -828,41 +800,38 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
         }
       } else {
         if (GuestRIPLookup.Entry) {
-          Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, DebugData->HostCodeSize, GuestRIPLookup.Entry->Filename,
+          Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, CompiledCode.Size, GuestRIPLookup.Entry->Filename,
                            GuestRIP - GuestRIPLookup.VAFileStart);
         } else {
-          Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, GuestRIP, DebugData->HostCodeSize);
+          Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, GuestRIP, CompiledCode.Size);
         }
       }
     }
   }
 
-  // Tell the object cache service to serialize the code if enabled
-  if (CodeObjectCacheService && Config.CacheObjectCodeCompilation == FEXCore::Config::ConfigObjectCodeHandler::CONFIG_READWRITE && DebugData) {
-    CodeObjectCacheService->AsyncAddSerializationJob(
-      fextl::make_unique<CodeSerialize::AsyncJobHandler::SerializationJobData>(CodeSerialize::AsyncJobHandler::SerializationJobData {
-        .GuestRIP = GuestRIP,
-        .GuestCodeLength = Length,
-        .GuestCodeHash = 0,
-        .HostCodeBegin = CodePtr,
-        .HostCodeLength = DebugData->HostCodeSize,
-        .HostCodeHash = 0,
-        .ThreadJobRefCount = &Thread->ObjectCacheRefCounter,
-        .Relocations = std::move(*DebugData->Relocations),
-      }));
-  }
-
   // Clear any relocations that might have been generated
   Thread->CPUBackend->ClearRelocations();
 
-  if (IRCaptureCache.PostCompileCode(Thread, CodePtr, GuestRIP, StartAddr, Length, {}, DebugData.get(), false)) {
+  if (IRCaptureCache.PostCompileCode(Thread, CompiledCode.BlockBegin, GuestRIP, StartAddr, Length, DebugData.get())) {
     // Early exit
     return (uintptr_t)CodePtr;
   }
 
+  if (NeedsAddGuestCodeRanges) {
+    // Track in the guest to host map all entrypoints for all pages the compiled block touches, if any page didn't previously
+    // contain code, inform the frontend so it can setup SMC detection.
+    auto BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo();
+    for (auto CodePage : BlockInfo->CodePages) {
+      if (Thread->LookupCache->AddBlockExecutableRange(BlockInfo->EntryPoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE)) {
+        SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE);
+      }
+    }
+  }
+
   // Insert to lookup cache
-  // Pages containing this block are added via AddBlockExecutableRange before each page gets accessed in the frontend
-  Thread->LookupCache->AddBlockMapping(GuestRIP, CodePtr);
+  for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) {
+    Thread->LookupCache->AddBlockMapping(GuestAddr, HostAddr);
+  }
 
   return (uintptr_t)CodePtr;
 }
@@ -876,7 +845,8 @@ uintptr_t ContextImpl::CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, ui
   // Invalidate might take a unique lock on this, to guarantee that during invalidation no code gets compiled
   auto lk = GuardSignalDeferringSection<std::shared_lock>(CodeInvalidationMutex, Thread);
 
-  auto [CodePtr, DebugData, StartAddr, Length] = CompileCode(Thread, GuestRIP, 1);
+  auto [CompiledCode, DebugData, StartAddr, Length, _] = CompileCode(Thread, GuestRIP, 1);
+  auto CodePtr = CompiledCode.EntryPoints[GuestRIP];
   if (CodePtr == nullptr) {
     return 0;
   }
@@ -887,22 +857,40 @@ uintptr_t ContextImpl::CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, ui
   return (uintptr_t)CodePtr;
 }
 
-static void InvalidateGuestThreadCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) {
-  std::lock_guard<std::recursive_mutex> lk(Thread->LookupCache->WriteLock);
+static void InvalidateGuestThreadCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator,
+                                           uint64_t Start, uint64_t Length) {
+  // Ensures now-modified mappings aren't cached as being in their previous non-executable state.
+  // Accessing FrontendDecoder is safe as the thread's code invalidation mutex must be locked here.
+  Thread->FrontendDecoder->ResetExecutableRangeCache();
 
-  auto lower = Thread->LookupCache->CodePages.lower_bound(Start >> 12);
-  auto upper = Thread->LookupCache->CodePages.upper_bound((Start + Length - 1) >> 12);
+  auto lk = Thread->LookupCache->AcquireLock();
+  auto& CodePages = Thread->LookupCache->Shared->CodePages;
+
+  auto lower = CodePages.lower_bound(Start >> 12);
+  auto upper = CodePages.upper_bound((Start + Length - 1) >> 12);
 
   for (auto it = lower; it != upper; it++) {
-    for (auto Address : it->second) {
-      ContextImpl::ThreadRemoveCodeEntry(Thread, Address);
+    Accumulator.emplace_back(std::move(it->second));
+  }
+
+  bool InvalidatedAnyEntries = false;
+  for (const auto& PageEntries : Accumulator) {
+    for (const auto& Entry : PageEntries) {
+      if (ContextImpl::ThreadRemoveCodeEntry(Thread, Entry)) {
+        InvalidatedAnyEntries = true;
+      }
     }
-    it->second.clear();
+  }
+
+  if (InvalidatedAnyEntries) {
+    // This may cause access violations in the thread on Windows as zeroing is not atomic, this is handled by the frontend
+    Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);
   }
 }
 
-void ContextImpl::InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) {
-  InvalidateGuestThreadCodeRange(Thread, Start, Length);
+void ContextImpl::InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator,
+                                           uint64_t Start, uint64_t Length) {
+  InvalidateGuestThreadCodeRange(Thread, Accumulator, Start, Length);
 }
 
 void ContextImpl::MarkMemoryShared(FEXCore::Core::InternalThreadState* Thread) {
@@ -915,18 +903,22 @@ void ContextImpl::MarkMemoryShared(FEXCore::Core::InternalThreadState* Thread) {
     UpdateAtomicTSOEmulationConfig();
 
     if (Config.TSOAutoMigration) {
-      // Only the lookup cache is cleared here, so that old code can keep running until next compilation
-      std::lock_guard<std::recursive_mutex> lkLookupCache(Thread->LookupCache->WriteLock);
+      // Only the lookup cache is cleared here, so that old code can keep running until next compilation.
+      // This will leak previously compiled blocks until the CodeBuffer is cleared for some other reason.
       Thread->LookupCache->ClearCache();
     }
   }
 }
 
-void ContextImpl::ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP) {
+bool ContextImpl::ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP) {
   LogMan::Throw::AFmt(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to "
                                                                                                          "be unique_locked here");
 
-  Thread->LookupCache->Erase(Thread->CurrentFrame, GuestRIP);
+  return Thread->LookupCache->Erase(Thread->CurrentFrame, GuestRIP);
+}
+
+void ContextImpl::ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP) {
+  static_cast<ContextImpl*>(Frame->Thread->CTX)->SyscallHandler->InvalidateGuestCodeRange(Frame->Thread, GuestRIP, 1);
 }
 
 std::optional<CustomIRResult>
@@ -935,7 +927,7 @@ ContextImpl::AddCustomIREntrypoint(uintptr_t Entrypoint, CustomIREntrypointHandl
 
   std::unique_lock lk(CustomIRMutex);
 
-  auto InsertedIterator = CustomIRHandlers.emplace(Entrypoint, std::tuple(Handler, Creator, Data));
+  auto InsertedIterator = CustomIRHandlers.emplace(Entrypoint, CustomIRHandlerEntry {Handler, Creator, Data});
   HasCustomIRHandlers = true;
 
   if (!InsertedIterator.second) {
@@ -959,20 +951,21 @@ void ContextImpl::AddThunkTrampolineIRHandler(uintptr_t Entrypoint, uintptr_t Gu
   auto Result = AddCustomIREntrypoint(
     Entrypoint,
     [this, GuestThunkEntrypoint](uintptr_t Entrypoint, FEXCore::IR::IREmitter* emit) {
-    auto IRHeader = emit->_IRHeader(emit->Invalid(), Entrypoint, 0, 0);
-    auto Block = emit->CreateCodeNode();
-    IRHeader.first->Blocks = emit->WrapNode(Block);
-    emit->SetCurrentCodeBlock(Block);
+      auto IRHeader = emit->_IRHeader(emit->Invalid(), Entrypoint, 0, 0, 0, 0);
+      auto Block = emit->CreateCodeNode(true, 0);
+      IRHeader.first->Blocks = emit->WrapNode(Block);
+      emit->SetCurrentCodeBlock(Block);
 
-    const auto GPRSize = GetGPROpSize();
+      const auto GPRSize = this->Config.Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;
 
-    if (GPRSize == IR::OpSize::i64Bit) {
-      emit->_StoreRegister(emit->_Constant(Entrypoint), X86State::REG_R11, IR::GPRClass, GPRSize);
-    } else {
-      emit->_StoreContext(GPRSize, IR::FPRClass, emit->_VCastFromGPR(IR::OpSize::i64Bit, IR::OpSize::i64Bit, emit->_Constant(Entrypoint)),
-                          offsetof(Core::CPUState, mm[0][0]));
-    }
-    emit->_ExitFunction(emit->_Constant(GuestThunkEntrypoint));
+      if (GPRSize == IR::OpSize::i64Bit) {
+        IR::Ref R = emit->_StoreRegister(emit->Constant(Entrypoint), GPRSize);
+        R->Reg = IR::PhysicalRegister(IR::GPRFixedClass, X86State::REG_R11).Raw;
+      } else {
+        emit->_StoreContext(GPRSize, IR::FPRClass, emit->_VCastFromGPR(IR::OpSize::i64Bit, IR::OpSize::i64Bit, emit->Constant(Entrypoint)),
+                            offsetof(Core::CPUState, mm[0][0]));
+      }
+      emit->_ExitFunction(IR::OpSize::i64Bit, emit->Constant(GuestThunkEntrypoint), IR::BranchHint::None, emit->Invalid(), emit->Invalid());
     },
     ThunkHandler, (void*)GuestThunkEntrypoint);
 
@@ -1001,15 +994,36 @@ void ContextImpl::RemoveForceTSOInformation(uint64_t Address, uint64_t Size) {
   ForceTSOInstructions.erase(ForceTSOInstructions.lower_bound(Address), ForceTSOInstructions.upper_bound(Address + Size));
 }
 
-void ContextImpl::RemoveCustomIREntrypoint(uintptr_t Entrypoint) {
+void ContextImpl::MarkMonoBackpatcherBlock(uint64_t BlockEntry) {
+  MonoBackpatcherBlock.store(BlockEntry, std::memory_order_relaxed);
+}
+
+void ContextImpl::RemoveCustomIREntrypoint(FEXCore::Core::InternalThreadState* Thread, uintptr_t Entrypoint) {
   LOGMAN_THROW_A_FMT(Config.Is64BitMode || !(Entrypoint >> 32), "64-bit Entrypoint in 32-bit mode {:x}", Entrypoint);
 
   std::scoped_lock lk(CustomIRMutex);
 
-  InvalidateGuestCodeRange(nullptr, Entrypoint, 1);
   CustomIRHandlers.erase(Entrypoint);
-
   HasCustomIRHandlers = !CustomIRHandlers.empty();
+  SyscallHandler->InvalidateGuestCodeRange(Thread, Entrypoint, 1);
+}
+
+void ContextImpl::MonoBackpatcherWrite(FEXCore::Core::CpuStateFrame* Frame, uint8_t Size, uint64_t Address, uint64_t Value) {
+  auto Thread = Frame->Thread;
+  auto CTX = static_cast<ContextImpl*>(Thread->CTX);
+  {
+    auto lk = GuardSignalDeferringSection(CTX->CodeInvalidationMutex, Thread);
+
+    if (Size == 8) {
+      *reinterpret_cast<uint64_t *>(Address) = Value;
+    } else if (Size == 4) {
+      *reinterpret_cast<uint32_t *>(Address) = Value;
+    } else {
+      ERROR_AND_DIE_FMT("Unexpected write size for backpatcher: {}", Size);
+    }
+  }
+
+  CTX->SyscallHandler->InvalidateGuestCodeRange(Thread, Address, Size);
 }
 
 IR::AOTIRCacheEntry* ContextImpl::LoadAOTIRCacheEntry(const fextl::string& filename) {
@@ -1017,9 +1031,7 @@ IR::AOTIRCacheEntry* ContextImpl::LoadAOTIRCacheEntry(const fextl::string& filen
   return rv;
 }
 
-void ContextImpl::UnloadAOTIRCacheEntry(IR::AOTIRCacheEntry* Entry) {
-  IRCaptureCache.UnloadAOTIRCacheEntry(Entry);
-}
+void ContextImpl::UnloadAOTIRCacheEntry(IR::AOTIRCacheEntry* Entry) {}
 
 void ContextImpl::ConfigureAOTGen(FEXCore::Core::InternalThreadState* Thread, fextl::set<uint64_t>* ExternalBranches, uint64_t SectionMaxAddress) {
   Thread->FrontendDecoder->SetExternalBranches(ExternalBranches);

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp

@@ -2,6 +2,7 @@
 
 #include "Common/SoftFloat.h"
 #include "Interface/Context/Context.h"
+#include "Interface/Core/CPUBackend.h"
 #include "Interface/Core/Dispatcher/Dispatcher.h"
 #include "Interface/Core/LookupCache.h"
 #include "Interface/Core/X86HelperGen.h"
@@ -19,11 +20,16 @@
 
 #include <CodeEmitter/Emitter.h>
 
+#ifdef VIXL_SIMULATOR
+#include <aarch64/simulator-aarch64.h>
+#endif
+
+#include <array>
 #include <atomic>
+#include <bit>
 #include <condition_variable>
 #include <csignal>
 #include <cstring>
-#include <signal.h>
 
 namespace FEXCore::CPU {
 
@@ -81,11 +87,14 @@ void Dispatcher::EmitDispatcher() {
   add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ARMEmitter::Reg::rsp, 0);
   str(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, ReturningStackLocation));
 
+  ARMEmitter::ForwardLabel CompileSingleStep;
   AbsoluteLoopTopAddressFillSRA = GetCursorAddress<uint64_t>();
 
   FillStaticRegs();
+  ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));
+  cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);
+
   ARMEmitter::BiDirectionalLabel LoopTop {};
-  ARMEmitter::ForwardLabel CompileSingleStep;
 
 #ifdef _M_ARM_64EC
   b(&LoopTop);
@@ -111,15 +120,26 @@ void Dispatcher::EmitDispatcher() {
   add(ARMEmitter::Size::i64Bit, StaticRegisters[X86State::REG_RSP], ARMEmitter::Reg::rsp, 0);
   add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, TMP1, 0);
 
+  ldr(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));
+
   FillSpecialRegs(TMP1, TMP2, false, true);
 
+  // As ARM64EC uses this as an entrypoint for both guest calls and host returns, opportunistically try to return
+  // using the call-ret stack to avoid unbalancing it.
+  ldp<ARMEmitter::IndexType::OFFSET>(TMP1, TMP2, REG_CALLRET_SP);
+  // EC_CALL_CHECKER_PC_REG is REG_PF which isn't touched by any of the above
+  sub(ARMEmitter::Size::i64Bit, TMP1, EC_CALL_CHECKER_PC_REG, TMP1);
+  cbnz(ARMEmitter::Size::i64Bit, TMP1, &LoopTop);
+
+  // If the entry at the TOS is for the target address, pop it and return to the JIT code
+  add(ARMEmitter::Size::i64Bit, REG_CALLRET_SP, REG_CALLRET_SP, 0x10);
+  ret(TMP2);
+
   // Enter JIT
 #endif
 
   // We want to ensure that we are 16 byte aligned at the top of this loop
   Align16B();
-  ARMEmitter::BiDirectionalLabel FullLookup {};
-  ARMEmitter::BiDirectionalLabel CallBlock {};
 
   Bind(&LoopTop);
   AbsoluteLoopTopAddress = GetCursorAddress<uint64_t>();
@@ -127,23 +147,33 @@ void Dispatcher::EmitDispatcher() {
   // Load in our RIP
   ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));
 
+#ifdef _M_ARM_64EC
+  // Clobbers TMP1/2
+  // Check the EC code bitmap incase we need to exit the JIT to call into native code.
+  ARMEmitter::ForwardLabel l_NotECCode;
+  ldr(TMP1, ARMEmitter::XReg::x18, TEB_PEB_OFFSET);
+  ldr(TMP1, TMP1, PEB_EC_CODE_BITMAP_OFFSET);
+
+  lsr(ARMEmitter::Size::i64Bit, TMP2, RipReg, 15);
+  and_(ARMEmitter::Size::i64Bit, TMP2, TMP2, 0x1fffffffffff8);
+  ldr(TMP1, TMP1, TMP2, ARMEmitter::ExtendedType::LSL_64, 0);
+  lsr(ARMEmitter::Size::i64Bit, TMP2, RipReg, 12);
+  lsrv(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP2);
+  tbz(TMP1, 0, &l_NotECCode);
+
+  str(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));
+
+  add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0);
+  mov(EC_CALL_CHECKER_PC_REG, RipReg);
+  ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionEC));
+  br(TMP2);
+
+  Bind(&l_NotECCode);
+#endif
+
   ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
   cbnz(ARMEmitter::Size::i32Bit, TMP1, &CompileSingleStep);
 
-  // L1 Cache
-  ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer));
-
-  and_(ARMEmitter::Size::i64Bit, TMP4, RipReg.R(), LookupCache::L1_ENTRIES_MASK);
-  add(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP4, ARMEmitter::ShiftType::LSL, 4);
-  ldp<ARMEmitter::IndexType::OFFSET>(TMP4, TMP1, TMP1, 0);
-  sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, RipReg);
-  cbnz(ARMEmitter::Size::i64Bit, TMP1, &FullLookup);
-
-  br(TMP4);
-
-  // L1C check failed, do a full lookup
-  Bind(&FullLookup);
-
   // This is the block cache lookup routine
   // It matches what is going on it LookupCache.h::FindBlock
   ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.L2Pointer));
@@ -271,38 +301,10 @@ void Dispatcher::EmitDispatcher() {
     br(TMP1);
   }
 
-#ifdef _M_ARM_64EC
-  // Clobbers TMP1/2
-  auto EmitECExitCheck = [&]() {
-    // Check the EC code bitmap incase we need to exit the JIT to call into native code.
-    ARMEmitter::ForwardLabel l_NotECCode;
-    ldr(TMP1, ARMEmitter::XReg::x18, TEB_PEB_OFFSET);
-    ldr(TMP1, TMP1, PEB_EC_CODE_BITMAP_OFFSET);
-
-    lsr(ARMEmitter::Size::i64Bit, TMP2, RipReg, 15);
-    and_(ARMEmitter::Size::i64Bit, TMP2, TMP2, 0x1fffffffffff8);
-    ldr(TMP1, TMP1, TMP2, ARMEmitter::ExtendedType::LSL_64, 0);
-    lsr(ARMEmitter::Size::i64Bit, TMP2, RipReg, 12);
-    lsrv(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP2);
-    tbz(TMP1, 0, &l_NotECCode);
-
-    add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0);
-    mov(EC_CALL_CHECKER_PC_REG, RipReg);
-    ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionEC));
-    br(TMP2);
-
-    Bind(&l_NotECCode);
-  };
-#endif
-
   // Need to create the block
   {
     Bind(&NoBlock);
 
-#ifdef _M_ARM_64EC
-    EmitECExitCheck();
-#endif
-
     EmitSignalGuardedRegion([&]() {
       SpillStaticRegs(TMP1);
 
@@ -337,10 +339,6 @@ void Dispatcher::EmitDispatcher() {
   {
     Bind(&CompileSingleStep);
 
-#ifdef _M_ARM_64EC
-    EmitECExitCheck();
-#endif
-
     EmitSignalGuardedRegion([&]() {
       SpillStaticRegs(TMP1);
 
@@ -498,6 +496,7 @@ void Dispatcher::EmitDispatcher() {
 
     // load static regs
     FillStaticRegs();
+    stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
 
     // Now go back to the regular dispatcher loop
     b(&LoopTop);
@@ -517,14 +516,15 @@ void Dispatcher::EmitDispatcher() {
 
     ldr(ARMEmitter::XReg::x3, R, Offset);
     if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
-      GenerateIndirectRuntimeCall<uint64_t, uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
+      GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
     } else {
       blr(ARMEmitter::Reg::r3);
     }
-    // Result is now in x0
 
+    // Result is now in x0, x1
     if (!TMP_ABIARGS) {
       mov(TMP1, ARMEmitter::XReg::x0);
+      mov(TMP2, ARMEmitter::XReg::x1);
     }
 
     FillStaticRegs();
@@ -539,8 +539,6 @@ void Dispatcher::EmitDispatcher() {
 
   LUDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.AArch64.LUDIV));
   LDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.AArch64.LDIV));
-  LUREMHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.AArch64.LUREM));
-  LREMHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.AArch64.LREM));
 
   // Interpreter fallbacks
   {
@@ -551,14 +549,16 @@ void Dispatcher::EmitDispatcher() {
       FABI_F80_I16_I32_PTR,
       FABI_F32_I16_F80_PTR,
       FABI_F64_I16_F80_PTR,
-      FABI_F64_I16_F64_PTR,
-      FABI_F64_I16_F64_F64_PTR,
+      FABI_F64_F64_PTR,
+      FABI_F64_F64_F64_PTR,
       FABI_I16_I16_F80_PTR,
       FABI_I32_I16_F80_PTR,
       FABI_I64_I16_F80_PTR,
       FABI_I64_I16_F80_F80_PTR,
       FABI_F80_I16_F80_PTR,
       FABI_F80_I16_F80_F80_PTR,
+      FABI_F80x2_I16_F80_PTR,
+      FABI_F64x2_F64_PTR,
       FABI_I32_I64_I64_V128_V128_I16,
       FABI_I32_V128_V128_I16,
     }};
@@ -606,6 +606,7 @@ void Dispatcher::EmitDispatcher() {
 #ifdef VIXL_SIMULATOR
 void Dispatcher::ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame) {
   Simulator.WriteXRegister(0, reinterpret_cast<int64_t>(Frame));
+  Simulator.WriteXRegister(1, 0);
   Simulator.RunFrom(reinterpret_cast< const vixl::aarch64::Instruction*>(DispatchPtr));
 }
 
@@ -627,6 +628,22 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
   constexpr static auto VABI1 = ARMEmitter::VReg::v0;
   constexpr static auto VABI2 = ARMEmitter::VReg::v1;
 
+  auto FillF80x2Result = [&]() {
+    if (!TMP_ABIARGS) {
+      mov(VTMP1.Q(), VABI1.Q());
+      mov(VTMP2.Q(), VABI2.Q());
+    }
+    FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true);
+  };
+
+  auto FillF64x2Result = [&]() {
+    if (!TMP_ABIARGS) {
+      fmov(VTMP1.D(), VABI1.D());
+      fmov(VTMP2.D(), VABI2.D());
+    }
+    FillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, true);
+  };
+
   auto FillF80Result = [&]() {
     if (VTMP1 != VABI1) {
       mov(VTMP1.Q(), VABI1.Q());
@@ -740,7 +757,7 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
 
     ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));
     if (!TMP_ABIARGS) {
-      fmov(VABI1.D(), VTMP1.D());
+      mov(VABI1.Q(), VTMP1.Q());
     }
 
     mov(ARMEmitter::XReg::x1, STATE);
@@ -773,7 +790,7 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
 
     FillF64Result();
   } break;
-  case FABI_F64_I16_F64_PTR: {
+  case FABI_F64_F64_PTR: {
     // Linux Reg/Win32 Reg:
     // tmp4 (x4/x13): FallbackHandler
     // x30: return
@@ -783,18 +800,17 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
     if (!TMP_ABIARGS) {
       fmov(VABI1.D(), VTMP1.D());
     }
-    ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));
-    mov(ARMEmitter::XReg::x1, STATE);
+    mov(ARMEmitter::XReg::x0, STATE);
 
     if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
-      GenerateIndirectRuntimeCall<double, uint16_t, double, uint64_t>(FallbackPointerReg);
+      GenerateIndirectRuntimeCall<double, double, uint64_t>(FallbackPointerReg);
     } else {
       blr(FallbackPointerReg);
     }
 
     FillF64Result();
   } break;
-  case FABI_F64_I16_F64_F64_PTR: {
+  case FABI_F64_F64_F64_PTR: {
     // Linux Reg/Win32 Reg:
     // tmp4 (x4/x13): FallbackHandler
     // x30: return
@@ -807,10 +823,9 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
       fmov(VABI2.D(), VTMP2.D());
     }
 
-    ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));
-    mov(ARMEmitter::XReg::x1, STATE);
+    mov(ARMEmitter::XReg::x0, STATE);
     if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
-      GenerateIndirectRuntimeCall<double, uint16_t, double, double, uint64_t>(FallbackPointerReg);
+      GenerateIndirectRuntimeCall<double, double, double, uint64_t>(FallbackPointerReg);
     } else {
       blr(FallbackPointerReg);
     }
@@ -947,6 +962,51 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
 
     FillF80Result();
   } break;
+  case FABI_F80x2_I16_F80_PTR: {
+    // Linux Reg/Win32 Reg:
+    // tmp4 (x4/x13): FallbackHandler
+    // x30: return
+    // vtmp1 (v0/v16): vector source 1
+    // vtmp2 (v1/v16): vector source 2
+
+    SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);
+
+    ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));
+    mov(ARMEmitter::XReg::x1, STATE);
+    if (!TMP_ABIARGS) {
+      mov(VABI1.Q(), VTMP1.Q());
+    }
+
+    if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
+      // GenerateIndirectRuntimeCall<FEXCore::VectorRegPairType, uint16_t, FEXCore::VectorRegType, uint64_t>(FallbackPointerReg);
+    } else {
+      blr(FallbackPointerReg);
+    }
+
+    FillF80x2Result();
+  } break;
+  case FABI_F64x2_F64_PTR: {
+    // Linux Reg/Win32 Reg:
+    // tmp4 (x4/x13): FallbackHandler
+    // x30: return
+    // vtmp1 (v0/v16): vector source 1
+    // vtmp2 (v1/v16): vector source 2
+
+    SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);
+
+    mov(ARMEmitter::XReg::x0, STATE);
+    if (!TMP_ABIARGS) {
+      fmov(VABI1.D(), VTMP1.D());
+    }
+
+    if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
+      // GenerateIndirectRuntimeCall<FEXCore::VectorScalarF64Pair, FEXCore::VectorRegType, uint64_t>(FallbackPointerReg);
+    } else {
+      blr(FallbackPointerReg);
+    }
+
+    FillF64x2Result();
+  } break;
   case FABI_I32_I64_I64_V128_V128_I16: {
     // Linux Reg/Win32 Reg:
     // stack: FallbackHandler
@@ -1036,14 +1096,59 @@ void Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState* Thread)
     auto& AArch64 = Thread->CurrentFrame->Pointers.AArch64;
     AArch64.LUDIVHandler = LUDIVHandlerAddress;
     AArch64.LDIVHandler = LDIVHandlerAddress;
-    AArch64.LUREMHandler = LUREMHandlerAddress;
-    AArch64.LREMHandler = LREMHandlerAddress;
 
     // Fill in the fallback handlers
     InterpreterOps::FillFallbackIndexPointers(Common.FallbackHandlerPointers, &ABIPointers[0]);
   }
 }
 
+SignalDelegatorConfig Dispatcher::MakeSignalDelegatorConfig() const {
+  // PF/AF are the final two SRA registers. We only want GPRs
+  const auto GPRCount = uint16_t(StaticRegisters.size() - 2);
+  const auto FPRCount = uint16_t(StaticFPRegisters.size());
+
+  const auto GetSRAGPRMapping = [GPRCount, this] {
+    SignalDelegatorConfig::SRAIndexMapping Mapping {};
+    for (size_t i = 0; i < GPRCount; ++i) {
+      Mapping[i] = StaticRegisters[i].Idx();
+    }
+    return Mapping;
+  };
+
+  const auto GetSRAFPRMapping = [FPRCount, this] {
+    SignalDelegatorConfig::SRAIndexMapping Mapping {};
+    for (size_t i = 0; i < FPRCount; ++i) {
+      Mapping[i] = StaticFPRegisters[i].Idx();
+    }
+    return Mapping;
+  };
+
+  return FEXCore::SignalDelegatorConfig {
+    .DispatcherBegin = Start,
+    .DispatcherEnd = End,
+
+    .AbsoluteLoopTopAddress = AbsoluteLoopTopAddress,
+    .AbsoluteLoopTopAddressFillSRA = AbsoluteLoopTopAddressFillSRA,
+    .SignalHandlerReturnAddress = SignalHandlerReturnAddress,
+    .SignalHandlerReturnAddressRT = SignalHandlerReturnAddressRT,
+
+    .PauseReturnInstruction = PauseReturnInstruction,
+    .ThreadPauseHandlerAddressSpillSRA = ThreadPauseHandlerAddressSpillSRA,
+    .ThreadPauseHandlerAddress = ThreadPauseHandlerAddress,
+
+    // Stop handlers.
+    .ThreadStopHandlerAddressSpillSRA = ThreadStopHandlerAddressSpillSRA,
+    .ThreadStopHandlerAddress = ThreadStopHandlerAddress,
+
+    // SRA information.
+    .SRAGPRCount = GPRCount,
+    .SRAFPRCount = FPRCount,
+
+    .SRAGPRMapping = GetSRAGPRMapping(),
+    .SRAFPRMapping = GetSRAFPRMapping(),
+  };
+}
+
 fextl::unique_ptr<Dispatcher> Dispatcher::Create(FEXCore::Context::ContextImpl* CTX) {
   return fextl::make_unique<Dispatcher>(CTX);
 }

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.h

@@ -2,25 +2,18 @@
 #pragma once
 
 #include "Interface/Core/ArchHelpers/Arm64Emitter.h"
-#include "Interface/Core/CPUBackend.h"
 #include "Interface/Core/Interpreter/InterpreterOps.h"
 
-#include <FEXCore/Core/CoreState.h>
 #include <FEXCore/fextl/memory.h>
 
-#ifdef VIXL_SIMULATOR
-#include <aarch64/simulator-aarch64.h>
-#endif
-
+#include <array>
+#include <cstddef>
 #include <cstdint>
-#include <signal.h>
-#include <stddef.h>
-#include <stack>
-#include <tuple>
 
 namespace FEXCore {
 struct GuestSigAction;
-}
+struct SignalDelegatorConfig;
+} // namespace FEXCore
 
 namespace FEXCore::Core {
 struct CpuStateFrame;
@@ -42,6 +35,32 @@ public:
   Dispatcher(FEXCore::Context::ContextImpl* ctx);
   ~Dispatcher();
 
+  void InitThreadPointers(FEXCore::Core::InternalThreadState* Thread);
+
+#ifdef VIXL_SIMULATOR
+  void ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame);
+  void ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP);
+#else
+  void ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame) {
+    DispatchPtr(Frame, false);
+  }
+
+  void ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP) {
+    CallbackPtr(Frame, RIP);
+  }
+#endif
+
+  SignalDelegatorConfig MakeSignalDelegatorConfig() const;
+
+protected:
+  FEXCore::Context::ContextImpl* CTX;
+
+  using AsmDispatch = void (*)(FEXCore::Core::CpuStateFrame* Frame, bool SingleInst);
+  using JITCallback = void (*)(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP);
+
+  AsmDispatch DispatchPtr;
+  JITCallback CallbackPtr;
+private:
   /**
    * @name Dispatch Helper functions
    * @{ */
@@ -59,67 +78,17 @@ public:
   uint64_t GuestSignal_SIGILL {};
   uint64_t GuestSignal_SIGTRAP {};
   uint64_t GuestSignal_SIGSEGV {};
-  uint64_t IntCallbackReturnAddress {};
 
   uint64_t PauseReturnInstruction {};
   std::array<uint64_t, FallbackABI::FABI_UNKNOWN> ABIPointers {};
-
   /**  @} */
 
   uint64_t Start {};
   uint64_t End {};
 
-  void InitThreadPointers(FEXCore::Core::InternalThreadState* Thread);
-
-#ifdef VIXL_SIMULATOR
-  void ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame);
-  void ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP);
-#else
-  void ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame) {
-    DispatchPtr(Frame);
-  }
-
-  void ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP) {
-    CallbackPtr(Frame, RIP);
-  }
-#endif
-
-  uint16_t GetSRAGPRCount() const {
-    // PF/AF are the final two SRA registers.
-    // Only return the SRA for GPRs.
-    return StaticRegisters.size() - 2;
-  }
-
-  uint16_t GetSRAFPRCount() const {
-    return StaticFPRegisters.size();
-  }
-
-  void GetSRAGPRMapping(uint8_t Mapping[16]) const {
-    for (size_t i = 0; i < StaticRegisters.size() - 2; ++i) {
-      Mapping[i] = StaticRegisters[i].Idx();
-    }
-  }
-
-  void GetSRAFPRMapping(uint8_t Mapping[16]) const {
-    for (size_t i = 0; i < StaticFPRegisters.size(); ++i) {
-      Mapping[i] = StaticFPRegisters[i].Idx();
-    }
-  }
-
-protected:
-  FEXCore::Context::ContextImpl* CTX;
-
-  using AsmDispatch = void (*)(FEXCore::Core::CpuStateFrame* Frame);
-  using JITCallback = void (*)(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP);
-
-  AsmDispatch DispatchPtr;
-  JITCallback CallbackPtr;
-private:
   // Long division helpers
   uint64_t LUDIVHandlerAddress {};
   uint64_t LDIVHandlerAddress {};
-  uint64_t LUREMHandlerAddress {};
-  uint64_t LREMHandlerAddress {};
 
   void EmitDispatcher();
   uint64_t GenerateABICall(FallbackABI ABI);

FEXCore/Source/Interface/Core/Frontend.cpp

@@ -9,6 +9,8 @@ $end_info$
 #include "Interface/Context/Context.h"
 #include "Interface/Core/Frontend.h"
 #include "Interface/Core/X86Tables/X86Tables.h"
+#include "Interface/Core/X86HelperGen.h"
+#include "Interface/Core/LookupCache.h"
 
 #include <array>
 #include <algorithm>
@@ -21,6 +23,7 @@ $end_info$
 #include <FEXCore/Utils/Profiler.h>
 #include <FEXCore/Utils/Telemetry.h>
 #include <FEXCore/Utils/TypeDefines.h>
+#include <FEXCore/Debug/InternalThreadState.h>
 #include <FEXCore/fextl/set.h>
 
 namespace FEXCore::Frontend {
@@ -64,33 +67,90 @@ static uint32_t MapVEXToReg(uint8_t vvvv, bool HasXMM) {
   }
 }
 
-Decoder::Decoder(FEXCore::Context::ContextImpl* ctx)
-  : CTX {ctx}
-  , OSABI {ctx->SyscallHandler ? ctx->SyscallHandler->GetOSABI() : FEXCore::HLE::SyscallOSABI::OS_UNKNOWN}
-  , PoolObject {ctx->FrontendAllocator, sizeof(FEXCore::X86Tables::DecodedInst) * DefaultDecodedBufferSize} {}
+Decoder::Decoder(FEXCore::Core::InternalThreadState* Thread)
+  : Thread {Thread}
+  , CTX {static_cast<FEXCore::Context::ContextImpl*>(Thread->CTX)}
+  , OSABI {CTX->SyscallHandler ? CTX->SyscallHandler->GetOSABI() : FEXCore::HLE::SyscallOSABI::OS_UNKNOWN}
+  , PoolObject {CTX->FrontendAllocator, sizeof(FEXCore::X86Tables::DecodedInst) * DefaultDecodedBufferSize} {
 
-Decoder::~Decoder() {
-  PoolObject.UnclaimBuffer();
+  FEX_CONFIG_OPT(ReducedPrecision, X87REDUCEDPRECISION);
+  if (ReducedPrecision) {
+    X87Table = &FEXCore::X86Tables::X87F64Ops;
+  } else {
+    X87Table = &FEXCore::X86Tables::X87F80Ops;
+  }
+
+  if (CTX->HostFeatures.SupportsAVX && CTX->HostFeatures.SupportsSVE256) {
+    VEXTable = &FEXCore::X86Tables::VEXTableOps;
+    VEXTableGroup = &FEXCore::X86Tables::VEXTableGroupOps;
+  } else if (CTX->HostFeatures.SupportsAVX) {
+    VEXTable = &FEXCore::X86Tables::VEXTableOps_AVX128;
+    VEXTableGroup = &FEXCore::X86Tables::VEXTableGroupOps_AVX128;
+  }
+}
+
+bool Decoder::CheckRangeExecutable(uint64_t Address, uint64_t Size) {
+  // Treat FEX-internal X86 callbacks as always executable
+  if (EntryPoint == CTX->X86CodeGen.CallbackReturn) {
+    return true;
+  }
+
+  while (Address < ExecutableRangeBase || Address + Size > ExecutableRangeEnd) {
+    auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, Address);
+    ExecutableRangeBase = RangeInfo.Base;
+    ExecutableRangeEnd = RangeInfo.Base + RangeInfo.Size;
+    ExecutableRangeWritable = RangeInfo.Writable;
+
+    if (RangeInfo.Size == 0) {
+      return false;
+    }
+
+    uint64_t RangeRemainingSize = ExecutableRangeEnd - Address;
+    if (Size > RangeRemainingSize) {
+      Size -= RangeRemainingSize;
+      Address += RangeRemainingSize;
+    }
+  }
+
+  return true;
 }
 
 uint8_t Decoder::ReadByte() {
-  uint8_t Byte = InstStream[InstructionSize];
   LOGMAN_THROW_A_FMT(InstructionSize < MAX_INST_SIZE, "Max instruction size exceeded!");
-  Instruction[InstructionSize] = Byte;
+  std::optional<uint8_t> Byte = PeekByte(0);
+  if (!Byte) {
+    HitNonExecutableRange = true;
+    // Pretend we read 0, the main decode loop will see HitNonExecutableRange and rollback the instruction.
+    return 0;
+  }
+
+  Instruction[InstructionSize] = *Byte;
   InstructionSize++;
-  return Byte;
+  return *Byte;
 }
 
-uint8_t Decoder::PeekByte(uint8_t Offset) const {
-  uint8_t Byte = InstStream[InstructionSize + Offset];
-  return Byte;
+std::optional<uint8_t> Decoder::PeekByte(uint8_t Offset) {
+  uint64_t ByteAddress = reinterpret_cast<uint64_t>(InstStream + InstructionSize + Offset);
+  if (CheckRangeExecutable(ByteAddress, 1)) {
+    return InstStream[InstructionSize + Offset];
+  } else {
+    return std::nullopt;
+  }
 }
 
 uint64_t Decoder::ReadData(uint8_t Size) {
   LOGMAN_THROW_A_FMT(Size != 0 && Size <= sizeof(uint64_t), "Unknown data size to read");
 
   uint64_t Res = 0;
-  std::memcpy(&Res, &InstStream[InstructionSize], Size);
+  uint64_t Address = reinterpret_cast<uint64_t>(InstStream + InstructionSize);
+  if (CheckRangeExecutable(Address, Size)) {
+    std::memcpy(&Res, &InstStream[InstructionSize], Size);
+  } else {
+    HitNonExecutableRange = true;
+    // See PeekByte, this specific case may cause some executable memory to read as 0 but it doesn't matter as the entire instruction will be rolled back anyway.
+    Res = 0;
+  }
+
 
 #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
   for (size_t i = 0; i < Size; ++i) {
@@ -271,6 +331,13 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand* Operand, X86Tables::ModR
 }
 
 bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op, DecodedHeader Options) {
+  if (Info->Type == FEXCore::X86Tables::TYPE_ARCH_DISPATCHER) [[unlikely]] {
+    // Dispatcher Op.
+    // TODO: Move this in to `NormalOpHeader`, Dispatch tables have a bug currently where some subtables don't inherit flags correctly.
+    // Can be seen by running FEX asm tests if this is removed.
+    return NormalOp(&Info->OpcodeDispatcher.Indirect[BlockInfo.Is64BitMode ? 1 : 0], Op);
+  }
+
   DecodeInst->OP = Op;
   DecodeInst->TableInfo = Info;
 
@@ -290,7 +357,7 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
   uint8_t DestSize {};
   const bool HasWideningDisplacement =
     (FEXCore::X86Tables::DecodeFlags::GetOpAddr(DecodeInst->Flags, 0) & FEXCore::X86Tables::DecodeFlags::FLAG_WIDENING_SIZE_LAST) != 0 ||
-    (Options.w && CTX->Config.Is64BitMode);
+    (Options.w && BlockInfo.Is64BitMode);
   const bool HasNarrowingDisplacement =
     (FEXCore::X86Tables::DecodeFlags::GetOpAddr(DecodeInst->Flags, 0) & FEXCore::X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST) != 0;
 
@@ -308,7 +375,21 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
   const bool HasMODRM = !!(Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_MODRM);
 
   const bool HasREX = !!(DecodeInst->Flags & DecodeFlags::FLAG_REX_PREFIX);
-  const bool Has16BitAddressing = !CTX->Config.Is64BitMode && DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;
+  const bool Has16BitAddressing = !BlockInfo.Is64BitMode && DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;
+
+  if (Options.w && (Info->Flags & InstFlags::FLAGS_REX_W_0)) {
+    return false;
+  } else if (!Options.w && (Info->Flags & InstFlags::FLAGS_REX_W_1)) {
+    return false;
+  }
+
+  if (Options.L && (Info->Flags & InstFlags::FLAGS_VEX_L_0)) {
+    return false;
+  } else if (!Options.L && (Info->Flags & InstFlags::FLAGS_VEX_L_1)) {
+    return false;
+  }
+
+  const bool UseVEXL = Options.L && !(Info->Flags & InstFlags::FLAGS_VEX_L_IGNORE);
 
   // This is used for ModRM register modification
   // For both modrm.reg and modrm.rm(when mod == 0b11) when value is >= 0b100
@@ -339,7 +420,7 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
       DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_16BIT);
       DestSize = 2;
     } else if (DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_128BIT) {
-      if (Options.L) {
+      if (UseVEXL) {
         DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_256BIT);
         DestSize = 32;
       } else {
@@ -355,7 +436,7 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
       // If the default operating mode is 32bit and we have the operand size flag then the operating size drops to 16bit
       DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_16BIT);
       DestSize = 2;
-    } else if ((HasXMMDst || HasMMDst || CTX->Config.Is64BitMode) &&
+    } else if ((HasXMMDst || HasMMDst || BlockInfo.Is64BitMode) &&
                (HasWideningDisplacement || DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BIT ||
                 DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BITDEF)) {
       DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_64BIT);
@@ -372,7 +453,7 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
     } else if (SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_16BIT) {
       DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_16BIT);
     } else if (SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_128BIT) {
-      if (Options.L) {
+      if (UseVEXL) {
         DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_256BIT);
       } else {
         DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_128BIT);
@@ -384,7 +465,7 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
       // See table 1-2. Operand-Size Overrides for this decoding
       // If the default operating mode is 32bit and we have the operand size flag then the operating size drops to 16bit
       DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_16BIT);
-    } else if ((HasXMMSrc || HasMMSrc || CTX->Config.Is64BitMode) &&
+    } else if ((HasXMMSrc || HasMMSrc || BlockInfo.Is64BitMode) &&
                (HasWideningDisplacement || SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BIT ||
                 SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BITDEF)) {
       DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_64BIT);
@@ -484,6 +565,9 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
   size_t CurrentSrc = 0;
 
   const auto VEXOperand = Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_VEX_SRC_MASK;
+  if (VEXOperand == FEXCore::X86Tables::InstFlags::FLAGS_VEX_NO_OPERAND && Options.vvvv) {
+    return false;
+  }
 
   if (VEXOperand == FEXCore::X86Tables::InstFlags::FLAGS_VEX_1ST_SRC) {
     DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;
@@ -566,7 +650,7 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
 }
 
 bool Decoder::NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op) {
-  DecodeInst->OP = Op;
+  DecodeInst->OPRaw = DecodeInst->OP = Op;
   DecodeInst->TableInfo = Info;
 
   if (Info->Type == FEXCore::X86Tables::TYPE_UNKNOWN) {
@@ -582,6 +666,9 @@ bool Decoder::NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16
   // A normal instruction is the most likely.
   if (Info->Type == FEXCore::X86Tables::TYPE_INST) [[likely]] {
     return NormalOp(Info, Op);
+  } else if (Info->Type == FEXCore::X86Tables::TYPE_ARCH_DISPATCHER) [[unlikely]] {
+    // Dispatcher Op.
+    return NormalOp(&Info->OpcodeDispatcher.Indirect[BlockInfo.Is64BitMode ? 1 : 0], Op);
   } else if (Info->Type >= FEXCore::X86Tables::TYPE_GROUP_1 && Info->Type <= FEXCore::X86Tables::TYPE_GROUP_11) {
     uint8_t ModRMByte = ReadByte();
     DecodeInst->ModRM = ModRMByte;
@@ -619,7 +706,7 @@ bool Decoder::NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16
     ModRM.Hex = DecodeInst->ModRM;
 
     uint16_t LocalOp = OPD(Info->Type, PrefixType, ModRM.reg);
-    FEXCore::X86Tables::X86InstInfo* LocalInfo = &SecondInstGroupOps[LocalOp];
+    const FEXCore::X86Tables::X86InstInfo* LocalInfo = &SecondInstGroupOps[LocalOp];
 #undef OPD
     if (LocalInfo->Type == FEXCore::X86Tables::TYPE_SECOND_GROUP_MODRM && ModRM.mod == 0b11) {
       // Everything in this group is privileged instructions aside from XGETBV
@@ -643,15 +730,20 @@ bool Decoder::NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16
     DecodeInst->DecodedModRM = true;
 
     uint16_t X87Op = ((Op - 0xD8) << 8) | ModRMByte;
-    return NormalOp(&X87Ops[X87Op], X87Op);
+    return NormalOp(&(*X87Table)[X87Op], X87Op);
   } else if (Info->Type == FEXCore::X86Tables::TYPE_VEX_TABLE_PREFIX) {
+    if (!VEXTable) {
+      // AVX not enabled.
+      return false;
+    }
+
     uint16_t map_select = 1;
     uint16_t pp = 0;
     const uint8_t Byte1 = ReadByte();
     DecodedHeader options {};
 
     if ((Byte1 & 0b10000000) == 0) {
-      if (!CTX->Config.Is64BitMode) {
+      if (!BlockInfo.Is64BitMode) {
         return false;
       }
 
@@ -670,19 +762,18 @@ bool Decoder::NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16
       options.w = (Byte2 & 0b10000000) != 0;
       options.L = (Byte2 & 0b100) != 0;
       if ((Byte1 & 0b01000000) == 0) {
-        if (!CTX->Config.Is64BitMode) {
+        if (!BlockInfo.Is64BitMode) {
           return false;
         }
         DecodeInst->Flags |= DecodeFlags::FLAG_REX_XGPR_X;
       }
-      if (CTX->Config.Is64BitMode && (Byte1 & 0b00100000) == 0) {
+      if (BlockInfo.Is64BitMode && (Byte1 & 0b00100000) == 0) {
         DecodeInst->Flags |= DecodeFlags::FLAG_REX_XGPR_B;
       }
       if (options.w) {
         DecodeInst->Flags |= DecodeFlags::FLAG_OPTION_AVX_W;
       }
       if (!(map_select >= 1 && map_select <= 3)) {
-        LogMan::Msg::EFmt("We don't understand a map_select of: {}", map_select);
         return false;
       }
     }
@@ -692,7 +783,7 @@ bool Decoder::NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16
     Op = OPD(map_select, pp, VEXOp);
 #undef OPD
 
-    FEXCore::X86Tables::X86InstInfo* LocalInfo = &VEXTableOps[Op];
+    const FEXCore::X86Tables::X86InstInfo* LocalInfo = &(*VEXTable)[Op];
 
     if (LocalInfo->Type >= FEXCore::X86Tables::TYPE_VEX_GROUP_12 && LocalInfo->Type <= FEXCore::X86Tables::TYPE_VEX_GROUP_17) {
       // We have ModRM
@@ -706,7 +797,7 @@ bool Decoder::NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16
 #define OPD(group, pp, opcode) (((group - TYPE_VEX_GROUP_12) << 4) | (pp << 3) | (opcode))
       Op = OPD(LocalInfo->Type, pp, ModRM.reg);
 #undef OPD
-      return NormalOp(&VEXTableGroupOps[Op], Op, options);
+      return NormalOp(&(*VEXTableGroup)[Op], Op, options);
     } else {
       return NormalOp(LocalInfo, Op, options);
     }
@@ -720,7 +811,7 @@ bool Decoder::NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16
   FEX_UNREACHABLE;
 }
 
-bool Decoder::DecodeInstruction(uint64_t PC) {
+bool Decoder::DecodeInstructionImpl(uint64_t PC) {
   InstructionSize = 0;
   Instruction.fill(0);
 
@@ -748,7 +839,7 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
           FEXCore::X86Tables::ModRMDecoded ModRM;
           ModRM.Hex = DecodeInst->ModRM;
 
-          const bool Has16BitAddressing = !CTX->Config.Is64BitMode && DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;
+          const bool Has16BitAddressing = !BlockInfo.Is64BitMode && DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;
 
           // All 3DNow! instructions have the second argument as the rm handler
           // We need to decode it upfront to get the displacement out of the way
@@ -857,29 +948,25 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
       DecodeInst->Flags |= DecodeFlags::FLAG_ADDRESS_SIZE;
       break;
     case 0x26: // ES legacy prefix
-      if (!CTX->Config.Is64BitMode) {
-        DecodeInst->Flags |= DecodeFlags::FLAG_ES_PREFIX;
+      if (!BlockInfo.Is64BitMode) {
+        DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_ES_PREFIX;
       }
       break;
     case 0x2E: // CS legacy prefix
-      if (!CTX->Config.Is64BitMode) {
-        DecodeInst->Flags |= DecodeFlags::FLAG_CS_PREFIX;
+      if (!BlockInfo.Is64BitMode) {
+        DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_CS_PREFIX;
       }
       break;
     case 0x36: // SS legacy prefix
-      if (!CTX->Config.Is64BitMode) {
-        DecodeInst->Flags |= DecodeFlags::FLAG_SS_PREFIX;
+      if (!BlockInfo.Is64BitMode) {
+        DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_SS_PREFIX;
       }
       break;
     case 0x3E: // DS legacy prefix
-      // Annoyingly GCC generates NOP ops with these prefixes
-      // Just ignore them for now
-      // eg. 66 2e 0f 1f 84 00 00 00 00 00 nop    WORD PTR cs:[rax+rax*1+0x0]
-      if (!CTX->Config.Is64BitMode) {
-        DecodeInst->Flags |= DecodeFlags::FLAG_DS_PREFIX;
+      if (!BlockInfo.Is64BitMode) {
+        DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_DS_PREFIX;
       }
       break;
-      break;
     case 0xF0: // LOCK prefix
       DecodeInst->Flags |= DecodeFlags::FLAG_LOCK;
       break;
@@ -892,19 +979,19 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
       DecodeInst->LastEscapePrefix = Op;
       break;
     case 0x64: // FS prefix
-      DecodeInst->Flags |= DecodeFlags::FLAG_FS_PREFIX;
+      DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_FS_PREFIX;
       break;
     case 0x65: // GS prefix
-      DecodeInst->Flags |= DecodeFlags::FLAG_GS_PREFIX;
+      DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_GS_PREFIX;
       break;
     default:
       [[likely]] { // Default base table
-        auto Info = &FEXCore::X86Tables::BaseOps[Op];
+        const X86InstInfo* Info = &FEXCore::X86Tables::BaseOps[Op];
+        if (Info->Type == FEXCore::X86Tables::TYPE_ARCH_DISPATCHER) {
+          Info = &Info->OpcodeDispatcher.Indirect[BlockInfo.Is64BitMode ? 1 : 0];
+        }
 
         if (Info->Type == FEXCore::X86Tables::TYPE_REX_PREFIX) {
-          if (!CTX->Config.Is64BitMode) {
-            return false;
-          }
           DecodeInst->Flags |= DecodeFlags::FLAG_REX_PREFIX;
 
           // Widening displacement
@@ -935,6 +1022,47 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
       }
     }
   }
+
+  if (DecodeInst->Dest.IsGPR()) {
+    return false;
+  }
+
+  return true;
+}
+
+Decoder::DecodedBlockStatus Decoder::DecodeInstruction(uint64_t PC) {
+  // Will be set if DecodeInstructionImpl tries to read non-executable memory
+  HitNonExecutableRange = false;
+  bool ErrorDuringDecoding = !DecodeInstructionImpl(PC);
+
+  if (ErrorDuringDecoding || HitNonExecutableRange) [[unlikely]] {
+    // Put an invalid instruction in the stream so the core can raise SIGILL if hit
+    // Error while decoding instruction. We don't know the table or instruction size
+    DecodeInst->TableInfo = nullptr;
+    DecodeInst->InstSize = 0;
+    return ErrorDuringDecoding ? DecodedBlockStatus::INVALID_INST : DecodedBlockStatus::NOEXEC_INST;
+  } else if (!DecodeInst->TableInfo || (DecodeInst->TableInfo->Type == TYPE_INST && !DecodeInst->TableInfo->OpcodeDispatcher.OpDispatch)) {
+    // If there wasn't an error during decoding but we have no dispatcher for the instruction then claim invalid instruction.
+    return DecodedBlockStatus::INVALID_INST;
+  }
+
+  if (CTX->AreMonoHacksActive()) {
+    // Unity uses a standard SPSC ringbuffer with cached read/write pointers and thread waiting flags at the following
+    // offsets, which are consistent between 32-bit and 64-bit Unity versions from 2015 onwards.
+    auto IsKnownAtomicDisplacement = [](uint64_t Displacement) {
+      return Displacement == 0x80 || Displacement == 0x84 || Displacement == 0xC0 || Displacement == 0xC4;
+    };
+
+    if (DecodeInst->OP == 0x8b && DecodeInst->Src[0].IsGPRIndirect() &&
+        IsKnownAtomicDisplacement(DecodeInst->Src[0].Data.GPRIndirect.Displacement)) {
+      DecodeInst->ForceTSO = true;
+    }
+    if (DecodeInst->OP == 0x89 && DecodeInst->Dest.IsGPRIndirect() && IsKnownAtomicDisplacement(DecodeInst->Dest.Data.GPRIndirect.Displacement)) {
+      DecodeInst->ForceTSO = true;
+    }
+  }
+
+  return DecodedBlockStatus::SUCCESS;
 }
 
 void Decoder::BranchTargetInMultiblockRange() {
@@ -944,16 +1072,30 @@ void Decoder::BranchTargetInMultiblockRange() {
 
   // If the RIP setting is conditional AND within our symbol range then it can be considered for multiblock
   uint64_t TargetRIP = 0;
-  const auto GPRSize = CTX->GetGPROpSize();
+  const auto GPRSize = GetGPROpSize();
   bool Conditional = true;
   const auto InstEnd = DecodeInst->PC + DecodeInst->InstSize;
 
+  if (DecodeInst->TableInfo->Flags & FEXCore::X86Tables::InstFlags::FLAGS_CALL) {
+    if (ExecutableRangeWritable && CTX->AreMonoHacksActive()) {
+      // Mono generated code often contains noreturn calls with garbage following them, and calls are always backpatched
+      // after CIL compilation leading to n recompiles for a multiblock with n calls. Choose to minimize stutters over
+      // raw performance and disable tracking past calls for mono generated code.
+      return;
+    }
+
+    AddBranchTarget(InstEnd);
+    BlockInfo.EntryPoints.emplace(InstEnd);
+    return;
+  }
+
+  // Calls are handled above
   switch (DecodeInst->OP) {
   case 0x70 ... 0x7F:   // Conditional JUMP
   case 0x80 ... 0x8F: { // More conditional
     // Source is a literal
     // auto RIPOffset = LoadSource(Op, Op->Src[0], Op->Flags);
-    // auto RIPTargetConst = _Constant(Op->PC + Op->InstSize);
+    // auto RIPTargetConst = Constant(Op->PC + Op->InstSize);
     // Target offset is PC + InstSize + Literal
     TargetRIP = InstEnd + DecodeInst->Src[0].Literal();
     break;
@@ -963,11 +1105,6 @@ void Decoder::BranchTargetInMultiblockRange() {
     TargetRIP = InstEnd + DecodeInst->Src[0].Literal();
     Conditional = false;
     break;
-  case 0xE8: // Call - Immediate target, We don't want to inline calls
-    if (ExternalBranches) {
-      ExternalBranches->insert(InstEnd);
-    }
-    [[fallthrough]];
   case 0xC2: // RET imm
   case 0xC3: // RET
   default: return; break;
@@ -978,10 +1115,16 @@ void Decoder::BranchTargetInMultiblockRange() {
     TargetRIP &= 0xFFFFFFFFU;
   }
 
+  if (Conditional) {
+    // If we are conditional then a target can be the instruction past the conditional instruction
+    AddBranchTarget(InstEnd);
+  }
+
   // If the target RIP is x86 code within the symbol ranges then we are golden
-  // Forbid cross-page branches to both avoid massive (range-wise) code blocks in highly fragmented code and trying to decode unmapped branch targets
-  bool ValidMultiblockMember =
-    TargetRIP >= SymbolMinAddress && TargetRIP < std::min(FEXCore::AlignUp(InstEnd, FEXCore::Utils::FEX_PAGE_SIZE), SymbolMaxAddress);
+  // Forbid distant branches to have the cost code better match the guest code layout, avoiding massive (range-wise) code
+  // blocks in highly fragmented guest code. Such branches are often not-taken branches to garbage in obfuscated code.
+  constexpr uint64_t MAX_FORWARD_BRANCH_DIST = FEXCore::Utils::FEX_PAGE_SIZE * 4;
+  bool ValidMultiblockMember = TargetRIP >= SymbolMinAddress && TargetRIP < std::min(InstEnd + MAX_FORWARD_BRANCH_DIST, SymbolMaxAddress);
 
 #ifdef _M_ARM_64EC
   ValidMultiblockMember = ValidMultiblockMember && !RtlIsEcCode(TargetRIP);
@@ -992,9 +1135,6 @@ void Decoder::BranchTargetInMultiblockRange() {
     if (Conditional) {
       MaxCondBranchForward = std::max(MaxCondBranchForward, TargetRIP);
       MaxCondBranchBackwards = std::min(MaxCondBranchBackwards, TargetRIP);
-
-      // If we are conditional then a target can be the instruction past the conditional instruction
-      AddBranchTarget(InstEnd);
     }
 
     AddBranchTarget(TargetRIP);
@@ -1005,6 +1145,60 @@ void Decoder::BranchTargetInMultiblockRange() {
   }
 }
 
+bool Decoder::IsBranchMonoTailcall(uint64_t NumInstructions) const {
+  // While the mono call backpatching block can easily be detected due it being the only one to contain SMC-faulting
+  // atomics, that can't be said for the tailcall jump backpatcher which has changed several times across versions and
+  // can be partially inlined. To work around this, instead detect the tailcall site itself and force full non-signal-based
+  // SMC detection for that single block.
+  if (!ExecutableRangeWritable) {
+    // We only care about jitted code
+    return false;
+  }
+
+  // See mini-{amd64,x86}.c in the mono codebase, specifically where METHOD_JUMP patches are emitted.
+  if (GetGPROpSize() == IR::OpSize::i32Bit) {
+    // Matches:
+    // LEAVE
+    // <none> / NOP / MOV EAX, EAX / LEA EBP, [EBP+0]
+    // JMP imm32
+    if (DecodeInst->OP != 0xE9 || NumInstructions < 2) {
+      return false;
+    }
+
+    auto PrevInst = std::prev(DecodeInst);
+    if (PrevInst->OP == 0xC9) {
+      return true;
+    }
+
+    if (NumInstructions < 3 || std::prev(PrevInst)->OP != 0xC9) {
+      return false;
+    }
+
+    return PrevInst->OP == 0x90 || (PrevInst->OP == 0x8B && PrevInst->ModRM == 0xC0) ||
+           (PrevInst->OP == 0x8D && PrevInst->ModRM == 0x6D && PrevInst->Src[1].IsLiteral() && PrevInst->Src[1].Literal() == 0);
+  } else {
+    FEXCore::X86Tables::ModRMDecoded ModRM;
+    ModRM.Hex = DecodeInst->ModRM;
+    if (DecodeInst->OPRaw == 0xFF && ModRM.reg == 4 && DecodeInst->Src[0].IsGPR()) {
+      if (DecodeInst->Src[0].Data.GPR.GPR == FEXCore::X86State::REG_RAX) {
+        // Found in versions of mono from 2024 onwards - matches:
+        // REX.W JMP rax
+        return (DecodeInst->Flags & (DecodeFlags::FLAG_REX_PREFIX | DecodeFlags::FLAG_REX_WIDENING | DecodeFlags::FLAG_REX_XGPR_B |
+                                     DecodeFlags::FLAG_REX_XGPR_X | DecodeFlags::FLAG_REX_XGPR_R)) ==
+               (DecodeFlags::FLAG_REX_PREFIX | DecodeFlags::FLAG_REX_WIDENING);
+      } else if (NumInstructions > 1 && DecodeInst->Src[0].Data.GPR.GPR == FEXCore::X86State::REG_R11) {
+        // Found in older versions of mono - match:
+        // MOV r11, imm64
+        // JMP r11
+        auto PrevInst = std::prev(DecodeInst);
+        return PrevInst->OP == 0xBB && PrevInst->Dest.IsGPR() && PrevInst->Dest.Data.GPR.GPR == FEXCore::X86State::REG_R11;
+      }
+    }
+  }
+
+  return false;
+}
+
 bool Decoder::InstCanContinue() const {
   if (DecodeInst->PC + DecodeInst->InstSize == NextBlockStartAddress) {
     return false;
@@ -1015,7 +1209,7 @@ bool Decoder::InstCanContinue() const {
   }
 
   uint64_t TargetRIP = 0;
-  const auto GPRSize = CTX->GetGPROpSize();
+  const auto GPRSize = GetGPROpSize();
 
   if (DecodeInst->OP == 0xE8) { // Call - immediate target
     const uint64_t NextRIP = DecodeInst->PC + DecodeInst->InstSize;
@@ -1067,7 +1261,7 @@ void Decoder::AddBranchTarget(uint64_t Target) {
           .Size = BlockIt->Size - SplitOffset,
           .NumInstructions = BlockIt->NumInstructions - SplitIdx,
           .DecodedInstructions = BlockIt->DecodedInstructions + SplitIdx,
-          .HasInvalidInstruction = BlockIt->HasInvalidInstruction,
+          .BlockStatus = BlockIt->BlockStatus,
         };
 
         BlockIt->Size = SplitOffset;
@@ -1107,12 +1301,10 @@ const uint8_t* Decoder::AdjustAddrForSpecialRegion(const uint8_t* _InstStream, u
   return _InstStream - EntryPoint + RIP;
 }
 
-void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC, uint64_t MaxInst,
-                                        std::function<void(uint64_t BlockEntry, uint64_t Start, uint64_t Length)> AddContainedCodePage) {
+void Decoder::DecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState *Thread, const uint8_t* _InstStream, uint64_t PC, uint64_t MaxInst) {
   FEXCORE_PROFILE_SCOPED("DecodeInstructions");
   BlockInfo.TotalInstructionCount = 0;
   BlockInfo.Blocks.clear();
-  BlocksToDecode.clear();
   VisitedBlocks.clear();
   // Reset internal state management
   DecodedSize = 0;
@@ -1120,9 +1312,15 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
   MaxCondBranchBackwards = ~0ULL;
   DecodedBuffer = PoolObject.ReownOrClaimBuffer();
 
+  // Decode operating mode from thread's CS segment.
+  const auto CSSegment = Core::CPUState::GetSegmentFromIndex(Thread->CurrentFrame->State, Thread->CurrentFrame->State.cs_idx);
+  BlockInfo.Is64BitMode = CSSegment->L == 1;
+  LOGMAN_THROW_A_FMT(BlockInfo.Is64BitMode == CTX->Config.Is64BitMode, "Expected operating mode to not change at runtime!");
+
   // XXX: Load symbol data
   SymbolAvailable = false;
   EntryPoint = PC;
+  BlockInfo.EntryPoints = {PC};
   InstStream = _InstStream;
 
   uint64_t TotalInstructions {};
@@ -1138,13 +1336,11 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
   DecodedMaxAddress = EntryPoint;
 
   // Entry is a jump target
-  BlocksToDecode.emplace(PC);
+  BlocksToDecode = {PC};
 
   uint64_t CurrentCodePage = PC & FEXCore::Utils::FEX_PAGE_MASK;
 
-  fextl::set<uint64_t> CodePages = {CurrentCodePage};
-
-  AddContainedCodePage(PC, CurrentCodePage, FEXCore::Utils::FEX_PAGE_SIZE);
+  BlockInfo.CodePages = {CurrentCodePage};
 
   if (MaxInst == 0) {
     MaxInst = CTX->Config.MaxInstPerBlock;
@@ -1179,6 +1375,7 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
 
     BlockIt->Entry = RIPToDecode;
     BlockIt->Size = 0;
+    BlockIt->IsEntryPoint = EntryBlock;
 
     uint64_t PCOffset = 0;
     uint64_t BlockStartOffset = DecodedSize;
@@ -1200,7 +1397,7 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
       auto OpMinPage = OpAddress & FEXCore::Utils::FEX_PAGE_MASK;
       auto OpMaxPage = OpMaxAddress & FEXCore::Utils::FEX_PAGE_MASK;
 
-      if (!EntryBlock && OpMinPage == OpMaxPage && PeekByte(0) == 0 && PeekByte(1) == 0) [[unlikely]] {
+      if (!EntryBlock && OpMinPage == OpMaxPage && PeekByte(0).value_or(0) == 0 && PeekByte(1).value_or(0) == 0) [[unlikely]] {
         // End the multiblock early if we hit 2 consecutive null bytes (add [rax], al) in the same page with the
         // assumption we are most likely trying to explore garbage code.
         break;
@@ -1208,31 +1405,17 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
 
       if (OpMinPage != CurrentCodePage) {
         CurrentCodePage = OpMinPage;
-        CodePages.insert(CurrentCodePage);
+        BlockInfo.CodePages.insert(CurrentCodePage);
       }
 
       if (OpMaxPage != CurrentCodePage) {
         CurrentCodePage = OpMaxPage;
-        CodePages.insert(CurrentCodePage);
+        BlockInfo.CodePages.insert(CurrentCodePage);
       }
 
-      bool ErrorDuringDecoding = !DecodeInstruction(OpAddress);
+      BlockIt->BlockStatus = DecodeInstruction(OpAddress);
       uint64_t OpEndAddress = OpAddress + DecodeInst->InstSize;
 
-      if (ErrorDuringDecoding) [[unlikely]] {
-        // Put an invalid instruction in the stream so the core can raise SIGILL if hit
-        BlockIt->HasInvalidInstruction = true;
-        // Error while decoding instruction. We don't know the table or instruction size
-        DecodeInst->TableInfo = nullptr;
-        DecodeInst->InstSize = 0;
-      } else {
-        // If there wasn't an error during decoding but we have no dispatcher for the instruction then claim invalid instruction.
-        auto TableInfo = DecodeInst->TableInfo;
-        if (!TableInfo || !TableInfo->OpcodeDispatcher) {
-          BlockIt->HasInvalidInstruction = true;
-        }
-      }
-
       DecodedMinAddress = std::min(DecodedMinAddress, OpAddress);
       DecodedMaxAddress = std::max(DecodedMaxAddress, OpEndAddress);
 
@@ -1248,7 +1431,7 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
       BlockIt->Size += DecodeInst->InstSize;
 
       // Can not continue this block at all on invalid instruction
-      if (BlockIt->HasInvalidInstruction) [[unlikely]] {
+      if (BlockIt->BlockStatus != DecodedBlockStatus::SUCCESS) [[unlikely]] {
         if (!EntryBlock) {
           // In multiblock configurations, we can early terminate any non-entrypoint blocks with the expectation that this won't get hit.
           // Improves compile-times.
@@ -1257,6 +1440,9 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
           DecodedSize = BlockStartOffset;
           InstStream -= PCOffset;
           EraseBlock = true;
+        } else {
+          LogMan::Msg::EFmt("{} instruction in entry block: {:X}",
+                            BlockIt->BlockStatus == DecodedBlockStatus::INVALID_INST ? "Invalid" : "NoExec", OpAddress);
         }
         break;
       }
@@ -1273,6 +1459,7 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
           // If the branch target is within our multiblock range then we can keep going on
           // We don't want to short circuit this since we want to calculate our ranges still
           // NOTE: This will invalidate BlockIt, this is fine as we immediately break from the loop and EraseBlock cannot be true
+          BlockIt->ForceFullSMCDetection = CTX->AreMonoHacksActive() && IsBranchMonoTailcall(BlockIt->NumInstructions);
           BranchTargetInMultiblockRange();
         }
 
@@ -1296,8 +1483,8 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
 
   BlockInfo.TotalInstructionCount = TotalInstructions;
 
-  for (auto CodePage : CodePages) {
-    AddContainedCodePage(PC, CodePage, FEXCore::Utils::FEX_PAGE_SIZE);
+  for (auto& Block : BlockInfo.Blocks) {
+    Block.IsEntryPoint = BlockInfo.EntryPoints.contains(Block.Entry);
   }
 }
 

FEXCore/Source/Interface/Core/Frontend.h

@@ -2,41 +2,54 @@
 #pragma once
 
 #include "Interface/Core/X86Tables/X86Tables.h"
+#include "Interface/IR/IR.h"
 
-#include <FEXCore/HLE/SyscallHandler.h>
-#include <FEXCore/Utils/Telemetry.h>
+#include <FEXCore/Utils/ThreadPoolAllocator.h>
 #include <FEXCore/fextl/set.h>
 #include <FEXCore/fextl/vector.h>
 
 #include <array>
+#include <cstddef>
 #include <cstdint>
-#include <stddef.h>
+#include <optional>
 
 namespace FEXCore::Context {
 class ContextImpl;
 }
+namespace FEXCore::HLE {
+enum class SyscallOSABI;
+}
 
 namespace FEXCore::Frontend {
 class Decoder final {
 public:
+  enum class DecodedBlockStatus {
+    SUCCESS,
+    INVALID_INST,
+    NOEXEC_INST,
+  };
+
   // New Frontend decoding
   struct DecodedBlocks final {
     uint64_t Entry {};
     uint64_t Size {};
     uint64_t NumInstructions {};
     FEXCore::X86Tables::DecodedInst* DecodedInstructions;
-    bool HasInvalidInstruction {};
+    DecodedBlockStatus BlockStatus;
+    bool IsEntryPoint {};
+    bool ForceFullSMCDetection {};
   };
 
   struct DecodedBlockInformation final {
     uint64_t TotalInstructionCount;
+    bool Is64BitMode {};
     fextl::vector<DecodedBlocks> Blocks;
+    fextl::set<uint64_t> EntryPoints;
+    fextl::set<uint64_t> CodePages; // Start addresses of all pages touching the block
   };
 
-  Decoder(FEXCore::Context::ContextImpl* ctx);
-  ~Decoder();
-  void DecodeInstructionsAtEntry(const uint8_t* InstStream, uint64_t PC, uint64_t MaxInst,
-                                 std::function<void(uint64_t BlockEntry, uint64_t Start, uint64_t Length)> AddContainedCodePage);
+  Decoder(FEXCore::Core::InternalThreadState* Thread);
+  void DecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState *Thread, const uint8_t* InstStream, uint64_t PC, uint64_t MaxInst);
 
   const DecodedBlockInformation* GetDecodedBlockInfo() const {
     return &BlockInfo;
@@ -56,6 +69,10 @@ public:
     PoolObject.DelayedDisownBuffer();
   }
 
+  void ResetExecutableRangeCache() {
+    ExecutableRangeBase = ExecutableRangeEnd = 0;
+  }
+
 private:
   // To pass any information from instruction prefixes
   // down into the actual instruction handling machinery.
@@ -65,18 +82,23 @@ private:
     bool L;       // VEX.L bit (if set then 256 bit operation, if unset then scalar or 128-bit operation)
   };
 
+  FEXCore::Core::InternalThreadState* Thread;
   FEXCore::Context::ContextImpl* CTX;
   const FEXCore::HLE::SyscallOSABI OSABI {};
 
-  bool DecodeInstruction(uint64_t PC);
+  bool DecodeInstructionImpl(uint64_t PC);
+  DecodedBlockStatus DecodeInstruction(uint64_t PC);
 
   void BranchTargetInMultiblockRange();
+  bool IsBranchMonoTailcall(uint64_t NumInstructions) const;
   bool InstCanContinue() const;
 
   void AddBranchTarget(uint64_t Target);
 
+  bool CheckRangeExecutable(uint64_t Address, uint64_t Size);
+
   uint8_t ReadByte();
-  uint8_t PeekByte(uint8_t Offset) const;
+  std::optional<uint8_t> PeekByte(uint8_t Offset);
   uint64_t ReadData(uint8_t Size);
   void SkipBytes(uint8_t Size) {
     InstructionSize += Size;
@@ -87,10 +109,18 @@ private:
 
   static constexpr size_t DefaultDecodedBufferSize = 0x10000;
   FEXCore::X86Tables::DecodedInst* DecodedBuffer {};
-  Utils::FixedSizePooledAllocation<FEXCore::X86Tables::DecodedInst*, 5000, 500> PoolObject;
+  Utils::PoolBufferWithTimedRetirement<FEXCore::X86Tables::DecodedInst*, 5000, 500> PoolObject;
   size_t DecodedSize {};
 
+  uint64_t ExecutableRangeBase {};
+  uint64_t ExecutableRangeEnd {};
+  bool ExecutableRangeWritable {};
+  bool HitNonExecutableRange {};
+
   const uint8_t* InstStream {};
+  IR::OpSize GetGPROpSize() const {
+    return BlockInfo.Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;
+  }
 
   static constexpr size_t MAX_INST_SIZE = 15;
   uint8_t InstructionSize {};
@@ -123,6 +153,11 @@ private:
     &FEXCore::Frontend::Decoder::DecodeModRM_16,
   };
 
+  const std::array<X86Tables::X86InstInfo, X86Tables::MAX_X87_TABLE_SIZE>* X87Table;
+
+  const std::array<X86Tables::X86InstInfo, X86Tables::MAX_VEX_TABLE_SIZE>* VEXTable {};
+  const std::array<X86Tables::X86InstInfo, X86Tables::MAX_VEX_GROUP_TABLE_SIZE>* VEXTableGroup {};
+
   const uint8_t* AdjustAddrForSpecialRegion(const uint8_t* _InstStream, uint64_t EntryPoint, uint64_t RIP);
 };
 } // namespace FEXCore::Frontend

FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h

@@ -6,7 +6,7 @@
 #include "Interface/IR/IR.h"
 
 #include <FEXCore/Debug/InternalThreadState.h>
-#include <FEXCore/Utils/Profiler.h>
+#include <FEXCore/Utils/SHMStats.h>
 
 namespace FEXCore::CPU {
 FEXCORE_PRESERVE_ALL_ATTR static softfloat_state SoftFloatStateFromFCW(uint16_t FCW, bool Force80BitPrecision = false) {
@@ -36,18 +36,48 @@ FEXCORE_PRESERVE_ALL_ATTR static softfloat_state SoftFloatStateFromFCW(uint16_t
   return State;
 }
 
+FEXCORE_PRESERVE_ALL_ATTR static void HandleX87Exception(const softfloat_state& State, FEXCore::Core::CpuStateFrame* Frame) {
+  // Check for Invalid Operation exception (bit 0 of X87 status word)
+  if (State.exceptionFlags & softfloat_flag_invalid) {
+    Frame->State.flags[FEXCore::X86State::X87FLAG_IE_LOC] = 1;
+  }
+}
+
+// Wrapper for SoftFloat state to handle X87 exceptions
+class ScopedSoftFloatState {
+public:
+  FEXCORE_PRESERVE_ALL_ATTR ScopedSoftFloatState(uint16_t FCW, FEXCore::Core::CpuStateFrame* Frame, bool Force80BitPrecision = false)
+    : State(SoftFloatStateFromFCW(FCW, Force80BitPrecision))
+    , Frame(Frame) {}
+
+  FEXCORE_PRESERVE_ALL_ATTR ~ScopedSoftFloatState() {
+    HandleX87Exception(State, Frame);
+  }
+
+  // Disable copy and move to ensure RAII semantics
+  ScopedSoftFloatState(const ScopedSoftFloatState&) = delete;
+  ScopedSoftFloatState& operator=(const ScopedSoftFloatState&) = delete;
+  ScopedSoftFloatState(ScopedSoftFloatState&&) = delete;
+  ScopedSoftFloatState& operator=(ScopedSoftFloatState&&) = delete;
+
+  softfloat_state State;
+
+private:
+  FEXCore::Core::CpuStateFrame* Frame;
+};
+
 template<>
 struct OpHandlers<IR::OP_F80CVTTO> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle4(uint16_t FCW, float src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(&State, src);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(&State.State, src);
   }
 
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle8(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(&State, src);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(&State.State, src);
   }
 };
 
@@ -55,12 +85,12 @@ template<>
 struct OpHandlers<IR::OP_F80CMP> {
   FEXCORE_PRESERVE_ALL_ATTR static uint64_t handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
+    ScopedSoftFloatState State {FCW, Frame};
 
     bool eq, lt, nan;
     uint64_t ResultFlags = 0;
 
-    X80SoftFloat::FCMP(&State, Src1, Src2, &eq, &lt, &nan);
+    X80SoftFloat::FCMP(&State.State, Src1, Src2, &eq, &lt, &nan);
     if (lt) {
       ResultFlags |= (1 << IR::FCMP_FLAG_LT);
     }
@@ -78,14 +108,14 @@ template<>
 struct OpHandlers<IR::OP_F80CVT> {
   FEXCORE_PRESERVE_ALL_ATTR static float handle4(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(src).ToF32(&State);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(src).ToF32(&State.State);
   }
 
   FEXCORE_PRESERVE_ALL_ATTR static double handle8(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(src).ToF64(&State);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(src).ToF64(&State.State);
   }
 };
 
@@ -93,26 +123,26 @@ template<>
 struct OpHandlers<IR::OP_F80CVTINT> {
   FEXCORE_PRESERVE_ALL_ATTR static int16_t handle2(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(src).ToI16(&State);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(src).ToI16(&State.State);
   }
 
   FEXCORE_PRESERVE_ALL_ATTR static int32_t handle4(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(src).ToI32(&State);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(src).ToI32(&State.State);
   }
 
   FEXCORE_PRESERVE_ALL_ATTR static int64_t handle8(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(src).ToI64(&State);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(src).ToI64(&State.State);
   }
 
   FEXCORE_PRESERVE_ALL_ATTR static int16_t handle2t(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    auto rv = extF80_to_i32(&State, X80SoftFloat(src), softfloat_round_minMag, false);
+    ScopedSoftFloatState State {FCW, Frame};
+    auto rv = extF80_to_i32(&State.State, X80SoftFloat(src), softfloat_round_minMag, false);
 
     if (rv > INT16_MAX || rv < INT16_MIN) {
       ///< Indefinite value for 16-bit conversions.
@@ -124,14 +154,14 @@ struct OpHandlers<IR::OP_F80CVTINT> {
 
   FEXCORE_PRESERVE_ALL_ATTR static int32_t handle4t(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return extF80_to_i32(&State, X80SoftFloat(src), softfloat_round_minMag, false);
+    ScopedSoftFloatState State {FCW, Frame};
+    return extF80_to_i32(&State.State, X80SoftFloat(src), softfloat_round_minMag, false);
   }
 
   FEXCORE_PRESERVE_ALL_ATTR static int64_t handle8t(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return extF80_to_i64(&State, X80SoftFloat(src), softfloat_round_minMag, false);
+    ScopedSoftFloatState State {FCW, Frame};
+    return extF80_to_i64(&State.State, X80SoftFloat(src), softfloat_round_minMag, false);
   }
 };
 
@@ -152,8 +182,8 @@ template<>
 struct OpHandlers<IR::OP_F80ROUND> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FRNDINT(&State, Src1);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FRNDINT(&State.State, Src1);
   }
 };
 
@@ -161,8 +191,8 @@ template<>
 struct OpHandlers<IR::OP_F80F2XM1> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::F2XM1(&State, Src1);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::F2XM1(&State.State, Src1);
   }
 };
 
@@ -170,8 +200,8 @@ template<>
 struct OpHandlers<IR::OP_F80TAN> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FTAN(&State, Src1);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FTAN(&State.State, Src1);
   }
 };
 
@@ -179,8 +209,8 @@ template<>
 struct OpHandlers<IR::OP_F80SQRT> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat::FSQRT(&State, Src1);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat::FSQRT(&State.State, Src1);
   }
 };
 
@@ -188,8 +218,8 @@ template<>
 struct OpHandlers<IR::OP_F80SIN> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FSIN(&State, Src1);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FSIN(&State.State, Src1);
   }
 };
 
@@ -197,8 +227,17 @@ template<>
 struct OpHandlers<IR::OP_F80COS> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FCOS(&State, Src1);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FCOS(&State.State, Src1);
+  }
+};
+
+template<>
+struct OpHandlers<IR::OP_F80SINCOS> {
+  FEXCORE_PRESERVE_ALL_ATTR static VectorRegPairType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
+    FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return FEXCore::MakeVectorRegPair(X80SoftFloat::FSIN(&State.State, Src1), X80SoftFloat::FCOS(&State.State, Src1));
   }
 };
 
@@ -222,8 +261,8 @@ template<>
 struct OpHandlers<IR::OP_F80ADD> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat::FADD(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat::FADD(&State.State, Src1, Src2);
   }
 };
 
@@ -231,8 +270,8 @@ template<>
 struct OpHandlers<IR::OP_F80SUB> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat::FSUB(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat::FSUB(&State.State, Src1, Src2);
   }
 };
 
@@ -240,8 +279,8 @@ template<>
 struct OpHandlers<IR::OP_F80MUL> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat::FMUL(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat::FMUL(&State.State, Src1, Src2);
   }
 };
 
@@ -249,8 +288,8 @@ template<>
 struct OpHandlers<IR::OP_F80DIV> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat::FDIV(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat::FDIV(&State.State, Src1, Src2);
   }
 };
 
@@ -258,8 +297,8 @@ template<>
 struct OpHandlers<IR::OP_F80FYL2X> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FYL2X(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FYL2X(&State.State, Src1, Src2);
   }
 };
 
@@ -267,8 +306,8 @@ template<>
 struct OpHandlers<IR::OP_F80ATAN> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FATAN(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FATAN(&State.State, Src1, Src2);
   }
 };
 
@@ -276,8 +315,8 @@ template<>
 struct OpHandlers<IR::OP_F80FPREM1> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FREM1(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FREM1(&State.State, Src1, Src2);
   }
 };
 
@@ -285,8 +324,8 @@ template<>
 struct OpHandlers<IR::OP_F80FPREM> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FREM(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FREM(&State.State, Src1, Src2);
   }
 };
 
@@ -294,14 +333,14 @@ template<>
 struct OpHandlers<IR::OP_F80SCALE> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FSCALE(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FSCALE(&State.State, Src1, Src2);
   }
 };
 
 template<>
 struct OpHandlers<IR::OP_F64SIN> {
-  FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
+  FEXCORE_PRESERVE_ALL_ATTR static double handle(double src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
     return sin(src);
   }
@@ -309,15 +348,30 @@ struct OpHandlers<IR::OP_F64SIN> {
 
 template<>
 struct OpHandlers<IR::OP_F64COS> {
-  FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
+  FEXCORE_PRESERVE_ALL_ATTR static double handle(double src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
     return cos(src);
   }
 };
 
+template<>
+struct OpHandlers<IR::OP_F64SINCOS> {
+  FEXCORE_PRESERVE_ALL_ATTR static VectorScalarF64Pair handle(double src, FEXCore::Core::CpuStateFrame* Frame) {
+    FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
+    double sin, cos;
+#ifdef _WIN32
+    sin = ::sin(src);
+    cos = ::cos(src);
+#else
+    sincos(src, &sin, &cos);
+#endif
+    return VectorScalarF64Pair {sin, cos};
+  }
+};
+
 template<>
 struct OpHandlers<IR::OP_F64TAN> {
-  FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
+  FEXCORE_PRESERVE_ALL_ATTR static double handle(double src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
     return tan(src);
   }
@@ -325,7 +379,7 @@ struct OpHandlers<IR::OP_F64TAN> {
 
 template<>
 struct OpHandlers<IR::OP_F64F2XM1> {
-  FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
+  FEXCORE_PRESERVE_ALL_ATTR static double handle(double src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
     return exp2(src) - 1.0;
   }
@@ -333,7 +387,7 @@ struct OpHandlers<IR::OP_F64F2XM1> {
 
 template<>
 struct OpHandlers<IR::OP_F64ATAN> {
-  FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
+  FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
     return atan2(src1, src2);
   }
@@ -341,7 +395,7 @@ struct OpHandlers<IR::OP_F64ATAN> {
 
 template<>
 struct OpHandlers<IR::OP_F64FPREM> {
-  FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
+  FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
     return fmod(src1, src2);
   }
@@ -349,7 +403,7 @@ struct OpHandlers<IR::OP_F64FPREM> {
 
 template<>
 struct OpHandlers<IR::OP_F64FPREM1> {
-  FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
+  FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
     return remainder(src1, src2);
   }
@@ -357,7 +411,7 @@ struct OpHandlers<IR::OP_F64FPREM1> {
 
 template<>
 struct OpHandlers<IR::OP_F64FYL2X> {
-  FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
+  FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
     return src2 * log2(src1);
   }
@@ -365,7 +419,7 @@ struct OpHandlers<IR::OP_F64FYL2X> {
 
 template<>
 struct OpHandlers<IR::OP_F64SCALE> {
-  FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
+  FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
     if (src1 == 0.0) { // src1 might be +/- zero
       return src1;     // this will return negative or positive zero if when appropriate
@@ -380,18 +434,17 @@ struct OpHandlers<IR::OP_F80BCDSTORE> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1q, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
     X80SoftFloat Src1 = Src1q;
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
+    ScopedSoftFloatState State {FCW, Frame};
     bool Negative = Src1.Sign;
 
-    Src1 = X80SoftFloat::FRNDINT(&State, Src1);
+    Src1 = X80SoftFloat::FRNDINT(&State.State, Src1);
 
     // Clear the Sign bit
     Src1.Sign = 0;
 
-    uint64_t Tmp = Src1.ToI64(&State);
+    uint64_t Tmp = Src1.ToI64(&State.State);
     X80SoftFloat Rv;
     uint8_t* BCD = reinterpret_cast<uint8_t*>(&Rv);
-    memset(BCD, 0, 10);
 
     for (size_t i = 0; i < 9; ++i) {
       if (Tmp == 0) {

FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp

@@ -50,6 +50,8 @@ void InterpreterOps::FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint
   Info[Core::OPINDEX_F80SQRT] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F80SQRT>::handle)};
   Info[Core::OPINDEX_F80SIN] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F80SIN>::handle)};
   Info[Core::OPINDEX_F80COS] = {ABIHandlers[FABI_F80_I16_F80_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F80COS>::handle)};
+  Info[Core::OPINDEX_F80SINCOS] = {ABIHandlers[FABI_F80x2_I16_F80_PTR],
+                                   reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F80SINCOS>::handle)};
   Info[Core::OPINDEX_F80XTRACT_EXP] = {ABIHandlers[FABI_F80_I16_F80_PTR],
                                        reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F80XTRACT_EXP>::handle)};
   Info[Core::OPINDEX_F80XTRACT_SIG] = {ABIHandlers[FABI_F80_I16_F80_PTR],
@@ -80,22 +82,24 @@ void InterpreterOps::FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint
                                   reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F80SCALE>::handle)};
 
   // Double Precision Unary
-  Info[Core::OPINDEX_F64SIN] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64SIN>::handle)};
-  Info[Core::OPINDEX_F64COS] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64COS>::handle)};
-  Info[Core::OPINDEX_F64TAN] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64TAN>::handle)};
-  Info[Core::OPINDEX_F64F2XM1] = {ABIHandlers[FABI_F64_I16_F64_PTR],
+  Info[Core::OPINDEX_F64SIN] = {ABIHandlers[FABI_F64_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64SIN>::handle)};
+  Info[Core::OPINDEX_F64COS] = {ABIHandlers[FABI_F64_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64COS>::handle)};
+  Info[Core::OPINDEX_F64SINCOS] = {ABIHandlers[FABI_F64x2_F64_PTR],
+                                   reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64SINCOS>::handle)};
+  Info[Core::OPINDEX_F64TAN] = {ABIHandlers[FABI_F64_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64TAN>::handle)};
+  Info[Core::OPINDEX_F64F2XM1] = {ABIHandlers[FABI_F64_F64_PTR],
                                   reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64F2XM1>::handle)};
 
   // Double Precision Binary
-  Info[Core::OPINDEX_F64ATAN] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR],
+  Info[Core::OPINDEX_F64ATAN] = {ABIHandlers[FABI_F64_F64_F64_PTR],
                                  reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64ATAN>::handle)};
-  Info[Core::OPINDEX_F64FPREM] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR],
+  Info[Core::OPINDEX_F64FPREM] = {ABIHandlers[FABI_F64_F64_F64_PTR],
                                   reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64FPREM>::handle)};
-  Info[Core::OPINDEX_F64FPREM1] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR],
+  Info[Core::OPINDEX_F64FPREM1] = {ABIHandlers[FABI_F64_F64_F64_PTR],
                                    reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64FPREM1>::handle)};
-  Info[Core::OPINDEX_F64FYL2X] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR],
+  Info[Core::OPINDEX_F64FYL2X] = {ABIHandlers[FABI_F64_F64_F64_PTR],
                                   reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64FYL2X>::handle)};
-  Info[Core::OPINDEX_F64SCALE] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR],
+  Info[Core::OPINDEX_F64SCALE] = {ABIHandlers[FABI_F64_F64_F64_PTR],
                                   reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64SCALE>::handle)};
 
   // SSE4.2 string instructions
@@ -198,6 +202,12 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf
     return true;                                           \
   }
 
+#define COMMON_UNARYPAIR_X87_OP(OP)                          \
+  case IR::OP_F80##OP: {                                     \
+    *Info = {FABI_F80x2_I16_F80_PTR, Core::OPINDEX_F80##OP}; \
+    return true;                                             \
+  }
+
 #define COMMON_BINARY_X87_OP(OP)                               \
   case IR::OP_F80##OP: {                                       \
     *Info = {FABI_F80_I16_F80_F80_PTR, Core::OPINDEX_F80##OP}; \
@@ -212,12 +222,18 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf
 
 #define COMMON_UNARY_F64_OP(OP)                            \
   case IR::OP_F64##OP: {                                   \
-    *Info = {FABI_F64_I16_F64_PTR, Core::OPINDEX_F64##OP}; \
+    *Info = {FABI_F64_F64_PTR, Core::OPINDEX_F64##OP}; \
     return true;                                           \
   }
+#define COMMON_UNARYPAIR_F64_OP(OP)                          \
+  case IR::OP_F64##OP: {                                     \
+    *Info = {FABI_F64x2_F64_PTR, Core::OPINDEX_F64##OP}; \
+    return true;                                             \
+  }
+
 #define COMMON_BINARY_F64_OP(OP)                               \
   case IR::OP_F64##OP: {                                       \
-    *Info = {FABI_F64_I16_F64_F64_PTR, Core::OPINDEX_F64##OP}; \
+    *Info = {FABI_F64_F64_F64_PTR, Core::OPINDEX_F64##OP}; \
     return true;                                               \
   }
 
@@ -228,6 +244,7 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf
     COMMON_UNARY_X87_OP(SQRT)
     COMMON_UNARY_X87_OP(SIN)
     COMMON_UNARY_X87_OP(COS)
+    COMMON_UNARYPAIR_X87_OP(SINCOS)
     COMMON_UNARY_X87_OP(XTRACT_EXP)
     COMMON_UNARY_X87_OP(XTRACT_SIG)
     COMMON_UNARY_X87_OP(BCDSTORE)
@@ -249,6 +266,7 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf
     COMMON_UNARY_F64_OP(TAN)
     COMMON_UNARY_F64_OP(SIN)
     COMMON_UNARY_F64_OP(COS)
+    COMMON_UNARYPAIR_F64_OP(SINCOS)
 
     // Double Precision Binary
     COMMON_BINARY_F64_OP(FYL2X)

FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h

@@ -19,14 +19,16 @@ enum FallbackABI {
   FABI_F80_I16_I32_PTR,
   FABI_F32_I16_F80_PTR,
   FABI_F64_I16_F80_PTR,
-  FABI_F64_I16_F64_PTR,
-  FABI_F64_I16_F64_F64_PTR,
+  FABI_F64_F64_PTR,
+  FABI_F64_F64_F64_PTR,
   FABI_I16_I16_F80_PTR,
   FABI_I32_I16_F80_PTR,
   FABI_I64_I16_F80_PTR,
   FABI_I64_I16_F80_F80_PTR,
   FABI_F80_I16_F80_PTR,
   FABI_F80_I16_F80_F80_PTR,
+  FABI_F80x2_I16_F80_PTR,
+  FABI_F64x2_F64_PTR,
   FABI_I32_I64_I64_V128_V128_I16,
   FABI_I32_V128_V128_I16,
   FABI_UNKNOWN,

FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp

@@ -33,7 +33,7 @@ void Arm64JITCore::InsertNamedThunkRelocation(ARMEmitter::Register Reg, const IR
 
   uint64_t Pointer = reinterpret_cast<uint64_t>(EmitterCTX->ThunkHandler->LookupThunk(Sum));
 
-  LoadConstant(ARMEmitter::Size::i64Bit, Reg, Pointer, EmitterCTX->Config.CacheObjectCodeCompilation());
+  LoadConstant(ARMEmitter::Size::i64Bit, Reg, Pointer, false);
   Relocations.emplace_back(MoveABI);
 }
 
@@ -77,7 +77,7 @@ void Arm64JITCore::InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constan
   MoveABI.GuestRIPMove.GuestRIP = Constant;
   MoveABI.GuestRIPMove.RegisterIndex = Reg.Idx();
 
-  LoadConstant(ARMEmitter::Size::i64Bit, Reg, Constant, EmitterCTX->Config.CacheObjectCodeCompilation());
+  LoadConstant(ARMEmitter::Size::i64Bit, Reg, Constant, false);
   Relocations.emplace_back(MoveABI);
 }
 

FEXCore/Source/Interface/Core/JIT/AtomicOps.cpp

@@ -10,18 +10,17 @@ $end_info$
 #include "Interface/Core/JIT/JITClass.h"
 
 namespace FEXCore::CPU {
-#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const* IROp, IR::NodeID Node)
 DEF_OP(CASPair) {
   auto Op = IROp->C<IR::IROp_CASPair>();
   LOGMAN_THROW_A_FMT(IROp->ElementSize == IR::OpSize::i32Bit || IROp->ElementSize == IR::OpSize::i64Bit, "Wrong element size");
   // Size is the size of each pair element
-  auto Dst0 = GetReg(Op->OutLo.ID());
-  auto Dst1 = GetReg(Op->OutHi.ID());
-  auto Expected0 = GetReg(Op->ExpectedLo.ID());
-  auto Expected1 = GetReg(Op->ExpectedHi.ID());
-  auto Desired0 = GetReg(Op->DesiredLo.ID());
-  auto Desired1 = GetReg(Op->DesiredHi.ID());
-  auto MemSrc = GetReg(Op->Addr.ID());
+  auto Dst0 = GetReg(Op->OutLo);
+  auto Dst1 = GetReg(Op->OutHi);
+  auto Expected0 = GetReg(Op->ExpectedLo);
+  auto Expected1 = GetReg(Op->ExpectedHi);
+  auto Desired0 = GetReg(Op->DesiredLo);
+  auto Desired1 = GetReg(Op->DesiredHi);
+  auto MemSrc = GetReg(Op->Addr);
 
   const auto EmitSize = IROp->ElementSize == IR::OpSize::i64Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;
   if (CTX->HostFeatures.SupportsAtomics) {
@@ -98,9 +97,9 @@ DEF_OP(CAS) {
   // if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc
   // This will write to memory! Careful!
 
-  auto Expected = GetReg(Op->Expected.ID());
-  auto Desired = GetReg(Op->Desired.ID());
-  auto MemSrc = GetReg(Op->Addr.ID());
+  auto Expected = GetReg(Op->Expected);
+  auto Desired = GetReg(Op->Desired);
+  auto MemSrc = GetReg(Op->Addr);
   auto Dst = GetReg(Node);
 
   if (CTX->HostFeatures.SupportsAtomics) {
@@ -139,115 +138,13 @@ DEF_OP(CAS) {
   }
 }
 
-DEF_OP(AtomicAdd) {
-  auto Op = IROp->C<IR::IROp_AtomicAdd>();
-  const auto EmitSize = ConvertSize(IROp);
-  const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
-
-  auto MemSrc = GetReg(Op->Addr.ID());
-  auto Src = GetReg(Op->Value.ID());
-
-  if (CTX->HostFeatures.SupportsAtomics) {
-    staddl(SubEmitSize, Src, MemSrc);
-  } else {
-    ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
-    ldaxr(SubEmitSize, TMP2, MemSrc);
-    add(EmitSize, TMP2, TMP2, Src);
-    stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
-  }
-}
-
-DEF_OP(AtomicSub) {
-  auto Op = IROp->C<IR::IROp_AtomicSub>();
-  const auto EmitSize = ConvertSize(IROp);
-  const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
-
-  auto MemSrc = GetReg(Op->Addr.ID());
-  auto Src = GetReg(Op->Value.ID());
-
-  if (CTX->HostFeatures.SupportsAtomics) {
-    neg(EmitSize, TMP2, Src);
-    staddl(SubEmitSize, TMP2, MemSrc);
-  } else {
-    ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
-    ldaxr(SubEmitSize, TMP2, MemSrc);
-    sub(EmitSize, TMP2, TMP2, Src);
-    stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
-  }
-}
-
-DEF_OP(AtomicAnd) {
-  auto Op = IROp->C<IR::IROp_AtomicAnd>();
-  const auto EmitSize = ConvertSize(IROp);
-  const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
-
-  auto MemSrc = GetReg(Op->Addr.ID());
-  auto Src = GetReg(Op->Value.ID());
-
-  if (CTX->HostFeatures.SupportsAtomics) {
-    mvn(EmitSize, TMP2, Src);
-    stclrl(SubEmitSize, TMP2, MemSrc);
-  } else {
-    ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
-    ldaxr(SubEmitSize, TMP2, MemSrc);
-    and_(EmitSize, TMP2, TMP2, Src);
-    stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
-  }
-}
-
-DEF_OP(AtomicCLR) {
-  auto Op = IROp->C<IR::IROp_AtomicCLR>();
-  const auto EmitSize = ConvertSize(IROp);
-  const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
-
-  auto MemSrc = GetReg(Op->Addr.ID());
-  auto Src = GetReg(Op->Value.ID());
-
-  if (CTX->HostFeatures.SupportsAtomics) {
-    stclrl(SubEmitSize, Src, MemSrc);
-  } else {
-    ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
-    ldaxr(SubEmitSize, TMP2, MemSrc);
-    bic(EmitSize, TMP2, TMP2, Src);
-    stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
-  }
-}
-
-DEF_OP(AtomicOr) {
-  auto Op = IROp->C<IR::IROp_AtomicOr>();
-  const auto EmitSize = ConvertSize(IROp);
-  const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
-
-  auto MemSrc = GetReg(Op->Addr.ID());
-  auto Src = GetReg(Op->Value.ID());
-
-  if (CTX->HostFeatures.SupportsAtomics) {
-    stsetl(SubEmitSize, Src, MemSrc);
-  } else {
-    ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
-    ldaxr(SubEmitSize, TMP2, MemSrc);
-    orr(EmitSize, TMP2, TMP2, Src);
-    stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
-  }
-}
-
 DEF_OP(AtomicXor) {
   auto Op = IROp->C<IR::IROp_AtomicXor>();
   const auto EmitSize = ConvertSize(IROp);
   const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
 
-  auto MemSrc = GetReg(Op->Addr.ID());
-  auto Src = GetReg(Op->Value.ID());
+  auto MemSrc = GetReg(Op->Addr);
+  auto Src = GetReg(Op->Value);
 
   if (CTX->HostFeatures.SupportsAtomics) {
     steorl(SubEmitSize, Src, MemSrc);
@@ -261,21 +158,6 @@ DEF_OP(AtomicXor) {
   }
 }
 
-DEF_OP(AtomicNeg) {
-  auto Op = IROp->C<IR::IROp_AtomicNeg>();
-  const auto EmitSize = ConvertSize(IROp);
-  const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
-
-  auto MemSrc = GetReg(Op->Addr.ID());
-
-  ARMEmitter::BackwardLabel LoopTop;
-  Bind(&LoopTop);
-  ldaxr(SubEmitSize, TMP2, MemSrc);
-  neg(EmitSize, TMP3, TMP2);
-  stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-  cbnz(EmitSize, TMP4, &LoopTop);
-}
-
 DEF_OP(AtomicSwap) {
   auto Op = IROp->C<IR::IROp_AtomicSwap>();
   const auto OpSize = IROp->Size;
@@ -284,8 +166,8 @@ DEF_OP(AtomicSwap) {
                                                                                                                                  "d CAS "
                                                                                                                                  "size");
 
-  auto MemSrc = GetReg(Op->Addr.ID());
-  auto Src = GetReg(Op->Value.ID());
+  auto MemSrc = GetReg(Op->Addr);
+  auto Src = GetReg(Op->Value);
 
   const auto EmitSize = ConvertSize(IROp);
   const auto SubEmitSize = OpSize == IR::OpSize::i64Bit ? ARMEmitter::SubRegSize::i64Bit :
@@ -310,8 +192,8 @@ DEF_OP(AtomicFetchAdd) {
   const auto EmitSize = ConvertSize(IROp);
   const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
 
-  auto MemSrc = GetReg(Op->Addr.ID());
-  auto Src = GetReg(Op->Value.ID());
+  auto MemSrc = GetReg(Op->Addr);
+  auto Src = GetReg(Op->Value);
 
   if (CTX->HostFeatures.SupportsAtomics) {
     ldaddal(SubEmitSize, Src, GetReg(Node), MemSrc);
@@ -331,8 +213,8 @@ DEF_OP(AtomicFetchSub) {
   const auto EmitSize = ConvertSize(IROp);
   const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
 
-  auto MemSrc = GetReg(Op->Addr.ID());
-  auto Src = GetReg(Op->Value.ID());
+  auto MemSrc = GetReg(Op->Addr);
+  auto Src = GetReg(Op->Value);
 
   if (CTX->HostFeatures.SupportsAtomics) {
     neg(EmitSize, TMP2, Src);
@@ -353,8 +235,8 @@ DEF_OP(AtomicFetchAnd) {
   const auto EmitSize = ConvertSize(IROp);
   const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
 
-  auto MemSrc = GetReg(Op->Addr.ID());
-  auto Src = GetReg(Op->Value.ID());
+  auto MemSrc = GetReg(Op->Addr);
+  auto Src = GetReg(Op->Value);
 
   if (CTX->HostFeatures.SupportsAtomics) {
     mvn(EmitSize, TMP2, Src);
@@ -375,8 +257,8 @@ DEF_OP(AtomicFetchCLR) {
   const auto EmitSize = ConvertSize(IROp);
   const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
 
-  auto MemSrc = GetReg(Op->Addr.ID());
-  auto Src = GetReg(Op->Value.ID());
+  auto MemSrc = GetReg(Op->Addr);
+  auto Src = GetReg(Op->Value);
 
   if (CTX->HostFeatures.SupportsAtomics) {
     ldclral(SubEmitSize, Src, GetReg(Node), MemSrc);
@@ -396,8 +278,8 @@ DEF_OP(AtomicFetchOr) {
   const auto EmitSize = ConvertSize(IROp);
   const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
 
-  auto MemSrc = GetReg(Op->Addr.ID());
-  auto Src = GetReg(Op->Value.ID());
+  auto MemSrc = GetReg(Op->Addr);
+  auto Src = GetReg(Op->Value);
 
   if (CTX->HostFeatures.SupportsAtomics) {
     ldsetal(SubEmitSize, Src, GetReg(Node), MemSrc);
@@ -417,8 +299,8 @@ DEF_OP(AtomicFetchXor) {
   const auto EmitSize = ConvertSize(IROp);
   const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
 
-  auto MemSrc = GetReg(Op->Addr.ID());
-  auto Src = GetReg(Op->Value.ID());
+  auto MemSrc = GetReg(Op->Addr);
+  auto Src = GetReg(Op->Value);
 
   if (CTX->HostFeatures.SupportsAtomics) {
     ldeoral(SubEmitSize, Src, GetReg(Node), MemSrc);
@@ -438,21 +320,34 @@ DEF_OP(AtomicFetchNeg) {
   const auto EmitSize = ConvertSize(IROp);
   const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
 
-  auto MemSrc = GetReg(Op->Addr.ID());
+  auto MemSrc = GetReg(Op->Addr);
 
-  ARMEmitter::BackwardLabel LoopTop;
-  Bind(&LoopTop);
-  ldaxr(SubEmitSize, TMP2, MemSrc);
-  neg(EmitSize, TMP3, TMP2);
-  stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-  cbnz(EmitSize, TMP4, &LoopTop);
-  mov(EmitSize, GetReg(Node), TMP2.R());
+  if (CTX->HostFeatures.SupportsAtomics) {
+    // Use a CAS loop to avoid needing to emulate unaligned LLSC atomics
+    ldr(SubEmitSize, TMP2, MemSrc);
+    ARMEmitter::BackwardLabel LoopTop;
+    Bind(&LoopTop);
+    mov(EmitSize, TMP4, TMP2);
+    neg(EmitSize, TMP3, TMP2);
+    casal(SubEmitSize, TMP2, TMP3, MemSrc);
+    sub(EmitSize, TMP3, TMP2, TMP4);
+    cbnz(EmitSize, TMP3, &LoopTop);
+    mov(EmitSize, GetReg(Node), TMP2.R());
+  } else {
+    ARMEmitter::BackwardLabel LoopTop;
+    Bind(&LoopTop);
+    ldaxr(SubEmitSize, TMP2, MemSrc);
+    neg(EmitSize, TMP3, TMP2);
+    stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
+    cbnz(EmitSize, TMP4, &LoopTop);
+    mov(EmitSize, GetReg(Node), TMP2.R());
+  }
 }
 
 DEF_OP(TelemetrySetValue) {
 #ifndef FEX_DISABLE_TELEMETRY
   auto Op = IROp->C<IR::IROp_TelemetrySetValue>();
-  auto Src = GetReg(Op->Value.ID());
+  auto Src = GetReg(Op->Value);
 
   ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.TelemetryValueAddresses[Op->TelemetryValueIndex]));
 
@@ -473,5 +368,4 @@ DEF_OP(TelemetrySetValue) {
 #endif
 }
 
-#undef DEF_OP
 } // namespace FEXCore::CPU

FEXCore/Source/Interface/Core/JIT/BranchOps.cpp

@@ -18,7 +18,6 @@ $end_info$
 #include <FEXCore/Utils/MathUtils.h>
 
 namespace FEXCore::CPU {
-#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const* IROp, IR::NodeID Node)
 
 DEF_OP(CallbackReturn) {
   // spill back to CTX
@@ -59,30 +58,120 @@ DEF_OP(ExitFunction) {
   if (IsInlineConstant(Op->NewRIP, &NewRIP) || IsInlineEntrypointOffset(Op->NewRIP, &NewRIP)) {
 #ifdef _M_ARM_64EC
     if (NewRIP < EC_CODE_BITMAP_MAX_ADDRESS && RtlIsEcCode(NewRIP)) {
+      str(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));
       add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0);
       LoadConstant(ARMEmitter::Size::i64Bit, EC_CALL_CHECKER_PC_REG, NewRIP);
       ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionEC));
       br(TMP2);
     } else {
 #endif
-      // Align to 16 byte to allow atomic patching of the following 16 byte
-      // of code (excluding the RIP data) on platforms that support LSE2
-      Align16B();
+      // In order to support direct branches without constantly hitting the L1 cache, we emit a call to a block linker,
+      // this will compile the branch target block when it is hit and replace the branch to the linker at the callsite
+      // with a direct branch to the destination block. Upon invalidation of the target block the backpatch is undone.
+      //
+      // In addition, to avoid needing to lookup in the cache for returns and any indirect branch prediction penalty,
+      // a shadow stack of <GuestReturnRIP, HostReturnPC> pairs is maintained, acting as a first level cache for any
+      // return operations. As the guest may not balance calls and returns exactly, an exception handler is expected to
+      // be installed by the frontend, to reset the shadow stack to the middle of its valid bounds on overflow/underflow.
+      // This shadow stack is also cleared on block invalidation operations or codebuffer switches, to ensure all pointed-to
+      // host code is always valid.
+
+      // This code will be backpatched by Arm64JITCore_ExitFunctionLink, below is an enumeration of all the possible cases.
+      // Jump thunks are emitted in JIT.cpp after compilation of the entire multiblock.
+      //
+      // Call with known return block - unlinked
+      //    00: adr TMP1, 0xC
+      //    04: stp RetReg, TMP1, [SpReg, -0x10]!
+      //    08: bl JmpThunk00
+      //    JmpThunk00:
+      //    00: b 0x8
+      //    04: br TMP1
+      //    08: ldr TMP1, <Shared exit linker>
+      //    0c: blr TMP1
+      //    10: HostCode
+      //    18: GuestRIP
+      //    20: CallerOffset
+      //
+      // Call with known return block after backpatching - linked in branch immediate range
+      //    00: adr TMP1, 0xC
+      //    04: stp RetReg, TMP1, [SpReg, -0x10]!
+      //    08: bl HostCode                                        - MODIFIED
+      //
+      // Call with known return block after backpatching - linked out of range
+      //    00: adr TMP1, 0xC
+      //    04: stp RetReg, TMP1, [SpReg, -0x10]!
+      //    08: bl JmpThunk00
+      //    JmpThunk00:
+      //    00: ldr TMP1, 0x10                                     - MODIFIED 2nd
+      //    04: br TMP1
+      //    08: ldr TMP1, <Shared exit linker>
+      //    0c: blr TMP1
+      //    10: HostCode                                           - MODIFIED 1st
+      //    18: GuestRIP
+      //    20: CallerOffset
+      //
+      // Jump - unlinked
+      //    00: b JmpThunk00
+      //    JmpThunk00:
+      //    00: b 0x8
+      //    04: br TMP1
+      //    08: ldr TMP1, <Shared exit linker>
+      //    0c: blr TMP1
+      //    10: HostCode
+      //    18: GuestRIP
+      //    20: CallerOffset
+      //
+      // Jump after backpatching - linked in branch immediate range
+      //    00: b HostCode                                         - MODIFIED
+      //
+      // Jump after backpatching - linked out of range
+      //    00: b JmpThunk00
+      //    JmpThunk00:
+      //    00: ldr TMP1, 0x10                                     - MODIFIED 2nd
+      //    04: br TMP1
+      //    08: ldr TMP1, <Shared exit linker>
+      //    0c: blr TMP1
+      //    10: HostCode                                           - MODIFIED 1st
+      //    18: GuestRIP
+      //    20: CallerOffset
 
       ARMEmitter::ForwardLabel l_BranchHost;
-      ldr(TMP1, &l_BranchHost);
-      blr(TMP1);
+      ARMEmitter::ForwardLabel l_CallReturn;
+      if (Op->Hint == IR::BranchHint::Call) {
+        if (!Op->CallReturnBlock.IsInvalid()) {
+          auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();
+          PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;
+          adr(TMP1, &l_CallReturn);
+          stp<ARMEmitter::IndexType::PRE>(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);
+        } else {
+          stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
+        }
+      } else if (Op->Hint == IR::BranchHint::CheckTF) {
+        ARMEmitter::ForwardLabel TFUnset;
+        ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
+        cbz(ARMEmitter::Size::i32Bit, TMP1, &TFUnset);
+        LoadConstant(ARMEmitter::Size::i64Bit, TMP1, NewRIP);
+        str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
+        ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop));
+        blr(TMP2);
+        Bind(&TFUnset);
+      }
 
-      Bind(&l_BranchHost);
-      dc64(ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker);
-      dc64(NewRIP);
+      EmitLinkedBranch(NewRIP, Op->Hint == IR::BranchHint::Call);
+      Bind(&l_CallReturn);
 #ifdef _M_ARM_64EC
     }
 #endif
   } else {
+    ARMEmitter::ForwardLabel SkipFullLookup;
+    auto RipReg = GetReg(Op->NewRIP);
 
-    ARMEmitter::ForwardLabel FullLookup;
-    auto RipReg = GetReg(Op->NewRIP.ID());
+    if (Op->Hint == IR::BranchHint::Return) {
+      // First try to pop from the call-ret stack, otherwise follow the normal path (but ending in a ret)
+      ldp<ARMEmitter::IndexType::POST>(TMP1, TMP2, REG_CALLRET_SP, 0x10);
+      sub(TMP1, TMP1, RipReg.X());
+      cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
+    }
 
     // L1 Cache
     ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.L1Pointer));
@@ -94,41 +183,56 @@ DEF_OP(ExitFunction) {
     ubfiz(ARMEmitter::Size::i64Bit, TMP4, RipReg, 4, 20);
     add(TMP1, TMP1, TMP4);
 
-    // Note: sub+cbnz used over cmp+br to preserve flags.
     ldp<ARMEmitter::IndexType::OFFSET>(TMP2, TMP1, TMP1, 0);
-    sub(TMP1, TMP1, RipReg.X());
-    cbnz(ARMEmitter::Size::i64Bit, TMP1, &FullLookup);
-    br(TMP2);
 
-    Bind(&FullLookup);
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop));
+    // Note: sub+cbnz used over cmp+br to preserve flags.
+    sub(TMP1, TMP1, RipReg.X());
+    cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
+    ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop));
     str(RipReg.X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
-    br(TMP1);
+
+    Bind(&SkipFullLookup);
+    if (Op->Hint == IR::BranchHint::Call) {
+      ARMEmitter::ForwardLabel l_CallReturn;
+      if (!Op->CallReturnBlock.IsInvalid()) {
+        auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();
+        PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;
+        adr(TMP1, &l_CallReturn);
+        stp<ARMEmitter::IndexType::PRE>(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);
+      } else {
+        stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
+      }
+      blr(TMP2);
+      Bind(&l_CallReturn);
+    } else if (Op->Hint == IR::BranchHint::Return) {
+      ret(TMP2);
+    } else {
+      br(TMP2);
+    }
   }
 }
 
 DEF_OP(Jump) {
   const auto Op = IROp->C<IR::IROp_Jump>();
-  const auto Target = Op->TargetBlock.ID();
 
-  PendingTargetLabel = &JumpTargets.try_emplace(Target).first->second;
+  PendingTargetLabel = JumpTarget(Op->TargetBlock);
 }
 
 DEF_OP(CondJump) {
   auto Op = IROp->C<IR::IROp_CondJump>();
 
-  auto TrueTargetLabel = &JumpTargets.try_emplace(Op->TrueBlock.ID()).first->second;
+  auto TrueTargetLabel = JumpTarget(Op->TrueBlock);
 
   if (Op->FromNZCV) {
     b(MapCC(Op->Cond), TrueTargetLabel);
   } else {
-    [[maybe_unused]] uint64_t Const;
-    [[maybe_unused]] const bool isConst = IsInlineConstant(Op->Cmp2, &Const);
+    uint64_t Const;
+    const bool isConst = IsInlineConstant(Op->Cmp2, &Const);
 
-    auto Reg = GetReg(Op->Cmp1.ID());
+    auto Reg = GetReg(Op->Cmp1);
     const auto Size = Op->CompareSize == IR::OpSize::i32Bit ? ARMEmitter::Size::i32Bit : ARMEmitter::Size::i64Bit;
 
-    LOGMAN_THROW_A_FMT(IsGPR(Op->Cmp1.ID()), "CondJump: Expected GPR");
+    LOGMAN_THROW_A_FMT(IsGPR(Op->Cmp1), "CondJump: Expected GPR");
     LOGMAN_THROW_A_FMT(isConst, "CondJump: Expected constant source");
 
     if (Op->Cond.Val == FEXCore::IR::COND_EQ) {
@@ -148,7 +252,7 @@ DEF_OP(CondJump) {
     }
   }
 
-  PendingTargetLabel = &JumpTargets.try_emplace(Op->FalseBlock.ID()).first->second;
+  PendingTargetLabel = JumpTarget(Op->FalseBlock);
 }
 
 DEF_OP(Syscall) {
@@ -184,7 +288,7 @@ DEF_OP(Syscall) {
     if (Op->Header.Args[i].IsInvalid()) {
       continue;
     }
-    str(GetReg(Op->Header.Args[i].ID()).X(), ARMEmitter::Reg::rsp, i * 8);
+    str(GetReg(Op->Header.Args[i]).X(), ARMEmitter::Reg::rsp, i * 8);
   }
 
   ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerObj));
@@ -244,7 +348,7 @@ DEF_OP(InlineSyscall) {
       break;
     }
 
-    auto Reg = GetReg(Op->Header.Args[i].ID());
+    auto Reg = GetReg(Op->Header.Args[i]);
     if (Reg == ARMEmitter::Reg::r8 || Reg == ARMEmitter::Reg::r4 || Reg == ARMEmitter::Reg::r5) {
 
       SpillMask |= (1U << Reg.Idx());
@@ -274,7 +378,7 @@ DEF_OP(InlineSyscall) {
         break;
       }
 
-      auto Reg = GetReg(Op->Header.Args[i].ID());
+      auto Reg = GetReg(Op->Header.Args[i]);
       if (SpillMask & (1U << Reg.Idx())) {
         // In the case of intersection with x4, x5, or x8 then these are currently SRA
         // for registers RAX, RDX, and RSP. Which have just been spilled
@@ -292,7 +396,7 @@ DEF_OP(InlineSyscall) {
         break;
       }
 
-      mov(EmitSize, RegArgs[i].R(), GetReg(Op->Header.Args[i].ID()));
+      mov(EmitSize, RegArgs[i].R(), GetReg(Op->Header.Args[i]));
     }
   }
 
@@ -325,7 +429,7 @@ DEF_OP(Thunk) {
 
   PushDynamicRegs(TMP1);
 
-  mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetReg(Op->ArgPtr.ID()));
+  mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetReg(Op->ArgPtr));
 
   auto thunkFn = static_cast<Context::ContextImpl*>(ThreadState->CTX)->ThunkHandler->LookupThunk(Op->ThunkNameHash);
   LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, (uintptr_t)thunkFn);
@@ -342,48 +446,50 @@ DEF_OP(Thunk) {
 
 DEF_OP(ValidateCode) {
   auto Op = IROp->C<IR::IROp_ValidateCode>();
-  const auto* OldCode = (const uint8_t*)&Op->CodeOriginalLow;
+  auto OldCode = Op->CodeOriginal.data();
+  auto Base = GetReg(Op->Header.Args[0]).X();
   int len = Op->CodeLength;
-  int idx = 0;
-
-  LoadConstant(ARMEmitter::Size::i64Bit, GetReg(Node), 0);
-  LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Entry + Op->Offset);
-  LoadConstant(ARMEmitter::Size::i64Bit, TMP2, 1);
+  int Offset = 0;
+  ARMEmitter::ForwardLabel Fail;
 
   const auto Dst = GetReg(Node);
 
-  while (len >= 8) {
-    ldr(ARMEmitter::XReg::x2, TMP1, idx);
-    LoadConstant(ARMEmitter::Size::i64Bit, TMP4, *(const uint64_t*)(OldCode + idx));
-    cmp(ARMEmitter::Size::i64Bit, TMP3, TMP4);
-    csel(ARMEmitter::Size::i64Bit, Dst, Dst, TMP2, ARMEmitter::Condition::CC_EQ);
-    len -= 8;
-    idx += 8;
-  }
-  while (len >= 4) {
-    ldr(ARMEmitter::WReg::w2, TMP1, idx);
-    LoadConstant(ARMEmitter::Size::i64Bit, TMP4, *(const uint32_t*)(OldCode + idx));
-    cmp(ARMEmitter::Size::i32Bit, TMP3, TMP4);
-    csel(ARMEmitter::Size::i64Bit, Dst, Dst, TMP2, ARMEmitter::Condition::CC_EQ);
-    len -= 4;
-    idx += 4;
-  }
-  while (len >= 2) {
-    ldrh(TMP3, TMP1, idx);
-    LoadConstant(ARMEmitter::Size::i64Bit, TMP4, *(const uint16_t*)(OldCode + idx));
-    cmp(ARMEmitter::Size::i32Bit, TMP3, TMP4);
-    csel(ARMEmitter::Size::i64Bit, Dst, Dst, TMP2, ARMEmitter::Condition::CC_EQ);
-    len -= 2;
-    idx += 2;
-  }
-  while (len >= 1) {
-    ldrb(TMP3, TMP1, idx);
-    LoadConstant(ARMEmitter::Size::i64Bit, TMP4, *(const uint8_t*)(OldCode + idx));
-    cmp(ARMEmitter::Size::i32Bit, TMP3, TMP4);
-    csel(ARMEmitter::Size::i64Bit, Dst, Dst, TMP2, ARMEmitter::Condition::CC_EQ);
-    len -= 1;
-    idx += 1;
-  }
+  auto EmitCheck = [&](size_t Size, auto&& LoadData) {
+    while (len >= Size) {
+      LoadData();
+      sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP2);
+      cbnz(ARMEmitter::Size::i64Bit, TMP1, &Fail);
+      len -= Size;
+      Offset += Size;
+    }
+  };
+
+  EmitCheck(8, [&]() {
+    ldr(TMP1, Base, Offset);
+    LoadConstant(ARMEmitter::Size::i64Bit, TMP2, *(const uint64_t*)(OldCode + Offset));
+  });
+
+  EmitCheck(4, [&]() {
+    ldr(TMP1.W(), Base, Offset);
+    LoadConstant(ARMEmitter::Size::i32Bit, TMP2, *(const uint32_t*)(OldCode + Offset));
+  });
+
+  EmitCheck(2, [&]() {
+    ldrh(TMP1.W(), Base, Offset);
+    LoadConstant(ARMEmitter::Size::i32Bit, TMP2, *(const uint16_t*)(OldCode + Offset));
+  });
+
+  EmitCheck(1, [&]() {
+    ldrb(TMP1.W(), Base, Offset);
+    LoadConstant(ARMEmitter::Size::i32Bit, TMP2, *(const uint8_t*)(OldCode + Offset));
+  });
+
+  ARMEmitter::ForwardLabel End;
+  LoadConstant(ARMEmitter::Size::i32Bit, Dst, 0);
+  b(&End);
+  Bind(&Fail);
+  LoadConstant(ARMEmitter::Size::i32Bit, Dst, 1);
+  Bind(&End);
 }
 
 DEF_OP(ThreadRemoveCodeEntry) {
@@ -412,8 +518,9 @@ DEF_OP(ThreadRemoveCodeEntry) {
 DEF_OP(CPUID) {
   auto Op = IROp->C<IR::IROp_CPUID>();
 
-  mov(ARMEmitter::Size::i64Bit, TMP2, GetReg(Op->Function.ID()));
-  mov(ARMEmitter::Size::i64Bit, TMP3, GetReg(Op->Leaf.ID()));
+  isb();
+  mov(ARMEmitter::Size::i64Bit, TMP2, GetReg(Op->Function));
+  mov(ARMEmitter::Size::i64Bit, TMP3, GetReg(Op->Leaf));
 
   PushDynamicRegs(TMP4);
   SpillStaticRegs(TMP4);
@@ -446,10 +553,10 @@ DEF_OP(CPUID) {
 
   // Results are in x0, x1
   // Results want to be 4xi32 scalars
-  mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutEAX.ID()), TMP1);
-  mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutECX.ID()), TMP2);
-  ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEBX.ID()), TMP1, 32, 32);
-  ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEDX.ID()), TMP2, 32, 32);
+  mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutEAX), TMP1);
+  mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutECX), TMP2);
+  ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEBX), TMP1, 32, 32);
+  ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEDX), TMP2, 32, 32);
 }
 
 DEF_OP(XGetBV) {
@@ -458,7 +565,7 @@ DEF_OP(XGetBV) {
   PushDynamicRegs(TMP4);
   SpillStaticRegs(TMP4);
 
-  mov(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r1, GetReg(Op->Function.ID()));
+  mov(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r1, GetReg(Op->Function));
 
   // x0 = CPUID Handler
   // x1 = XCR Function
@@ -479,9 +586,8 @@ DEF_OP(XGetBV) {
   PopDynamicRegs();
 
   // Results are in x0, need to split into i32 parts
-  mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutEAX.ID()), TMP1);
-  ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEDX.ID()), TMP1, 32, 32);
+  mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutEAX), TMP1);
+  ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEDX), TMP1, 32, 32);
 }
 
-#undef DEF_OP
 } // namespace FEXCore::CPU

FEXCore/Source/Interface/Core/JIT/ConversionOps.cpp

@@ -9,7 +9,6 @@ $end_info$
 #include "Interface/Context/Context.h"
 
 namespace FEXCore::CPU {
-#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const* IROp, IR::NodeID Node)
 DEF_OP(VInsGPR) {
   const auto Op = IROp->C<IR::IROp_VInsGPR>();
   const auto OpSize = IROp->Size;
@@ -23,8 +22,8 @@ DEF_OP(VInsGPR) {
   const auto ElementsPer128Bit = IR::NumElements(IR::OpSize::i128Bit, ElementSize);
 
   const auto Dst = GetVReg(Node);
-  const auto DestVector = GetVReg(Op->DestVector.ID());
-  const auto Src = GetReg(Op->Src.ID());
+  const auto DestVector = GetVReg(Op->DestVector);
+  const auto Src = GetReg(Op->Src);
 
   if (HostSupportsSVE256 && Is256Bit) {
     const auto ElementSizeBits = IR::OpSizeAsBits(ElementSize);
@@ -88,7 +87,7 @@ DEF_OP(VInsGPR) {
 DEF_OP(VCastFromGPR) {
   auto Op = IROp->C<IR::IROp_VCastFromGPR>();
   auto Dst = GetVReg(Node);
-  auto Src = GetReg(Op->Src.ID());
+  auto Src = GetReg(Op->Src);
 
   switch (Op->Header.ElementSize) {
   case IR::OpSize::i8Bit:
@@ -109,8 +108,8 @@ DEF_OP(VLoadTwoGPRs) {
   const auto Op = IROp->C<IR::IROp_VLoadTwoGPRs>();
 
   const auto Dst = GetVReg(Node);
-  const auto SrcLower = GetReg(Op->Lower.ID());
-  const auto SrcUpper = GetReg(Op->Upper.ID());
+  const auto SrcLower = GetReg(Op->Lower);
+  const auto SrcUpper = GetReg(Op->Upper);
   fmov(ARMEmitter::Size::i64Bit, Dst.D(), SrcLower);
   fmov(ARMEmitter::Size::i64Bit, Dst.D(), SrcUpper, true);
 }
@@ -120,7 +119,7 @@ DEF_OP(VDupFromGPR) {
   const auto OpSize = IROp->Size;
 
   const auto Dst = GetVReg(Node);
-  const auto Src = GetReg(Op->Src.ID());
+  const auto Src = GetReg(Op->Src);
 
   const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
   LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
@@ -141,7 +140,7 @@ DEF_OP(Float_FromGPR_S) {
   const uint16_t Conv = (ElementSize << 8) | IR::OpSizeToSize(Op->SrcElementSize);
 
   auto Dst = GetVReg(Node);
-  auto Src = GetReg(Op->Src.ID());
+  auto Src = GetReg(Op->Src);
 
   switch (Conv) {
   case 0x0204: { // Half <- int32_t
@@ -179,7 +178,7 @@ DEF_OP(Float_FToF) {
   const uint16_t Conv = (IR::OpSizeToSize(Op->Header.ElementSize) << 8) | IR::OpSizeToSize(Op->SrcElementSize);
 
   auto Dst = GetVReg(Node);
-  auto Src = GetVReg(Op->Scalar.ID());
+  auto Src = GetVReg(Op->Scalar);
 
   switch (Conv) {
   case 0x0204: { // Half <- Float
@@ -220,7 +219,7 @@ DEF_OP(Vector_SToF) {
   LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
 
   const auto Dst = GetVReg(Node);
-  const auto Vector = GetVReg(Op->Vector.ID());
+  const auto Vector = GetVReg(Op->Vector);
   if (HostSupportsSVE256 && Is256Bit) {
     const auto Mask = PRED_TMP_32B;
     scvtf(Dst.Z(), SubEmitSize, Mask.Merging(), Vector.Z(), SubEmitSize);
@@ -253,7 +252,7 @@ DEF_OP(Vector_FToZS) {
   LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
 
   const auto Dst = GetVReg(Node);
-  const auto Vector = GetVReg(Op->Vector.ID());
+  const auto Vector = GetVReg(Op->Vector);
   if (HostSupportsSVE256 && Is256Bit) {
     const auto Mask = PRED_TMP_32B;
     fcvtzs(Dst.Z(), SubEmitSize, Mask.Merging(), Vector.Z(), SubEmitSize);
@@ -286,7 +285,7 @@ DEF_OP(Vector_FToS) {
   const auto SubEmitSize = ConvertSubRegSize248(IROp);
 
   const auto Dst = GetVReg(Node);
-  const auto Vector = GetVReg(Op->Vector.ID());
+  const auto Vector = GetVReg(Op->Vector);
 
   if (HostSupportsSVE256 && Is256Bit) {
     const auto Mask = PRED_TMP_32B;
@@ -294,7 +293,7 @@ DEF_OP(Vector_FToS) {
     fcvtzs(Dst.Z(), SubEmitSize, Mask.Merging(), Dst.Z(), SubEmitSize);
   } else {
     const auto Dst = GetVReg(Node);
-    const auto Vector = GetVReg(Op->Vector.ID());
+    const auto Vector = GetVReg(Op->Vector);
     if (OpSize == IR::OpSize::i64Bit) {
       frinti(SubEmitSize, Dst.D(), Vector.D());
       fcvtzs(SubEmitSize, Dst.D(), Dst.D());
@@ -317,7 +316,7 @@ DEF_OP(Vector_FToF) {
   const auto Conv = (IR::OpSizeToSize(ElementSize) << 8) | IR::OpSizeToSize(Op->SrcElementSize);
 
   const auto Dst = GetVReg(Node);
-  const auto Vector = GetVReg(Op->Vector.ID());
+  const auto Vector = GetVReg(Op->Vector);
 
   if (HostSupportsSVE256 && Is256Bit) {
     // Curiously, FCVTLT and FCVTNT have no bottom variants,
@@ -381,7 +380,7 @@ DEF_OP(VFCVTL2) {
   const auto SubEmitSize = ConvertSubRegSize248(IROp);
 
   const auto Dst = GetVReg(Node);
-  const auto Vector = GetVReg(Op->Vector.ID());
+  const auto Vector = GetVReg(Op->Vector);
 
   fcvtl2(SubEmitSize, Dst.D(), Vector.D());
 }
@@ -392,8 +391,8 @@ DEF_OP(VFCVTN2) {
   const auto SubEmitSize = ConvertSubRegSize248(IROp);
 
   const auto Dst = GetVReg(Node);
-  const auto VectorLower = GetVReg(Op->VectorLower.ID());
-  const auto VectorUpper = GetVReg(Op->VectorUpper.ID());
+  const auto VectorLower = GetVReg(Op->VectorLower);
+  const auto VectorUpper = GetVReg(Op->VectorUpper);
 
   auto Lower = VectorLower;
   if (Dst != VectorLower) {
@@ -418,7 +417,7 @@ DEF_OP(Vector_FToI) {
   LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
 
   const auto Dst = GetVReg(Node);
-  const auto Vector = GetVReg(Op->Vector.ID());
+  const auto Vector = GetVReg(Op->Vector);
 
   if (HostSupportsSVE256 && Is256Bit) {
     const auto Mask = PRED_TMP_32B.Merging();
@@ -479,7 +478,7 @@ DEF_OP(Vector_FToISized) {
   LOGMAN_THROW_A_FMT(CTX->HostFeatures.SupportsFRINTTS, "Need FRINTTS for Vector_FToISized");
 
   const auto Dst = GetVReg(Node);
-  const auto Vector = GetVReg(Op->Vector.ID());
+  const auto Vector = GetVReg(Op->Vector);
 
   if (ElementSize == IROp->Size) {
 // See above
@@ -533,7 +532,7 @@ DEF_OP(Vector_F64ToI32) {
   LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
 
   const auto Dst = GetVReg(Node);
-  const auto Vector = GetVReg(Op->Vector.ID());
+  const auto Vector = GetVReg(Op->Vector);
   if (HostSupportsSVE128 || HostSupportsSVE256) {
     const auto Mask = Is256Bit ? PRED_TMP_32B.Merging() : PRED_TMP_16B.Merging();
     // First step is to round the f64 values to integrals (frint*)
@@ -583,5 +582,4 @@ DEF_OP(Vector_F64ToI32) {
   }
 }
 
-#undef DEF_OP
 } // namespace FEXCore::CPU

FEXCore/Source/Interface/Core/JIT/EncryptionOps.cpp

@@ -8,21 +8,20 @@ $end_info$
 #include "Interface/Core/JIT/JITClass.h"
 
 namespace FEXCore::CPU {
-#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const* IROp, IR::NodeID Node)
 
 DEF_OP(VAESImc) {
   auto Op = IROp->C<IR::IROp_VAESImc>();
-  aesimc(GetVReg(Node), GetVReg(Op->Vector.ID()));
+  aesimc(GetVReg(Node), GetVReg(Op->Vector));
 }
 
 DEF_OP(VAESEnc) {
   const auto Op = IROp->C<IR::IROp_VAESEnc>();
-  [[maybe_unused]] const auto OpSize = IROp->Size;
+  const auto OpSize = IROp->Size;
 
   const auto Dst = GetVReg(Node);
-  const auto Key = GetVReg(Op->Key.ID());
-  const auto State = GetVReg(Op->State.ID());
-  const auto ZeroReg = GetVReg(Op->ZeroReg.ID());
+  const auto Key = GetVReg(Op->Key);
+  const auto State = GetVReg(Op->State);
+  const auto ZeroReg = GetVReg(Op->ZeroReg);
 
   LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit, "Currently only supports 128-bit operations.");
 
@@ -42,12 +41,12 @@ DEF_OP(VAESEnc) {
 
 DEF_OP(VAESEncLast) {
   const auto Op = IROp->C<IR::IROp_VAESEncLast>();
-  [[maybe_unused]] const auto OpSize = IROp->Size;
+  const auto OpSize = IROp->Size;
 
   const auto Dst = GetVReg(Node);
-  const auto Key = GetVReg(Op->Key.ID());
-  const auto State = GetVReg(Op->State.ID());
-  const auto ZeroReg = GetVReg(Op->ZeroReg.ID());
+  const auto Key = GetVReg(Op->Key);
+  const auto State = GetVReg(Op->State);
+  const auto ZeroReg = GetVReg(Op->ZeroReg);
 
   LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit, "Currently only supports 128-bit operations.");
 
@@ -65,12 +64,12 @@ DEF_OP(VAESEncLast) {
 
 DEF_OP(VAESDec) {
   const auto Op = IROp->C<IR::IROp_VAESDec>();
-  [[maybe_unused]] const auto OpSize = IROp->Size;
+  const auto OpSize = IROp->Size;
 
   const auto Dst = GetVReg(Node);
-  const auto Key = GetVReg(Op->Key.ID());
-  const auto State = GetVReg(Op->State.ID());
-  const auto ZeroReg = GetVReg(Op->ZeroReg.ID());
+  const auto Key = GetVReg(Op->Key);
+  const auto State = GetVReg(Op->State);
+  const auto ZeroReg = GetVReg(Op->ZeroReg);
 
   LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit, "Currently only supports 128-bit operations.");
 
@@ -90,12 +89,12 @@ DEF_OP(VAESDec) {
 
 DEF_OP(VAESDecLast) {
   const auto Op = IROp->C<IR::IROp_VAESDecLast>();
-  [[maybe_unused]] const auto OpSize = IROp->Size;
+  const auto OpSize = IROp->Size;
 
   const auto Dst = GetVReg(Node);
-  const auto Key = GetVReg(Op->Key.ID());
-  const auto State = GetVReg(Op->State.ID());
-  const auto ZeroReg = GetVReg(Op->ZeroReg.ID());
+  const auto Key = GetVReg(Op->Key);
+  const auto State = GetVReg(Op->State);
+  const auto ZeroReg = GetVReg(Op->ZeroReg);
 
   LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit, "Currently only supports 128-bit operations.");
 
@@ -114,9 +113,9 @@ DEF_OP(VAESDecLast) {
 DEF_OP(VAESKeyGenAssist) {
   auto Op = IROp->C<IR::IROp_VAESKeyGenAssist>();
   const auto Dst = GetVReg(Node);
-  const auto Src = GetVReg(Op->Src.ID());
-  const auto Swizzle = GetVReg(Op->KeyGenTBLSwizzle.ID());
-  auto ZeroReg = GetVReg(Op->ZeroReg.ID());
+  const auto Src = GetVReg(Op->Src);
+  const auto Swizzle = GetVReg(Op->KeyGenTBLSwizzle);
+  auto ZeroReg = GetVReg(Op->ZeroReg);
 
   if (Dst == ZeroReg) {
     // Seriously? ZeroReg ended up being the destination register?
@@ -148,8 +147,8 @@ DEF_OP(CRC32) {
   auto Op = IROp->C<IR::IROp_CRC32>();
 
   const auto Dst = GetReg(Node);
-  const auto Src1 = GetReg(Op->Src1.ID());
-  const auto Src2 = GetReg(Op->Src2.ID());
+  const auto Src1 = GetReg(Op->Src1);
+  const auto Src2 = GetReg(Op->Src2);
 
   switch (Op->SrcSize) {
   case IR::OpSize::i8Bit: crc32cb(Dst.W(), Src1.W(), Src2.W()); break;
@@ -164,7 +163,7 @@ DEF_OP(VSha1H) {
   auto Op = IROp->C<IR::IROp_VSha1H>();
 
   const auto Dst = GetVReg(Node);
-  const auto Src = GetVReg(Op->Src.ID());
+  const auto Src = GetVReg(Op->Src);
 
   sha1h(Dst.S(), Src.S());
 }
@@ -173,9 +172,9 @@ DEF_OP(VSha1C) {
   auto Op = IROp->C<IR::IROp_VSha1C>();
 
   const auto Dst = GetVReg(Node);
-  const auto Src1 = GetVReg(Op->Src1.ID());
-  const auto Src2 = GetVReg(Op->Src2.ID());
-  const auto Src3 = GetVReg(Op->Src3.ID());
+  const auto Src1 = GetVReg(Op->Src1);
+  const auto Src2 = GetVReg(Op->Src2);
+  const auto Src3 = GetVReg(Op->Src3);
 
   if (Dst == Src1) {
     sha1c(Dst, Src2.S(), Src3);
@@ -193,9 +192,9 @@ DEF_OP(VSha1M) {
   auto Op = IROp->C<IR::IROp_VSha1M>();
 
   const auto Dst = GetVReg(Node);
-  const auto Src1 = GetVReg(Op->Src1.ID());
-  const auto Src2 = GetVReg(Op->Src2.ID());
-  const auto Src3 = GetVReg(Op->Src3.ID());
+  const auto Src1 = GetVReg(Op->Src1);
+  const auto Src2 = GetVReg(Op->Src2);
+  const auto Src3 = GetVReg(Op->Src3);
 
   if (Dst == Src1) {
     sha1m(Dst, Src2.S(), Src3);
@@ -213,9 +212,9 @@ DEF_OP(VSha1P) {
   auto Op = IROp->C<IR::IROp_VSha1P>();
 
   const auto Dst = GetVReg(Node);
-  const auto Src1 = GetVReg(Op->Src1.ID());
-  const auto Src2 = GetVReg(Op->Src2.ID());
-  const auto Src3 = GetVReg(Op->Src3.ID());
+  const auto Src1 = GetVReg(Op->Src1);
+  const auto Src2 = GetVReg(Op->Src2);
+  const auto Src3 = GetVReg(Op->Src3);
 
   if (Dst == Src1) {
     sha1p(Dst, Src2.S(), Src3);
@@ -233,8 +232,8 @@ DEF_OP(VSha1SU1) {
   auto Op = IROp->C<IR::IROp_VSha1SU1>();
 
   const auto Dst = GetVReg(Node);
-  const auto Src1 = GetVReg(Op->Src1.ID());
-  const auto Src2 = GetVReg(Op->Src2.ID());
+  const auto Src1 = GetVReg(Op->Src1);
+  const auto Src2 = GetVReg(Op->Src2);
 
   if (Dst == Src1) {
     sha1su1(Dst, Src2);
@@ -252,9 +251,9 @@ DEF_OP(VSha256H) {
   auto Op = IROp->C<IR::IROp_VSha256H>();
 
   const auto Dst = GetVReg(Node);
-  const auto Src1 = GetVReg(Op->Src1.ID());
-  const auto Src2 = GetVReg(Op->Src2.ID());
-  const auto Src3 = GetVReg(Op->Src3.ID());
+  const auto Src1 = GetVReg(Op->Src1);
+  const auto Src2 = GetVReg(Op->Src2);
+  const auto Src3 = GetVReg(Op->Src3);
 
   if (Dst == Src1) {
     sha256h(Dst, Src2, Src3);
@@ -272,9 +271,9 @@ DEF_OP(VSha256H2) {
   auto Op = IROp->C<IR::IROp_VSha256H2>();
 
   const auto Dst = GetVReg(Node);
-  const auto Src1 = GetVReg(Op->Src1.ID());
-  const auto Src2 = GetVReg(Op->Src2.ID());
-  const auto Src3 = GetVReg(Op->Src3.ID());
+  const auto Src1 = GetVReg(Op->Src1);
+  const auto Src2 = GetVReg(Op->Src2);
+  const auto Src3 = GetVReg(Op->Src3);
 
   if (Dst == Src1) {
     sha256h2(Dst, Src2, Src3);
@@ -292,8 +291,8 @@ DEF_OP(VSha256U0) {
   auto Op = IROp->C<IR::IROp_VSha256U0>();
 
   const auto Dst = GetVReg(Node);
-  const auto Src1 = GetVReg(Op->Src1.ID());
-  const auto Src2 = GetVReg(Op->Src2.ID());
+  const auto Src1 = GetVReg(Op->Src1);
+  const auto Src2 = GetVReg(Op->Src2);
 
   if (Dst == Src1) {
     sha256su0(Dst, Src2);
@@ -308,8 +307,8 @@ DEF_OP(VSha256U1) {
   auto Op = IROp->C<IR::IROp_VSha256U1>();
 
   const auto Dst = GetVReg(Node);
-  const auto Src1 = GetVReg(Op->Src1.ID());
-  const auto Src2 = GetVReg(Op->Src2.ID());
+  const auto Src1 = GetVReg(Op->Src1);
+  const auto Src2 = GetVReg(Op->Src2);
 
   if (Dst != Src1 && Dst != Src2) {
     movi(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), 0);
@@ -323,11 +322,11 @@ DEF_OP(VSha256U1) {
 
 DEF_OP(PCLMUL) {
   const auto Op = IROp->C<IR::IROp_PCLMUL>();
-  [[maybe_unused]] const auto OpSize = IROp->Size;
+  const auto OpSize = IROp->Size;
 
   const auto Dst = GetVReg(Node);
-  const auto Src1 = GetVReg(Op->Src1.ID());
-  const auto Src2 = GetVReg(Op->Src2.ID());
+  const auto Src1 = GetVReg(Op->Src1);
+  const auto Src2 = GetVReg(Op->Src2);
 
   LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i128Bit, "Currently only supports 128-bit operations.");
 
@@ -346,5 +345,4 @@ DEF_OP(PCLMUL) {
   }
 }
 
-#undef DEF_OP
 } // namespace FEXCore::CPU

FEXCore/Source/Interface/Core/JIT/JIT.cpp

@@ -40,36 +40,33 @@ $end_info$
 #include <string.h>
 #include <limits>
 
-static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16;
-// We don't want to move above 128MB atm because that means we will have to encode longer jumps
-static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 128;
-
 namespace {
-static uint64_t LUDIV(uint64_t SrcHigh, uint64_t SrcLow, uint64_t Divisor) {
+struct DivRem {
+  uint64_t Quotient;
+  uint64_t Remainder;
+};
+
+static struct DivRem LUDIV(uint64_t SrcHigh, uint64_t SrcLow, uint64_t Divisor) {
   __uint128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;
-  __uint128_t Res = Source / Divisor;
-  return Res;
+
+  return {
+    .Quotient = (uint64_t)(Source / Divisor),
+    .Remainder = (uint64_t)(Source % Divisor),
+  };
 }
 
-static int64_t LDIV(uint64_t SrcHigh, uint64_t SrcLow, int64_t Divisor) {
+static struct DivRem
+LDIV(uint64_t SrcHigh, uint64_t SrcLow, int64_t Divisor) {
   __int128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;
-  __int128_t Res = Source / Divisor;
-  return Res;
+
+  return {
+    .Quotient = (uint64_t)(Source / Divisor),
+    .Remainder = (uint64_t)(Source % Divisor),
+  };
 }
 
-static uint64_t LUREM(uint64_t SrcHigh, uint64_t SrcLow, uint64_t Divisor) {
-  __uint128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;
-  __uint128_t Res = Source % Divisor;
-  return Res;
-}
-
-static int64_t LREM(uint64_t SrcHigh, uint64_t SrcLow, int64_t Divisor) {
-  __int128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;
-  __int128_t Res = Source % Divisor;
-  return Res;
-}
-
-static void PrintValue(uint64_t Value) {
+static void
+PrintValue(uint64_t Value) {
   LogMan::Msg::DFmt("Value: 0x{:x}", Value);
 }
 
@@ -80,13 +77,23 @@ static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) {
 
 namespace FEXCore::CPU {
 
-void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
+void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
   FallbackInfo Info;
   if (!InterpreterOps::GetFallbackHandler(IROp, &Info)) {
 #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
     LOGMAN_MSG_A_FMT("Unhandled IR Op: {}", FEXCore::IR::GetName(IROp->Op));
 #endif
   } else {
+    auto FillF80x2Result = [&](auto DstLo, auto DstHi) {
+      mov(DstLo.Q(), VTMP1.Q());
+      mov(DstHi.Q(), VTMP2.Q());
+    };
+
+    auto FillF64x2Result = [&](auto DstLo, auto DstHi) {
+      fmov(DstLo.D(), VTMP1.D());
+      fmov(DstHi.D(), VTMP2.D());
+    };
+
     auto FillF80Result = [&]() {
       const auto Dst = GetVReg(Node);
       mov(Dst.Q(), VTMP1.Q());
@@ -125,7 +132,7 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       // vtmp1 (v0/v16): source
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetVReg(IROp->Args[0].ID());
+      const auto Src1 = GetVReg(IROp->Args[0]);
       fmov(VTMP1.S(), Src1.S());
 
       ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
@@ -143,7 +150,7 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       // vtmp1 (v0/v16): source
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetVReg(IROp->Args[0].ID());
+      const auto Src1 = GetVReg(IROp->Args[0]);
       fmov(VTMP1.D(), Src1.D());
 
       ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
@@ -162,7 +169,7 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       // tmp2 (x1/x11): source
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetReg(IROp->Args[0].ID());
+      const auto Src1 = GetReg(IROp->Args[0]);
 
       // Need to sign or zero extend this for the dispatcher handler.
       if (Info.ABI == FABI_F80_I16_I16_PTR) {
@@ -186,7 +193,7 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       // vtmp1 (v0/v16): source
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetVReg(IROp->Args[0].ID());
+      const auto Src1 = GetVReg(IROp->Args[0]);
       mov(VTMP1.Q(), Src1.Q());
 
       ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
@@ -204,7 +211,7 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       // vtmp1 (v0/v16): source
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetVReg(IROp->Args[0].ID());
+      const auto Src1 = GetVReg(IROp->Args[0]);
       mov(VTMP1.Q(), Src1.Q());
 
       ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
@@ -215,14 +222,14 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       FillF64Result();
     } break;
 
-    case FABI_F64_I16_F64_PTR: {
+    case FABI_F64_F64_PTR: {
       // Linux Reg/Win32 Reg:
       // tmp4 (x4/x13): FallbackHandler
       // x30: return
       // vtmp1 (v0/v16): vector source
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetVReg(IROp->Args[0].ID());
+      const auto Src1 = GetVReg(IROp->Args[0]);
       fmov(VTMP1.D(), Src1.D());
 
       ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
@@ -232,8 +239,32 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
       FillF64Result();
     } break;
+    case FABI_F64x2_F64_PTR: {
+      // Linux Reg/Win32 Reg:
+      // tmp4 (x4/x13): FallbackHandler
+      // x30: return
+      // vtmp1 (v0/v16): vector source
+      // vtmp2 (v1/v16): vector source
+#ifdef VIXL_SIMULATOR
+      LOGMAN_THROW_A_FMT(CTX->Config.DisableVixlIndirectCalls, "Vector register pairs unsupported by simulator currently");
+#endif
+      str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-    case FABI_F64_I16_F64_F64_PTR: {
+      const auto Src1 = GetVReg(IROp->Args[0]);
+      const auto DstLo = GetVReg(IROp->Args[1]);
+      const auto DstHi = GetVReg(IROp->Args[2]);
+
+      fmov(VTMP1.D(), Src1.D());
+
+      ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
+      ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
+      blr(TMP1);
+
+      ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
+      FillF64x2Result(DstLo, DstHi);
+    } break;
+
+    case FABI_F64_F64_F64_PTR: {
       // Linux Reg/Win32 Reg:
       // tmp4 (x4/x13): FallbackHandler
       // x30: return
@@ -241,8 +272,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       // vtmp2 (v1/v17): vector source 2
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetVReg(IROp->Args[0].ID());
-      const auto Src2 = GetVReg(IROp->Args[1].ID());
+      const auto Src1 = GetVReg(IROp->Args[0]);
+      const auto Src2 = GetVReg(IROp->Args[1]);
 
       fmov(VTMP1.D(), Src1.D());
       fmov(VTMP2.D(), Src2.D());
@@ -262,7 +293,7 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       // vtmp1 (v0/v16): source
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetVReg(IROp->Args[0].ID());
+      const auto Src1 = GetVReg(IROp->Args[0]);
       mov(VTMP1.Q(), Src1.Q());
 
       ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
@@ -280,7 +311,7 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       // vtmp1 (v0/v16): source
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetVReg(IROp->Args[0].ID());
+      const auto Src1 = GetVReg(IROp->Args[0]);
       mov(VTMP1.Q(), Src1.Q());
 
       ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
@@ -298,7 +329,7 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       // vtmp1 (v0/v16): source
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetVReg(IROp->Args[0].ID());
+      const auto Src1 = GetVReg(IROp->Args[0]);
       mov(VTMP1.Q(), Src1.Q());
 
       ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
@@ -317,8 +348,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       // vtmp2 (v1/v17): vector source 2
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetVReg(IROp->Args[0].ID());
-      const auto Src2 = GetVReg(IROp->Args[1].ID());
+      const auto Src1 = GetVReg(IROp->Args[0]);
+      const auto Src2 = GetVReg(IROp->Args[1]);
       mov(VTMP1.Q(), Src1.Q());
       mov(VTMP2.Q(), Src2.Q());
 
@@ -337,7 +368,7 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       // vtmp1 (v0/v16): vector source 1
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetVReg(IROp->Args[0].ID());
+      const auto Src1 = GetVReg(IROp->Args[0]);
       mov(VTMP1.Q(), Src1.Q());
 
       ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
@@ -348,6 +379,31 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       FillF80Result();
     } break;
 
+    case FABI_F80x2_I16_F80_PTR: {
+      // Linux Reg/Win32 Reg:
+      // tmp4 (x4/x13): FallbackHandler
+      // x30: return
+      // vtmp1 (v0/v16): vector source 1
+      // vtmp2 (v1/v16): vector source 2
+#ifdef VIXL_SIMULATOR
+      LOGMAN_THROW_A_FMT(CTX->Config.DisableVixlIndirectCalls, "Vector register pairs unsupported by simulator currently");
+#endif
+      str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
+
+      const auto Src1 = GetVReg(IROp->Args[0]);
+      const auto DstLo = GetVReg(IROp->Args[1]);
+      const auto DstHi = GetVReg(IROp->Args[2]);
+
+      mov(VTMP1.Q(), Src1.Q());
+
+      ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
+      ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
+      blr(TMP1);
+
+      ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
+      FillF80x2Result(DstLo, DstHi);
+    } break;
+
     case FABI_F80_I16_F80_F80_PTR: {
       // Linux Reg/Win32 Reg:
       // tmp4 (x4/x13): FallbackHandler
@@ -356,8 +412,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       // vtmp2 (v1/v17): vector source 2
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetVReg(IROp->Args[0].ID());
-      const auto Src2 = GetVReg(IROp->Args[1].ID());
+      const auto Src1 = GetVReg(IROp->Args[0]);
+      const auto Src2 = GetVReg(IROp->Args[1]);
 
       mov(VTMP1.Q(), Src1.Q());
       mov(VTMP2.Q(), Src2.Q());
@@ -385,16 +441,16 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
 
       stp<ARMEmitter::IndexType::PRE>(TMP1, ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto SrcRAX = GetReg(Op->RAX.ID());
-      const auto SrcRDX = GetReg(Op->RDX.ID());
+      const auto SrcRAX = GetReg(Op->RAX);
+      const auto SrcRDX = GetReg(Op->RDX);
       const auto Control = Op->Control;
 
       mov(TMP1, SrcRAX.X());
       mov(TMP2, SrcRDX.X());
       movz(ARMEmitter::Size::i32Bit, TMP3, Control);
 
-      const auto Src1 = GetVReg(Op->LHS.ID());
-      const auto Src2 = GetVReg(Op->RHS.ID());
+      const auto Src1 = GetVReg(Op->LHS);
+      const auto Src2 = GetVReg(Op->RHS);
 
       mov(VTMP1.Q(), Src1.Q());
       mov(VTMP2.Q(), Src2.Q());
@@ -414,8 +470,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
       const auto Op = IROp->C<IR::IROp_VPCMPISTRX>();
       str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
 
-      const auto Src1 = GetVReg(Op->LHS.ID());
-      const auto Src2 = GetVReg(Op->RHS.ID());
+      const auto Src1 = GetVReg(Op->LHS);
+      const auto Src2 = GetVReg(Op->RHS);
       const auto Control = Op->Control;
 
       mov(VTMP1.Q(), Src1.Q());
@@ -439,71 +495,132 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::NodeID Node) {
   }
 }
 
-static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
-  // Emit new 16 bytes of code to a temporary patch, then atomically apply it
-  __uint128_t Patch;
-  ARMEmitter::Emitter emit((uint8_t*)&Patch, sizeof(Patch));
-  emit.ldr(TMP1, 8); // PC-relative value pointing to constant after blr
-  emit.blr(TMP1);
-  emit.dc64(Frame->Pointers.Common.ExitFunctionLinker);
+static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record, bool Call) {
+  uintptr_t JumpThunkStartAddress = reinterpret_cast<uintptr_t>(Record) - 0x10;
+  uintptr_t CallerAddress = JumpThunkStartAddress + Record->CallerOffset;
+  auto BranchOffset = JumpThunkStartAddress / 4 - CallerAddress / 4;
 
-  auto branch = reinterpret_cast<__uint128_t*>((uintptr_t)Record - 8);
-  std::atomic_ref<__uint128_t>(*branch).store(Patch, std::memory_order::relaxed);
-  ARMEmitter::Emitter::ClearICache((void*)branch, sizeof(*branch));
+  // Replace the patched callsite with a branch to the jump thunk.
+  uint32_t BranchInst = 0;
+  ARMEmitter::Emitter BranchEmit(reinterpret_cast<uint8_t*>(&BranchInst), 4);
+  if (Call) {
+    BranchEmit.bl(BranchOffset);
+  } else {
+    BranchEmit.b(BranchOffset);
+  }
+
+  std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(CallerAddress)).store(BranchInst, std::memory_order::relaxed);
+  ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(CallerAddress), 4);
 }
 
-static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
+static void IndirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
+  uintptr_t JumpThunkStartAddress = reinterpret_cast<uintptr_t>(Record) - 0x10;
+  uint32_t BranchInst = 0;
+  ARMEmitter::Emitter BranchEmit(reinterpret_cast<uint8_t*>(&BranchInst), 4);
+  BranchEmit.b(0x8);
+
+  std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(JumpThunkStartAddress)).store(BranchInst, std::memory_order::relaxed);
+  ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(JumpThunkStartAddress), 4);
+
+  // No need to reset HostCode here as the exit linker pointer is stored separately, and if the block is relinked it will be updated.
+}
+
+uint64_t Arm64JITCore::ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
   auto Thread = Frame->Thread;
   bool TFSet = Thread->CurrentFrame->State.flags[X86State::RFLAG_TF_RAW_LOC];
   uintptr_t HostCode {};
   auto GuestRip = Record->GuestRIP;
 
-  if (!TFSet) {
-    HostCode = Thread->LookupCache->FindBlock(GuestRip);
-  }
-
-  if (TFSet || !HostCode) {
+  if (TFSet) {
     // If TF is set, the cache must be skipped as different code needs to be generated.
     Frame->State.rip = GuestRip;
     return Frame->Pointers.Common.DispatcherLoopTop;
+  } else {
+    {
+      // Guard the LookupCache lock with the code invalidation mutex, to avoid issues with forking
+      auto lk_inval = GuardSignalDeferringSection<std::shared_lock>(static_cast<Context::ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
+      HostCode = Thread->LookupCache->FindBlock(GuestRip);
+    }
+    if (!HostCode) {
+      // Hold a reference to the code buffer, to avoid linking unmapped code if compilation triggers a recreation.
+      auto CodeBuffer = static_cast<Arm64JITCore*>(Thread->CPUBackend.get())->CurrentCodeBuffer;
+      HostCode = static_cast<Context::ContextImpl*>(Thread->CTX)->CompileBlock(Frame, GuestRip, 0);
+      if (Thread->LookupCache->Shared != CodeBuffer->LookupCache.get()) {
+        return HostCode;
+      }
+    }
   }
 
-  uintptr_t branch = (uintptr_t)(Record)-8;
-  LOGMAN_THROW_A_FMT((branch % 16) == 0, "Incorrect alignment for block linking record");
+  // See ExitFunction in BranchOps.cpp for an assembly level view of the handled cases.
+  uintptr_t JumpThunkStartAddress = reinterpret_cast<uintptr_t>(Record) - 0x10;
+  uintptr_t CallerAddress = JumpThunkStartAddress + Record->CallerOffset;
+  auto BranchOffset = HostCode / 4 - CallerAddress / 4;
 
-  auto offset = HostCode / 4 - branch / 4;
-  if (ARMEmitter::Emitter::IsInt26(offset)) {
-    // This is the optimal case, where the target can be encoded in a single instruction.
-    // Atomically patch the code with a relative branch.
-    const uint32_t Patch = (0b0001'01 << 26) | (offset & ((1u << 26) - 1));
-    std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(branch)).store(Patch, std::memory_order::relaxed);
-    ARMEmitter::Emitter::ClearICache((void*)branch, 4);
+  uint32_t ExpectedKnownCallMarkerInst = 0;
+  ARMEmitter::Emitter ExpectedKnownCallMarkerEmit(reinterpret_cast<uint8_t*>(&ExpectedKnownCallMarkerInst), 4);
+  ExpectedKnownCallMarkerEmit.adr(TMP1, 0xC);
+
+  // Guard the LookupCache lock with the code invalidation mutex, to avoid issues with forking
+  auto lk_inval = GuardSignalDeferringSection<std::shared_lock>(static_cast<Context::ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
+
+  // Lock here is necessary to prevent simultaneous linking and delinking
+  auto lk = Thread->LookupCache->AcquireLock();
+
+  // For non-calls, this would extend into the block's code, however that's fine as an out-of-range adr would never
+  // be generated avoiding any false positives.
+  uintptr_t KnownCallMarkerAddr = CallerAddress - 0x8;
+  uint32_t KnownCallMarkerInst = *reinterpret_cast<uint32_t*>(KnownCallMarkerAddr);
+  if (ARMEmitter::Emitter::IsInt26(BranchOffset)) {
+    // Directly patch the callsite with the appropriate branch instruction.
+    uint32_t BranchInst = 0;
+    ARMEmitter::Emitter BranchEmit(reinterpret_cast<uint8_t*>(&BranchInst), 4);
+
+    if (KnownCallMarkerInst == ExpectedKnownCallMarkerInst) {
+      BranchEmit.bl(BranchOffset);
+      Thread->LookupCache->AddBlockLink(GuestRip, Record, [](FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
+        DirectBlockDelinker(Frame, Record, true);
+      });
+    } else {
+      BranchEmit.b(BranchOffset);
+      Thread->LookupCache->AddBlockLink(GuestRip, Record, [](FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
+        DirectBlockDelinker(Frame, Record, false);
+      });
+    }
+
+    std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(CallerAddress)).store(BranchInst, std::memory_order::relaxed);
+    ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(CallerAddress), 4);
   } else {
-    // fallback case - do a soft-er link by patching the pointer
-    std::atomic_ref<uint64_t>(Record->HostBranch).store(HostCode, std::memory_order::seq_cst);
+    // This case is common between calls and jumps as the thunk callsite can be left untouched.
+    std::atomic_ref<uint64_t>(Record->HostCode).store(HostCode, std::memory_order::seq_cst);
 #ifdef _M_ARM_64
     // Make memory write visible to other threads reading the same location
-    asm volatile("dc cvau, %0; dsb ish" : : "r"(Record->HostBranch) :);
+    asm volatile("dc cvau, %0; dsb ish" : : "r"(Record->HostCode) :);
 #endif
-  }
 
-  // Add de-linking handler
-  Thread->LookupCache->AddBlockLink(GuestRip, Record, DirectBlockDelinker);
+    uint32_t LdrInst = 0;
+    ARMEmitter::Emitter LdrEmit(reinterpret_cast<uint8_t*>(&LdrInst), 4);
+    LdrEmit.ldr(TMP1, reinterpret_cast<uint64_t>(&Record->HostCode) - JumpThunkStartAddress);
+    std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(JumpThunkStartAddress)).store(LdrInst, std::memory_order::relaxed);
+    ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(JumpThunkStartAddress), 4);
+
+    Thread->LookupCache->AddBlockLink(GuestRip, Record, IndirectBlockDelinker);
+  }
 
   return HostCode;
 }
 
-void Arm64JITCore::Op_NoOp(const IR::IROp_Header* IROp, IR::NodeID Node) {}
+void Arm64JITCore::Op_NoOp(const IR::IROp_Header* IROp, IR::Ref Node) {}
 
 Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::InternalThreadState* Thread)
-  : CPUBackend(Thread, INITIAL_CODE_SIZE, MAX_CODE_SIZE)
+  : CPUBackend(*ctx, Thread)
   , Arm64Emitter(ctx)
   , HostSupportsSVE128 {ctx->HostFeatures.SupportsSVE128}
   , HostSupportsSVE256 {ctx->HostFeatures.SupportsSVE256}
   , HostSupportsAVX256 {ctx->HostFeatures.SupportsAVX && ctx->HostFeatures.SupportsSVE256}
   , HostSupportsRPRES {ctx->HostFeatures.SupportsRPRES}
   , HostSupportsAFP {ctx->HostFeatures.SupportsAFP}
-  , CTX {ctx} {
+  , CTX {ctx}
+  , TempAllocator(ctx->CPUBackendAllocator, 0) {
 
   RAPass = Thread->PassManager->GetPass<IR::RegisterAllocationPass>("RA");
 
@@ -522,6 +639,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::In
     Common.PrintValue = reinterpret_cast<uint64_t>(PrintValue);
     Common.PrintVectorValue = reinterpret_cast<uint64_t>(PrintVectorValue);
     Common.ThreadRemoveCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::ContextImpl::ThreadRemoveCodeEntryFromJit);
+    Common.MonoBackpatcherWrite = reinterpret_cast<uint64_t>(&Context::ContextImpl::MonoBackpatcherWrite);
     Common.CPUIDObj = reinterpret_cast<uint64_t>(&CTX->CPUID);
 
     {
@@ -539,19 +657,17 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::In
       Common.SyscallHandlerObj = reinterpret_cast<uint64_t>(CTX->SyscallHandler);
       Common.SyscallHandlerFunc = PMF.GetVTableEntry(CTX->SyscallHandler);
     }
-    Common.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Context::ContextImpl::ThreadExitFunctionLink<Arm64JITCore_ExitFunctionLink>);
+    Common.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Arm64JITCore::ExitFunctionLink);
 
     // Platform Specific
     auto& AArch64 = ThreadState->CurrentFrame->Pointers.AArch64;
 
     AArch64.LUDIV = reinterpret_cast<uint64_t>(LUDIV);
     AArch64.LDIV = reinterpret_cast<uint64_t>(LDIV);
-    AArch64.LUREM = reinterpret_cast<uint64_t>(LUREM);
-    AArch64.LREM = reinterpret_cast<uint64_t>(LREM);
   }
 
-  // Must be done after Dispatcher init
-  ClearCache();
+  CurrentCodeBuffer = CodeBuffers.GetLatest();
+  ThreadState->LookupCache->Shared = CurrentCodeBuffer->LookupCache.get();
 
   // Setup dynamic dispatch.
   if (ParanoidTSO()) {
@@ -570,16 +686,24 @@ void Arm64JITCore::EmitDetectionString() {
 }
 
 void Arm64JITCore::ClearCache() {
-  // Get the backing code buffer
+  // NOTE: Holding on to the reference here is required to ensure validity of the WriteLock mutex
+  auto PrevCodeBuffer = CurrentCodeBuffer;
+  std::lock_guard lk(PrevCodeBuffer->LookupCache->WriteLock);
 
   auto CodeBuffer = GetEmptyCodeBuffer();
   SetBuffer(CodeBuffer->Ptr, CodeBuffer->Size);
   EmitDetectionString();
+
+  ThreadState->LookupCache->ChangeGuestToHostMapping(*PrevCodeBuffer, *CurrentCodeBuffer->LookupCache);
 }
 
 Arm64JITCore::~Arm64JITCore() {}
 
 bool Arm64JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {
+  if (WNode.IsImmediate()) {
+    return false;
+  }
+
   auto OpHeader = IR->GetOp<IR::IROp_Header>(WNode);
 
   if (OpHeader->Op == IR::IROps::OP_INLINECONSTANT) {
@@ -594,6 +718,10 @@ bool Arm64JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_
 }
 
 bool Arm64JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {
+  if (WNode.IsImmediate()) {
+    return false;
+  }
+
   auto OpHeader = IR->GetOp<IR::IROp_Header>(WNode);
 
   if (OpHeader->Op == IR::IROps::OP_INLINEENTRYPOINTOFFSET) {
@@ -612,22 +740,6 @@ bool Arm64JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode,
   }
 }
 
-FEXCore::IR::RegisterClassType Arm64JITCore::GetRegClass(IR::NodeID Node) const {
-  return FEXCore::IR::RegisterClassType {GetPhys(Node).Class};
-}
-
-bool Arm64JITCore::IsFPR(IR::NodeID Node) const {
-  auto Class = GetRegClass(Node);
-
-  return Class == IR::FPRClass || Class == IR::FPRFixedClass;
-}
-
-bool Arm64JITCore::IsGPR(IR::NodeID Node) const {
-  auto Class = GetRegClass(Node);
-
-  return Class == IR::GPRClass || Class == IR::GPRFixedClass;
-}
-
 void Arm64JITCore::EmitInterruptChecks(bool CheckTF) {
   if (CheckTF) {
     ARMEmitter::ForwardLabel l_TFUnset;
@@ -688,24 +800,48 @@ void Arm64JITCore::EmitInterruptChecks(bool CheckTF) {
 #endif
 }
 
+void Arm64JITCore::EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF) {
+  // Get the address of the JITCodeHeader and store in to the core state.
+  // Two instruction cost, each 1 cycle.
+  adr(TMP1, &HeaderLabel);
+  str(TMP1, STATE, offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader));
+
+  EmitInterruptChecks(CheckTF);
+
+  if (SpillSlots) {
+    const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize;
+
+    if (ARMEmitter::IsImmAddSub(TotalSpillSlotsSize)) {
+      sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, TotalSpillSlotsSize);
+    } else {
+      LoadConstant(ARMEmitter::Size::i64Bit, TMP1, TotalSpillSlotsSize);
+      sub(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, TMP1, ARMEmitter::ExtendedType::LSL_64, 0);
+    }
+  }
+}
+
 CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
-                                                   FEXCore::Core::DebugData* DebugData, const FEXCore::IR::RegisterAllocationData* RAData,
-                                                   bool CheckTF) {
+                                                   FEXCore::Core::DebugData* DebugData, bool CheckTF) {
   FEXCORE_PROFILE_SCOPED("Arm64::CompileCode");
 
   JumpTargets.clear();
+  CallReturnTargets.clear();
+  PendingJumpThunks.clear();
   uint32_t SSACount = IR->GetSSACount();
+  JumpTargets.resize(IR->GetHeader()->BlockCount, {});
 
   this->Entry = Entry;
-  this->RAData = RAData;
   this->DebugData = DebugData;
   this->IR = IR;
+  CodeData.EntryPoints.clear();
 
   // Fairly excessive buffer range to make sure we don't overflow
-  uint32_t BufferRange = SSACount * 16;
-  if ((GetCursorOffset() + BufferRange) > (CurrentCodeBuffer->Size - Utils::FEX_PAGE_SIZE)) {
-    CTX->ClearCodeCache(ThreadState);
-  }
+  uint32_t BufferRange = 0x1000 + SSACount * 24;
+
+  // JIT output is first written to a temporary buffer and later relocated to the CodeBuffer.
+  // This minimizes lock contention of CodeBufferWriteMutex.
+  auto TempCodeBuffer = TempAllocator.ReownOrClaimBuffer(BufferRange);
+  SetBuffer(TempCodeBuffer, BufferRange);
 
   CodeData.BlockBegin = GetCursorAddress<uint8_t*>();
 
@@ -715,9 +851,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
   JITCodeHeader* CodeHeader = GetCursorAddress<JITCodeHeader*>();
   CursorIncrement(sizeof(JITCodeHeader));
 
-#ifdef VIXL_DISASSEMBLER
-  const auto DisasmBegin = GetCursorAddress<const vixl::aarch64::Instruction*>();
-#endif
+  auto CodeBegin = GetCursorAddress<uint8_t*>();
 
   // AAPCS64
   // r30      = LR
@@ -739,71 +873,76 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
   // X1-X3 = Temp
   // X4-r18 = RA
 
-  CodeData.BlockEntry = GetCursorAddress<uint8_t*>();
-
-  // Get the address of the JITCodeHeader and store in to the core state.
-  // Two instruction cost, each 1 cycle.
-  adr(TMP1, &JITCodeHeaderLabel);
-  str(TMP1, STATE, offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader));
-
-  EmitInterruptChecks(CheckTF);
-
-  SpillSlots = RAData->SpillSlots();
-
-  if (SpillSlots) {
-    const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize;
-
-    if (ARMEmitter::IsImmAddSub(TotalSpillSlotsSize)) {
-      sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, TotalSpillSlotsSize);
-    } else {
-      LoadConstant(ARMEmitter::Size::i64Bit, TMP1, TotalSpillSlotsSize);
-      sub(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, TMP1, ARMEmitter::ExtendedType::LSL_64, 0);
-    }
-  }
+  SpillSlots = IR->SpillSlots();
 
   PendingTargetLabel = nullptr;
+  PendingCallReturnTargetLabel = nullptr;
 
   for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {
     using namespace FEXCore::IR;
-#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
     auto BlockIROp = BlockHeader->CW<FEXCore::IR::IROp_CodeBlock>();
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
     LOGMAN_THROW_A_FMT(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
 #endif
 
     auto BlockStartHostCode = GetCursorAddress<uint8_t*>();
     {
       const auto Node = IR->GetID(BlockNode);
-      const auto IsTarget = JumpTargets.try_emplace(Node).first;
+      const auto Target = &JumpTargets[BlockIROp->ID];
 
       // if there's a pending branch, and it is not fall-through
-      if (PendingTargetLabel && PendingTargetLabel != &IsTarget->second) {
+      if (PendingTargetLabel && PendingTargetLabel != Target) {
         b(PendingTargetLabel);
+        PendingTargetLabel = nullptr;
+      }
+
+      if (BlockIROp->EntryPoint) {
+        uint64_t BlockStartRIP = Entry + BlockIROp->GuestEntryOffset;
+
+        const auto IsReturnTarget = CallReturnTargets.try_emplace(Node).first;
+        if (PendingTargetLabel) {
+          // If there is a fallthrough branch to this block, skip over the entrypoint code.
+          b(Target);
+        } else if (PendingCallReturnTargetLabel && PendingCallReturnTargetLabel != &IsReturnTarget->second) {
+          // If we just emitted a call, but the block we're now emitting is not the return block so don't fallthrough.
+          b(PendingCallReturnTargetLabel);
+        }
+        PendingCallReturnTargetLabel = nullptr;
+
+        Bind(&IsReturnTarget->second);
+        CodeData.EntryPoints.emplace(BlockStartRIP, GetCursorAddress<uint8_t*>());
+        DebugData->GuestOpcodes.push_back({BlockIROp->GuestEntryOffset, GetCursorAddress<uint8_t*>() - CodeData.BlockBegin});
+
+        EmitEntryPoint(JITCodeHeaderLabel, CheckTF);
+      }
+
+      if (PendingCallReturnTargetLabel) {
+        // If there is still a pending call return target, then the block we're emitting is not the return block so don't fallthrough.
+        b(PendingCallReturnTargetLabel);
+        PendingCallReturnTargetLabel = nullptr;
       }
       PendingTargetLabel = nullptr;
 
-      Bind(&IsTarget->second);
+      Bind(Target);
     }
 
     for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {
-      const auto ID = IR->GetID(CodeNode);
       switch (IROp->Op) {
 #define REGISTER_OP_RT(op, x) \
-  case FEXCore::IR::IROps::OP_##op: std::invoke(RT_##x, this, IROp, ID); break
+  case FEXCore::IR::IROps::OP_##op: std::invoke(RT_##x, this, IROp, CodeNode); break
 #define REGISTER_OP(op, x) \
-  case FEXCore::IR::IROps::OP_##op: Op_##x(IROp, ID); break
+  case FEXCore::IR::IROps::OP_##op: Op_##x(IROp, CodeNode); break
 
 #define IROP_DISPATCH_DISPATCH
 #include <FEXCore/IR/IRDefines_Dispatch.inc>
 #undef REGISTER_OP
 
-      default: Op_Unhandled(IROp, ID); break;
+      default: Op_Unhandled(IROp, CodeNode); break;
       }
     }
 
-    if (DebugData) {
-      DebugData->Subblocks.push_back({static_cast<uint32_t>(BlockStartHostCode - CodeData.BlockEntry),
-                                      static_cast<uint32_t>(GetCursorAddress<uint8_t*>() - BlockStartHostCode)});
-    }
+    DebugData->Subblocks.push_back({static_cast<uint32_t>(BlockStartHostCode - CodeData.BlockBegin),
+                                    static_cast<uint32_t>(GetCursorAddress<uint8_t*>() - BlockStartHostCode)});
   }
 
   // Make sure last branch is generated. It certainly can't be eliminated here.
@@ -812,8 +951,32 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
   }
   PendingTargetLabel = nullptr;
 
-  // CodeSize not including the tail data.
-  const uint64_t CodeOnlySize = GetCursorAddress<uint8_t*>() - CodeData.BlockBegin;
+  ARMEmitter::ForwardLabel l_ExitLink;
+  for (auto& PendingJumpThunk : PendingJumpThunks) {
+    // Align as 64-bit atomics are used on the HostCode field.
+    Align(8);
+
+    ARMEmitter::ForwardLabel l_DoLink;
+    uint64_t ThunkAddress = GetCursorAddress<uint64_t>();
+    Bind(&PendingJumpThunk.Label);
+    b(&l_DoLink);
+    br(TMP1);
+    Bind(&l_DoLink);
+    ldr(TMP1, &l_ExitLink);
+    blr(TMP1);
+
+    // This is a ExitFunctionLinkData struct
+    Bind(&l_ExitLink);
+    dc64(0);                                             // HostCode
+    dc64(PendingJumpThunk.GuestRIP);                     // GuestRIP
+    dc64(PendingJumpThunk.CallerAddress - ThunkAddress); // CallerOffset
+  }
+
+  Bind(&l_ExitLink);
+  dc64(ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker);
+
+  // CodeSize not including the header or tail data.
+  const uint64_t CodeOnlySize = GetCursorAddress<uint8_t*>() - CodeBegin;
 
   // Add the JitCodeTail
   Align(alignof(JITCodeTail));
@@ -878,7 +1041,54 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
 
   JITBlockTail->Size = CodeData.Size;
 
-  ClearICache(CodeData.BlockBegin, CodeOnlySize);
+  // Migrate the compile output from temporary storage to the actual CodeBuffer.
+  // This can block progress in other compiling threads, so the duration of the lock should be as small as possible.
+  {
+    auto CodeBufferLock = std::unique_lock {CodeBuffers.CodeBufferWriteMutex};
+
+    // Query size of generated code
+    const auto TempSize = GetCursorOffset();
+    LOGMAN_THROW_A_FMT(TempSize <= BufferRange, "Exceeded bounds of temporary buffer ({:#x} vs {:#x})", TempSize, BufferRange);
+
+    // Bring CodeBuffer up to date
+    {
+      LOGMAN_THROW_A_FMT(CurrentCodeBuffer->LookupCache.get() == ThreadState->LookupCache->Shared, "INVARIANT VIOLATED: SharedLookupCache "
+                                                                                                   "doesn't match up!\n");
+      if (auto Prev = CheckCodeBufferUpdate()) {
+        Allocator::VirtualDontNeed(ThreadState->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);
+        ThreadState->LookupCache->ChangeGuestToHostMapping(*Prev, *CurrentCodeBuffer->LookupCache);
+      }
+
+      // NOTE: 16-byte alignment of the new cursor offset must be preserved for block linking records
+      SetBuffer(CurrentCodeBuffer->Ptr, CurrentCodeBuffer->Size);
+      SetCursorOffset(AlignUp(CodeBuffers.LatestOffset, 16));
+      if ((GetCursorOffset() + TempSize) > (CurrentCodeBuffer->Size - Utils::FEX_PAGE_SIZE)) {
+        CTX->ClearCodeCache(ThreadState);
+      }
+
+      Align16B();
+
+      CodeBuffers.LatestOffset = GetCursorOffset();
+    }
+
+    // Adjust host addresses
+    const auto Delta = GetCursorAddress<uint8_t*>() - CodeData.BlockBegin;
+    CodeData.BlockBegin += Delta;
+    for (auto& EntryPoint : CodeData.EntryPoints) {
+      EntryPoint.second += Delta;
+    }
+    CodeBegin += Delta;
+
+    // Copy over CodeBuffer contents
+    memcpy(GetCursorAddress<uint8_t*>(), TempCodeBuffer, TempSize);
+    SetCursorOffset(CodeBuffers.LatestOffset + TempSize);
+
+    CodeBuffers.LatestOffset = GetCursorOffset();
+  }
+
+  TempAllocator.DelayedDisownBuffer();
+
+  ClearICache(CodeBegin, CodeOnlySize);
 
 #ifdef VIXL_DISASSEMBLER
   if (Disassemble() & FEXCore::Config::Disassemble::STATS) {
@@ -892,7 +1102,8 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
   }
 
   if (Disassemble() & FEXCore::Config::Disassemble::BLOCKS) {
-    const auto DisasmEnd = reinterpret_cast<const vixl::aarch64::Instruction*>(JITBlockTailLocation);
+    const auto DisasmBegin = reinterpret_cast<const vixl::aarch64::Instruction*>(CodeBegin);
+    const auto DisasmEnd = reinterpret_cast<const vixl::aarch64::Instruction*>(CodeBegin + CodeOnlySize);
     LogMan::Msg::IFmt("Disassemble Begin");
     for (auto PCToDecode = DisasmBegin; PCToDecode < DisasmEnd; PCToDecode += 4) {
       DisasmDecoder->Decode(PCToDecode);
@@ -903,14 +1114,12 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
   }
 #endif
 
-  if (DebugData) {
-    DebugData->HostCodeSize = CodeData.Size;
-    DebugData->Relocations = &Relocations;
-  }
+  DebugData->HostCodeSize = CodeData.Size;
+  DebugData->Relocations = &Relocations;
 
   this->IR = nullptr;
 
-  return CodeData;
+  return std::move(CodeData);
 }
 
 void Arm64JITCore::ResetStack() {

FEXCore/Source/Interface/Core/JIT/JITClass.h

@@ -24,6 +24,7 @@ $end_info$
 
 #include <array>
 #include <cstdint>
+#include <functional>
 #include <utility>
 #include <variant>
 
@@ -31,6 +32,10 @@ namespace FEXCore::Core {
 struct InternalThreadState;
 }
 
+namespace FEXCore::Context {
+struct ExitFunctionLinkData;
+}
+
 namespace FEXCore::CPU {
 class Arm64JITCore final : public CPUBackend, public Arm64Emitter {
 public:
@@ -38,9 +43,8 @@ public:
   ~Arm64JITCore() override;
 
   [[nodiscard]]
-  CPUBackend::CompiledCode
-  CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR, FEXCore::Core::DebugData* DebugData,
-              const FEXCore::IR::RegisterAllocationData* RAData, bool CheckTF) override;
+  CPUBackend::CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
+                                       FEXCore::Core::DebugData* DebugData, bool CheckTF) override;
 
   void ClearCache() override;
 
@@ -58,17 +62,34 @@ private:
   const bool HostSupportsAFP {};
 
   ARMEmitter::BiDirectionalLabel* PendingTargetLabel {};
+  ARMEmitter::BiDirectionalLabel* PendingCallReturnTargetLabel {};
   FEXCore::Context::ContextImpl* CTX {};
   const FEXCore::IR::IRListView* IR {};
   uint64_t Entry {};
   CPUBackend::CompiledCode CodeData {};
 
-  fextl::map<IR::NodeID, ARMEmitter::BiDirectionalLabel> JumpTargets;
+  fextl::vector<ARMEmitter::BiDirectionalLabel> JumpTargets;
+
+  ARMEmitter::BiDirectionalLabel* JumpTarget(IR::OrderedNodeWrapper Node) {
+    auto Block = IR->GetOp<IR::IROp_CodeBlock>(Node);
+    return &JumpTargets[Block->ID];
+  }
+
+  fextl::map<IR::NodeID, ARMEmitter::BiDirectionalLabel> CallReturnTargets;
+
+  struct PendingJumpThunk {
+    uint64_t CallerAddress;
+    uint64_t GuestRIP;
+    ARMEmitter::ForwardLabel Label;
+  };
+  fextl::vector<PendingJumpThunk> PendingJumpThunks;
+
+  Utils::PoolBufferWithTimedRetirement<uint8_t*, 5000, 500> TempAllocator;
+
+  static uint64_t ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record);
 
   [[nodiscard]]
-  ARMEmitter::Register GetReg(IR::NodeID Node) const {
-    const auto Reg = GetPhys(Node);
-
+  ARMEmitter::Register GetReg(IR::PhysicalRegister Reg) const {
     LOGMAN_THROW_A_FMT(Reg.Class == IR::GPRFixedClass.Val || Reg.Class == IR::GPRClass.Val, "Unexpected Class: {}", Reg.Class);
 
     if (Reg.Class == IR::GPRFixedClass.Val) {
@@ -81,9 +102,17 @@ private:
   }
 
   [[nodiscard]]
-  ARMEmitter::VRegister GetVReg(IR::NodeID Node) const {
-    const auto Reg = GetPhys(Node);
+  ARMEmitter::Register GetReg(IR::Ref Node) const {
+    return GetReg(IR::PhysicalRegister(Node));
+  }
 
+  [[nodiscard]]
+  ARMEmitter::Register GetReg(IR::OrderedNodeWrapper Wrap) const {
+    return GetReg(IR::PhysicalRegister(Wrap));
+  }
+
+  [[nodiscard]]
+  ARMEmitter::VRegister GetVReg(IR::PhysicalRegister Reg) const {
     LOGMAN_THROW_A_FMT(Reg.Class == IR::FPRFixedClass.Val || Reg.Class == IR::FPRClass.Val, "Unexpected Class: {}", Reg.Class);
 
     if (Reg.Class == IR::FPRFixedClass.Val) {
@@ -96,15 +125,18 @@ private:
   }
 
   [[nodiscard]]
-  FEXCore::IR::RegisterClassType GetRegClass(IR::NodeID Node) const;
+  ARMEmitter::VRegister GetVReg(IR::Ref Node) const {
+    return GetVReg(IR::PhysicalRegister(Node));
+  }
 
   [[nodiscard]]
-  IR::PhysicalRegister GetPhys(IR::NodeID Node) const {
-    auto PhyReg = RAData->GetNodeRegister(Node);
+  ARMEmitter::VRegister GetVReg(IR::OrderedNodeWrapper Wrap) const {
+    return GetVReg(IR::PhysicalRegister(Wrap));
+  }
 
-    LOGMAN_THROW_A_FMT(!PhyReg.IsInvalid(), "Couldn't Allocate register for node: ssa{}. Class: {}", Node, PhyReg.Class);
-
-    return PhyReg;
+  [[nodiscard]]
+  FEXCore::IR::RegisterClassType GetRegClass(IR::Ref Node) const {
+    return FEXCore::IR::RegisterClassType {IR::PhysicalRegister(Node).Class};
   }
 
   [[nodiscard]]
@@ -114,7 +146,7 @@ private:
       LOGMAN_THROW_A_FMT(Const == 0, "Only valid constant");
       return ARMEmitter::Reg::zr;
     } else {
-      return GetReg(Src.ID());
+      return GetReg(Src);
     }
   }
 
@@ -213,10 +245,10 @@ private:
     case FEXCore::IR::COND_FGE: return ARMEmitter::Condition::CC_GE;
     case FEXCore::IR::COND_FLEU: return ARMEmitter::Condition::CC_LE;
     case FEXCore::IR::COND_FGT: return ARMEmitter::Condition::CC_GT;
-    case FEXCore::IR::COND_FU: return ARMEmitter::Condition::CC_VS;
-    case FEXCore::IR::COND_FNU: return ARMEmitter::Condition::CC_VC;
-    case FEXCore::IR::COND_VS:
-    case FEXCore::IR::COND_VC:
+    case FEXCore::IR::COND_FU:
+    case FEXCore::IR::COND_VS: return ARMEmitter::Condition::CC_VS;
+    case FEXCore::IR::COND_FNU:
+    case FEXCore::IR::COND_VC: return ARMEmitter::Condition::CC_VC;
     case FEXCore::IR::COND_MI: return ARMEmitter::Condition::CC_MI;
     case FEXCore::IR::COND_PL: return ARMEmitter::Condition::CC_PL;
     default: LOGMAN_MSG_A_FMT("Unsupported compare type"); return ARMEmitter::Condition::CC_NV;
@@ -224,9 +256,34 @@ private:
   }
 
   [[nodiscard]]
-  bool IsFPR(IR::NodeID Node) const;
+  bool IsFPR(IR::RegisterClassType Class) const {
+    return Class == IR::FPRClass || Class == IR::FPRFixedClass;
+  }
+
   [[nodiscard]]
-  bool IsGPR(IR::NodeID Node) const;
+  bool IsGPR(IR::RegisterClassType Class) const {
+    return Class == IR::GPRClass || Class == IR::GPRFixedClass;
+  }
+
+  [[nodiscard]]
+  bool IsGPR(IR::Ref Node) {
+    return IsGPR(GetRegClass(Node));
+  }
+
+  [[nodiscard]]
+  bool IsFPR(IR::Ref Node) {
+    return IsFPR(GetRegClass(Node));
+  }
+
+  [[nodiscard]]
+  bool IsGPR(IR::OrderedNodeWrapper Wrap) {
+    return IsGPR(IR::RegisterClassType {IR::PhysicalRegister(Wrap).Class});
+  }
+
+  [[nodiscard]]
+  bool IsFPR(IR::OrderedNodeWrapper Wrap) {
+    return IsFPR(IR::RegisterClassType {IR::PhysicalRegister(Wrap).Class});
+  }
 
   [[nodiscard]]
   ARMEmitter::ExtendedMemOperand GenerateMemOperand(IR::OpSize AccessSize, ARMEmitter::Register Base, IR::OrderedNodeWrapper Offset,
@@ -255,10 +312,20 @@ private:
     uint32_t End;
   };
 
+  void EmitLinkedBranch(uint64_t GuestRIP, bool Call) {
+    PendingJumpThunks.push_back({GetCursorAddress<uint64_t>(), GuestRIP, {}});
+    auto& Thunk = PendingJumpThunks.back();
+    Bind(&Thunk.Label);
+    if (Call) {
+      bl(&Thunk.Label);
+    } else {
+      b(&Thunk.Label);
+    }
+  }
+
   // This is purely a debugging aid for developers to see if they are in JIT code space when inspecting raw memory
   void EmitDetectionString();
   IR::RegisterAllocationPass* RAPass {};
-  const IR::RegisterAllocationData* RAData {};
   FEXCore::Core::DebugData* DebugData {};
 
   void ResetStack();
@@ -319,7 +386,7 @@ private:
   /**  @} */
 
   uint32_t SpillSlots {};
-  using OpType = void (Arm64JITCore::*)(const IR::IROp_Header* IROp, IR::NodeID Node);
+  using OpType = void (Arm64JITCore::*)(const IR::IROp_Header* IROp, IR::Ref Node);
 
   using ScalarFMAOpCaller =
     std::function<void(ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2, ARMEmitter::VRegister Src3)>;
@@ -341,12 +408,14 @@ private:
 
   void EmitInterruptChecks(bool CheckTF);
 
+  void EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF);
+
   // Runtime selection;
   // Load and store TSO memory style
   OpType RT_LoadMemTSO;
   OpType RT_StoreMemTSO;
 
-#define DEF_OP(x) void Op_##x(IR::IROp_Header const* IROp, IR::NodeID Node)
+#define DEF_OP(x) void Op_##x(IR::IROp_Header const* IROp, IR::Ref Node)
 
   // Dynamic Dispatcher supporting operations
   DEF_OP(ParanoidLoadMemTSO);
@@ -363,6 +432,8 @@ private:
 #undef DEF_OP
 };
 
+#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const* IROp, IR::Ref Node)
+
 [[nodiscard]]
 fextl::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::InternalThreadState* Thread);
 

FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp

@@ -11,11 +11,11 @@ $end_info$
 #include "Interface/Core/ArchHelpers/Arm64Emitter.h"
 #include "Interface/Core/CPUID.h"
 #include "Interface/Core/JIT/JITClass.h"
+#include "Interface/IR/RegisterAllocationData.h"
 #include <FEXCore/Utils/CompilerDefs.h>
 #include <FEXCore/Utils/MathUtils.h>
 
 namespace FEXCore::CPU {
-#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const* IROp, IR::NodeID Node)
 
 DEF_OP(LoadContext) {
   const auto Op = IROp->C<IR::IROp_LoadContext>();
@@ -53,8 +53,8 @@ DEF_OP(LoadContextPair) {
   const auto Op = IROp->C<IR::IROp_LoadContextPair>();
 
   if (Op->Class == FEXCore::IR::GPRClass) {
-    const auto Dst1 = GetReg(Op->OutValue1.ID());
-    const auto Dst2 = GetReg(Op->OutValue2.ID());
+    const auto Dst1 = GetReg(Op->OutValue1);
+    const auto Dst2 = GetReg(Op->OutValue2);
 
     switch (IROp->Size) {
     case IR::OpSize::i32Bit: ldp<ARMEmitter::IndexType::OFFSET>(Dst1.W(), Dst2.W(), STATE, Op->Offset); break;
@@ -62,8 +62,8 @@ DEF_OP(LoadContextPair) {
     default: LOGMAN_MSG_A_FMT("Unhandled LoadMemPair size: {}", IROp->Size); break;
     }
   } else {
-    const auto Dst1 = GetVReg(Op->OutValue1.ID());
-    const auto Dst2 = GetVReg(Op->OutValue2.ID());
+    const auto Dst1 = GetVReg(Op->OutValue1);
+    const auto Dst2 = GetVReg(Op->OutValue2);
 
     switch (IROp->Size) {
     case IR::OpSize::i32Bit: ldp<ARMEmitter::IndexType::OFFSET>(Dst1.S(), Dst2.S(), STATE, Op->Offset); break;
@@ -89,7 +89,7 @@ DEF_OP(StoreContext) {
     default: LOGMAN_MSG_A_FMT("Unhandled StoreContext size: {}", OpSize); break;
     }
   } else {
-    const auto Src = GetVReg(Op->Value.ID());
+    const auto Src = GetVReg(Op->Value);
 
     switch (OpSize) {
     case IR::OpSize::i8Bit: strb(Src, STATE, Op->Offset); break;
@@ -120,8 +120,8 @@ DEF_OP(StoreContextPair) {
     default: LOGMAN_MSG_A_FMT("Unhandled StoreContext size: {}", OpSize); break;
     }
   } else {
-    const auto Src1 = GetVReg(Op->Value1.ID());
-    const auto Src2 = GetVReg(Op->Value2.ID());
+    const auto Src1 = GetVReg(Op->Value1);
+    const auto Src2 = GetVReg(Op->Value2);
 
     switch (OpSize) {
     case IR::OpSize::i32Bit: stp<ARMEmitter::IndexType::OFFSET>(Src1.S(), Src2.S(), STATE, Op->Offset); break;
@@ -137,25 +137,20 @@ DEF_OP(LoadRegister) {
 
   if (Op->Class == IR::GPRClass) {
     LOGMAN_THROW_A_FMT(Op->Reg < StaticRegisters.size(), "out of range reg");
-    const auto reg = StaticRegisters[Op->Reg];
 
-    if (GetReg(Node).Idx() != reg.Idx()) {
-      mov(GetReg(Node).X(), reg.X());
-    }
+    mov(GetReg(Node).X(), StaticRegisters[Op->Reg].X());
   } else if (Op->Class == IR::FPRClass) {
-    [[maybe_unused]] const auto regSize = HostSupportsAVX256 ? IR::OpSize::i256Bit : IR::OpSize::i128Bit;
+    const auto regSize = HostSupportsAVX256 ? IR::OpSize::i256Bit : IR::OpSize::i128Bit;
     LOGMAN_THROW_A_FMT(Op->Reg < StaticFPRegisters.size(), "out of range reg");
     LOGMAN_THROW_A_FMT(IROp->Size == regSize, "expected sized");
 
     const auto guest = StaticFPRegisters[Op->Reg];
     const auto host = GetVReg(Node);
 
-    if (host.Idx() != guest.Idx()) {
-      if (HostSupportsAVX256) {
-        mov(ARMEmitter::SubRegSize::i64Bit, host.Z(), PRED_TMP_32B.Merging(), guest.Z());
-      } else {
-        mov(host.Q(), guest.Q());
-      }
+    if (HostSupportsAVX256) {
+      mov(ARMEmitter::SubRegSize::i64Bit, host.Z(), PRED_TMP_32B.Merging(), guest.Z());
+    } else {
+      mov(host.Q(), guest.Q());
     }
   } else {
     LOGMAN_THROW_A_FMT(false, "Unhandled Op->Class {}", Op->Class);
@@ -180,44 +175,32 @@ DEF_OP(LoadAF) {
 
 DEF_OP(StoreRegister) {
   const auto Op = IROp->C<IR::IROp_StoreRegister>();
+  auto Reg = IR::PhysicalRegister(Node);
 
-  if (Op->Class == IR::GPRClass) {
-    unsigned Reg = Op->Reg == Core::CPUState::PF_AS_GREG ? (StaticRegisters.size() - 2) :
-                   Op->Reg == Core::CPUState::AF_AS_GREG ? (StaticRegisters.size() - 1) :
-                                                           Op->Reg;
-
-    LOGMAN_THROW_A_FMT(Reg < StaticRegisters.size(), "out of range reg");
-    const auto reg = StaticRegisters[Reg];
-    const auto Src = GetReg(Op->Value.ID());
-
-    if (Src.Idx() != reg.Idx()) {
-      // Always use 64-bit, it's faster. Upper bits ignored for 32-bit mode.
-      mov(ARMEmitter::Size::i64Bit, reg, Src);
-    }
-  } else if (Op->Class == IR::FPRClass) {
-    [[maybe_unused]] const auto regSize = HostSupportsAVX256 ? IR::OpSize::i256Bit : IR::OpSize::i128Bit;
-    LOGMAN_THROW_A_FMT(Op->Reg < StaticFPRegisters.size(), "reg out of range");
+  if (Reg.Class == IR::GPRFixedClass) {
+    // Always use 64-bit, it's faster. Upper bits ignored for 32-bit mode.
+    mov(ARMEmitter::Size::i64Bit, GetReg(Reg), GetReg(Op->Value));
+  } else if (Reg.Class == IR::FPRFixedClass) {
+    const auto regSize = HostSupportsAVX256 ? IR::OpSize::i256Bit : IR::OpSize::i128Bit;
     LOGMAN_THROW_A_FMT(IROp->Size == regSize, "expected sized");
 
-    const auto guest = StaticFPRegisters[Op->Reg];
-    const auto host = GetVReg(Op->Value.ID());
+    const auto guest = GetVReg(Reg);
+    const auto host = GetVReg(Op->Value);
 
-    if (guest.Idx() != host.Idx()) {
-      if (HostSupportsAVX256) {
-        mov(ARMEmitter::SubRegSize::i64Bit, guest.Z(), PRED_TMP_32B.Merging(), host.Z());
-      } else {
-        mov(guest.Q(), host.Q());
-      }
+    if (HostSupportsAVX256) {
+      mov(ARMEmitter::SubRegSize::i64Bit, guest.Z(), PRED_TMP_32B.Merging(), host.Z());
+    } else {
+      mov(guest.Q(), host.Q());
     }
   } else {
-    LOGMAN_THROW_A_FMT(false, "Unhandled Op->Class {}", Op->Class);
+    LOGMAN_THROW_A_FMT(false, "Unhandled Op->Class {}", Reg.Class);
   }
 }
 
 DEF_OP(StorePF) {
   const auto Op = IROp->C<IR::IROp_StorePF>();
   const auto reg = StaticRegisters[StaticRegisters.size() - 2];
-  const auto Src = GetReg(Op->Value.ID());
+  const auto Src = GetReg(Op->Value);
 
   if (Src.Idx() != reg.Idx()) {
     // Always use 64-bit, it's faster. Upper bits ignored for 32-bit mode.
@@ -228,7 +211,7 @@ DEF_OP(StorePF) {
 DEF_OP(StoreAF) {
   const auto Op = IROp->C<IR::IROp_StoreAF>();
   const auto reg = StaticRegisters[StaticRegisters.size() - 1];
-  const auto Src = GetReg(Op->Value.ID());
+  const auto Src = GetReg(Op->Value);
 
   if (Src.Idx() != reg.Idx()) {
     // Always use 64-bit, it's faster. Upper bits ignored for 32-bit mode.
@@ -240,7 +223,7 @@ DEF_OP(LoadContextIndexed) {
   const auto Op = IROp->C<IR::IROp_LoadContextIndexed>();
   const auto OpSize = IROp->Size;
 
-  const auto Index = GetReg(Op->Index.ID());
+  const auto Index = GetReg(Op->Index);
 
   if (Op->Class == FEXCore::IR::GPRClass) {
     switch (Op->Stride) {
@@ -303,10 +286,10 @@ DEF_OP(StoreContextIndexed) {
   const auto Op = IROp->C<IR::IROp_StoreContextIndexed>();
   const auto OpSize = IROp->Size;
 
-  const auto Index = GetReg(Op->Index.ID());
+  const auto Index = GetReg(Op->Index);
 
   if (Op->Class == FEXCore::IR::GPRClass) {
-    const auto Value = GetReg(Op->Value.ID());
+    const auto Value = GetReg(Op->Value);
 
     switch (Op->Stride) {
     case 1:
@@ -328,7 +311,7 @@ DEF_OP(StoreContextIndexed) {
     default: LOGMAN_MSG_A_FMT("Unhandled StoreContextIndexed stride: {}", Op->Stride); break;
     }
   } else {
-    const auto Value = GetVReg(Op->Value.ID());
+    const auto Value = GetVReg(Op->Value);
 
     switch (Op->Stride) {
     case 1:
@@ -371,7 +354,7 @@ DEF_OP(SpillRegister) {
   const uint32_t SlotOffset = Op->Slot * MaxSpillSlotSize;
 
   if (Op->Class == FEXCore::IR::GPRClass) {
-    const auto Src = GetReg(Op->Value.ID());
+    const auto Src = GetReg(Op->Value);
     switch (OpSize) {
     case IR::OpSize::i8Bit: {
       if (SlotOffset > LSByteMaxUnsignedOffset) {
@@ -412,7 +395,7 @@ DEF_OP(SpillRegister) {
     default: LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize); break;
     }
   } else if (Op->Class == FEXCore::IR::FPRClass) {
-    const auto Src = GetVReg(Op->Value.ID());
+    const auto Src = GetVReg(Op->Value);
 
     switch (OpSize) {
     case IR::OpSize::i32Bit: {
@@ -552,7 +535,7 @@ DEF_OP(LoadNZCV) {
 DEF_OP(StoreNZCV) {
   auto Op = IROp->C<IR::IROp_StoreNZCV>();
 
-  msr(ARMEmitter::SystemRegister::NZCV, GetReg(Op->Value.ID()));
+  msr(ARMEmitter::SystemRegister::NZCV, GetReg(Op->Value));
 }
 
 DEF_OP(LoadDF) {
@@ -575,7 +558,7 @@ ARMEmitter::ExtendedMemOperand Arm64JITCore::GenerateMemOperand(
     if (IsInlineConstant(Offset, &Const)) {
       return ARMEmitter::ExtendedMemOperand(Base.X(), ARMEmitter::IndexType::OFFSET, Const);
     } else {
-      auto RegOffset = GetReg(Offset.ID());
+      auto RegOffset = GetReg(Offset);
       switch (OffsetType.Val) {
       case IR::MEM_OFFSET_SXTX.Val:
         return ARMEmitter::ExtendedMemOperand(Base.X(), RegOffset.X(), ARMEmitter::ExtendedType::SXTX, FEXCore::ilog2(OffsetScale));
@@ -609,7 +592,7 @@ ARMEmitter::Register Arm64JITCore::ApplyMemOperand(IR::OpSize AccessSize, ARMEmi
     LoadConstant(ARMEmitter::Size::i64Bit, Tmp, Const);
     add(ARMEmitter::Size::i64Bit, Tmp, Base, Tmp, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(OffsetScale));
   } else {
-    auto RegOffset = GetReg(Offset.ID());
+    auto RegOffset = GetReg(Offset);
     switch (OffsetType.Val) {
     case IR::MEM_OFFSET_SXTX.Val:
       add(ARMEmitter::Size::i64Bit, Tmp, Base, RegOffset, ARMEmitter::ExtendedType::SXTX, FEXCore::ilog2(OffsetScale));
@@ -676,7 +659,7 @@ ARMEmitter::SVEMemOperand Arm64JITCore::GenerateSVEMemOperand(IR::OpSize AccessS
   // optional extension or shift as part of their behavior.
   LOGMAN_THROW_A_FMT(OffsetType.Val == IR::MEM_OFFSET_SXTX.Val, "Currently only the default offset type (SXTX) is supported.");
 
-  const auto RegOffset = GetReg(Offset.ID());
+  const auto RegOffset = GetReg(Offset);
   return ARMEmitter::SVEMemOperand(Base.X(), RegOffset.X());
 }
 
@@ -684,7 +667,7 @@ DEF_OP(LoadMem) {
   const auto Op = IROp->C<IR::IROp_LoadMem>();
   const auto OpSize = IROp->Size;
 
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto MemReg = GetReg(Op->Addr);
   const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
 
   if (Op->Class == FEXCore::IR::GPRClass) {
@@ -719,11 +702,11 @@ DEF_OP(LoadMem) {
 
 DEF_OP(LoadMemPair) {
   const auto Op = IROp->C<IR::IROp_LoadMemPair>();
-  const auto Addr = GetReg(Op->Addr.ID());
+  const auto Addr = GetReg(Op->Addr);
 
   if (Op->Class == FEXCore::IR::GPRClass) {
-    const auto Dst1 = GetReg(Op->OutValue1.ID());
-    const auto Dst2 = GetReg(Op->OutValue2.ID());
+    const auto Dst1 = GetReg(Op->OutValue1);
+    const auto Dst2 = GetReg(Op->OutValue2);
 
     switch (IROp->Size) {
     case IR::OpSize::i32Bit: ldp<ARMEmitter::IndexType::OFFSET>(Dst1.W(), Dst2.W(), Addr, Op->Offset); break;
@@ -731,8 +714,8 @@ DEF_OP(LoadMemPair) {
     default: LOGMAN_MSG_A_FMT("Unhandled LoadMemPair size: {}", IROp->Size); break;
     }
   } else {
-    const auto Dst1 = GetVReg(Op->OutValue1.ID());
-    const auto Dst2 = GetVReg(Op->OutValue2.ID());
+    const auto Dst1 = GetVReg(Op->OutValue1);
+    const auto Dst2 = GetVReg(Op->OutValue2);
 
     switch (IROp->Size) {
     case IR::OpSize::i32Bit: ldp<ARMEmitter::IndexType::OFFSET>(Dst1.S(), Dst2.S(), Addr, Op->Offset); break;
@@ -747,7 +730,7 @@ DEF_OP(LoadMemTSO) {
   const auto Op = IROp->C<IR::IROp_LoadMemTSO>();
   const auto OpSize = IROp->Size;
 
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto MemReg = GetReg(Op->Addr);
 
   if (Op->Class == FEXCore::IR::GPRClass) {
     LOGMAN_THROW_A_FMT(Op->Offset.IsInvalid() || CTX->HostFeatures.SupportsTSOImm9, "unexpected offset");
@@ -759,7 +742,7 @@ DEF_OP(LoadMemTSO) {
     const auto Dst = GetReg(Node);
     uint64_t Offset = 0;
     if (!Op->Offset.IsInvalid()) {
-      [[maybe_unused]] bool IsInline = IsInlineConstant(Op->Offset, &Offset);
+      bool IsInline = IsInlineConstant(Op->Offset, &Offset);
       LOGMAN_THROW_A_FMT(IsInline, "expected immediate");
     }
 
@@ -844,8 +827,8 @@ DEF_OP(VLoadVectorMasked) {
   const auto GoverningPredicate = Is256Bit ? PRED_TMP_32B : PRED_TMP_16B;
 
   const auto Dst = GetVReg(Node);
-  const auto MaskReg = GetVReg(Op->Mask.ID());
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto MaskReg = GetVReg(Op->Mask);
+  const auto MemReg = GetReg(Op->Addr);
 
   if (HostSupportsSVE128 || HostSupportsSVE256) {
     const auto MemSrc = GenerateSVEMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
@@ -946,9 +929,9 @@ DEF_OP(VStoreVectorMasked) {
   const auto CMPPredicate = ARMEmitter::PReg::p0;
   const auto GoverningPredicate = Is256Bit ? PRED_TMP_32B : PRED_TMP_16B;
 
-  const auto RegData = GetVReg(Op->Data.ID());
-  const auto MaskReg = GetVReg(Op->Mask.ID());
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto RegData = GetVReg(Op->Data);
+  const auto MaskReg = GetVReg(Op->Mask);
+  const auto MemReg = GetReg(Op->Addr);
   if (HostSupportsSVE128 || HostSupportsSVE256) {
     const auto MemDst = GenerateSVEMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
 
@@ -1171,13 +1154,13 @@ DEF_OP(VLoadVectorGatherMasked) {
   LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
 
   const auto Dst = GetVReg(Node);
-  const auto IncomingDst = GetVReg(Op->Incoming.ID());
+  const auto IncomingDst = GetVReg(Op->Incoming);
 
-  const auto MaskReg = GetVReg(Op->Mask.ID());
-  std::optional<ARMEmitter::Register> BaseAddr = !Op->AddrBase.IsInvalid() ? std::make_optional(GetReg(Op->AddrBase.ID())) : std::nullopt;
-  const auto VectorIndexLow = GetVReg(Op->VectorIndexLow.ID());
+  const auto MaskReg = GetVReg(Op->Mask);
+  std::optional<ARMEmitter::Register> BaseAddr = !Op->AddrBase.IsInvalid() ? std::make_optional(GetReg(Op->AddrBase)) : std::nullopt;
+  const auto VectorIndexLow = GetVReg(Op->VectorIndexLow);
   std::optional<ARMEmitter::VRegister> VectorIndexHigh =
-    !Op->VectorIndexHigh.IsInvalid() ? std::make_optional(GetVReg(Op->VectorIndexHigh.ID())) : std::nullopt;
+    !Op->VectorIndexHigh.IsInvalid() ? std::make_optional(GetVReg(Op->VectorIndexHigh)) : std::nullopt;
 
   ///< If the host supports SVE and the offset scale matches SVE limitations then it can do an SVE style load.
   const bool SupportsSVELoad = (HostSupportsSVE128 || HostSupportsSVE256) &&
@@ -1206,7 +1189,7 @@ DEF_OP(VLoadVectorGatherMasked) {
     if (BaseAddr.has_value() || OffsetScale != 1) {
       ARMEmitter::Register AddrReg = TMP1;
       if (BaseAddr.has_value()) {
-        AddrReg = GetReg(Op->AddrBase.ID());
+        AddrReg = GetReg(Op->AddrBase);
       } else {
         ///< OpcodeDispatcher didn't provide a Base address while SVE requires one.
         LoadConstant(ARMEmitter::Size::i64Bit, AddrReg, 0);
@@ -1255,13 +1238,13 @@ DEF_OP(VLoadVectorGatherMaskedQPS) {
   /// - Matches VGATHERQPS/VPGATHERQD behaviour!
   const auto OffsetScale = Op->OffsetScale;
   const auto Dst = GetVReg(Node);
-  const auto IncomingDst = GetVReg(Op->Incoming.ID());
+  const auto IncomingDst = GetVReg(Op->Incoming);
 
-  const auto MaskReg = GetVReg(Op->MaskReg.ID());
-  std::optional<ARMEmitter::Register> BaseAddr = !Op->AddrBase.IsInvalid() ? std::make_optional(GetReg(Op->AddrBase.ID())) : std::nullopt;
-  const auto VectorIndexLow = GetVReg(Op->VectorIndexLow.ID());
+  const auto MaskReg = GetVReg(Op->MaskReg);
+  std::optional<ARMEmitter::Register> BaseAddr = !Op->AddrBase.IsInvalid() ? std::make_optional(GetReg(Op->AddrBase)) : std::nullopt;
+  const auto VectorIndexLow = GetVReg(Op->VectorIndexLow);
   std::optional<ARMEmitter::VRegister> VectorIndexHigh =
-    !Op->VectorIndexHigh.IsInvalid() ? std::make_optional(GetVReg(Op->VectorIndexHigh.ID())) : std::nullopt;
+    !Op->VectorIndexHigh.IsInvalid() ? std::make_optional(GetVReg(Op->VectorIndexHigh)) : std::nullopt;
 
   ///< If the host supports SVE and the offset scale matches SVE limitations then it can do an SVE style load.
   if (HostSupportsSVE128 && (OffsetScale == 1 || OffsetScale == 4)) {
@@ -1330,8 +1313,8 @@ DEF_OP(VLoadVectorElement) {
   const auto ElementSize = IROp->ElementSize;
 
   const auto Dst = GetVReg(Node);
-  const auto DstSrc = GetVReg(Op->DstSrc.ID());
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto DstSrc = GetVReg(Op->DstSrc);
+  const auto MemReg = GetReg(Op->Addr);
 
   LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i8Bit || ElementSize == IR::OpSize::i16Bit || ElementSize == IR::OpSize::i32Bit ||
                        ElementSize == IR::OpSize::i64Bit || ElementSize == IR::OpSize::i128Bit,
@@ -1367,8 +1350,8 @@ DEF_OP(VStoreVectorElement) {
   const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
   const auto ElementSize = IROp->ElementSize;
 
-  const auto Value = GetVReg(Op->Value.ID());
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto Value = GetVReg(Op->Value);
+  const auto MemReg = GetReg(Op->Addr);
 
   LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i8Bit || ElementSize == IR::OpSize::i16Bit || ElementSize == IR::OpSize::i32Bit ||
                        ElementSize == IR::OpSize::i64Bit || ElementSize == IR::OpSize::i128Bit,
@@ -1403,7 +1386,7 @@ DEF_OP(VBroadcastFromMem) {
   const auto ElementSize = IROp->ElementSize;
 
   const auto Dst = GetVReg(Node);
-  const auto MemReg = GetReg(Op->Address.ID());
+  const auto MemReg = GetReg(Op->Address);
 
   LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i8Bit || ElementSize == IR::OpSize::i16Bit || ElementSize == IR::OpSize::i32Bit ||
                        ElementSize == IR::OpSize::i64Bit || ElementSize == IR::OpSize::i128Bit,
@@ -1444,8 +1427,8 @@ DEF_OP(VBroadcastFromMem) {
 DEF_OP(Push) {
   const auto Op = IROp->C<IR::IROp_Push>();
   const auto ValueSize = IR::OpSizeToSize(Op->ValueSize);
-  auto Src = GetReg(Op->Value.ID());
-  const auto AddrSrc = GetReg(Op->Addr.ID());
+  auto Src = GetReg(Op->Value);
+  const auto AddrSrc = GetReg(Op->Addr);
   const auto Dst = GetReg(Node);
 
   bool NeedsMoveAfterwards = false;
@@ -1526,11 +1509,34 @@ DEF_OP(Push) {
   }
 }
 
+DEF_OP(PushTwo) {
+  const auto Op = IROp->C<IR::IROp_PushTwo>();
+  const auto ValueSize = IR::OpSizeToSize(Op->ValueSize);
+  auto Src1 = GetReg(Op->Value1);
+  auto Src2 = GetReg(Op->Value2);
+  const auto Dst = GetReg(Op->Addr);
+
+  switch (ValueSize) {
+  case 4: {
+    stp<ARMEmitter::IndexType::PRE>(Src1.W(), Src2.W(), Dst, -2 * ValueSize);
+    break;
+  }
+  case 8: {
+    stp<ARMEmitter::IndexType::PRE>(Src1.X(), Src2.X(), Dst, -2 * ValueSize);
+    break;
+  }
+  default: {
+    LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, ValueSize);
+    break;
+  }
+  }
+}
+
 DEF_OP(Pop) {
   const auto Op = IROp->C<IR::IROp_Pop>();
   const auto Size = IR::OpSizeToSize(Op->Size);
-  const auto Addr = GetReg(Op->InoutAddr.ID());
-  const auto Dst = GetReg(Op->OutValue.ID());
+  const auto Addr = GetReg(Op->InoutAddr);
+  const auto Dst = GetReg(Op->OutValue);
 
   LOGMAN_THROW_A_FMT(Dst != Addr, "Invalid");
 
@@ -1558,11 +1564,42 @@ DEF_OP(Pop) {
   }
 }
 
+DEF_OP(PopTwo) {
+  const auto Op = IROp->C<IR::IROp_PopTwo>();
+  const auto Size = IR::OpSizeToSize(Op->Size);
+  const auto Addr = GetReg(Op->InoutAddr);
+  auto Dst1 = GetReg(Op->OutValue1);
+  const auto Dst2 = GetReg(Op->OutValue2);
+
+  // ldp x, x is invalid. Explicitly discard the first destination to encode.
+  if (Dst1 == Dst2) {
+    Dst1 = ARMEmitter::Reg::zr;
+  }
+
+  LOGMAN_THROW_A_FMT(Dst1 != Addr && Dst2 != Addr, "Invalid");
+  LOGMAN_THROW_A_FMT(Dst1 != Dst2, "Invalid");
+
+  switch (Size) {
+  case 4: {
+    ldp<ARMEmitter::IndexType::POST>(Dst1.W(), Dst2.W(), Addr, 2 * Size);
+    break;
+  }
+  case 8: {
+    ldp<ARMEmitter::IndexType::POST>(Dst1.X(), Dst2.X(), Addr, 2 * Size);
+    break;
+  }
+  default: {
+    LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Op->Size);
+    break;
+  }
+  }
+}
+
 DEF_OP(StoreMem) {
   const auto Op = IROp->C<IR::IROp_StoreMem>();
   const auto OpSize = IROp->Size;
 
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto MemReg = GetReg(Op->Addr);
   const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
 
   if (Op->Class == FEXCore::IR::GPRClass) {
@@ -1575,7 +1612,7 @@ DEF_OP(StoreMem) {
     default: LOGMAN_MSG_A_FMT("Unhandled StoreMem size: {}", OpSize); break;
     }
   } else {
-    const auto Src = GetVReg(Op->Value.ID());
+    const auto Src = GetVReg(Op->Value);
 
     switch (OpSize) {
     case IR::OpSize::i8Bit: {
@@ -1615,8 +1652,8 @@ DEF_OP(StoreMemX87SVEOptPredicate) {
 
   LOGMAN_THROW_A_FMT(HostSupportsSVE128 || HostSupportsSVE256, "StoreMemX87SVEOptPredicate needs SVE support");
 
-  const auto RegData = GetVReg(Op->Value.ID());
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto RegData = GetVReg(Op->Value);
+  const auto MemReg = GetReg(Op->Addr);
   const auto MemDst = ARMEmitter::SVEMemOperand(MemReg.X(), 0);
 
   switch (IROp->ElementSize) {
@@ -1644,7 +1681,7 @@ DEF_OP(LoadMemX87SVEOptPredicate) {
   const auto Op = IROp->C<IR::IROp_LoadMemX87SVEOptPredicate>();
   const auto Dst = GetVReg(Node);
   const auto Predicate = PRED_X87_SVEOPT;
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto MemReg = GetReg(Op->Addr);
 
   LOGMAN_THROW_A_FMT(HostSupportsSVE128 || HostSupportsSVE256, "LoadMemX87SVEOptPredicate needs SVE support");
 
@@ -1674,7 +1711,7 @@ DEF_OP(LoadMemX87SVEOptPredicate) {
 DEF_OP(StoreMemPair) {
   const auto Op = IROp->C<IR::IROp_StoreMemPair>();
   const auto OpSize = IROp->Size;
-  const auto Addr = GetReg(Op->Addr.ID());
+  const auto Addr = GetReg(Op->Addr);
 
   if (Op->Class == FEXCore::IR::GPRClass) {
     const auto Src1 = GetZeroableReg(Op->Value1);
@@ -1685,8 +1722,8 @@ DEF_OP(StoreMemPair) {
     default: LOGMAN_MSG_A_FMT("Unhandled StoreMem size: {}", OpSize); break;
     }
   } else {
-    const auto Src1 = GetVReg(Op->Value1.ID());
-    const auto Src2 = GetVReg(Op->Value2.ID());
+    const auto Src1 = GetVReg(Op->Value1);
+    const auto Src2 = GetVReg(Op->Value2);
 
     switch (OpSize) {
     case IR::OpSize::i32Bit: stp<ARMEmitter::IndexType::OFFSET>(Src1.S(), Src2.S(), Addr, Op->Offset); break;
@@ -1701,7 +1738,7 @@ DEF_OP(StoreMemTSO) {
   const auto Op = IROp->C<IR::IROp_StoreMemTSO>();
   const auto OpSize = IROp->Size;
 
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto MemReg = GetReg(Op->Addr);
 
   if (Op->Class == FEXCore::IR::GPRClass) {
     LOGMAN_THROW_A_FMT(Op->Offset.IsInvalid() || CTX->HostFeatures.SupportsTSOImm9, "unexpected offset");
@@ -1713,7 +1750,7 @@ DEF_OP(StoreMemTSO) {
     const auto Src = GetZeroableReg(Op->Value);
     uint64_t Offset = 0;
     if (!Op->Offset.IsInvalid()) {
-      [[maybe_unused]] bool IsInline = IsInlineConstant(Op->Offset, &Offset);
+      bool IsInline = IsInlineConstant(Op->Offset, &Offset);
       LOGMAN_THROW_A_FMT(IsInline, "expected immediate");
     }
 
@@ -1751,7 +1788,7 @@ DEF_OP(StoreMemTSO) {
       // Half-Barrier.
       dmb(ARMEmitter::BarrierScope::ISH);
     }
-    const auto Src = GetVReg(Op->Value.ID());
+    const auto Src = GetVReg(Op->Value);
     const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
     switch (OpSize) {
     case IR::OpSize::i8Bit: strb(Src, MemSrc); break;
@@ -1782,16 +1819,16 @@ DEF_OP(MemSet) {
 
   const bool IsAtomic = CTX->IsMemcpyAtomicTSOEnabled();
   const auto Size = IR::OpSizeToSize(Op->Size);
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto MemReg = GetReg(Op->Addr);
   const auto Value = GetZeroableReg(Op->Value);
-  const auto Length = GetReg(Op->Length.ID());
+  const auto Length = GetReg(Op->Length);
   const auto Dst = GetReg(Node);
 
   uint64_t DirectionConstant;
   bool DirectionIsInline = IsInlineConstant(Op->Direction, &DirectionConstant);
   ARMEmitter::Register DirectionReg = ARMEmitter::Reg::r0;
   if (!DirectionIsInline) {
-    DirectionReg = GetReg(Op->Direction.ID());
+    DirectionReg = GetReg(Op->Direction);
   }
 
   // If Direction > 0 then:
@@ -1808,7 +1845,7 @@ DEF_OP(MemSet) {
   if (Op->Prefix.IsInvalid()) {
     mov(TMP2, MemReg.X());
   } else {
-    const auto Prefix = GetReg(Op->Prefix.ID());
+    const auto Prefix = GetReg(Op->Prefix);
     add(TMP2, Prefix.X(), MemReg.X());
   }
 
@@ -1971,19 +2008,19 @@ DEF_OP(MemCpy) {
 
   const bool IsAtomic = CTX->IsMemcpyAtomicTSOEnabled();
   const auto Size = IR::OpSizeToSize(Op->Size);
-  const auto MemRegDest = GetReg(Op->Dest.ID());
-  const auto MemRegSrc = GetReg(Op->Src.ID());
+  const auto MemRegDest = GetReg(Op->Dest);
+  const auto MemRegSrc = GetReg(Op->Src);
 
-  const auto Length = GetReg(Op->Length.ID());
+  const auto Length = GetReg(Op->Length);
   uint64_t DirectionConstant;
   bool DirectionIsInline = IsInlineConstant(Op->Direction, &DirectionConstant);
   ARMEmitter::Register DirectionReg = ARMEmitter::Reg::r0;
   if (!DirectionIsInline) {
-    DirectionReg = GetReg(Op->Direction.ID());
+    DirectionReg = GetReg(Op->Direction);
   }
 
-  auto Dst0 = GetReg(Op->OutDstAddress.ID());
-  auto Dst1 = GetReg(Op->OutSrcAddress.ID());
+  auto Dst0 = GetReg(Op->OutDstAddress);
+  auto Dst1 = GetReg(Op->OutSrcAddress);
   // If Direction > 0 then:
   //   MemRegDest is incremented (by size)
   //   MemRegSrc is incremented (by size)
@@ -2241,7 +2278,7 @@ DEF_OP(ParanoidLoadMemTSO) {
   const auto Op = IROp->C<IR::IROp_LoadMemTSO>();
   const auto OpSize = IROp->Size;
 
-  auto MemReg = GetReg(Op->Addr.ID());
+  auto MemReg = GetReg(Op->Addr);
 
   if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == FEXCore::IR::GPRClass) {
     const auto Dst = GetReg(Node);
@@ -2329,7 +2366,7 @@ DEF_OP(ParanoidStoreMemTSO) {
   const auto Op = IROp->C<IR::IROp_StoreMemTSO>();
   const auto OpSize = IROp->Size;
 
-  auto MemReg = GetReg(Op->Addr.ID());
+  auto MemReg = GetReg(Op->Addr);
 
   if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == FEXCore::IR::GPRClass) {
     const auto Src = GetZeroableReg(Op->Value);
@@ -2362,7 +2399,7 @@ DEF_OP(ParanoidStoreMemTSO) {
     default: LOGMAN_MSG_A_FMT("Unhandled ParanoidStoreMemTSO size: {}", OpSize); break;
     }
   } else {
-    const auto Src = GetVReg(Op->Value.ID());
+    const auto Src = GetVReg(Op->Value);
 
     MemReg = ApplyMemOperand(OpSize, MemReg, TMP4, Op->Offset, Op->OffsetType, Op->OffsetScale);
 
@@ -2416,7 +2453,7 @@ DEF_OP(CacheLineClear) {
 
   auto Op = IROp->C<IR::IROp_CacheLineClear>();
 
-  auto MemReg = GetReg(Op->Addr.ID());
+  auto MemReg = GetReg(Op->Addr);
 
   // Clear dcache only
   // icache doesn't matter here since the guest application shouldn't be calling clflush on JIT code.
@@ -2445,7 +2482,7 @@ DEF_OP(CacheLineClean) {
 
   auto Op = IROp->C<IR::IROp_CacheLineClean>();
 
-  auto MemReg = GetReg(Op->Addr.ID());
+  auto MemReg = GetReg(Op->Addr);
 
   // Clean dcache only
   if (CTX->HostFeatures.DCacheLineSize >= 64U) {
@@ -2463,7 +2500,7 @@ DEF_OP(CacheLineClean) {
 DEF_OP(CacheLineZero) {
   auto Op = IROp->C<IR::IROp_CacheLineZero>();
 
-  auto MemReg = GetReg(Op->Addr.ID());
+  auto MemReg = GetReg(Op->Addr);
 
   if (CTX->HostFeatures.SupportsCLZERO) {
     // We can use this instruction directly
@@ -2483,7 +2520,7 @@ DEF_OP(CacheLineZero) {
 
 DEF_OP(Prefetch) {
   auto Op = IROp->C<IR::IROp_Prefetch>();
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto MemReg = GetReg(Op->Addr);
 
   // Access size is only ever handled as 8-byte. Even though it is accesssed as a cacheline.
   const auto MemSrc = GenerateMemOperand(IR::OpSize::i64Bit, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
@@ -2527,8 +2564,8 @@ DEF_OP(VStoreNonTemporal) {
   LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
   const auto Is128Bit = OpSize == IR::OpSize::i128Bit;
 
-  const auto Value = GetVReg(Op->Value.ID());
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto Value = GetVReg(Op->Value);
+  const auto MemReg = GetReg(Op->Addr);
   const auto Offset = Op->Offset;
 
   if (Is256Bit) {
@@ -2549,13 +2586,13 @@ DEF_OP(VStoreNonTemporalPair) {
   const auto Op = IROp->C<IR::IROp_VStoreNonTemporalPair>();
   const auto OpSize = IROp->Size;
 
-  [[maybe_unused]] const auto Is128Bit = OpSize == IR::OpSize::i128Bit;
+  const auto Is128Bit = OpSize == IR::OpSize::i128Bit;
   LOGMAN_THROW_A_FMT(Is128Bit, "This IR operation only operates at 128-bit wide");
 
-  const auto ValueLow = GetVReg(Op->ValueLow.ID());
-  const auto ValueHigh = GetVReg(Op->ValueHigh.ID());
+  const auto ValueLow = GetVReg(Op->ValueLow);
+  const auto ValueHigh = GetVReg(Op->ValueHigh);
 
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto MemReg = GetReg(Op->Addr);
   const auto Offset = Op->Offset;
 
   stnp(ValueLow.Q(), ValueHigh.Q(), MemReg, Offset);
@@ -2570,7 +2607,7 @@ DEF_OP(VLoadNonTemporal) {
   const auto Is128Bit = OpSize == IR::OpSize::i128Bit;
 
   const auto Dst = GetVReg(Node);
-  const auto MemReg = GetReg(Op->Addr.ID());
+  const auto MemReg = GetReg(Op->Addr);
   const auto Offset = Op->Offset;
 
   if (Is256Bit) {
@@ -2587,5 +2624,4 @@ DEF_OP(VLoadNonTemporal) {
   }
 }
 
-#undef DEF_OP
 } // namespace FEXCore::CPU

FEXCore/Source/Interface/Core/JIT/MiscOps.cpp

@@ -16,11 +16,26 @@ $end_info$
 #include <FEXCore/Core/SignalDelegator.h>
 
 namespace FEXCore::CPU {
-#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const* IROp, IR::NodeID Node)
 
-DEF_OP(AllocateGPR) {}
-DEF_OP(AllocateGPRAfter) {}
-DEF_OP(AllocateFPR) {}
+DEF_OP(WFET) {
+  auto Op = IROp->C<IR::IROp_WFET>();
+  const auto Lower = GetReg(Op->Lower);
+  const auto Upper = GetReg(Op->Upper);
+
+  // Combine registers.
+  mov(ARMEmitter::Size::i64Bit, TMP1, Lower);
+  bfi(ARMEmitter::Size::i64Bit, TMP1, Upper, 32, 32);
+  if (CTX->Config.TSCScale) {
+    // Scale back to ARM64 TSC scale if necessary
+    lsr(ARMEmitter::Size::i64Bit, TMP1, TMP1, CTX->Config.TSCScale);
+  }
+
+  // Clear the exclusive monitor so it can't spuriously wake up with that event.
+  clrex();
+
+  // Execute wfet to wait until the TSC.
+  wfet(TMP1);
+}
 
 DEF_OP(GuestOpcode) {
   auto Op = IROp->C<IR::IROp_GuestOpcode>();
@@ -102,8 +117,8 @@ DEF_OP(GetRoundingMode) {
 
 DEF_OP(SetRoundingMode) {
   auto Op = IROp->C<IR::IROp_SetRoundingMode>();
-  auto Src = GetReg(Op->RoundMode.ID());
-  auto MXCSR = GetReg(Op->MXCSR.ID());
+  auto Src = GetReg(Op->RoundMode);
+  auto MXCSR = GetReg(Op->MXCSR);
 
   // As above, setup the rounding flags in [31:30]
   rbit(ARMEmitter::Size::i32Bit, TMP2, Src);
@@ -161,7 +176,7 @@ DEF_OP(PushRoundingMode) {
 
 DEF_OP(PopRoundingMode) {
   auto Op = IROp->C<IR::IROp_PopRoundingMode>();
-  msr(ARMEmitter::SystemRegister::FPCR, GetReg(Op->FPCR.ID()));
+  msr(ARMEmitter::SystemRegister::FPCR, GetReg(Op->FPCR));
 }
 
 DEF_OP(Print) {
@@ -170,17 +185,17 @@ DEF_OP(Print) {
   PushDynamicRegs(TMP1);
   SpillStaticRegs(TMP1);
 
-  if (IsGPR(Op->Value.ID())) {
-    mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetReg(Op->Value.ID()));
+  if (IsGPR(Op->Value)) {
+    mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetReg(Op->Value));
     ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintValue));
   } else {
-    fmov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetVReg(Op->Value.ID()), false);
-    fmov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, GetVReg(Op->Value.ID()), true);
+    fmov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetVReg(Op->Value), false);
+    fmov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, GetVReg(Op->Value), true);
     ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintVectorValue));
   }
 
   if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
-    if (IsGPR(Op->Value.ID())) {
+    if (IsGPR(Op->Value)) {
       GenerateIndirectRuntimeCall<void, uint64_t>(ARMEmitter::Reg::r3);
     } else {
       GenerateIndirectRuntimeCall<void, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
@@ -271,5 +286,43 @@ DEF_OP(Yield) {
   yield();
 }
 
-#undef DEF_OP
+DEF_OP(MonoBackpatcherWrite) {
+  auto Op = IROp->C<IR::IROp_MonoBackpatcherWrite>();
+
+  mov(ARMEmitter::Size::i64Bit, TMP3, GetReg(Op->Addr));
+  mov(ARMEmitter::Size::i64Bit, TMP4, GetReg(Op->Value));
+
+  PushDynamicRegs(TMP1);
+  SpillStaticRegs(TMP1);
+
+  mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, STATE.R());
+  mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, IR::OpSizeToSize(Op->Size));
+
+  if (!TMP_ABIARGS) {
+    mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, TMP3);
+    mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, TMP4);
+  }
+
+#ifdef _M_ARM_64EC
+  ldr(TMP2, ARMEmitter::XReg::x18, TEB_CPU_AREA_OFFSET);
+  LoadConstant(ARMEmitter::Size::i32Bit, TMP1, 1);
+  strb(TMP1.W(), TMP2, CPU_AREA_IN_SYSCALL_CALLBACK_OFFSET);
+#endif
+
+  ldr(ARMEmitter::XReg::x4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.MonoBackpatcherWrite));
+  if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
+    GenerateIndirectRuntimeCall<void, void*, uint8_t, uint64_t, uint64_t>(ARMEmitter::Reg::r4);
+  } else {
+    blr(ARMEmitter::Reg::r4);
+  }
+
+#ifdef _M_ARM_64EC
+  ldr(TMP2, ARMEmitter::XReg::x18, TEB_CPU_AREA_OFFSET);
+  strb(ARMEmitter::WReg::zr, TMP2, CPU_AREA_IN_SYSCALL_CALLBACK_OFFSET);
+#endif
+
+  FillStaticRegs();
+  PopDynamicRegs();
+}
+
 } // namespace FEXCore::CPU

FEXCore/Source/Interface/Core/JIT/MoveOps.cpp

@@ -8,26 +8,19 @@ $end_info$
 #include "Interface/Core/JIT/JITClass.h"
 
 namespace FEXCore::CPU {
-#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const* IROp, IR::NodeID Node)
 DEF_OP(Copy) {
   auto Op = IROp->C<IR::IROp_Copy>();
 
-  mov(ARMEmitter::Size::i64Bit, GetReg(Node), GetReg(Op->Source.ID()));
+  mov(ARMEmitter::Size::i64Bit, GetReg(Node), GetReg(Op->Source));
 }
 
 DEF_OP(RMWHandle) {
-  auto Op = IROp->C<IR::IROp_RMWHandle>();
-  auto Dest = GetReg(Node);
-  auto Src = GetReg(Op->Value.ID());
-
-  if (Dest != Src) {
-    mov(ARMEmitter::Size::i64Bit, Dest, Src);
-  }
+  mov(ARMEmitter::Size::i64Bit, GetReg(Node), GetReg(IROp->Args[0]));
 }
 
 DEF_OP(Swap1) {
   auto Op = IROp->C<IR::IROp_Swap1>();
-  auto A = GetReg(Op->A.ID()), B = GetReg(Op->B.ID());
+  auto A = GetReg(Op->A), B = GetReg(Op->B);
   LOGMAN_THROW_A_FMT(B == GetReg(Node), "Invariant");
 
   mov(ARMEmitter::Size::i64Bit, TMP1, A);
@@ -39,5 +32,4 @@ DEF_OP(Swap2) {
   // Implemented above
 }
 
-#undef DEF_OP
 } // namespace FEXCore::CPU

FEXCore/Source/Interface/Core/LookupCache.cpp

@@ -14,14 +14,17 @@ $end_info$
 #include "Interface/Core/LookupCache.h"
 
 namespace FEXCore {
-LookupCache::LookupCache(FEXCore::Context::ContextImpl* CTX)
-  : BlockLinks_mbr {fextl::pmr::get_default_resource()}
-  , ctx {CTX} {
-
-  TotalCacheSize = ctx->Config.VirtualMemSize / 4096 * 8 + CODE_SIZE + L1_SIZE;
+GuestToHostMap::GuestToHostMap()
+  : BlockLinks_mbr {fextl::pmr::get_default_resource()} {
   BlockLinks_pma = fextl::make_unique<std::pmr::polymorphic_allocator<std::byte>>(&BlockLinks_mbr);
   // Setup our PMR map.
   BlockLinks = BlockLinks_pma->new_object<BlockLinksMapType>();
+}
+
+LookupCache::LookupCache(FEXCore::Context::ContextImpl* CTX)
+  : ctx {CTX} {
+
+  TotalCacheSize = ctx->Config.VirtualMemSize / 4096 * 8 + CODE_SIZE + L1_SIZE;
 
   // Block cache ends up looking like this
   // PageMemoryMap[VirtualMemoryRegion >> 12]
@@ -62,18 +65,30 @@ LookupCache::~LookupCache() {
 }
 
 void LookupCache::ClearL2Cache() {
-  std::lock_guard<std::recursive_mutex> lk(WriteLock);
+  auto lk = Shared->AcquireLock();
   // Clear out the page memory
   // PagePointer and PageMemory are sequential with each other. Clear both at once.
   FEXCore::Allocator::VirtualDontNeed(reinterpret_cast<void*>(PagePointer), ctx->Config.VirtualMemSize / 4096 * 8 + CODE_SIZE, false);
   AllocateOffset = 0;
 }
 
-void LookupCache::ClearCache() {
-  std::lock_guard<std::recursive_mutex> lk(WriteLock);
+void LookupCache::ClearThreadLocalCaches() {
+  auto lk = Shared->AcquireLock();
 
   // Clear L1 and L2 by clearing the full cache.
   FEXCore::Allocator::VirtualDontNeed(reinterpret_cast<void*>(PagePointer), TotalCacheSize, false);
+}
+
+void LookupCache::ClearCache() {
+  auto lk = Shared->AcquireLock();
+
+  // Clear L1 and L2 by clearing the full cache.
+  FEXCore::Allocator::VirtualDontNeed(reinterpret_cast<void*>(PagePointer), TotalCacheSize, false);
+
+  Shared->ClearCache(lk);
+}
+
+void GuestToHostMap::ClearCache(const LockToken&) {
   // Allocate a new pointer from the BlockLinks pma again.
   BlockLinks = BlockLinks_pma->new_object<BlockLinksMapType>();
   // All code is gone, clear the block list

FEXCore/Source/Interface/Core/LookupCache.h

@@ -16,6 +16,100 @@
 
 namespace FEXCore {
 
+struct GuestToHostMap {
+  std::recursive_mutex WriteLock;
+
+  struct LockToken {
+    std::lock_guard<std::recursive_mutex> Lock;
+  };
+
+  [[nodiscard]]
+  LockToken AcquireLock() {
+    return LockToken {std::lock_guard {WriteLock}};
+  }
+
+  struct BlockLinkTag {
+    uint64_t GuestDestination;
+    FEXCore::Context::ExitFunctionLinkData* HostLink;
+
+    bool operator<(const BlockLinkTag& other) const {
+      if (GuestDestination < other.GuestDestination) {
+        return true;
+      } else if (GuestDestination == other.GuestDestination) {
+        return HostLink < other.HostLink;
+      } else {
+        return false;
+      }
+    }
+  };
+
+  // Use a monotonic buffer resource to allocate both the std::pmr::map and its members.
+  // This allows us to quickly clear the block link map by clearing the monotonic allocator.
+  // If we had allocated the block link map without the MBR, then clearing the map would require slowly
+  // walking each block member and destructing objects.
+  //
+  // This makes `BlockLinks` look like a raw pointer that could memory leak, but since it is backed by the MBR, it won't.
+  std::pmr::monotonic_buffer_resource BlockLinks_mbr;
+  using BlockLinksMapType = std::pmr::map<BlockLinkTag, FEXCore::Context::BlockDelinkerFunc>;
+  fextl::unique_ptr<std::pmr::polymorphic_allocator<std::byte>> BlockLinks_pma;
+  BlockLinksMapType* BlockLinks;
+
+  fextl::robin_map<uint64_t, uint64_t> BlockList;
+
+  fextl::map<uint64_t, fextl::vector<uint64_t>> CodePages;
+
+  GuestToHostMap();
+
+  // Adds to Guest -> Host code mapping
+  void AddBlockMapping(uint64_t Address, void* HostCode, const LockToken&) {
+    // This may replace an existing mapping
+    // NOTE: Generally no previous entry should exist, however there is one exception:
+    //       If the backend updates the active thread's CodeBuffer, the new associated LookupCache
+    //       may already contain the block address. Since is comparatively rare, we'll just leak
+    //       one of the two blocks in this case.
+    BlockList[Address] = (uintptr_t)HostCode;
+  }
+
+  std::optional<uintptr_t> FindBlock(uint64_t Address, const LockToken&) {
+    auto HostCode = BlockList.find(Address);
+    if (HostCode == BlockList.end()) {
+      return std::nullopt;
+    }
+    return HostCode->second;
+  }
+
+  bool Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address, const LockToken&) {
+    // Sever any links to this block
+    auto lower = BlockLinks->lower_bound({Address, nullptr});
+    auto upper = BlockLinks->upper_bound({Address, reinterpret_cast<FEXCore::Context::ExitFunctionLinkData*>(UINTPTR_MAX)});
+    for (auto it = lower; it != upper; it = BlockLinks->erase(it)) {
+      it->second(Frame, it->first.HostLink);
+    }
+
+    // Remove from BlockList
+    return BlockList.erase(Address) != 0;
+  }
+
+  void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink,
+                    const FEXCore::Context::BlockDelinkerFunc& delinker, const LockToken&) {
+    BlockLinks->insert({{GuestDestination, HostLink}, delinker});
+  }
+
+  bool AddBlockExecutableRange(const fextl::set<uint64_t>& Addresses, uint64_t Start, uint64_t Length, const LockToken&) {
+    bool rv = false;
+
+    for (auto CurrentPage = Start >> 12, EndPage = (Start + Length - 1) >> 12; CurrentPage <= EndPage; CurrentPage++) {
+      auto& CodePage = CodePages[CurrentPage];
+      rv |= CodePage.empty();
+      CodePage.insert(CodePage.end(), Addresses.begin(), Addresses.end());
+    }
+
+    return rv;
+  }
+
+  void ClearCache(const LockToken&);
+};
+
 class LookupCache {
 public:
   struct LookupCacheEntry {
@@ -26,6 +120,13 @@ public:
   LookupCache(FEXCore::Context::ContextImpl* CTX);
   ~LookupCache();
 
+  // Swaps out the underlying GuestToHostMap and clears all associated caches.
+  // This interface requires the previous CodeBuffer to be provided despite not using it. This ensures the shared write lock is still valid.
+  void ChangeGuestToHostMapping([[maybe_unused]] CPU::CodeBuffer& Prev, GuestToHostMap& NewMap) {
+    ClearThreadLocalCaches();
+    Shared = &NewMap;
+  }
+
   uintptr_t FindBlock(uint64_t Address) {
     // Try L1, no lock needed
     auto& L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
@@ -34,7 +135,7 @@ public:
     }
 
     // L2 and L3 need to be locked
-    std::lock_guard<std::recursive_mutex> lk(WriteLock);
+    auto lk = Shared->AcquireLock();
 
     // Try L2
     const auto PageIndex = (Address & (VirtualMemSize - 1)) >> 12;
@@ -56,41 +157,30 @@ public:
     }
 
     // Try L3
-    auto HostCode = BlockList.find(Address);
-
-    if (HostCode != BlockList.end()) {
-      CacheBlockMapping(Address, HostCode->second);
-      return HostCode->second;
+    auto HostCode = Shared->FindBlock(Address, lk);
+    if (HostCode) {
+      CacheBlockMapping(Address, HostCode.value());
+      return HostCode.value();
     }
 
     // Failed to find
     return 0;
   }
 
-  fextl::map<uint64_t, fextl::vector<uint64_t>> CodePages;
+  GuestToHostMap* Shared = nullptr;
 
-  // Appends Block {Address} to CodePages [Start, Start + Length)
+  // Appends a list of Block {Address} to CodePages [Start, Start + Length)
   // Returns true if new pages are marked as containing code
-  bool AddBlockExecutableRange(uint64_t Address, uint64_t Start, uint64_t Length) {
-    std::lock_guard<std::recursive_mutex> lk(WriteLock);
-
-    bool rv = false;
-
-    for (auto CurrentPage = Start >> 12, EndPage = (Start + Length - 1) >> 12; CurrentPage <= EndPage; CurrentPage++) {
-      auto& CodePage = CodePages[CurrentPage];
-      rv |= CodePage.size() == 0;
-      CodePage.push_back(Address);
-    }
-
-    return rv;
+  bool AddBlockExecutableRange(const fextl::set<uint64_t>& Addresses, uint64_t Start, uint64_t Length) {
+    auto lk = Shared->AcquireLock();
+    return Shared->AddBlockExecutableRange(Addresses, Start, Length, lk);
   }
 
   // Adds to Guest -> Host code mapping
   void AddBlockMapping(uint64_t Address, void* HostCode) {
-    std::lock_guard<std::recursive_mutex> lk(WriteLock);
+    auto lk = Shared->AcquireLock();
 
-    [[maybe_unused]] auto Inserted = BlockList.emplace(Address, (uintptr_t)HostCode).second;
-    LOGMAN_THROW_A_FMT(Inserted, "Duplicate block mapping added");
+    Shared->AddBlockMapping(Address, HostCode, lk);
 
     // There is no need to update L1 or L2, they will get updated on first lookup
     // However, adding to L1 here increases performance
@@ -99,24 +189,19 @@ public:
     L1Entry.HostCode = (uintptr_t)HostCode;
   }
 
-  void Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address) {
+  // NOTE: It's the caller's responsibility to call Erase() for all other
+  //       GuestToHostMaps that share the same LookupCache. Otherwise, the
+  //       L1/L2 caches will contain stale references to deallocated memory.
+  bool Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address) {
+    auto lk = Shared->AcquireLock();
 
-    std::lock_guard<std::recursive_mutex> lk(WriteLock);
-
-    // Sever any links to this block
-    auto lower = BlockLinks->lower_bound({Address, nullptr});
-    auto upper = BlockLinks->upper_bound({Address, reinterpret_cast<FEXCore::Context::ExitFunctionLinkData*>(UINTPTR_MAX)});
-    for (auto it = lower; it != upper; it = BlockLinks->erase(it)) {
-      it->second(Frame, it->first.HostLink);
-    }
-
-    // Remove from BlockList
-    BlockList.erase(Address);
+    bool ErasedAny = Shared->Erase(Frame, Address, lk);
 
     // Do L1
     auto& L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
     if (L1Entry.GuestCode == Address) {
       L1Entry.GuestCode = 0;
+      ErasedAny = true;
       // Leave L1Entry.HostCode as is, so that concurrent lookups won't read a null pointer
       // This is a soft guarantee for cross thread invalidation, as atomics are not used
       // and it hasn't been thoroughly tested
@@ -131,23 +216,24 @@ public:
     uint64_t LocalPagePointer = Pointers[Address];
     if (!LocalPagePointer) {
       // Page for this code didn't even exist, nothing to do
-      return;
+      return ErasedAny;
     }
 
     // Page exists, just set the offset to zero
     auto BlockPointers = reinterpret_cast<LookupCacheEntry*>(LocalPagePointer);
     BlockPointers[PageOffset].GuestCode = 0;
     BlockPointers[PageOffset].HostCode = 0;
+    return true;
   }
 
   void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink, const FEXCore::Context::BlockDelinkerFunc& delinker) {
-    std::lock_guard<std::recursive_mutex> lk(WriteLock);
-
-    BlockLinks->insert({{GuestDestination, HostLink}, delinker});
+    auto lk = Shared->AcquireLock();
+    Shared->AddBlockLink(GuestDestination, HostLink, delinker, lk);
   }
 
   void ClearCache();
   void ClearL2Cache();
+  void ClearThreadLocalCaches();
 
   uintptr_t GetL1Pointer() const {
     return L1Pointer;
@@ -169,7 +255,9 @@ public:
   // Some care is taken so that L1 lookups can be done without locks, and even tearing is unlikely to lead to a crash.
   // This approach has not been fully vetted yet.
   // Also note that L1 lookups might be inlined in the JIT Dispatcher and/or block ends.
-  std::recursive_mutex WriteLock;
+  auto AcquireLock() {
+    return Shared->AcquireLock();
+  }
 
 private:
   void CacheBlockMapping(uint64_t Address, uintptr_t HostCode) {
@@ -226,34 +314,6 @@ private:
   uintptr_t PageMemory;
   uintptr_t L1Pointer;
 
-  struct BlockLinkTag {
-    uint64_t GuestDestination;
-    FEXCore::Context::ExitFunctionLinkData* HostLink;
-
-    bool operator<(const BlockLinkTag& other) const {
-      if (GuestDestination < other.GuestDestination) {
-        return true;
-      } else if (GuestDestination == other.GuestDestination) {
-        return HostLink < other.HostLink;
-      } else {
-        return false;
-      }
-    }
-  };
-
-  // Use a monotonic buffer resource to allocate both the std::pmr::map and its members.
-  // This allows us to quickly clear the block link map by clearing the monotonic allocator.
-  // If we had allocated the block link map without the MBR, then clearing the map would require slowly
-  // walking each block member and destructing objects.
-  //
-  // This makes `BlockLinks` look like a raw pointer that could memory leak, but since it is backed by the MBR, it won't.
-  std::pmr::monotonic_buffer_resource BlockLinks_mbr;
-  using BlockLinksMapType = std::pmr::map<BlockLinkTag, FEXCore::Context::BlockDelinkerFunc>;
-  fextl::unique_ptr<std::pmr::polymorphic_allocator<std::byte>> BlockLinks_pma;
-  BlockLinksMapType* BlockLinks;
-
-  fextl::robin_map<uint64_t, uint64_t> BlockList;
-
   size_t TotalCacheSize;
 
   constexpr static size_t CODE_SIZE = 128 * 1024 * 1024;

FEXCore/Source/Interface/Core/ObjectCache/CodeObjectSerializationConfig.h

@@ -1,86 +0,0 @@
-// SPDX-License-Identifier: MIT
-#pragma once
-
-#include <FEXCore/Utils/CompilerDefs.h>
-
-#include <cstdint>
-
-namespace FEXCore::CodeSerialize {
-// If any of the config options mismatch on load then the cache won't be used
-// Any of these will result in codegen changes
-struct FEX_PACKED CodeObjectSerializationConfig {
-  // Cookie in the header of the file, isn't part of the config hash
-  uint64_t Cookie {};
-
-  // Instructions per block configuration
-  int32_t MaxInstPerBlock {};
-
-  // Follows CPUID 4000_0001_EAX[3:0]
-  unsigned Arch : 4;
-
-  // Multiblock enabled
-  unsigned MultiBlock : 1;
-
-  // Hardware TSO enabled
-  unsigned HardwareTSOEnabled : 1;
-
-  // TSO enabled
-  unsigned TSOEnabled : 1;
-
-  // ABI local flag unsafe optimization
-  unsigned ABILocalFlags : 1;
-
-  // Paranoid TSO mode enabled
-  unsigned ParanoidTSO : 1;
-
-  // Guest code execution mode (We don't support live mode switch)
-  unsigned Is64BitMode : 1;
-
-  // SMC checks style
-  unsigned SMCChecks : 2;
-
-  // x87 reduced precision
-  unsigned x87ReducedPrecision : 1;
-
-  // Padding to remove uninitialized data warning from asan
-  // Shows remaining amount of bits available for config
-  unsigned _Pad : 19;
-
-  bool operator==(const CodeObjectSerializationConfig& other) const {
-    return Cookie == other.Cookie && MaxInstPerBlock == other.MaxInstPerBlock && Arch == other.Arch && MultiBlock == other.MultiBlock &&
-           HardwareTSOEnabled == other.HardwareTSOEnabled && TSOEnabled == other.TSOEnabled && ABILocalFlags == other.ABILocalFlags &&
-           ParanoidTSO == other.ParanoidTSO && Is64BitMode == other.Is64BitMode && SMCChecks == other.SMCChecks &&
-           x87ReducedPrecision == other.x87ReducedPrecision;
-  }
-  static uint64_t GetHash(const CodeObjectSerializationConfig& other) {
-    // For < 64-bits of data just pack directly
-    // Skip the cookie
-    uint64_t Hash {};
-    Hash <<= 32;
-    Hash |= other.MaxInstPerBlock;
-    Hash <<= 1;
-    Hash |= other.Arch;
-    Hash <<= 1;
-    Hash |= other.MultiBlock;
-    Hash <<= 1;
-    Hash |= other.HardwareTSOEnabled;
-    Hash <<= 1;
-    Hash |= other.TSOEnabled;
-    Hash <<= 1;
-    Hash |= other.ABILocalFlags;
-    Hash <<= 1;
-    Hash |= other.ParanoidTSO;
-    Hash <<= 1;
-    Hash |= other.Is64BitMode;
-    Hash <<= 2;
-    Hash |= other.SMCChecks;
-    Hash <<= 1;
-    Hash |= other.x87ReducedPrecision;
-    return Hash;
-  }
-};
-
-static_assert(sizeof(CodeObjectSerializationConfig) == 16, "Size changed");
-static_assert((sizeof(CodeObjectSerializationConfig) - sizeof(uint64_t)) == 8, "Config size exceeded 64its. Need to change how the hash is "
-                                                                               "generated!");
-} // namespace FEXCore::CodeSerialize

FEXCore/Source/Interface/Core/ObjectCache/JobHandling.cpp

@@ -1,121 +0,0 @@
-// SPDX-License-Identifier: MIT
-#include "Interface/Core/ObjectCache/ObjectCacheService.h"
-
-#include <FEXCore/Config/Config.h>
-#include <FEXCore/fextl/memory.h>
-#include <FEXCore/fextl/string.h>
-#include <FEXHeaderUtils/Filesystem.h>
-
-#include <fcntl.h>
-#include <xxhash.h>
-
-namespace FEXCore::CodeSerialize {
-void AsyncJobHandler::AsyncAddNamedRegionJob(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const fextl::string& filename) {
-#ifndef _WIN32
-  // This function adds a named region *JOB* to our named region handler
-  // This needs to be as fast as possible to keep out of the way of the JIT
-
-  const fextl::string BaseFilename = FHU::Filesystem::GetFilename(filename);
-
-  if (!BaseFilename.empty()) {
-    // Create a new entry that once set up will be put in to our section object map
-    auto Entry = fextl::make_unique<CodeRegionEntry>(Base, Size, Offset, filename, NamedRegionHandler->DefaultCodeHeader(Base, Offset));
-
-    // Lock the job ref counter so we can block anything attempting to use the entry before it is loaded
-    Entry->NamedJobRefCountMutex.lock();
-
-    CodeRegionMapType::iterator EntryIterator;
-    {
-      std::unique_lock lk {CodeObjectCacheService->GetEntryMapMutex()};
-
-      auto& EntryMap = CodeObjectCacheService->GetEntryMap();
-
-      auto it = EntryMap.emplace(Base, std::move(Entry));
-      if (!it.second) {
-        // This happens when an application overwrites a previous region without unmapping what was there
-
-        // Lock this entry's Named job reference counter.
-        // Once this passes then we know that this section has been loaded.
-        it.first->second->NamedJobRefCountMutex.lock();
-
-        // Finalize anything the region needs to do first.
-        CodeObjectCacheService->DoCodeRegionClosure(it.first->second->Base, it.first->second.get());
-
-        // munmap the file that was mapped
-        FEXCore::Allocator::munmap(it.first->second->CodeData, it.first->second->FileSize);
-
-        // Remove this entry from the unrelocated map as well
-        {
-          std::unique_lock lk2 {CodeObjectCacheService->GetUnrelocatedEntryMapMutex()};
-          CodeObjectCacheService->GetUnrelocatedEntryMap().erase(it.first->second->EntryHeader.OriginalBase);
-        }
-
-        // Now overwrite the entry in the map
-        it = EntryMap.insert_or_assign(Base, std::move(Entry));
-        EntryIterator = it.first;
-      } else {
-        // No overwrite, just insert
-        EntryIterator = it.first;
-      }
-    }
-
-    // Now that this entry has been added to the map, we can insert a load job using the entry iterator.
-    // This allows us to quickly unblock the JIT thread when it is loading multiple regions and have the async thread
-    // do the loading for us.
-    //
-    // Create the async work queue job now so it can load
-    NamedRegionHandler->AsyncAddNamedRegionWorkItem(BaseFilename, filename, true, EntryIterator);
-
-    // Tell the async thread that it has work to do
-    CodeObjectCacheService->NotifyWork();
-  }
-#endif
-}
-
-void AsyncJobHandler::AsyncRemoveNamedRegionJob(uintptr_t Base, uintptr_t Size) {
-#ifndef _WIN32
-  // Removing a named region through the job system
-  // We need to find the entry that we are deleting first
-  fextl::unique_ptr<CodeRegionEntry> EntryPointer;
-  {
-    std::unique_lock lk {CodeObjectCacheService->GetEntryMapMutex()};
-
-    auto& EntryMap = CodeObjectCacheService->GetEntryMap();
-    auto it = EntryMap.find(Base);
-    if (it != EntryMap.end()) {
-      // Lock the job ref counter since we are erasing it
-      // Once this passes it will have been loaded
-      it->second->NamedJobRefCountMutex.lock();
-
-      // Take the pointer from the map
-      EntryPointer = std::move(it->second);
-
-      // We can now unmap the file data
-      FEXCore::Allocator::munmap(EntryPointer->CodeData, EntryPointer->FileSize);
-
-      // Remove this from the entry map
-      EntryMap.erase(it);
-
-      // Remove this entry from the unrelocated map as well
-      {
-        std::unique_lock lk2 {CodeObjectCacheService->GetUnrelocatedEntryMapMutex()};
-        CodeObjectCacheService->GetUnrelocatedEntryMap().erase(EntryPointer->EntryHeader.OriginalBase);
-      }
-    } else {
-      // Tried to remove something that wasn't in our code object tracking
-      return;
-    }
-
-    // Create the async work queue job now so it can finalize what it needs to do
-    NamedRegionHandler->AsyncRemoveNamedRegionWorkItem(Base, Size, std::move(EntryPointer));
-
-    // Tell the async thread that it has work to do
-    CodeObjectCacheService->NotifyWork();
-  }
-#endif
-}
-
-void AsyncJobHandler::AsyncAddSerializationJob(fextl::unique_ptr<SerializationJobData> Data) {
-  // XXX: Actually add serialization job
-}
-} // namespace FEXCore::CodeSerialize

FEXCore/Source/Interface/Core/ObjectCache/NamedRegionObjectHandler.cpp

@@ -1,71 +0,0 @@
-// SPDX-License-Identifier: MIT
-#include "Interface/Core/ObjectCache/ObjectCacheService.h"
-
-#include <FEXCore/fextl/memory.h>
-#include <FEXCore/fextl/string.h>
-
-namespace FEXCore::CodeSerialize {
-NamedRegionObjectHandler::NamedRegionObjectHandler(FEXCore::Context::ContextImpl* ctx) {
-  DefaultSerializationConfig.Cookie = CODE_COOKIE;
-
-  // Initialize the Arch from CPUID
-  uint32_t Arch = ctx->CPUID.RunFunction(0x4000'0001, 0).eax & 0xF;
-  DefaultSerializationConfig.Arch = Arch;
-
-  DefaultSerializationConfig.MaxInstPerBlock = ctx->Config.MaxInstPerBlock;
-  DefaultSerializationConfig.MultiBlock = ctx->Config.Multiblock;
-  DefaultSerializationConfig.TSOEnabled = ctx->Config.TSOEnabled;
-  DefaultSerializationConfig.ABILocalFlags = ctx->Config.ABILocalFlags;
-  DefaultSerializationConfig.ParanoidTSO = ctx->Config.ParanoidTSO;
-  DefaultSerializationConfig.Is64BitMode = ctx->Config.Is64BitMode;
-  DefaultSerializationConfig.SMCChecks = ctx->Config.SMCChecks;
-  DefaultSerializationConfig.x87ReducedPrecision = ctx->Config.x87ReducedPrecision;
-}
-
-void NamedRegionObjectHandler::AddNamedRegionObject(CodeRegionMapType::iterator Entry, const fextl::string& base_filename,
-                                                    const fextl::string& filename, bool Executable) {
-  // XXX: Add named region objects
-
-  // XXX: Until entry loading is complete just claim it is loaded
-  Entry->second->NamedJobRefCountMutex.unlock();
-}
-
-void NamedRegionObjectHandler::RemoveNamedRegionObject(uintptr_t Base, uintptr_t Size, fextl::unique_ptr<CodeRegionEntry> Entry) {
-  // XXX: Remove named region objects
-
-  // XXX: Until entry loading is complete just claim it is loaded
-  Entry->NamedJobRefCountMutex.unlock();
-}
-
-void NamedRegionObjectHandler::HandleNamedRegionObjectJobs() {
-  // Walk through all of our jobs sequentially until the work queue is empty
-  while (NamedWorkQueueJobs.load()) {
-    fextl::unique_ptr<AsyncJobHandler::NamedRegionWorkItem> WorkItem;
-
-    {
-      // Lock the work queue mutex for a short moment and grab an item from the list
-      std::unique_lock lk {NamedWorkQueueMutex};
-      size_t WorkItems = WorkQueue.size();
-      if (WorkItems != 0) {
-        WorkItem = std::move(WorkQueue.front());
-        WorkQueue.pop();
-      }
-
-      // Atomically update the number of jobs
-      --NamedWorkQueueJobs;
-    }
-
-    if (WorkItem) {
-      if (WorkItem->GetType() == AsyncJobHandler::NamedRegionJobType::JOB_ADD_NAMED_REGION) {
-        auto WorkAdd = static_cast<AsyncJobHandler::WorkItemAddNamedRegion*>(WorkItem.get());
-        AddNamedRegionObject(WorkAdd->Entry, WorkAdd->BaseFilename, WorkAdd->Filename, WorkAdd->Executable);
-      }
-
-      if (WorkItem->GetType() == AsyncJobHandler::NamedRegionJobType::JOB_REMOVE_NAMED_REGION) {
-        auto WorkRemove = static_cast<AsyncJobHandler::WorkItemRemoveNamedRegion*>(WorkItem.get());
-        RemoveNamedRegionObject(WorkRemove->Base, WorkRemove->Size, std::move(WorkRemove->Entry));
-      }
-    }
-  }
-}
-} // namespace FEXCore::CodeSerialize

FEXCore/Source/Interface/Core/ObjectCache/ObjectCacheService.cpp

@@ -1,85 +0,0 @@
-// SPDX-License-Identifier: MIT
-#include "Interface/Core/ObjectCache/ObjectCacheService.h"
-
-#include <FEXCore/Config/Config.h>
-#include <FEXCore/fextl/memory.h>
-#include <FEXCore/Utils/Threads.h>
-
-namespace {
-static void* ThreadHandler(void* Arg) {
-  FEXCore::CodeSerialize::CodeObjectSerializeService* This = reinterpret_cast<FEXCore::CodeSerialize::CodeObjectSerializeService*>(Arg);
-  This->ExecutionThread();
-  return nullptr;
-}
-} // namespace
-
-namespace FEXCore::CodeSerialize {
-CodeObjectSerializeService::CodeObjectSerializeService(FEXCore::Context::ContextImpl* ctx)
-  : CTX {ctx}
-  , AsyncHandler {&NamedRegionHandler, this}
-  , NamedRegionHandler {ctx} {
-  Initialize();
-}
-
-void CodeObjectSerializeService::Shutdown() {
-  if (CTX->Config.CacheObjectCodeCompilation() == FEXCore::Config::ConfigObjectCodeHandler::CONFIG_NONE) {
-    return;
-  }
-
-  WorkerThreadShuttingDown = true;
-
-  // Kick the working thread
-  WorkAvailable.NotifyAll();
-
-  if (WorkerThread->joinable()) {
-    // Wait for worker thread to close down
-    WorkerThread->join(nullptr);
-  }
-}
-
-void CodeObjectSerializeService::Initialize() {
-  // Add a canary so we don't crash on empty map iterator handling
-  auto it = AddressToEntryMap.insert_or_assign(~0ULL, fextl::make_unique<CodeRegionEntry>());
-  UnrelocatedAddressToEntryMap.insert_or_assign(~0ULL, it.first->second.get());
-
-  uint64_t OldMask = FEXCore::Threads::SetSignalMask(~0ULL);
-  WorkerThread = FEXCore::Threads::Thread::Create(ThreadHandler, this);
-  FEXCore::Threads::SetSignalMask(OldMask);
-}
-
-void CodeObjectSerializeService::DoCodeRegionClosure(uint64_t Base, CodeRegionEntry* it) {
-  if (Base == ~0ULL) {
-    // Don't do closure on canary
-    return;
-  }
-  // XXX: Do code region closure
-}
-
-const CodeObjectFileSection* CodeObjectSerializeService::FetchCodeObjectFromCache(uint64_t GuestRIP) {
-  // XXX: Actually fetch code objects from cache
-  return nullptr;
-}
-
-void CodeObjectSerializeService::ExecutionThread() {
-  // Set our thread name so we can see its relation
-  FEXCore::Threads::SetThreadName("ObjectCodeSeri\0");
-  while (WorkerThreadShuttingDown.load() != true) {
-    // Wait for work
-    WorkAvailable.Wait();
-
-    // Handle named region async jobs first. Highest priority
-    NamedRegionHandler.HandleNamedRegionObjectJobs();
-
-    // XXX: Handle code serialization jobs second.
-  }
-
-  // Do final code region closures on thread shutdown
-  for (auto& it : AddressToEntryMap) {
-    DoCodeRegionClosure(it.first, it.second.get());
-  }
-
-  // Safely clear our maps now
-  AddressToEntryMap.clear();
-  UnrelocatedAddressToEntryMap.clear();
-}
-} // namespace FEXCore::CodeSerialize

FEXCore/Source/Interface/Core/ObjectCache/ObjectCacheService.h

@@ -1,457 +0,0 @@
-// SPDX-License-Identifier: MIT
-#pragma once
-#include "Interface/Context/Context.h"
-#include "Interface/Core/ObjectCache/Relocations.h"
-#include "Interface/Core/ObjectCache/CodeObjectSerializationConfig.h"
-#include "Interface/IR/AOTIR.h"
-
-#include <FEXCore/Utils/Event.h>
-#include <FEXCore/Utils/Threads.h>
-#include <FEXCore/fextl/map.h>
-#include <FEXCore/fextl/memory.h>
-#include <FEXCore/fextl/queue.h>
-#include <FEXCore/fextl/robin_map.h>
-#include <FEXCore/fextl/string.h>
-#include <FEXCore/fextl/vector.h>
-
-#include <shared_mutex>
-
-namespace FEXCore::CodeSerialize {
-// XXX: Does this need to be signal safe?
-using CodeSerializationMutex = std::shared_mutex;
-struct CodeSerializationData {};
-
-struct CodeObjectFileSection {
-  bool Serialized;
-  bool Invalid;
-  const CodeSerializationData* Data;
-  const char* HostCode;
-  uint64_t NumRelocations;
-  const char* Relocations;
-};
-
-/**
- * @brief This is the file header that lives at the start of an object cache file
- *
- * This header is updated from multiple processes!
- * Care must be taken to use OS locks when updating the file backing including this header
- */
-struct CodeObjectSerializationHeader {
-  // The configuration that this file has
-  CodeObjectSerializationConfig Config;
-  // The original RIP that this object section was mapped at
-  uint64_t OriginalBase {};
-  // The original offset in to the file that this object section was loaded from
-  uint64_t OriginalOffset {};
-  // Total amount of code that should be in this file
-  uint64_t TotalCodeSize {};
-  // Used to reserve the TSL map
-  uint64_t NumCodeEntries {};
-  // The number of relocations that point to this section
-  uint64_t NumRelocationsTo {};
-  // Total relocations in this file
-  uint64_t TotalRelocationsCount {};
-};
-
-struct CodeRegionEntry {
-  /**
-   * @name Threaded initialization objects for the initial object creation
-   * @{ */
-  // Base address in memory where the code region is at
-  uint64_t Base {};
-
-  // Size of this code entry
-  uint64_t Size {};
-
-  // The offset inside the file that is mapped to Base
-  uint64_t Offset {};
-
-  // Filename of the object
-  fextl::string Filename {};
-
-  CodeObjectSerializationHeader EntryHeader {};
-  /**  @} */
-
-  // The filename of the object cache for this entry
-  fextl::string ObjectEntrySourceFilename {};
-
-  // In the case of file corruption that we can detect, we can disable serialization early for an entry
-  // We should be resiliant to corruption but things happen
-  bool StillSerializing {true};
-
-  // Long lived FD for serialization if we have multiple jobs to serialize
-  // Bursts of code entries are common and this reduces file lock overhead
-  //
-  // Especially useful over network mounts where file locks are very slow
-  int CurrentSerializedFD {-1};
-
-  /**
-   * @name Objects required to sync objects between threads
-   * @{ */
-  // Refcount for the number of outstanding code entries waiting to be written for this object section
-  CodeSerializationMutex ObjectJobRefCountMutex;
-
-  // Refcount for outstanding named object region entry loading itself
-  // Will block JIT code cache look up when this has a unique_lock held
-  CodeSerializationMutex NamedJobRefCountMutex;
-  /**  @} */
-
-  /**
-   * @name Object Entry data management
-   * @{ */
-
-  /**
-   * @name This is the raw file data that we loaded from the code region entry file
-   * @{ */
-  char* CodeData {};
-  size_t FileSize {};
-
-  fextl::vector<CodeObjectFileSection> FileCodeSections;
-  /**  @} */
-
-  // This per section map takes the most time to load and needs to be quick
-  // This is the map of all code segments for this entry
-  fextl::robin_map<uint64_t, CodeObjectFileSection*> SectionLookupMap {};
-  /**  @} */
-
-  // Default initialization
-  CodeRegionEntry() = default;
-
-  // Initializer specifically for threaded loading
-  CodeRegionEntry(uint64_t Base, uint64_t Size, uint64_t Offset, const fextl::string& Filename, const CodeObjectSerializationHeader& DefaultHeader)
-    : Base {Base}
-    , Size {Size}
-    , Offset {Offset}
-    , Filename {Filename}
-    , EntryHeader {DefaultHeader} {}
-};
-
-// Map type must use an interator that isn't invalidation on erase/insert
-using CodeRegionMapType = fextl::map<uint64_t, fextl::unique_ptr<CodeRegionEntry>>;
-using CodeRegionPtrMapType = fextl::map<uint64_t, CodeRegionEntry*>;
-
-class NamedRegionObjectHandler;
-class CodeObjectSerializeService;
-
-class AsyncJobHandler final {
-public:
-  /**
-   * @brief Structure containing all the data required to async serialize code objects
-   */
-  struct SerializationJobData {
-    uint64_t GuestRIP; ///< The RIP for the guest
-    // XXX: Support multiblock
-    uint64_t GuestCodeLength; ///< The Guest's code length
-    uint64_t GuestCodeHash;   ///< Hash of the guest code
-
-    void* HostCodeBegin;   ///< Host JIT code starting memory address
-    size_t HostCodeLength; ///< Host JIT code length
-    uint64_t HostCodeHash; ///< Host JIT code hash before any backpatching
-
-    // This is the thread specific ref counter for outstanding jobs.
-    // This shared mutex is incremented when the job is added, then decremented when the job is complete.
-    // If a thread is shutting down or clearing code cache then the thread will pull a unique lock on this mutex.
-    // This way it will wait until the async job handler is complete with it.
-    CodeSerializationMutex* ThreadJobRefCount;
-
-    // These are the reolocations for this serialization job
-    // Relatively small number of entries most of the time
-    fextl::vector<FEXCore::CPU::Relocation> Relocations;
-
-    /**
-     * @name Objects filled in from the Code Object Serialization service when a job is added
-     * @{ */
-    // This is the code region's ref counter for outstanding jobs.
-    // This shared mutex is incremented when the job is added, then decremented when the job is complete.
-    // If a named region is being removed then a unique lock will be pulled to wait for all jobs to complete and no new jobs to be added.
-    CodeSerializationMutex* ObjectJobRefCountMutexPtr;
-
-    // This is the code region iterator to reduce the number of map lookups
-    // This will remain valid while jobs are outstanding for this region
-    CodeRegionMapType::iterator CodeRegionIterator;
-    /**  @} */
-  };
-
-  AsyncJobHandler(NamedRegionObjectHandler* NamedRegionHandler, CodeObjectSerializeService* CodeObjectCacheService)
-    : NamedRegionHandler {NamedRegionHandler}
-    , CodeObjectCacheService {CodeObjectCacheService} {}
-
-protected:
-  friend class CodeObjectSerializeService;
-  friend class NamedRegionObjectHandler;
-  /**
-   * @name Async job submission functions
-   * @{ */
-  void AsyncAddNamedRegionJob(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const fextl::string& filename);
-  void AsyncRemoveNamedRegionJob(uintptr_t Base, uintptr_t Size);
-  void AsyncAddSerializationJob(fextl::unique_ptr<SerializationJobData> Data);
-  /**  @} */
-
-  /**
-   * @name Async named region handling
-   * @{ */
-  /**
-   * @brief The async named region jobs to handle.
-   *
-   * Only two, Code serialization goes in to a different queue.
-   */
-  enum class NamedRegionJobType {
-    JOB_ADD_NAMED_REGION,
-    JOB_REMOVE_NAMED_REGION,
-  };
-
-  class NamedRegionWorkItem {
-  public:
-    NamedRegionJobType GetType() const {
-      return Type;
-    }
-
-  protected:
-    friend class WorkItemAddNamedRegion;
-    NamedRegionWorkItem(NamedRegionJobType type)
-      : Type {type} {}
-
-  private:
-    NamedRegionJobType Type;
-  };
-
-  class WorkItemAddNamedRegion : public NamedRegionWorkItem {
-  public:
-    WorkItemAddNamedRegion(const fextl::string& base, const fextl::string& filename, bool executable, CodeRegionMapType::iterator entry)
-      : NamedRegionWorkItem {NamedRegionJobType::JOB_ADD_NAMED_REGION}
-      , BaseFilename {base}
-      , Filename {filename}
-      , Executable {executable}
-      , Entry {entry} {}
-    const fextl::string BaseFilename;
-    const fextl::string Filename;
-    bool Executable;
-    CodeRegionMapType::iterator Entry;
-  };
-
-  class WorkItemRemoveNamedRegion : public NamedRegionWorkItem {
-  public:
-    WorkItemRemoveNamedRegion(uint64_t base, uint64_t size, fextl::unique_ptr<CodeRegionEntry> entry)
-      : NamedRegionWorkItem {NamedRegionJobType::JOB_REMOVE_NAMED_REGION}
-      , Base {base}
-      , Size {size}
-      , Entry {std::move(entry)} {}
-
-    uint64_t Base;
-    uint64_t Size;
-    fextl::unique_ptr<CodeRegionEntry> Entry;
-  };
-  /**  @} */
-
-private:
-  NamedRegionObjectHandler* NamedRegionHandler;
-  CodeObjectSerializeService* CodeObjectCacheService;
-};
-
-class NamedRegionObjectHandler final {
-public:
-  NamedRegionObjectHandler(FEXCore::Context::ContextImpl* ctx);
-
-  void HandleNamedRegionObjectJobs();
-
-  const CodeObjectSerializationConfig& GetDefaultSerializationConfig() const {
-    return DefaultSerializationConfig;
-  }
-
-protected:
-  friend class AsyncJobHandler;
-
-  // Return a default code header based off the default serialization config
-  CodeObjectSerializationHeader DefaultCodeHeader(uint64_t Base, uint64_t Offset) const {
-    return CodeObjectSerializationHeader {
-      .Config = DefaultSerializationConfig,
-      .OriginalBase = Base,
-      .OriginalOffset = Offset,
-      .NumCodeEntries = 0,
-      .NumRelocationsTo = 0,
-      .TotalRelocationsCount = 0,
-    };
-  }
-
-  /**
-   * @brief Adds an asynchronous add named region work item to the object queue
-   *
-   * This adds the job that will do the loading of file resources and data tracking.
-   */
-  void AsyncAddNamedRegionWorkItem(const fextl::string& base, const fextl::string& filename, bool executable, CodeRegionMapType::iterator entry) {
-    std::unique_lock lk {NamedWorkQueueMutex};
-    WorkQueue.emplace(fextl::make_unique<AsyncJobHandler::WorkItemAddNamedRegion>(base, filename, executable, entry));
-    ++NamedWorkQueueJobs;
-  }
-
-  void AsyncRemoveNamedRegionWorkItem(uint64_t Base, uint64_t Size, fextl::unique_ptr<CodeRegionEntry> Entry) {
-    std::unique_lock lk {NamedWorkQueueMutex};
-    WorkQueue.emplace(fextl::make_unique<AsyncJobHandler::WorkItemRemoveNamedRegion>(Base, Size, std::move(Entry)));
-    ++NamedWorkQueueJobs;
-  }
-
-private:
-  // Code version. If the code emission changes then this needs to increment
-  constexpr static uint32_t CODE_VERSION = 0x0;
-
-  // Default cookie header for the file header
-  constexpr static uint64_t CODE_COOKIE = FEXCore::IR::COOKIE_VERSION("FEXC", CODE_VERSION);
-
-  // Code serialization config for our current process configuration
-  CodeObjectSerializationConfig DefaultSerializationConfig;
-
-  // Atomic counter for number of jobs in the queue without needing to pull the mutex to check
-  std::atomic<uint64_t> NamedWorkQueueJobs {};
-
-  // Mutex for ading new jobs to the work queue
-  std::mutex NamedWorkQueueMutex {};
-
-  // The job queue itself
-  // Jobs get consumed as a FIFO
-  // Jobs always get appended to the end
-  fextl::queue<fextl::unique_ptr<AsyncJobHandler::NamedRegionWorkItem>> WorkQueue {};
-
-  /**
-   * @name Named Region object handling
-   * @{ */
-  void AddNamedRegionObject(CodeRegionMapType::iterator Entry, const fextl::string& base_filename, const fextl::string& filename, bool Executable);
-  void RemoveNamedRegionObject(uintptr_t Base, uintptr_t Size, fextl::unique_ptr<CodeRegionEntry> Entry);
-  /**  @} */
-};
-
-/**
- * @brief Context specific code object serialization class
- *
- * Contains everything required for FEXCore to serialize code objects
- */
-class CodeObjectSerializeService final {
-public:
-  CodeObjectSerializeService(FEXCore::Context::ContextImpl* ctx);
-
-  /**
-   * @brief Initialize the internal interface
-   *
-   * Is a public interface to allow the service to reinitialize after forking
-   */
-  void Initialize();
-
-  /**
-   * @brief Safely shut down the Code Object serialization service.
-   *
-   * This service needs to be resiliant to application crashes, but shutting down safely is still preferred.
-   */
-  void Shutdown();
-
-  /**
-   * @name Async interface
-   * @{ */
-  /**
-   * @brief Loads a named region in to the code serialization service. As async as possible.
-   *
-   * @param Base - Virtual address that this named region is loaded
-   * @param Size - The size of the region
-   * @param Offset - The offset from the file
-   * @param filename - The filename itself
-   */
-  void AsyncAddNamedRegionJob(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const fextl::string& filename) {
-    AsyncHandler.AsyncAddNamedRegionJob(Base, Size, Offset, filename);
-  }
-
-  /**
-   * @brief Unloads a named region from the code serialization service. As async as possible.
-   *
-   * @param Base - Virtual address of the named region
-   * @param Size - The size of the region
-   */
-  void AsyncRemoveNamedRegionJob(uintptr_t Base, uintptr_t Size) {
-    AsyncHandler.AsyncRemoveNamedRegionJob(Base, Size);
-  }
-
-  /**
-   * @brief Adds a code object serialization job. As async as possible.
-   * Code hashing happens prior to async job serialization to catch invalidations due to backpatching.
-   *
-   * @param Data - A fully filled out struct containing all the code serialization
-   */
-  void AsyncAddSerializationJob(fextl::unique_ptr<AsyncJobHandler::SerializationJobData> Data) {
-    AsyncHandler.AsyncAddSerializationJob(std::move(Data));
-  }
-  /**  @} */
-
-  /**
-   * @name Synchronous interface
-   * @{ */
-  /**
-   * @brief Synchronously waits for this thread's job queue to become empty.
-   *
-   * This is necessary for when a thread is shutting down
-   *
-   * @param ThreadJobRefCount - The shared mutex to wait on until to be empty
-   */
-  static void WaitForEmptyJobQueue(CodeSerializationMutex* ThreadJobRefCount) {
-    // Once the shared mutex is empty this unique lock will be gained
-    std::unique_lock lk {*ThreadJobRefCount};
-  }
-
-  /**
-   * @brief Fetches object code from the Code Object Cache for JIT.
-   *
-   * @param GuestRIP - Which GuestRIP to search the cache for
-   *
-   * @return Data required for the JIT to relocate the Object code.
-   */
-  const CodeObjectFileSection* FetchCodeObjectFromCache(uint64_t GuestRIP);
-  /**  @} */
-
-  // Public for threading
-  void ExecutionThread();
-
-protected:
-  friend class AsyncJobHandler;
-
-  /**
-   * @brief Safely closes out code object regions from the map
-   *
-   * @param it - iterator to do a closure on
-   */
-  void DoCodeRegionClosure(uint64_t Base, CodeRegionEntry* it);
-
-  CodeSerializationMutex& GetEntryMapMutex() {
-    return EntryMapMutex;
-  }
-  CodeSerializationMutex& GetUnrelocatedEntryMapMutex() {
-    return EntryMapMutex;
-  }
-
-  CodeRegionMapType& GetEntryMap() {
-    return AddressToEntryMap;
-  }
-  CodeRegionPtrMapType& GetUnrelocatedEntryMap() {
-    return UnrelocatedAddressToEntryMap;
-  }
-
-  /**
-   * @brief Notify the async thread that it has work to do
-   */
-  void NotifyWork() {
-    WorkAvailable.NotifyOne();
-  }
-
-private:
-  FEXCore::Context::ContextImpl* CTX;
-
-  Event WorkAvailable {};
-  fextl::unique_ptr<FEXCore::Threads::Thread> WorkerThread;
-  std::atomic_bool WorkerThreadShuttingDown {false};
-  AsyncJobHandler AsyncHandler;
-  NamedRegionObjectHandler NamedRegionHandler;
-
-  // Mutex to hold when modifying the entry maps
-  CodeSerializationMutex EntryMapMutex;
-  CodeSerializationMutex UnrelocatedEntryMapMutex;
-
-  // Entry maps
-  CodeRegionMapType AddressToEntryMap;
-  CodeRegionPtrMapType UnrelocatedAddressToEntryMap;
-};
-} // namespace FEXCore::CodeSerialize

FEXCore/Source/Interface/Core/OpcodeDispatcher.h

@@ -7,6 +7,7 @@
 #include "Interface/Context/Context.h"
 #include "Interface/IR/IR.h"
 #include "Interface/IR/IREmitter.h"
+#include "Interface/IR/RegisterAllocationData.h"
 
 #include <FEXCore/Config/Config.h>
 #include <FEXCore/Core/Context.h>
@@ -26,8 +27,6 @@
 #include <xxhash.h>
 
 namespace FEXCore::IR {
-class Pass;
-class PassManager;
 
 enum class MemoryAccessType {
   // Choose TSO or Non-TSO depending on access type
@@ -79,10 +78,13 @@ struct LoadSourceOptions {
   bool AllowUpperGarbage = false;
 };
 
-class OpDispatchBuilder final : public IREmitter {
-  friend class FEXCore::IR::Pass;
-  friend class FEXCore::IR::PassManager;
+struct DispatchTableEntry {
+  uint16_t Op;
+  uint8_t Count;
+  X86Tables::OpDispatchPtr Ptr;
+};
 
+class OpDispatchBuilder final : public IREmitter {
 public:
   Ref GetNewJumpBlock(uint64_t RIP) {
     auto it = JumpTargets.find(RIP);
@@ -159,9 +161,13 @@ public:
     auto InlineConst = _InlineConstant(Bit);
     return _CondJump(Src, InlineConst, InvalidNode, InvalidNode, {Set ? COND_TSTNZ : COND_TSTZ}, OpSize::iInvalid, false);
   }
-  IRPair<IROp_ExitFunction> ExitFunction(Ref NewRIP) {
+  IRPair<IROp_ExitFunction> ExitFunction(Ref NewRIP, BranchHint Hint = BranchHint::None) {
     FlushRegisterCache();
-    return _ExitFunction(NewRIP);
+    return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, InvalidNode, InvalidNode);
+  }
+  IRPair<IROp_ExitFunction> ExitFunction(Ref NewRIP, BranchHint Hint, Ref CallReturnAddress, Ref CallReturnBlock) {
+    FlushRegisterCache();
+    return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, CallReturnAddress, CallReturnBlock);
   }
   IRPair<IROp_Break> Break(BreakDefinition Reason) {
     FlushRegisterCache();
@@ -187,11 +193,10 @@ public:
       auto it = JumpTargets.find(NextRIP);
       if (it == JumpTargets.end()) {
 
-        const auto GPRSize = CTX->GetGPROpSize();
+        const auto GPRSize = GetGPROpSize();
         // If we don't have a jump target to a new block then we have to leave
         // Set the RIP to the next instruction and leave
-        auto RelocatedNextRIP = _EntrypointOffset(GPRSize, NextRIP - Entry);
-        ExitFunction(RelocatedNextRIP);
+        ExitFunction(_InlineEntrypointOffset(GPRSize, NextRIP - Entry));
       } else if (it != JumpTargets.end()) {
         Jump(it->second.BlockEntry);
         return true;
@@ -231,7 +236,7 @@ public:
   template<typename F>
   void ForeachDirection(F&& Routine) {
     // Otherwise, prepare to branch.
-    auto Zero = _Constant(0);
+    auto Zero = Constant(0);
 
     // If the shift is zero, do not touch the flags.
     auto ForwardBlock = CreateNewCodeBlockAfter(GetCurrentBlock());
@@ -255,7 +260,6 @@ public:
   }
 
   OpDispatchBuilder(FEXCore::Context::ContextImpl* ctx);
-  OpDispatchBuilder(FEXCore::Utils::IntrusivePooledAllocator& Allocator);
 
   void ResetWorkingList();
   void ResetDecodeFailure() {
@@ -289,7 +293,7 @@ public:
     return ShouldDump;
   }
 
-  void BeginFunction(uint64_t RIP, const fextl::vector<FEXCore::Frontend::Decoder::DecodedBlocks>* Blocks, uint32_t NumInstructions);
+  void BeginFunction(uint64_t RIP, const fextl::vector<FEXCore::Frontend::Decoder::DecodedBlocks>* Blocks, uint32_t NumInstructions, bool Is64BitMode, bool MonoBackpatcherBlock);
   void Finalize();
 
   // Dispatch builder functions
@@ -339,6 +343,9 @@ public:
   void LoopOp(OpcodeArgs);
   void JUMPOp(OpcodeArgs);
   void JUMPAbsoluteOp(OpcodeArgs);
+  void JUMPFARIndirectOp(OpcodeArgs);
+  void CALLFARIndirectOp(OpcodeArgs);
+  void RETFARIndirectOp(OpcodeArgs);
   void TESTOp(OpcodeArgs, uint32_t SrcIndex);
   void MOVSXDOp(OpcodeArgs);
   void MOVSXOp(OpcodeArgs);
@@ -701,7 +708,7 @@ public:
   Ref ReconstructX87StateFromFSW_Helper(Ref FSW);
   void FLD(OpcodeArgs, IR::OpSize Width);
   void FLDFromStack(OpcodeArgs);
-  void FLD_Const(OpcodeArgs, NamedVectorConstant Constant);
+  void FLD_Const(OpcodeArgs, NamedVectorConstant K);
 
   void FBLD(OpcodeArgs);
   void FBSTP(OpcodeArgs);
@@ -724,6 +731,7 @@ public:
   void FADD(OpcodeArgs, IR::OpSize Width, bool Integer, OpResult ResInST0);
   void FDIV(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpResult ResInST0);
   void FMUL(OpcodeArgs, IR::OpSize Width, bool Integer, OpResult ResInST0);
+  void FNCLEX(OpcodeArgs);
   void FNINIT(OpcodeArgs);
   void FSUB(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpResult ResInST0);
   void FTST(OpcodeArgs);
@@ -823,11 +831,13 @@ public:
   void PHADDS(OpcodeArgs);
   void PHSUBS(OpcodeArgs);
 
-  void CLWB(OpcodeArgs);
+  void CLWBOrTPause(OpcodeArgs);
   void CLFLUSHOPT(OpcodeArgs);
   void LoadFenceOrXRSTOR(OpcodeArgs);
   void MemFenceOrXSAVEOPT(OpcodeArgs);
   void StoreFenceOrCLFlush(OpcodeArgs);
+  void UMonitorOrCLRSSBSY(OpcodeArgs);
+  void UMWaitOp(OpcodeArgs);
   void CLZeroOp(OpcodeArgs);
   void RDTSCPOp(OpcodeArgs);
   void RDPIDOp(OpcodeArgs);
@@ -836,8 +846,6 @@ public:
 
   void PSADBW(OpcodeArgs);
 
-  Ref BitwiseAtLeastTwo(Ref A, Ref B, Ref C);
-
   void SHA1NEXTEOp(OpcodeArgs);
   void SHA1MSG1Op(OpcodeArgs);
   void SHA1MSG2Op(OpcodeArgs);
@@ -927,7 +935,6 @@ public:
   RefVSIB AVX128_LoadVSIB(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags, bool NeedsHigh);
   void AVX128_StoreResult_WithOpSize(FEXCore::X86Tables::DecodedOp Op, const FEXCore::X86Tables::DecodedOperand& Operand, const RefPair Src,
                                      MemoryAccessType AccessType = MemoryAccessType::DEFAULT);
-  void InstallAVX128Handlers();
   void AVX128_VMOVScalarImpl(OpcodeArgs, IR::OpSize ElementSize);
   void AVX128_VectorALU(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
   void AVX128_VectorUnary(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
@@ -956,39 +963,26 @@ public:
   void AVX128_VMOVDDUP(OpcodeArgs);
   void AVX128_VMOVSLDUP(OpcodeArgs);
   void AVX128_VMOVSHDUP(OpcodeArgs);
-  template<IR::OpSize ElementSize>
-  void AVX128_VBROADCAST(OpcodeArgs);
-  template<IR::OpSize ElementSize>
-  void AVX128_VPUNPCKL(OpcodeArgs);
-  template<IR::OpSize ElementSize>
-  void AVX128_VPUNPCKH(OpcodeArgs);
+  void AVX128_VBROADCAST(OpcodeArgs, IR::OpSize ElementSize);
+  void AVX128_VPUNPCKL(OpcodeArgs, IR::OpSize ElementSize);
+  void AVX128_VPUNPCKH(OpcodeArgs, IR::OpSize ElementSize);
   void AVX128_MOVVectorUnaligned(OpcodeArgs);
-  template<IR::OpSize DstElementSize>
-  void AVX128_InsertCVTGPR_To_FPR(OpcodeArgs);
-  template<IR::OpSize SrcElementSize, bool HostRoundingMode>
-  void AVX128_CVTFPR_To_GPR(OpcodeArgs);
+  void AVX128_InsertCVTGPR_To_FPR(OpcodeArgs, IR::OpSize DstElementSize);
+  void AVX128_CVTFPR_To_GPR(OpcodeArgs, IR::OpSize SrcElementSize, bool HostRoundingMode);
   void AVX128_VANDN(OpcodeArgs);
-  template<IR::OpSize ElementSize>
-  void AVX128_VPACKSS(OpcodeArgs);
-  template<IR::OpSize ElementSize>
-  void AVX128_VPACKUS(OpcodeArgs);
+  void AVX128_VPACKSS(OpcodeArgs, IR::OpSize ElementSize);
+  void AVX128_VPACKUS(OpcodeArgs, IR::OpSize ElementSize);
   Ref AVX128_PSIGNImpl(IR::OpSize ElementSize, Ref Src1, Ref Src2);
-  template<IR::OpSize ElementSize>
-  void AVX128_VPSIGN(OpcodeArgs);
-  template<IR::OpSize ElementSize>
-  void AVX128_UCOMISx(OpcodeArgs);
+  void AVX128_VPSIGN(OpcodeArgs, IR::OpSize ElementSize);
+  void AVX128_UCOMISx(OpcodeArgs, IR::OpSize ElementSize);
   void AVX128_VectorScalarInsertALU(OpcodeArgs, FEXCore::IR::IROps IROp, IR::OpSize ElementSize);
   Ref AVX128_VFCMPImpl(IR::OpSize ElementSize, Ref Src1, Ref Src2, uint8_t CompType);
-  template<IR::OpSize ElementSize>
-  void AVX128_VFCMP(OpcodeArgs);
-  template<IR::OpSize ElementSize>
-  void AVX128_InsertScalarFCMP(OpcodeArgs);
+  void AVX128_VFCMP(OpcodeArgs, IR::OpSize ElementSize);
+  void AVX128_InsertScalarFCMP(OpcodeArgs, IR::OpSize ElementSize);
   void AVX128_MOVBetweenGPR_FPR(OpcodeArgs);
-  template<IR::OpSize ElementSize>
-  void AVX128_PExtr(OpcodeArgs);
+  void AVX128_PExtr(OpcodeArgs, IR::OpSize ElementSize);
   void AVX128_ExtendVectorElements(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DstElementSize, bool Signed);
-  template<IR::OpSize ElementSize>
-  void AVX128_MOVMSK(OpcodeArgs);
+  void AVX128_MOVMSK(OpcodeArgs, IR::OpSize ElementSize);
   void AVX128_MOVMSKB(OpcodeArgs);
   void AVX128_PINSRImpl(OpcodeArgs, IR::OpSize ElementSize, const X86Tables::DecodedOperand& Src1Op,
                         const X86Tables::DecodedOperand& Src2Op, const X86Tables::DecodedOperand& Imm);
@@ -1002,33 +996,25 @@ public:
   void AVX128_VINSERTPS(OpcodeArgs);
 
   Ref AVX128_PHSUBImpl(Ref Src1, Ref Src2, size_t ElementSize);
-  template<IR::OpSize ElementSize>
-  void AVX128_VPHSUB(OpcodeArgs);
+  void AVX128_VPHSUB(OpcodeArgs, IR::OpSize ElementSize);
 
   void AVX128_VPHSUBSW(OpcodeArgs);
 
-  template<IR::OpSize ElementSize>
-  void AVX128_VADDSUBP(OpcodeArgs);
+  void AVX128_VADDSUBP(OpcodeArgs, IR::OpSize ElementSize);
 
-  template<IR::OpSize ElementSize, bool Signed>
-  void AVX128_VPMULL(OpcodeArgs);
+  void AVX128_VPMULL(OpcodeArgs, IR::OpSize ElementSize, bool Signed);
 
   void AVX128_VPMULHRSW(OpcodeArgs);
 
-  template<bool Signed>
-  void AVX128_VPMULHW(OpcodeArgs);
+  void AVX128_VPMULHW(OpcodeArgs, bool Signed);
 
-  template<IR::OpSize DstElementSize, IR::OpSize SrcElementSize>
-  void AVX128_InsertScalar_CVT_Float_To_Float(OpcodeArgs);
+  void AVX128_InsertScalar_CVT_Float_To_Float(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize);
 
-  template<IR::OpSize DstElementSize, IR::OpSize SrcElementSize>
-  void AVX128_Vector_CVT_Float_To_Float(OpcodeArgs);
+  void AVX128_Vector_CVT_Float_To_Float(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize);
 
-  template<IR::OpSize SrcElementSize, bool HostRoundingMode>
-  void AVX128_Vector_CVT_Float_To_Int(OpcodeArgs);
+  void AVX128_Vector_CVT_Float_To_Int(OpcodeArgs, IR::OpSize SrcElementSize, bool HostRoundingMode);
 
-  template<IR::OpSize SrcElementSize, bool Widen>
-  void AVX128_Vector_CVT_Int_To_Float(OpcodeArgs);
+  void AVX128_Vector_CVT_Int_To_Float(OpcodeArgs, IR::OpSize SrcElementSize, bool Widen);
 
   void AVX128_VEXTRACT128(OpcodeArgs);
   void AVX128_VAESImc(OpcodeArgs);
@@ -1045,36 +1031,28 @@ public:
 
   void AVX128_PHMINPOSUW(OpcodeArgs);
 
-  template<IR::OpSize ElementSize>
-  void AVX128_VectorRound(OpcodeArgs);
-  template<IR::OpSize ElementSize>
-  void AVX128_InsertScalarRound(OpcodeArgs);
+  void AVX128_VectorRound(OpcodeArgs, IR::OpSize ElementSize);
+  void AVX128_InsertScalarRound(OpcodeArgs, IR::OpSize ElementSize);
 
-  template<IR::OpSize ElementSize>
-  void AVX128_VDPP(OpcodeArgs);
+  void AVX128_VDPP(OpcodeArgs, IR::OpSize ElementSize);
   void AVX128_VPERMQ(OpcodeArgs);
 
   void AVX128_VPSHUFW(OpcodeArgs, bool Low);
 
-  template<IR::OpSize ElementSize>
-  void AVX128_VSHUF(OpcodeArgs);
+  void AVX128_VSHUF(OpcodeArgs, IR::OpSize ElementSize);
 
-  template<IR::OpSize ElementSize>
-  void AVX128_VPERMILImm(OpcodeArgs);
+  void AVX128_VPERMILImm(OpcodeArgs, IR::OpSize ElementSize);
 
-  template<IROps IROp, IR::OpSize ElementSize>
-  void AVX128_VHADDP(OpcodeArgs);
+  void AVX128_VHADDP(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
 
   void AVX128_VPHADDSW(OpcodeArgs);
 
   void AVX128_VPMADDUBSW(OpcodeArgs);
   void AVX128_VPMADDWD(OpcodeArgs);
 
-  template<IR::OpSize ElementSize>
-  void AVX128_VBLEND(OpcodeArgs);
+  void AVX128_VBLEND(OpcodeArgs, IR::OpSize ElementSize);
 
-  template<IR::OpSize ElementSize>
-  void AVX128_VHSUBP(OpcodeArgs);
+  void AVX128_VHSUBP(OpcodeArgs, IR::OpSize ElementSize);
 
   void AVX128_VPSHUFB(OpcodeArgs);
   void AVX128_VPSADBW(OpcodeArgs);
@@ -1085,28 +1063,23 @@ public:
   void AVX128_VMASKMOVImpl(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DstSize, bool IsStore, const X86Tables::DecodedOperand& MaskOp,
                            const X86Tables::DecodedOperand& DataOp);
 
-  template<bool IsStore>
-  void AVX128_VPMASKMOV(OpcodeArgs);
+  void AVX128_VPMASKMOV(OpcodeArgs, bool IsStore);
 
-  template<IR::OpSize ElementSize, bool IsStore>
-  void AVX128_VMASKMOV(OpcodeArgs);
+  void AVX128_VMASKMOV(OpcodeArgs, IR::OpSize ElementSize, bool IsStore);
 
   void AVX128_MASKMOV(OpcodeArgs);
 
-  template<IR::OpSize ElementSize>
-  void AVX128_VectorVariableBlend(OpcodeArgs);
+  void AVX128_VectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize);
 
   void AVX128_SaveAVXState(Ref MemBase);
   void AVX128_RestoreAVXState(Ref MemBase);
   void AVX128_DefaultAVXState();
 
   void AVX128_VPERM2(OpcodeArgs);
-  template<IR::OpSize ElementSize>
-  void AVX128_VTESTP(OpcodeArgs);
+  void AVX128_VTESTP(OpcodeArgs, IR::OpSize ElementSize);
   void AVX128_PTest(OpcodeArgs);
 
-  template<IR::OpSize ElementSize>
-  void AVX128_VPERMILReg(OpcodeArgs);
+  void AVX128_VPERMILReg(OpcodeArgs, IR::OpSize ElementSize);
 
   void AVX128_VPERMD(OpcodeArgs);
 
@@ -1119,8 +1092,7 @@ public:
   RefPair AVX128_VPGatherQPSImpl(Ref Dest, Ref Mask, RefVSIB VSIB);
   RefPair AVX128_VPGatherImpl(OpSize Size, OpSize ElementLoadSize, OpSize AddrElementSize, RefPair Dest, RefPair Mask, RefVSIB VSIB);
 
-  template<OpSize AddrElementSize>
-  void AVX128_VPGATHER(OpcodeArgs);
+  void AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize);
 
   void AVX128_VCVTPH2PS(OpcodeArgs);
   void AVX128_VCVTPS2PH(OpcodeArgs);
@@ -1154,9 +1126,14 @@ public:
     StoreXMMRegister(XMM, Value);
   }
 
+  void AVXVectorALUOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
+  void AVXVectorUnaryOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
+  void AVXVectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize);
+
   // End of AVX 256-bit implementation
 
   void InvalidOp(OpcodeArgs);
+  void NoExecOp(OpcodeArgs);
 
   void SetPackedRFLAG(bool Lower8, Ref Src);
   Ref GetPackedRFLAG(uint32_t FlagsMask = ~0U);
@@ -1175,15 +1152,44 @@ public:
     }
   }
 
-  void FlushRegisterCache(bool SRAOnly = false) {
+  void StoreContextHelper(IR::OpSize Size, RegisterClassType Class, Ref Value, uint32_t Offset) {
+    // For i128Bit, we won't see a normal Constant to inline, but as a special
+    // case we can replace with a 2x64-bit store which can use inline zeroes.
+    if (Size == OpSize::i128Bit) {
+      auto Header = GetOpHeader(WrapNode(Value));
+      const auto MAX_STP_OFFSET = (252 * 4);
+
+      if (Offset <= MAX_STP_OFFSET && Header->Op == OP_LOADNAMEDVECTORCONSTANT) {
+        auto Const = Header->C<IR::IROp_LoadNamedVectorConstant>();
+
+        if (Const->Constant == IR::NamedVectorConstant::NAMED_VECTOR_ZERO) {
+          Ref Zero = _Constant(0);
+          Ref STP = _StoreContextPair(IR::OpSize::i64Bit, GPRClass, Zero, Zero, Offset);
+
+          // XXX: This works around InlineConstant not having an associated
+          // register class, else we'd just do InlineConstant above.
+          Ref InlineZero = _InlineConstant(0);
+          ReplaceNodeArgument(STP, 0, InlineZero);
+          ReplaceNodeArgument(STP, 1, InlineZero);
+          return;
+        }
+      }
+    }
+
+    _StoreContext(Size, Class, Value, Offset);
+  }
+
+  void FlushRegisterCache(bool SRAOnly = false, bool MMXOnly = false) {
     // At block boundaries, fix up the carry flag.
     if (!SRAOnly) {
       RectifyCarryInvert(CFInvertedABI);
     }
 
-    CalculateDeferredFlags();
+    if (!MMXOnly) {
+      CalculateDeferredFlags();
+    }
 
-    const auto GPRSize = CTX->GetGPROpSize();
+    const auto GPRSize = GetGPROpSize();
     const auto VectorSize = GetGuestVectorLength();
 
     // Write backwards. This is a heuristic to improve coalescing, since we
@@ -1203,18 +1209,25 @@ public:
       Bits &= Mask;
     }
 
+    if (MMXOnly) {
+      Mask &= ((1ull << (MM7Index - MM0Index + 1)) - 1) << MM0Index;
+      Bits &= Mask;
+    }
+
     while (Bits != 0) {
       uint32_t Index = 63 - std::countl_zero(Bits);
       Ref Value = RegCache.Value[Index];
 
       if (Index >= GPR0Index && Index <= GPR15Index) {
-        _StoreRegister(Value, Index - GPR0Index, GPRClass, GPRSize);
+        Ref R = _StoreRegister(Value, GPRSize);
+        R->Reg = PhysicalRegister(GPRFixedClass, Index - GPR0Index).Raw;
       } else if (Index == PFIndex) {
         _StorePF(Value, GPRSize);
       } else if (Index == AFIndex) {
         _StoreAF(Value, GPRSize);
       } else if (Index >= FPR0Index && Index <= FPR15Index) {
-        _StoreRegister(Value, Index - FPR0Index, FPRClass, VectorSize);
+        Ref R = _StoreRegister(Value, VectorSize);
+        R->Reg = PhysicalRegister(FPRFixedClass, Index - FPR0Index).Raw;
       } else if (Index == DFIndex) {
         _StoreContext(OpSize::i8Bit, GPRClass, Value, offsetof(Core::CPUState, flags[X86State::RFLAG_DF_RAW_LOC]));
       } else {
@@ -1238,10 +1251,10 @@ public:
           _StoreContextPair(Size, Class, ValueNext, Value, Offset - SizeInt);
           Bits &= ~NextBit;
         } else {
-          _StoreContext(Size, Class, Value, Offset);
+          StoreContextHelper(Size, Class, Value, Offset);
           // If Partial and MMX register, then we need to store all 1s in bits 64-80
           if (Partial && Index >= MM0Index && Index <= MM7Index) {
-            _StoreContext(OpSize::i16Bit, IR::GPRClass, _Constant(0xFFFF), Offset + 8);
+            _StoreContext(OpSize::i16Bit, IR::GPRClass, Constant(0xFFFF), Offset + 8);
           }
         }
       }
@@ -1254,6 +1267,10 @@ public:
     RegCache.Partial &= ~Mask;
   }
 
+  IR::OpSize GetGPROpSize() const {
+    return Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;
+  }
+
 protected:
   void RecordX87Use() override {
     CurrentHeader->HasX87 = true;
@@ -1307,6 +1324,7 @@ private:
   struct JumpTargetInfo {
     Ref BlockEntry;
     bool HaveEmitted;
+    bool IsEntryPoint;
   };
 
   FEXCore::Context::ContextImpl* CTX {};
@@ -1355,11 +1373,6 @@ private:
 
   Ref ADDSUBPOpImpl(OpSize Size, IR::OpSize ElementSize, Ref Src1, Ref Src2);
 
-  void AVXVectorALUOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
-  void AVXVectorUnaryOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
-
-  void AVXVectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize);
-
   void AVXVariableShiftImpl(OpcodeArgs, IROps IROp);
 
   Ref AESKeyGenAssistImpl(OpcodeArgs);
@@ -1492,7 +1505,7 @@ private:
 #undef OpcodeArgs
 
   Ref AppendSegmentOffset(Ref Value, uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false);
-  Ref GetSegment(uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false);
+  Ref GetSegment(uint32_t Flags, uint32_t DefaultPrefix = FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX, bool Override = false);
 
   void UpdatePrefixFromSegment(Ref Segment, uint32_t SegmentReg);
 
@@ -1500,14 +1513,32 @@ private:
   void StoreGPRRegister(uint32_t GPR, const Ref Src, IR::OpSize Size = OpSize::iInvalid, uint8_t Offset = 0);
   void StoreXMMRegister(uint32_t XMM, const Ref Src);
 
-  Ref GetRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset = 0);
+  Ref _GetRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset, bool Inline) {
+    const auto GPRSize = GetGPROpSize();
+    const auto Offs = Op->PC + Op->InstSize + Offset - Entry;
+    return Inline ? _InlineEntrypointOffset(GPRSize, Offs) : _EntrypointOffset(GPRSize, Offs);
+  }
 
-  bool IsOperandMem(const X86Tables::DecodedOperand& Operand, bool Load) {
+  Ref GetRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset = 0) {
+    return _GetRelocatedPC(Op, Offset, false);
+  }
+
+  void ExitRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset = 0) {
+    ExitFunction(_GetRelocatedPC(Op, Offset, true /* Inline */));
+  }
+
+  void ExitRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset, BranchHint Hint, Ref CallReturnAddress, Ref CallReturnBlock) {
+    ExitFunction(_GetRelocatedPC(Op, Offset, true /* Inline */), Hint, CallReturnAddress, CallReturnBlock);
+  }
+
+  [[nodiscard]]
+  static bool IsOperandMem(const X86Tables::DecodedOperand& Operand, bool Load) {
     // Literals are immediates as sources but memory addresses as destinations.
     return !(Load && Operand.IsLiteral()) && !Operand.IsGPR();
   }
 
-  bool IsNonTSOReg(MemoryAccessType Access, uint8_t Reg) {
+  [[nodiscard]]
+  static bool IsNonTSOReg(MemoryAccessType Access, uint8_t Reg) {
     return Access == MemoryAccessType::DEFAULT && Reg == X86State::REG_RSP;
   }
 
@@ -1614,7 +1645,7 @@ private:
   }
 
   void ZeroNZCV() {
-    CachedNZCV = _Constant(0);
+    CachedNZCV = Constant(0);
     NZCVDirty = true;
   }
 
@@ -1627,9 +1658,9 @@ private:
     // This is currently worse for 8/16-bit, but that should be optimized. TODO
     if (SrcSize >= OpSize::i32Bit) {
       if (SetPF) {
-        CalculatePF(_SubWithFlags(SrcSize, Res, _Constant(0)));
+        CalculatePF(SubWithFlags(SrcSize, Res, (uint64_t)0));
       } else {
-        _SubNZCV(SrcSize, Res, _Constant(0));
+        _SubNZCV(SrcSize, Res, Constant(0));
       }
 
       CFInverted = true;
@@ -1700,7 +1731,7 @@ private:
       } else {
         // Invert as a GPR
         unsigned Bit = IndexNZCV(FEXCore::X86State::RFLAG_CF_RAW_LOC);
-        SetNZCV(_Xor(OpSize::i32Bit, GetNZCV(), _Constant(1u << Bit)));
+        SetNZCV(_Xor(OpSize::i32Bit, GetNZCV(), Constant(1u << Bit)));
         CalculateDeferredFlags();
       }
 
@@ -1738,7 +1769,7 @@ private:
     }
 
     HandleNZCVWrite();
-    _SubNZCV(OpSize::i32Bit, _Constant(0), Value);
+    _SubNZCV(OpSize::i32Bit, Constant(0), Value);
     CFInverted = true;
   }
 
@@ -1763,25 +1794,25 @@ private:
       StoreRegister(Core::CPUState::AF_AS_GREG, false, Value);
     } else if (BitOffset == FEXCore::X86State::RFLAG_DF_RAW_LOC) {
       // For DF, we need to transform 0/1 into 1/-1
-      StoreDF(_SubShift(OpSize::i64Bit, _Constant(1), Value, ShiftType::LSL, 1));
+      StoreDF(_SubShift(OpSize::i64Bit, Constant(1), Value, ShiftType::LSL, 1));
     } else if (BitOffset == FEXCore::X86State::RFLAG_TF_RAW_LOC) {
       auto PackedTF = _LoadContext(OpSize::i8Bit, GPRClass, offsetof(FEXCore::Core::CPUState, flags[BitOffset]));
       // An exception should still be raised after an instruction that unsets TF, leave the unblocked bit set but unset
       // the TF bit to cause such behaviour. The handling code at the start of the next block will then unset the
       // unblocked bit before raising the exception.
-      auto NewPackedTF = _Select(FEXCore::IR::COND_EQ, Value, _Constant(0), _And(OpSize::i32Bit, PackedTF, _Constant(~1)), _Constant(1));
+      auto NewPackedTF = _Select(FEXCore::IR::COND_EQ, Value, Constant(0), _And(OpSize::i32Bit, PackedTF, Constant(~1)), Constant(1));
       _StoreContext(OpSize::i8Bit, GPRClass, NewPackedTF, offsetof(FEXCore::Core::CPUState, flags[BitOffset]));
     } else {
       _StoreContext(OpSize::i8Bit, GPRClass, Value, offsetof(FEXCore::Core::CPUState, flags[BitOffset]));
     }
   }
 
-  void SetAF(unsigned Constant) {
+  void SetAF(unsigned K) {
     // AF is stored in bit 4 of the AF flag byte, with garbage in the other
     // bits. This allows us to defer the extract in the usual case. When it is
     // read, bit 4 is extracted.  In order to write a constant value of AF, that
     // means we need to left-shift here to compensate.
-    SetRFLAG<FEXCore::X86State::RFLAG_AF_RAW_LOC>(_Constant(Constant << 4));
+    SetRFLAG<FEXCore::X86State::RFLAG_AF_RAW_LOC>(Constant(K << 4));
   }
 
   void ZeroPF_AF();
@@ -1797,7 +1828,8 @@ private:
     InvalidateReg(Core::CPUState::AF_AS_GREG);
   }
 
-  CondClassType CondForNZCVBit(unsigned BitOffset, bool Invert) {
+  [[nodiscard]]
+  static CondClassType CondForNZCVBit(unsigned BitOffset, bool Invert) {
     switch (BitOffset) {
     case X86State::RFLAG_SF_RAW_LOC: return {Invert ? COND_PL : COND_MI};
     case X86State::RFLAG_ZF_RAW_LOC: return {Invert ? COND_NEQ : COND_EQ};
@@ -1823,7 +1855,8 @@ private:
   static const int AVXHigh0Index = 48;
   static const int AVXHigh15Index = 63;
 
-  uint32_t CacheIndexToContextOffset(int Index) {
+  [[nodiscard]]
+  static uint32_t CacheIndexToContextOffset(int Index) {
     switch (Index) {
     case MM0Index ... MM7Index: return offsetof(FEXCore::Core::CPUState, mm[Index - MM0Index]);
     case AVXHigh0Index ... AVXHigh15Index: return offsetof(FEXCore::Core::CPUState, avx_high[Index - AVXHigh0Index][0]);
@@ -1832,7 +1865,8 @@ private:
     }
   }
 
-  RegisterClassType CacheIndexClass(int Index) {
+  [[nodiscard]]
+  static RegisterClassType CacheIndexClass(int Index) {
     if ((Index >= MM0Index && Index <= MM7Index) || Index >= FPR0Index) {
       return FPRClass;
     } else {
@@ -1840,7 +1874,8 @@ private:
     }
   }
 
-  IR::OpSize CacheIndexToOpSize(int Index) {
+  [[nodiscard]]
+  static IR::OpSize CacheIndexToOpSize(int Index) {
     // MMX registers are rounded up to 128-bit since they are shared with 80-bit
     // x87 registers, even though MMX is logically only 64-bit.
     if (Index >= AVXHigh0Index || ((Index >= MM0Index && Index <= MM7Index))) {
@@ -1948,7 +1983,7 @@ private:
   }
 
   Ref LoadGPR(uint8_t Reg) {
-    return LoadRegCache(Reg, GPR0Index + Reg, GPRClass, CTX->GetGPROpSize());
+    return LoadRegCache(Reg, GPR0Index + Reg, GPRClass, GetGPROpSize());
   }
 
   Ref LoadContext(IR::OpSize Size, uint8_t Index) {
@@ -2002,14 +2037,14 @@ private:
         auto Value = _Bfe(OpSize::i32Bit, 1, IndexNZCV(BitOffset), GetNZCV());
 
         if (Invert) {
-          return _Xor(OpSize::i32Bit, Value, _Constant(1));
+          return _Xor(OpSize::i32Bit, Value, Constant(1));
         } else {
           return Value;
         }
       } else {
         // Because we explicitly inverted for CF above, we use the unsafe
         // _NZCVSelect rather than the safe CF-aware version.
-        return _NZCVSelect(OpSize::i32Bit, CondForNZCVBit(BitOffset, Invert), _Constant(1), _Constant(0));
+        return _NZCVSelect01(CondForNZCVBit(BitOffset, Invert));
       }
     } else if (BitOffset == FEXCore::X86State::RFLAG_PF_RAW_LOC) {
       return LoadGPR(Core::CPUState::PF_AS_GREG);
@@ -2017,7 +2052,7 @@ private:
       return LoadGPR(Core::CPUState::AF_AS_GREG);
     } else if (BitOffset == FEXCore::X86State::RFLAG_DF_RAW_LOC) {
       // Recover the sign bit, it is the logical DF value
-      return _Lshr(OpSize::i64Bit, LoadDF(), _Constant(63));
+      return _Lshr(OpSize::i64Bit, LoadDF(), Constant(63));
     } else {
       return _LoadContext(OpSize::i8Bit, GPRClass, offsetof(Core::CPUState, flags[BitOffset]));
     }
@@ -2025,14 +2060,7 @@ private:
 
   // Returns (DF ? -Size : Size)
   Ref LoadDir(const unsigned Size) {
-    auto Dir = LoadDF();
-    auto Shift = FEXCore::ilog2(Size);
-
-    if (Shift) {
-      return _Lshl(CTX->GetGPROpSize(), Dir, _Constant(Shift));
-    } else {
-      return Dir;
-    }
+    return ARef(LoadDF()).Lshl(FEXCore::ilog2(Size)).Ref();
   }
 
   // Returns DF ? (X - Size) : (X + Size)
@@ -2096,7 +2124,7 @@ private:
       // Zero AF. Note that the comparison sets the raw PF to 0/1 above, so
       // PF[4] is 0 so the XOR with PF will have no effect, so setting the AF
       // byte to zero will indeed zero AF as intended.
-      SetRFLAG<FEXCore::X86State::RFLAG_AF_RAW_LOC>(_Constant(0));
+      SetRFLAG<FEXCore::X86State::RFLAG_AF_RAW_LOC>(Constant(0));
     }
 
     // Convert NZCV from the Arm representation to an eXternal representation
@@ -2113,7 +2141,7 @@ private:
   void ConvertNZCVToX87() {
     LOGMAN_THROW_A_FMT(NZCVDirty && CachedNZCV, "NZCV must be saved");
 
-    Ref V = _NZCVSelect(OpSize::i32Bit, CondForNZCVBit(FEXCore::X86State::RFLAG_OF_RAW_LOC, false), _Constant(1), _Constant(0));
+    Ref V = _NZCVSelect01(CondForNZCVBit(FEXCore::X86State::RFLAG_OF_RAW_LOC, false));
 
     if (CTX->HostFeatures.SupportsFlagM2) {
       // Convert to x86 flags, saves us from or'ing after.
@@ -2121,8 +2149,8 @@ private:
     }
 
     // CF is inverted after FCMP
-    Ref C = _NZCVSelect(OpSize::i32Bit, CondForNZCVBit(FEXCore::X86State::RFLAG_CF_RAW_LOC, true), _Constant(1), _Constant(0));
-    Ref Z = _NZCVSelect(OpSize::i32Bit, CondForNZCVBit(FEXCore::X86State::RFLAG_ZF_RAW_LOC, false), _Constant(1), _Constant(0));
+    Ref C = _NZCVSelect01(CondForNZCVBit(FEXCore::X86State::RFLAG_CF_RAW_LOC, true));
+    Ref Z = _NZCVSelect01(CondForNZCVBit(FEXCore::X86State::RFLAG_ZF_RAW_LOC, false));
 
     if (!CTX->HostFeatures.SupportsFlagM2) {
       C = _Or(OpSize::i32Bit, C, V);
@@ -2130,7 +2158,7 @@ private:
     }
 
     SetRFLAG<FEXCore::X86State::X87FLAG_C0_LOC>(C);
-    SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(_Constant(0));
+    SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Constant(0));
     SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(V);
     SetRFLAG<FEXCore::X86State::X87FLAG_C3_LOC>(Z);
   }
@@ -2180,9 +2208,9 @@ private:
       return CachedNamedVectorConstants[NamedConstant][log2_size_bytes];
     }
 
-    auto Constant = _LoadNamedVectorConstant(Size, NamedConstant);
-    CachedNamedVectorConstants[NamedConstant][log2_size_bytes] = Constant;
-    return Constant;
+    auto K = _LoadNamedVectorConstant(Size, NamedConstant);
+    CachedNamedVectorConstants[NamedConstant][log2_size_bytes] = K;
+    return K;
   }
   Ref LoadAndCacheIndexedNamedVectorConstant(IR::OpSize Size, FEXCore::IR::IndexNamedVectorConstant NamedIndexedConstant, uint32_t Index) {
     IndexNamedVectorMapKey Key {
@@ -2196,9 +2224,9 @@ private:
       return it->second;
     }
 
-    auto Constant = _LoadNamedVectorIndexedConstant(Size, NamedIndexedConstant, Index);
-    CachedIndexedNamedVectorConstants.insert_or_assign(Key, Constant);
-    return Constant;
+    auto K = _LoadNamedVectorIndexedConstant(Size, NamedIndexedConstant, Index);
+    CachedIndexedNamedVectorConstants.insert_or_assign(Key, K);
+    return K;
   }
 
   Ref LoadUncachedZeroVector(IR::OpSize Size) {
@@ -2216,9 +2244,9 @@ private:
     CachedIndexedNamedVectorConstants.clear();
   }
 
-  std::pair<bool, CondClassType> DecodeNZCVCondition(uint8_t OP);
+  std::optional<CondClassType> DecodeNZCVCondition(uint8_t OP);
   Ref SelectBit(Ref Cmp, IR::OpSize ResultSize, Ref TrueValue, Ref FalseValue);
-  Ref SelectCC(uint8_t OP, IR::OpSize ResultSize, Ref TrueValue, Ref FalseValue);
+  Ref SelectCC0All1(uint8_t OP);
 
   /**
    * @brief Flushes NZCV. Mostly vestigial.
@@ -2253,7 +2281,7 @@ private:
     }
 
     // Otherwise, prepare to branch.
-    auto Zero = _Constant(0);
+    auto Zero = Constant(0);
 
     // If the shift is zero, do not touch the flags.
     auto SetBlock = CreateNewCodeBlockAfter(GetCurrentBlock());
@@ -2293,7 +2321,6 @@ private:
    * @name These functions are used by the deferred flag handling while it is calculating and storing flags in to RFLAGs.
    * @{ */
   Ref LoadPFRaw(bool Mask, bool Invert);
-  Ref SelectPF(bool Invert, IR::OpSize ResultSize, Ref TrueValue, Ref FalseValue);
   Ref LoadAF();
   void FixupAF();
   void SetAFAndFixup(Ref AF);
@@ -2310,7 +2337,7 @@ private:
   Ref CalculateFlags_ADD(IR::OpSize SrcSize, Ref Src1, Ref Src2, bool UpdateCF = true);
   void CalculateFlags_MUL(IR::OpSize SrcSize, Ref Res, Ref High);
   void CalculateFlags_UMUL(Ref High);
-  void CalculateFlags_Logical(IR::OpSize SrcSize, Ref Res, Ref Src1, Ref Src2);
+  void CalculateFlags_Logical(IR::OpSize SrcSize, Ref Res);
   void CalculateFlags_ShiftLeft(IR::OpSize SrcSize, Ref Res, Ref Src1, Ref Src2);
   void CalculateFlags_ShiftLeftImmediate(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift);
   void CalculateFlags_ShiftRight(IR::OpSize SrcSize, Ref Res, Ref Src1, Ref Src2);
@@ -2321,20 +2348,7 @@ private:
   void CalculateFlags_ZCNT(IR::OpSize SrcSize, Ref Result);
   /**  @} */
 
-  Ref AndConst(FEXCore::IR::OpSize Size, Ref Node, uint64_t Const) {
-    uint64_t NodeConst;
-
-    if (IsValueConstant(WrapNode(Node), &NodeConst)) {
-      return _Constant(NodeConst & Const);
-    } else {
-      return _And(Size, Node, _Constant(Const));
-    }
-  }
-
-  /**  @} */
-
   Ref GetX87Top();
-  Ref GetX87Tag(Ref Value, Ref AbridgedFTW);
   void SetX87FTW(Ref FTW);
   Ref GetX87FTW_Helper();
   void SetX87Top(Ref Value);
@@ -2342,17 +2356,19 @@ private:
   void ChgStateX87_MMX() override {
     LOGMAN_THROW_A_FMT(MMXState == MMXState_X87, "Expected state to be x87");
     _StackForceSlow();
-    SetX87Top(_Constant(0));                             // top reset to zero
-    StoreContext(AbridgedFTWIndex, _Constant(0xFFFFUL)); // all valid
+    SetX87Top(Constant(0));                             // top reset to zero
+    StoreContext(AbridgedFTWIndex, Constant(0xFFFFUL)); // all valid
     MMXState = MMXState_MMX;
   }
 
   void ChgStateMMX_X87() override {
     LOGMAN_THROW_A_FMT(MMXState == MMXState_MMX, "Expected state to be MMX");
+    // The opcode dispatcher register cache is used for MMX, but the x87 pass register cache is used for x87, spill to
+    // context to ensure coherence.
+    FlushRegisterCache(false, true);
     // We explicitly initialize to x87 state in StartNewBlock.
     // So if we ever change this to do something else, we need to
     // make sure that we consider if we need to explicitly set it there.
-    FlushRegisterCache();
     MMXState = MMXState_X87;
   }
 
@@ -2368,10 +2384,17 @@ private:
   bool BlockSetRIP {false};
 
   bool Multiblock {};
+  bool Is64BitMode {};
   uint64_t Entry {};
+
+  // Set if mono hacks are enabled and the current block is the mono callsite backpatcher, in which case the
+  // XCHG ops that would patch code are replaced with a hook that performs the write and manually invalidates
+  // the target address.
+  bool IsMonoBackpatcherBlock {false};
   IROp_IRHeader* CurrentHeader {};
 
-  bool IsTSOEnabled(FEXCore::IR::RegisterClassType Class) {
+  [[nodiscard]]
+  bool IsTSOEnabled(FEXCore::IR::RegisterClassType Class) const {
     if (ForceTSO == ForceTSOMode::ForceEnabled) {
       return true;
     } else if (ForceTSO == ForceTSOMode::ForceDisabled) {
@@ -2401,7 +2424,7 @@ private:
 
   Ref _LoadMemAutoTSO(FEXCore::IR::RegisterClassType Class, IR::OpSize Size, AddressMode A, IR::OpSize Align = IR::OpSize::i8Bit) {
     bool AtomicTSO = IsTSOEnabled(Class) && !A.NonTSO;
-    A = SelectAddressMode(this, A, CTX->GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, AtomicTSO, Class != GPRClass, Size);
+    A = SelectAddressMode(this, A, GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, AtomicTSO, Class != GPRClass, Size);
 
     if (AtomicTSO) {
       return _LoadMemTSO(Class, Size, A.Base, A.Index, Align, A.IndexType, A.IndexScale);
@@ -2421,7 +2444,7 @@ private:
       A.Offset = 0;
     }
 
-    Out.Base = LoadEffectiveAddress(this, A, CTX->GetGPROpSize(), true, false);
+    Out.Base = LoadEffectiveAddress(this, A, GetGPROpSize(), true, false);
     return Out;
   }
 
@@ -2452,7 +2475,7 @@ private:
 
   Ref _StoreMemAutoTSO(FEXCore::IR::RegisterClassType Class, IR::OpSize Size, AddressMode A, Ref Value, IR::OpSize Align = IR::OpSize::i8Bit) {
     bool AtomicTSO = IsTSOEnabled(Class) && !A.NonTSO;
-    A = SelectAddressMode(this, A, CTX->GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, AtomicTSO, Class != GPRClass, Size);
+    A = SelectAddressMode(this, A, GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, AtomicTSO, Class != GPRClass, Size);
 
     if (AtomicTSO) {
       return _StoreMemTSO(Class, Size, Value, A.Base, A.Index, Align, A.IndexType, A.IndexScale);
@@ -2494,13 +2517,140 @@ private:
     return Value;
   }
 
-  void Push(IR::OpSize Size, Ref Value) {
-    auto OldSP = LoadGPRRegister(X86State::REG_RSP);
-    auto NewSP = _Push(CTX->GetGPROpSize(), Size, Value, OldSP);
-    StoreGPRRegister(X86State::REG_RSP, NewSP);
+  Ref VZeroExtendOperand(OpSize Size, X86Tables::DecodedOperand Op, Ref Value) {
+    bool IsMMX = Op.IsGPR() && Op.Data.GPR.GPR >= X86State::REG_MM_0;
+    bool AlreadyExtended = Op.IsGPRDirect() || Op.IsGPRIndirect() || IsMMX;
+
+    return AlreadyExtended ? Value : _VMov(Size, Value);
   }
 
-  void InstallHostSpecificOpcodeHandlers();
+  void Push(IR::OpSize Size, Ref Value) {
+    auto OldSP = LoadGPRRegister(X86State::REG_RSP);
+    auto NewSP = _Push(GetGPROpSize(), Size, Value, OldSP);
+    StoreGPRRegister(X86State::REG_RSP, NewSP);
+    FlushRegisterCache();
+  }
+
+  struct ArithRef {
+    IREmitter* E {};
+    bool IsConstant {};
+    union {
+      Ref R {};
+      uint64_t C;
+    };
+
+    ArithRef() {}
+
+    ArithRef(IREmitter* IREmit, Ref Reference)
+      : E(IREmit)
+      , IsConstant(false)
+      , R(Reference) {}
+
+    ArithRef(IREmitter* IREmit, uint64_t K)
+      : E(IREmit)
+      , IsConstant(true)
+      , C(K) {}
+
+    ArithRef Neg() {
+      return IsConstant ? ArithRef(E, -C) : ArithRef(E, E->_Neg(OpSize::i64Bit, R));
+    }
+
+    ArithRef And(uint64_t K) {
+      return IsConstant ? ArithRef(E, C & K) : ArithRef(E, E->_And(OpSize::i64Bit, R, E->Constant(K)));
+    }
+
+    ArithRef Presub(uint64_t K) {
+      return IsConstant ? ArithRef(E, K - C) : ArithRef(E, E->Sub(OpSize::i64Bit, E->Constant(K), R));
+    }
+
+    ArithRef Lshl(uint64_t Shift) {
+      if (Shift == 0) {
+        return *this;
+      } else if (IsConstant) {
+        return ArithRef(E, C << Shift);
+      } else {
+        return ArithRef(E, E->_Lshl(OpSize::i64Bit, R, E->Constant(Shift)));
+      }
+    }
+
+    ArithRef Bfe(unsigned Start, unsigned Size) {
+      if (IsConstant) {
+        return ArithRef(E, (C >> Start) & ((1ull << Size) - 1));
+      } else {
+        return ArithRef(E, E->_Bfe(OpSize::i64Bit, Size, Start, R));
+      }
+    }
+
+    ArithRef Sbfe(unsigned Start, unsigned Size) {
+      if (IsConstant) {
+        uint64_t SourceMask = Size == 64 ? ~0ULL : ((1ULL << Size) - 1);
+        SourceMask <<= Start;
+
+        int64_t NewConstant = (C & SourceMask) >> Start;
+        NewConstant <<= 64 - Size;
+        NewConstant >>= 64 - Size;
+
+        return ArithRef(E, NewConstant);
+      } else {
+        return ArithRef(E, E->_Sbfe(OpSize::i64Bit, Size, Start, R));
+      }
+    }
+
+    Ref BfiInto(Ref Bitfield, unsigned Start, unsigned Size) {
+      if (IsConstant && (Size > 0 && Size < 64)) {
+        uint64_t SourceMask = (1ULL << Size) - 1;
+        uint64_t SourceMaskShifted = SourceMask << Start;
+
+        if (C == 0) {
+          return E->_And(OpSize::i64Bit, Bitfield, E->_InlineConstant(~SourceMaskShifted));
+        } else if (C == SourceMask) {
+          return E->_Or(OpSize::i64Bit, Bitfield, E->_InlineConstant(SourceMaskShifted));
+        }
+      }
+
+      if (IsConstant) {
+        return E->_Bfi(OpSize::i64Bit, Size, Start, Bitfield, E->Constant(C));
+      } else {
+        return E->_Bfi(OpSize::i64Bit, Size, Start, Bitfield, R);
+      }
+    }
+
+    ArithRef MaskBit(OpSize Size) {
+      if (IsConstant) {
+        uint64_t ShiftMask = Size == OpSize::i64Bit ? 63 : 31;
+        uint64_t Result = 1ull << (C & ShiftMask);
+        if (ShiftMask == 31) {
+          Result &= ((1ull << 32) - 1);
+        }
+
+        return ArithRef(E, Result);
+      } else {
+        return ArithRef(E, E->_Lshl(Size, E->Constant(1), R));
+      }
+    }
+
+    Ref Ref() {
+      return IsConstant ? E->Constant(C) : R;
+    }
+
+    bool IsDefinitelyZero() const {
+      return IsConstant && C == 0;
+    }
+  };
+
+  ArithRef ARef(Ref R) {
+    uint64_t C;
+
+    if (IsValueConstant(WrapNode(R), &C)) {
+      return ARef(C);
+    } else {
+      return ArithRef(this, R);
+    }
+  }
+
+  ArithRef ARef(uint64_t K) {
+    return ArithRef(this, K);
+  }
 
   ///< Segment telemetry tracking
   uint32_t SegmentsNeedReadCheck {~0U};
@@ -2510,21 +2660,19 @@ private:
 
 constexpr inline void InstallToTable(auto& FinalTable, const auto& LocalTable) {
   for (const auto& Op : LocalTable) {
-    auto OpNum = std::get<0>(Op);
-    auto Dispatcher = std::get<2>(Op);
-    for (uint8_t i = 0; i < std::get<1>(Op); ++i) {
+    auto OpNum = Op.Op;
+    auto Dispatcher = Op.Ptr;
+    for (uint8_t i = 0; i < Op.Count; ++i) {
       auto& TableOp = FinalTable[OpNum + i];
 #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
-      if (TableOp.OpcodeDispatcher) {
+      if (TableOp.OpcodeDispatcher.OpDispatch) {
         ERROR_AND_DIE_FMT("Duplicate Entry {}", TableOp.Name);
       }
 #endif
 
-      TableOp.OpcodeDispatcher = Dispatcher;
+      TableOp.OpcodeDispatcher.OpDispatch = Dispatcher;
     }
   }
 }
 
-void InstallOpcodeHandlers(Context::OperatingMode Mode);
-
 } // namespace FEXCore::IR

FEXCore/Source/Interface/Core/OpcodeDispatcher/BaseTables.h

@@ -3,7 +3,7 @@
 #include "Interface/Core/OpcodeDispatcher.h"
 
 namespace FEXCore::IR {
-constexpr inline std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr> OpDispatch_BaseOpTable[] = {
+constexpr inline DispatchTableEntry OpDispatch_BaseOpTable[] = {
   // Instructions
   {0x00, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ADD, FEXCore::IR::IROps::OP_ATOMICFETCHADD, 0>},
 
@@ -57,6 +57,7 @@ constexpr inline std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr> OpDispat
   {0xC2, 2, &OpDispatchBuilder::RETOp},
   {0xC8, 1, &OpDispatchBuilder::EnterOp},
   {0xC9, 1, &OpDispatchBuilder::LEAVEOp},
+  {0xCA, 2, &OpDispatchBuilder::RETFARIndirectOp},
   {0xCC, 2, &OpDispatchBuilder::INTOp},
   {0xCF, 1, &OpDispatchBuilder::IRETOp},
   {0xD7, 2, &OpDispatchBuilder::XLATOp},
@@ -75,33 +76,4 @@ constexpr inline std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr> OpDispat
   {0xFA, 2, &OpDispatchBuilder::PermissionRestrictedOp},
   {0xFC, 2, &OpDispatchBuilder::FLAGControlOp},
 };
-
-constexpr inline std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr> OpDispatch_BaseOpTable_64[] = {
-  {0x63, 1, &OpDispatchBuilder::MOVSXDOp},
-  {0xA0, 4, &OpDispatchBuilder::MOVOffsetOp},
-};
-
-constexpr inline std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr> OpDispatch_BaseOpTable_32[] = {
-  {0x06, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX>},
-  {0x07, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX>},
-  {0x0E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX>},
-  {0x16, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX>},
-  {0x17, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX>},
-  {0x1E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX>},
-  {0x1F, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX>},
-  {0x27, 1, &OpDispatchBuilder::DAAOp},
-  {0x2F, 1, &OpDispatchBuilder::DASOp},
-  {0x37, 1, &OpDispatchBuilder::AAAOp},
-  {0x3F, 1, &OpDispatchBuilder::AASOp},
-  {0x40, 8, &OpDispatchBuilder::INCOp},
-  {0x48, 8, &OpDispatchBuilder::DECOp},
-
-  {0x60, 1, &OpDispatchBuilder::PUSHAOp},
-  {0x61, 1, &OpDispatchBuilder::POPAOp},
-  {0xA0, 4, &OpDispatchBuilder::MOVOffsetOp},
-  {0xCE, 1, &OpDispatchBuilder::INTOp},
-  {0xD4, 1, &OpDispatchBuilder::AAMOp},
-  {0xD5, 1, &OpDispatchBuilder::AADOp},
-  {0xD6, 1, &OpDispatchBuilder::SALCOp},
-};
 } // namespace FEXCore::IR

FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp

@@ -11,10 +11,7 @@ $end_info$
 #include <FEXCore/Utils/LogManager.h>
 #include "Interface/Core/OpcodeDispatcher.h"
 
-#include <array>
 #include <cstdint>
-#include <tuple>
-#include <utility>
 
 namespace FEXCore::IR {
 class OrderedNode;
@@ -22,24 +19,20 @@ class OrderedNode;
 #define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op
 
 void OpDispatchBuilder::SHA1NEXTEOp(OpcodeArgs) {
+  if (!CTX->HostFeatures.SupportsSHA) {
+    UnimplementedOp(Op);
+    return;
+  }
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
 
-  Ref RotatedNode {};
-  if (CTX->HostFeatures.SupportsSHA) {
-    // ARMv8 SHA1 extension provides a `SHA1H` instruction which does a fixed rotate by 30.
-    // This only operates on element 0 rather than element 3. We don't have the luxury of rewriting the x86 SHA algorithm to take advantage of this.
-    // Move the element to zero, rotate, and then move back (Using duplicates).
-    // Saves one instruction versus that path that doesn't support SHA extension.
-    auto Duplicated = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
-    auto Sha1HRotated = _VSha1H(Duplicated);
-    RotatedNode = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Sha1HRotated, 0);
-  } else {
-    // SHA1 extension missing, manually rotate.
-    // Emulate rotate.
-    auto ShiftLeft = _VShlI(OpSize::i128Bit, OpSize::i32Bit, Dest, 30);
-    RotatedNode = _VUShraI(OpSize::i128Bit, OpSize::i32Bit, ShiftLeft, Dest, 2);
-  }
+  // ARMv8 SHA1 extension provides a `SHA1H` instruction which does a fixed rotate by 30.
+  // This only operates on element 0 rather than element 3. We don't have the luxury of rewriting the x86 SHA algorithm to take advantage of this.
+  // Move the element to zero, rotate, and then move back (Using duplicates).
+  // Saves one instruction versus that path that doesn't support SHA extension.
+  auto Duplicated = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
+  auto Sha1HRotated = _VSha1H(Duplicated);
+  auto RotatedNode = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Sha1HRotated, 0);
   auto Tmp = _VAdd(OpSize::i128Bit, OpSize::i32Bit, Src, RotatedNode);
   auto Result = _VInsElement(OpSize::i128Bit, OpSize::i32Bit, 3, 3, Src, Tmp);
 
@@ -47,6 +40,10 @@ void OpDispatchBuilder::SHA1NEXTEOp(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::SHA1MSG1Op(OpcodeArgs) {
+  if (!CTX->HostFeatures.SupportsSHA) {
+    UnimplementedOp(Op);
+    return;
+  }
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
 
@@ -59,334 +56,145 @@ void OpDispatchBuilder::SHA1MSG1Op(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::SHA1MSG2Op(OpcodeArgs) {
+  if (!CTX->HostFeatures.SupportsSHA) {
+    UnimplementedOp(Op);
+    return;
+  }
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
 
-  Ref Result;
-  if (CTX->HostFeatures.SupportsSHA) {
-    // ARM SHA1 mostly matches x86 semantics, except the input and outputs are both flipped from elements 0,1,2,3 to 3,2,1,0.
-    auto Src1 = SHADataShuffle(Dest);
-    auto Src2 = SHADataShuffle(Src);
-
-    // The result is swizzled differently than expected
-    Result = SHADataShuffle(_VSha1SU1(Src1, Src2));
-  } else {
-    // Shift the incoming source left by a 32-bit element, inserting Zeros.
-    // This could be slightly improved to use a VInsGPR with the zero register.
-    const auto ZeroRegister = LoadZeroVector(OpSize::i32Bit);
-    auto Src2Shift = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, ZeroRegister, 12);
-    auto Xor1 = _VXor(OpSize::i128Bit, OpSize::i8Bit, Dest, Src2Shift);
-
-    // Emulate rotate.
-    auto ShiftLeftXor1 = _VShlI(OpSize::i128Bit, OpSize::i32Bit, Xor1, 1);
-    auto RotatedXor1 = _VUShraI(OpSize::i128Bit, OpSize::i32Bit, ShiftLeftXor1, Xor1, 31);
-
-    // Element0 didn't get XOR'd with anything, so do it now.
-    auto ExtractUpper = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, RotatedXor1, 3);
-    auto XorLower = _VXor(OpSize::i128Bit, OpSize::i8Bit, Dest, ExtractUpper);
-
-    // Emulate rotate.
-    auto ShiftLeftXorLower = _VShlI(OpSize::i128Bit, OpSize::i32Bit, XorLower, 1);
-    auto RotatedXorLower = _VUShraI(OpSize::i128Bit, OpSize::i32Bit, ShiftLeftXorLower, XorLower, 31);
-
-    Result = _VInsElement(OpSize::i128Bit, OpSize::i32Bit, 0, 0, RotatedXor1, RotatedXorLower);
-  }
+  // ARM SHA1 mostly matches x86 semantics, except the input and outputs are both flipped from elements 0,1,2,3 to 3,2,1,0.
+  auto Src1 = SHADataShuffle(Dest);
+  auto Src2 = SHADataShuffle(Src);
 
+  // The result is swizzled differently than expected
+  auto Result = SHADataShuffle(_VSha1SU1(Src1, Src2));
 
   StoreResult(FPRClass, Op, Result, OpSize::iInvalid);
 }
 
 void OpDispatchBuilder::SHA1RNDS4Op(OpcodeArgs) {
-  using FnType = Ref (*)(OpDispatchBuilder&, Ref, Ref, Ref);
-
-  const auto f0 = [](OpDispatchBuilder& Self, Ref B, Ref C, Ref D) -> Ref { // sha1c?
-    return Self._Xor(OpSize::i32Bit, Self._And(OpSize::i32Bit, B, C), Self._Andn(OpSize::i32Bit, D, B));
-  };
-  const auto f1 = [](OpDispatchBuilder& Self, Ref B, Ref C, Ref D) -> Ref { // sha1p with different key
-    return Self._Xor(OpSize::i32Bit, Self._Xor(OpSize::i32Bit, B, C), D);
-  };
-  const auto f2 = [](OpDispatchBuilder& Self, Ref B, Ref C, Ref D) -> Ref { // sha1m
-    return Self.BitwiseAtLeastTwo(B, C, D);
-  };
-  const auto f3 = [](OpDispatchBuilder& Self, Ref B, Ref C, Ref D) -> Ref { // sha1p
-    return Self._Xor(OpSize::i32Bit, Self._Xor(OpSize::i32Bit, B, C), D);
-  };
-
-  constexpr std::array<uint32_t, 4> k_array {
-    0x5A827999U,
-    0x6ED9EBA1U,
-    0x8F1BBCDCU,
-    0xCA62C1D6U,
-  };
-
-  constexpr std::array<FnType, 4> fn_array {
-    f0,
-    f1,
-    f2,
-    f3,
-  };
-
-
+  if (!CTX->HostFeatures.SupportsSHA) {
+    UnimplementedOp(Op);
+    return;
+  }
   const uint64_t Imm8 = Op->Src[1].Literal() & 0b11;
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
 
   Ref Result {};
-  if (CTX->HostFeatures.SupportsSHA) {
-    Ref ConstantVector {};
-    switch (Imm8) {
-    case 0:
-      ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K0);
-      break;
-    case 1:
-      ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K1);
-      break;
-    case 2:
-      ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K2);
-      break;
-    case 3:
-      ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K3);
-      break;
-    }
+  Ref ConstantVector {};
+  switch (Imm8) {
+  case 0:
+    ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K0);
+    break;
+  case 1:
+    ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K1);
+    break;
+  case 2:
+    ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K2);
+    break;
+  case 3:
+    ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K3);
+    break;
+  }
 
-    const auto ZeroRegister = LoadZeroVector(OpSize::i32Bit);
+  const auto ZeroRegister = LoadZeroVector(OpSize::i32Bit);
 
-    Ref Src1 = SHADataShuffle(Dest);
-    Ref Src2 = SHADataShuffle(Src);
-    Src2 = _VAdd(OpSize::i128Bit, OpSize::i32Bit, Src2, ConstantVector);
+  Ref Src1 = SHADataShuffle(Dest);
+  Ref Src2 = SHADataShuffle(Src);
+  Src2 = _VAdd(OpSize::i128Bit, OpSize::i32Bit, Src2, ConstantVector);
 
-    switch (Imm8) {
-    case 0: Result = SHADataShuffle(_VSha1C(Src1, ZeroRegister, Src2)); break;
-    case 2: Result = SHADataShuffle(_VSha1M(Src1, ZeroRegister, Src2)); break;
-    case 1:
-    case 3: Result = SHADataShuffle(_VSha1P(Src1, ZeroRegister, Src2)); break;
-    }
-  } else {
-    const FnType Fn = fn_array[Imm8];
-    auto K = _Constant(OpSize::i32Bit, k_array[Imm8]);
-    auto W0E = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 3);
-
-    using RoundResult = std::tuple<Ref, Ref, Ref, Ref, Ref>;
-
-    const auto Round0 = [&]() -> RoundResult {
-      auto A = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
-      auto B = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 2);
-      auto C = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 1);
-      auto D = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 0);
-
-      auto A1 =
-        _Add(OpSize::i32Bit,
-             _Add(OpSize::i32Bit, _Add(OpSize::i32Bit, Fn(*this, B, C, D), _Ror(OpSize::i32Bit, A, _Constant(OpSize::i32Bit, 27))), W0E), K);
-      auto B1 = A;
-      auto C1 = _Ror(OpSize::i32Bit, B, _Constant(OpSize::i32Bit, 2));
-      auto D1 = C;
-      auto E1 = D;
-
-      return {A1, B1, C1, D1, E1};
-    };
-    const auto Round1To3 = [&](Ref A, Ref B, Ref C, Ref D, Ref E, Ref Src, unsigned W_idx) -> RoundResult {
-      // Kill W and E at the beginning
-      auto W = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, W_idx);
-      auto Q = _Add(OpSize::i32Bit, W, E);
-
-      auto ANext =
-        _Add(OpSize::i32Bit,
-             _Add(OpSize::i32Bit, _Add(OpSize::i32Bit, Fn(*this, B, C, D), _Ror(OpSize::i32Bit, A, _Constant(OpSize::i32Bit, 27))), Q), K);
-      auto BNext = A;
-      auto CNext = _Ror(OpSize::i32Bit, B, _Constant(OpSize::i32Bit, 2));
-      auto DNext = C;
-      auto ENext = D;
-
-      return {ANext, BNext, CNext, DNext, ENext};
-    };
-
-    auto [A1, B1, C1, D1, E1] = Round0();
-    auto [A2, B2, C2, D2, E2] = Round1To3(A1, B1, C1, D1, E1, Src, 2);
-    auto [A3, B3, C3, D3, E3] = Round1To3(A2, B2, C2, D2, E2, Src, 1);
-    auto Final = Round1To3(A3, B3, C3, D3, E3, Src, 0);
-
-    auto Dest3 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 3, Dest, std::get<0>(Final));
-    auto Dest2 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 2, Dest3, std::get<1>(Final));
-    auto Dest1 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 1, Dest2, std::get<2>(Final));
-    Result = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 0, Dest1, std::get<3>(Final));
+  switch (Imm8) {
+  case 0: Result = SHADataShuffle(_VSha1C(Src1, ZeroRegister, Src2)); break;
+  case 2: Result = SHADataShuffle(_VSha1M(Src1, ZeroRegister, Src2)); break;
+  case 1:
+  case 3: Result = SHADataShuffle(_VSha1P(Src1, ZeroRegister, Src2)); break;
   }
 
   StoreResult(FPRClass, Op, Result, OpSize::iInvalid);
 }
 
 void OpDispatchBuilder::SHA256MSG1Op(OpcodeArgs) {
+  if (!CTX->HostFeatures.SupportsSHA) {
+    UnimplementedOp(Op);
+    return;
+  }
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
 
-  Ref Result {};
-
-  if (CTX->HostFeatures.SupportsSHA) {
-    Result = _VSha256U0(Dest, Src);
-  } else {
-    const auto Sigma0 = [this](Ref W) -> Ref {
-      return _Xor(
-        OpSize::i32Bit,
-        _Xor(OpSize::i32Bit, _Ror(OpSize::i32Bit, W, _Constant(OpSize::i32Bit, 7)), _Ror(OpSize::i32Bit, W, _Constant(OpSize::i32Bit, 18))),
-        _Lshr(OpSize::i32Bit, W, _Constant(OpSize::i32Bit, 3)));
-    };
-
-    auto W4 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 0);
-    auto W3 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
-    auto W2 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 2);
-    auto W1 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 1);
-    auto W0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 0);
-
-    auto Sig3 = _Add(OpSize::i32Bit, W3, Sigma0(W4));
-    auto Sig2 = _Add(OpSize::i32Bit, W2, Sigma0(W3));
-    auto Sig1 = _Add(OpSize::i32Bit, W1, Sigma0(W2));
-    auto Sig0 = _Add(OpSize::i32Bit, W0, Sigma0(W1));
-
-    auto D3 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 3, Dest, Sig3);
-    auto D2 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 2, D3, Sig2);
-    auto D1 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 1, D2, Sig1);
-    Result = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 0, D1, Sig0);
-  }
+  auto Result = _VSha256U0(Dest, Src);
 
   StoreResult(FPRClass, Op, Result, OpSize::iInvalid);
 }
 
 void OpDispatchBuilder::SHA256MSG2Op(OpcodeArgs) {
-  const auto Sigma1 = [this](Ref W) -> Ref {
-    return _Xor(
-      OpSize::i32Bit,
-      _Xor(OpSize::i32Bit, _Ror(OpSize::i32Bit, W, _Constant(OpSize::i32Bit, 17)), _Ror(OpSize::i32Bit, W, _Constant(OpSize::i32Bit, 19))),
-      _Lshr(OpSize::i32Bit, W, _Constant(OpSize::i32Bit, 10)));
-  };
-
+  if (!CTX->HostFeatures.SupportsSHA) {
+    UnimplementedOp(Op);
+    return;
+  }
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
 
-  Ref Result;
-  if (CTX->HostFeatures.SupportsSHA) {
-    auto Src1 = _VExtr(OpSize::i128Bit, OpSize::i32Bit, Dest, Dest, 3);
-    auto DupDst = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
-    auto Src2 = _VZip2(OpSize::i128Bit, OpSize::i64Bit, DupDst, Src);
+  auto Src1 = _VExtr(OpSize::i128Bit, OpSize::i32Bit, Dest, Dest, 3);
+  auto DupDst = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
+  auto Src2 = _VZip2(OpSize::i128Bit, OpSize::i64Bit, DupDst, Src);
 
-    Result = _VSha256U1(Src1, Src2);
-  } else {
-    auto W14 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 2);
-    auto W15 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 3);
-    auto W16 = _Add(OpSize::i32Bit, _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 0), Sigma1(W14));
-    auto W17 = _Add(OpSize::i32Bit, _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 1), Sigma1(W15));
-    auto W18 = _Add(OpSize::i32Bit, _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 2), Sigma1(W16));
-    auto W19 = _Add(OpSize::i32Bit, _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 3), Sigma1(W17));
-
-    auto D3 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 3, Dest, W19);
-    auto D2 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 2, D3, W18);
-    auto D1 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 1, D2, W17);
-    Result = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 0, D1, W16);
-  }
+  auto Result = _VSha256U1(Src1, Src2);
 
   StoreResult(FPRClass, Op, Result, OpSize::iInvalid);
 }
 
-Ref OpDispatchBuilder::BitwiseAtLeastTwo(Ref A, Ref B, Ref C) {
-  // Returns whether at least 2/3 of A/B/C is true.
-  // Expressed as (A & (B | C)) | (B & C)
-  //
-  // Equivalent to expression in SHA calculations: (A & B) ^ (A & C) ^ (B & C)
-  auto And = _And(OpSize::i32Bit, B, C);
-  auto Or = _Or(OpSize::i32Bit, B, C);
-  return _Or(OpSize::i32Bit, _And(OpSize::i32Bit, A, Or), And);
-}
-
 void OpDispatchBuilder::SHA256RNDS2Op(OpcodeArgs) {
+  if (!CTX->HostFeatures.SupportsSHA) {
+    UnimplementedOp(Op);
+    return;
+  }
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
   // Hardcoded to XMM0
   auto XMM0 = LoadXMMRegister(0);
 
-  Ref Result;
-  if (CTX->HostFeatures.SupportsSHA) {
-    auto shuffle_abcd = [this](Ref Src1, Ref Src2) -> Ref {
-      // Generates a suitable SHA256 `abcd` configuration from x86 format.
-      auto Tmp = _VZip2(OpSize::i128Bit, OpSize::i64Bit, Src2, Src1);
-      return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);
-    };
+  auto shuffle_abcd = [this](Ref Src1, Ref Src2) -> Ref {
+    // Generates a suitable SHA256 `abcd` configuration from x86 format.
+    auto Tmp = _VZip2(OpSize::i128Bit, OpSize::i64Bit, Src2, Src1);
+    return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);
+  };
 
-    auto shuffle_efgh = [this](Ref Src1, Ref Src2) -> Ref {
-      // Generates a suitable SHA256 `efgh` configuration from x86 format.
-      auto Tmp = _VZip(OpSize::i128Bit, OpSize::i64Bit, Src2, Src1);
-      return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);
-    };
+  auto shuffle_efgh = [this](Ref Src1, Ref Src2) -> Ref {
+    // Generates a suitable SHA256 `efgh` configuration from x86 format.
+    auto Tmp = _VZip(OpSize::i128Bit, OpSize::i64Bit, Src2, Src1);
+    return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);
+  };
 
-    auto ABCD = shuffle_abcd(Dest, Src);
-    auto EFGH = shuffle_efgh(Dest, Src);
+  auto ABCD = shuffle_abcd(Dest, Src);
+  auto EFGH = shuffle_efgh(Dest, Src);
 
-    // x86 uses only the bottom 64-bits of the key, so duplicate to match ARM64 semantics.
-    auto Key = _VDupElement(OpSize::i128Bit, OpSize::i64Bit, XMM0, 0);
+  // x86 uses only the bottom 64-bits of the key, so duplicate to match ARM64 semantics.
+  auto Key = _VDupElement(OpSize::i128Bit, OpSize::i64Bit, XMM0, 0);
 
-    auto A = _VSha256H(ABCD, EFGH, Key);
-    auto B = _VSha256H2(EFGH, ABCD, Key);
-    Result = shuffle_abcd(A, B);
-  } else {
-    const auto Ch = [this](Ref E, Ref F, Ref G) -> Ref {
-      return _Xor(OpSize::i32Bit, _And(OpSize::i32Bit, E, F), _Andn(OpSize::i32Bit, G, E));
-    };
-    const auto Sigma0 = [this](Ref A) -> Ref {
-      return _XorShift(OpSize::i32Bit, _XorShift(OpSize::i32Bit, _Ror(OpSize::i32Bit, A, _Constant(OpSize::i32Bit, 2)), A, ShiftType::ROR, 13),
-                       A, ShiftType::ROR, 22);
-    };
-    const auto Sigma1 = [this](Ref E) -> Ref {
-      return _XorShift(OpSize::i32Bit, _XorShift(OpSize::i32Bit, _Ror(OpSize::i32Bit, E, _Constant(OpSize::i32Bit, 6)), E, ShiftType::ROR, 11),
-                       E, ShiftType::ROR, 25);
-    };
-
-    auto E0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 1);
-    auto F0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 0);
-    auto G0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 1);
-    Ref Q0 = _Add(OpSize::i32Bit, Ch(E0, F0, G0), Sigma1(E0));
-
-    auto WK0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, XMM0, 0);
-    Q0 = _Add(OpSize::i32Bit, Q0, WK0);
-
-    auto H0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 0);
-    Q0 = _Add(OpSize::i32Bit, Q0, H0);
-
-    auto A0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 3);
-    auto B0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 2);
-    auto C0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
-    auto A1 = _Add(OpSize::i32Bit, _Add(OpSize::i32Bit, Q0, BitwiseAtLeastTwo(A0, B0, C0)), Sigma0(A0));
-
-    auto D0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 2);
-    auto E1 = _Add(OpSize::i32Bit, Q0, D0);
-
-    Ref Q1 = _Add(OpSize::i32Bit, Ch(E1, E0, F0), Sigma1(E1));
-
-    auto WK1 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, XMM0, 1);
-    Q1 = _Add(OpSize::i32Bit, Q1, WK1);
-
-    // Rematerialize G0. Costs a move but saves spilling, coming out ahead.
-    G0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 1);
-    Q1 = _Add(OpSize::i32Bit, Q1, G0);
-
-    auto A2 = _Add(OpSize::i32Bit, _Add(OpSize::i32Bit, Q1, BitwiseAtLeastTwo(A1, A0, B0)), Sigma0(A1));
-
-    // Rematerialize C0. As with G0.
-    C0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
-    auto E2 = _Add(OpSize::i32Bit, Q1, C0);
-
-    auto Res3 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 3, Dest, A2);
-    auto Res2 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 2, Res3, A1);
-    auto Res1 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 1, Res2, E2);
-    Result = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 0, Res1, E1);
-  }
+  auto A = _VSha256H(ABCD, EFGH, Key);
+  auto B = _VSha256H2(EFGH, ABCD, Key);
+  auto Result = shuffle_abcd(A, B);
 
   StoreResult(FPRClass, Op, Result, OpSize::iInvalid);
 }
 
 void OpDispatchBuilder::AESImcOp(OpcodeArgs) {
+  if (!CTX->HostFeatures.SupportsAES) {
+    UnimplementedOp(Op);
+    return;
+  }
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
   Ref Result = _VAESImc(Src);
   StoreResult(FPRClass, Op, Result, OpSize::iInvalid);
 }
 
 void OpDispatchBuilder::AESEncOp(OpcodeArgs) {
+  if (!CTX->HostFeatures.SupportsAES) {
+    UnimplementedOp(Op);
+    return;
+  }
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
   Ref Result = _VAESEnc(OpSize::i128Bit, Dest, Src, LoadZeroVector(OpSize::i128Bit));
@@ -395,7 +203,7 @@ void OpDispatchBuilder::AESEncOp(OpcodeArgs) {
 
 void OpDispatchBuilder::VAESEncOp(OpcodeArgs) {
   const auto DstSize = OpSizeFromDst(Op);
-  [[maybe_unused]] const auto Is128Bit = DstSize == OpSize::i128Bit;
+  const auto Is128Bit = DstSize == OpSize::i128Bit;
 
   // TODO: Handle 256-bit VAESENC.
   LOGMAN_THROW_A_FMT(Is128Bit, "256-bit VAESENC unimplemented");
@@ -408,6 +216,10 @@ void OpDispatchBuilder::VAESEncOp(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::AESEncLastOp(OpcodeArgs) {
+  if (!CTX->HostFeatures.SupportsAES) {
+    UnimplementedOp(Op);
+    return;
+  }
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
   Ref Result = _VAESEncLast(OpSize::i128Bit, Dest, Src, LoadZeroVector(OpSize::i128Bit));
@@ -416,7 +228,7 @@ void OpDispatchBuilder::AESEncLastOp(OpcodeArgs) {
 
 void OpDispatchBuilder::VAESEncLastOp(OpcodeArgs) {
   const auto DstSize = OpSizeFromDst(Op);
-  [[maybe_unused]] const auto Is128Bit = DstSize == OpSize::i128Bit;
+  const auto Is128Bit = DstSize == OpSize::i128Bit;
 
   // TODO: Handle 256-bit VAESENCLAST.
   LOGMAN_THROW_A_FMT(Is128Bit, "256-bit VAESENCLAST unimplemented");
@@ -429,6 +241,10 @@ void OpDispatchBuilder::VAESEncLastOp(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::AESDecOp(OpcodeArgs) {
+  if (!CTX->HostFeatures.SupportsAES) {
+    UnimplementedOp(Op);
+    return;
+  }
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
   Ref Result = _VAESDec(OpSize::i128Bit, Dest, Src, LoadZeroVector(OpSize::i128Bit));
@@ -437,7 +253,7 @@ void OpDispatchBuilder::AESDecOp(OpcodeArgs) {
 
 void OpDispatchBuilder::VAESDecOp(OpcodeArgs) {
   const auto DstSize = OpSizeFromDst(Op);
-  [[maybe_unused]] const auto Is128Bit = DstSize == OpSize::i128Bit;
+  const auto Is128Bit = DstSize == OpSize::i128Bit;
 
   // TODO: Handle 256-bit VAESDEC.
   LOGMAN_THROW_A_FMT(Is128Bit, "256-bit VAESDEC unimplemented");
@@ -450,6 +266,10 @@ void OpDispatchBuilder::VAESDecOp(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::AESDecLastOp(OpcodeArgs) {
+  if (!CTX->HostFeatures.SupportsAES) {
+    UnimplementedOp(Op);
+    return;
+  }
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
   Ref Result = _VAESDecLast(OpSize::i128Bit, Dest, Src, LoadZeroVector(OpSize::i128Bit));
@@ -458,7 +278,7 @@ void OpDispatchBuilder::AESDecLastOp(OpcodeArgs) {
 
 void OpDispatchBuilder::VAESDecLastOp(OpcodeArgs) {
   const auto DstSize = OpSizeFromDst(Op);
-  [[maybe_unused]] const auto Is128Bit = DstSize == OpSize::i128Bit;
+  const auto Is128Bit = DstSize == OpSize::i128Bit;
 
   // TODO: Handle 256-bit VAESDECLAST.
   LOGMAN_THROW_A_FMT(Is128Bit, "256-bit VAESDECLAST unimplemented");
@@ -479,11 +299,20 @@ Ref OpDispatchBuilder::AESKeyGenAssistImpl(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::AESKeyGenAssist(OpcodeArgs) {
+  if (!CTX->HostFeatures.SupportsAES) {
+    UnimplementedOp(Op);
+    return;
+  }
+
   Ref Result = AESKeyGenAssistImpl(Op);
   StoreResult(FPRClass, Op, Result, OpSize::iInvalid);
 }
 
 void OpDispatchBuilder::PCLMULQDQOp(OpcodeArgs) {
+  if (!CTX->HostFeatures.SupportsPMULL_128Bit) {
+    UnimplementedOp(Op);
+    return;
+  }
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
   const auto Selector = static_cast<uint8_t>(Op->Src[1].Literal());
@@ -493,6 +322,10 @@ void OpDispatchBuilder::PCLMULQDQOp(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::VPCLMULQDQOp(OpcodeArgs) {
+  if (!CTX->HostFeatures.SupportsPMULL_128Bit) {
+    UnimplementedOp(Op);
+    return;
+  }
   const auto DstSize = OpSizeFromDst(Op);
 
   Ref Src1 = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);