Docs: Update for release FEX-2508

RegisterAllocationPass: defer next-use analysis

.clang-format

@@ -32,7 +32,7 @@ AttributeMacros:
 BinPackArguments: true
 BinPackParameters: true
 BitFieldColonSpacing: Both
-BreakAfterAttributes: Always # clang 16 required
+BreakAfterAttributes: Leave
 BreakBeforeBraces: Attach
 BreakBeforeBinaryOperators: None
 BreakBeforeInlineASMColon: OnlyMultiline # clang 16 required
@@ -60,7 +60,7 @@ IndentRequires: false
 IndentWidth: 2
 InsertBraces: true
 KeepEmptyLinesAtTheStartOfBlocks: true
-LambdaBodyIndentation: OuterScope
+LambdaBodyIndentation: Signature
 LineEnding: LF # clang 16 required
 MaxEmptyLinesToKeep: 2
 NamespaceIndentation: Inner

.clang-format-ignore

@@ -1,8 +1,4 @@
 # This file is used to ignore files and directories from clang-format
-
-# Ignore all files in the External directory
-External/*
-
 Source/Common/cpp-optparse/*
 
 # Files with human-indented tables for readability - don't mess with these

.git-blame-ignore-revs

@@ -16,3 +16,7 @@
 
 # Reformat of CodeEmitter inl files
 8760c593ece92d7e9fa94c40da0368fd367c9cad
+
+# Whole-tree reformat with clang-format-19
+5267cde60e7642852d18f20ae8568643bb5293d5
+

.github/workflows/pr-code-format.yml

@@ -40,11 +40,8 @@ jobs:
           echo "Formatting files:"
           echo "$CHANGED_FILES"
 
-      - name: Check for correct clang-format version
-        run: clang-format --version | grep -qF '16.0.6'
-
-      - name: Check git-clang-format-16 exists
-        run: which git-clang-format-16
+      - name: Check git-clang-format-19 exists
+        run: which git-clang-format-19
 
       - name: Setup Python env
         uses: actions/setup-python@v4
@@ -58,19 +55,16 @@ jobs:
 
       - name: Run code formatter
         env:
-          CLANG_FORMAT_PATH: 'git-clang-format-16'
+          CLANG_FORMAT_PATH: 'git-clang-format-19'
           GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
           START_REV: ${{ github.event.pull_request.base.sha }}
           END_REV: ${{ github.event.pull_request.head.sha }}
           CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
-        # TODO(pmatos): Once we adopt v18, we should be able
-        # to take advantage of the new --diff_from_common_commit option
-        # explicitly in code-format-helper.py and not have to diff starting at
-        # the merge base.
+        # Using --diff_from_common_commit option available in clang-format-19
         run: |
           python ./External/code-format-helper/code-format-helper.py \
             --repo "FEX-emu/FEX" \
             --issue-number $GITHUB_PR_NUMBER \
-            --start-rev $(git merge-base $START_REV $END_REV) \
+            --start-rev $START_REV \
             --end-rev $END_REV \
             --changed-files "$CHANGED_FILES"

.github/workflows/wine_dll_artifacts.yml

@@ -0,0 +1,88 @@
+name: Wine DLL artifacts
+
+on:
+  push:
+    branches:
+      - main
+
+env:
+  BUILD_TYPE: Release
+
+jobs:
+  wine_dll_artifacts:
+    runs-on: ${{ matrix.arch }}
+    strategy:
+      matrix:
+        arch: [[self-hosted, ARM64, mingw]]
+      fail-fast: false
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Add MingGW to PATH
+      run: echo "$HOME/llvm-mingw/build/bin/" >> $GITHUB_PATH
+
+    - name : submodule checkout
+      # Need to update submodules
+      run: |
+        git submodule sync --recursive
+        git submodule update --init --depth 1
+
+    - name: Clean install directory
+      run: |
+        rm -Rf ${{runner.workspace}}/build_install
+        mkdir ${{runner.workspace}}/build_install
+
+    - name: Clean Build Environment
+      run: |
+        rm -Rf ${{runner.workspace}}/build_arm64ec
+        rm -Rf ${{runner.workspace}}/build_wow64
+
+    - name: Create Build Environment arm64ec
+      run: |
+        cmake -E make_directory ${{runner.workspace}}/build_arm64ec
+        cmake -E make_directory ${{runner.workspace}}/build_wow64
+
+    - name: Configure CMake arm64ec
+      shell: bash
+      working-directory: ${{runner.workspace}}/build_arm64ec
+      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/Data/CMake/toolchain_mingw.cmake -DMINGW_TRIPLE=arm64ec-w64-mingw32 -DCMAKE_INSTALL_LIBDIR=/usr/lib/wine/aarch64-windows -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=False -DENABLE_JEMALLOC_GLIBC_ALLOC=False -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_TESTS=False -DCMAKE_INSTALL_PREFIX=/usr
+
+    - name: Configure CMake wow64
+      shell: bash
+      working-directory: ${{runner.workspace}}/build_wow64
+      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/Data/CMake/toolchain_mingw.cmake -DMINGW_TRIPLE=aarch64-w64-mingw32 -DCMAKE_INSTALL_LIBDIR=/usr/lib/wine/aarch64-windows -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=False -DENABLE_JEMALLOC_GLIBC_ALLOC=False -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_TESTS=False -DCMAKE_INSTALL_PREFIX=/usr
+
+    - name: Build arm64ec
+      working-directory: ${{runner.workspace}}/build_arm64ec
+      shell: bash
+      run: cmake --build . --config $BUILD_TYPE
+
+    - name: install arm64ec
+      working-directory: ${{runner.workspace}}/build_arm64ec
+      shell: bash
+      env:
+        DESTDIR: ${{runner.workspace}}/build_install
+      run: cmake --build . --config $BUILD_TYPE -t install
+
+    - name: Build wow64
+      working-directory: ${{runner.workspace}}/build_wow64
+      shell: bash
+      run: cmake --build . --config $BUILD_TYPE
+
+    - name: install wow64
+      working-directory: ${{runner.workspace}}/build_wow64
+      shell: bash
+      env:
+        DESTDIR: ${{runner.workspace}}/build_install
+      run: cmake --build . --config $BUILD_TYPE -t install
+
+    - name: Upload libraries
+      uses: 'actions/upload-artifact@v4'
+      timeout-minutes: 1
+      with:
+        overwrite: true
+        name: wine_dll_artifacts
+        path: ${{runner.workspace}}/build_install/usr/lib/wine/aarch64-windows/lib*.dll
+        retention-days: 60
+        compression-level: 9

CMakeLists.txt

@@ -39,10 +39,16 @@ set (X86_32_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86
 set (X86_64_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_64.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
 set (X86_DEV_ROOTFS "/" CACHE FILEPATH "Path to the sysroot used for cross-compiling for i686 and x86_64")
 set (DATA_DIRECTORY "" CACHE PATH "Global data directory (override)")
+set (HOSTLIBS_DATA_DIRECTORY "" CACHE PATH "Global data directory (override)")
 if (NOT DATA_DIRECTORY)
   set (DATA_DIRECTORY "${CMAKE_INSTALL_PREFIX}/share/fex-emu")
 endif()
 
+include(GNUInstallDirs)
+if (NOT HOSTLIBS_DATA_DIRECTORY)
+  set(HOSTLIBS_DATA_DIRECTORY "${CMAKE_INSTALL_FULL_LIBDIR}/fex-emu")
+endif()
+
 string(FIND ${CMAKE_BASE_NAME} mingw CONTAINS_MINGW)
 if (NOT CONTAINS_MINGW EQUAL -1)
   message (STATUS "Mingw build")
@@ -406,6 +412,13 @@ if (TUNE_CPU STREQUAL "native")
 
       string(STRIP ${AARCH64_CPU} AARCH64_CPU)
 
+      execute_process(COMMAND python3 "${PROJECT_SOURCE_DIR}/Scripts/NeedDisabledSVE.py"
+        RESULT_VARIABLE NEEDS_SVE_DISABLED)
+      if (NEEDS_SVE_DISABLED)
+        message(STATUS "Platform has bugged SVE. Disabling")
+        set(AARCH64_CPU "cortex-a78")
+      endif()
+
       check_cxx_compiler_flag("-mcpu=${AARCH64_CPU}" COMPILER_SUPPORTS_CPU_TYPE)
       if(COMPILER_SUPPORTS_CPU_TYPE)
         list(APPEND FEX_TUNE_COMPILE_FLAGS "-mcpu=${AARCH64_CPU}")

CodeEmitter/CodeEmitter/Emitter.h

@@ -86,6 +86,14 @@ constexpr size_t SubRegSizeInBits(SubRegSize size) {
   return size_t {8} << FEXCore::ToUnderlying(size);
 }
 
+// Many floating point operations constrain their element sizes to the
+// main three float sizes half, single, and double precision. This just
+// combines all the checks together for brevity.
+[[nodiscard]]
+constexpr bool IsStandardFloatSize(SubRegSize size) {
+  return size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit;
+}
+
 /* This `ScalarRegSize` enum is used for most scalar float
  * operations.
  *

CodeEmitter/CodeEmitter/SVEOps.inl

@@ -60,8 +60,7 @@ public:
   }
 
   void fcmla(SubRegSize size, ZRegister zda, PRegisterMerge pv, ZRegister zn, ZRegister zm, Rotation rot) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
     LOGMAN_THROW_A_FMT(pv <= PReg::p7.Merging(), "fcmla can only use p0 to p7");
 
     uint32_t Op = 0b0110'0100'0000'0000'0000'0000'0000'0000;
@@ -76,8 +75,7 @@ public:
   }
 
   void fcadd(SubRegSize size, ZRegister zd, PRegisterMerge pv, ZRegister zn, ZRegister zm, Rotation rot) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
     LOGMAN_THROW_A_FMT(pv <= PReg::p7.Merging(), "fcadd can only use p0 to p7");
     LOGMAN_THROW_A_FMT(rot == Rotation::ROTATE_90 || rot == Rotation::ROTATE_270, "fcadd rotation may only be 90 or 270 degrees");
     LOGMAN_THROW_A_FMT(zd == zn, "fcadd zd and zn must be the same register");
@@ -815,16 +813,12 @@ public:
   // SVE Integer Misc - Unpredicated
   // SVE floating-point trig select coefficient
   void ftssel(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "ftssel may only have "
-                                                                                                               "16-bit, 32-bit, or 64-bit "
-                                                                                                               "element sizes");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "ftssel may only use 16/32/64-bit element sizes");
     SVEIntegerMiscUnpredicated(0b00, zm.Idx(), FEXCore::ToUnderlying(size), zd, zn);
   }
   // SVE floating-point exponential accelerator
   void fexpa(SubRegSize size, ZRegister zd, ZRegister zn) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "fexpa may only have "
-                                                                                                               "16-bit, 32-bit, or 64-bit "
-                                                                                                               "element sizes");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "fexpa may only use 16/32/64-bit element sizes");
     SVEIntegerMiscUnpredicated(0b10, 0b00000, FEXCore::ToUnderlying(size), zd, zn);
   }
   // SVE constructive prefix (unpredicated)
@@ -1503,9 +1497,9 @@ public:
   }
 
   // SVE broadcast floating-point immediate (unpredicated)
-  void fdup(ARMEmitter::SubRegSize size, ARMEmitter::ZRegister zd, float Value) {
-    LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i16Bit || size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit,
-                       "Unsupported fmov size");
+  void fdup(SubRegSize size, ZRegister zd, float Value) {
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Unsupported fmov size");
+
     uint32_t Imm {};
     if (size == SubRegSize::i16Bit) {
       LOGMAN_MSG_A_FMT("Unsupported");
@@ -1518,7 +1512,7 @@ public:
 
     SVEBroadcastFloatImmUnpredicated(0b00, 0, Imm, size, zd);
   }
-  void fmov(ARMEmitter::SubRegSize size, ARMEmitter::ZRegister zd, float Value) {
+  void fmov(SubRegSize size, ZRegister zd, float Value) {
     fdup(size, zd, Value);
   }
 
@@ -3401,8 +3395,8 @@ private:
   }
 
   void SVEBroadcastFloatImmPredicated(SubRegSize size, ZRegister zd, PRegister pg, float value) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Unsupported fcpy/fmov "
-                                                                                                               "size");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Unsupported fcpy/fmov size");
+
     uint32_t imm {};
     if (size == SubRegSize::i16Bit) {
       LOGMAN_MSG_A_FMT("Unsupported");
@@ -3578,7 +3572,7 @@ private:
   // SVE2 floating-point pairwise operations
   void SVEFloatPairwiseArithmetic(uint32_t opc, SubRegSize size, PRegister pg, ZRegister zd, ZRegister zn, ZRegister zm) {
     LOGMAN_THROW_A_FMT(zd == zn, "zd needs to equal zn");
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Invalid float size");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Invalid float size");
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
 
     uint32_t Instr = 0b0110'0100'0001'0000'1000'0000'0000'0000;
@@ -3592,7 +3586,7 @@ private:
 
   // SVE floating-point arithmetic (unpredicated)
   void SVEFloatArithmeticUnpredicated(uint32_t opc, SubRegSize size, ZRegister zm, ZRegister zn, ZRegister zd) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Invalid float size");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Invalid float size");
 
     uint32_t Instr = 0b0110'0101'0000'0000'0000'0000'0000'0000;
     Instr |= FEXCore::ToUnderlying(size) << 22;
@@ -3700,7 +3694,7 @@ private:
   // SVE floating-point arithmetic (predicated)
   void SVEFloatArithmeticPredicated(uint32_t opc, SubRegSize size, PRegister pg, ZRegister zd, ZRegister zn, ZRegister zm) {
     LOGMAN_THROW_A_FMT(zd == zn, "zn needs to equal zd");
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Invalid float size");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Invalid float size");
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
 
     uint32_t Instr = 0b0110'0101'0000'0000'1000'0000'0000'0000;
@@ -3728,9 +3722,7 @@ private:
   }
 
   void SVEFPRecursiveReduction(uint32_t opc, SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "FP reduction operation can "
-                                                                                                               "only use 16-bit, 32-bit, "
-                                                                                                               "or 64-bit element sizes");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "FP reduction operation can only use 16/32/64-bit element sizes");
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "FP reduction operation can only use p0-p7 as a governing predicate");
 
     uint32_t Instr = 0b0110'0101'0000'0000'0010'0000'0000'0000;
@@ -4112,7 +4104,7 @@ private:
     // 0b111 - I - Current
 
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Unsupported size in {}", __func__);
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Unsupported size in {}", __func__);
 
     uint32_t Instr = 0b0110'0101'0000'0000'1010'0000'0000'0000;
     Instr |= FEXCore::ToUnderlying(size) << 22;
@@ -4721,7 +4713,7 @@ private:
 
   void SVEFloatUnary(uint32_t opc, SubRegSize size, PRegister pg, ZRegister zn, ZRegister zd) {
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Unsupported size in {}", __func__);
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Unsupported size in {}", __func__);
 
     uint32_t Instr = 0b0110'0101'0000'1100'1010'0000'0000'0000;
     Instr |= FEXCore::ToUnderlying(size) << 22;
@@ -4809,8 +4801,7 @@ private:
   }
 
   void SVEFPUnaryOpsUnpredicated(uint32_t opc, SubRegSize size, ZRegister zd, ZRegister zn) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
 
     uint32_t Instr = 0b0110'0101'0000'1000'0011'0000'0000'0000;
     Instr |= FEXCore::ToUnderlying(size) << 22;
@@ -4821,8 +4812,7 @@ private:
   }
 
   void SVEFPSerialReductionPredicated(uint32_t opc, SubRegSize size, VRegister vd, PRegister pg, VRegister vn, ZRegister zm) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
     LOGMAN_THROW_A_FMT(vd == vn, "vn must be the same as vd");
 
@@ -4836,8 +4826,7 @@ private:
   }
 
   void SVEFPCompareWithZero(uint32_t eqlt, uint32_t ne, SubRegSize size, PRegister pd, PRegister pg, ZRegister zn) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
 
     uint32_t Instr = 0b0110'0101'0001'0000'0010'0000'0000'0000;
@@ -4852,8 +4841,7 @@ private:
 
   void SVEFPMultiplyAdd(uint32_t opc, SubRegSize size, ZRegister zd, PRegister pg, ZRegister zn, ZRegister zm) {
     // NOTE: opc also includes the op0 bit (bit 15) like op0:opc, since the fields are adjacent
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
     LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
 
     uint32_t Instr = 0b0110'0101'0010'0000'0000'0000'0000'0000;
@@ -4867,8 +4855,7 @@ private:
   }
 
   void SVEFPMultiplyAddIndexed(uint32_t op, SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {
-    LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
-                                                                                                               "32-bit, or 64-bit");
+    LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
     LOGMAN_THROW_A_FMT((size <= SubRegSize::i32Bit && zm <= ZReg::z7) || (size == SubRegSize::i64Bit && zm <= ZReg::z15),
                        "16-bit and 32-bit indexed variants may only use Zm between z0-z7\n"
                        "64-bit variants may only use Zm between z0-z15");

CodeEmitter/CodeEmitter/ScalarOps.inl

@@ -27,8 +27,6 @@ struct EmitterOps : Emitter {
 public:
   // Advanced SIMD scalar copy
   void dup(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Index) {
-    constexpr uint32_t Op = 0b0101'1110'0000'0000'0000'01 << 10;
-
     const uint32_t SizeImm = FEXCore::ToUnderlying(size);
     const uint32_t IndexShift = SizeImm + 1;
     const uint32_t ElementSize = 1U << SizeImm;
@@ -38,10 +36,10 @@ public:
 
     const uint32_t imm5 = (Index << IndexShift) | ElementSize;
 
-    ASIMDScalarCopy(Op, 1, imm5, 0b0000, rd, rn);
+    ASIMDScalarCopy(1, 1, imm5, 0b0000, rd, rn);
   }
 
-  void mov(ARMEmitter::ScalarRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, uint32_t Index) {
+  void mov(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Index) {
     dup(size, rd, rn, Index);
   }
 
@@ -1282,10 +1280,10 @@ public:
 
 private:
   // Advanced SIMD scalar copy
-  void ASIMDScalarCopy(uint32_t Op, uint32_t Q, uint32_t imm5, uint32_t imm4, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn) {
-    uint32_t Instr = Op;
-
+  void ASIMDScalarCopy(uint32_t Q, uint32_t b28, uint32_t imm5, uint32_t imm4, VRegister rd, VRegister rn) {
+    uint32_t Instr = 0b0000'1110'0000'0000'0000'01U << 10;
     Instr |= Q << 30;
+    Instr |= b28 << 28;
     Instr |= imm5 << 16;
     Instr |= imm4 << 11;
     Instr |= Encode_rn(rn);

CodeEmitter/CodeEmitter/VixlUtils.inl

@@ -224,9 +224,13 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
   //    11110s     2    UInt(s)
   //
   // So we 'or' (2 * -d) with our computed s to form imms.
-  if ((n != NULL) || (imm_s != NULL) || (imm_r != NULL)) {
+  if (n != nullptr) {
     *n = out_n;
+  }
+  if (imm_s != nullptr) {
     *imm_s = ((2 * -d) | (s - 1)) & 0x3f;
+  }
+  if (imm_r != nullptr) {
     *imm_r = r;
   }
 

Data/CMake/toolchain_mingw.cmake

@@ -4,6 +4,7 @@ set(CMAKE_RC_COMPILER ${MINGW_TRIPLE}-windres)
 set(CMAKE_C_COMPILER ${MINGW_TRIPLE}-clang)
 set(CMAKE_CXX_COMPILER ${MINGW_TRIPLE}-clang++)
 set(CMAKE_DLLTOOL ${MINGW_TRIPLE}-dlltool)
+set(CMAKE_AR ${MINGW_TRIPLE}-ar)
 
 # Compile everything as static to avoid requiring the MinGW runtime libraries, force page aligned sections so that
 # debug symbols work correctly, and disable loop alignment to workaround an LLVM bug

External/.clang-format

@@ -0,0 +1 @@
+DisableFormat: true

External/code-format-helper/code-format-helper.py

@@ -169,14 +169,9 @@ View the diff from {self.name} here.
 
 
 class ClangFormatHelper(FormatHelper):
-    name = "clang-format"
+    name = "git-clang-format"
     friendly_name = "C/C++ code formatter"
 
-    @property
-    def cformat_wrapper_path(self) -> str:
-        relpath = "../../Scripts/clang-format.py"
-        curpath = os.path.dirname(os.path.abspath(__file__))
-        return os.path.abspath(os.path.normpath(os.path.join(curpath, relpath)))
 
     @property
     def instructions(self) -> str:
@@ -199,7 +194,7 @@ class ClangFormatHelper(FormatHelper):
     def clang_fmt_path(self) -> str:
         if "CLANG_FORMAT_PATH" in os.environ:
             return os.environ["CLANG_FORMAT_PATH"]
-        return "git-clang-format"
+        return "git-clang-format-19"
 
     def has_tool(self) -> bool:
         cmd = [self.clang_fmt_path, "-h"]
@@ -217,8 +212,9 @@ class ClangFormatHelper(FormatHelper):
 
         cf_cmd = [
             self.clang_fmt_path,
-            f"--binary={self.cformat_wrapper_path}",
+            "--binary=clang-format-19",
             "--diff",
+            "--diff_from_common_commit",
         ]
 
         if args.start_rev and args.end_rev:

External/code-format-helper/requirements_formatting.txt

@@ -1,52 +1,392 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.13
 # by the following command:
 #
-#    pip-compile --output-file=llvm/utils/git/requirements_formatting.txt llvm/utils/git/requirements_formatting.txt.in
+#    pip-compile --generate-hashes --output-file=requirements_formatting.txt --strip-extras requirements_formatting.txt.in
 #
-black==23.9.1
+black==25.1.0 \
+    --hash=sha256:030b9759066a4ee5e5aca28c3c77f9c64789cdd4de8ac1df642c40b708be6171 \
+    --hash=sha256:055e59b198df7ac0b7efca5ad7ff2516bca343276c466be72eb04a3bcc1f82d7 \
+    --hash=sha256:0e519ecf93120f34243e6b0054db49c00a35f84f195d5bce7e9f5cfc578fc2da \
+    --hash=sha256:172b1dbff09f86ce6f4eb8edf9dede08b1fce58ba194c87d7a4f1a5aa2f5b3c2 \
+    --hash=sha256:1e2978f6df243b155ef5fa7e558a43037c3079093ed5d10fd84c43900f2d8ecc \
+    --hash=sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666 \
+    --hash=sha256:3b48735872ec535027d979e8dcb20bf4f70b5ac75a8ea99f127c106a7d7aba9f \
+    --hash=sha256:4b60580e829091e6f9238c848ea6750efed72140b91b048770b64e74fe04908b \
+    --hash=sha256:759e7ec1e050a15f89b770cefbf91ebee8917aac5c20483bc2d80a6c3a04df32 \
+    --hash=sha256:8f0b18a02996a836cc9c9c78e5babec10930862827b1b724ddfe98ccf2f2fe4f \
+    --hash=sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717 \
+    --hash=sha256:96c1c7cd856bba8e20094e36e0f948718dc688dba4a9d78c3adde52b9e6c2299 \
+    --hash=sha256:a1ee0a0c330f7b5130ce0caed9936a904793576ef4d2b98c40835d6a65afa6a0 \
+    --hash=sha256:a22f402b410566e2d1c950708c77ebf5ebd5d0d88a6a2e87c86d9fb48afa0d18 \
+    --hash=sha256:a39337598244de4bae26475f77dda852ea00a93bd4c728e09eacd827ec929df0 \
+    --hash=sha256:afebb7098bfbc70037a053b91ae8437c3857482d3a690fefc03e9ff7aa9a5fd3 \
+    --hash=sha256:bacabb307dca5ebaf9c118d2d2f6903da0d62c9faa82bd21a33eecc319559355 \
+    --hash=sha256:bce2e264d59c91e52d8000d507eb20a9aca4a778731a08cfff7e5ac4a4bb7096 \
+    --hash=sha256:d9e6827d563a2c820772b32ce8a42828dc6790f095f441beef18f96aa6f8294e \
+    --hash=sha256:db8ea9917d6f8fc62abd90d944920d95e73c83a5ee3383493e35d271aca872e9 \
+    --hash=sha256:ea0213189960bda9cf99be5b8c8ce66bb054af5e9e861249cd23471bd7b0b3ba \
+    --hash=sha256:f3df5f1bf91d36002b0a75389ca8663510cf0531cca8aa5c1ef695b46d98655f
     # via
-    #   -r llvm/utils/git/requirements_formatting.txt.in
+    #   -r requirements_formatting.txt.in
     #   darker
-certifi==2023.7.22
-    # via requests
-cffi==1.15.1
+certifi==2025.7.14 \
+    --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \
+    --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995
+    # via
+    #   -r requirements_formatting.txt.in
+    #   requests
+cffi==1.15.1 \
+    --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \
+    --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \
+    --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \
+    --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \
+    --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \
+    --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \
+    --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \
+    --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \
+    --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \
+    --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \
+    --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \
+    --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \
+    --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \
+    --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \
+    --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \
+    --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \
+    --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \
+    --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \
+    --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \
+    --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \
+    --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \
+    --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \
+    --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \
+    --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \
+    --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \
+    --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \
+    --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \
+    --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \
+    --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \
+    --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \
+    --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \
+    --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \
+    --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \
+    --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \
+    --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \
+    --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \
+    --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \
+    --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \
+    --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \
+    --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \
+    --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \
+    --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \
+    --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \
+    --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \
+    --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \
+    --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \
+    --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \
+    --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \
+    --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \
+    --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \
+    --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \
+    --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \
+    --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \
+    --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \
+    --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \
+    --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \
+    --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \
+    --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \
+    --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \
+    --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \
+    --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \
+    --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \
+    --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \
+    --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0
     # via
     #   cryptography
     #   pynacl
-charset-normalizer==3.2.0
+charset-normalizer==3.2.0 \
+    --hash=sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96 \
+    --hash=sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c \
+    --hash=sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710 \
+    --hash=sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706 \
+    --hash=sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020 \
+    --hash=sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252 \
+    --hash=sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad \
+    --hash=sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329 \
+    --hash=sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a \
+    --hash=sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f \
+    --hash=sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6 \
+    --hash=sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4 \
+    --hash=sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a \
+    --hash=sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46 \
+    --hash=sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2 \
+    --hash=sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23 \
+    --hash=sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace \
+    --hash=sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd \
+    --hash=sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982 \
+    --hash=sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10 \
+    --hash=sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2 \
+    --hash=sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea \
+    --hash=sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09 \
+    --hash=sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5 \
+    --hash=sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149 \
+    --hash=sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489 \
+    --hash=sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9 \
+    --hash=sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80 \
+    --hash=sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592 \
+    --hash=sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3 \
+    --hash=sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6 \
+    --hash=sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed \
+    --hash=sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c \
+    --hash=sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200 \
+    --hash=sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a \
+    --hash=sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e \
+    --hash=sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d \
+    --hash=sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6 \
+    --hash=sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623 \
+    --hash=sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669 \
+    --hash=sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3 \
+    --hash=sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa \
+    --hash=sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9 \
+    --hash=sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2 \
+    --hash=sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f \
+    --hash=sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1 \
+    --hash=sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4 \
+    --hash=sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a \
+    --hash=sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8 \
+    --hash=sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3 \
+    --hash=sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029 \
+    --hash=sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f \
+    --hash=sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959 \
+    --hash=sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22 \
+    --hash=sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7 \
+    --hash=sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952 \
+    --hash=sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346 \
+    --hash=sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e \
+    --hash=sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d \
+    --hash=sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299 \
+    --hash=sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd \
+    --hash=sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a \
+    --hash=sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3 \
+    --hash=sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037 \
+    --hash=sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94 \
+    --hash=sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c \
+    --hash=sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858 \
+    --hash=sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a \
+    --hash=sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449 \
+    --hash=sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c \
+    --hash=sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918 \
+    --hash=sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1 \
+    --hash=sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c \
+    --hash=sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac \
+    --hash=sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa
     # via requests
-click==8.1.7
+click==8.1.7 \
+    --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \
+    --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de
     # via black
-cryptography==41.0.3
-    # via pyjwt
-darker==1.7.2
-    # via -r llvm/utils/git/requirements_formatting.txt.in
-deprecated==1.2.14
+cryptography==45.0.5 \
+    --hash=sha256:0027d566d65a38497bc37e0dd7c2f8ceda73597d2ac9ba93810204f56f52ebc7 \
+    --hash=sha256:101ee65078f6dd3e5a028d4f19c07ffa4dd22cce6a20eaa160f8b5219911e7d8 \
+    --hash=sha256:12e55281d993a793b0e883066f590c1ae1e802e3acb67f8b442e721e475e6463 \
+    --hash=sha256:14d96584701a887763384f3c47f0ca7c1cce322aa1c31172680eb596b890ec30 \
+    --hash=sha256:1e1da5accc0c750056c556a93c3e9cb828970206c68867712ca5805e46dc806f \
+    --hash=sha256:206210d03c1193f4e1ff681d22885181d47efa1ab3018766a7b32a7b3d6e6afd \
+    --hash=sha256:2089cc8f70a6e454601525e5bf2779e665d7865af002a5dec8d14e561002e135 \
+    --hash=sha256:3a264aae5f7fbb089dbc01e0242d3b67dffe3e6292e1f5182122bdf58e65215d \
+    --hash=sha256:3af26738f2db354aafe492fb3869e955b12b2ef2e16908c8b9cb928128d42c57 \
+    --hash=sha256:3fcfbefc4a7f332dece7272a88e410f611e79458fab97b5efe14e54fe476f4fd \
+    --hash=sha256:460f8c39ba66af7db0545a8c6f2eabcbc5a5528fc1cf6c3fa9a1e44cec33385e \
+    --hash=sha256:57c816dfbd1659a367831baca4b775b2a5b43c003daf52e9d57e1d30bc2e1b0e \
+    --hash=sha256:5aa1e32983d4443e310f726ee4b071ab7569f58eedfdd65e9675484a4eb67bd1 \
+    --hash=sha256:6ff8728d8d890b3dda5765276d1bc6fb099252915a2cd3aff960c4c195745dd0 \
+    --hash=sha256:7259038202a47fdecee7e62e0fd0b0738b6daa335354396c6ddebdbe1206af2a \
+    --hash=sha256:72e76caa004ab63accdf26023fccd1d087f6d90ec6048ff33ad0445abf7f605a \
+    --hash=sha256:7760c1c2e1a7084153a0f68fab76e754083b126a47d0117c9ed15e69e2103492 \
+    --hash=sha256:8c4a6ff8a30e9e3d38ac0539e9a9e02540ab3f827a3394f8852432f6b0ea152e \
+    --hash=sha256:9024beb59aca9d31d36fcdc1604dd9bbeed0a55bface9f1908df19178e2f116e \
+    --hash=sha256:90cb0a7bb35959f37e23303b7eed0a32280510030daba3f7fdfbb65defde6a97 \
+    --hash=sha256:91098f02ca81579c85f66df8a588c78f331ca19089763d733e34ad359f474174 \
+    --hash=sha256:926c3ea71a6043921050eaa639137e13dbe7b4ab25800932a8498364fc1abec9 \
+    --hash=sha256:982518cd64c54fcada9d7e5cf28eabd3ee76bd03ab18e08a48cad7e8b6f31b18 \
+    --hash=sha256:9b4cf6318915dccfe218e69bbec417fdd7c7185aa7aab139a2c0beb7468c89f0 \
+    --hash=sha256:ad0caded895a00261a5b4aa9af828baede54638754b51955a0ac75576b831b27 \
+    --hash=sha256:b85980d1e345fe769cfc57c57db2b59cff5464ee0c045d52c0df087e926fbe63 \
+    --hash=sha256:b8fa8b0a35a9982a3c60ec79905ba5bb090fc0b9addcfd3dc2dd04267e45f25e \
+    --hash=sha256:b9e38e0a83cd51e07f5a48ff9691cae95a79bea28fe4ded168a8e5c6c77e819d \
+    --hash=sha256:bd4c45986472694e5121084c6ebbd112aa919a25e783b87eb95953c9573906d6 \
+    --hash=sha256:be97d3a19c16a9be00edf79dca949c8fa7eff621763666a145f9f9535a5d7f42 \
+    --hash=sha256:c648025b6840fe62e57107e0a25f604db740e728bd67da4f6f060f03017d5097 \
+    --hash=sha256:d05a38884db2ba215218745f0781775806bde4f32e07b135348355fe8e4991d9 \
+    --hash=sha256:dd420e577921c8c2d31289536c386aaa30140b473835e97f83bc71ea9d2baf2d \
+    --hash=sha256:e357286c1b76403dd384d938f93c46b2b058ed4dfcdce64a770f0537ed3feb6f \
+    --hash=sha256:e6c00130ed423201c5bc5544c23359141660b07999ad82e34e7bb8f882bb78e0 \
+    --hash=sha256:e74d30ec9c7cb2f404af331d5b4099a9b322a8a6b25c4632755c8757345baac5 \
+    --hash=sha256:f3562c2f23c612f2e4a6964a61d942f891d29ee320edb62ff48ffb99f3de9ae8
+    # via
+    #   -r requirements_formatting.txt.in
+    #   pyjwt
+darker==2.1.1 \
+    --hash=sha256:a6e6a682c0604e76fe9aec7650e96a944f517563c69b28fcc076db9d957d98ea \
+    --hash=sha256:ead701414c45359fc0312bc285614d3285fc135476d43f3bc08d989ee19d9020
+    # via -r requirements_formatting.txt.in
+darkgraylib==1.2.1 \
+    --hash=sha256:60c59de69842367ce0c78c32c451fa8e9d29500e681312d9864a7416bcdb7792 \
+    --hash=sha256:a5dd6a2015a470d9047278cdd01a91ccb1d746675f8fd4562b3b5f6b8cbda930
+    # via
+    #   darker
+    #   graylint
+deprecated==1.2.14 \
+    --hash=sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c \
+    --hash=sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3
     # via pygithub
-idna==3.4
-    # via requests
-mypy-extensions==1.0.0
-    # via black
-packaging==23.1
-    # via black
-pathspec==0.11.2
-    # via black
-platformdirs==3.10.0
-    # via black
-pycparser==2.21
-    # via cffi
-pygithub==1.59.1
-    # via -r llvm/utils/git/requirements_formatting.txt.in
-pyjwt[crypto]==2.8.0
-    # via pygithub
-pynacl==1.5.0
-    # via pygithub
-requests==2.31.0
-    # via pygithub
-toml==0.10.2
+graylint==1.1.1 \
+    --hash=sha256:0fd8e02972ca03d0ef2bf0adea76b5343efcd492d7afb5f658f3e3a724f55a36 \
+    --hash=sha256:b7e0eab6c159684dbf5ef84e942c3340f6a6549b02a3d11b1a1763cc4f8f0593
     # via darker
-urllib3==2.0.4
-    # via requests
-wrapt==1.15.0
+idna==3.10 \
+    --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
+    --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
+    # via
+    #   -r requirements_formatting.txt.in
+    #   requests
+mypy-extensions==1.0.0 \
+    --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \
+    --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782
+    # via black
+packaging==23.1 \
+    --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
+    --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
+    # via black
+pathspec==0.11.2 \
+    --hash=sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20 \
+    --hash=sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3
+    # via black
+platformdirs==3.10.0 \
+    --hash=sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d \
+    --hash=sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d
+    # via black
+pycparser==2.21 \
+    --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \
+    --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206
+    # via cffi
+pygithub==2.6.1 \
+    --hash=sha256:6f2fa6d076ccae475f9fc392cc6cdbd54db985d4f69b8833a28397de75ed6ca3 \
+    --hash=sha256:b5c035392991cca63959e9453286b41b54d83bf2de2daa7d7ff7e4312cebf3bf
+    # via -r requirements_formatting.txt.in
+pyjwt==2.8.0 \
+    --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \
+    --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320
+    # via pygithub
+pynacl==1.5.0 \
+    --hash=sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858 \
+    --hash=sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d \
+    --hash=sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93 \
+    --hash=sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1 \
+    --hash=sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92 \
+    --hash=sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff \
+    --hash=sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba \
+    --hash=sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394 \
+    --hash=sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b \
+    --hash=sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543
+    # via pygithub
+requests==2.32.4 \
+    --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \
+    --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422
+    # via
+    #   -r requirements_formatting.txt.in
+    #   pygithub
+toml==0.10.2 \
+    --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
+    --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
+    # via
+    #   darker
+    #   darkgraylib
+typing-extensions==4.14.1 \
+    --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \
+    --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76
+    # via pygithub
+urllib3==2.5.0 \
+    --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \
+    --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc
+    # via
+    #   -r requirements_formatting.txt.in
+    #   pygithub
+    #   requests
+wrapt==1.15.0 \
+    --hash=sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0 \
+    --hash=sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420 \
+    --hash=sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a \
+    --hash=sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c \
+    --hash=sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079 \
+    --hash=sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923 \
+    --hash=sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f \
+    --hash=sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1 \
+    --hash=sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8 \
+    --hash=sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86 \
+    --hash=sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0 \
+    --hash=sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364 \
+    --hash=sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e \
+    --hash=sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c \
+    --hash=sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e \
+    --hash=sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c \
+    --hash=sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727 \
+    --hash=sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff \
+    --hash=sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e \
+    --hash=sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29 \
+    --hash=sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7 \
+    --hash=sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72 \
+    --hash=sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475 \
+    --hash=sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a \
+    --hash=sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317 \
+    --hash=sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2 \
+    --hash=sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd \
+    --hash=sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640 \
+    --hash=sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98 \
+    --hash=sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248 \
+    --hash=sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e \
+    --hash=sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d \
+    --hash=sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec \
+    --hash=sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1 \
+    --hash=sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e \
+    --hash=sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9 \
+    --hash=sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92 \
+    --hash=sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb \
+    --hash=sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094 \
+    --hash=sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46 \
+    --hash=sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29 \
+    --hash=sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd \
+    --hash=sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705 \
+    --hash=sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8 \
+    --hash=sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975 \
+    --hash=sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb \
+    --hash=sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e \
+    --hash=sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b \
+    --hash=sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418 \
+    --hash=sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019 \
+    --hash=sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1 \
+    --hash=sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba \
+    --hash=sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6 \
+    --hash=sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2 \
+    --hash=sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3 \
+    --hash=sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7 \
+    --hash=sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752 \
+    --hash=sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416 \
+    --hash=sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f \
+    --hash=sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1 \
+    --hash=sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc \
+    --hash=sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145 \
+    --hash=sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee \
+    --hash=sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a \
+    --hash=sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7 \
+    --hash=sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b \
+    --hash=sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653 \
+    --hash=sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0 \
+    --hash=sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90 \
+    --hash=sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29 \
+    --hash=sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6 \
+    --hash=sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034 \
+    --hash=sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09 \
+    --hash=sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559 \
+    --hash=sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639
     # via deprecated

External/code-format-helper/requirements_formatting.txt.in

@@ -0,0 +1,8 @@
+black~=25.1
+darker==2.1.1
+PyGithub==2.6.1
+cryptography>=43.0.1
+urllib3>=2.5.0
+requests>=2.32.4
+idna>=3.7
+certifi>=2024.7.4

FEXCore/Source/CMakeLists.txt

@@ -1,4 +1,3 @@
-include(GNUInstallDirs)
 set (MAN_DIR share/man CACHE PATH "MAN_DIR")
 
 set (FEXCORE_BASE_SRCS

FEXCore/Source/Common/SoftFloat.h

@@ -157,7 +157,30 @@ struct FEX_PACKED X80SoftFloat {
     return Result;
 #else
     /*
-     * FPREM is not an IEEE-754 remainder.  From the spec:
+     * Check for invalid operation cases first - Intel FPREM sets Invalid Operation
+     * for several cases including infinity dividend and zero divisor.
+     */
+    X80SoftFloat result = 0;
+    if (HandleInfinityOp(state, lhs, result)) {
+      return result;
+    } else if (lhs.Exponent == 0x7FFF && (lhs.Significand & 0x7FFFFFFFFFFFFFFFULL)) { // NaN
+      // propagate NaN
+      state->exceptionFlags |= softfloat_flag_invalid;
+      return lhs;
+    }
+
+    // Check for zero divisor - fprem(x, 0) is invalid operation
+    if (rhs.Exponent == 0 && rhs.Significand == 0) {
+      state->exceptionFlags |= softfloat_flag_invalid;
+      // Return QNaN
+      result.Sign = 0;
+      result.Exponent = 0x7FFF;
+      result.Significand = 0xC000000000000000ULL;
+      return result;
+    }
+
+    /*
+     * FPREM is not an IEEE-754 remainder.  From the Intel spec:
      *
      *    Computes the remainder obtained from dividing the value in the ST(0)
      *    register (the dividend) by the value in the ST(1) register (the divisor
@@ -390,6 +413,11 @@ struct FEX_PACKED X80SoftFloat {
 
     return Result;
 #else
+    X80SoftFloat result;
+    if (HandleInfinityOp(state, lhs, result)) {
+      return result;
+    }
+
     BIGFLOAT Src_d = lhs.ToFMax(state);
     Src_d = FEXCore::cephes_128bit::tanl(Src_d);
     return X80SoftFloat(state, Src_d);
@@ -411,6 +439,11 @@ struct FEX_PACKED X80SoftFloat {
 
     return Result;
 #else
+    X80SoftFloat result;
+    if (HandleInfinityOp(state, lhs, result)) {
+      return result;
+    }
+
     BIGFLOAT Src_d = lhs.ToFMax(state);
     Src_d = FEXCore::cephes_128bit::sinl(Src_d);
     return X80SoftFloat(state, Src_d);
@@ -432,6 +465,11 @@ struct FEX_PACKED X80SoftFloat {
 
     return Result;
 #else
+    X80SoftFloat result;
+    if (HandleInfinityOp(state, lhs, result)) {
+      return result;
+    }
+
     BIGFLOAT Src_d = lhs.ToFMax(state);
     Src_d = FEXCore::cephes_128bit::cosl(Src_d);
     return X80SoftFloat(state, Src_d);
@@ -591,6 +629,20 @@ private:
   static constexpr uint64_t IntegerBit = (1ULL << 63);
   static constexpr uint64_t Bottom62Significand = ((1ULL << 62) - 1);
   static constexpr uint32_t ExponentBias = 16383;
+
+  // Helper function to check for infinity and set invalid operation flag.
+  // Returns true if infinity is dealt with, false otherwise.
+  FEXCORE_PRESERVE_ALL_ATTR static bool HandleInfinityOp(softfloat_state* state, const X80SoftFloat& arg, X80SoftFloat& result) {
+    if (arg.Exponent == 0x7FFF && arg.Significand == 0x8000000000000000ULL) {
+      state->exceptionFlags |= softfloat_flag_invalid;
+      // Return QNaN.
+      result.Sign = 0;
+      result.Exponent = 0x7FFF;
+      result.Significand = 0xC000000000000000ULL;
+      return true;
+    }
+    return false;
+  }
 };
 
 #ifndef _WIN32

FEXCore/Source/Interface/Config/Config.json.in

@@ -68,7 +68,9 @@
           "ENABLESVEBITPERM": "enablesvebitperm",
           "DISABLESVEBITPERM": "disablesvebitperm",
           "ENABLEPRESERVEALLABI": "enablepreserveallabi",
-          "DISABLEPRESERVEALLABI": "disablepreserveallabi"
+          "DISABLEPRESERVEALLABI": "disablepreserveallabi",
+          "ENABLEWFXT": "enablewfxt",
+          "DISABLEWFXT": "disablewfxt"
         },
         "Desc": [
           "Allows controlling of the CPU features in the JIT.",
@@ -89,7 +91,8 @@
           "\t{enable,disable}crypto: Will force enable or disable crypto extensions even if the host doesn't support it",
           "\t{enable,disable}rpres: Will force enable or disable rpres even if the host doesn't support it",
           "\t{enable,disable}svebitperm: Will force enable or disable svebitperm even if the host doesn't support it",
-          "\t{enable,disable}preserveallabi: Will force enable or disable preserve_all abi even if the host doesn't support it"
+          "\t{enable,disable}preserveallabi: Will force enable or disable preserve_all abi even if the host doesn't support it",
+          "\t{enable,disable}wfxt: Will force enable or disable wfxt even if the host doesn't support it"
         ]
       },
       "SmallTSCScale": {

FEXCore/Source/Interface/Context/Context.h

@@ -59,8 +59,9 @@ namespace Validation {
 
 namespace FEXCore::Context {
 struct FEX_PACKED ExitFunctionLinkData {
-  uint64_t HostBranch;
+  uint64_t HostCode;
   uint64_t GuestRIP;
+  int64_t CallerOffset;
 };
 
 struct CustomIRResult {
@@ -166,7 +167,8 @@ public:
 
   void OnCodeBufferAllocated(CPU::CodeBuffer&) override;
   void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) override;
-  void InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) override;
+  void InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, uint64_t Start,
+                                uint64_t Length) override;
   FEXCore::ForkableSharedMutex& GetCodeInvalidationMutex() override {
     return CodeInvalidationMutex;
   }
@@ -248,15 +250,7 @@ public:
   ContextImpl(const FEXCore::HostFeatures& Features);
   ~ContextImpl();
 
-  static void ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP);
-
-  template<auto Fn>
-  static uint64_t ThreadExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, ExitFunctionLinkData* Record) {
-    auto Thread = Frame->Thread;
-    auto lk = GuardSignalDeferringSection<std::shared_lock>(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
-
-    return Fn(Frame, Record);
-  }
+  static bool ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP);
 
   // Wrapper which takes CpuStateFrame instead of InternalThreadState and unique_locks CodeInvalidationMutex
   // Must be called from owning thread
@@ -279,25 +273,23 @@ public:
     uint64_t TotalInstructionsLength;
     uint64_t StartAddr;
     uint64_t Length;
+    bool NeedsAddGuestCodeRanges;
   };
   [[nodiscard]]
   GenerateIRResult GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, bool ExtendedDebugInfo, uint64_t MaxInst);
 
   struct CompileCodeResult {
-    void* CompiledCode;
+    CPU::CPUBackend::CompiledCode CompiledCode;
     fextl::unique_ptr<FEXCore::Core::DebugData> DebugData;
     uint64_t StartAddr;
     uint64_t Length;
+    bool NeedsAddGuestCodeRanges;
   };
   [[nodiscard]]
   CompileCodeResult CompileCode(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, uint64_t MaxInst = 0);
   uintptr_t CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP, uint64_t MaxInst = 0);
   uintptr_t CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP);
 
-  IR::OpSize GetGPROpSize() const {
-    return Config.Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;
-  }
-
   FEXCore::JITSymbols Symbols;
 
   FEXCore::Utils::PooledAllocatorVirtual OpDispatcherAllocator;
@@ -377,7 +369,12 @@ private:
 
   std::shared_mutex CustomIRMutex;
   std::atomic<bool> HasCustomIRHandlers {};
-  fextl::unordered_map<uint64_t, std::tuple<CustomIREntrypointHandler, void*, void*>> CustomIRHandlers;
+  struct CustomIRHandlerEntry final {
+    CustomIREntrypointHandler Handler;
+    void *Creator;
+    void *Data;
+  };
+  fextl::unordered_map<uint64_t, CustomIRHandlerEntry> CustomIRHandlers;
   IntervalList<uint64_t> ForceTSOValidRanges; // The ranges for which ForceTSOInstructions has populated data
   fextl::set<uint64_t> ForceTSOInstructions;
 };

FEXCore/Source/Interface/Core/Addressing.cpp

@@ -11,7 +11,7 @@ Ref LoadEffectiveAddress(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSize, b
   Ref Tmp = A.Base;
 
   if (A.Offset) {
-    Ref Offset = IREmit->_Constant(A.Offset);
+    Ref Offset = IREmit->Constant(A.Offset);
     Tmp = Tmp ? IREmit->_Add(GPRSize, Tmp, Offset) : Offset;
   }
 
@@ -22,7 +22,7 @@ Ref LoadEffectiveAddress(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSize, b
       if (Tmp) {
         Tmp = IREmit->_AddShift(GPRSize, Tmp, A.Index, ShiftType::LSL, Log2);
       } else {
-        Tmp = IREmit->_Lshl(GPRSize, A.Index, IREmit->_Constant(Log2));
+        Tmp = IREmit->_Lshl(GPRSize, A.Index, IREmit->Constant(Log2));
       }
     } else {
       Tmp = Tmp ? IREmit->_Add(GPRSize, Tmp, A.Index) : A.Index;
@@ -41,7 +41,7 @@ Ref LoadEffectiveAddress(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSize, b
     } else if (A.Offset) {
       uint64_t X = A.Offset;
       X &= (1ull << Bits) - 1;
-      Tmp = IREmit->_Constant(X);
+      Tmp = IREmit->Constant(X);
     }
   }
 
@@ -49,7 +49,7 @@ Ref LoadEffectiveAddress(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSize, b
     Tmp = Tmp ? IREmit->_Add(GPRSize, Tmp, A.Segment) : A.Segment;
   }
 
-  return Tmp ?: IREmit->_Constant(0);
+  return Tmp ?: IREmit->Constant(0);
 }
 
 AddressMode SelectAddressMode(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSize, bool HostSupportsTSOImm9, bool AtomicTSO, bool Vector,
@@ -107,7 +107,7 @@ AddressMode SelectAddressMode(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSi
 
     return {
       .Base = LoadEffectiveAddress(IREmit, B, GPRSize, true /* AddSegmentBase */, false),
-      .Index = IREmit->_Constant(A.Offset),
+      .Index = IREmit->Constant(A.Offset),
       .IndexType = MEM_OFFSET_SXTX,
       .IndexScale = 1,
     };
@@ -150,7 +150,7 @@ AddressMode SelectAddressMode(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSi
 
         return {
           .Base = LoadEffectiveAddress(IREmit, B, GPRSize, true /* AddSegmentBase */, false),
-          .Index = IREmit->_Constant(A.Offset),
+          .Index = IREmit->Constant(A.Offset),
           .IndexType = MEM_OFFSET_SXTX,
           .IndexScale = 1,
         };

FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp

@@ -73,13 +73,13 @@ namespace x64 {
     ARMEmitter::Reg::r8, ARMEmitter::Reg::r16, ARMEmitter::Reg::r17,
   };
 
-  constexpr std::array<ARMEmitter::Register, 8> RA = {
+  constexpr std::array<ARMEmitter::Register, 7> RA = {
     // All these callee saved
     ARMEmitter::Reg::r20, ARMEmitter::Reg::r21, ARMEmitter::Reg::r22, ARMEmitter::Reg::r23,
-    ARMEmitter::Reg::r24, ARMEmitter::Reg::r25, ARMEmitter::Reg::r30, ARMEmitter::Reg::r18,
+    ARMEmitter::Reg::r24, ARMEmitter::Reg::r30, ARMEmitter::Reg::r18,
   };
 
-  constexpr unsigned RAPairs = 6;
+  constexpr unsigned RAPairs = 4;
 
   // Dynamic GPRs
   constexpr std::array<ARMEmitter::Register, 2> PreserveAll_Dynamic = {
@@ -143,18 +143,18 @@ namespace x64 {
     ARMEmitter::Reg::r4, ARMEmitter::Reg::r5, ARMEmitter::Reg::r8,
   };
 
-  constexpr std::array<ARMEmitter::Register, 7> RA = {
-    ARMEmitter::Reg::r6,  ARMEmitter::Reg::r7,  ARMEmitter::Reg::r14, ARMEmitter::Reg::r15,
-    ARMEmitter::Reg::r16, ARMEmitter::Reg::r17, ARMEmitter::Reg::r30,
+  constexpr std::array<ARMEmitter::Register, 6> RA = {
+    ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r14, ARMEmitter::Reg::r15, ARMEmitter::Reg::r16, ARMEmitter::Reg::r30,
   };
 
-  constexpr std::array<ARMEmitter::Register, 5> PreserveAll_Dynamic = {
-    ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r16, ARMEmitter::Reg::r17, ARMEmitter::Reg::r30,
-  };
+  constexpr std::array<ARMEmitter::Register, 5> PreserveAll_Dynamic = {ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r16,
+                                                                       ARMEmitter::Reg::r17, ARMEmitter::Reg::r30};
 
-  constexpr std::array<ARMEmitter::Register, 7> NotPreserved_Dynamic = RA;
+  constexpr std::array<ARMEmitter::Register, 7> NotPreserved_Dynamic = {ARMEmitter::Reg::r6,  ARMEmitter::Reg::r7,  ARMEmitter::Reg::r14,
+                                                                        ARMEmitter::Reg::r15, ARMEmitter::Reg::r16, ARMEmitter::Reg::r17,
+                                                                        ARMEmitter::Reg::r30};
 
-  constexpr unsigned RAPairs = 6;
+  constexpr unsigned RAPairs = 4;
 
   constexpr std::array<ARMEmitter::VRegister, 16> SRAFPR = {
     ARMEmitter::VReg::v0,  ARMEmitter::VReg::v1,  ARMEmitter::VReg::v2,  ARMEmitter::VReg::v3,
@@ -245,14 +245,12 @@ namespace x32 {
     REG_AF,
   };
 
-  constexpr std::array<ARMEmitter::Register, 15> RA = {
+  constexpr std::array<ARMEmitter::Register, 14> RA = {
     // All these callee saved
     ARMEmitter::Reg::r20,
     ARMEmitter::Reg::r21,
     ARMEmitter::Reg::r22,
     ARMEmitter::Reg::r23,
-    ARMEmitter::Reg::r24,
-    ARMEmitter::Reg::r25,
 
     // Registers only available on 32-bit
     // All these are caller saved (except for r19).
@@ -265,6 +263,7 @@ namespace x32 {
     ARMEmitter::Reg::r29,
     ARMEmitter::Reg::r30,
 
+    ARMEmitter::Reg::r24,
     ARMEmitter::Reg::r19,
   };
 
@@ -273,7 +272,7 @@ namespace x32 {
     ARMEmitter::Reg::r16, ARMEmitter::Reg::r17, ARMEmitter::Reg::r30,
   };
 
-  constexpr unsigned RAPairs = 12;
+  constexpr unsigned RAPairs = 10;
 
   // All are caller saved
   constexpr std::array<ARMEmitter::VRegister, 8> SRAFPR = {
@@ -370,6 +369,8 @@ Arm64Emitter::Arm64Emitter(FEXCore::Context::ContextImpl* ctx, void* EmissionPtr
   // Hardcode a 256-bit vector width if we are running in the simulator.
   // Allow the user to override this.
   Simulator.SetVectorLengthInBits(ForceSVEWidth() ? ForceSVEWidth() : 256);
+  // FEX doesn't support GCS.
+  Simulator.DisableGCSCheck();
 #endif
 #ifdef VIXL_DISASSEMBLER
   // Only setup the disassembler if enabled.
@@ -495,7 +496,7 @@ void Arm64Emitter::LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, ui
   uint64_t AlignedPC = PC & ~0xFFFULL;
 
   // Offset from aligned PC
-  int64_t AlignedOffset = static_cast<int64_t>(Constant) - static_cast<int64_t>(AlignedPC);
+  auto AlignedOffset = std::bit_cast<int64_t>(Constant - AlignedPC);
 
   int NumMoves = 0;
 
@@ -511,7 +512,7 @@ void Arm64Emitter::LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, ui
     } else {
       // If the constant is within 1MB of PC then we can still use ADR to load in a single instruction
       // 21-bit signed integer here
-      int64_t SmallOffset = static_cast<int64_t>(Constant) - static_cast<int64_t>(PC);
+      auto SmallOffset = std::bit_cast<int64_t>(Constant - PC);
       if (ARMEmitter::Emitter::IsInt21(SmallOffset)) {
         adr(Reg, SmallOffset);
       } else {
@@ -694,6 +695,8 @@ void Arm64Emitter::SpillStaticRegs(ARMEmitter::Register TmpReg, bool FPRs, uint3
   unsigned PFAFSpillMask = GPRSpillMask & PFAFMask;
   GPRSpillMask &= ~PFAFSpillMask;
 
+  str(REG_CALLRET_SP, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.callret_sp));
+
   for (size_t i = 0; i < StaticRegisters.size(); i += 2) {
     auto Reg1 = StaticRegisters[i];
     auto Reg2 = StaticRegisters[i + 1];
@@ -789,6 +792,8 @@ void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t GPRFillMask, uint32_t FPRF
   ldr(STATE, TmpReg, CPU_AREA_EMULATOR_DATA_OFFSET);
 #endif
 
+  ldr(REG_CALLRET_SP, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.callret_sp));
+
   // Regardless of what GPRs/FPRs we're filling, we need to fill NZCV since it
   // is always static and was almost certainly clobbered.
   //

FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h

@@ -43,6 +43,8 @@ constexpr bool TMP_ABIARGS = true;
 constexpr auto REG_PF = ARMEmitter::Reg::r26;
 constexpr auto REG_AF = ARMEmitter::Reg::r27;
 
+constexpr auto REG_CALLRET_SP = ARMEmitter::XReg::x25;
+
 // Vector temporaries
 constexpr auto VTMP1 = ARMEmitter::VReg::v0;
 constexpr auto VTMP2 = ARMEmitter::VReg::v1;
@@ -61,6 +63,8 @@ constexpr bool TMP_ABIARGS = false;
 constexpr auto REG_PF = ARMEmitter::Reg::r9;
 constexpr auto REG_AF = ARMEmitter::Reg::r24;
 
+constexpr auto REG_CALLRET_SP = ARMEmitter::XReg::x17;
+
 // Vector temporaries
 constexpr auto VTMP1 = ARMEmitter::VReg::v16;
 constexpr auto VTMP2 = ARMEmitter::VReg::v17;
@@ -84,7 +88,8 @@ constexpr uint64_t EC_CODE_BITMAP_MAX_ADDRESS = 1ULL << 47;
 #endif
 
 // Will force one single instruction block to be generated first if set when entering the JIT filling SRA.
-constexpr auto ENTRY_FILL_SRA_SINGLE_INST_REG = TMP1;
+// FillStaticRegs must preserve this
+constexpr auto ENTRY_FILL_SRA_SINGLE_INST_REG = TMP2;
 
 // Predicate to use in the X87 SVE optimization
 constexpr ARMEmitter::PRegister PRED_X87_SVEOPT = ARMEmitter::PReg::p2;

FEXCore/Source/Interface/Core/CPUBackend.h

@@ -13,6 +13,7 @@ $end_info$
 #include <FEXCore/fextl/memory.h>
 #include <FEXCore/fextl/string.h>
 #include <FEXCore/fextl/vector.h>
+#include <FEXCore/fextl/map.h>
 
 #include <cstdint>
 
@@ -94,15 +95,7 @@ namespace CPU {
     struct CompiledCode {
       // Where this code block begins.
       uint8_t* BlockBegin;
-      /**
-       * The function entrypoint to this codeblock.
-       *
-       * This may or may not equal `BlockBegin` above. Depending on the CPU backend, it may stick data
-       * prior to the BlockEntry.
-       *
-       * Is actually a function pointer of type `void (FEXCore::Core::ThreadState *Thread)`
-       */
-      uint8_t* BlockEntry;
+      fextl::map<uint64_t, uint8_t*> EntryPoints;
       // The total size of the codeblock from [BlockBegin, BlockBegin+Size).
       size_t Size;
     };

FEXCore/Source/Interface/Core/CPUID.cpp

@@ -626,6 +626,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
     // Only enable EnhancedREPMOVS if atomic memcpy tso emulation isn't enabled.
     const uint32_t SupportsEnhancedREPMOVS = CTX->IsMemcpyAtomicTSOEnabled() == false;
     const uint32_t SupportsVPCLMULQDQ = CTX->HostFeatures.SupportsPMULL_128Bit && SupportsAVX();
+    const uint32_t SupportsWFXT = CTX->HostFeatures.SupportsWFXT;
 
     // Number of subfunctions
     Res.eax = 0x0;
@@ -645,39 +646,39 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (1 << 13) |                              // Deprecates FPU CS and DS
               (0 << 14) |                              // Intel MPX
               (0 << 15) |                              // Intel Resource Directory Technology Allocation
-              (0 << 16) |                              // Reserved
-              (0 << 17) |                              // Reserved
+              (0 << 16) |                              // AVX512-F
+              (0 << 17) |                              // AVX512-DQ
               (CTX->HostFeatures.SupportsRAND << 18) | // RDSEED
               (1 << 19) |                              // ADCX and ADOX instructions
               (0 << 20) |                              // SMAP Supervisor mode access prevention and CLAC/STAC instructions
-              (0 << 21) |                              // Reserved
-              (0 << 22) |                              // Reserved
+              (0 << 21) |                              // AVX512-IFMA
+              (0 << 22) |                              // PCOMMIT (deprecated?)
               (1 << 23) |                              // CLFLUSHOPT instruction
               (1 << 24) |                              // CLWB instruction
               (0 << 25) |                              // Intel processor trace
-              (0 << 26) |                              // Reserved
-              (0 << 27) |                              // Reserved
-              (0 << 28) |                              // Reserved
+              (0 << 26) |                              // AVX512-PF
+              (0 << 27) |                              // AVX512-ER
+              (0 << 28) |                              // AVX512-CD
               (Features.SHA << 29) |                   // SHA instructions
-              (0 << 30) |                              // Reserved
-              (0 << 31);                               // Reserved
+              (0 << 30) |                              // AVX512-BW
+              (0 << 31);                               // AVX512-VL
 
     Res.ecx = (1 << 0) |                                // PREFETCHWT1
               (0 << 1) |                                // AVX512VBMI
               (0 << 2) |                                // Usermode instruction prevention
               (0 << 3) |                                // Protection keys for user mode pages
               (0 << 4) |                                // OS protection keys
-              (0 << 5) |                                // waitpkg
-              (0 << 6) |                                // AVX512_VBMI2
+              (SupportsWFXT << 5) |                     // waitpkg
+              (0 << 6) |                                // AVX512-VBMI2
               (0 << 7) |                                // CET shadow stack
               (0 << 8) |                                // GFNI
               (CTX->HostFeatures.SupportsAES256 << 9) | // VAES
               (SupportsVPCLMULQDQ << 10) |              // VPCLMULQDQ
-              (0 << 11) |                               // AVX512_VNNI
-              (0 << 12) |                               // AVX512_BITALG
+              (0 << 11) |                               // AVX512-VNNI
+              (0 << 12) |                               // AVX512-BITALG
               (0 << 13) |                               // Intel Total Memory Encryption
-              (0 << 14) |                               // AVX512_VPOPCNTDQ
-              (0 << 15) |                               // Reserved
+              (0 << 14) |                               // AVX512-VPOPCNTDQ
+              (0 << 15) |                               // FZM (TDX)
               (0 << 16) |                               // 5 Level page tables
               (0 << 17) |                               // MPX MAWAU
               (0 << 18) |                               // MPX MAWAU
@@ -685,28 +686,28 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (0 << 20) |                               // MPX MAWAU
               (0 << 21) |                               // MPX MAWAU
               (1 << 22) |                               // RDPID Read Processor ID
-              (0 << 23) |                               // Reserved
-              (0 << 24) |                               // Reserved
+              (0 << 23) |                               // AES Key Locker
+              (1 << 24) |                               // bus-lock-detect
               (0 << 25) |                               // CLDEMOTE
-              (0 << 26) |                               // Reserved
+              (0 << 26) |                               // MPRR (TDX)
               (0 << 27) |                               // MOVDIRI
               (0 << 28) |                               // MOVDIR64B
-              (0 << 29) |                               // Reserved
+              (0 << 29) |                               // ENQCMD
               (0 << 30) |                               // SGX Launch configuration
-              (0 << 31);                                // Reserved
+              (0 << 31);                                // PKS
 
-    Res.edx = (0 << 0) |                   // Reserved
-              (0 << 1) |                   // Reserved
-              (0 << 2) |                   // AVX512_4VNNIW
-              (0 << 3) |                   // AVX512_4FMAPS
+    Res.edx = (0 << 0) |                   // SGX-TEM (TDX)
+              (0 << 1) |                   // SGX-KEYS
+              (0 << 2) |                   // AVX512-4VNNIW
+              (0 << 3) |                   // AVX512-4FMAPS
               (1 << 4) |                   // Fast Short Rep Mov
-              (0 << 5) |                   // Reserved
+              (0 << 5) |                   // UINTR
               (0 << 6) |                   // Reserved
               (0 << 7) |                   // Reserved
-              (0 << 8) |                   // AVX512_VP2INTERSECT
+              (0 << 8) |                   // AVX512-VP2INTERSECT
               (0 << 9) |                   // SRBDS_CTRL (Special Register Buffer Data Sampling Mitigations)
               (0 << 10) |                  // VERW clears CPU buffers
-              (0 << 11) |                  // Reserved
+              (0 << 11) |                  // rtm-always-abort
               (0 << 12) |                  // Reserved
               (0 << 13) |                  // TSX Force Abort (TSX will force abort if attempted)
               (0 << 14) |                  // SERIALIZE instruction
@@ -718,7 +719,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (0 << 20) |                  // Intel CET
               (0 << 21) |                  // Reserved
               (0 << 22) |                  // AMX-BF16 - Tile computation on bfloat16
-              (0 << 23) |                  // AVX512_FP16 - FP16 AVX512 instructions
+              (0 << 23) |                  // AVX512-FP16 - FP16 AVX512 instructions
               (0 << 24) |                  // AMX-tile - If AMX is implemented
               (0 << 25) |                  // AMX-int8 - AMX on 8-bit integers
               (0 << 26) |                  // IBRS_IBPB - Speculation control
@@ -754,7 +755,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_0Dh(uint32_t Leaf) const {
     // XFeatureSupportedMask[63:32]
     Res.edx = 0; // Upper 32-bits of XFeatureSupportedMask
   } else if (Leaf == 1) {
-    Res.eax = (0 << 0) | // XSAVEOPT
+    Res.eax = (1 << 0) | // XSAVEOPT
               (0 << 1) | // XSAVEC (and XRSTOR)
               (0 << 2) | // XGETBV - XGETBV with ECX=1 supported
               (0 << 3);  // XSAVES - XSAVES, XRSTORS, and IA32_XSS supported

FEXCore/Source/Interface/Core/Core.cpp

@@ -46,13 +46,13 @@ $end_info$
 #include "FEXCore/Utils/SignalScopeGuards.h"
 #include <FEXCore/Utils/Threads.h>
 #include <FEXCore/Utils/Profiler.h>
+#include <FEXCore/Utils/SHMStats.h>
 #include <FEXCore/fextl/fmt.h>
 #include <FEXCore/fextl/memory.h>
 #include <FEXCore/fextl/set.h>
 #include <FEXCore/fextl/sstream.h>
 #include <FEXCore/fextl/vector.h>
 #include <FEXHeaderUtils/Syscalls.h>
-#include <FEXHeaderUtils/TodoDefines.h>
 
 #include <algorithm>
 #include <array>
@@ -405,14 +405,14 @@ void ContextImpl::ExecuteThread(FEXCore::Core::InternalThreadState* Thread) {
   }
 
   // If it is the parent thread that died then just leave
-  FEX_TODO("This doesn't make sense when the parent thread doesn't outlive its children");
+  // TODO: This doesn't make sense when the parent thread doesn't outlive its children
 }
 
 void ContextImpl::InitializeCompiler(FEXCore::Core::InternalThreadState* Thread) {
   Thread->OpDispatcher = fextl::make_unique<FEXCore::IR::OpDispatchBuilder>(this);
   Thread->OpDispatcher->SetMultiblock(Config.Multiblock);
   Thread->LookupCache = fextl::make_unique<FEXCore::LookupCache>(this);
-  Thread->FrontendDecoder = fextl::make_unique<FEXCore::Frontend::Decoder>(this);
+  Thread->FrontendDecoder = fextl::make_unique<FEXCore::Frontend::Decoder>(Thread);
   Thread->PassManager = fextl::make_unique<FEXCore::IR::PassManager>();
 
   Thread->CurrentFrame->Pointers.Common.L1Pointer = Thread->LookupCache->GetL1Pointer();
@@ -441,6 +441,22 @@ ContextImpl::CreateThread(uint64_t InitialRIP, uint64_t StackPointer, const FEXC
   Thread->CurrentFrame->State.gregs[X86State::REG_RSP] = StackPointer;
   Thread->CurrentFrame->State.rip = InitialRIP;
 
+  // Set up default code segment.
+  // Default code segment indexes match the numbers that the Linux kernel uses.
+  Thread->CurrentFrame->State.cs_idx = 6 << 3;
+  auto &GDT = Thread->CurrentFrame->State.gdt[Thread->CurrentFrame->State.cs_idx >> 3];
+  Thread->CurrentFrame->State.SetGDTBase(&GDT, 0);
+  Thread->CurrentFrame->State.SetGDTLimit(&GDT, 0xF'FFFFU);
+
+  if (Config.Is64BitMode) {
+    GDT.L = 1; // L = Long Mode = 64-bit
+    GDT.D = 0; // D = Default Operand SIze = Reserved
+  }
+  else {
+    GDT.L = 0; // L = Long Mode = 32-bit
+    GDT.D = 1; // D = Default Operand Size = 32-bit
+  }
+
   // Copy over the new thread state to the new object
   if (NewThreadState) {
     memcpy(&Thread->CurrentFrame->State, NewThreadState, sizeof(FEXCore::Core::CPUState));
@@ -517,6 +533,7 @@ void ContextImpl::ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, boo
     // Clear L1+L2 cache of this thread, and clear L3 cache across any threads using it
     Thread->LookupCache->ClearCache();
   }
+  Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);
 }
 
 static void IRDumper(FEXCore::Core::InternalThreadState* Thread, IR::IREmitter* IREmitter, uint64_t GuestRIP) {
@@ -545,7 +562,7 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
     if (Handler != CustomIRHandlers.end()) {
       TotalInstructions = 1;
       TotalInstructionsLength = 1;
-      std::get<0>(Handler->second)(GuestRIP, Thread->OpDispatcher.get());
+      Handler->second.Handler(GuestRIP, Thread->OpDispatcher.get());
       HasCustomIR = true;
     }
   }
@@ -557,19 +574,14 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
     bool HadDispatchError {false};
     bool HadInvalidInst {false};
 
-    Thread->FrontendDecoder->DecodeInstructionsAtEntry(GuestCode, GuestRIP, MaxInst,
-                                                       [Thread](uint64_t BlockEntry, uint64_t Start, uint64_t Length) {
-      if (Thread->LookupCache->AddBlockExecutableRange(BlockEntry, Start, Length)) {
-        static_cast<ContextImpl*>(Thread->CTX)->SyscallHandler->MarkGuestExecutableRange(Thread, Start, Length);
-      }
-    });
+    Thread->FrontendDecoder->DecodeInstructionsAtEntry(Thread, GuestCode, GuestRIP, MaxInst);
 
     auto BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo();
     auto CodeBlocks = &BlockInfo->Blocks;
 
-    Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks, BlockInfo->TotalInstructionCount);
+    Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks, BlockInfo->TotalInstructionCount, BlockInfo->Is64BitMode);
 
-    const auto GPRSize = GetGPROpSize();
+    const auto GPRSize = Thread->OpDispatcher->GetGPROpSize();
 
     for (size_t j = 0; j < CodeBlocks->size(); ++j) {
       const FEXCore::Frontend::Decoder::DecodedBlocks& Block = CodeBlocks->at(j);
@@ -683,7 +695,11 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
               LogMan::Msg::EFmt("Invalid or Unknown instruction: {} 0x{:x}", TableInfo->Name ?: "UND", Block.Entry - GuestRIP);
             }
 
-            Thread->OpDispatcher->InvalidOp(DecodedInfo);
+            if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::NOEXEC_INST) {
+              Thread->OpDispatcher->NoExecOp(DecodedInfo);
+            } else {
+              Thread->OpDispatcher->InvalidOp(DecodedInfo);
+            }
           }
 
           HadInvalidInst = true;
@@ -739,6 +755,7 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
     .TotalInstructionsLength = TotalInstructionsLength,
     .StartAddr = Thread->FrontendDecoder->DecodedMinAddress,
     .Length = Thread->FrontendDecoder->DecodedMaxAddress - Thread->FrontendDecoder->DecodedMinAddress,
+    .NeedsAddGuestCodeRanges = !HasCustomIR,
   };
 }
 
@@ -750,10 +767,11 @@ ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalT
       auto CompiledCode = Thread->CPUBackend->RelocateJITObjectCode(GuestRIP, CodeCacheEntry);
       if (CompiledCode) {
         return {
-          .CompiledCode = CompiledCode,
+          .CompiledCode = {},
           .DebugData = nullptr, // nullptr here ensures that code serialization doesn't occur on from cache read
           .StartAddr = 0,       // Unused
           .Length = 0,          // Unused
+          .NeedsAddGuestCodeRanges = false,
         };
       }
     }
@@ -767,9 +785,10 @@ ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalT
   }
 
   // Generate IR + Meta Info
-  auto [IRView, TotalInstructions, TotalInstructionsLength, StartAddr, Length] = GenerateIR(Thread, GuestRIP, Config.GDBSymbols(), MaxInst);
+  auto [IRView, TotalInstructions, TotalInstructionsLength, StartAddr, Length, NeedsAddGuestCodeRanges] =
+    GenerateIR(Thread, GuestRIP, Config.GDBSymbols(), MaxInst);
   if (!IRView) {
-    return {nullptr, nullptr, 0, 0};
+    return {{}, nullptr, 0, 0, false};
   }
 
   // Attempt to get the CPU backend to compile this code
@@ -780,7 +799,11 @@ ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalT
   if (MaxInst != 1) {
     if (auto Block = Thread->LookupCache->FindBlock(GuestRIP)) {
       Thread->OpDispatcher->DelayedDisownBuffer();
-      return {.CompiledCode = reinterpret_cast<uint8_t*>(Block), .DebugData = nullptr, .StartAddr = 0, .Length = 0};
+      return {.CompiledCode = {.BlockBegin = reinterpret_cast<uint8_t*>(Block), .EntryPoints = {{GuestRIP, reinterpret_cast<uint8_t*>(Block)}}},
+              .DebugData = nullptr,
+              .StartAddr = 0,
+              .Length = 0,
+              .NeedsAddGuestCodeRanges = false};
     }
   }
 
@@ -795,13 +818,11 @@ ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalT
   Thread->OpDispatcher->DelayedDisownBuffer();
 
   return {
-    // FEX currently throws away the CPUBackend::CompiledCode object other than the entrypoint
-    // In the future with code caching getting wired up, we will pass the rest of the data forward.
-    // TODO: Pass the data forward when code caching is wired up to this.
-    .CompiledCode = CompiledCode.BlockEntry,
+    .CompiledCode = std::move(CompiledCode),
     .DebugData = std::move(DebugData),
     .StartAddr = StartAddr,
     .Length = Length,
+    .NeedsAddGuestCodeRanges = NeedsAddGuestCodeRanges,
   };
 }
 
@@ -821,7 +842,8 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
     return HostCode;
   }
 
-  auto [CodePtr, DebugData, StartAddr, Length] = CompileCode(Thread, GuestRIP, MaxInst);
+  auto [CompiledCode, DebugData, StartAddr, Length, NeedsAddGuestCodeRanges] = CompileCode(Thread, GuestRIP, MaxInst);
+  auto CodePtr = CompiledCode.EntryPoints[GuestRIP];
   if (CodePtr == nullptr) {
     return 0;
   } else if (!DebugData) {
@@ -831,7 +853,7 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
 
   // The core managed to compile the code.
   if (Config.BlockJITNaming()) {
-    auto FragmentBasePtr = reinterpret_cast<uint8_t*>(CodePtr);
+    auto FragmentBasePtr = CompiledCode.BlockBegin;
 
     if (DebugData) {
       auto GuestRIPLookup = SyscallHandler->LookupAOTIRCacheEntry(Thread, GuestRIP);
@@ -840,7 +862,7 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
         for (auto& Subblock : DebugData->Subblocks) {
           auto BlockBasePtr = FragmentBasePtr + Subblock.HostCodeOffset;
           if (GuestRIPLookup.Entry) {
-            Symbols.Register(Thread->SymbolBuffer.get(), BlockBasePtr, DebugData->HostCodeSize, GuestRIPLookup.Entry->Filename,
+            Symbols.Register(Thread->SymbolBuffer.get(), BlockBasePtr, CompiledCode.Size, GuestRIPLookup.Entry->Filename,
                              GuestRIP - GuestRIPLookup.VAFileStart);
           } else {
             Symbols.Register(Thread->SymbolBuffer.get(), BlockBasePtr, GuestRIP, Subblock.HostCodeSize);
@@ -848,10 +870,10 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
         }
       } else {
         if (GuestRIPLookup.Entry) {
-          Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, DebugData->HostCodeSize, GuestRIPLookup.Entry->Filename,
+          Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, CompiledCode.Size, GuestRIPLookup.Entry->Filename,
                            GuestRIP - GuestRIPLookup.VAFileStart);
         } else {
-          Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, GuestRIP, DebugData->HostCodeSize);
+          Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, GuestRIP, CompiledCode.Size);
         }
       }
     }
@@ -864,8 +886,8 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
         .GuestRIP = GuestRIP,
         .GuestCodeLength = Length,
         .GuestCodeHash = 0,
-        .HostCodeBegin = CodePtr,
-        .HostCodeLength = DebugData->HostCodeSize,
+        .HostCodeBegin = CompiledCode.BlockBegin,
+        .HostCodeLength = CompiledCode.Size,
         .HostCodeHash = 0,
         .ThreadJobRefCount = &Thread->ObjectCacheRefCounter,
         .Relocations = std::move(*DebugData->Relocations),
@@ -875,14 +897,26 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
   // Clear any relocations that might have been generated
   Thread->CPUBackend->ClearRelocations();
 
-  if (IRCaptureCache.PostCompileCode(Thread, CodePtr, GuestRIP, StartAddr, Length, {}, DebugData.get(), false)) {
+  if (IRCaptureCache.PostCompileCode(Thread, CompiledCode.BlockBegin, GuestRIP, StartAddr, Length, {}, DebugData.get(), false)) {
     // Early exit
     return (uintptr_t)CodePtr;
   }
 
+  if (NeedsAddGuestCodeRanges) {
+    // Track in the guest to host map all entrypoints for all pages the compiled block touches, if any page didn't previously
+    // contain code, inform the frontend so it can setup SMC detection.
+    auto BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo();
+    for (auto CodePage : BlockInfo->CodePages) {
+      if (Thread->LookupCache->AddBlockExecutableRange(BlockInfo->EntryPoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE)) {
+        SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE);
+      }
+    }
+  }
+
   // Insert to lookup cache
-  // Pages containing this block are added via AddBlockExecutableRange before each page gets accessed in the frontend
-  Thread->LookupCache->AddBlockMapping(GuestRIP, CodePtr);
+  for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) {
+    Thread->LookupCache->AddBlockMapping(GuestAddr, HostAddr);
+  }
 
   return (uintptr_t)CodePtr;
 }
@@ -896,7 +930,8 @@ uintptr_t ContextImpl::CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, ui
   // Invalidate might take a unique lock on this, to guarantee that during invalidation no code gets compiled
   auto lk = GuardSignalDeferringSection<std::shared_lock>(CodeInvalidationMutex, Thread);
 
-  auto [CodePtr, DebugData, StartAddr, Length] = CompileCode(Thread, GuestRIP, 1);
+  auto [CompiledCode, DebugData, StartAddr, Length, _] = CompileCode(Thread, GuestRIP, 1);
+  auto CodePtr = CompiledCode.EntryPoints[GuestRIP];
   if (CodePtr == nullptr) {
     return 0;
   }
@@ -907,22 +942,40 @@ uintptr_t ContextImpl::CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, ui
   return (uintptr_t)CodePtr;
 }
 
-static void InvalidateGuestThreadCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) {
-  auto lk = Thread->LookupCache->AcquireLock();
+static void InvalidateGuestThreadCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator,
+                                           uint64_t Start, uint64_t Length) {
+  // Ensures now-modified mappings aren't cached as being in their previous non-executable state.
+  // Accessing FrontendDecoder is safe as the thread's code invalidation mutex must be locked here.
+  Thread->FrontendDecoder->ResetExecutableRangeCache();
 
-  auto lower = Thread->LookupCache->CodePages.lower_bound(Start >> 12);
-  auto upper = Thread->LookupCache->CodePages.upper_bound((Start + Length - 1) >> 12);
+  auto lk = Thread->LookupCache->AcquireLock();
+  auto& CodePages = Thread->LookupCache->Shared->CodePages;
+
+  auto lower = CodePages.lower_bound(Start >> 12);
+  auto upper = CodePages.upper_bound((Start + Length - 1) >> 12);
 
   for (auto it = lower; it != upper; it++) {
-    for (auto Address : it->second) {
-      ContextImpl::ThreadRemoveCodeEntry(Thread, Address);
+    Accumulator.emplace_back(std::move(it->second));
+  }
+
+  bool InvalidatedAnyEntries = false;
+  for (const auto& PageEntries : Accumulator) {
+    for (const auto& Entry : PageEntries) {
+      if (ContextImpl::ThreadRemoveCodeEntry(Thread, Entry)) {
+        InvalidatedAnyEntries = true;
+      }
     }
-    it->second.clear();
+  }
+
+  if (InvalidatedAnyEntries) {
+    // This may cause access violations in the thread on Windows as zeroing is not atomic, this is handled by the frontend
+    Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);
   }
 }
 
-void ContextImpl::InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) {
-  InvalidateGuestThreadCodeRange(Thread, Start, Length);
+void ContextImpl::InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator,
+                                           uint64_t Start, uint64_t Length) {
+  InvalidateGuestThreadCodeRange(Thread, Accumulator, Start, Length);
 }
 
 void ContextImpl::MarkMemoryShared(FEXCore::Core::InternalThreadState* Thread) {
@@ -942,11 +995,11 @@ void ContextImpl::MarkMemoryShared(FEXCore::Core::InternalThreadState* Thread) {
   }
 }
 
-void ContextImpl::ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP) {
+bool ContextImpl::ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP) {
   LogMan::Throw::AFmt(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to "
                                                                                                          "be unique_locked here");
 
-  Thread->LookupCache->Erase(Thread->CurrentFrame, GuestRIP);
+  return Thread->LookupCache->Erase(Thread->CurrentFrame, GuestRIP);
 }
 
 std::optional<CustomIRResult>
@@ -955,7 +1008,7 @@ ContextImpl::AddCustomIREntrypoint(uintptr_t Entrypoint, CustomIREntrypointHandl
 
   std::unique_lock lk(CustomIRMutex);
 
-  auto InsertedIterator = CustomIRHandlers.emplace(Entrypoint, std::tuple(Handler, Creator, Data));
+  auto InsertedIterator = CustomIRHandlers.emplace(Entrypoint, CustomIRHandlerEntry {Handler, Creator, Data});
   HasCustomIRHandlers = true;
 
   if (!InsertedIterator.second) {
@@ -979,21 +1032,21 @@ void ContextImpl::AddThunkTrampolineIRHandler(uintptr_t Entrypoint, uintptr_t Gu
   auto Result = AddCustomIREntrypoint(
     Entrypoint,
     [this, GuestThunkEntrypoint](uintptr_t Entrypoint, FEXCore::IR::IREmitter* emit) {
-    auto IRHeader = emit->_IRHeader(emit->Invalid(), Entrypoint, 0, 0, 0, 0);
-    auto Block = emit->CreateCodeNode();
-    IRHeader.first->Blocks = emit->WrapNode(Block);
-    emit->SetCurrentCodeBlock(Block);
+      auto IRHeader = emit->_IRHeader(emit->Invalid(), Entrypoint, 0, 0, 0, 0);
+      auto Block = emit->CreateCodeNode(true, 0);
+      IRHeader.first->Blocks = emit->WrapNode(Block);
+      emit->SetCurrentCodeBlock(Block);
 
-    const auto GPRSize = GetGPROpSize();
+      const auto GPRSize = this->Config.Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;
 
-    if (GPRSize == IR::OpSize::i64Bit) {
-      IR::Ref R = emit->_StoreRegister(emit->_Constant(Entrypoint), GPRSize);
-      R->Reg = IR::PhysicalRegister(IR::GPRFixedClass, X86State::REG_R11).Raw;
-    } else {
-      emit->_StoreContext(GPRSize, IR::FPRClass, emit->_VCastFromGPR(IR::OpSize::i64Bit, IR::OpSize::i64Bit, emit->_Constant(Entrypoint)),
-                          offsetof(Core::CPUState, mm[0][0]));
-    }
-    emit->_ExitFunction(IR::OpSize::i64Bit, emit->_Constant(GuestThunkEntrypoint));
+      if (GPRSize == IR::OpSize::i64Bit) {
+        IR::Ref R = emit->_StoreRegister(emit->Constant(Entrypoint), GPRSize);
+        R->Reg = IR::PhysicalRegister(IR::GPRFixedClass, X86State::REG_R11).Raw;
+      } else {
+        emit->_StoreContext(GPRSize, IR::FPRClass, emit->_VCastFromGPR(IR::OpSize::i64Bit, IR::OpSize::i64Bit, emit->Constant(Entrypoint)),
+                            offsetof(Core::CPUState, mm[0][0]));
+      }
+      emit->_ExitFunction(IR::OpSize::i64Bit, emit->Constant(GuestThunkEntrypoint), IR::BranchHint::None, emit->Invalid(), emit->Invalid());
     },
     ThunkHandler, (void*)GuestThunkEntrypoint);
 
@@ -1027,7 +1080,8 @@ void ContextImpl::RemoveCustomIREntrypoint(uintptr_t Entrypoint) {
 
   std::scoped_lock lk(CustomIRMutex);
 
-  InvalidateGuestCodeRange(nullptr, Entrypoint, 1);
+  InvalidatedEntryAccumulator Accumulator;
+  InvalidateGuestCodeRange(nullptr, Accumulator, Entrypoint, 1);
   CustomIRHandlers.erase(Entrypoint);
 
   HasCustomIRHandlers = !CustomIRHandlers.empty();

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp

@@ -81,11 +81,14 @@ void Dispatcher::EmitDispatcher() {
   add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ARMEmitter::Reg::rsp, 0);
   str(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, ReturningStackLocation));
 
+  ARMEmitter::ForwardLabel CompileSingleStep;
   AbsoluteLoopTopAddressFillSRA = GetCursorAddress<uint64_t>();
 
   FillStaticRegs();
+  ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));
+  cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);
+
   ARMEmitter::BiDirectionalLabel LoopTop {};
-  ARMEmitter::ForwardLabel CompileSingleStep;
 
 #ifdef _M_ARM_64EC
   b(&LoopTop);
@@ -111,8 +114,21 @@ void Dispatcher::EmitDispatcher() {
   add(ARMEmitter::Size::i64Bit, StaticRegisters[X86State::REG_RSP], ARMEmitter::Reg::rsp, 0);
   add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, TMP1, 0);
 
+  ldr(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));
+
   FillSpecialRegs(TMP1, TMP2, false, true);
 
+  // As ARM64EC uses this as an entrypoint for both guest calls and host returns, opportunistically try to return
+  // using the call-ret stack to avoid unbalancing it.
+  ldp<ARMEmitter::IndexType::OFFSET>(TMP1, TMP2, REG_CALLRET_SP);
+  // EC_CALL_CHECKER_PC_REG is REG_PF which isn't touched by any of the above
+  sub(ARMEmitter::Size::i64Bit, TMP1, EC_CALL_CHECKER_PC_REG, TMP1);
+  cbnz(ARMEmitter::Size::i64Bit, TMP1, &LoopTop);
+
+  // If the entry at the TOS is for the target address, pop it and return to the JIT code
+  add(ARMEmitter::Size::i64Bit, REG_CALLRET_SP, REG_CALLRET_SP, 0x10);
+  ret(TMP2);
+
   // Enter JIT
 #endif
 
@@ -286,6 +302,8 @@ void Dispatcher::EmitDispatcher() {
     lsrv(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP2);
     tbz(TMP1, 0, &l_NotECCode);
 
+    str(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));
+
     add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0);
     mov(EC_CALL_CHECKER_PC_REG, RipReg);
     ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionEC));
@@ -498,6 +516,7 @@ void Dispatcher::EmitDispatcher() {
 
     // load static regs
     FillStaticRegs();
+    stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
 
     // Now go back to the regular dispatcher loop
     b(&LoopTop);
@@ -607,6 +626,7 @@ void Dispatcher::EmitDispatcher() {
 #ifdef VIXL_SIMULATOR
 void Dispatcher::ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame) {
   Simulator.WriteXRegister(0, reinterpret_cast<int64_t>(Frame));
+  Simulator.WriteXRegister(1, 0);
   Simulator.RunFrom(reinterpret_cast< const vixl::aarch64::Instruction*>(DispatchPtr));
 }
 

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.h

@@ -76,7 +76,7 @@ public:
   void ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP);
 #else
   void ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame) {
-    DispatchPtr(Frame);
+    DispatchPtr(Frame, false);
   }
 
   void ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP) {
@@ -109,7 +109,7 @@ public:
 protected:
   FEXCore::Context::ContextImpl* CTX;
 
-  using AsmDispatch = void (*)(FEXCore::Core::CpuStateFrame* Frame);
+  using AsmDispatch = void (*)(FEXCore::Core::CpuStateFrame* Frame, bool SingleInst);
   using JITCallback = void (*)(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP);
 
   AsmDispatch DispatchPtr;

FEXCore/Source/Interface/Core/Frontend.cpp

@@ -9,6 +9,8 @@ $end_info$
 #include "Interface/Context/Context.h"
 #include "Interface/Core/Frontend.h"
 #include "Interface/Core/X86Tables/X86Tables.h"
+#include "Interface/Core/X86HelperGen.h"
+#include "Interface/Core/LookupCache.h"
 
 #include <array>
 #include <algorithm>
@@ -21,6 +23,7 @@ $end_info$
 #include <FEXCore/Utils/Profiler.h>
 #include <FEXCore/Utils/Telemetry.h>
 #include <FEXCore/Utils/TypeDefines.h>
+#include <FEXCore/Debug/InternalThreadState.h>
 #include <FEXCore/fextl/set.h>
 
 namespace FEXCore::Frontend {
@@ -64,33 +67,73 @@ static uint32_t MapVEXToReg(uint8_t vvvv, bool HasXMM) {
   }
 }
 
-Decoder::Decoder(FEXCore::Context::ContextImpl* ctx)
-  : CTX {ctx}
-  , OSABI {ctx->SyscallHandler ? ctx->SyscallHandler->GetOSABI() : FEXCore::HLE::SyscallOSABI::OS_UNKNOWN}
-  , PoolObject {ctx->FrontendAllocator, sizeof(FEXCore::X86Tables::DecodedInst) * DefaultDecodedBufferSize} {}
+Decoder::Decoder(FEXCore::Core::InternalThreadState* Thread)
+  : Thread {Thread}
+  , CTX {static_cast<FEXCore::Context::ContextImpl*>(Thread->CTX)}
+  , OSABI {CTX->SyscallHandler ? CTX->SyscallHandler->GetOSABI() : FEXCore::HLE::SyscallOSABI::OS_UNKNOWN}
+  , PoolObject {CTX->FrontendAllocator, sizeof(FEXCore::X86Tables::DecodedInst) * DefaultDecodedBufferSize} {}
 
-Decoder::~Decoder() {
-  PoolObject.UnclaimBuffer();
+bool Decoder::CheckRangeExecutable(uint64_t Address, uint64_t Size) {
+  // Treat FEX-internal X86 callbacks as always executable
+  if (EntryPoint == CTX->X86CodeGen.CallbackReturn) {
+    return true;
+  }
+
+  while (Address < ExecutableRangeBase || Address + Size > ExecutableRangeEnd) {
+    auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, Address);
+    ExecutableRangeBase = RangeInfo.Base;
+    ExecutableRangeEnd = RangeInfo.Base + RangeInfo.Size;
+
+    if (RangeInfo.Size == 0) {
+      return false;
+    }
+
+    uint64_t RangeRemainingSize = ExecutableRangeEnd - Address;
+    if (Size > RangeRemainingSize) {
+      Size -= RangeRemainingSize;
+      Address += RangeRemainingSize;
+    }
+  }
+
+  return true;
 }
 
 uint8_t Decoder::ReadByte() {
-  uint8_t Byte = InstStream[InstructionSize];
   LOGMAN_THROW_A_FMT(InstructionSize < MAX_INST_SIZE, "Max instruction size exceeded!");
-  Instruction[InstructionSize] = Byte;
+  std::optional<uint8_t> Byte = PeekByte(0);
+  if (!Byte) {
+    HitNonExecutableRange = true;
+    // Pretend we read 0, the main decode loop will see HitNonExecutableRange and rollback the instruction.
+    return 0;
+  }
+
+  Instruction[InstructionSize] = *Byte;
   InstructionSize++;
-  return Byte;
+  return *Byte;
 }
 
-uint8_t Decoder::PeekByte(uint8_t Offset) const {
-  uint8_t Byte = InstStream[InstructionSize + Offset];
-  return Byte;
+std::optional<uint8_t> Decoder::PeekByte(uint8_t Offset) {
+  uint64_t ByteAddress = reinterpret_cast<uint64_t>(InstStream + InstructionSize + Offset);
+  if (CheckRangeExecutable(ByteAddress, 1)) {
+    return InstStream[InstructionSize + Offset];
+  } else {
+    return std::nullopt;
+  }
 }
 
 uint64_t Decoder::ReadData(uint8_t Size) {
   LOGMAN_THROW_A_FMT(Size != 0 && Size <= sizeof(uint64_t), "Unknown data size to read");
 
   uint64_t Res = 0;
-  std::memcpy(&Res, &InstStream[InstructionSize], Size);
+  uint64_t Address = reinterpret_cast<uint64_t>(InstStream + InstructionSize);
+  if (CheckRangeExecutable(Address, Size)) {
+    std::memcpy(&Res, &InstStream[InstructionSize], Size);
+  } else {
+    HitNonExecutableRange = true;
+    // See PeekByte, this specific case may cause some executable memory to read as 0 but it doesn't matter as the entire instruction will be rolled back anyway.
+    Res = 0;
+  }
+
 
 #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
   for (size_t i = 0; i < Size; ++i) {
@@ -290,7 +333,7 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
   uint8_t DestSize {};
   const bool HasWideningDisplacement =
     (FEXCore::X86Tables::DecodeFlags::GetOpAddr(DecodeInst->Flags, 0) & FEXCore::X86Tables::DecodeFlags::FLAG_WIDENING_SIZE_LAST) != 0 ||
-    (Options.w && CTX->Config.Is64BitMode);
+    (Options.w && BlockInfo.Is64BitMode);
   const bool HasNarrowingDisplacement =
     (FEXCore::X86Tables::DecodeFlags::GetOpAddr(DecodeInst->Flags, 0) & FEXCore::X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST) != 0;
 
@@ -308,7 +351,21 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
   const bool HasMODRM = !!(Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_MODRM);
 
   const bool HasREX = !!(DecodeInst->Flags & DecodeFlags::FLAG_REX_PREFIX);
-  const bool Has16BitAddressing = !CTX->Config.Is64BitMode && DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;
+  const bool Has16BitAddressing = !BlockInfo.Is64BitMode && DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;
+
+  if (Options.w && (Info->Flags & InstFlags::FLAGS_REX_W_0)) {
+    return false;
+  } else if (!Options.w && (Info->Flags & InstFlags::FLAGS_REX_W_1)) {
+    return false;
+  }
+
+  if (Options.L && (Info->Flags & InstFlags::FLAGS_VEX_L_0)) {
+    return false;
+  } else if (!Options.L && (Info->Flags & InstFlags::FLAGS_VEX_L_1)) {
+    return false;
+  }
+
+  const bool UseVEXL = Options.L && !(Info->Flags & InstFlags::FLAGS_VEX_L_IGNORE);
 
   // This is used for ModRM register modification
   // For both modrm.reg and modrm.rm(when mod == 0b11) when value is >= 0b100
@@ -339,7 +396,7 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
       DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_16BIT);
       DestSize = 2;
     } else if (DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_128BIT) {
-      if (Options.L) {
+      if (UseVEXL) {
         DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_256BIT);
         DestSize = 32;
       } else {
@@ -355,7 +412,7 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
       // If the default operating mode is 32bit and we have the operand size flag then the operating size drops to 16bit
       DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_16BIT);
       DestSize = 2;
-    } else if ((HasXMMDst || HasMMDst || CTX->Config.Is64BitMode) &&
+    } else if ((HasXMMDst || HasMMDst || BlockInfo.Is64BitMode) &&
                (HasWideningDisplacement || DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BIT ||
                 DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BITDEF)) {
       DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_64BIT);
@@ -372,7 +429,7 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
     } else if (SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_16BIT) {
       DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_16BIT);
     } else if (SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_128BIT) {
-      if (Options.L) {
+      if (UseVEXL) {
         DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_256BIT);
       } else {
         DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_128BIT);
@@ -384,7 +441,7 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
       // See table 1-2. Operand-Size Overrides for this decoding
       // If the default operating mode is 32bit and we have the operand size flag then the operating size drops to 16bit
       DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_16BIT);
-    } else if ((HasXMMSrc || HasMMSrc || CTX->Config.Is64BitMode) &&
+    } else if ((HasXMMSrc || HasMMSrc || BlockInfo.Is64BitMode) &&
                (HasWideningDisplacement || SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BIT ||
                 SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_64BITDEF)) {
       DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_64BIT);
@@ -484,6 +541,9 @@ bool Decoder::NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op,
   size_t CurrentSrc = 0;
 
   const auto VEXOperand = Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_VEX_SRC_MASK;
+  if (VEXOperand == FEXCore::X86Tables::InstFlags::FLAGS_VEX_NO_OPERAND && Options.vvvv) {
+    return false;
+  }
 
   if (VEXOperand == FEXCore::X86Tables::InstFlags::FLAGS_VEX_1ST_SRC) {
     DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;
@@ -651,7 +711,7 @@ bool Decoder::NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16
     DecodedHeader options {};
 
     if ((Byte1 & 0b10000000) == 0) {
-      if (!CTX->Config.Is64BitMode) {
+      if (!BlockInfo.Is64BitMode) {
         return false;
       }
 
@@ -670,12 +730,12 @@ bool Decoder::NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16
       options.w = (Byte2 & 0b10000000) != 0;
       options.L = (Byte2 & 0b100) != 0;
       if ((Byte1 & 0b01000000) == 0) {
-        if (!CTX->Config.Is64BitMode) {
+        if (!BlockInfo.Is64BitMode) {
           return false;
         }
         DecodeInst->Flags |= DecodeFlags::FLAG_REX_XGPR_X;
       }
-      if (CTX->Config.Is64BitMode && (Byte1 & 0b00100000) == 0) {
+      if (BlockInfo.Is64BitMode && (Byte1 & 0b00100000) == 0) {
         DecodeInst->Flags |= DecodeFlags::FLAG_REX_XGPR_B;
       }
       if (options.w) {
@@ -720,7 +780,7 @@ bool Decoder::NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16
   FEX_UNREACHABLE;
 }
 
-bool Decoder::DecodeInstruction(uint64_t PC) {
+bool Decoder::DecodeInstructionImpl(uint64_t PC) {
   InstructionSize = 0;
   Instruction.fill(0);
 
@@ -748,7 +808,7 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
           FEXCore::X86Tables::ModRMDecoded ModRM;
           ModRM.Hex = DecodeInst->ModRM;
 
-          const bool Has16BitAddressing = !CTX->Config.Is64BitMode && DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;
+          const bool Has16BitAddressing = !BlockInfo.Is64BitMode && DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;
 
           // All 3DNow! instructions have the second argument as the rm handler
           // We need to decode it upfront to get the displacement out of the way
@@ -857,29 +917,25 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
       DecodeInst->Flags |= DecodeFlags::FLAG_ADDRESS_SIZE;
       break;
     case 0x26: // ES legacy prefix
-      if (!CTX->Config.Is64BitMode) {
-        DecodeInst->Flags |= DecodeFlags::FLAG_ES_PREFIX;
+      if (!BlockInfo.Is64BitMode) {
+        DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_ES_PREFIX;
       }
       break;
     case 0x2E: // CS legacy prefix
-      if (!CTX->Config.Is64BitMode) {
-        DecodeInst->Flags |= DecodeFlags::FLAG_CS_PREFIX;
+      if (!BlockInfo.Is64BitMode) {
+        DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_CS_PREFIX;
       }
       break;
     case 0x36: // SS legacy prefix
-      if (!CTX->Config.Is64BitMode) {
-        DecodeInst->Flags |= DecodeFlags::FLAG_SS_PREFIX;
+      if (!BlockInfo.Is64BitMode) {
+        DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_SS_PREFIX;
       }
       break;
     case 0x3E: // DS legacy prefix
-      // Annoyingly GCC generates NOP ops with these prefixes
-      // Just ignore them for now
-      // eg. 66 2e 0f 1f 84 00 00 00 00 00 nop    WORD PTR cs:[rax+rax*1+0x0]
-      if (!CTX->Config.Is64BitMode) {
-        DecodeInst->Flags |= DecodeFlags::FLAG_DS_PREFIX;
+      if (!BlockInfo.Is64BitMode) {
+        DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_DS_PREFIX;
       }
       break;
-      break;
     case 0xF0: // LOCK prefix
       DecodeInst->Flags |= DecodeFlags::FLAG_LOCK;
       break;
@@ -892,19 +948,16 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
       DecodeInst->LastEscapePrefix = Op;
       break;
     case 0x64: // FS prefix
-      DecodeInst->Flags |= DecodeFlags::FLAG_FS_PREFIX;
+      DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_FS_PREFIX;
       break;
     case 0x65: // GS prefix
-      DecodeInst->Flags |= DecodeFlags::FLAG_GS_PREFIX;
+      DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_GS_PREFIX;
       break;
     default:
       [[likely]] { // Default base table
         auto Info = &FEXCore::X86Tables::BaseOps[Op];
 
         if (Info->Type == FEXCore::X86Tables::TYPE_REX_PREFIX) {
-          if (!CTX->Config.Is64BitMode) {
-            return false;
-          }
           DecodeInst->Flags |= DecodeFlags::FLAG_REX_PREFIX;
 
           // Widening displacement
@@ -935,6 +988,31 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
       }
     }
   }
+
+  if (DecodeInst->Dest.IsGPR()) {
+    return false;
+  }
+
+  return true;
+}
+
+Decoder::DecodedBlockStatus Decoder::DecodeInstruction(uint64_t PC) {
+  // Will be set if DecodeInstructionImpl tries to read non-executable memory
+  HitNonExecutableRange = false;
+  bool ErrorDuringDecoding = !DecodeInstructionImpl(PC);
+
+  if (ErrorDuringDecoding || HitNonExecutableRange) [[unlikely]] {
+    // Put an invalid instruction in the stream so the core can raise SIGILL if hit
+    // Error while decoding instruction. We don't know the table or instruction size
+    DecodeInst->TableInfo = nullptr;
+    DecodeInst->InstSize = 0;
+    return ErrorDuringDecoding ? DecodedBlockStatus::INVALID_INST : DecodedBlockStatus::NOEXEC_INST;
+  } else if (!DecodeInst->TableInfo || !DecodeInst->TableInfo->OpcodeDispatcher) {
+    // If there wasn't an error during decoding but we have no dispatcher for the instruction then claim invalid instruction.
+    return DecodedBlockStatus::INVALID_INST;
+  }
+
+  return DecodedBlockStatus::SUCCESS;
 }
 
 void Decoder::BranchTargetInMultiblockRange() {
@@ -944,16 +1022,23 @@ void Decoder::BranchTargetInMultiblockRange() {
 
   // If the RIP setting is conditional AND within our symbol range then it can be considered for multiblock
   uint64_t TargetRIP = 0;
-  const auto GPRSize = CTX->GetGPROpSize();
+  const auto GPRSize = GetGPROpSize();
   bool Conditional = true;
   const auto InstEnd = DecodeInst->PC + DecodeInst->InstSize;
 
+  if (DecodeInst->TableInfo->Flags & FEXCore::X86Tables::InstFlags::FLAGS_CALL) {
+    AddBranchTarget(InstEnd);
+    BlockInfo.EntryPoints.emplace(InstEnd);
+    return;
+  }
+
+  // Calls are handled above
   switch (DecodeInst->OP) {
   case 0x70 ... 0x7F:   // Conditional JUMP
   case 0x80 ... 0x8F: { // More conditional
     // Source is a literal
     // auto RIPOffset = LoadSource(Op, Op->Src[0], Op->Flags);
-    // auto RIPTargetConst = _Constant(Op->PC + Op->InstSize);
+    // auto RIPTargetConst = Constant(Op->PC + Op->InstSize);
     // Target offset is PC + InstSize + Literal
     TargetRIP = InstEnd + DecodeInst->Src[0].Literal();
     break;
@@ -963,11 +1048,6 @@ void Decoder::BranchTargetInMultiblockRange() {
     TargetRIP = InstEnd + DecodeInst->Src[0].Literal();
     Conditional = false;
     break;
-  case 0xE8: // Call - Immediate target, We don't want to inline calls
-    if (ExternalBranches) {
-      ExternalBranches->insert(InstEnd);
-    }
-    [[fallthrough]];
   case 0xC2: // RET imm
   case 0xC3: // RET
   default: return; break;
@@ -1015,7 +1095,7 @@ bool Decoder::InstCanContinue() const {
   }
 
   uint64_t TargetRIP = 0;
-  const auto GPRSize = CTX->GetGPROpSize();
+  const auto GPRSize = GetGPROpSize();
 
   if (DecodeInst->OP == 0xE8) { // Call - immediate target
     const uint64_t NextRIP = DecodeInst->PC + DecodeInst->InstSize;
@@ -1067,7 +1147,7 @@ void Decoder::AddBranchTarget(uint64_t Target) {
           .Size = BlockIt->Size - SplitOffset,
           .NumInstructions = BlockIt->NumInstructions - SplitIdx,
           .DecodedInstructions = BlockIt->DecodedInstructions + SplitIdx,
-          .HasInvalidInstruction = BlockIt->HasInvalidInstruction,
+          .BlockStatus = BlockIt->BlockStatus,
         };
 
         BlockIt->Size = SplitOffset;
@@ -1107,12 +1187,10 @@ const uint8_t* Decoder::AdjustAddrForSpecialRegion(const uint8_t* _InstStream, u
   return _InstStream - EntryPoint + RIP;
 }
 
-void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC, uint64_t MaxInst,
-                                        std::function<void(uint64_t BlockEntry, uint64_t Start, uint64_t Length)> AddContainedCodePage) {
+void Decoder::DecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState *Thread, const uint8_t* _InstStream, uint64_t PC, uint64_t MaxInst) {
   FEXCORE_PROFILE_SCOPED("DecodeInstructions");
   BlockInfo.TotalInstructionCount = 0;
   BlockInfo.Blocks.clear();
-  BlocksToDecode.clear();
   VisitedBlocks.clear();
   // Reset internal state management
   DecodedSize = 0;
@@ -1120,9 +1198,15 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
   MaxCondBranchBackwards = ~0ULL;
   DecodedBuffer = PoolObject.ReownOrClaimBuffer();
 
+  // Decode operating mode from thread's CS segment.
+  const auto CSSegment = Thread->CurrentFrame->State.gdt[Thread->CurrentFrame->State.cs_idx >> 3];
+  BlockInfo.Is64BitMode = CSSegment.L == 1;
+  LOGMAN_THROW_A_FMT(BlockInfo.Is64BitMode == CTX->Config.Is64BitMode, "Expected operating mode to not change at runtime!");
+
   // XXX: Load symbol data
   SymbolAvailable = false;
   EntryPoint = PC;
+  BlockInfo.EntryPoints = {PC};
   InstStream = _InstStream;
 
   uint64_t TotalInstructions {};
@@ -1138,13 +1222,11 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
   DecodedMaxAddress = EntryPoint;
 
   // Entry is a jump target
-  BlocksToDecode.emplace(PC);
+  BlocksToDecode = {PC};
 
   uint64_t CurrentCodePage = PC & FEXCore::Utils::FEX_PAGE_MASK;
 
-  fextl::set<uint64_t> CodePages = {CurrentCodePage};
-
-  AddContainedCodePage(PC, CurrentCodePage, FEXCore::Utils::FEX_PAGE_SIZE);
+  BlockInfo.CodePages = {CurrentCodePage};
 
   if (MaxInst == 0) {
     MaxInst = CTX->Config.MaxInstPerBlock;
@@ -1179,6 +1261,7 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
 
     BlockIt->Entry = RIPToDecode;
     BlockIt->Size = 0;
+    BlockIt->IsEntryPoint = EntryBlock;
 
     uint64_t PCOffset = 0;
     uint64_t BlockStartOffset = DecodedSize;
@@ -1200,7 +1283,7 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
       auto OpMinPage = OpAddress & FEXCore::Utils::FEX_PAGE_MASK;
       auto OpMaxPage = OpMaxAddress & FEXCore::Utils::FEX_PAGE_MASK;
 
-      if (!EntryBlock && OpMinPage == OpMaxPage && PeekByte(0) == 0 && PeekByte(1) == 0) [[unlikely]] {
+      if (!EntryBlock && OpMinPage == OpMaxPage && PeekByte(0).value_or(0) == 0 && PeekByte(1).value_or(0) == 0) [[unlikely]] {
         // End the multiblock early if we hit 2 consecutive null bytes (add [rax], al) in the same page with the
         // assumption we are most likely trying to explore garbage code.
         break;
@@ -1208,31 +1291,17 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
 
       if (OpMinPage != CurrentCodePage) {
         CurrentCodePage = OpMinPage;
-        CodePages.insert(CurrentCodePage);
+        BlockInfo.CodePages.insert(CurrentCodePage);
       }
 
       if (OpMaxPage != CurrentCodePage) {
         CurrentCodePage = OpMaxPage;
-        CodePages.insert(CurrentCodePage);
+        BlockInfo.CodePages.insert(CurrentCodePage);
       }
 
-      bool ErrorDuringDecoding = !DecodeInstruction(OpAddress);
+      BlockIt->BlockStatus = DecodeInstruction(OpAddress);
       uint64_t OpEndAddress = OpAddress + DecodeInst->InstSize;
 
-      if (ErrorDuringDecoding) [[unlikely]] {
-        // Put an invalid instruction in the stream so the core can raise SIGILL if hit
-        BlockIt->HasInvalidInstruction = true;
-        // Error while decoding instruction. We don't know the table or instruction size
-        DecodeInst->TableInfo = nullptr;
-        DecodeInst->InstSize = 0;
-      } else {
-        // If there wasn't an error during decoding but we have no dispatcher for the instruction then claim invalid instruction.
-        auto TableInfo = DecodeInst->TableInfo;
-        if (!TableInfo || !TableInfo->OpcodeDispatcher) {
-          BlockIt->HasInvalidInstruction = true;
-        }
-      }
-
       DecodedMinAddress = std::min(DecodedMinAddress, OpAddress);
       DecodedMaxAddress = std::max(DecodedMaxAddress, OpEndAddress);
 
@@ -1248,7 +1317,7 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
       BlockIt->Size += DecodeInst->InstSize;
 
       // Can not continue this block at all on invalid instruction
-      if (BlockIt->HasInvalidInstruction) [[unlikely]] {
+      if (BlockIt->BlockStatus != DecodedBlockStatus::SUCCESS) [[unlikely]] {
         if (!EntryBlock) {
           // In multiblock configurations, we can early terminate any non-entrypoint blocks with the expectation that this won't get hit.
           // Improves compile-times.
@@ -1257,6 +1326,9 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
           DecodedSize = BlockStartOffset;
           InstStream -= PCOffset;
           EraseBlock = true;
+        } else {
+          LogMan::Msg::EFmt("{} instruction in entry block: {:X}",
+                            BlockIt->BlockStatus == DecodedBlockStatus::INVALID_INST ? "Invalid" : "NoExec", OpAddress);
         }
         break;
       }
@@ -1296,8 +1368,8 @@ void Decoder::DecodeInstructionsAtEntry(const uint8_t* _InstStream, uint64_t PC,
 
   BlockInfo.TotalInstructionCount = TotalInstructions;
 
-  for (auto CodePage : CodePages) {
-    AddContainedCodePage(PC, CodePage, FEXCore::Utils::FEX_PAGE_SIZE);
+  for (auto& Block : BlockInfo.Blocks) {
+    Block.IsEntryPoint = BlockInfo.EntryPoints.contains(Block.Entry);
   }
 }
 

FEXCore/Source/Interface/Core/Frontend.h

@@ -2,6 +2,7 @@
 #pragma once
 
 #include "Interface/Core/X86Tables/X86Tables.h"
+#include "Interface/IR/IR.h"
 
 #include <FEXCore/HLE/SyscallHandler.h>
 #include <FEXCore/Utils/Telemetry.h>
@@ -19,24 +20,32 @@ class ContextImpl;
 namespace FEXCore::Frontend {
 class Decoder final {
 public:
+  enum class DecodedBlockStatus {
+    SUCCESS,
+    INVALID_INST,
+    NOEXEC_INST,
+  };
+
   // New Frontend decoding
   struct DecodedBlocks final {
     uint64_t Entry {};
     uint64_t Size {};
     uint64_t NumInstructions {};
     FEXCore::X86Tables::DecodedInst* DecodedInstructions;
-    bool HasInvalidInstruction {};
+    DecodedBlockStatus BlockStatus;
+    bool IsEntryPoint {};
   };
 
   struct DecodedBlockInformation final {
     uint64_t TotalInstructionCount;
+    bool Is64BitMode {};
     fextl::vector<DecodedBlocks> Blocks;
+    fextl::set<uint64_t> EntryPoints;
+    fextl::set<uint64_t> CodePages; // Start addresses of all pages touching the block
   };
 
-  Decoder(FEXCore::Context::ContextImpl* ctx);
-  ~Decoder();
-  void DecodeInstructionsAtEntry(const uint8_t* InstStream, uint64_t PC, uint64_t MaxInst,
-                                 std::function<void(uint64_t BlockEntry, uint64_t Start, uint64_t Length)> AddContainedCodePage);
+  Decoder(FEXCore::Core::InternalThreadState* Thread);
+  void DecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState *Thread, const uint8_t* InstStream, uint64_t PC, uint64_t MaxInst);
 
   const DecodedBlockInformation* GetDecodedBlockInfo() const {
     return &BlockInfo;
@@ -56,6 +65,10 @@ public:
     PoolObject.DelayedDisownBuffer();
   }
 
+  void ResetExecutableRangeCache() {
+    ExecutableRangeBase = ExecutableRangeEnd = 0;
+  }
+
 private:
   // To pass any information from instruction prefixes
   // down into the actual instruction handling machinery.
@@ -65,18 +78,22 @@ private:
     bool L;       // VEX.L bit (if set then 256 bit operation, if unset then scalar or 128-bit operation)
   };
 
+  FEXCore::Core::InternalThreadState* Thread;
   FEXCore::Context::ContextImpl* CTX;
   const FEXCore::HLE::SyscallOSABI OSABI {};
 
-  bool DecodeInstruction(uint64_t PC);
+  bool DecodeInstructionImpl(uint64_t PC);
+  DecodedBlockStatus DecodeInstruction(uint64_t PC);
 
   void BranchTargetInMultiblockRange();
   bool InstCanContinue() const;
 
   void AddBranchTarget(uint64_t Target);
 
+  bool CheckRangeExecutable(uint64_t Address, uint64_t Size);
+
   uint8_t ReadByte();
-  uint8_t PeekByte(uint8_t Offset) const;
+  std::optional<uint8_t> PeekByte(uint8_t Offset);
   uint64_t ReadData(uint8_t Size);
   void SkipBytes(uint8_t Size) {
     InstructionSize += Size;
@@ -90,7 +107,14 @@ private:
   Utils::PoolBufferWithTimedRetirement<FEXCore::X86Tables::DecodedInst*, 5000, 500> PoolObject;
   size_t DecodedSize {};
 
+  uint64_t ExecutableRangeBase {};
+  uint64_t ExecutableRangeEnd {};
+  bool HitNonExecutableRange {};
+
   const uint8_t* InstStream {};
+  IR::OpSize GetGPROpSize() const {
+    return BlockInfo.Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;
+  }
 
   static constexpr size_t MAX_INST_SIZE = 15;
   uint8_t InstructionSize {};

FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h

@@ -6,7 +6,7 @@
 #include "Interface/IR/IR.h"
 
 #include <FEXCore/Debug/InternalThreadState.h>
-#include <FEXCore/Utils/Profiler.h>
+#include <FEXCore/Utils/SHMStats.h>
 
 namespace FEXCore::CPU {
 FEXCORE_PRESERVE_ALL_ATTR static softfloat_state SoftFloatStateFromFCW(uint16_t FCW, bool Force80BitPrecision = false) {
@@ -36,18 +36,48 @@ FEXCORE_PRESERVE_ALL_ATTR static softfloat_state SoftFloatStateFromFCW(uint16_t
   return State;
 }
 
+FEXCORE_PRESERVE_ALL_ATTR static void HandleX87Exception(const softfloat_state& State, FEXCore::Core::CpuStateFrame* Frame) {
+  // Check for Invalid Operation exception (bit 0 of X87 status word)
+  if (State.exceptionFlags & softfloat_flag_invalid) {
+    Frame->State.flags[FEXCore::X86State::X87FLAG_IE_LOC] = 1;
+  }
+}
+
+// Wrapper for SoftFloat state to handle X87 exceptions
+class ScopedSoftFloatState {
+public:
+  FEXCORE_PRESERVE_ALL_ATTR ScopedSoftFloatState(uint16_t FCW, FEXCore::Core::CpuStateFrame* Frame, bool Force80BitPrecision = false)
+    : State(SoftFloatStateFromFCW(FCW, Force80BitPrecision))
+    , Frame(Frame) {}
+
+  FEXCORE_PRESERVE_ALL_ATTR ~ScopedSoftFloatState() {
+    HandleX87Exception(State, Frame);
+  }
+
+  // Disable copy and move to ensure RAII semantics
+  ScopedSoftFloatState(const ScopedSoftFloatState&) = delete;
+  ScopedSoftFloatState& operator=(const ScopedSoftFloatState&) = delete;
+  ScopedSoftFloatState(ScopedSoftFloatState&&) = delete;
+  ScopedSoftFloatState& operator=(ScopedSoftFloatState&&) = delete;
+
+  softfloat_state State;
+
+private:
+  FEXCore::Core::CpuStateFrame* Frame;
+};
+
 template<>
 struct OpHandlers<IR::OP_F80CVTTO> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle4(uint16_t FCW, float src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(&State, src);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(&State.State, src);
   }
 
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle8(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(&State, src);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(&State.State, src);
   }
 };
 
@@ -55,12 +85,12 @@ template<>
 struct OpHandlers<IR::OP_F80CMP> {
   FEXCORE_PRESERVE_ALL_ATTR static uint64_t handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
+    ScopedSoftFloatState State {FCW, Frame};
 
     bool eq, lt, nan;
     uint64_t ResultFlags = 0;
 
-    X80SoftFloat::FCMP(&State, Src1, Src2, &eq, &lt, &nan);
+    X80SoftFloat::FCMP(&State.State, Src1, Src2, &eq, &lt, &nan);
     if (lt) {
       ResultFlags |= (1 << IR::FCMP_FLAG_LT);
     }
@@ -78,14 +108,14 @@ template<>
 struct OpHandlers<IR::OP_F80CVT> {
   FEXCORE_PRESERVE_ALL_ATTR static float handle4(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(src).ToF32(&State);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(src).ToF32(&State.State);
   }
 
   FEXCORE_PRESERVE_ALL_ATTR static double handle8(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(src).ToF64(&State);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(src).ToF64(&State.State);
   }
 };
 
@@ -93,26 +123,26 @@ template<>
 struct OpHandlers<IR::OP_F80CVTINT> {
   FEXCORE_PRESERVE_ALL_ATTR static int16_t handle2(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(src).ToI16(&State);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(src).ToI16(&State.State);
   }
 
   FEXCORE_PRESERVE_ALL_ATTR static int32_t handle4(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(src).ToI32(&State);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(src).ToI32(&State.State);
   }
 
   FEXCORE_PRESERVE_ALL_ATTR static int64_t handle8(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat(src).ToI64(&State);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat(src).ToI64(&State.State);
   }
 
   FEXCORE_PRESERVE_ALL_ATTR static int16_t handle2t(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    auto rv = extF80_to_i32(&State, X80SoftFloat(src), softfloat_round_minMag, false);
+    ScopedSoftFloatState State {FCW, Frame};
+    auto rv = extF80_to_i32(&State.State, X80SoftFloat(src), softfloat_round_minMag, false);
 
     if (rv > INT16_MAX || rv < INT16_MIN) {
       ///< Indefinite value for 16-bit conversions.
@@ -124,14 +154,14 @@ struct OpHandlers<IR::OP_F80CVTINT> {
 
   FEXCORE_PRESERVE_ALL_ATTR static int32_t handle4t(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return extF80_to_i32(&State, X80SoftFloat(src), softfloat_round_minMag, false);
+    ScopedSoftFloatState State {FCW, Frame};
+    return extF80_to_i32(&State.State, X80SoftFloat(src), softfloat_round_minMag, false);
   }
 
   FEXCORE_PRESERVE_ALL_ATTR static int64_t handle8t(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return extF80_to_i64(&State, X80SoftFloat(src), softfloat_round_minMag, false);
+    ScopedSoftFloatState State {FCW, Frame};
+    return extF80_to_i64(&State.State, X80SoftFloat(src), softfloat_round_minMag, false);
   }
 };
 
@@ -152,8 +182,8 @@ template<>
 struct OpHandlers<IR::OP_F80ROUND> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FRNDINT(&State, Src1);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FRNDINT(&State.State, Src1);
   }
 };
 
@@ -161,8 +191,8 @@ template<>
 struct OpHandlers<IR::OP_F80F2XM1> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::F2XM1(&State, Src1);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::F2XM1(&State.State, Src1);
   }
 };
 
@@ -170,8 +200,8 @@ template<>
 struct OpHandlers<IR::OP_F80TAN> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FTAN(&State, Src1);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FTAN(&State.State, Src1);
   }
 };
 
@@ -179,8 +209,8 @@ template<>
 struct OpHandlers<IR::OP_F80SQRT> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat::FSQRT(&State, Src1);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat::FSQRT(&State.State, Src1);
   }
 };
 
@@ -188,8 +218,8 @@ template<>
 struct OpHandlers<IR::OP_F80SIN> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FSIN(&State, Src1);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FSIN(&State.State, Src1);
   }
 };
 
@@ -197,8 +227,8 @@ template<>
 struct OpHandlers<IR::OP_F80COS> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FCOS(&State, Src1);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FCOS(&State.State, Src1);
   }
 };
 
@@ -206,8 +236,8 @@ template<>
 struct OpHandlers<IR::OP_F80SINCOS> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegPairType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return FEXCore::MakeVectorRegPair(X80SoftFloat::FSIN(&State, Src1), X80SoftFloat::FCOS(&State, Src1));
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return FEXCore::MakeVectorRegPair(X80SoftFloat::FSIN(&State.State, Src1), X80SoftFloat::FCOS(&State.State, Src1));
   }
 };
 
@@ -231,8 +261,8 @@ template<>
 struct OpHandlers<IR::OP_F80ADD> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat::FADD(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat::FADD(&State.State, Src1, Src2);
   }
 };
 
@@ -240,8 +270,8 @@ template<>
 struct OpHandlers<IR::OP_F80SUB> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat::FSUB(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat::FSUB(&State.State, Src1, Src2);
   }
 };
 
@@ -249,8 +279,8 @@ template<>
 struct OpHandlers<IR::OP_F80MUL> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat::FMUL(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat::FMUL(&State.State, Src1, Src2);
   }
 };
 
@@ -258,8 +288,8 @@ template<>
 struct OpHandlers<IR::OP_F80DIV> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
-    return X80SoftFloat::FDIV(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame};
+    return X80SoftFloat::FDIV(&State.State, Src1, Src2);
   }
 };
 
@@ -267,8 +297,8 @@ template<>
 struct OpHandlers<IR::OP_F80FYL2X> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FYL2X(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FYL2X(&State.State, Src1, Src2);
   }
 };
 
@@ -276,8 +306,8 @@ template<>
 struct OpHandlers<IR::OP_F80ATAN> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FATAN(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FATAN(&State.State, Src1, Src2);
   }
 };
 
@@ -285,8 +315,8 @@ template<>
 struct OpHandlers<IR::OP_F80FPREM1> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FREM1(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FREM1(&State.State, Src1, Src2);
   }
 };
 
@@ -294,8 +324,8 @@ template<>
 struct OpHandlers<IR::OP_F80FPREM> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FREM(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FREM(&State.State, Src1, Src2);
   }
 };
 
@@ -303,8 +333,8 @@ template<>
 struct OpHandlers<IR::OP_F80SCALE> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
-    softfloat_state State = SoftFloatStateFromFCW(FCW, true);
-    return X80SoftFloat::FSCALE(&State, Src1, Src2);
+    ScopedSoftFloatState State {FCW, Frame, true};
+    return X80SoftFloat::FSCALE(&State.State, Src1, Src2);
   }
 };
 
@@ -404,15 +434,15 @@ struct OpHandlers<IR::OP_F80BCDSTORE> {
   FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1q, FEXCore::Core::CpuStateFrame* Frame) {
     FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
     X80SoftFloat Src1 = Src1q;
-    softfloat_state State = SoftFloatStateFromFCW(FCW);
+    ScopedSoftFloatState State {FCW, Frame};
     bool Negative = Src1.Sign;
 
-    Src1 = X80SoftFloat::FRNDINT(&State, Src1);
+    Src1 = X80SoftFloat::FRNDINT(&State.State, Src1);
 
     // Clear the Sign bit
     Src1.Sign = 0;
 
-    uint64_t Tmp = Src1.ToI64(&State);
+    uint64_t Tmp = Src1.ToI64(&State.State);
     X80SoftFloat Rv;
     uint8_t* BCD = reinterpret_cast<uint8_t*>(&Rv);
     memset(BCD, 0, 10);

FEXCore/Source/Interface/Core/JIT/ALUOps.cpp

@@ -969,11 +969,15 @@ DEF_OP(UDiv) {
     break;
   }
   case IR::OpSize::i32Bit: {
+    // We need to mask divisor if we have Upper bits, since the frontend does
+    // not on the hope that we can optimize to use the path above.
+    mov(ARMEmitter::Size::i32Bit, TMP2, Divisor);
+
     // TODO: 32-bit operation should be guaranteed not to leave garbage in the upper bits.
     mov(EmitSize, TMP1, Lower);
     bfi(EmitSize, TMP1, Upper, 32, 32);
-    udiv(EmitSize, Quotient, TMP1, Divisor);
-    msub(EmitSize, Remainder, Quotient, Divisor, TMP1);
+    udiv(EmitSize, Quotient, TMP1, TMP2);
+    msub(EmitSize, Remainder, Quotient, TMP2, TMP1);
     break;
   }
   case IR::OpSize::i64Bit: {
@@ -1034,34 +1038,55 @@ DEF_OP(Popcount) {
   const auto Dst = GetReg(Node);
   const auto Src = GetReg(Op->Src);
 
-  switch (OpSize) {
-  case IR::OpSize::i8Bit:
-    fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);
-    // only use lowest byte
-    cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
-    break;
-  case IR::OpSize::i16Bit:
-    fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);
-    cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
-    // only count two lowest bytes
-    addp(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D(), VTMP1.D());
-    break;
-  case IR::OpSize::i32Bit:
-    fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);
-    cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
-    // fmov has zero extended, unused bytes are zero
-    addv(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
-    break;
-  case IR::OpSize::i64Bit:
-    fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), Src);
-    cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
-    // fmov has zero extended, unused bytes are zero
-    addv(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
-    break;
-  default: LOGMAN_MSG_A_FMT("Unsupported Popcount size: {}", OpSize);
+  if (CTX->HostFeatures.SupportsCSSC) {
+    switch (OpSize) {
+      case IR::OpSize::i8Bit:
+        uxtb(ARMEmitter::Size::i32Bit, Dst, Src);
+        cnt(ARMEmitter::Size::i32Bit, Dst, Dst);
+        break;
+      case IR::OpSize::i16Bit:
+        uxth(ARMEmitter::Size::i32Bit, Dst, Src);
+        cnt(ARMEmitter::Size::i32Bit, Dst, Dst);
+        break;
+      case IR::OpSize::i32Bit:
+        cnt(ARMEmitter::Size::i32Bit, Dst, Src);
+        break;
+      case IR::OpSize::i64Bit:
+        cnt(ARMEmitter::Size::i64Bit, Dst, Src);
+        break;
+      default: LOGMAN_MSG_A_FMT("Unsupported Popcount size: {}", OpSize);
+    }
   }
+  else {
+    switch (OpSize) {
+    case IR::OpSize::i8Bit:
+      fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);
+      // only use lowest byte
+      cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
+      break;
+    case IR::OpSize::i16Bit:
+      fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);
+      cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
+      // only count two lowest bytes
+      addp(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D(), VTMP1.D());
+      break;
+    case IR::OpSize::i32Bit:
+      fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);
+      cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
+      // fmov has zero extended, unused bytes are zero
+      addv(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
+      break;
+    case IR::OpSize::i64Bit:
+      fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), Src);
+      cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
+      // fmov has zero extended, unused bytes are zero
+      addv(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
+      break;
+    default: LOGMAN_MSG_A_FMT("Unsupported Popcount size: {}", OpSize);
+    }
 
-  umov<ARMEmitter::SubRegSize::i8Bit>(Dst, VTMP1, 0);
+    umov<ARMEmitter::SubRegSize::i8Bit>(Dst, VTMP1, 0);
+  }
 }
 
 DEF_OP(FindLSB) {

FEXCore/Source/Interface/Core/JIT/AtomicOps.cpp

@@ -138,108 +138,6 @@ DEF_OP(CAS) {
   }
 }
 
-DEF_OP(AtomicAdd) {
-  auto Op = IROp->C<IR::IROp_AtomicAdd>();
-  const auto EmitSize = ConvertSize(IROp);
-  const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
-
-  auto MemSrc = GetReg(Op->Addr);
-  auto Src = GetReg(Op->Value);
-
-  if (CTX->HostFeatures.SupportsAtomics) {
-    staddl(SubEmitSize, Src, MemSrc);
-  } else {
-    ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
-    ldaxr(SubEmitSize, TMP2, MemSrc);
-    add(EmitSize, TMP2, TMP2, Src);
-    stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
-  }
-}
-
-DEF_OP(AtomicSub) {
-  auto Op = IROp->C<IR::IROp_AtomicSub>();
-  const auto EmitSize = ConvertSize(IROp);
-  const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
-
-  auto MemSrc = GetReg(Op->Addr);
-  auto Src = GetReg(Op->Value);
-
-  if (CTX->HostFeatures.SupportsAtomics) {
-    neg(EmitSize, TMP2, Src);
-    staddl(SubEmitSize, TMP2, MemSrc);
-  } else {
-    ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
-    ldaxr(SubEmitSize, TMP2, MemSrc);
-    sub(EmitSize, TMP2, TMP2, Src);
-    stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
-  }
-}
-
-DEF_OP(AtomicAnd) {
-  auto Op = IROp->C<IR::IROp_AtomicAnd>();
-  const auto EmitSize = ConvertSize(IROp);
-  const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
-
-  auto MemSrc = GetReg(Op->Addr);
-  auto Src = GetReg(Op->Value);
-
-  if (CTX->HostFeatures.SupportsAtomics) {
-    mvn(EmitSize, TMP2, Src);
-    stclrl(SubEmitSize, TMP2, MemSrc);
-  } else {
-    ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
-    ldaxr(SubEmitSize, TMP2, MemSrc);
-    and_(EmitSize, TMP2, TMP2, Src);
-    stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
-  }
-}
-
-DEF_OP(AtomicCLR) {
-  auto Op = IROp->C<IR::IROp_AtomicCLR>();
-  const auto EmitSize = ConvertSize(IROp);
-  const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
-
-  auto MemSrc = GetReg(Op->Addr);
-  auto Src = GetReg(Op->Value);
-
-  if (CTX->HostFeatures.SupportsAtomics) {
-    stclrl(SubEmitSize, Src, MemSrc);
-  } else {
-    ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
-    ldaxr(SubEmitSize, TMP2, MemSrc);
-    bic(EmitSize, TMP2, TMP2, Src);
-    stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
-  }
-}
-
-DEF_OP(AtomicOr) {
-  auto Op = IROp->C<IR::IROp_AtomicOr>();
-  const auto EmitSize = ConvertSize(IROp);
-  const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
-
-  auto MemSrc = GetReg(Op->Addr);
-  auto Src = GetReg(Op->Value);
-
-  if (CTX->HostFeatures.SupportsAtomics) {
-    stsetl(SubEmitSize, Src, MemSrc);
-  } else {
-    ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
-    ldaxr(SubEmitSize, TMP2, MemSrc);
-    orr(EmitSize, TMP2, TMP2, Src);
-    stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
-  }
-}
-
 DEF_OP(AtomicXor) {
   auto Op = IROp->C<IR::IROp_AtomicXor>();
   const auto EmitSize = ConvertSize(IROp);
@@ -260,21 +158,6 @@ DEF_OP(AtomicXor) {
   }
 }
 
-DEF_OP(AtomicNeg) {
-  auto Op = IROp->C<IR::IROp_AtomicNeg>();
-  const auto EmitSize = ConvertSize(IROp);
-  const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
-
-  auto MemSrc = GetReg(Op->Addr);
-
-  ARMEmitter::BackwardLabel LoopTop;
-  Bind(&LoopTop);
-  ldaxr(SubEmitSize, TMP2, MemSrc);
-  neg(EmitSize, TMP3, TMP2);
-  stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-  cbnz(EmitSize, TMP4, &LoopTop);
-}
-
 DEF_OP(AtomicSwap) {
   auto Op = IROp->C<IR::IROp_AtomicSwap>();
   const auto OpSize = IROp->Size;

FEXCore/Source/Interface/Core/JIT/BranchOps.cpp

@@ -58,31 +58,112 @@ DEF_OP(ExitFunction) {
   if (IsInlineConstant(Op->NewRIP, &NewRIP) || IsInlineEntrypointOffset(Op->NewRIP, &NewRIP)) {
 #ifdef _M_ARM_64EC
     if (NewRIP < EC_CODE_BITMAP_MAX_ADDRESS && RtlIsEcCode(NewRIP)) {
+      str(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));
       add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0);
       LoadConstant(ARMEmitter::Size::i64Bit, EC_CALL_CHECKER_PC_REG, NewRIP);
       ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionEC));
       br(TMP2);
     } else {
 #endif
-      // Align to 16 byte to allow atomic patching of the following 16 byte
-      // of code (excluding the RIP data) on platforms that support LSE2
-      Align16B();
+      // In order to support direct branches without constantly hitting the L1 cache, we emit a call to a block linker,
+      // this will compile the branch target block when it is hit and replace the branch to the linker at the callsite
+      // with a direct branch to the destination block. Upon invalidation of the target block the backpatch is undone.
+      //
+      // In addition, to avoid needing to lookup in the cache for returns and any indirect branch prediction penalty,
+      // a shadow stack of <GuestReturnRIP, HostReturnPC> pairs is maintained, acting as a first level cache for any
+      // return operations. As the guest may not balance calls and returns exactly, an exception handler is expected to
+      // be installed by the frontend, to reset the shadow stack to the middle of its valid bounds on overflow/underflow.
+      // This shadow stack is also cleared on block invalidation operations or codebuffer switches, to ensure all pointed-to
+      // host code is always valid.
+
+      // This code will be backpatched by Arm64JITCore_ExitFunctionLink, below is an enumeration of all the possible cases.
+      // Jump thunks are emitted in JIT.cpp after compilation of the entire multiblock.
+      //
+      // Call with known return block - unlinked
+      //    00: adr TMP1, 0xC
+      //    04: stp RetReg, TMP1, [SpReg, -0x10]!
+      //    08: bl JmpThunk00
+      //    JmpThunk00:
+      //    00: b 0x8
+      //    04: br TMP1
+      //    08: ldr TMP1, <Shared exit linker>
+      //    0c: blr TMP1
+      //    10: HostCode
+      //    18: GuestRIP
+      //    20: CallerOffset
+      //
+      // Call with known return block after backpatching - linked in branch immediate range
+      //    00: adr TMP1, 0xC
+      //    04: stp RetReg, TMP1, [SpReg, -0x10]!
+      //    08: bl HostCode                                        - MODIFIED
+      //
+      // Call with known return block after backpatching - linked out of range
+      //    00: adr TMP1, 0xC
+      //    04: stp RetReg, TMP1, [SpReg, -0x10]!
+      //    08: bl JmpThunk00
+      //    JmpThunk00:
+      //    00: ldr TMP1, 0x10                                     - MODIFIED 2nd
+      //    04: br TMP1
+      //    08: ldr TMP1, <Shared exit linker>
+      //    0c: blr TMP1
+      //    10: HostCode                                           - MODIFIED 1st
+      //    18: GuestRIP
+      //    20: CallerOffset
+      //
+      // Jump - unlinked
+      //    00: b JmpThunk00
+      //    JmpThunk00:
+      //    00: b 0x8
+      //    04: br TMP1
+      //    08: ldr TMP1, <Shared exit linker>
+      //    0c: blr TMP1
+      //    10: HostCode
+      //    18: GuestRIP
+      //    20: CallerOffset
+      //
+      // Jump after backpatching - linked in branch immediate range
+      //    00: b HostCode                                         - MODIFIED
+      //
+      // Jump after backpatching - linked out of range
+      //    00: b JmpThunk00
+      //    JmpThunk00:
+      //    00: ldr TMP1, 0x10                                     - MODIFIED 2nd
+      //    04: br TMP1
+      //    08: ldr TMP1, <Shared exit linker>
+      //    0c: blr TMP1
+      //    10: HostCode                                           - MODIFIED 1st
+      //    18: GuestRIP
+      //    20: CallerOffset
 
       ARMEmitter::ForwardLabel l_BranchHost;
-      ldr(TMP1, &l_BranchHost);
-      blr(TMP1);
+      ARMEmitter::ForwardLabel l_CallReturn;
+      if (Op->Hint == IR::BranchHint::Call) {
+        if (!Op->CallReturnBlock.IsInvalid()) {
+          auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();
+          PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;
+          adr(TMP1, &l_CallReturn);
+          stp<ARMEmitter::IndexType::PRE>(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);
+        } else {
+          stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
+        }
+      }
 
-      Bind(&l_BranchHost);
-      dc64(ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker);
-      dc64(NewRIP);
+      EmitLinkedBranch(NewRIP, Op->Hint == IR::BranchHint::Call);
+      Bind(&l_CallReturn);
 #ifdef _M_ARM_64EC
     }
 #endif
   } else {
-
-    ARMEmitter::ForwardLabel FullLookup;
+    ARMEmitter::ForwardLabel SkipFullLookup;
     auto RipReg = GetReg(Op->NewRIP);
 
+    if (Op->Hint == IR::BranchHint::Return) {
+      // First try to pop from the call-ret stack, otherwise follow the normal path (but ending in a ret)
+      ldp<ARMEmitter::IndexType::POST>(TMP1, TMP2, REG_CALLRET_SP, 0x10);
+      sub(TMP1, TMP1, RipReg.X());
+      cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
+    }
+
     // L1 Cache
     ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.L1Pointer));
 
@@ -93,16 +174,32 @@ DEF_OP(ExitFunction) {
     ubfiz(ARMEmitter::Size::i64Bit, TMP4, RipReg, 4, 20);
     add(TMP1, TMP1, TMP4);
 
-    // Note: sub+cbnz used over cmp+br to preserve flags.
     ldp<ARMEmitter::IndexType::OFFSET>(TMP2, TMP1, TMP1, 0);
-    sub(TMP1, TMP1, RipReg.X());
-    cbnz(ARMEmitter::Size::i64Bit, TMP1, &FullLookup);
-    br(TMP2);
 
-    Bind(&FullLookup);
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop));
+    // Note: sub+cbnz used over cmp+br to preserve flags.
+    sub(TMP1, TMP1, RipReg.X());
+    cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
+    ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop));
     str(RipReg.X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
-    br(TMP1);
+
+    Bind(&SkipFullLookup);
+    if (Op->Hint == IR::BranchHint::Call) {
+      ARMEmitter::ForwardLabel l_CallReturn;
+      if (!Op->CallReturnBlock.IsInvalid()) {
+        auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();
+        PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;
+        adr(TMP1, &l_CallReturn);
+        stp<ARMEmitter::IndexType::PRE>(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);
+      } else {
+        stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
+      }
+      blr(TMP2);
+      Bind(&l_CallReturn);
+    } else if (Op->Hint == IR::BranchHint::Return) {
+      ret(TMP2);
+    } else {
+      br(TMP2);
+    }
   }
 }
 

FEXCore/Source/Interface/Core/JIT/JIT.cpp

@@ -495,58 +495,116 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
   }
 }
 
-static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
-  // Emit new 16 bytes of code to a temporary patch, then atomically apply it
-  __uint128_t Patch;
-  ARMEmitter::Emitter emit((uint8_t*)&Patch, sizeof(Patch));
-  emit.ldr(TMP1, 8); // PC-relative value pointing to constant after blr
-  emit.blr(TMP1);
-  emit.dc64(Frame->Pointers.Common.ExitFunctionLinker);
+static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record, bool Call) {
+  uintptr_t JumpThunkStartAddress = reinterpret_cast<uintptr_t>(Record) - 0x10;
+  uintptr_t CallerAddress = JumpThunkStartAddress + Record->CallerOffset;
+  auto BranchOffset = JumpThunkStartAddress / 4 - CallerAddress / 4;
 
-  auto branch = reinterpret_cast<__uint128_t*>((uintptr_t)Record - 8);
-  std::atomic_ref<__uint128_t>(*branch).store(Patch, std::memory_order::relaxed);
-  ARMEmitter::Emitter::ClearICache((void*)branch, sizeof(*branch));
+  // Replace the patched callsite with a branch to the jump thunk.
+  uint32_t BranchInst = 0;
+  ARMEmitter::Emitter BranchEmit(reinterpret_cast<uint8_t*>(&BranchInst), 4);
+  if (Call) {
+    BranchEmit.bl(BranchOffset);
+  } else {
+    BranchEmit.b(BranchOffset);
+  }
+
+  std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(CallerAddress)).store(BranchInst, std::memory_order::relaxed);
+  ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(CallerAddress), 4);
 }
 
-static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
-  auto Thread = Frame->Thread;
-  auto Lock = Thread->LookupCache->AcquireLock();
+static void IndirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
+  uintptr_t JumpThunkStartAddress = reinterpret_cast<uintptr_t>(Record) - 0x10;
+  uint32_t BranchInst = 0;
+  ARMEmitter::Emitter BranchEmit(reinterpret_cast<uint8_t*>(&BranchInst), 4);
+  BranchEmit.b(0x8);
 
+  std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(JumpThunkStartAddress)).store(BranchInst, std::memory_order::relaxed);
+  ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(JumpThunkStartAddress), 4);
+
+  // No need to reset HostCode here as the exit linker pointer is stored separately, and if the block is relinked it will be updated.
+}
+
+uint64_t Arm64JITCore::ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
+  auto Thread = Frame->Thread;
   bool TFSet = Thread->CurrentFrame->State.flags[X86State::RFLAG_TF_RAW_LOC];
   uintptr_t HostCode {};
   auto GuestRip = Record->GuestRIP;
 
-  if (!TFSet) {
-    HostCode = Thread->LookupCache->FindBlock(GuestRip);
-  }
-
-  if (TFSet || !HostCode) {
+  if (TFSet) {
     // If TF is set, the cache must be skipped as different code needs to be generated.
     Frame->State.rip = GuestRip;
     return Frame->Pointers.Common.DispatcherLoopTop;
+  } else {
+    {
+      // Guard the LookupCache lock with the code invalidation mutex, to avoid issues with forking
+      auto lk_inval = GuardSignalDeferringSection<std::shared_lock>(static_cast<Context::ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
+      HostCode = Thread->LookupCache->FindBlock(GuestRip);
+    }
+    if (!HostCode) {
+      // Hold a reference to the code buffer, to avoid linking unmapped code if compilation triggers a recreation.
+      auto CodeBuffer = static_cast<Arm64JITCore*>(Thread->CPUBackend.get())->CurrentCodeBuffer;
+      HostCode = static_cast<Context::ContextImpl*>(Thread->CTX)->CompileBlock(Frame, GuestRip, 0);
+      if (Thread->LookupCache->Shared != CodeBuffer->LookupCache.get()) {
+        return HostCode;
+      }
+    }
   }
 
-  uintptr_t branch = (uintptr_t)(Record)-8;
-  LOGMAN_THROW_A_FMT((branch % 16) == 0, "Incorrect alignment for block linking record");
+  // See ExitFunction in BranchOps.cpp for an assembly level view of the handled cases.
+  uintptr_t JumpThunkStartAddress = reinterpret_cast<uintptr_t>(Record) - 0x10;
+  uintptr_t CallerAddress = JumpThunkStartAddress + Record->CallerOffset;
+  auto BranchOffset = HostCode / 4 - CallerAddress / 4;
 
-  auto offset = HostCode / 4 - branch / 4;
-  if (ARMEmitter::Emitter::IsInt26(offset)) {
-    // This is the optimal case, where the target can be encoded in a single instruction.
-    // Atomically patch the code with a relative branch.
-    const uint32_t Patch = (0b0001'01 << 26) | (offset & ((1u << 26) - 1));
-    std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(branch)).store(Patch, std::memory_order::relaxed);
-    ARMEmitter::Emitter::ClearICache((void*)branch, 4);
+  uint32_t ExpectedKnownCallMarkerInst = 0;
+  ARMEmitter::Emitter ExpectedKnownCallMarkerEmit(reinterpret_cast<uint8_t*>(&ExpectedKnownCallMarkerInst), 4);
+  ExpectedKnownCallMarkerEmit.adr(TMP1, 0xC);
+
+  // Guard the LookupCache lock with the code invalidation mutex, to avoid issues with forking
+  auto lk_inval = GuardSignalDeferringSection<std::shared_lock>(static_cast<Context::ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
+
+  // Lock here is necessary to prevent simultaneous linking and delinking
+  auto lk = Thread->LookupCache->AcquireLock();
+
+  // For non-calls, this would extend into the block's code, however that's fine as an out-of-range adr would never
+  // be generated avoiding any false positives.
+  uintptr_t KnownCallMarkerAddr = CallerAddress - 0x8;
+  uint32_t KnownCallMarkerInst = *reinterpret_cast<uint32_t*>(KnownCallMarkerAddr);
+  if (ARMEmitter::Emitter::IsInt26(BranchOffset)) {
+    // Directly patch the callsite with the appropriate branch instruction.
+    uint32_t BranchInst = 0;
+    ARMEmitter::Emitter BranchEmit(reinterpret_cast<uint8_t*>(&BranchInst), 4);
+
+    if (KnownCallMarkerInst == ExpectedKnownCallMarkerInst) {
+      BranchEmit.bl(BranchOffset);
+      Thread->LookupCache->AddBlockLink(GuestRip, Record, [](FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
+        DirectBlockDelinker(Frame, Record, true);
+      });
+    } else {
+      BranchEmit.b(BranchOffset);
+      Thread->LookupCache->AddBlockLink(GuestRip, Record, [](FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
+        DirectBlockDelinker(Frame, Record, false);
+      });
+    }
+
+    std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(CallerAddress)).store(BranchInst, std::memory_order::relaxed);
+    ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(CallerAddress), 4);
   } else {
-    // fallback case - do a soft-er link by patching the pointer
-    std::atomic_ref<uint64_t>(Record->HostBranch).store(HostCode, std::memory_order::seq_cst);
+    // This case is common between calls and jumps as the thunk callsite can be left untouched.
+    std::atomic_ref<uint64_t>(Record->HostCode).store(HostCode, std::memory_order::seq_cst);
 #ifdef _M_ARM_64
     // Make memory write visible to other threads reading the same location
-    asm volatile("dc cvau, %0; dsb ish" : : "r"(Record->HostBranch) :);
+    asm volatile("dc cvau, %0; dsb ish" : : "r"(Record->HostCode) :);
 #endif
-  }
 
-  // Add de-linking handler
-  Thread->LookupCache->AddBlockLink(GuestRip, Record, DirectBlockDelinker);
+    uint32_t LdrInst = 0;
+    ARMEmitter::Emitter LdrEmit(reinterpret_cast<uint8_t*>(&LdrInst), 4);
+    LdrEmit.ldr(TMP1, reinterpret_cast<uint64_t>(&Record->HostCode) - JumpThunkStartAddress);
+    std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(JumpThunkStartAddress)).store(LdrInst, std::memory_order::relaxed);
+    ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(JumpThunkStartAddress), 4);
+
+    Thread->LookupCache->AddBlockLink(GuestRip, Record, IndirectBlockDelinker);
+  }
 
   return HostCode;
 }
@@ -598,7 +656,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::In
       Common.SyscallHandlerObj = reinterpret_cast<uint64_t>(CTX->SyscallHandler);
       Common.SyscallHandlerFunc = PMF.GetVTableEntry(CTX->SyscallHandler);
     }
-    Common.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Context::ContextImpl::ThreadExitFunctionLink<Arm64JITCore_ExitFunctionLink>);
+    Common.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Arm64JITCore::ExitFunctionLink);
 
     // Platform Specific
     auto& AArch64 = ThreadState->CurrentFrame->Pointers.AArch64;
@@ -741,19 +799,42 @@ void Arm64JITCore::EmitInterruptChecks(bool CheckTF) {
 #endif
 }
 
+void Arm64JITCore::EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF) {
+  // Get the address of the JITCodeHeader and store in to the core state.
+  // Two instruction cost, each 1 cycle.
+  adr(TMP1, &HeaderLabel);
+  str(TMP1, STATE, offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader));
+
+  EmitInterruptChecks(CheckTF);
+
+  if (SpillSlots) {
+    const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize;
+
+    if (ARMEmitter::IsImmAddSub(TotalSpillSlotsSize)) {
+      sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, TotalSpillSlotsSize);
+    } else {
+      LoadConstant(ARMEmitter::Size::i64Bit, TMP1, TotalSpillSlotsSize);
+      sub(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, TMP1, ARMEmitter::ExtendedType::LSL_64, 0);
+    }
+  }
+}
+
 CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
                                                    FEXCore::Core::DebugData* DebugData, bool CheckTF) {
   FEXCORE_PROFILE_SCOPED("Arm64::CompileCode");
 
   JumpTargets.clear();
+  CallReturnTargets.clear();
+  PendingJumpThunks.clear();
   uint32_t SSACount = IR->GetSSACount();
 
   this->Entry = Entry;
   this->DebugData = DebugData;
   this->IR = IR;
+  CodeData.EntryPoints.clear();
 
   // Fairly excessive buffer range to make sure we don't overflow
-  uint32_t BufferRange = 0x100 + SSACount * 24;
+  uint32_t BufferRange = 0x1000 + SSACount * 24;
 
   // JIT output is first written to a temporary buffer and later relocated to the CodeBuffer.
   // This minimizes lock contention of CodeBufferWriteMutex.
@@ -768,9 +849,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
   JITCodeHeader* CodeHeader = GetCursorAddress<JITCodeHeader*>();
   CursorIncrement(sizeof(JITCodeHeader));
 
-#ifdef VIXL_DISASSEMBLER
-  const auto DisasmBegin = GetCursorAddress<const vixl::aarch64::Instruction*>();
-#endif
+  auto CodeBegin = GetCursorAddress<uint8_t*>();
 
   // AAPCS64
   // r30      = LR
@@ -792,34 +871,15 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
   // X1-X3 = Temp
   // X4-r18 = RA
 
-  CodeData.BlockEntry = GetCursorAddress<uint8_t*>();
-
-  // Get the address of the JITCodeHeader and store in to the core state.
-  // Two instruction cost, each 1 cycle.
-  adr(TMP1, &JITCodeHeaderLabel);
-  str(TMP1, STATE, offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader));
-
-  EmitInterruptChecks(CheckTF);
-
   SpillSlots = IR->SpillSlots();
 
-  if (SpillSlots) {
-    const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize;
-
-    if (ARMEmitter::IsImmAddSub(TotalSpillSlotsSize)) {
-      sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, TotalSpillSlotsSize);
-    } else {
-      LoadConstant(ARMEmitter::Size::i64Bit, TMP1, TotalSpillSlotsSize);
-      sub(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, TMP1, ARMEmitter::ExtendedType::LSL_64, 0);
-    }
-  }
-
   PendingTargetLabel = nullptr;
+  PendingCallReturnTargetLabel = nullptr;
 
   for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {
     using namespace FEXCore::IR;
-#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
     auto BlockIROp = BlockHeader->CW<FEXCore::IR::IROp_CodeBlock>();
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
     LOGMAN_THROW_A_FMT(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
 #endif
 
@@ -831,6 +891,33 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
       // if there's a pending branch, and it is not fall-through
       if (PendingTargetLabel && PendingTargetLabel != &IsTarget->second) {
         b(PendingTargetLabel);
+        PendingTargetLabel = nullptr;
+      }
+
+      if (BlockIROp->EntryPoint) {
+        uint64_t BlockStartRIP = Entry + BlockIROp->GuestEntryOffset;
+
+        const auto IsReturnTarget = CallReturnTargets.try_emplace(Node).first;
+        if (PendingTargetLabel) {
+          // If there is a fallthrough branch to this block, skip over the entrypoint code.
+          b(&IsTarget->second);
+        } else if (PendingCallReturnTargetLabel && PendingCallReturnTargetLabel != &IsReturnTarget->second) {
+          // If we just emitted a call, but the block we're now emitting is not the return block so don't fallthrough.
+          b(PendingCallReturnTargetLabel);
+        }
+        PendingCallReturnTargetLabel = nullptr;
+
+        Bind(&IsReturnTarget->second);
+        CodeData.EntryPoints.emplace(BlockStartRIP, GetCursorAddress<uint8_t*>());
+        DebugData->GuestOpcodes.push_back({BlockIROp->GuestEntryOffset, GetCursorAddress<uint8_t*>() - CodeData.BlockBegin});
+
+        EmitEntryPoint(JITCodeHeaderLabel, CheckTF);
+      }
+
+      if (PendingCallReturnTargetLabel) {
+        // If there is still a pending call return target, then the block we're emitting is not the return block so don't fallthrough.
+        b(PendingCallReturnTargetLabel);
+        PendingCallReturnTargetLabel = nullptr;
       }
       PendingTargetLabel = nullptr;
 
@@ -852,7 +939,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
       }
     }
 
-    DebugData->Subblocks.push_back({static_cast<uint32_t>(BlockStartHostCode - CodeData.BlockEntry),
+    DebugData->Subblocks.push_back({static_cast<uint32_t>(BlockStartHostCode - CodeData.BlockBegin),
                                     static_cast<uint32_t>(GetCursorAddress<uint8_t*>() - BlockStartHostCode)});
   }
 
@@ -862,8 +949,32 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
   }
   PendingTargetLabel = nullptr;
 
-  // CodeSize not including the tail data.
-  const uint64_t CodeOnlySize = GetCursorAddress<uint8_t*>() - CodeData.BlockBegin;
+  ARMEmitter::ForwardLabel l_ExitLink;
+  for (auto& PendingJumpThunk : PendingJumpThunks) {
+    // Align as 64-bit atomics are used on the HostCode field.
+    Align(8);
+
+    ARMEmitter::ForwardLabel l_DoLink;
+    uint64_t ThunkAddress = GetCursorAddress<uint64_t>();
+    Bind(&PendingJumpThunk.Label);
+    b(&l_DoLink);
+    br(TMP1);
+    Bind(&l_DoLink);
+    ldr(TMP1, &l_ExitLink);
+    blr(TMP1);
+
+    // This is a ExitFunctionLinkData struct
+    Bind(&l_ExitLink);
+    dc64(0);                                             // HostCode
+    dc64(PendingJumpThunk.GuestRIP);                     // GuestRIP
+    dc64(PendingJumpThunk.CallerAddress - ThunkAddress); // CallerOffset
+  }
+
+  Bind(&l_ExitLink);
+  dc64(ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker);
+
+  // CodeSize not including the header or tail data.
+  const uint64_t CodeOnlySize = GetCursorAddress<uint8_t*>() - CodeBegin;
 
   // Add the JitCodeTail
   Align(alignof(JITCodeTail));
@@ -942,6 +1053,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
       LOGMAN_THROW_A_FMT(CurrentCodeBuffer->LookupCache.get() == ThreadState->LookupCache->Shared, "INVARIANT VIOLATED: SharedLookupCache "
                                                                                                    "doesn't match up!\n");
       if (auto Prev = CheckCodeBufferUpdate()) {
+        Allocator::VirtualDontNeed(ThreadState->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);
         ThreadState->LookupCache->ChangeGuestToHostMapping(*Prev, *CurrentCodeBuffer->LookupCache);
       }
 
@@ -960,7 +1072,10 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
     // Adjust host addresses
     const auto Delta = GetCursorAddress<uint8_t*>() - CodeData.BlockBegin;
     CodeData.BlockBegin += Delta;
-    CodeData.BlockEntry += Delta;
+    for (auto& EntryPoint : CodeData.EntryPoints) {
+      EntryPoint.second += Delta;
+    }
+    CodeBegin += Delta;
 
     // Copy over CodeBuffer contents
     memcpy(GetCursorAddress<uint8_t*>(), TempCodeBuffer, TempSize);
@@ -971,7 +1086,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
 
   TempAllocator.DelayedDisownBuffer();
 
-  ClearICache(CodeData.BlockBegin, CodeOnlySize);
+  ClearICache(CodeBegin, CodeOnlySize);
 
 #ifdef VIXL_DISASSEMBLER
   if (Disassemble() & FEXCore::Config::Disassemble::STATS) {
@@ -985,7 +1100,8 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
   }
 
   if (Disassemble() & FEXCore::Config::Disassemble::BLOCKS) {
-    const auto DisasmEnd = reinterpret_cast<const vixl::aarch64::Instruction*>(JITBlockTailLocation);
+    const auto DisasmBegin = reinterpret_cast<const vixl::aarch64::Instruction*>(CodeBegin);
+    const auto DisasmEnd = reinterpret_cast<const vixl::aarch64::Instruction*>(CodeBegin + CodeOnlySize);
     LogMan::Msg::IFmt("Disassemble Begin");
     for (auto PCToDecode = DisasmBegin; PCToDecode < DisasmEnd; PCToDecode += 4) {
       DisasmDecoder->Decode(PCToDecode);
@@ -1001,7 +1117,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
 
   this->IR = nullptr;
 
-  return CodeData;
+  return std::move(CodeData);
 }
 
 void Arm64JITCore::ResetStack() {

FEXCore/Source/Interface/Core/JIT/JITClass.h

@@ -31,6 +31,10 @@ namespace FEXCore::Core {
 struct InternalThreadState;
 }
 
+namespace FEXCore::Context {
+struct ExitFunctionLinkData;
+}
+
 namespace FEXCore::CPU {
 class Arm64JITCore final : public CPUBackend, public Arm64Emitter {
 public:
@@ -57,15 +61,26 @@ private:
   const bool HostSupportsAFP {};
 
   ARMEmitter::BiDirectionalLabel* PendingTargetLabel {};
+  ARMEmitter::BiDirectionalLabel* PendingCallReturnTargetLabel {};
   FEXCore::Context::ContextImpl* CTX {};
   const FEXCore::IR::IRListView* IR {};
   uint64_t Entry {};
   CPUBackend::CompiledCode CodeData {};
 
   fextl::map<IR::NodeID, ARMEmitter::BiDirectionalLabel> JumpTargets;
+  fextl::map<IR::NodeID, ARMEmitter::BiDirectionalLabel> CallReturnTargets;
+
+  struct PendingJumpThunk {
+    uint64_t CallerAddress;
+    uint64_t GuestRIP;
+    ARMEmitter::ForwardLabel Label;
+  };
+  fextl::vector<PendingJumpThunk> PendingJumpThunks;
 
   Utils::PoolBufferWithTimedRetirement<uint8_t*, 5000, 500> TempAllocator;
 
+  static uint64_t ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record);
+
   [[nodiscard]]
   ARMEmitter::Register GetReg(IR::PhysicalRegister Reg) const {
     LOGMAN_THROW_A_FMT(Reg.Class == IR::GPRFixedClass.Val || Reg.Class == IR::GPRClass.Val, "Unexpected Class: {}", Reg.Class);
@@ -290,6 +305,17 @@ private:
     uint32_t End;
   };
 
+  void EmitLinkedBranch(uint64_t GuestRIP, bool Call) {
+    PendingJumpThunks.push_back({GetCursorAddress<uint64_t>(), GuestRIP, {}});
+    auto& Thunk = PendingJumpThunks.back();
+    Bind(&Thunk.Label);
+    if (Call) {
+      bl(&Thunk.Label);
+    } else {
+      b(&Thunk.Label);
+    }
+  }
+
   // This is purely a debugging aid for developers to see if they are in JIT code space when inspecting raw memory
   void EmitDetectionString();
   IR::RegisterAllocationPass* RAPass {};
@@ -375,6 +401,8 @@ private:
 
   void EmitInterruptChecks(bool CheckTF);
 
+  void EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF);
+
   // Runtime selection;
   // Load and store TSO memory style
   OpType RT_LoadMemTSO;

FEXCore/Source/Interface/Core/JIT/MiscOps.cpp

@@ -17,6 +17,26 @@ $end_info$
 
 namespace FEXCore::CPU {
 
+DEF_OP(WFET) {
+  auto Op = IROp->C<IR::IROp_WFET>();
+  const auto Lower = GetReg(Op->Lower);
+  const auto Upper = GetReg(Op->Upper);
+
+  // Combine registers.
+  mov(ARMEmitter::Size::i64Bit, TMP1, Lower);
+  bfi(ARMEmitter::Size::i64Bit, TMP1, Upper, 32, 32);
+  if (CTX->Config.TSCScale) {
+    // Scale back to ARM64 TSC scale if necessary
+    lsr(ARMEmitter::Size::i64Bit, TMP1, TMP1, CTX->Config.TSCScale);
+  }
+
+  // Clear the exclusive monitor so it can't spuriously wake up with that event.
+  clrex();
+
+  // Execute wfet to wait until the TSC.
+  wfet(TMP1);
+}
+
 DEF_OP(GuestOpcode) {
   auto Op = IROp->C<IR::IROp_GuestOpcode>();
   // metadata

FEXCore/Source/Interface/Core/JIT/VectorOps.cpp

@@ -193,29 +193,29 @@ namespace FEXCore::CPU {
     VFScalarOperation(IROp->Size, ElementSize, Op->ZeroUpperBits, ScalarEmit, Dst, Vector1, Vector2);                         \
   }
 
-#define DEF_FMAOP_SCALAR_INSERT(FEXOp, ARMOp)                                                                                              \
-  DEF_OP(FEXOp) {                                                                                                                          \
-    const auto Op = IROp->C<IR::IROp_##FEXOp>();                                                                                           \
-    const auto ElementSize = Op->Header.ElementSize;                                                                                       \
-                                                                                                                                           \
-    auto ScalarEmit =                                                                                                                      \
-      [this, ElementSize](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2, ARMEmitter::VRegister Src3) { \
-      if (ElementSize == IR::OpSize::i16Bit) {                                                                                             \
-        ARMOp(Dst.H(), Src1.H(), Src2.H(), Src3.H());                                                                                      \
-      } else if (ElementSize == IR::OpSize::i32Bit) {                                                                                      \
-        ARMOp(Dst.S(), Src1.S(), Src2.S(), Src3.S());                                                                                      \
-      } else if (ElementSize == IR::OpSize::i64Bit) {                                                                                      \
-        ARMOp(Dst.D(), Src1.D(), Src2.D(), Src3.D());                                                                                      \
-      }                                                                                                                                    \
-    };                                                                                                                                     \
-                                                                                                                                           \
-    const auto Dst = GetVReg(Node);                                                                                                        \
-    const auto Upper = GetVReg(Op->Upper);                                                                                                 \
-    const auto Vector1 = GetVReg(Op->Vector1);                                                                                             \
-    const auto Vector2 = GetVReg(Op->Vector2);                                                                                             \
-    const auto Addend = GetVReg(Op->Addend);                                                                                               \
-                                                                                                                                           \
-    VFScalarFMAOperation(IROp->Size, ElementSize, ScalarEmit, Dst, Upper, Vector1, Vector2, Addend);                                       \
+#define DEF_FMAOP_SCALAR_INSERT(FEXOp, ARMOp)                                                                                \
+  DEF_OP(FEXOp) {                                                                                                            \
+    const auto Op = IROp->C<IR::IROp_##FEXOp>();                                                                             \
+    const auto ElementSize = Op->Header.ElementSize;                                                                         \
+                                                                                                                             \
+    auto ScalarEmit = [this, ElementSize](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2, \
+                                          ARMEmitter::VRegister Src3) {                                                      \
+      if (ElementSize == IR::OpSize::i16Bit) {                                                                               \
+        ARMOp(Dst.H(), Src1.H(), Src2.H(), Src3.H());                                                                        \
+      } else if (ElementSize == IR::OpSize::i32Bit) {                                                                        \
+        ARMOp(Dst.S(), Src1.S(), Src2.S(), Src3.S());                                                                        \
+      } else if (ElementSize == IR::OpSize::i64Bit) {                                                                        \
+        ARMOp(Dst.D(), Src1.D(), Src2.D(), Src3.D());                                                                        \
+      }                                                                                                                      \
+    };                                                                                                                       \
+                                                                                                                             \
+    const auto Dst = GetVReg(Node);                                                                                          \
+    const auto Upper = GetVReg(Op->Upper);                                                                                   \
+    const auto Vector1 = GetVReg(Op->Vector1);                                                                               \
+    const auto Vector2 = GetVReg(Op->Vector2);                                                                               \
+    const auto Addend = GetVReg(Op->Addend);                                                                                 \
+                                                                                                                             \
+    VFScalarFMAOperation(IROp->Size, ElementSize, ScalarEmit, Dst, Upper, Vector1, Vector2, Addend);                         \
   }
 
 DEF_UNOP(VAbs, abs, true)
@@ -803,8 +803,8 @@ DEF_OP(VFCMPScalarInsert) {
     default: break;
     }
   };
-  auto ScalarEmitUNO =
-    [this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2) {
+  auto ScalarEmitUNO = [this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1,
+                                                                   ARMEmitter::VRegister Src2) {
     switch (SubRegSize.Scalar) {
     case ARMEmitter::ScalarRegSize::i16Bit: {
       fcmge(VTMP1.H(), Src1.H(), Src2.H());
@@ -838,8 +838,8 @@ DEF_OP(VFCMPScalarInsert) {
       }
     }
   };
-  auto ScalarEmitNEQ =
-    [this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2) {
+  auto ScalarEmitNEQ = [this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1,
+                                                                   ARMEmitter::VRegister Src2) {
     switch (SubRegSize.Scalar) {
     case ARMEmitter::ScalarRegSize::i16Bit: {
       fcmeq(VTMP1.H(), Src2.H(), Src1.H());
@@ -868,8 +868,8 @@ DEF_OP(VFCMPScalarInsert) {
       }
     }
   };
-  auto ScalarEmitORD =
-    [this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2) {
+  auto ScalarEmitORD = [this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1,
+                                                                   ARMEmitter::VRegister Src2) {
     switch (SubRegSize.Scalar) {
     case ARMEmitter::ScalarRegSize::i16Bit: {
       fcmge(VTMP1.H(), Src1.H(), Src2.H());
@@ -1115,7 +1115,7 @@ DEF_OP(VAddP) {
 }
 
 DEF_OP(VFAddV) {
-  const auto Op = IROp->C<IR::IROp_VAddV>();
+  const auto Op = IROp->C<IR::IROp_VFAddV>();
   const auto OpSize = IROp->Size;
   const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
   LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);

FEXCore/Source/Interface/Core/LookupCache.h

@@ -56,6 +56,8 @@ struct GuestToHostMap {
 
   fextl::robin_map<uint64_t, uint64_t> BlockList;
 
+  fextl::map<uint64_t, fextl::vector<uint64_t>> CodePages;
+
   GuestToHostMap();
 
   // Adds to Guest -> Host code mapping
@@ -76,7 +78,7 @@ struct GuestToHostMap {
     return HostCode->second;
   }
 
-  void Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address, const LockToken&) {
+  bool Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address, const LockToken&) {
     // Sever any links to this block
     auto lower = BlockLinks->lower_bound({Address, nullptr});
     auto upper = BlockLinks->upper_bound({Address, reinterpret_cast<FEXCore::Context::ExitFunctionLinkData*>(UINTPTR_MAX)});
@@ -85,7 +87,7 @@ struct GuestToHostMap {
     }
 
     // Remove from BlockList
-    BlockList.erase(Address);
+    return BlockList.erase(Address) != 0;
   }
 
   void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink,
@@ -93,6 +95,18 @@ struct GuestToHostMap {
     BlockLinks->insert({{GuestDestination, HostLink}, delinker});
   }
 
+  bool AddBlockExecutableRange(const fextl::set<uint64_t>& Addresses, uint64_t Start, uint64_t Length, const LockToken&) {
+    bool rv = false;
+
+    for (auto CurrentPage = Start >> 12, EndPage = (Start + Length - 1) >> 12; CurrentPage <= EndPage; CurrentPage++) {
+      auto& CodePage = CodePages[CurrentPage];
+      rv |= CodePage.empty();
+      CodePage.insert(CodePage.end(), Addresses.begin(), Addresses.end());
+    }
+
+    return rv;
+  }
+
   void ClearCache(const LockToken&);
 };
 
@@ -155,22 +169,11 @@ public:
 
   GuestToHostMap* Shared = nullptr;
 
-  fextl::map<uint64_t, fextl::vector<uint64_t>> CodePages;
-
-  // Appends Block {Address} to CodePages [Start, Start + Length)
+  // Appends a list of Block {Address} to CodePages [Start, Start + Length)
   // Returns true if new pages are marked as containing code
-  bool AddBlockExecutableRange(uint64_t Address, uint64_t Start, uint64_t Length) {
+  bool AddBlockExecutableRange(const fextl::set<uint64_t>& Addresses, uint64_t Start, uint64_t Length) {
     auto lk = Shared->AcquireLock();
-
-    bool rv = false;
-
-    for (auto CurrentPage = Start >> 12, EndPage = (Start + Length - 1) >> 12; CurrentPage <= EndPage; CurrentPage++) {
-      auto& CodePage = CodePages[CurrentPage];
-      rv |= CodePage.empty();
-      CodePage.push_back(Address);
-    }
-
-    return rv;
+    return Shared->AddBlockExecutableRange(Addresses, Start, Length, lk);
   }
 
   // Adds to Guest -> Host code mapping
@@ -189,15 +192,16 @@ public:
   // NOTE: It's the caller's responsibility to call Erase() for all other
   //       GuestToHostMaps that share the same LookupCache. Otherwise, the
   //       L1/L2 caches will contain stale references to deallocated memory.
-  void Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address) {
+  bool Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address) {
     auto lk = Shared->AcquireLock();
 
-    Shared->Erase(Frame, Address, lk);
+    bool ErasedAny = Shared->Erase(Frame, Address, lk);
 
     // Do L1
     auto& L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
     if (L1Entry.GuestCode == Address) {
       L1Entry.GuestCode = 0;
+      ErasedAny = true;
       // Leave L1Entry.HostCode as is, so that concurrent lookups won't read a null pointer
       // This is a soft guarantee for cross thread invalidation, as atomics are not used
       // and it hasn't been thoroughly tested
@@ -212,13 +216,14 @@ public:
     uint64_t LocalPagePointer = Pointers[Address];
     if (!LocalPagePointer) {
       // Page for this code didn't even exist, nothing to do
-      return;
+      return ErasedAny;
     }
 
     // Page exists, just set the offset to zero
     auto BlockPointers = reinterpret_cast<LookupCacheEntry*>(LocalPagePointer);
     BlockPointers[PageOffset].GuestCode = 0;
     BlockPointers[PageOffset].HostCode = 0;
+    return true;
   }
 
   void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink, const FEXCore::Context::BlockDelinkerFunc& delinker) {

FEXCore/Source/Interface/Core/OpcodeDispatcher.h

@@ -80,6 +80,12 @@ struct LoadSourceOptions {
   bool AllowUpperGarbage = false;
 };
 
+struct DispatchTableEntry {
+  uint16_t Op;
+  uint8_t Count;
+  X86Tables::OpDispatchPtr Ptr;
+};
+
 class OpDispatchBuilder final : public IREmitter {
   friend class FEXCore::IR::Pass;
   friend class FEXCore::IR::PassManager;
@@ -160,9 +166,13 @@ public:
     auto InlineConst = _InlineConstant(Bit);
     return _CondJump(Src, InlineConst, InvalidNode, InvalidNode, {Set ? COND_TSTNZ : COND_TSTZ}, OpSize::iInvalid, false);
   }
-  IRPair<IROp_ExitFunction> ExitFunction(Ref NewRIP) {
+  IRPair<IROp_ExitFunction> ExitFunction(Ref NewRIP, BranchHint Hint = BranchHint::None) {
     FlushRegisterCache();
-    return _ExitFunction(GetOpSize(NewRIP), NewRIP);
+    return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, InvalidNode, InvalidNode);
+  }
+  IRPair<IROp_ExitFunction> ExitFunction(Ref NewRIP, BranchHint Hint, Ref CallReturnAddress, Ref CallReturnBlock) {
+    FlushRegisterCache();
+    return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, CallReturnAddress, CallReturnBlock);
   }
   IRPair<IROp_Break> Break(BreakDefinition Reason) {
     FlushRegisterCache();
@@ -188,7 +198,7 @@ public:
       auto it = JumpTargets.find(NextRIP);
       if (it == JumpTargets.end()) {
 
-        const auto GPRSize = CTX->GetGPROpSize();
+        const auto GPRSize = GetGPROpSize();
         // If we don't have a jump target to a new block then we have to leave
         // Set the RIP to the next instruction and leave
         auto RelocatedNextRIP = _EntrypointOffset(GPRSize, NextRIP - Entry);
@@ -232,7 +242,7 @@ public:
   template<typename F>
   void ForeachDirection(F&& Routine) {
     // Otherwise, prepare to branch.
-    auto Zero = _Constant(0);
+    auto Zero = Constant(0);
 
     // If the shift is zero, do not touch the flags.
     auto ForwardBlock = CreateNewCodeBlockAfter(GetCurrentBlock());
@@ -290,7 +300,7 @@ public:
     return ShouldDump;
   }
 
-  void BeginFunction(uint64_t RIP, const fextl::vector<FEXCore::Frontend::Decoder::DecodedBlocks>* Blocks, uint32_t NumInstructions);
+  void BeginFunction(uint64_t RIP, const fextl::vector<FEXCore::Frontend::Decoder::DecodedBlocks>* Blocks, uint32_t NumInstructions, bool Is64BitMode);
   void Finalize();
 
   // Dispatch builder functions
@@ -702,7 +712,7 @@ public:
   Ref ReconstructX87StateFromFSW_Helper(Ref FSW);
   void FLD(OpcodeArgs, IR::OpSize Width);
   void FLDFromStack(OpcodeArgs);
-  void FLD_Const(OpcodeArgs, NamedVectorConstant Constant);
+  void FLD_Const(OpcodeArgs, NamedVectorConstant K);
 
   void FBLD(OpcodeArgs);
   void FBSTP(OpcodeArgs);
@@ -824,11 +834,13 @@ public:
   void PHADDS(OpcodeArgs);
   void PHSUBS(OpcodeArgs);
 
-  void CLWB(OpcodeArgs);
+  void CLWBOrTPause(OpcodeArgs);
   void CLFLUSHOPT(OpcodeArgs);
   void LoadFenceOrXRSTOR(OpcodeArgs);
   void MemFenceOrXSAVEOPT(OpcodeArgs);
   void StoreFenceOrCLFlush(OpcodeArgs);
+  void UMonitorOrCLRSSBSY(OpcodeArgs);
+  void UMWaitOp(OpcodeArgs);
   void CLZeroOp(OpcodeArgs);
   void RDTSCPOp(OpcodeArgs);
   void RDPIDOp(OpcodeArgs);
@@ -1158,6 +1170,7 @@ public:
   // End of AVX 256-bit implementation
 
   void InvalidOp(OpcodeArgs);
+  void NoExecOp(OpcodeArgs);
 
   void SetPackedRFLAG(bool Lower8, Ref Src);
   Ref GetPackedRFLAG(uint32_t FlagsMask = ~0U);
@@ -1184,7 +1197,7 @@ public:
 
     CalculateDeferredFlags();
 
-    const auto GPRSize = CTX->GetGPROpSize();
+    const auto GPRSize = GetGPROpSize();
     const auto VectorSize = GetGuestVectorLength();
 
     // Write backwards. This is a heuristic to improve coalescing, since we
@@ -1244,7 +1257,7 @@ public:
           _StoreContext(Size, Class, Value, Offset);
           // If Partial and MMX register, then we need to store all 1s in bits 64-80
           if (Partial && Index >= MM0Index && Index <= MM7Index) {
-            _StoreContext(OpSize::i16Bit, IR::GPRClass, _Constant(0xFFFF), Offset + 8);
+            _StoreContext(OpSize::i16Bit, IR::GPRClass, Constant(0xFFFF), Offset + 8);
           }
         }
       }
@@ -1257,6 +1270,10 @@ public:
     RegCache.Partial &= ~Mask;
   }
 
+  IR::OpSize GetGPROpSize() const {
+    return Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;
+  }
+
 protected:
   void RecordX87Use() override {
     CurrentHeader->HasX87 = true;
@@ -1310,6 +1327,7 @@ private:
   struct JumpTargetInfo {
     Ref BlockEntry;
     bool HaveEmitted;
+    bool IsEntryPoint;
   };
 
   FEXCore::Context::ContextImpl* CTX {};
@@ -1495,7 +1513,7 @@ private:
 #undef OpcodeArgs
 
   Ref AppendSegmentOffset(Ref Value, uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false);
-  Ref GetSegment(uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false);
+  Ref GetSegment(uint32_t Flags, uint32_t DefaultPrefix = FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX, bool Override = false);
 
   void UpdatePrefixFromSegment(Ref Segment, uint32_t SegmentReg);
 
@@ -1505,12 +1523,14 @@ private:
 
   Ref GetRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset = 0);
 
-  bool IsOperandMem(const X86Tables::DecodedOperand& Operand, bool Load) {
+  [[nodiscard]]
+  static bool IsOperandMem(const X86Tables::DecodedOperand& Operand, bool Load) {
     // Literals are immediates as sources but memory addresses as destinations.
     return !(Load && Operand.IsLiteral()) && !Operand.IsGPR();
   }
 
-  bool IsNonTSOReg(MemoryAccessType Access, uint8_t Reg) {
+  [[nodiscard]]
+  static bool IsNonTSOReg(MemoryAccessType Access, uint8_t Reg) {
     return Access == MemoryAccessType::DEFAULT && Reg == X86State::REG_RSP;
   }
 
@@ -1617,7 +1637,7 @@ private:
   }
 
   void ZeroNZCV() {
-    CachedNZCV = _Constant(0);
+    CachedNZCV = Constant(0);
     NZCVDirty = true;
   }
 
@@ -1630,9 +1650,9 @@ private:
     // This is currently worse for 8/16-bit, but that should be optimized. TODO
     if (SrcSize >= OpSize::i32Bit) {
       if (SetPF) {
-        CalculatePF(_SubWithFlags(SrcSize, Res, _Constant(0)));
+        CalculatePF(_SubWithFlags(SrcSize, Res, Constant(0)));
       } else {
-        _SubNZCV(SrcSize, Res, _Constant(0));
+        _SubNZCV(SrcSize, Res, Constant(0));
       }
 
       CFInverted = true;
@@ -1703,7 +1723,7 @@ private:
       } else {
         // Invert as a GPR
         unsigned Bit = IndexNZCV(FEXCore::X86State::RFLAG_CF_RAW_LOC);
-        SetNZCV(_Xor(OpSize::i32Bit, GetNZCV(), _Constant(1u << Bit)));
+        SetNZCV(_Xor(OpSize::i32Bit, GetNZCV(), Constant(1u << Bit)));
         CalculateDeferredFlags();
       }
 
@@ -1741,7 +1761,7 @@ private:
     }
 
     HandleNZCVWrite();
-    _SubNZCV(OpSize::i32Bit, _Constant(0), Value);
+    _SubNZCV(OpSize::i32Bit, Constant(0), Value);
     CFInverted = true;
   }
 
@@ -1766,25 +1786,25 @@ private:
       StoreRegister(Core::CPUState::AF_AS_GREG, false, Value);
     } else if (BitOffset == FEXCore::X86State::RFLAG_DF_RAW_LOC) {
       // For DF, we need to transform 0/1 into 1/-1
-      StoreDF(_SubShift(OpSize::i64Bit, _Constant(1), Value, ShiftType::LSL, 1));
+      StoreDF(_SubShift(OpSize::i64Bit, Constant(1), Value, ShiftType::LSL, 1));
     } else if (BitOffset == FEXCore::X86State::RFLAG_TF_RAW_LOC) {
       auto PackedTF = _LoadContext(OpSize::i8Bit, GPRClass, offsetof(FEXCore::Core::CPUState, flags[BitOffset]));
       // An exception should still be raised after an instruction that unsets TF, leave the unblocked bit set but unset
       // the TF bit to cause such behaviour. The handling code at the start of the next block will then unset the
       // unblocked bit before raising the exception.
-      auto NewPackedTF = _Select(FEXCore::IR::COND_EQ, Value, _Constant(0), _And(OpSize::i32Bit, PackedTF, _Constant(~1)), _Constant(1));
+      auto NewPackedTF = _Select(FEXCore::IR::COND_EQ, Value, Constant(0), _And(OpSize::i32Bit, PackedTF, Constant(~1)), Constant(1));
       _StoreContext(OpSize::i8Bit, GPRClass, NewPackedTF, offsetof(FEXCore::Core::CPUState, flags[BitOffset]));
     } else {
       _StoreContext(OpSize::i8Bit, GPRClass, Value, offsetof(FEXCore::Core::CPUState, flags[BitOffset]));
     }
   }
 
-  void SetAF(unsigned Constant) {
+  void SetAF(unsigned K) {
     // AF is stored in bit 4 of the AF flag byte, with garbage in the other
     // bits. This allows us to defer the extract in the usual case. When it is
     // read, bit 4 is extracted.  In order to write a constant value of AF, that
     // means we need to left-shift here to compensate.
-    SetRFLAG<FEXCore::X86State::RFLAG_AF_RAW_LOC>(_Constant(Constant << 4));
+    SetRFLAG<FEXCore::X86State::RFLAG_AF_RAW_LOC>(Constant(K << 4));
   }
 
   void ZeroPF_AF();
@@ -1800,7 +1820,8 @@ private:
     InvalidateReg(Core::CPUState::AF_AS_GREG);
   }
 
-  CondClassType CondForNZCVBit(unsigned BitOffset, bool Invert) {
+  [[nodiscard]]
+  static CondClassType CondForNZCVBit(unsigned BitOffset, bool Invert) {
     switch (BitOffset) {
     case X86State::RFLAG_SF_RAW_LOC: return {Invert ? COND_PL : COND_MI};
     case X86State::RFLAG_ZF_RAW_LOC: return {Invert ? COND_NEQ : COND_EQ};
@@ -1826,7 +1847,8 @@ private:
   static const int AVXHigh0Index = 48;
   static const int AVXHigh15Index = 63;
 
-  uint32_t CacheIndexToContextOffset(int Index) {
+  [[nodiscard]]
+  static uint32_t CacheIndexToContextOffset(int Index) {
     switch (Index) {
     case MM0Index ... MM7Index: return offsetof(FEXCore::Core::CPUState, mm[Index - MM0Index]);
     case AVXHigh0Index ... AVXHigh15Index: return offsetof(FEXCore::Core::CPUState, avx_high[Index - AVXHigh0Index][0]);
@@ -1835,7 +1857,8 @@ private:
     }
   }
 
-  RegisterClassType CacheIndexClass(int Index) {
+  [[nodiscard]]
+  static RegisterClassType CacheIndexClass(int Index) {
     if ((Index >= MM0Index && Index <= MM7Index) || Index >= FPR0Index) {
       return FPRClass;
     } else {
@@ -1843,7 +1866,8 @@ private:
     }
   }
 
-  IR::OpSize CacheIndexToOpSize(int Index) {
+  [[nodiscard]]
+  static IR::OpSize CacheIndexToOpSize(int Index) {
     // MMX registers are rounded up to 128-bit since they are shared with 80-bit
     // x87 registers, even though MMX is logically only 64-bit.
     if (Index >= AVXHigh0Index || ((Index >= MM0Index && Index <= MM7Index))) {
@@ -1951,7 +1975,7 @@ private:
   }
 
   Ref LoadGPR(uint8_t Reg) {
-    return LoadRegCache(Reg, GPR0Index + Reg, GPRClass, CTX->GetGPROpSize());
+    return LoadRegCache(Reg, GPR0Index + Reg, GPRClass, GetGPROpSize());
   }
 
   Ref LoadContext(IR::OpSize Size, uint8_t Index) {
@@ -2005,14 +2029,14 @@ private:
         auto Value = _Bfe(OpSize::i32Bit, 1, IndexNZCV(BitOffset), GetNZCV());
 
         if (Invert) {
-          return _Xor(OpSize::i32Bit, Value, _Constant(1));
+          return _Xor(OpSize::i32Bit, Value, Constant(1));
         } else {
           return Value;
         }
       } else {
         // Because we explicitly inverted for CF above, we use the unsafe
         // _NZCVSelect rather than the safe CF-aware version.
-        return _NZCVSelect(OpSize::i32Bit, CondForNZCVBit(BitOffset, Invert), _Constant(1), _Constant(0));
+        return _NZCVSelect(OpSize::i32Bit, CondForNZCVBit(BitOffset, Invert), Constant(1), Constant(0));
       }
     } else if (BitOffset == FEXCore::X86State::RFLAG_PF_RAW_LOC) {
       return LoadGPR(Core::CPUState::PF_AS_GREG);
@@ -2020,7 +2044,7 @@ private:
       return LoadGPR(Core::CPUState::AF_AS_GREG);
     } else if (BitOffset == FEXCore::X86State::RFLAG_DF_RAW_LOC) {
       // Recover the sign bit, it is the logical DF value
-      return _Lshr(OpSize::i64Bit, LoadDF(), _Constant(63));
+      return _Lshr(OpSize::i64Bit, LoadDF(), Constant(63));
     } else {
       return _LoadContext(OpSize::i8Bit, GPRClass, offsetof(Core::CPUState, flags[BitOffset]));
     }
@@ -2092,7 +2116,7 @@ private:
       // Zero AF. Note that the comparison sets the raw PF to 0/1 above, so
       // PF[4] is 0 so the XOR with PF will have no effect, so setting the AF
       // byte to zero will indeed zero AF as intended.
-      SetRFLAG<FEXCore::X86State::RFLAG_AF_RAW_LOC>(_Constant(0));
+      SetRFLAG<FEXCore::X86State::RFLAG_AF_RAW_LOC>(Constant(0));
     }
 
     // Convert NZCV from the Arm representation to an eXternal representation
@@ -2109,7 +2133,7 @@ private:
   void ConvertNZCVToX87() {
     LOGMAN_THROW_A_FMT(NZCVDirty && CachedNZCV, "NZCV must be saved");
 
-    Ref V = _NZCVSelect(OpSize::i32Bit, CondForNZCVBit(FEXCore::X86State::RFLAG_OF_RAW_LOC, false), _Constant(1), _Constant(0));
+    Ref V = _NZCVSelect(OpSize::i32Bit, CondForNZCVBit(FEXCore::X86State::RFLAG_OF_RAW_LOC, false), Constant(1), Constant(0));
 
     if (CTX->HostFeatures.SupportsFlagM2) {
       // Convert to x86 flags, saves us from or'ing after.
@@ -2117,8 +2141,8 @@ private:
     }
 
     // CF is inverted after FCMP
-    Ref C = _NZCVSelect(OpSize::i32Bit, CondForNZCVBit(FEXCore::X86State::RFLAG_CF_RAW_LOC, true), _Constant(1), _Constant(0));
-    Ref Z = _NZCVSelect(OpSize::i32Bit, CondForNZCVBit(FEXCore::X86State::RFLAG_ZF_RAW_LOC, false), _Constant(1), _Constant(0));
+    Ref C = _NZCVSelect(OpSize::i32Bit, CondForNZCVBit(FEXCore::X86State::RFLAG_CF_RAW_LOC, true), Constant(1), Constant(0));
+    Ref Z = _NZCVSelect(OpSize::i32Bit, CondForNZCVBit(FEXCore::X86State::RFLAG_ZF_RAW_LOC, false), Constant(1), Constant(0));
 
     if (!CTX->HostFeatures.SupportsFlagM2) {
       C = _Or(OpSize::i32Bit, C, V);
@@ -2126,7 +2150,7 @@ private:
     }
 
     SetRFLAG<FEXCore::X86State::X87FLAG_C0_LOC>(C);
-    SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(_Constant(0));
+    SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Constant(0));
     SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(V);
     SetRFLAG<FEXCore::X86State::X87FLAG_C3_LOC>(Z);
   }
@@ -2176,9 +2200,9 @@ private:
       return CachedNamedVectorConstants[NamedConstant][log2_size_bytes];
     }
 
-    auto Constant = _LoadNamedVectorConstant(Size, NamedConstant);
-    CachedNamedVectorConstants[NamedConstant][log2_size_bytes] = Constant;
-    return Constant;
+    auto K = _LoadNamedVectorConstant(Size, NamedConstant);
+    CachedNamedVectorConstants[NamedConstant][log2_size_bytes] = K;
+    return K;
   }
   Ref LoadAndCacheIndexedNamedVectorConstant(IR::OpSize Size, FEXCore::IR::IndexNamedVectorConstant NamedIndexedConstant, uint32_t Index) {
     IndexNamedVectorMapKey Key {
@@ -2192,9 +2216,9 @@ private:
       return it->second;
     }
 
-    auto Constant = _LoadNamedVectorIndexedConstant(Size, NamedIndexedConstant, Index);
-    CachedIndexedNamedVectorConstants.insert_or_assign(Key, Constant);
-    return Constant;
+    auto K = _LoadNamedVectorIndexedConstant(Size, NamedIndexedConstant, Index);
+    CachedIndexedNamedVectorConstants.insert_or_assign(Key, K);
+    return K;
   }
 
   Ref LoadUncachedZeroVector(IR::OpSize Size) {
@@ -2212,7 +2236,7 @@ private:
     CachedIndexedNamedVectorConstants.clear();
   }
 
-  std::pair<bool, CondClassType> DecodeNZCVCondition(uint8_t OP);
+  std::optional<CondClassType> DecodeNZCVCondition(uint8_t OP);
   Ref SelectBit(Ref Cmp, IR::OpSize ResultSize, Ref TrueValue, Ref FalseValue);
   Ref SelectCC(uint8_t OP, IR::OpSize ResultSize, Ref TrueValue, Ref FalseValue);
 
@@ -2249,7 +2273,7 @@ private:
     }
 
     // Otherwise, prepare to branch.
-    auto Zero = _Constant(0);
+    auto Zero = Constant(0);
 
     // If the shift is zero, do not touch the flags.
     auto SetBlock = CreateNewCodeBlockAfter(GetCurrentBlock());
@@ -2325,8 +2349,8 @@ private:
   void ChgStateX87_MMX() override {
     LOGMAN_THROW_A_FMT(MMXState == MMXState_X87, "Expected state to be x87");
     _StackForceSlow();
-    SetX87Top(_Constant(0));                             // top reset to zero
-    StoreContext(AbridgedFTWIndex, _Constant(0xFFFFUL)); // all valid
+    SetX87Top(Constant(0));                             // top reset to zero
+    StoreContext(AbridgedFTWIndex, Constant(0xFFFFUL)); // all valid
     MMXState = MMXState_MMX;
   }
 
@@ -2351,10 +2375,12 @@ private:
   bool BlockSetRIP {false};
 
   bool Multiblock {};
+  bool Is64BitMode {};
   uint64_t Entry {};
   IROp_IRHeader* CurrentHeader {};
 
-  bool IsTSOEnabled(FEXCore::IR::RegisterClassType Class) {
+  [[nodiscard]]
+  bool IsTSOEnabled(FEXCore::IR::RegisterClassType Class) const {
     if (ForceTSO == ForceTSOMode::ForceEnabled) {
       return true;
     } else if (ForceTSO == ForceTSOMode::ForceDisabled) {
@@ -2384,7 +2410,7 @@ private:
 
   Ref _LoadMemAutoTSO(FEXCore::IR::RegisterClassType Class, IR::OpSize Size, AddressMode A, IR::OpSize Align = IR::OpSize::i8Bit) {
     bool AtomicTSO = IsTSOEnabled(Class) && !A.NonTSO;
-    A = SelectAddressMode(this, A, CTX->GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, AtomicTSO, Class != GPRClass, Size);
+    A = SelectAddressMode(this, A, GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, AtomicTSO, Class != GPRClass, Size);
 
     if (AtomicTSO) {
       return _LoadMemTSO(Class, Size, A.Base, A.Index, Align, A.IndexType, A.IndexScale);
@@ -2404,7 +2430,7 @@ private:
       A.Offset = 0;
     }
 
-    Out.Base = LoadEffectiveAddress(this, A, CTX->GetGPROpSize(), true, false);
+    Out.Base = LoadEffectiveAddress(this, A, GetGPROpSize(), true, false);
     return Out;
   }
 
@@ -2435,7 +2461,7 @@ private:
 
   Ref _StoreMemAutoTSO(FEXCore::IR::RegisterClassType Class, IR::OpSize Size, AddressMode A, Ref Value, IR::OpSize Align = IR::OpSize::i8Bit) {
     bool AtomicTSO = IsTSOEnabled(Class) && !A.NonTSO;
-    A = SelectAddressMode(this, A, CTX->GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, AtomicTSO, Class != GPRClass, Size);
+    A = SelectAddressMode(this, A, GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, AtomicTSO, Class != GPRClass, Size);
 
     if (AtomicTSO) {
       return _StoreMemTSO(Class, Size, Value, A.Base, A.Index, Align, A.IndexType, A.IndexScale);
@@ -2486,7 +2512,7 @@ private:
 
   void Push(IR::OpSize Size, Ref Value) {
     auto OldSP = LoadGPRRegister(X86State::REG_RSP);
-    auto NewSP = _Push(CTX->GetGPROpSize(), Size, Value, OldSP);
+    auto NewSP = _Push(GetGPROpSize(), Size, Value, OldSP);
     StoreGPRRegister(X86State::REG_RSP, NewSP);
     FlushRegisterCache();
   }
@@ -2516,11 +2542,11 @@ private:
     }
 
     ArithRef And(uint64_t K) {
-      return IsConstant ? ArithRef(E, C & K) : ArithRef(E, E->_And(OpSize::i64Bit, R, E->_Constant(K)));
+      return IsConstant ? ArithRef(E, C & K) : ArithRef(E, E->_And(OpSize::i64Bit, R, E->Constant(K)));
     }
 
     ArithRef Presub(uint64_t K) {
-      return IsConstant ? ArithRef(E, K - C) : ArithRef(E, E->_Sub(OpSize::i64Bit, E->_Constant(K), R));
+      return IsConstant ? ArithRef(E, K - C) : ArithRef(E, E->_Sub(OpSize::i64Bit, E->Constant(K), R));
     }
 
     ArithRef Lshl(uint64_t Shift) {
@@ -2529,7 +2555,7 @@ private:
       } else if (IsConstant) {
         return ArithRef(E, C << Shift);
       } else {
-        return ArithRef(E, E->_Lshl(OpSize::i64Bit, R, E->_Constant(Shift)));
+        return ArithRef(E, E->_Lshl(OpSize::i64Bit, R, E->Constant(Shift)));
       }
     }
 
@@ -2569,7 +2595,7 @@ private:
       }
 
       if (IsConstant) {
-        return E->_Bfi(OpSize::i64Bit, Size, Start, Bitfield, E->_Constant(C));
+        return E->_Bfi(OpSize::i64Bit, Size, Start, Bitfield, E->Constant(C));
       } else {
         return E->_Bfi(OpSize::i64Bit, Size, Start, Bitfield, R);
       }
@@ -2585,15 +2611,15 @@ private:
 
         return ArithRef(E, Result);
       } else {
-        return ArithRef(E, E->_Lshl(Size, E->_Constant(1), R));
+        return ArithRef(E, E->_Lshl(Size, E->Constant(1), R));
       }
     }
 
     Ref Ref() {
-      return IsConstant ? E->_Constant(C) : R;
+      return IsConstant ? E->Constant(C) : R;
     }
 
-    bool IsDefinitelyZero() {
+    bool IsDefinitelyZero() const {
       return IsConstant && C == 0;
     }
   };
@@ -2622,9 +2648,9 @@ private:
 
 constexpr inline void InstallToTable(auto& FinalTable, const auto& LocalTable) {
   for (const auto& Op : LocalTable) {
-    auto OpNum = std::get<0>(Op);
-    auto Dispatcher = std::get<2>(Op);
-    for (uint8_t i = 0; i < std::get<1>(Op); ++i) {
+    auto OpNum = Op.Op;
+    auto Dispatcher = Op.Ptr;
+    for (uint8_t i = 0; i < Op.Count; ++i) {
       auto& TableOp = FinalTable[OpNum + i];
 #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
       if (TableOp.OpcodeDispatcher) {

FEXCore/Source/Interface/Core/OpcodeDispatcher/AVX_128.cpp

@@ -23,7 +23,7 @@ class OrderedNode;
 
 void OpDispatchBuilder::InstallAVX128Handlers() {
 #define OPD(map_select, pp, opcode) (((map_select - 1) << 10) | (pp << 8) | (opcode))
-  static constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> AVX128Table[] = {
+  static constexpr DispatchTableEntry AVX128Table[] = {
     {OPD(1, 0b00, 0x10), 1, &OpDispatchBuilder::AVX128_VMOVAPS},
     {OPD(1, 0b01, 0x10), 1, &OpDispatchBuilder::AVX128_VMOVAPS},
     {OPD(1, 0b10, 0x10), 1, &OpDispatchBuilder::AVX128_VMOVSS},
@@ -426,7 +426,7 @@ void OpDispatchBuilder::InstallAVX128Handlers() {
 #undef OPD
 
 #define OPD(group, pp, opcode) (((group - X86Tables::TYPE_VEX_GROUP_12) << 4) | (pp << 3) | (opcode))
-  static constexpr std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr> VEX128TableGroupOps[] {
+  static constexpr DispatchTableEntry VEX128TableGroupOps[] {
     // VPSRLI
     {OPD(X86Tables::TYPE_VEX_GROUP_12, 1, 0b010), 1,
      &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VectorShiftImmImpl, OpSize::i16Bit, IROps::OP_VUSHRI>},
@@ -465,7 +465,7 @@ void OpDispatchBuilder::InstallAVX128Handlers() {
 #undef OPD
 
 #define OPD(map_select, pp, opcode) (((map_select - 1) << 10) | (pp << 8) | (opcode))
-  constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> VEX128_PCLMUL[] = {
+  constexpr DispatchTableEntry VEX128_PCLMUL[] = {
     {OPD(3, 0b01, 0x44), 1, &OpDispatchBuilder::AVX128_VPCLMULQDQ},
   };
 #undef OPD
@@ -778,7 +778,7 @@ void OpDispatchBuilder::AVX128_VectorXOR(OpcodeArgs) {
 void OpDispatchBuilder::AVX128_VZERO(OpcodeArgs) {
   const auto DstSize = GetDstSize(Op);
   const auto IsVZEROALL = DstSize == Core::CPUState::XMM_AVX_REG_SIZE;
-  const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U;
+  const auto NumRegs = Is64BitMode ? 16U : 8U;
 
   if (IsVZEROALL) {
     // NOTE: Despite the name being VZEROALL, this will still only ever
@@ -968,7 +968,7 @@ void OpDispatchBuilder::AVX128_VBROADCAST(OpcodeArgs) {
     }
   } else {
     // Get the address to broadcast from into a GPR.
-    Ref Address = MakeSegmentAddress(Op, Op->Src[0], CTX->GetGPROpSize());
+    Ref Address = MakeSegmentAddress(Op, Op->Src[0], GetGPROpSize());
     Src.Low = _VBroadcastFromMem(OpSize::i128Bit, ElementSize, Address);
   }
 
@@ -1022,7 +1022,7 @@ void OpDispatchBuilder::AVX128_InsertCVTGPR_To_FPR(OpcodeArgs) {
 
   if (Op->Src[1].IsGPR()) {
     // If the source is a GPR then convert directly from the GPR.
-    auto Src2 = LoadSource_WithOpSize(GPRClass, Op, Op->Src[1], CTX->GetGPROpSize(), Op->Flags);
+    auto Src2 = LoadSource_WithOpSize(GPRClass, Op, Op->Src[1], GetGPROpSize(), Op->Flags);
     Result.Low = _VSToFGPRInsert(OpSize::i128Bit, DstElementSize, SrcSize, Src1.Low, Src2, false);
   } else if (SrcSize != DstElementSize) {
     // If the source is from memory but the Source size and destination size aren't the same,
@@ -1054,7 +1054,7 @@ void OpDispatchBuilder::AVX128_CVTFPR_To_GPR(OpcodeArgs) {
   if (Op->Src[0].IsGPR()) {
     Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false);
   } else {
-    Src.Low = LoadSource_WithOpSize(FPRClass, Op, Op->Src[0], OpSizeFromSrc(Op), Op->Flags);
+    Src.Low = LoadSource_WithOpSize(FPRClass, Op, Op->Src[0], SrcElementSize, Op->Flags);
   }
 
   Ref Result = CVTFPR_To_GPRImpl(Op, Src.Low, SrcElementSize, HostRoundingMode);
@@ -1094,7 +1094,7 @@ void OpDispatchBuilder::AVX128_VPSIGN(OpcodeArgs) {
 
 template<IR::OpSize ElementSize>
 void OpDispatchBuilder::AVX128_UCOMISx(OpcodeArgs) {
-  const auto SrcSize = Op->Src[0].IsGPR() ? GetGuestVectorLength() : OpSizeFromSrc(Op);
+  const auto SrcSize = Op->Src[0].IsGPR() ? GetGuestVectorLength() : ElementSize;
 
   auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Dest, Op->Flags, false);
 
@@ -1179,7 +1179,7 @@ void OpDispatchBuilder::AVX128_MOVBetweenGPR_FPR(OpcodeArgs) {
     RefPair Result {};
     if (Op->Src[0].IsGPR()) {
       // Loading from GPR and moving to Vector.
-      Ref Src = LoadSource_WithOpSize(FPRClass, Op, Op->Src[0], CTX->GetGPROpSize(), Op->Flags);
+      Ref Src = LoadSource_WithOpSize(FPRClass, Op, Op->Src[0], GetGPROpSize(), Op->Flags);
       // zext to 128bit
       Result.Low = _VCastFromGPR(OpSize::i128Bit, OpSizeFromSrc(Op), Src);
     } else {
@@ -1227,7 +1227,7 @@ void OpDispatchBuilder::AVX128_PExtr(OpcodeArgs) {
   Index &= NumElements - 1;
 
   if (Op->Dest.IsGPR()) {
-    const auto GPRSize = CTX->GetGPROpSize();
+    const auto GPRSize = GetGPROpSize();
     // Extract already zero extends the result.
     Ref Result = _VExtractToGPR(OpSize::i128Bit, OverridenElementSize, Src.Low, Index);
     StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Result, GPRSize, OpSize::iInvalid);
@@ -1309,7 +1309,7 @@ void OpDispatchBuilder::AVX128_MOVMSK(OpcodeArgs) {
     // Inserting the full lower 32-bits offset 31 so the sign bit ends up at offset 63.
     GPR = _Bfi(OpSize::i64Bit, 32, 31, GPR, GPR);
     // Shift right to only get the two sign bits we care about.
-    return _Lshr(OpSize::i64Bit, GPR, _Constant(62));
+    return _Lshr(OpSize::i64Bit, GPR, Constant(62));
   };
 
   auto Mask4Byte = [this](Ref Src) {
@@ -1341,7 +1341,7 @@ void OpDispatchBuilder::AVX128_MOVMSK(OpcodeArgs) {
     auto GPRHigh = Mask8Byte(Src.High);
     GPR = _Orlshl(OpSize::i64Bit, GPRLow, GPRHigh, 2);
   }
-  StoreResult_WithOpSize(GPRClass, Op, Op->Dest, GPR, CTX->GetGPROpSize(), OpSize::iInvalid);
+  StoreResult_WithOpSize(GPRClass, Op, Op->Dest, GPR, GetGPROpSize(), OpSize::iInvalid);
 }
 
 void OpDispatchBuilder::AVX128_MOVMSKB(OpcodeArgs) {
@@ -1383,7 +1383,7 @@ void OpDispatchBuilder::AVX128_PINSRImpl(OpcodeArgs, IR::OpSize ElementSize, con
 
   if (Src2Op.IsGPR()) {
     // If the source is a GPR then convert directly from the GPR.
-    auto Src2 = LoadSource_WithOpSize(GPRClass, Op, Src2Op, CTX->GetGPROpSize(), Op->Flags);
+    auto Src2 = LoadSource_WithOpSize(GPRClass, Op, Src2Op, GetGPROpSize(), Op->Flags);
     Result.Low = _VInsGPR(OpSize::i128Bit, ElementSize, Index, Src1.Low, Src2);
   } else {
     // If loading from memory then we only load the element size
@@ -2055,7 +2055,7 @@ void OpDispatchBuilder::AVX128_VMASKMOVImpl(OpcodeArgs, IR::OpSize ElementSize,
   auto Mask = AVX128_LoadSource_WithOpSize(Op, MaskOp, Op->Flags, !Is128Bit);
 
   const auto MakeAddress = [this, Op](const X86Tables::DecodedOperand& Data) {
-    return MakeSegmentAddress(Op, Data, CTX->GetGPROpSize());
+    return MakeSegmentAddress(Op, Data, GetGPROpSize());
   };
 
   if (IsStore) {
@@ -2148,7 +2148,7 @@ void OpDispatchBuilder::AVX128_VectorVariableBlend(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::AVX128_SaveAVXState(Ref MemBase) {
-  const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U;
+  const auto NumRegs = Is64BitMode ? 16U : 8U;
 
   for (uint32_t i = 0; i < NumRegs; i += 2) {
     RefPair Pair = LoadContextPair(OpSize::i128Bit, AVXHigh0Index + i);
@@ -2157,7 +2157,7 @@ void OpDispatchBuilder::AVX128_SaveAVXState(Ref MemBase) {
 }
 
 void OpDispatchBuilder::AVX128_RestoreAVXState(Ref MemBase) {
-  const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U;
+  const auto NumRegs = Is64BitMode ? 16U : 8U;
 
   for (uint32_t i = 0; i < NumRegs; i += 2) {
     auto YMMHRegs = LoadMemPair(FPRClass, OpSize::i128Bit, MemBase, i * 16 + 576);
@@ -2168,7 +2168,7 @@ void OpDispatchBuilder::AVX128_RestoreAVXState(Ref MemBase) {
 }
 
 void OpDispatchBuilder::AVX128_DefaultAVXState() {
-  const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U;
+  const auto NumRegs = Is64BitMode ? 16U : 8U;
 
   auto ZeroRegister = LoadZeroVector(OpSize::i128Bit);
   for (uint32_t i = 0; i < NumRegs; i++) {
@@ -2212,8 +2212,8 @@ void OpDispatchBuilder::AVX128_VTESTP(OpcodeArgs) {
   // For 256-bit, we need to split up the operation. This is nontrivial.
   // Let's go the simple route here.
   Ref ZF, CFInv;
-  Ref ZeroConst = _Constant(0);
-  Ref OneConst = _Constant(1);
+  Ref ZeroConst = Constant(0);
+  Ref OneConst = Constant(1);
 
   const auto ElementSizeInBits = IR::OpSizeAsBits(ElementSize);
 
@@ -2294,8 +2294,8 @@ void OpDispatchBuilder::AVX128_PTest(OpcodeArgs) {
   Test1 = _VExtractToGPR(OpSize::i128Bit, OpSize::i16Bit, Test1, 0);
   Test2 = _VExtractToGPR(OpSize::i128Bit, OpSize::i16Bit, Test2, 0);
 
-  auto ZeroConst = _Constant(0);
-  auto OneConst = _Constant(1);
+  auto ZeroConst = Constant(0);
+  auto OneConst = Constant(1);
 
   Test2 = _Select(FEXCore::IR::COND_NEQ, Test2, ZeroConst, OneConst, ZeroConst);
 
@@ -2328,7 +2328,7 @@ void OpDispatchBuilder::AVX128_VPERMD(OpcodeArgs) {
   RefPair Result {};
 
   Ref IndexMask = _VectorImm(OpSize::i128Bit, OpSize::i32Bit, 0b111);
-  Ref AddConst = _Constant(0x03020100);
+  Ref AddConst = Constant(0x03020100);
   Ref Repeating3210 = _VDupFromGPR(OpSize::i128Bit, OpSize::i32Bit, AddConst);
 
   Result.Low = DoPerm(Src, Indices.Low, IndexMask, Repeating3210);
@@ -2454,20 +2454,21 @@ void OpDispatchBuilder::AVX128_VFMAImpl(OpcodeArgs, IROps IROp, uint8_t Src1Idx,
 }
 
 void OpDispatchBuilder::AVX128_VFMAScalarImpl(OpcodeArgs, IROps IROp, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx) {
-  const auto Size = GetDstSize(Op);
-  const auto Is128Bit = Size == Core::CPUState::XMM_SSE_REG_SIZE;
-  LOGMAN_THROW_A_FMT(Is128Bit, "This can't be 256-bit");
+  const auto SrcSize = OpSizeFromSrc(Op);
 
-  const OpSize ElementSize = Op->Flags & X86Tables::DecodeFlags::FLAG_OPTION_AVX_W ? OpSize::i64Bit : OpSize::i32Bit;
-
-  auto Dest = AVX128_LoadSource_WithOpSize(Op, Op->Dest, Op->Flags, !Is128Bit).Low;
-  auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit).Low;
-  auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit).Low;
+  auto Dest = AVX128_LoadSource_WithOpSize(Op, Op->Dest, Op->Flags, false).Low;
+  auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, false).Low;
+  Ref Src2 {};
+  if (Op->Src[1].IsGPR()) {
+    Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, false).Low;
+  } else {
+    Src2 = LoadSource_WithOpSize(FPRClass, Op, Op->Src[1], SrcSize, Op->Flags);
+  }
 
   Ref Sources[3] = {Dest, Src1, Src2};
 
   DeriveOp(Result_Low, IROp,
-           _VFMLAScalarInsert(OpSize::i128Bit, ElementSize, Dest, Sources[Src1Idx - 1], Sources[Src2Idx - 1], Sources[AddendIdx - 1]));
+           _VFMLAScalarInsert(OpSize::i128Bit, SrcSize, Dest, Sources[Src1Idx - 1], Sources[Src2Idx - 1], Sources[AddendIdx - 1]));
   AVX128_StoreResult_WithOpSize(Op, Op->Dest, AVX128_Zext(Result_Low));
 }
 
@@ -2517,9 +2518,9 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpSize Size, O
   ///< BaseAddr doesn't need to exist, calculate that here.
   Ref BaseAddr = VSIB.BaseAddr;
   if (BaseAddr && VSIB.Displacement) {
-    BaseAddr = _Add(OpSize::i64Bit, BaseAddr, _Constant(VSIB.Displacement));
+    BaseAddr = _Add(OpSize::i64Bit, BaseAddr, Constant(VSIB.Displacement));
   } else if (VSIB.Displacement) {
-    BaseAddr = _Constant(VSIB.Displacement);
+    BaseAddr = Constant(VSIB.Displacement);
   } else if (!BaseAddr) {
     BaseAddr = Invalid();
   }
@@ -2612,9 +2613,9 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(Ref Dest, R
   ///< BaseAddr doesn't need to exist, calculate that here.
   Ref BaseAddr = VSIB.BaseAddr;
   if (BaseAddr && VSIB.Displacement) {
-    BaseAddr = _Add(OpSize::i64Bit, BaseAddr, _Constant(VSIB.Displacement));
+    BaseAddr = _Add(OpSize::i64Bit, BaseAddr, Constant(VSIB.Displacement));
   } else if (VSIB.Displacement) {
-    BaseAddr = _Constant(VSIB.Displacement);
+    BaseAddr = Constant(VSIB.Displacement);
   } else if (!BaseAddr) {
     BaseAddr = Invalid();
   }

FEXCore/Source/Interface/Core/OpcodeDispatcher/BaseTables.h

@@ -3,7 +3,7 @@
 #include "Interface/Core/OpcodeDispatcher.h"
 
 namespace FEXCore::IR {
-constexpr inline std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr> OpDispatch_BaseOpTable[] = {
+constexpr inline DispatchTableEntry OpDispatch_BaseOpTable[] = {
   // Instructions
   {0x00, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ADD, FEXCore::IR::IROps::OP_ATOMICFETCHADD, 0>},
 
@@ -76,12 +76,12 @@ constexpr inline std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr> OpDispat
   {0xFC, 2, &OpDispatchBuilder::FLAGControlOp},
 };
 
-constexpr inline std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr> OpDispatch_BaseOpTable_64[] = {
+constexpr inline DispatchTableEntry OpDispatch_BaseOpTable_64[] = {
   {0x63, 1, &OpDispatchBuilder::MOVSXDOp},
   {0xA0, 4, &OpDispatchBuilder::MOVOffsetOp},
 };
 
-constexpr inline std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr> OpDispatch_BaseOpTable_32[] = {
+constexpr inline DispatchTableEntry OpDispatch_BaseOpTable_32[] = {
   {0x06, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX>},
   {0x07, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX>},
   {0x0E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX>},

FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp

@@ -11,10 +11,7 @@ $end_info$
 #include <FEXCore/Utils/LogManager.h>
 #include "Interface/Core/OpcodeDispatcher.h"
 
-#include <array>
 #include <cstdint>
-#include <tuple>
-#include <utility>
 
 namespace FEXCore::IR {
 class OrderedNode;
@@ -25,21 +22,13 @@ void OpDispatchBuilder::SHA1NEXTEOp(OpcodeArgs) {
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
 
-  Ref RotatedNode {};
-  if (CTX->HostFeatures.SupportsSHA) {
-    // ARMv8 SHA1 extension provides a `SHA1H` instruction which does a fixed rotate by 30.
-    // This only operates on element 0 rather than element 3. We don't have the luxury of rewriting the x86 SHA algorithm to take advantage of this.
-    // Move the element to zero, rotate, and then move back (Using duplicates).
-    // Saves one instruction versus that path that doesn't support SHA extension.
-    auto Duplicated = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
-    auto Sha1HRotated = _VSha1H(Duplicated);
-    RotatedNode = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Sha1HRotated, 0);
-  } else {
-    // SHA1 extension missing, manually rotate.
-    // Emulate rotate.
-    auto ShiftLeft = _VShlI(OpSize::i128Bit, OpSize::i32Bit, Dest, 30);
-    RotatedNode = _VUShraI(OpSize::i128Bit, OpSize::i32Bit, ShiftLeft, Dest, 2);
-  }
+  // ARMv8 SHA1 extension provides a `SHA1H` instruction which does a fixed rotate by 30.
+  // This only operates on element 0 rather than element 3. We don't have the luxury of rewriting the x86 SHA algorithm to take advantage of this.
+  // Move the element to zero, rotate, and then move back (Using duplicates).
+  // Saves one instruction versus that path that doesn't support SHA extension.
+  auto Duplicated = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
+  auto Sha1HRotated = _VSha1H(Duplicated);
+  auto RotatedNode = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Sha1HRotated, 0);
   auto Tmp = _VAdd(OpSize::i128Bit, OpSize::i32Bit, Src, RotatedNode);
   auto Result = _VInsElement(OpSize::i128Bit, OpSize::i32Bit, 3, 3, Src, Tmp);
 
@@ -62,153 +51,49 @@ void OpDispatchBuilder::SHA1MSG2Op(OpcodeArgs) {
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
 
-  Ref Result;
-  if (CTX->HostFeatures.SupportsSHA) {
-    // ARM SHA1 mostly matches x86 semantics, except the input and outputs are both flipped from elements 0,1,2,3 to 3,2,1,0.
-    auto Src1 = SHADataShuffle(Dest);
-    auto Src2 = SHADataShuffle(Src);
-
-    // The result is swizzled differently than expected
-    Result = SHADataShuffle(_VSha1SU1(Src1, Src2));
-  } else {
-    // Shift the incoming source left by a 32-bit element, inserting Zeros.
-    // This could be slightly improved to use a VInsGPR with the zero register.
-    const auto ZeroRegister = LoadZeroVector(OpSize::i32Bit);
-    auto Src2Shift = _VExtr(OpSize::i128Bit, OpSize::i8Bit, Src, ZeroRegister, 12);
-    auto Xor1 = _VXor(OpSize::i128Bit, OpSize::i8Bit, Dest, Src2Shift);
-
-    // Emulate rotate.
-    auto ShiftLeftXor1 = _VShlI(OpSize::i128Bit, OpSize::i32Bit, Xor1, 1);
-    auto RotatedXor1 = _VUShraI(OpSize::i128Bit, OpSize::i32Bit, ShiftLeftXor1, Xor1, 31);
-
-    // Element0 didn't get XOR'd with anything, so do it now.
-    auto ExtractUpper = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, RotatedXor1, 3);
-    auto XorLower = _VXor(OpSize::i128Bit, OpSize::i8Bit, Dest, ExtractUpper);
-
-    // Emulate rotate.
-    auto ShiftLeftXorLower = _VShlI(OpSize::i128Bit, OpSize::i32Bit, XorLower, 1);
-    auto RotatedXorLower = _VUShraI(OpSize::i128Bit, OpSize::i32Bit, ShiftLeftXorLower, XorLower, 31);
-
-    Result = _VInsElement(OpSize::i128Bit, OpSize::i32Bit, 0, 0, RotatedXor1, RotatedXorLower);
-  }
+  // ARM SHA1 mostly matches x86 semantics, except the input and outputs are both flipped from elements 0,1,2,3 to 3,2,1,0.
+  auto Src1 = SHADataShuffle(Dest);
+  auto Src2 = SHADataShuffle(Src);
 
+  // The result is swizzled differently than expected
+  auto Result = SHADataShuffle(_VSha1SU1(Src1, Src2));
 
   StoreResult(FPRClass, Op, Result, OpSize::iInvalid);
 }
 
 void OpDispatchBuilder::SHA1RNDS4Op(OpcodeArgs) {
-  using FnType = Ref (*)(OpDispatchBuilder&, Ref, Ref, Ref);
-
-  const auto f0 = [](OpDispatchBuilder& Self, Ref B, Ref C, Ref D) -> Ref { // sha1c?
-    return Self._Xor(OpSize::i32Bit, Self._And(OpSize::i32Bit, B, C), Self._Andn(OpSize::i32Bit, D, B));
-  };
-  const auto f1 = [](OpDispatchBuilder& Self, Ref B, Ref C, Ref D) -> Ref { // sha1p with different key
-    return Self._Xor(OpSize::i32Bit, Self._Xor(OpSize::i32Bit, B, C), D);
-  };
-  const auto f2 = [](OpDispatchBuilder& Self, Ref B, Ref C, Ref D) -> Ref { // sha1m
-    return Self.BitwiseAtLeastTwo(B, C, D);
-  };
-  const auto f3 = [](OpDispatchBuilder& Self, Ref B, Ref C, Ref D) -> Ref { // sha1p
-    return Self._Xor(OpSize::i32Bit, Self._Xor(OpSize::i32Bit, B, C), D);
-  };
-
-  constexpr std::array<uint32_t, 4> k_array {
-    0x5A827999U,
-    0x6ED9EBA1U,
-    0x8F1BBCDCU,
-    0xCA62C1D6U,
-  };
-
-  constexpr std::array<FnType, 4> fn_array {
-    f0,
-    f1,
-    f2,
-    f3,
-  };
-
-
   const uint64_t Imm8 = Op->Src[1].Literal() & 0b11;
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
 
   Ref Result {};
-  if (CTX->HostFeatures.SupportsSHA) {
-    Ref ConstantVector {};
-    switch (Imm8) {
-    case 0:
-      ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K0);
-      break;
-    case 1:
-      ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K1);
-      break;
-    case 2:
-      ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K2);
-      break;
-    case 3:
-      ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K3);
-      break;
-    }
+  Ref ConstantVector {};
+  switch (Imm8) {
+  case 0:
+    ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K0);
+    break;
+  case 1:
+    ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K1);
+    break;
+  case 2:
+    ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K2);
+    break;
+  case 3:
+    ConstantVector = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_SHA1RNDS_K3);
+    break;
+  }
 
-    const auto ZeroRegister = LoadZeroVector(OpSize::i32Bit);
+  const auto ZeroRegister = LoadZeroVector(OpSize::i32Bit);
 
-    Ref Src1 = SHADataShuffle(Dest);
-    Ref Src2 = SHADataShuffle(Src);
-    Src2 = _VAdd(OpSize::i128Bit, OpSize::i32Bit, Src2, ConstantVector);
+  Ref Src1 = SHADataShuffle(Dest);
+  Ref Src2 = SHADataShuffle(Src);
+  Src2 = _VAdd(OpSize::i128Bit, OpSize::i32Bit, Src2, ConstantVector);
 
-    switch (Imm8) {
-    case 0: Result = SHADataShuffle(_VSha1C(Src1, ZeroRegister, Src2)); break;
-    case 2: Result = SHADataShuffle(_VSha1M(Src1, ZeroRegister, Src2)); break;
-    case 1:
-    case 3: Result = SHADataShuffle(_VSha1P(Src1, ZeroRegister, Src2)); break;
-    }
-  } else {
-    const FnType Fn = fn_array[Imm8];
-    auto K = _Constant(OpSize::i32Bit, k_array[Imm8]);
-    auto W0E = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 3);
-
-    using RoundResult = std::tuple<Ref, Ref, Ref, Ref, Ref>;
-
-    const auto Round0 = [&]() -> RoundResult {
-      auto A = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
-      auto B = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 2);
-      auto C = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 1);
-      auto D = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 0);
-
-      auto A1 =
-        _Add(OpSize::i32Bit,
-             _Add(OpSize::i32Bit, _Add(OpSize::i32Bit, Fn(*this, B, C, D), _Ror(OpSize::i32Bit, A, _Constant(OpSize::i32Bit, 27))), W0E), K);
-      auto B1 = A;
-      auto C1 = _Ror(OpSize::i32Bit, B, _Constant(OpSize::i32Bit, 2));
-      auto D1 = C;
-      auto E1 = D;
-
-      return {A1, B1, C1, D1, E1};
-    };
-    const auto Round1To3 = [&](Ref A, Ref B, Ref C, Ref D, Ref E, Ref Src, unsigned W_idx) -> RoundResult {
-      // Kill W and E at the beginning
-      auto W = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, W_idx);
-      auto Q = _Add(OpSize::i32Bit, W, E);
-
-      auto ANext =
-        _Add(OpSize::i32Bit,
-             _Add(OpSize::i32Bit, _Add(OpSize::i32Bit, Fn(*this, B, C, D), _Ror(OpSize::i32Bit, A, _Constant(OpSize::i32Bit, 27))), Q), K);
-      auto BNext = A;
-      auto CNext = _Ror(OpSize::i32Bit, B, _Constant(OpSize::i32Bit, 2));
-      auto DNext = C;
-      auto ENext = D;
-
-      return {ANext, BNext, CNext, DNext, ENext};
-    };
-
-    auto [A1, B1, C1, D1, E1] = Round0();
-    auto [A2, B2, C2, D2, E2] = Round1To3(A1, B1, C1, D1, E1, Src, 2);
-    auto [A3, B3, C3, D3, E3] = Round1To3(A2, B2, C2, D2, E2, Src, 1);
-    auto Final = Round1To3(A3, B3, C3, D3, E3, Src, 0);
-
-    auto Dest3 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 3, Dest, std::get<0>(Final));
-    auto Dest2 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 2, Dest3, std::get<1>(Final));
-    auto Dest1 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 1, Dest2, std::get<2>(Final));
-    Result = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 0, Dest1, std::get<3>(Final));
+  switch (Imm8) {
+  case 0: Result = SHADataShuffle(_VSha1C(Src1, ZeroRegister, Src2)); break;
+  case 2: Result = SHADataShuffle(_VSha1M(Src1, ZeroRegister, Src2)); break;
+  case 1:
+  case 3: Result = SHADataShuffle(_VSha1P(Src1, ZeroRegister, Src2)); break;
   }
 
   StoreResult(FPRClass, Op, Result, OpSize::iInvalid);
@@ -218,69 +103,20 @@ void OpDispatchBuilder::SHA256MSG1Op(OpcodeArgs) {
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
 
-  Ref Result {};
-
-  if (CTX->HostFeatures.SupportsSHA) {
-    Result = _VSha256U0(Dest, Src);
-  } else {
-    const auto Sigma0 = [this](Ref W) -> Ref {
-      return _Xor(
-        OpSize::i32Bit,
-        _Xor(OpSize::i32Bit, _Ror(OpSize::i32Bit, W, _Constant(OpSize::i32Bit, 7)), _Ror(OpSize::i32Bit, W, _Constant(OpSize::i32Bit, 18))),
-        _Lshr(OpSize::i32Bit, W, _Constant(OpSize::i32Bit, 3)));
-    };
-
-    auto W4 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 0);
-    auto W3 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
-    auto W2 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 2);
-    auto W1 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 1);
-    auto W0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 0);
-
-    auto Sig3 = _Add(OpSize::i32Bit, W3, Sigma0(W4));
-    auto Sig2 = _Add(OpSize::i32Bit, W2, Sigma0(W3));
-    auto Sig1 = _Add(OpSize::i32Bit, W1, Sigma0(W2));
-    auto Sig0 = _Add(OpSize::i32Bit, W0, Sigma0(W1));
-
-    auto D3 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 3, Dest, Sig3);
-    auto D2 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 2, D3, Sig2);
-    auto D1 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 1, D2, Sig1);
-    Result = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 0, D1, Sig0);
-  }
+  auto Result = _VSha256U0(Dest, Src);
 
   StoreResult(FPRClass, Op, Result, OpSize::iInvalid);
 }
 
 void OpDispatchBuilder::SHA256MSG2Op(OpcodeArgs) {
-  const auto Sigma1 = [this](Ref W) -> Ref {
-    return _Xor(
-      OpSize::i32Bit,
-      _Xor(OpSize::i32Bit, _Ror(OpSize::i32Bit, W, _Constant(OpSize::i32Bit, 17)), _Ror(OpSize::i32Bit, W, _Constant(OpSize::i32Bit, 19))),
-      _Lshr(OpSize::i32Bit, W, _Constant(OpSize::i32Bit, 10)));
-  };
-
   Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
   Ref Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
 
-  Ref Result;
-  if (CTX->HostFeatures.SupportsSHA) {
-    auto Src1 = _VExtr(OpSize::i128Bit, OpSize::i32Bit, Dest, Dest, 3);
-    auto DupDst = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
-    auto Src2 = _VZip2(OpSize::i128Bit, OpSize::i64Bit, DupDst, Src);
+  auto Src1 = _VExtr(OpSize::i128Bit, OpSize::i32Bit, Dest, Dest, 3);
+  auto DupDst = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
+  auto Src2 = _VZip2(OpSize::i128Bit, OpSize::i64Bit, DupDst, Src);
 
-    Result = _VSha256U1(Src1, Src2);
-  } else {
-    auto W14 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 2);
-    auto W15 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 3);
-    auto W16 = _Add(OpSize::i32Bit, _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 0), Sigma1(W14));
-    auto W17 = _Add(OpSize::i32Bit, _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 1), Sigma1(W15));
-    auto W18 = _Add(OpSize::i32Bit, _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 2), Sigma1(W16));
-    auto W19 = _Add(OpSize::i32Bit, _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 3), Sigma1(W17));
-
-    auto D3 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 3, Dest, W19);
-    auto D2 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 2, D3, W18);
-    auto D1 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 1, D2, W17);
-    Result = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 0, D1, W16);
-  }
+  auto Result = _VSha256U1(Src1, Src2);
 
   StoreResult(FPRClass, Op, Result, OpSize::iInvalid);
 }
@@ -301,81 +137,27 @@ void OpDispatchBuilder::SHA256RNDS2Op(OpcodeArgs) {
   // Hardcoded to XMM0
   auto XMM0 = LoadXMMRegister(0);
 
-  Ref Result;
-  if (CTX->HostFeatures.SupportsSHA) {
-    auto shuffle_abcd = [this](Ref Src1, Ref Src2) -> Ref {
-      // Generates a suitable SHA256 `abcd` configuration from x86 format.
-      auto Tmp = _VZip2(OpSize::i128Bit, OpSize::i64Bit, Src2, Src1);
-      return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);
-    };
+  auto shuffle_abcd = [this](Ref Src1, Ref Src2) -> Ref {
+    // Generates a suitable SHA256 `abcd` configuration from x86 format.
+    auto Tmp = _VZip2(OpSize::i128Bit, OpSize::i64Bit, Src2, Src1);
+    return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);
+  };
 
-    auto shuffle_efgh = [this](Ref Src1, Ref Src2) -> Ref {
-      // Generates a suitable SHA256 `efgh` configuration from x86 format.
-      auto Tmp = _VZip(OpSize::i128Bit, OpSize::i64Bit, Src2, Src1);
-      return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);
-    };
+  auto shuffle_efgh = [this](Ref Src1, Ref Src2) -> Ref {
+    // Generates a suitable SHA256 `efgh` configuration from x86 format.
+    auto Tmp = _VZip(OpSize::i128Bit, OpSize::i64Bit, Src2, Src1);
+    return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);
+  };
 
-    auto ABCD = shuffle_abcd(Dest, Src);
-    auto EFGH = shuffle_efgh(Dest, Src);
+  auto ABCD = shuffle_abcd(Dest, Src);
+  auto EFGH = shuffle_efgh(Dest, Src);
 
-    // x86 uses only the bottom 64-bits of the key, so duplicate to match ARM64 semantics.
-    auto Key = _VDupElement(OpSize::i128Bit, OpSize::i64Bit, XMM0, 0);
+  // x86 uses only the bottom 64-bits of the key, so duplicate to match ARM64 semantics.
+  auto Key = _VDupElement(OpSize::i128Bit, OpSize::i64Bit, XMM0, 0);
 
-    auto A = _VSha256H(ABCD, EFGH, Key);
-    auto B = _VSha256H2(EFGH, ABCD, Key);
-    Result = shuffle_abcd(A, B);
-  } else {
-    const auto Ch = [this](Ref E, Ref F, Ref G) -> Ref {
-      return _Xor(OpSize::i32Bit, _And(OpSize::i32Bit, E, F), _Andn(OpSize::i32Bit, G, E));
-    };
-    const auto Sigma0 = [this](Ref A) -> Ref {
-      return _XorShift(OpSize::i32Bit, _XorShift(OpSize::i32Bit, _Ror(OpSize::i32Bit, A, _Constant(OpSize::i32Bit, 2)), A, ShiftType::ROR, 13),
-                       A, ShiftType::ROR, 22);
-    };
-    const auto Sigma1 = [this](Ref E) -> Ref {
-      return _XorShift(OpSize::i32Bit, _XorShift(OpSize::i32Bit, _Ror(OpSize::i32Bit, E, _Constant(OpSize::i32Bit, 6)), E, ShiftType::ROR, 11),
-                       E, ShiftType::ROR, 25);
-    };
-
-    auto E0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 1);
-    auto F0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 0);
-    auto G0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 1);
-    Ref Q0 = _Add(OpSize::i32Bit, Ch(E0, F0, G0), Sigma1(E0));
-
-    auto WK0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, XMM0, 0);
-    Q0 = _Add(OpSize::i32Bit, Q0, WK0);
-
-    auto H0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 0);
-    Q0 = _Add(OpSize::i32Bit, Q0, H0);
-
-    auto A0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 3);
-    auto B0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Src, 2);
-    auto C0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
-    auto A1 = _Add(OpSize::i32Bit, _Add(OpSize::i32Bit, Q0, BitwiseAtLeastTwo(A0, B0, C0)), Sigma0(A0));
-
-    auto D0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 2);
-    auto E1 = _Add(OpSize::i32Bit, Q0, D0);
-
-    Ref Q1 = _Add(OpSize::i32Bit, Ch(E1, E0, F0), Sigma1(E1));
-
-    auto WK1 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, XMM0, 1);
-    Q1 = _Add(OpSize::i32Bit, Q1, WK1);
-
-    // Rematerialize G0. Costs a move but saves spilling, coming out ahead.
-    G0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 1);
-    Q1 = _Add(OpSize::i32Bit, Q1, G0);
-
-    auto A2 = _Add(OpSize::i32Bit, _Add(OpSize::i32Bit, Q1, BitwiseAtLeastTwo(A1, A0, B0)), Sigma0(A1));
-
-    // Rematerialize C0. As with G0.
-    C0 = _VExtractToGPR(OpSize::i128Bit, OpSize::i32Bit, Dest, 3);
-    auto E2 = _Add(OpSize::i32Bit, Q1, C0);
-
-    auto Res3 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 3, Dest, A2);
-    auto Res2 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 2, Res3, A1);
-    auto Res1 = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 1, Res2, E2);
-    Result = _VInsGPR(OpSize::i128Bit, OpSize::i32Bit, 0, Res1, E1);
-  }
+  auto A = _VSha256H(ABCD, EFGH, Key);
+  auto B = _VSha256H2(EFGH, ABCD, Key);
+  auto Result = shuffle_abcd(A, B);
 
   StoreResult(FPRClass, Op, Result, OpSize::iInvalid);
 }

FEXCore/Source/Interface/Core/OpcodeDispatcher/DDDTables.h

@@ -3,7 +3,7 @@
 #include "Interface/Core/OpcodeDispatcher.h"
 
 namespace FEXCore::IR {
-constexpr std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_DDDTable[] = {
+constexpr DispatchTableEntry OpDispatch_DDDTable[] = {
   {0x0C, 1, &OpDispatchBuilder::PI2FWOp},
   {0x0D, 1, &OpDispatchBuilder::Vector_CVT_Int_To_Float<OpSize::i32Bit, false>},
   {0x1C, 1, &OpDispatchBuilder::PF2IWOp},

FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp

@@ -28,7 +28,7 @@ constexpr std::array<uint32_t, 17> FlagOffsets = {
 
 void OpDispatchBuilder::ZeroPF_AF() {
   // PF is stored inverted, so invert it when we zero.
-  SetRFLAG<FEXCore::X86State::RFLAG_PF_RAW_LOC>(_Constant(1));
+  SetRFLAG<FEXCore::X86State::RFLAG_PF_RAW_LOC>(Constant(1));
   SetAF(0);
 }
 
@@ -247,7 +247,7 @@ void OpDispatchBuilder::CalculateAF(Ref Src1, Ref Src2) {
   // We store the XOR of the arguments. At read time, we XOR with the
   // appropriate bit of the result (available as the PF flag) and extract the
   // appropriate bit. Again 64-bit to avoid masking.
-  Ref XorRes = Src1 == Src2 ? _Constant(0) : _Xor(OpSize::i64Bit, Src1, Src2);
+  Ref XorRes = Src1 == Src2 ? Constant(0) : _Xor(OpSize::i64Bit, Src1, Src2);
   SetRFLAG<FEXCore::X86State::RFLAG_AF_RAW_LOC>(XorRes);
 }
 

FEXCore/Source/Interface/Core/OpcodeDispatcher/H0F38Tables.h

@@ -8,7 +8,7 @@ constexpr uint16_t PF_38_NONE = 0;
 constexpr uint16_t PF_38_66 = (1U << 0);
 constexpr uint16_t PF_38_F3 = (1U << 2);
 
-constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_H0F38Table[] = {
+constexpr DispatchTableEntry OpDispatch_H0F38Table[] = {
   {OPD(PF_38_NONE, 0x00), 1, &OpDispatchBuilder::PSHUFBOp},
   {OPD(PF_38_66, 0x00), 1, &OpDispatchBuilder::PSHUFBOp},
   {OPD(PF_38_NONE, 0x01), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADDP, OpSize::i16Bit>},

FEXCore/Source/Interface/Core/OpcodeDispatcher/H0F3ATables.h

@@ -8,7 +8,7 @@ namespace FEXCore::IR {
 #define PF_3A_66 1
 constexpr auto OpDispatchTableGenH0F3A = []() consteval {
   constexpr auto OpDispatchTableGenH0F3AREX = []<uint16_t REX>() consteval {
-    constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> Table[] = {
+    constexpr DispatchTableEntry Table[] = {
       {OPD(REX, PF_3A_66, 0x08), 1, &OpDispatchBuilder::VectorRound<OpSize::i32Bit>},
       {OPD(REX, PF_3A_66, 0x09), 1, &OpDispatchBuilder::VectorRound<OpSize::i64Bit>},
       {OPD(REX, PF_3A_66, 0x0A), 1, &OpDispatchBuilder::InsertScalarRound<OpSize::i32Bit>},
@@ -42,8 +42,8 @@ constexpr auto OpDispatchTableGenH0F3A = []() consteval {
 
   auto REX0 = OpDispatchTableGenH0F3AREX.template operator()<0>();
   auto REX1 = OpDispatchTableGenH0F3AREX.template operator()<1>();
-  auto concat = []<typename T, size_t N1, size_t N2>(std::array<T, N1> const& lhs,
-                                                     std::array<T, N2> const& rhs) consteval -> std::array<T, N1 + N2> {
+  auto concat = []<typename T, size_t N1, size_t N2>(const std::array<T, N1>& lhs,
+                                                     const std::array<T, N2>& rhs) consteval -> std::array<T, N1 + N2> {
     std::array<T, N1 + N2> Table {};
     for (size_t i = 0; i < N1; ++i) {
       Table[i] = lhs[i];
@@ -60,12 +60,12 @@ constexpr auto OpDispatchTableGenH0F3A = []() consteval {
 
 constexpr auto OpDispatch_H0F3ATableIgnoreREX = OpDispatchTableGenH0F3A();
 
-constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_H0F3ATableNeedsREX0[] = {
+constexpr DispatchTableEntry OpDispatch_H0F3ATableNeedsREX0[] = {
   {OPD(0, PF_3A_66, 0x16), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i32Bit>},
   {OPD(0, PF_3A_66, 0x22), 1, &OpDispatchBuilder::PINSROp<OpSize::i32Bit>},
 };
 
-constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_H0F3ATable_64[] = {
+constexpr DispatchTableEntry OpDispatch_H0F3ATable_64[] = {
   {OPD(1, PF_3A_66, 0x16), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i64Bit>},
   {OPD(1, PF_3A_66, 0x22), 1, &OpDispatchBuilder::PINSROp<OpSize::i64Bit>},
 };

FEXCore/Source/Interface/Core/OpcodeDispatcher/PrimaryGroupTables.h

@@ -5,7 +5,7 @@
 namespace FEXCore::IR {
 using X86Tables::OpToIndex;
 #define OPD(group, prefix, Reg) (((group - FEXCore::X86Tables::TYPE_GROUP_1) << 6) | (prefix) << 3 | (Reg))
-constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_PrimaryGroupTables[] = {
+constexpr DispatchTableEntry OpDispatch_PrimaryGroupTables[] = {
   // GROUP 1
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 0), 1, &OpDispatchBuilder::SecondaryALUOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 1), 1, &OpDispatchBuilder::SecondaryALUOp},

FEXCore/Source/Interface/Core/OpcodeDispatcher/SecondaryGroupTables.h

@@ -8,7 +8,7 @@ constexpr uint16_t PF_NONE = 0;
 constexpr uint16_t PF_F3 = 1;
 constexpr uint16_t PF_66 = 2;
 constexpr uint16_t PF_F2 = 3;
-constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_SecondaryGroupTables[] = {
+constexpr DispatchTableEntry OpDispatch_SecondaryGroupTables[] = {
   // GROUP 6
   {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_NONE, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_F3, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},
@@ -113,11 +113,13 @@ constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDis
   {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 7), 1, &OpDispatchBuilder::StoreFenceOrCLFlush}, // SFENCE (or CLFLUSH)
 
   {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F3, 5), 1, &OpDispatchBuilder::UnimplementedOp},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F3, 6), 1, &OpDispatchBuilder::UnimplementedOp},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F3, 6), 1, &OpDispatchBuilder::UMonitorOrCLRSSBSY},
 
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_66, 6), 1, &OpDispatchBuilder::CLWB},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_66, 6), 1, &OpDispatchBuilder::CLWBOrTPause},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_66, 7), 1, &OpDispatchBuilder::CLFLUSHOPT},
 
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F2, 6), 1, &OpDispatchBuilder::UMWaitOp},
+
   // GROUP 16
   {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_NONE, 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, true, 1>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_NONE, 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 1>},
@@ -154,7 +156,7 @@ constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDis
   {OPD(FEXCore::X86Tables::TYPE_GROUP_P, PF_F2, 0), 8, &OpDispatchBuilder::NOPOp},
 };
 
-constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_SecondaryGroupTables_64[] = {
+constexpr DispatchTableEntry OpDispatch_SecondaryGroupTables_64[] = {
   // GROUP 15
   {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F3, 0), 1,
    &OpDispatchBuilder::Bind<&OpDispatchBuilder::ReadSegmentReg, OpDispatchBuilder::Segment::FS>},

FEXCore/Source/Interface/Core/OpcodeDispatcher/SecondaryModRMTables.h

@@ -3,7 +3,7 @@
 #include "Interface/Core/OpcodeDispatcher.h"
 
 namespace FEXCore::IR {
-constexpr std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_SecondaryModRMTables[] = {
+constexpr DispatchTableEntry OpDispatch_SecondaryModRMTables[] = {
   // REG /1
   {((0 << 3) | 0), 1, &OpDispatchBuilder::UnimplementedOp},
   {((0 << 3) | 1), 1, &OpDispatchBuilder::UnimplementedOp},

FEXCore/Source/Interface/Core/OpcodeDispatcher/SecondaryTables.h

@@ -3,7 +3,7 @@
 #include "Interface/Core/OpcodeDispatcher.h"
 
 namespace FEXCore::IR {
-constexpr std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_TwoByteOpTable[] = {
+constexpr DispatchTableEntry OpDispatch_TwoByteOpTable[] = {
   // Instructions
   {0x03, 1, &OpDispatchBuilder::LSLOp},
   {0x06, 1, &OpDispatchBuilder::PermissionRestrictedOp},
@@ -150,7 +150,7 @@ constexpr std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDisp
 #endif
 };
 
-constexpr std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_SecondaryRepModTables[] = {
+constexpr DispatchTableEntry OpDispatch_SecondaryRepModTables[] = {
   {0x10, 2, &OpDispatchBuilder::MOVSSOp},
   {0x12, 1, &OpDispatchBuilder::VMOVSLDUPOp},
   {0x16, 1, &OpDispatchBuilder::VMOVSHDUPOp},
@@ -181,7 +181,7 @@ constexpr std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDisp
   {0xE6, 1, &OpDispatchBuilder::Vector_CVT_Int_To_Float<OpSize::i32Bit, true>},
 };
 
-constexpr std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_SecondaryRepNEModTables[] = {
+constexpr DispatchTableEntry OpDispatch_SecondaryRepNEModTables[] = {
   {0x10, 2, &OpDispatchBuilder::MOVSDOp},
   {0x12, 1, &OpDispatchBuilder::MOVDDUPOp},
   {0x2A, 1, &OpDispatchBuilder::InsertCVTGPR_To_FPR<OpSize::i64Bit>},
@@ -207,7 +207,7 @@ constexpr std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDisp
   {0xF0, 1, &OpDispatchBuilder::MOVVectorUnalignedOp},
 };
 
-constexpr std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_SecondaryOpSizeModTables[] = {
+constexpr DispatchTableEntry OpDispatch_SecondaryOpSizeModTables[] = {
   {0x10, 2, &OpDispatchBuilder::MOVVectorUnalignedOp},
   {0x12, 2, &OpDispatchBuilder::MOVLPOp},
   {0x14, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUNPCKLOp, OpSize::i64Bit>},
@@ -314,7 +314,7 @@ constexpr std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDisp
   {0xFE, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VectorALUOp, IR::OP_VADD, OpSize::i32Bit>},
 };
 
-constexpr std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_TwoByteOpTable_64[] = {
+constexpr DispatchTableEntry OpDispatch_TwoByteOpTable_64[] = {
   {0x05, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SyscallOp, true>},
   {0xA0, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX>},
   {0xA1, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX>},
@@ -322,7 +322,7 @@ constexpr std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDisp
   {0xA9, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX>},
 };
 
-constexpr std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_TwoByteOpTable_32[] = {
+constexpr DispatchTableEntry OpDispatch_TwoByteOpTable_32[] = {
   {0x05, 1, &OpDispatchBuilder::NOPOp},
   {0xA0, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX>},
   {0xA1, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::POPSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX>},

FEXCore/Source/Interface/Core/OpcodeDispatcher/VEXTables.h

@@ -4,7 +4,7 @@
 
 namespace FEXCore::IR {
 #define OPD(map_select, pp, opcode) (((map_select - 1) << 10) | (pp << 8) | (opcode))
-constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDispatch_VEXTable[] = {
+constexpr DispatchTableEntry OpDispatch_VEXTable[] = {
   {OPD(2, 0b00, 0xF2), 1, &OpDispatchBuilder::ANDNBMIOp}, {OPD(2, 0b00, 0xF5), 1, &OpDispatchBuilder::BZHI},
   {OPD(2, 0b10, 0xF5), 1, &OpDispatchBuilder::PEXT},      {OPD(2, 0b11, 0xF5), 1, &OpDispatchBuilder::PDEP},
   {OPD(2, 0b11, 0xF6), 1, &OpDispatchBuilder::MULX},      {OPD(2, 0b00, 0xF7), 1, &OpDispatchBuilder::BEXTRBMIOp},
@@ -16,7 +16,7 @@ constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDis
 #undef OPD
 
 #define OPD(group, pp, opcode) (((group - X86Tables::InstType::TYPE_VEX_GROUP_12) << 4) | (pp << 3) | (opcode))
-constexpr std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr> OpDispatch_VEXGroupTable[] = {
+constexpr DispatchTableEntry OpDispatch_VEXGroupTable[] = {
   {OPD(X86Tables::InstType::TYPE_VEX_GROUP_17, 0, 0b001), 1, &OpDispatchBuilder::BLSRBMIOp},
   {OPD(X86Tables::InstType::TYPE_VEX_GROUP_17, 0, 0b010), 1, &OpDispatchBuilder::BLSMSKBMIOp},
   {OPD(X86Tables::InstType::TYPE_VEX_GROUP_17, 0, 0b011), 1, &OpDispatchBuilder::BLSIBMIOp},

FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp

@@ -434,7 +434,7 @@ Ref OpDispatchBuilder::InsertCVTGPR_To_FPRImpl(OpcodeArgs, IR::OpSize DstSize, I
 
   if (Src2Op.IsGPR()) {
     // If the source is a GPR then convert directly from the GPR.
-    auto Src2 = LoadSource_WithOpSize(GPRClass, Op, Src2Op, CTX->GetGPROpSize(), Op->Flags);
+    auto Src2 = LoadSource_WithOpSize(GPRClass, Op, Src2Op, GetGPROpSize(), Op->Flags);
     return _VSToFGPRInsert(DstSize, DstElementSize, SrcSize, Src1, Src2, ZeroUpperBits);
   } else if (SrcSize != DstElementSize) {
     // If the source is from memory but the Source size and destination size aren't the same,
@@ -740,8 +740,8 @@ void OpDispatchBuilder::MOVMSKOp(OpcodeArgs, IR::OpSize ElementSize) {
     // Inserting the full lower 32-bits offset 31 so the sign bit ends up at offset 63.
     GPR = _Bfi(OpSize::i64Bit, 32, 31, GPR, GPR);
     // Shift right to only get the two sign bits we care about.
-    GPR = _Lshr(OpSize::i64Bit, GPR, _Constant(62));
-    StoreResult_WithOpSize(GPRClass, Op, Op->Dest, GPR, CTX->GetGPROpSize(), OpSize::iInvalid);
+    GPR = _Lshr(OpSize::i64Bit, GPR, Constant(62));
+    StoreResult_WithOpSize(GPRClass, Op, Op->Dest, GPR, GetGPROpSize(), OpSize::iInvalid);
   } else if (Size == OpSize::i128Bit && ElementSize == OpSize::i32Bit) {
     // Shift all the sign bits to the bottom of their respective elements.
     Src = _VUShrI(Size, OpSize::i32Bit, Src, 31);
@@ -753,9 +753,9 @@ void OpDispatchBuilder::MOVMSKOp(OpcodeArgs, IR::OpSize ElementSize) {
     Src = _VAddV(Size, OpSize::i32Bit, Src);
     // Extract to a GPR.
     Ref GPR = _VExtractToGPR(Size, OpSize::i32Bit, Src, 0);
-    StoreResult_WithOpSize(GPRClass, Op, Op->Dest, GPR, CTX->GetGPROpSize(), OpSize::iInvalid);
+    StoreResult_WithOpSize(GPRClass, Op, Op->Dest, GPR, GetGPROpSize(), OpSize::iInvalid);
   } else {
-    Ref CurrentVal = _Constant(0);
+    Ref CurrentVal = Constant(0);
 
     for (unsigned i = 0; i < NumElements; ++i) {
       // Extract the top bit of the element
@@ -1504,7 +1504,7 @@ void OpDispatchBuilder::VBROADCASTOp(OpcodeArgs, IR::OpSize ElementSize) {
     Result = _VDupElement(DstSize, ElementSize, Src, 0);
   } else {
     // Get the address to broadcast from into a GPR.
-    Ref Address = MakeSegmentAddress(Op, Op->Src[0], CTX->GetGPROpSize());
+    Ref Address = MakeSegmentAddress(Op, Op->Src[0], GetGPROpSize());
     Result = _VBroadcastFromMem(DstSize, ElementSize, Address);
   }
 
@@ -1523,7 +1523,7 @@ Ref OpDispatchBuilder::PINSROpImpl(OpcodeArgs, IR::OpSize ElementSize, const X86
 
   if (Src2Op.IsGPR()) {
     // If the source is a GPR then convert directly from the GPR.
-    auto Src2 = LoadSource_WithOpSize(GPRClass, Op, Src2Op, CTX->GetGPROpSize(), Op->Flags);
+    auto Src2 = LoadSource_WithOpSize(GPRClass, Op, Src2Op, GetGPROpSize(), Op->Flags);
     return _VInsGPR(Size, ElementSize, Index, Src1, Src2);
   }
 
@@ -1644,7 +1644,7 @@ void OpDispatchBuilder::PExtrOp(OpcodeArgs, IR::OpSize ElementSize) {
   Index &= NumElements - 1;
 
   if (Op->Dest.IsGPR()) {
-    const auto GPRSize = CTX->GetGPROpSize();
+    const auto GPRSize = GetGPROpSize();
     // Extract already zero extends the result.
     Ref Result = _VExtractToGPR(OpSize::i128Bit, OverridenElementSize, Src, Index);
     StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Result, GPRSize, OpSize::iInvalid);
@@ -2065,7 +2065,7 @@ Ref OpDispatchBuilder::CVTGPR_To_FPRImpl(OpcodeArgs, IR::OpSize DstElementSize,
   Ref Converted {};
   if (Src2Op.IsGPR()) {
     // If the source is a GPR then convert directly from the GPR.
-    auto Src2 = LoadSource_WithOpSize(GPRClass, Op, Src2Op, CTX->GetGPROpSize(), Op->Flags);
+    auto Src2 = LoadSource_WithOpSize(GPRClass, Op, Src2Op, GetGPROpSize(), Op->Flags);
     Converted = _Float_FromGPR_S(DstElementSize, SrcSize, Src2);
   } else if (SrcSize != DstElementSize) {
     // If the source is from memory but the Source size and destination size aren't the same,
@@ -2121,7 +2121,7 @@ Ref OpDispatchBuilder::CVTFPR_To_GPRImpl(OpcodeArgs, Ref Src, IR::OpSize SrcElem
     Ref Converted = _Float_ToGPR_ZS(GPRSize, SrcElementSize, Src);
 
     bool Dst32 = GPRSize == OpSize::i32Bit;
-    Ref MaxI = Dst32 ? _Constant(0x80000000) : _Constant(0x8000000000000000);
+    Ref MaxI = Dst32 ? Constant(0x80000000) : Constant(0x8000000000000000);
     Ref MaxF = LoadAndCacheNamedVectorConstant(SrcElementSize, (SrcElementSize == OpSize::i32Bit) ?
                                                                  (Dst32 ? NAMED_VECTOR_CVTMAX_F32_I32 : NAMED_VECTOR_CVTMAX_F32_I64) :
                                                                  (Dst32 ? NAMED_VECTOR_CVTMAX_F64_I32 : NAMED_VECTOR_CVTMAX_F64_I64));
@@ -2134,7 +2134,7 @@ void OpDispatchBuilder::CVTFPR_To_GPR(OpcodeArgs) {
   // If loading a vector, use the full size, so we don't
   // unnecessarily zero extend the vector. Otherwise, if
   // memory, then we want to load the element size exactly.
-  const auto SrcSize = Op->Src[0].IsGPR() ? OpSize::i128Bit : OpSizeFromSrc(Op);
+  const auto SrcSize = Op->Src[0].IsGPR() ? OpSize::i128Bit : SrcElementSize;
   Ref Src = LoadSource_WithOpSize(FPRClass, Op, Op->Src[0], SrcSize, Op->Flags);
   Ref Result = CVTFPR_To_GPRImpl(Op, Src, SrcElementSize, HostRoundingMode);
   StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
@@ -2355,7 +2355,7 @@ void OpDispatchBuilder::VMASKMOVOpImpl(OpcodeArgs, IR::OpSize ElementSize, IR::O
                                        const X86Tables::DecodedOperand& MaskOp, const X86Tables::DecodedOperand& DataOp) {
 
   const auto MakeAddress = [this, Op](const X86Tables::DecodedOperand& Data) {
-    return MakeSegmentAddress(Op, Data, CTX->GetGPROpSize());
+    return MakeSegmentAddress(Op, Data, GetGPROpSize());
   };
 
   Ref Mask = LoadSource_WithOpSize(FPRClass, Op, MaskOp, DataSize, Op->Flags);
@@ -2398,7 +2398,7 @@ void OpDispatchBuilder::MOVBetweenGPR_FPR(OpcodeArgs, VectorOpType VectorType) {
     Ref Result {};
     if (Op->Src[0].IsGPR()) {
       // Loading from GPR and moving to Vector.
-      Ref Src = LoadSource_WithOpSize(FPRClass, Op, Op->Src[0], CTX->GetGPROpSize(), Op->Flags);
+      Ref Src = LoadSource_WithOpSize(FPRClass, Op, Op->Src[0], GetGPROpSize(), Op->Flags);
       // zext to 128bit
       Result = _VCastFromGPR(OpSize::i128Bit, OpSizeFromSrc(Op), Src);
     } else {
@@ -2504,7 +2504,7 @@ Ref OpDispatchBuilder::XSaveBase(X86Tables::DecodedOp Op) {
 void OpDispatchBuilder::XSaveOpImpl(OpcodeArgs) {
   // NOTE: Mask should be EAX and EDX concatenated, but we only need to test
   //       for features that are in the lower 32 bits, so EAX only is sufficient.
-  const auto OpSize = CTX->GetGPROpSize();
+  const auto OpSize = GetGPROpSize();
 
   const auto StoreIfFlagSet = [this, OpSize](uint32_t BitIndex, auto fn, uint32_t FieldSize = 1) {
     Ref Mask = LoadGPRRegister(X86State::REG_RAX);
@@ -2539,8 +2539,7 @@ void OpDispatchBuilder::XSaveOpImpl(OpcodeArgs) {
 
   // We need to save MXCSR and MXCSR_MASK if either SSE or AVX are requested to be saved
   {
-    StoreIfFlagSet(
-      1, [this, Op] { SaveMXCSRState(XSaveBase(Op)); }, 2);
+    StoreIfFlagSet(1, [this, Op] { SaveMXCSRState(XSaveBase(Op)); }, 2);
   }
 
   // Update XSTATE_BV region of the XSAVE header
@@ -2553,7 +2552,7 @@ void OpDispatchBuilder::XSaveOpImpl(OpcodeArgs) {
 
     // XSTATE_BV section of the header is 8 bytes in size, but we only really
     // care about setting at most 3 bits in the first byte. We zero out the rest.
-    _StoreMem(GPRClass, OpSize::i64Bit, RequestedFeatures, Base, _Constant(512), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
+    _StoreMem(GPRClass, OpSize::i64Bit, RequestedFeatures, Base, Constant(512), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
   }
 }
 
@@ -2577,11 +2576,11 @@ void OpDispatchBuilder::SaveX87State(OpcodeArgs, Ref MemBase) {
     _StoreMem(GPRClass, OpSize::i16Bit, MemBase, FCW, OpSize::i16Bit);
   }
 
-  { _StoreMem(GPRClass, OpSize::i16Bit, ReconstructFSW_Helper(), MemBase, _Constant(2), OpSize::i16Bit, MEM_OFFSET_SXTX, 1); }
+  { _StoreMem(GPRClass, OpSize::i16Bit, ReconstructFSW_Helper(), MemBase, Constant(2), OpSize::i16Bit, MEM_OFFSET_SXTX, 1); }
 
   {
     // Abridged FTW
-    _StoreMem(GPRClass, OpSize::i8Bit, LoadContext(AbridgedFTWIndex), MemBase, _Constant(4), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
+    _StoreMem(GPRClass, OpSize::i8Bit, LoadContext(AbridgedFTWIndex), MemBase, Constant(4), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
   }
 
   // BYTE | 0 1 | 2 3 | 4   | 5     | 6 7 | 8 9 | a b | c d | e f |
@@ -2635,7 +2634,7 @@ void OpDispatchBuilder::SaveX87State(OpcodeArgs, Ref MemBase) {
 }
 
 void OpDispatchBuilder::SaveSSEState(Ref MemBase) {
-  const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U;
+  const auto NumRegs = Is64BitMode ? 16U : 8U;
 
   for (uint32_t i = 0; i < NumRegs; i += 2) {
     _StoreMemPair(FPRClass, OpSize::i128Bit, LoadXMMRegister(i), LoadXMMRegister(i + 1), MemBase, i * 16 + 160);
@@ -2644,11 +2643,11 @@ void OpDispatchBuilder::SaveSSEState(Ref MemBase) {
 
 void OpDispatchBuilder::SaveMXCSRState(Ref MemBase) {
   // Store MXCSR and the mask for all bits.
-  _StoreMemPair(GPRClass, OpSize::i32Bit, GetMXCSR(), _Constant(0xFFFF), MemBase, 24);
+  _StoreMemPair(GPRClass, OpSize::i32Bit, GetMXCSR(), Constant(0xFFFF), MemBase, 24);
 }
 
 void OpDispatchBuilder::SaveAVXState(Ref MemBase) {
-  const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U;
+  const auto NumRegs = Is64BitMode ? 16U : 8U;
 
   for (uint32_t i = 0; i < NumRegs; i += 2) {
     Ref Upper0 = _VDupElement(OpSize::i256Bit, OpSize::i128Bit, LoadXMMRegister(i + 0), 1);
@@ -2662,7 +2661,7 @@ Ref OpDispatchBuilder::GetMXCSR() {
   Ref MXCSR = _LoadContext(OpSize::i32Bit, GPRClass, offsetof(FEXCore::Core::CPUState, mxcsr));
   // Mask out unsupported bits
   // Keeps FZ, RC, exception masks, and DAZ
-  MXCSR = _And(OpSize::i32Bit, MXCSR, _Constant(0xFFC0));
+  MXCSR = _And(OpSize::i32Bit, MXCSR, Constant(0xFFC0));
   return MXCSR;
 }
 
@@ -2672,12 +2671,12 @@ void OpDispatchBuilder::FXRStoreOp(OpcodeArgs) {
   RestoreX87State(Mem);
   RestoreSSEState(Mem);
 
-  Ref MXCSR = _LoadMem(GPRClass, OpSize::i32Bit, Mem, _Constant(24), OpSize::i32Bit, MEM_OFFSET_SXTX, 1);
+  Ref MXCSR = _LoadMem(GPRClass, OpSize::i32Bit, Mem, Constant(24), OpSize::i32Bit, MEM_OFFSET_SXTX, 1);
   RestoreMXCSRState(MXCSR);
 }
 
 void OpDispatchBuilder::XRstorOpImpl(OpcodeArgs) {
-  const auto OpSize = CTX->GetGPROpSize();
+  const auto OpSize = GetGPROpSize();
 
   // If a bit in our XSTATE_BV is set, then we restore from that region of the XSAVE area,
   // otherwise, if not set, then we need to set the relevant data the bit corresponds to
@@ -2689,7 +2688,7 @@ void OpDispatchBuilder::XRstorOpImpl(OpcodeArgs) {
     // Note: we rematerialize Base/Mask in each block to avoid crossblock
     // liveness.
     Ref Base = XSaveBase(Op);
-    Ref Mask = _LoadMem(GPRClass, OpSize::i64Bit, Base, _Constant(512), OpSize::i64Bit, MEM_OFFSET_SXTX, 1);
+    Ref Mask = _LoadMem(GPRClass, OpSize::i64Bit, Base, Constant(512), OpSize::i64Bit, MEM_OFFSET_SXTX, 1);
 
     Ref BitFlag = _Bfe(OpSize, FieldSize, BitIndex, Mask);
     auto CondJump_ = CondJump(BitFlag, {COND_NEQ});
@@ -2715,13 +2714,11 @@ void OpDispatchBuilder::XRstorOpImpl(OpcodeArgs) {
 
   // x87
   {
-    RestoreIfFlagSetOrDefault(
-      0, [this, Op] { RestoreX87State(XSaveBase(Op)); }, [this, Op] { DefaultX87State(Op); });
+    RestoreIfFlagSetOrDefault(0, [this, Op] { RestoreX87State(XSaveBase(Op)); }, [this, Op] { DefaultX87State(Op); });
   }
   // SSE
   {
-    RestoreIfFlagSetOrDefault(
-      1, [this, Op] { RestoreSSEState(XSaveBase(Op)); }, [this] { DefaultSSEState(); });
+    RestoreIfFlagSetOrDefault(1, [this, Op] { RestoreSSEState(XSaveBase(Op)); }, [this] { DefaultSSEState(); });
   }
   // AVX
   if (CTX->HostFeatures.SupportsAVX) {
@@ -2734,9 +2731,9 @@ void OpDispatchBuilder::XRstorOpImpl(OpcodeArgs) {
     RestoreIfFlagSetOrDefault(
       1,
       [this, Op] {
-      Ref Base = XSaveBase(Op);
-      Ref MXCSR = _LoadMem(GPRClass, OpSize::i32Bit, Base, _Constant(24), OpSize::i32Bit, MEM_OFFSET_SXTX, 1);
-      RestoreMXCSRState(MXCSR);
+        Ref Base = XSaveBase(Op);
+        Ref MXCSR = _LoadMem(GPRClass, OpSize::i32Bit, Base, Constant(24), OpSize::i32Bit, MEM_OFFSET_SXTX, 1);
+        RestoreMXCSRState(MXCSR);
       },
       [] { /* Intentionally do nothing*/ }, 2);
   }
@@ -2747,13 +2744,13 @@ void OpDispatchBuilder::RestoreX87State(Ref MemBase) {
   _StoreContext(OpSize::i16Bit, GPRClass, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));
 
   {
-    auto NewFSW = _LoadMem(GPRClass, OpSize::i16Bit, MemBase, _Constant(2), OpSize::i16Bit, MEM_OFFSET_SXTX, 1);
+    auto NewFSW = _LoadMem(GPRClass, OpSize::i16Bit, MemBase, Constant(2), OpSize::i16Bit, MEM_OFFSET_SXTX, 1);
     ReconstructX87StateFromFSW_Helper(NewFSW);
   }
 
   {
     // Abridged FTW
-    StoreContext(AbridgedFTWIndex, _LoadMem(GPRClass, OpSize::i8Bit, MemBase, _Constant(4), OpSize::i8Bit, MEM_OFFSET_SXTX, 1));
+    StoreContext(AbridgedFTWIndex, _LoadMem(GPRClass, OpSize::i8Bit, MemBase, Constant(4), OpSize::i8Bit, MEM_OFFSET_SXTX, 1));
   }
 
   for (uint32_t i = 0; i < Core::CPUState::NUM_MMS; i += 2) {
@@ -2765,7 +2762,7 @@ void OpDispatchBuilder::RestoreX87State(Ref MemBase) {
 }
 
 void OpDispatchBuilder::RestoreSSEState(Ref MemBase) {
-  const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U;
+  const auto NumRegs = Is64BitMode ? 16U : 8U;
 
   for (uint32_t i = 0; i < NumRegs; i += 2) {
     auto XMMRegs = LoadMemPair(FPRClass, OpSize::i128Bit, MemBase, i * 16 + 160);
@@ -2777,7 +2774,7 @@ void OpDispatchBuilder::RestoreSSEState(Ref MemBase) {
 
 void OpDispatchBuilder::RestoreMXCSRState(Ref MXCSR) {
   // Mask out unsupported bits
-  MXCSR = _And(OpSize::i32Bit, MXCSR, _Constant(0xFFC0));
+  MXCSR = _And(OpSize::i32Bit, MXCSR, Constant(0xFFC0));
 
   _StoreContext(OpSize::i32Bit, GPRClass, MXCSR, offsetof(FEXCore::Core::CPUState, mxcsr));
   // We only support the rounding mode and FTZ bit being set
@@ -2786,7 +2783,7 @@ void OpDispatchBuilder::RestoreMXCSRState(Ref MXCSR) {
 }
 
 void OpDispatchBuilder::RestoreAVXState(Ref MemBase) {
-  const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U;
+  const auto NumRegs = Is64BitMode ? 16U : 8U;
 
   for (uint32_t i = 0; i < NumRegs; i += 2) {
     Ref XMMReg0 = LoadXMMRegister(i + 0);
@@ -2811,7 +2808,7 @@ void OpDispatchBuilder::DefaultX87State(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::DefaultSSEState() {
-  const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U;
+  const auto NumRegs = Is64BitMode ? 16U : 8U;
 
   Ref ZeroVector = LoadZeroVector(OpSize::i128Bit);
   for (uint32_t i = 0; i < NumRegs; ++i) {
@@ -2820,7 +2817,7 @@ void OpDispatchBuilder::DefaultSSEState() {
 }
 
 void OpDispatchBuilder::DefaultAVXState() {
-  const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U;
+  const auto NumRegs = Is64BitMode ? 16U : 8U;
 
   for (uint32_t i = 0; i < NumRegs; i++) {
     Ref Reg = LoadXMMRegister(i);
@@ -2877,7 +2874,7 @@ void OpDispatchBuilder::VPALIGNROp(OpcodeArgs) {
 
 template<IR::OpSize ElementSize>
 void OpDispatchBuilder::UCOMISxOp(OpcodeArgs) {
-  const auto SrcSize = Op->Src[0].IsGPR() ? GetGuestVectorLength() : OpSizeFromSrc(Op);
+  const auto SrcSize = Op->Src[0].IsGPR() ? GetGuestVectorLength() : ElementSize;
   Ref Src1 = LoadSource_WithOpSize(FPRClass, Op, Op->Dest, GetGuestVectorLength(), Op->Flags);
   Ref Src2 = LoadSource_WithOpSize(FPRClass, Op, Op->Src[0], SrcSize, Op->Flags);
 
@@ -2888,12 +2885,12 @@ template void OpDispatchBuilder::UCOMISxOp<OpSize::i32Bit>(OpcodeArgs);
 template void OpDispatchBuilder::UCOMISxOp<OpSize::i64Bit>(OpcodeArgs);
 
 void OpDispatchBuilder::LDMXCSR(OpcodeArgs) {
-  Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags);
+  Ref Dest = LoadSource_WithOpSize(GPRClass, Op, Op->Dest, OpSize::i32Bit, Op->Flags);
   RestoreMXCSRState(Dest);
 }
 
 void OpDispatchBuilder::STMXCSR(OpcodeArgs) {
-  StoreResult(GPRClass, Op, GetMXCSR(), OpSize::iInvalid);
+  StoreResult_WithOpSize(GPRClass, Op, Op->Dest, GetMXCSR(), OpSize::i32Bit, OpSize::iInvalid);
 }
 
 template<IR::OpSize ElementSize>
@@ -3953,8 +3950,8 @@ void OpDispatchBuilder::PTestOpImpl(OpSize Size, Ref Dest, Ref Src) {
   Test1 = _VExtractToGPR(Size, OpSize::i16Bit, Test1, 0);
   Test2 = _VExtractToGPR(Size, OpSize::i16Bit, Test2, 0);
 
-  auto ZeroConst = _Constant(0);
-  auto OneConst = _Constant(1);
+  auto ZeroConst = Constant(0);
+  auto OneConst = Constant(1);
 
   Test2 = _Select(FEXCore::IR::COND_NEQ, Test2, ZeroConst, OneConst, ZeroConst);
 
@@ -3979,7 +3976,7 @@ void OpDispatchBuilder::VTESTOpImpl(OpSize SrcSize, IR::OpSize ElementSize, Ref
   const auto ElementSizeInBits = IR::OpSizeAsBits(ElementSize);
   const auto MaskConstant = uint64_t {1} << (ElementSizeInBits - 1);
 
-  Ref Mask = _VDupFromGPR(SrcSize, ElementSize, _Constant(MaskConstant));
+  Ref Mask = _VDupFromGPR(SrcSize, ElementSize, Constant(MaskConstant));
 
   Ref AndTest = _VAnd(SrcSize, OpSize::i8Bit, Src2, Src1);
   Ref AndNotTest = _VAndn(SrcSize, OpSize::i8Bit, Src2, Src1);
@@ -3993,8 +3990,8 @@ void OpDispatchBuilder::VTESTOpImpl(OpSize SrcSize, IR::OpSize ElementSize, Ref
   Ref AndGPR = _VExtractToGPR(SrcSize, OpSize::i16Bit, MaxAnd, 0);
   Ref AndNotGPR = _VExtractToGPR(SrcSize, OpSize::i16Bit, MaxAndNot, 0);
 
-  Ref ZeroConst = _Constant(0);
-  Ref OneConst = _Constant(1);
+  Ref ZeroConst = Constant(0);
+  Ref OneConst = Constant(1);
 
   Ref CFInv = _Select(IR::COND_NEQ, AndNotGPR, ZeroConst, OneConst, ZeroConst);
 
@@ -4583,7 +4580,7 @@ void OpDispatchBuilder::VPERMDOp(OpcodeArgs) {
   // Get rid of any junk unrelated to the relevant selector index bits (bits [2:0])
   Ref IndexMask = _VectorImm(DstSize, OpSize::i32Bit, 0b111);
 
-  Ref AddConst = _Constant(0x03020100);
+  Ref AddConst = Constant(0x03020100);
   Ref Repeating3210 = _VDupFromGPR(DstSize, OpSize::i32Bit, AddConst);
   Ref FinalIndices = VPERMDIndices(OpSizeFromDst(Op), Indices, IndexMask, Repeating3210);
 
@@ -4732,7 +4729,7 @@ void OpDispatchBuilder::VPBLENDWOp(OpcodeArgs) {
 void OpDispatchBuilder::VZEROOp(OpcodeArgs) {
   const auto DstSize = OpSizeFromDst(Op);
   const auto IsVZEROALL = DstSize == OpSize::i256Bit;
-  const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U;
+  const auto NumRegs = Is64BitMode ? 16U : 8U;
 
   if (IsVZEROALL) {
     // NOTE: Despite the name being VZEROALL, this will still only ever
@@ -4818,7 +4815,7 @@ Ref OpDispatchBuilder::VPERMILRegOpImpl(OpSize DstSize, IR::OpSize ElementSize,
   Ref ShiftedIndices = _VShlI(DstSize, OpSize::i8Bit, IndexTrn3, IndexShift);
 
   uint64_t VConstant = IsPD ? 0x0706050403020100 : 0x03020100;
-  Ref VectorConst = _VDupFromGPR(DstSize, ElementSize, _Constant(VConstant));
+  Ref VectorConst = _VDupFromGPR(DstSize, ElementSize, Constant(VConstant));
   Ref FinalIndices {};
 
   if (Is256Bit) {
@@ -4877,7 +4874,7 @@ void OpDispatchBuilder::PCMPXSTRXOpImpl(OpcodeArgs, bool IsExplicit, bool IsMask
     IntermediateResult = _VPCMPISTRX(Src1, Src2, Control);
   }
 
-  Ref ZeroConst = _Constant(0);
+  Ref ZeroConst = Constant(0);
 
   if (IsMask) {
     // For the masked variant of the instructions, if control[6] is set, then we
@@ -4914,7 +4911,7 @@ void OpDispatchBuilder::PCMPXSTRXOpImpl(OpcodeArgs, bool IsExplicit, bool IsMask
 
     Ref ResultNoFlags = _Bfe(OpSize::i32Bit, 16, 0, IntermediateResult);
 
-    Ref IfZero = _Constant(16 >> (Control & 1));
+    Ref IfZero = Constant(16 >> (Control & 1));
     Ref IfNotZero = UseMSBIndex ? _FindMSB(IR::OpSize::i32Bit, ResultNoFlags) : _FindLSB(IR::OpSize::i32Bit, ResultNoFlags);
     Ref Result = _Select(IR::COND_EQ, ResultNoFlags, ZeroConst, IfZero, IfNotZero);
 
@@ -4947,9 +4944,14 @@ void OpDispatchBuilder::VFMAImpl(OpcodeArgs, IROps IROp, bool Scalar, uint8_t Sr
 
   const OpSize ElementSize = Op->Flags & X86Tables::DecodeFlags::FLAG_OPTION_AVX_W ? OpSize::i64Bit : OpSize::i32Bit;
 
-  Ref Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags);
-  Ref Src1 = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags);
-  Ref Src2 = LoadSource(FPRClass, Op, Op->Src[1], Op->Flags);
+  Ref Dest = LoadSource_WithOpSize(FPRClass, Op, Op->Dest, Size, Op->Flags);
+  Ref Src1 = LoadSource_WithOpSize(FPRClass, Op, Op->Src[0], Size, Op->Flags);
+  Ref Src2 {};
+  if (Op->Src[1].IsGPR()) {
+    Src2 = LoadSource_WithOpSize(FPRClass, Op, Op->Src[1], Size, Op->Flags);
+  } else {
+    Src2 = LoadSource(FPRClass, Op, Op->Src[1], Op->Flags);
+  }
 
   Ref Sources[3] = {
     Dest,
@@ -5082,9 +5084,9 @@ void OpDispatchBuilder::VPGATHER(OpcodeArgs) {
     ///< BaseAddr doesn't need to exist, calculate that here.
     Ref BaseAddr = VSIB.BaseAddr;
     if (BaseAddr && VSIB.Displacement) {
-      BaseAddr = _Add(OpSize::i64Bit, BaseAddr, _Constant(VSIB.Displacement));
+      BaseAddr = _Add(OpSize::i64Bit, BaseAddr, Constant(VSIB.Displacement));
     } else if (VSIB.Displacement) {
-      BaseAddr = _Constant(VSIB.Displacement);
+      BaseAddr = Constant(VSIB.Displacement);
     } else if (!BaseAddr) {
       BaseAddr = Invalid();
     }

FEXCore/Source/Interface/Core/OpcodeDispatcher/X87.cpp

@@ -32,12 +32,12 @@ Ref OpDispatchBuilder::GetX87Top() {
 }
 
 void OpDispatchBuilder::SetX87FTW(Ref FTW) {
-  Ref X87Empty = _Constant(static_cast<uint8_t>(FPState::X87Tag::Empty));
+  Ref X87Empty = Constant(static_cast<uint8_t>(FPState::X87Tag::Empty));
   Ref NewAbridgedFTW {};
 
   for (int i = 0; i < 8; i++) {
     Ref RegTag = _Bfe(OpSize::i32Bit, 2, i * 2, FTW);
-    Ref RegValid = _Select(FEXCore::IR::COND_NEQ, RegTag, X87Empty, _Constant(1), _Constant(0));
+    Ref RegValid = _Select(FEXCore::IR::COND_NEQ, RegTag, X87Empty, Constant(1), Constant(0));
 
     if (i) {
       NewAbridgedFTW = _Orlshl(OpSize::i32Bit, NewAbridgedFTW, RegValid, i);
@@ -84,9 +84,9 @@ void OpDispatchBuilder::FBSTP(OpcodeArgs) {
   _PopStackDestroy();
 }
 
-void OpDispatchBuilder::FLD_Const(OpcodeArgs, NamedVectorConstant Constant) {
+void OpDispatchBuilder::FLD_Const(OpcodeArgs, NamedVectorConstant K) {
   // Update TOP
-  Ref Data = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, Constant);
+  Ref Data = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, K);
   _PushStack(Data, Data, OpSize::i128Bit, true);
 }
 
@@ -104,16 +104,16 @@ void OpDispatchBuilder::FILD(OpcodeArgs) {
   SaveNZCV();
 
   // Extract sign and make integer absolute
-  auto zero = _Constant(0);
+  auto zero = Constant(0);
   _SubNZCV(OpSize::i64Bit, Data, zero);
-  auto sign = _NZCVSelect(OpSize::i64Bit, CondClassType {COND_SLT}, _Constant(0x8000), zero);
+  auto sign = _NZCVSelect(OpSize::i64Bit, CondClassType {COND_SLT}, Constant(0x8000), zero);
   auto absolute = _Neg(OpSize::i64Bit, Data, CondClassType {COND_MI});
 
   // left justify the absolute integer
-  auto shift = _Sub(OpSize::i64Bit, _Constant(63), _FindMSB(IR::OpSize::i64Bit, absolute));
+  auto shift = _Sub(OpSize::i64Bit, Constant(63), _FindMSB(IR::OpSize::i64Bit, absolute));
   auto shifted = _Lshl(OpSize::i64Bit, absolute, shift);
 
-  auto adjusted_exponent = _Sub(OpSize::i64Bit, _Constant(0x3fff + 63), shift);
+  auto adjusted_exponent = _Sub(OpSize::i64Bit, Constant(0x3fff + 63), shift);
   auto zeroed_exponent = _Select(COND_EQ, absolute, zero, zero, adjusted_exponent);
   auto upper = _Or(OpSize::i64Bit, sign, zeroed_exponent);
 
@@ -125,7 +125,7 @@ void OpDispatchBuilder::FST(OpcodeArgs, IR::OpSize Width) {
   const auto SourceSize = ReducedPrecisionMode ? OpSize::i64Bit : OpSize::i128Bit;
   AddressMode A = DecodeAddress(Op, Op->Dest, MemoryAccessType::DEFAULT, false);
 
-  A = SelectAddressMode(this, A, CTX->GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, false, false, Width);
+  A = SelectAddressMode(this, A, GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, false, false, Width);
   _StoreStackMem(SourceSize, Width, A.Base, A.Index, OpSize::iInvalid, A.IndexType, A.IndexScale, /*Float=*/true);
 
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
@@ -148,6 +148,30 @@ void OpDispatchBuilder::FSTToStack(OpcodeArgs) {
 void OpDispatchBuilder::FIST(OpcodeArgs, bool Truncate) {
   const auto Size = OpSizeFromSrc(Op);
   Ref Data = _ReadStackValue(0);
+
+  // For 16-bit integers, we need to manually check for overflow
+  // since _F80CVTInt doesn't handle 16-bit overflow detection properly
+  if (Size == OpSize::i16Bit) {
+    // Extract the 80-bit float value to check for special cases
+    // Get the upper 64 bits which contain sign and exponent and then the exponent from upper.
+    Ref Upper = _VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, Data, 1);
+    Ref Exponent = _And(OpSize::i64Bit, Upper, Constant(0x7fff));
+
+    // Check for NaN/Infinity: exponent = 0x7fff
+    SaveNZCV();
+    _TestNZ(OpSize::i64Bit, Exponent, Constant(0x7fff));
+    Ref IsSpecial = _NZCVSelect(OpSize::i64Bit, {COND_EQ}, Constant(1), Constant(0));
+
+    // For overflow detection, check if exponent indicates a value >= 2^15
+    // Biased exponent for 2^15 is 0x3fff + 15 = 0x400e
+    _SubWithFlags(OpSize::i64Bit, Exponent, Constant(0x400e));
+    Ref IsOverflow = _NZCVSelect(OpSize::i64Bit, {COND_UGE}, Constant(1), Constant(0));
+
+    // Set Invalid Operation flag if overflow or special value
+    Ref InvalidFlag = _Or(OpSize::i64Bit, IsSpecial, IsOverflow);
+    SetRFLAG<FEXCore::X86State::X87FLAG_IE_LOC>(InvalidFlag);
+  }
+
   Data = _F80CVTInt(Size, Data, Truncate);
 
   StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Data, Size, OpSize::i8Bit);
@@ -312,17 +336,17 @@ Ref OpDispatchBuilder::GetX87FTW_Helper() {
   // https://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN
   Ref X = LoadContext(AbridgedFTWIndex);
   X = _Orlshl(OpSize::i32Bit, X, X, 4);
-  X = _And(OpSize::i32Bit, X, _Constant(0x0f0f0f0f));
+  X = _And(OpSize::i32Bit, X, Constant(0x0f0f0f0f));
   X = _Orlshl(OpSize::i32Bit, X, X, 2);
-  X = _And(OpSize::i32Bit, X, _Constant(0x33333333));
+  X = _And(OpSize::i32Bit, X, Constant(0x33333333));
   X = _Orlshl(OpSize::i32Bit, X, X, 1);
-  X = _And(OpSize::i32Bit, X, _Constant(0x55555555));
+  X = _And(OpSize::i32Bit, X, Constant(0x55555555));
   X = _Orlshl(OpSize::i32Bit, X, X, 1);
 
   // The above sequence sets valid to 11 and empty to 00, so invert to finalize.
   static_assert(static_cast<uint8_t>(FPState::X87Tag::Valid) == 0b00);
   static_assert(static_cast<uint8_t>(FPState::X87Tag::Empty) == 0b11);
-  return _Xor(OpSize::i32Bit, X, _Constant(0xffff));
+  return _Xor(OpSize::i32Bit, X, Constant(0xffff));
 }
 
 void OpDispatchBuilder::X87FNSTENV(OpcodeArgs) {
@@ -359,33 +383,33 @@ void OpDispatchBuilder::X87FNSTENV(OpcodeArgs) {
     _StoreMem(GPRClass, Size, Mem, FCW, Size);
   }
 
-  { _StoreMem(GPRClass, Size, ReconstructFSW_Helper(), Mem, _Constant(IR::OpSizeToSize(Size) * 1), Size, MEM_OFFSET_SXTX, 1); }
+  { _StoreMem(GPRClass, Size, ReconstructFSW_Helper(), Mem, Constant(IR::OpSizeToSize(Size) * 1), Size, MEM_OFFSET_SXTX, 1); }
 
-  auto ZeroConst = _Constant(0);
+  auto ZeroConst = Constant(0);
 
   {
     // FTW
-    _StoreMem(GPRClass, Size, GetX87FTW_Helper(), Mem, _Constant(IR::OpSizeToSize(Size) * 2), Size, MEM_OFFSET_SXTX, 1);
+    _StoreMem(GPRClass, Size, GetX87FTW_Helper(), Mem, Constant(IR::OpSizeToSize(Size) * 2), Size, MEM_OFFSET_SXTX, 1);
   }
 
   {
     // Instruction Offset
-    _StoreMem(GPRClass, Size, ZeroConst, Mem, _Constant(IR::OpSizeToSize(Size) * 3), Size, MEM_OFFSET_SXTX, 1);
+    _StoreMem(GPRClass, Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MEM_OFFSET_SXTX, 1);
   }
 
   {
     // Instruction CS selector (+ Opcode)
-    _StoreMem(GPRClass, Size, ZeroConst, Mem, _Constant(IR::OpSizeToSize(Size) * 4), Size, MEM_OFFSET_SXTX, 1);
+    _StoreMem(GPRClass, Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 4), Size, MEM_OFFSET_SXTX, 1);
   }
 
   {
     // Data pointer offset
-    _StoreMem(GPRClass, Size, ZeroConst, Mem, _Constant(IR::OpSizeToSize(Size) * 5), Size, MEM_OFFSET_SXTX, 1);
+    _StoreMem(GPRClass, Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 5), Size, MEM_OFFSET_SXTX, 1);
   }
 
   {
     // Data pointer selector
-    _StoreMem(GPRClass, Size, ZeroConst, Mem, _Constant(IR::OpSizeToSize(Size) * 6), Size, MEM_OFFSET_SXTX, 1);
+    _StoreMem(GPRClass, Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 6), Size, MEM_OFFSET_SXTX, 1);
   }
 }
 
@@ -415,13 +439,13 @@ void OpDispatchBuilder::X87LDENV(OpcodeArgs) {
   auto NewFCW = _LoadMem(GPRClass, OpSize::i16Bit, Mem, OpSize::i16Bit);
   _StoreContext(OpSize::i16Bit, GPRClass, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));
 
-  Ref MemLocation = _Add(OpSize::i64Bit, Mem, _Constant(IR::OpSizeToSize(Size) * 1));
+  Ref MemLocation = _Add(OpSize::i64Bit, Mem, Constant(IR::OpSizeToSize(Size) * 1));
   auto NewFSW = _LoadMem(GPRClass, Size, MemLocation, Size);
   ReconstructX87StateFromFSW_Helper(NewFSW);
 
   {
     // FTW
-    Ref MemLocation = _Add(OpSize::i64Bit, Mem, _Constant(IR::OpSizeToSize(Size) * 2));
+    Ref MemLocation = _Add(OpSize::i64Bit, Mem, Constant(IR::OpSizeToSize(Size) * 2));
     SetX87FTW(_LoadMem(GPRClass, Size, MemLocation, Size));
   }
 }
@@ -455,44 +479,44 @@ void OpDispatchBuilder::X87FNSAVE(OpcodeArgs) {
     _StoreMem(GPRClass, Size, Mem, FCW, Size);
   }
 
-  { _StoreMem(GPRClass, Size, ReconstructFSW_Helper(), Mem, _Constant(IR::OpSizeToSize(Size) * 1), Size, MEM_OFFSET_SXTX, 1); }
+  { _StoreMem(GPRClass, Size, ReconstructFSW_Helper(), Mem, Constant(IR::OpSizeToSize(Size) * 1), Size, MEM_OFFSET_SXTX, 1); }
 
-  auto ZeroConst = _Constant(0);
+  auto ZeroConst = Constant(0);
 
   {
     // FTW
-    _StoreMem(GPRClass, Size, GetX87FTW_Helper(), Mem, _Constant(IR::OpSizeToSize(Size) * 2), Size, MEM_OFFSET_SXTX, 1);
+    _StoreMem(GPRClass, Size, GetX87FTW_Helper(), Mem, Constant(IR::OpSizeToSize(Size) * 2), Size, MEM_OFFSET_SXTX, 1);
   }
 
   {
     // Instruction Offset
-    _StoreMem(GPRClass, Size, ZeroConst, Mem, _Constant(IR::OpSizeToSize(Size) * 3), Size, MEM_OFFSET_SXTX, 1);
+    _StoreMem(GPRClass, Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MEM_OFFSET_SXTX, 1);
   }
 
   {
     // Instruction CS selector (+ Opcode)
-    _StoreMem(GPRClass, Size, ZeroConst, Mem, _Constant(IR::OpSizeToSize(Size) * 4), Size, MEM_OFFSET_SXTX, 1);
+    _StoreMem(GPRClass, Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 4), Size, MEM_OFFSET_SXTX, 1);
   }
 
   {
     // Data pointer offset
-    _StoreMem(GPRClass, Size, ZeroConst, Mem, _Constant(IR::OpSizeToSize(Size) * 5), Size, MEM_OFFSET_SXTX, 1);
+    _StoreMem(GPRClass, Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 5), Size, MEM_OFFSET_SXTX, 1);
   }
 
   {
     // Data pointer selector
-    _StoreMem(GPRClass, Size, ZeroConst, Mem, _Constant(IR::OpSizeToSize(Size) * 6), Size, MEM_OFFSET_SXTX, 1);
+    _StoreMem(GPRClass, Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 6), Size, MEM_OFFSET_SXTX, 1);
   }
 
-  auto OneConst = _Constant(1);
-  auto SevenConst = _Constant(7);
+  auto OneConst = Constant(1);
+  auto SevenConst = Constant(7);
   const auto LoadSize = ReducedPrecisionMode ? OpSize::i64Bit : OpSize::i128Bit;
   for (int i = 0; i < 7; ++i) {
     Ref data = _LoadContextIndexed(Top, LoadSize, MMBaseOffset(), IR::OpSizeToSize(OpSize::i128Bit), FPRClass);
     if (ReducedPrecisionMode) {
       data = _F80CVTTo(data, OpSize::i64Bit);
     }
-    _StoreMem(FPRClass, OpSize::i128Bit, data, Mem, _Constant((IR::OpSizeToSize(Size) * 7) + (10 * i)), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
+    _StoreMem(FPRClass, OpSize::i128Bit, data, Mem, Constant((IR::OpSizeToSize(Size) * 7) + (10 * i)), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
     Top = _And(OpSize::i32Bit, _Add(OpSize::i32Bit, Top, OneConst), SevenConst);
   }
 
@@ -504,9 +528,9 @@ void OpDispatchBuilder::X87FNSAVE(OpcodeArgs) {
   // ST7 broken in to two parts
   // Lower 64bits [63:0]
   // upper 16 bits [79:64]
-  _StoreMem(FPRClass, OpSize::i64Bit, data, Mem, _Constant((IR::OpSizeToSize(Size) * 7) + (7 * 10)), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
+  _StoreMem(FPRClass, OpSize::i64Bit, data, Mem, Constant((IR::OpSizeToSize(Size) * 7) + (7 * 10)), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
   auto topBytes = _VDupElement(OpSize::i128Bit, OpSize::i16Bit, data, 4);
-  _StoreMem(FPRClass, OpSize::i16Bit, topBytes, Mem, _Constant((IR::OpSizeToSize(Size) * 7) + (7 * 10) + 8), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
+  _StoreMem(FPRClass, OpSize::i16Bit, topBytes, Mem, Constant((IR::OpSizeToSize(Size) * 7) + (7 * 10) + 8), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
 
   // reset to default
   FNINIT(Op);
@@ -523,29 +547,29 @@ void OpDispatchBuilder::X87FRSTOR(OpcodeArgs) {
     // ignore the rounding precision, we're always 64-bit in F64.
     // extract rounding mode
     Ref roundingMode = NewFCW;
-    auto roundShift = _Constant(10);
-    auto roundMask = _Constant(3);
+    auto roundShift = Constant(10);
+    auto roundMask = Constant(3);
     roundingMode = _Lshr(OpSize::i32Bit, roundingMode, roundShift);
     roundingMode = _And(OpSize::i32Bit, roundingMode, roundMask);
     _SetRoundingMode(roundingMode, false, roundingMode);
   }
 
-  auto NewFSW = _LoadMem(GPRClass, Size, Mem, _Constant(IR::OpSizeToSize(Size) * 1), Size, MEM_OFFSET_SXTX, 1);
+  auto NewFSW = _LoadMem(GPRClass, Size, Mem, Constant(IR::OpSizeToSize(Size) * 1), Size, MEM_OFFSET_SXTX, 1);
   Ref Top = ReconstructX87StateFromFSW_Helper(NewFSW);
   {
     // FTW
-    SetX87FTW(_LoadMem(GPRClass, Size, Mem, _Constant(IR::OpSizeToSize(Size) * 2), Size, MEM_OFFSET_SXTX, 1));
+    SetX87FTW(_LoadMem(GPRClass, Size, Mem, Constant(IR::OpSizeToSize(Size) * 2), Size, MEM_OFFSET_SXTX, 1));
   }
 
-  auto OneConst = _Constant(1);
-  auto SevenConst = _Constant(7);
+  auto OneConst = Constant(1);
+  auto SevenConst = Constant(7);
 
-  auto low = _Constant(~0ULL);
-  auto high = _Constant(0xFFFF);
+  auto low = Constant(~0ULL);
+  auto high = Constant(0xFFFF);
   Ref Mask = _VLoadTwoGPRs(low, high);
   const auto StoreSize = ReducedPrecisionMode ? OpSize::i64Bit : OpSize::i128Bit;
   for (int i = 0; i < 7; ++i) {
-    Ref Reg = _LoadMem(FPRClass, OpSize::i128Bit, Mem, _Constant((IR::OpSizeToSize(Size) * 7) + (10 * i)), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
+    Ref Reg = _LoadMem(FPRClass, OpSize::i128Bit, Mem, Constant((IR::OpSizeToSize(Size) * 7) + (10 * i)), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
     // Mask off the top bits
     Reg = _VAnd(OpSize::i128Bit, OpSize::i128Bit, Reg, Mask);
     if (ReducedPrecisionMode) {
@@ -561,9 +585,9 @@ void OpDispatchBuilder::X87FRSTOR(OpcodeArgs) {
   // ST7 broken in to two parts
   // Lower 64bits [63:0]
   // upper 16 bits [79:64]
-  Ref Reg = _LoadMem(FPRClass, OpSize::i64Bit, Mem, _Constant((IR::OpSizeToSize(Size) * 7) + (10 * 7)), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
+  Ref Reg = _LoadMem(FPRClass, OpSize::i64Bit, Mem, Constant((IR::OpSizeToSize(Size) * 7) + (10 * 7)), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
   Ref RegHigh =
-    _LoadMem(FPRClass, OpSize::i16Bit, Mem, _Constant((IR::OpSizeToSize(Size) * 7) + (10 * 7) + 8), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
+    _LoadMem(FPRClass, OpSize::i16Bit, Mem, Constant((IR::OpSizeToSize(Size) * 7) + (10 * 7) + 8), OpSize::i8Bit, MEM_OFFSET_SXTX, 1);
   Reg = _VInsElement(OpSize::i128Bit, OpSize::i16Bit, 4, 0, Reg, RegHigh);
   if (ReducedPrecisionMode) {
     Reg = _F80CVT(OpSize::i64Bit, Reg); // Convert to double precision
@@ -592,13 +616,13 @@ void OpDispatchBuilder::FXCH(OpcodeArgs) {
   if (Offset != 0) {
     _F80StackXchange(Offset);
   }
-  SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(_Constant(0));
+  SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Constant(0));
 }
 
 void OpDispatchBuilder::X87FYL2X(OpcodeArgs, bool IsFYL2XP1) {
   if (IsFYL2XP1) {
     // create an add between top of stack and 1.
-    Ref One = ReducedPrecisionMode ? _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, _Constant(0x3FF0000000000000)) :
+    Ref One = ReducedPrecisionMode ? _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, Constant(0x3FF0000000000000)) :
                                      LoadAndCacheNamedVectorConstant(OpSize::i128Bit, NamedVectorConstant::NAMED_VECTOR_X87_ONE);
     _F80AddValue(0, One);
   }
@@ -639,7 +663,7 @@ void OpDispatchBuilder::FCOMI(OpcodeArgs, IR::OpSize Width, bool Integer, OpDisp
 
   if (WhichFlags == FCOMIFlags::FLAGS_X87) {
     SetRFLAG<FEXCore::X86State::X87FLAG_C0_LOC>(HostFlag_CF);
-    SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(_Constant(0));
+    SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Constant(0));
     SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(HostFlag_Unordered);
     SetRFLAG<FEXCore::X86State::X87FLAG_C3_LOC>(HostFlag_ZF);
   } else {
@@ -649,7 +673,7 @@ void OpDispatchBuilder::FCOMI(OpcodeArgs, IR::OpSize Width, bool Integer, OpDisp
 
     // PF is stored inverted, so invert from the host flag.
     // TODO: This could perhaps be optimized?
-    auto PF = _Xor(OpSize::i32Bit, HostFlag_Unordered, _Constant(1));
+    auto PF = _Xor(OpSize::i32Bit, HostFlag_Unordered, Constant(1));
     SetRFLAG<FEXCore::X86State::RFLAG_PF_RAW_LOC>(PF);
   }
 
@@ -671,7 +695,7 @@ void OpDispatchBuilder::FTST(OpcodeArgs) {
   HostFlag_ZF = _Or(OpSize::i32Bit, HostFlag_ZF, HostFlag_Unordered);
 
   SetRFLAG<FEXCore::X86State::X87FLAG_C0_LOC>(HostFlag_CF);
-  SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(_Constant(0));
+  SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Constant(0));
   SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(HostFlag_Unordered);
   SetRFLAG<FEXCore::X86State::X87FLAG_C3_LOC>(HostFlag_ZF);
 }
@@ -679,7 +703,7 @@ void OpDispatchBuilder::FTST(OpcodeArgs) {
 void OpDispatchBuilder::X87OpHelper(OpcodeArgs, FEXCore::IR::IROps IROp, bool ZeroC2) {
   DeriveOp(Result, IROp, _F80SCALEStack());
   if (ZeroC2) {
-    SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(_Constant(0));
+    SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(Constant(0));
   }
 }
 
@@ -705,7 +729,7 @@ void OpDispatchBuilder::X87ModifySTP(OpcodeArgs, bool Inc) {
 Ref OpDispatchBuilder::ReconstructFSW_Helper(Ref T) {
   // Start with the top value
   auto Top = T ? T : GetX87Top();
-  Ref FSW = _Lshl(OpSize::i64Bit, Top, _Constant(11));
+  Ref FSW = _Lshl(OpSize::i64Bit, Top, Constant(11));
 
   // We must construct the FSW from our various bits
   auto C0 = GetRFLAG(FEXCore::X86State::X87FLAG_C0_LOC);
@@ -720,6 +744,9 @@ Ref OpDispatchBuilder::ReconstructFSW_Helper(Ref T) {
   auto C3 = GetRFLAG(FEXCore::X86State::X87FLAG_C3_LOC);
   FSW = _Orlshl(OpSize::i64Bit, FSW, C3, 14);
 
+  auto IE = GetRFLAG(FEXCore::X86State::X87FLAG_IE_LOC);
+  FSW = _Or(OpSize::i64Bit, FSW, IE);
+
   return FSW;
 }
 
@@ -733,14 +760,14 @@ void OpDispatchBuilder::X87FNSTSW(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::FNINIT(OpcodeArgs) {
-  auto Zero = _Constant(0);
+  auto Zero = Constant(0);
 
   if (ReducedPrecisionMode) {
     _SetRoundingMode(Zero, false, Zero);
   }
 
   // Init FCW to 0x037F
-  auto NewFCW = _Constant(OpSize::i16Bit, 0x037F);
+  auto NewFCW = Constant(0x037F);
   _StoreContext(OpSize::i16Bit, GPRClass, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));
 
   // Set top to zero
@@ -800,8 +827,8 @@ void OpDispatchBuilder::X87FCMOV(OpcodeArgs) {
   default: LOGMAN_MSG_A_FMT("Unhandled FCMOV op: 0x{:x}", Opcode); break;
   }
 
-  auto ZeroConst = _Constant(0);
-  auto AllOneConst = _Constant(0xffff'ffff'ffff'ffffull);
+  auto ZeroConst = Constant(0);
+  auto AllOneConst = Constant(0xffff'ffff'ffff'ffffull);
 
   Ref SrcCond = SelectCC(CC, OpSize::i64Bit, AllOneConst, ZeroConst);
   Ref VecCond = _VDupFromGPR(OpSize::i128Bit, OpSize::i64Bit, SrcCond);
@@ -820,8 +847,8 @@ void OpDispatchBuilder::X87FXAM(OpcodeArgs) {
   // Claim this is a normal number
   // We don't support anything else
   auto TopValid = _StackValidTag(0);
-  auto ZeroConst = _Constant(0);
-  auto OneConst = _Constant(1);
+  auto ZeroConst = Constant(0);
+  auto OneConst = Constant(1);
 
   // In the case of top being invalid then C3:C2:C0 is 0b101
   auto C3 = _Select(FEXCore::IR::COND_NEQ, TopValid, OneConst, OneConst, ZeroConst);

FEXCore/Source/Interface/Core/OpcodeDispatcher/X87F64.cpp

@@ -36,12 +36,12 @@ void OpDispatchBuilder::X87LDENVF64(OpcodeArgs) {
   _SetRoundingMode(roundingMode, false, roundingMode);
   _StoreContext(OpSize::i16Bit, GPRClass, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));
 
-  auto NewFSW = _LoadMem(GPRClass, Size, Mem, _Constant(IR::OpSizeToSize(Size)), Size, MEM_OFFSET_SXTX, 1);
+  auto NewFSW = _LoadMem(GPRClass, Size, Mem, Constant(IR::OpSizeToSize(Size)), Size, MEM_OFFSET_SXTX, 1);
   ReconstructX87StateFromFSW_Helper(NewFSW);
 
   {
     // FTW
-    SetX87FTW(_LoadMem(GPRClass, Size, Mem, _Constant(IR::OpSizeToSize(Size) * 2), Size, MEM_OFFSET_SXTX, 1));
+    SetX87FTW(_LoadMem(GPRClass, Size, Mem, Constant(IR::OpSizeToSize(Size) * 2), Size, MEM_OFFSET_SXTX, 1));
   }
 }
 
@@ -87,7 +87,7 @@ void OpDispatchBuilder::FBSTPF64(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::FLDF64_Const(OpcodeArgs, uint64_t Num) {
-  auto Data = _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, _Constant(Num));
+  auto Data = _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, Constant(Num));
   _PushStack(Data, Data, OpSize::i64Bit, true);
 }
 
@@ -377,21 +377,21 @@ void OpDispatchBuilder::X87FXTRACTF64(OpcodeArgs) {
   Ref Gpr = _VExtractToGPR(OpSize::i64Bit, OpSize::i64Bit, Node, 0);
 
   // zero case
-  Ref ExpZV = _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, _Constant(0xfff0'0000'0000'0000UL));
+  Ref ExpZV = _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, Constant(0xfff0'0000'0000'0000UL));
   Ref SigZV = Node;
 
   // non zero case
   Ref ExpNZ = _Bfe(OpSize::i64Bit, 11, 52, Gpr);
-  ExpNZ = _Sub(OpSize::i64Bit, ExpNZ, _Constant(1023));
+  ExpNZ = _Sub(OpSize::i64Bit, ExpNZ, Constant(1023));
   Ref ExpNZV = _Float_FromGPR_S(OpSize::i64Bit, OpSize::i64Bit, ExpNZ);
 
-  Ref SigNZ = _And(OpSize::i64Bit, Gpr, _Constant(0x800f'ffff'ffff'ffffLL));
-  SigNZ = _Or(OpSize::i64Bit, SigNZ, _Constant(0x3ff0'0000'0000'0000LL));
+  Ref SigNZ = _And(OpSize::i64Bit, Gpr, Constant(0x800f'ffff'ffff'ffffLL));
+  SigNZ = _Or(OpSize::i64Bit, SigNZ, Constant(0x3ff0'0000'0000'0000LL));
   Ref SigNZV = _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, SigNZ);
 
   // Comparison and select to push onto stack
   SaveNZCV();
-  _TestNZ(OpSize::i64Bit, Gpr, _Constant(0x7fff'ffff'ffff'ffffUL));
+  _TestNZ(OpSize::i64Bit, Gpr, Constant(0x7fff'ffff'ffff'ffffUL));
 
   Ref Sig = _NZCVSelectV(OpSize::i64Bit, {COND_EQ}, SigZV, SigNZV);
   Ref Exp = _NZCVSelectV(OpSize::i64Bit, {COND_EQ}, ExpZV, ExpNZV);

FEXCore/Source/Interface/Core/X86Tables/BaseTables.cpp

@@ -187,7 +187,7 @@ std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
     {0xE4, 2, X86InstInfo{"IN",     TYPE_INST, FLAGS_BLOCK_END,                                                                                                      1, nullptr}},
     {0xE6, 2, X86InstInfo{"OUT",    TYPE_INST, FLAGS_BLOCK_END,                                                                                                      1, nullptr}},
 
-    {0xE8, 1, X86InstInfo{"CALL",   TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_BLOCK_END , 4, nullptr}},
+    {0xE8, 1, X86InstInfo{"CALL",   TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_BLOCK_END | FLAGS_CALL , 4, nullptr}},
     {0xE9, 1, X86InstInfo{"JMP",    TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_BLOCK_END , 4, nullptr}},
     {0xEB, 1, X86InstInfo{"JMP",    TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_BLOCK_END ,                             1, nullptr}},
 

FEXCore/Source/Interface/Core/X86Tables/PrimaryGroupTables.cpp

@@ -128,7 +128,7 @@ std::array<X86InstInfo, MAX_INST_GROUP_TABLE_SIZE> PrimaryInstGroupOps = []() co
     // GROUP 5
     {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 0), 1, X86InstInfo{"INC",   TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,                                                     0, nullptr}},
     {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 1), 1, X86InstInfo{"DEC",   TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,                                                     0, nullptr}},
-    {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 2), 1, X86InstInfo{"CALL",  TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_MODRM | FLAGS_BLOCK_END , 0, nullptr}},
+    {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 2), 1, X86InstInfo{"CALL",  TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_MODRM | FLAGS_BLOCK_END | FLAGS_CALL , 0, nullptr}},
     {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 3), 1, X86InstInfo{"CALLF", TYPE_INST, FLAGS_SETS_RIP | FLAGS_MODRM | FLAGS_BLOCK_END,                  0, nullptr}},
     {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 4), 1, X86InstInfo{"JMP",   TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_MODRM | FLAGS_BLOCK_END , 0, nullptr}},
     {OPD(TYPE_GROUP_5, OpToIndex(0xFF), 5), 1, X86InstInfo{"JMPF",  TYPE_INST, FLAGS_SETS_RIP | FLAGS_MODRM | FLAGS_BLOCK_END,                  0, nullptr}},

FEXCore/Source/Interface/Core/X86Tables/SecondaryGroupTables.cpp

@@ -223,11 +223,11 @@ std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGroupOps = [
     // GROUP 12
     {OPD(TYPE_GROUP_12, PF_NONE, 0), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
     {OPD(TYPE_GROUP_12, PF_NONE, 1), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
-    {OPD(TYPE_GROUP_12, PF_NONE, 2), 1, X86InstInfo{"PSRLW", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
+    {OPD(TYPE_GROUP_12, PF_NONE, 2), 1, X86InstInfo{"PSRLW", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
     {OPD(TYPE_GROUP_12, PF_NONE, 3), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
-    {OPD(TYPE_GROUP_12, PF_NONE, 4), 1, X86InstInfo{"PSRAW", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
+    {OPD(TYPE_GROUP_12, PF_NONE, 4), 1, X86InstInfo{"PSRAW", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
     {OPD(TYPE_GROUP_12, PF_NONE, 5), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
-    {OPD(TYPE_GROUP_12, PF_NONE, 6), 1, X86InstInfo{"PSLLW", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
+    {OPD(TYPE_GROUP_12, PF_NONE, 6), 1, X86InstInfo{"PSLLW", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
     {OPD(TYPE_GROUP_12, PF_NONE, 7), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
 
     {OPD(TYPE_GROUP_12, PF_66, 0), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                        0, nullptr}},
@@ -260,11 +260,11 @@ std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGroupOps = [
     // GROUP 13
     {OPD(TYPE_GROUP_13, PF_NONE, 0), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
     {OPD(TYPE_GROUP_13, PF_NONE, 1), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
-    {OPD(TYPE_GROUP_13, PF_NONE, 2), 1, X86InstInfo{"PSRLD", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
+    {OPD(TYPE_GROUP_13, PF_NONE, 2), 1, X86InstInfo{"PSRLD", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
     {OPD(TYPE_GROUP_13, PF_NONE, 3), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
-    {OPD(TYPE_GROUP_13, PF_NONE, 4), 1, X86InstInfo{"PSRAD", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
+    {OPD(TYPE_GROUP_13, PF_NONE, 4), 1, X86InstInfo{"PSRAD", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
     {OPD(TYPE_GROUP_13, PF_NONE, 5), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
-    {OPD(TYPE_GROUP_13, PF_NONE, 6), 1, X86InstInfo{"PSLLD", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
+    {OPD(TYPE_GROUP_13, PF_NONE, 6), 1, X86InstInfo{"PSLLD", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
     {OPD(TYPE_GROUP_13, PF_NONE, 7), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
 
     {OPD(TYPE_GROUP_13, PF_66, 0), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                        0, nullptr}},
@@ -297,11 +297,11 @@ std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGroupOps = [
     // GROUP 14
     {OPD(TYPE_GROUP_14, PF_NONE, 0), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
     {OPD(TYPE_GROUP_14, PF_NONE, 1), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
-    {OPD(TYPE_GROUP_14, PF_NONE, 2), 1, X86InstInfo{"PSRLQ", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
+    {OPD(TYPE_GROUP_14, PF_NONE, 2), 1, X86InstInfo{"PSRLQ", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
     {OPD(TYPE_GROUP_14, PF_NONE, 3), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
     {OPD(TYPE_GROUP_14, PF_NONE, 4), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
     {OPD(TYPE_GROUP_14, PF_NONE, 5), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
-    {OPD(TYPE_GROUP_14, PF_NONE, 6), 1, X86InstInfo{"PSLLQ", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
+    {OPD(TYPE_GROUP_14, PF_NONE, 6), 1, X86InstInfo{"PSLLQ", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
     {OPD(TYPE_GROUP_14, PF_NONE, 7), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                      0, nullptr}},
 
     {OPD(TYPE_GROUP_14, PF_66, 0), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,                                        0, nullptr}},
@@ -347,7 +347,7 @@ std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGroupOps = [
     {OPD(TYPE_GROUP_15, PF_F3, 3), 1, X86InstInfo{"WRGSBASE", TYPE_INST, GenFlagsDstSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY,          0, nullptr}},
     {OPD(TYPE_GROUP_15, PF_F3, 4), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
     {OPD(TYPE_GROUP_15, PF_F3, 5), 1, X86InstInfo{"INCSSPQ", TYPE_INST, FLAGS_MODRM,                    0, nullptr}},
-    {OPD(TYPE_GROUP_15, PF_F3, 6), 1, X86InstInfo{"CLRSSBSY", TYPE_INST, FLAGS_NONE,                    0, nullptr}},
+    {OPD(TYPE_GROUP_15, PF_F3, 6), 1, X86InstInfo{"UMONITOR/CLRSSBSY", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,       0, nullptr}},
     {OPD(TYPE_GROUP_15, PF_F3, 7), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
 
     {OPD(TYPE_GROUP_15, PF_66, 0), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
@@ -365,7 +365,7 @@ std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGroupOps = [
     {OPD(TYPE_GROUP_15, PF_F2, 3), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
     {OPD(TYPE_GROUP_15, PF_F2, 4), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
     {OPD(TYPE_GROUP_15, PF_F2, 5), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
-    {OPD(TYPE_GROUP_15, PF_F2, 6), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
+    {OPD(TYPE_GROUP_15, PF_F2, 6), 1, X86InstInfo{"UMWAIT", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,       0, nullptr}},
     {OPD(TYPE_GROUP_15, PF_F2, 7), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
 
     // GROUP 16

FEXCore/Source/Interface/Core/X86Tables/SecondaryTables.cpp

@@ -384,8 +384,8 @@ std::array<X86InstInfo, MAX_REPNE_MOD_TABLE_SIZE> RepNEModOps = []() consteval {
     {0x24, 6, X86InstInfo{"",          TYPE_INVALID, FLAGS_NONE,                                                         0, nullptr}},
     {0x2A, 1, X86InstInfo{"CVTSI2SD",  TYPE_INST, GenFlagsDstSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR,                    0, nullptr}},
     {0x2B, 1, X86InstInfo{"MOVNTSD",   TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {0x2C, 1, X86InstInfo{"CVTTSD2SI", TYPE_INST, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0, nullptr}},
-    {0x2D, 1, X86InstInfo{"CVTSD2SI",  TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0, nullptr}},
+    {0x2C, 1, X86InstInfo{"CVTTSD2SI", TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0, nullptr}},
+    {0x2D, 1, X86InstInfo{"CVTSD2SI",  TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0, nullptr}},
     {0x2E, 2, X86InstInfo{"",          TYPE_INVALID, FLAGS_NONE,                                                         0, nullptr}},
 
     {0x30, 16, X86InstInfo{"",         TYPE_COPY_OTHER, FLAGS_NONE,                                                      0, nullptr}},

FEXCore/Source/Interface/Core/X86Tables/VEXTables.cpp

@@ -20,21 +20,21 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     // VEX Map 1
     {OPD(1, 0b00, 0x10), 1, X86InstInfo{"VMOVUPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x10), 1, X86InstInfo{"VMOVUPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b10, 0x10), 1, X86InstInfo{"VMOVSS",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b11, 0x10), 1, X86InstInfo{"VMOVSD",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b10, 0x10), 1, X86InstInfo{"VMOVSS",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b11, 0x10), 1, X86InstInfo{"VMOVSD",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(1, 0b00, 0x11), 1, X86InstInfo{"VMOVUPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x11), 1, X86InstInfo{"VMOVUPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b10, 0x11), 1, X86InstInfo{"VMOVSS",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b11, 0x11), 1, X86InstInfo{"VMOVSD",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b10, 0x11), 1, X86InstInfo{"VMOVSS",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b11, 0x11), 1, X86InstInfo{"VMOVSD",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
-    {OPD(1, 0b00, 0x12), 1, X86InstInfo{"VMOVLPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_1ST_SRC, 0, nullptr}},
-    {OPD(1, 0b01, 0x12), 1, X86InstInfo{"VMOVLPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_1ST_SRC, 0, nullptr}},
+    {OPD(1, 0b00, 0x12), 1, X86InstInfo{"VMOVLPS",TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_1ST_SRC | FLAGS_VEX_L_0, 0, nullptr}},
+    {OPD(1, 0b01, 0x12), 1, X86InstInfo{"VMOVLPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MOD_MEM_ONLY | FLAGS_VEX_1ST_SRC | FLAGS_VEX_L_0, 0, nullptr}},
     {OPD(1, 0b10, 0x12), 1, X86InstInfo{"VMOVSLDUP", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b11, 0x12), 1, X86InstInfo{"VMOVDDUP",  TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
 
-    {OPD(1, 0b00, 0x13), 1, X86InstInfo{"VMOVLPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b01, 0x13), 1, X86InstInfo{"VMOVLPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b00, 0x13), 1, X86InstInfo{"VMOVLPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 0, nullptr}},
+    {OPD(1, 0b01, 0x13), 1, X86InstInfo{"VMOVLPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 0, nullptr}},
 
     {OPD(1, 0b00, 0x14), 1, X86InstInfo{"VUNPCKLPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x14), 1, X86InstInfo{"VUNPCKLPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -42,26 +42,26 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(1, 0b00, 0x15), 1, X86InstInfo{"VUNPCKHPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x15), 1, X86InstInfo{"VUNPCKHPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
 
-    {OPD(1, 0b00, 0x16), 1, X86InstInfo{"VMOVHPS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_1ST_SRC, 0, nullptr}},
-    {OPD(1, 0b01, 0x16), 1, X86InstInfo{"VMOVHPD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS | FLAGS_VEX_1ST_SRC, 0, nullptr}},
+    {OPD(1, 0b00, 0x16), 1, X86InstInfo{"VMOV(L)HPS",TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_1ST_SRC | FLAGS_VEX_L_0, 0, nullptr}},
+    {OPD(1, 0b01, 0x16), 1, X86InstInfo{"VMOVHPD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS | FLAGS_VEX_1ST_SRC | FLAGS_VEX_L_0, 0, nullptr}},
     {OPD(1, 0b10, 0x16), 1, X86InstInfo{"VMOVSHDUP", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
 
-    {OPD(1, 0b00, 0x17), 1, X86InstInfo{"VMOVHPS",   TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b01, 0x17), 1, X86InstInfo{"VMOVHPD",   TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b00, 0x17), 1, X86InstInfo{"VMOVHPS",   TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 0, nullptr}},
+    {OPD(1, 0b01, 0x17), 1, X86InstInfo{"VMOVHPD",   TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 0, nullptr}},
 
     {OPD(1, 0b00, 0x50), 1, X86InstInfo{"VMOVMSKPS", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0, nullptr}},
     {OPD(1, 0b01, 0x50), 1, X86InstInfo{"VMOVMSKPD", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0, nullptr}},
 
     {OPD(1, 0b00, 0x51), 1, X86InstInfo{"VSQRTPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x51), 1, X86InstInfo{"VSQRTPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b10, 0x51), 1, X86InstInfo{"VSQRTSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b11, 0x51), 1, X86InstInfo{"VSQRTSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b10, 0x51), 1, X86InstInfo{"VSQRTSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b11, 0x51), 1, X86InstInfo{"VSQRTSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(1, 0b00, 0x52), 1, X86InstInfo{"VRSQRTPS",  TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b10, 0x52), 1, X86InstInfo{"VRSQRTSS",  TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b10, 0x52), 1, X86InstInfo{"VRSQRTSS",  TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(1, 0b00, 0x53), 1, X86InstInfo{"VRCPPS",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b10, 0x53), 1, X86InstInfo{"VRCPSS",    TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b10, 0x53), 1, X86InstInfo{"VRCPSS",    TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(1, 0b00, 0x54), 1, X86InstInfo{"VANDPS",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x54), 1, X86InstInfo{"VANDPD",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -100,11 +100,11 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
 
     {OPD(1, 0b00, 0xC2), 1, X86InstInfo{"VCMPccPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
     {OPD(1, 0b01, 0xC2), 1, X86InstInfo{"VCMPccPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(1, 0b10, 0xC2), 1, X86InstInfo{"VCMPccSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(1, 0b11, 0xC2), 1, X86InstInfo{"VCMPccSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(1, 0b10, 0xC2), 1, X86InstInfo{"VCMPccSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_L_IGNORE | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(1, 0b11, 0xC2), 1, X86InstInfo{"VCMPccSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_L_IGNORE | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
 
-    {OPD(1, 0b01, 0xC4), 1, X86InstInfo{"VPINSRW",    TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_16BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_SF_SRC_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(1, 0b01, 0xC5), 1, X86InstInfo{"VPEXTRW",    TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(1, 0b01, 0xC4), 1, X86InstInfo{"VPINSRW",    TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_16BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_SF_SRC_GPR | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1, nullptr}},
+    {OPD(1, 0b01, 0xC5), 1, X86InstInfo{"VPEXTRW",    TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1, nullptr}},
 
     {OPD(1, 0b00, 0xC6), 1, X86InstInfo{"VSHUFPS",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
     {OPD(1, 0b01, 0xC6), 1, X86InstInfo{"VSHUFPD",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
@@ -119,38 +119,38 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(1, 0b00, 0x29), 1, X86InstInfo{"VMOVAPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x29), 1, X86InstInfo{"VMOVAPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
 
-    {OPD(1, 0b10, 0x2A), 1, X86InstInfo{"VCVTSI2SS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 0, nullptr}},
-    {OPD(1, 0b11, 0x2A), 1, X86InstInfo{"VCVTSI2SD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 0, nullptr}},
+    {OPD(1, 0b10, 0x2A), 1, X86InstInfo{"VCVTSI2SS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b11, 0x2A), 1, X86InstInfo{"VCVTSI2SD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(1, 0b00, 0x2B), 1, X86InstInfo{"VMOVNTPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x2B), 1, X86InstInfo{"VMOVNTPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
 
-    {OPD(1, 0b10, 0x2C), 1, X86InstInfo{"VCVTTSS2SI",   TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0, nullptr}},
-    {OPD(1, 0b11, 0x2C), 1, X86InstInfo{"VCVTTSD2SI",   TYPE_INST, GenFlagsSrcSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0, nullptr}},
+    {OPD(1, 0b10, 0x2C), 1, X86InstInfo{"VCVTTSS2SI",   TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b11, 0x2C), 1, X86InstInfo{"VCVTTSD2SI",   TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
-    {OPD(1, 0b10, 0x2D), 1, X86InstInfo{"VCVTSS2SI",   TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0, nullptr}},
-    {OPD(1, 0b11, 0x2D), 1, X86InstInfo{"VCVTSD2SI",   TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR, 0, nullptr}},
+    {OPD(1, 0b10, 0x2D), 1, X86InstInfo{"VCVTSS2SI",   TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b11, 0x2D), 1, X86InstInfo{"VCVTSD2SI",   TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
-    {OPD(1, 0b00, 0x2E), 1, X86InstInfo{"VUCOMISS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b01, 0x2E), 1, X86InstInfo{"VUCOMISD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b00, 0x2E), 1, X86InstInfo{"VUCOMISS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b01, 0x2E), 1, X86InstInfo{"VUCOMISD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
-    {OPD(1, 0b00, 0x2F), 1, X86InstInfo{"VCOMISS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b01, 0x2F), 1, X86InstInfo{"VCOMISD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b00, 0x2F), 1, X86InstInfo{"VCOMISS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b01, 0x2F), 1, X86InstInfo{"VCOMISD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(1, 0b00, 0x58), 1, X86InstInfo{"VADDPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x58), 1, X86InstInfo{"VADDPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b10, 0x58), 1, X86InstInfo{"VADDSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b11, 0x58), 1, X86InstInfo{"VADDSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b10, 0x58), 1, X86InstInfo{"VADDSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b11, 0x58), 1, X86InstInfo{"VADDSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(1, 0b00, 0x59), 1, X86InstInfo{"VMULPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x59), 1, X86InstInfo{"VMULPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b10, 0x59), 1, X86InstInfo{"VMULSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b11, 0x59), 1, X86InstInfo{"VMULSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b10, 0x59), 1, X86InstInfo{"VMULSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b11, 0x59), 1, X86InstInfo{"VMULSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(1, 0b00, 0x5A), 1, X86InstInfo{"VCVTPS2PD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x5A), 1, X86InstInfo{"VCVTPD2PS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b10, 0x5A), 1, X86InstInfo{"VCVTSS2SD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b11, 0x5A), 1, X86InstInfo{"VCVTSD2SS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b10, 0x5A), 1, X86InstInfo{"VCVTSS2SD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_L_IGNORE | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b11, 0x5A), 1, X86InstInfo{"VCVTSD2SS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_L_IGNORE |FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
 
     {OPD(1, 0b00, 0x5B), 1, X86InstInfo{"VCVTDQ2PS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x5B), 1, X86InstInfo{"VCVTPS2DQ",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -158,23 +158,23 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
 
     {OPD(1, 0b00, 0x5C), 1, X86InstInfo{"VSUBPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x5C), 1, X86InstInfo{"VSUBPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b10, 0x5C), 1, X86InstInfo{"VSUBSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b11, 0x5C), 1, X86InstInfo{"VSUBSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b10, 0x5C), 1, X86InstInfo{"VSUBSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b11, 0x5C), 1, X86InstInfo{"VSUBSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(1, 0b00, 0x5D), 1, X86InstInfo{"VMINPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x5D), 1, X86InstInfo{"VMINPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b10, 0x5D), 1, X86InstInfo{"VMINSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b11, 0x5D), 1, X86InstInfo{"VMINSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b10, 0x5D), 1, X86InstInfo{"VMINSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b11, 0x5D), 1, X86InstInfo{"VMINSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(1, 0b00, 0x5E), 1, X86InstInfo{"VDIVPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x5E), 1, X86InstInfo{"VDIVPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b10, 0x5E), 1, X86InstInfo{"VDIVSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b11, 0x5E), 1, X86InstInfo{"VDIVSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b10, 0x5E), 1, X86InstInfo{"VDIVSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b11, 0x5E), 1, X86InstInfo{"VDIVSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(1, 0b00, 0x5F), 1, X86InstInfo{"VMAXPS",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x5F), 1, X86InstInfo{"VMAXPD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b10, 0x5F), 1, X86InstInfo{"VMAXSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b11, 0x5F), 1, X86InstInfo{"VMAXSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b10, 0x5F), 1, X86InstInfo{"VMAXSS",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
+    {OPD(1, 0b11, 0x5F), 1, X86InstInfo{"VMAXSD",   TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(1, 0b01, 0x68), 1, X86InstInfo{"VPUNPCKHBW",  TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x69), 1, X86InstInfo{"VPUNPCKHWD",  TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -182,7 +182,7 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(1, 0b01, 0x6B), 1, X86InstInfo{"VPACKSSDW",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x6C), 1, X86InstInfo{"VPUNPCKLQDQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0x6D), 1, X86InstInfo{"VPUNPCKHQDQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b01, 0x6E), 1, X86InstInfo{"VMOV*",       TYPE_INST, GenFlagsDstSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 0, nullptr}},
+    {OPD(1, 0b01, 0x6E), 1, X86InstInfo{"VMOV*",       TYPE_INST, GenFlagsDstSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0 | FLAGS_SF_SRC_GPR, 0, nullptr}},
 
     {OPD(1, 0b01, 0x6F), 1, X86InstInfo{"VMOVDQA",     TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b10, 0x6F), 1, X86InstInfo{"VMOVDQU",     TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -193,8 +193,8 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(1, 0b01, 0x7D), 1, X86InstInfo{"VHSUBPD",     TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b11, 0x7D), 1, X86InstInfo{"VHSUBPS",     TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
 
-    {OPD(1, 0b01, 0x7E), 1, X86InstInfo{"VMOV*",     TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b10, 0x7E), 1, X86InstInfo{"VMOVQ",     TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b01, 0x7E), 1, X86InstInfo{"VMOV*",     TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_VEX_L_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b10, 0x7E), 1, X86InstInfo{"VMOVQ",     TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
 
     {OPD(1, 0b01, 0x7F), 1, X86InstInfo{"VMOVDQA",     TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b10, 0x7F), 1, X86InstInfo{"VMOVDQU",     TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -212,7 +212,7 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(1, 0b01, 0xD3), 1, X86InstInfo{"VPSRLQ",      TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0xD4), 1, X86InstInfo{"VPADDQ",      TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0xD5), 1, X86InstInfo{"VPMULLW",     TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b01, 0xD6), 1, X86InstInfo{"VMOVQ",       TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b01, 0xD6), 1, X86InstInfo{"VMOVQ",       TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_L_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0xD7), 1, X86InstInfo{"VPMOVMSKB",   TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_DST_GPR | FLAGS_SF_MOD_REG_ONLY, 0, nullptr}},
 
     {OPD(1, 0b01, 0xD8), 1, X86InstInfo{"VPSUBUSB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -246,7 +246,7 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(1, 0b01, 0xEE), 1, X86InstInfo{"VPMAXSW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0xEF), 1, X86InstInfo{"VPXOR",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
 
-    {OPD(1, 0b11, 0xF0), 1, X86InstInfo{"VLDDQU",      TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b11, 0xF0), 1, X86InstInfo{"VLDDQU",      TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS, 0, nullptr}},
 
     {OPD(1, 0b01, 0xF1), 1, X86InstInfo{"VPSLLW",      TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0xF2), 1, X86InstInfo{"VPSLLD",      TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -254,7 +254,7 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(1, 0b01, 0xF4), 1, X86InstInfo{"VPMULUDQ",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0xF5), 1, X86InstInfo{"VPMADDWD",    TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0xF6), 1, X86InstInfo{"VPSADBW",     TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(1, 0b01, 0xF7), 1, X86InstInfo{"VMASKMOVDQU", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(1, 0b01, 0xF7), 1, X86InstInfo{"VMASKMOVDQU", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 0, nullptr}},
 
     {OPD(1, 0b01, 0xF8), 1, X86InstInfo{"VPSUBB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(1, 0b01, 0xF9), 1, X86InstInfo{"VPSUBW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -278,18 +278,18 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(2, 0b01, 0x09), 1, X86InstInfo{"VPSIGNW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x0A), 1, X86InstInfo{"VPSIGND", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x0B), 1, X86InstInfo{"VPMULHRSW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x0C), 1, X86InstInfo{"VPERMILPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x0D), 1, X86InstInfo{"VPERMILPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x0E), 1, X86InstInfo{"VTESTPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x0F), 1, X86InstInfo{"VTESTPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x0C), 1, X86InstInfo{"VPERMILPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x0D), 1, X86InstInfo{"VPERMILPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x0E), 1, X86InstInfo{"VTESTPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x0F), 1, X86InstInfo{"VTESTPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
 
-    {OPD(2, 0b01, 0x13), 1, X86InstInfo{"VCVTPH2PS", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x16), 1, X86InstInfo{"VPERMPS", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x13), 1, X86InstInfo{"VCVTPH2PS", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x16), 1, X86InstInfo{"VPERMPS", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x17), 1, X86InstInfo{"VPTEST", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
 
-    {OPD(2, 0b01, 0x18), 1, X86InstInfo{"VBROADCASTSS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x19), 1, X86InstInfo{"VBROADCASTSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x1A), 1, X86InstInfo{"VBROADCASTF128", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x18), 1, X86InstInfo{"VBROADCASTSS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x19), 1, X86InstInfo{"VBROADCASTSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x1A), 1, X86InstInfo{"VBROADCASTF128", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_SF_MOD_MEM_ONLY | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x1C), 1, X86InstInfo{"VPABSB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x1D), 1, X86InstInfo{"VPABSW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x1E), 1, X86InstInfo{"VPABSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -305,10 +305,10 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(2, 0b01, 0x29), 1, X86InstInfo{"VPCMPEQQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x2A), 1, X86InstInfo{"VMOVNTDQA", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x2B), 1, X86InstInfo{"VPACKUSDW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x2C), 1, X86InstInfo{"VMASKMOVPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x2D), 1, X86InstInfo{"VMASKMOVPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x2E), 1, X86InstInfo{"VMASKMOVPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x2F), 1, X86InstInfo{"VMASKMOVPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x2C), 1, X86InstInfo{"VMASKMOVPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x2D), 1, X86InstInfo{"VMASKMOVPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x2E), 1, X86InstInfo{"VMASKMOVPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x2F), 1, X86InstInfo{"VMASKMOVPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
 
     {OPD(2, 0b01, 0x30), 1, X86InstInfo{"VPMOVZXBW", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x31), 1, X86InstInfo{"VPMOVZXBD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -316,7 +316,7 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(2, 0b01, 0x33), 1, X86InstInfo{"VPMOVZXWD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x34), 1, X86InstInfo{"VPMOVZXWQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x35), 1, X86InstInfo{"VPMOVZXDQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x36), 1, X86InstInfo{"VPERMD", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x36), 1, X86InstInfo{"VPERMD", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x37), 1, X86InstInfo{"VPCMPGTQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
 
     {OPD(2, 0b01, 0x38), 1, X86InstInfo{"VPMINSB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -329,17 +329,17 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(2, 0b01, 0x3F), 1, X86InstInfo{"VPMAXUD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
 
     {OPD(2, 0b01, 0x40), 1, X86InstInfo{"VPMULLD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x41), 1, X86InstInfo{"VPHMINPOSUW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x41), 1, X86InstInfo{"VPHMINPOSUW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 0, nullptr}},
     {OPD(2, 0b01, 0x45), 1, X86InstInfo{"VPSRLV", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x46), 1, X86InstInfo{"VPSRAVD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x46), 1, X86InstInfo{"VPSRAVD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x47), 1, X86InstInfo{"VPSLLV", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
 
-    {OPD(2, 0b01, 0x58), 1, X86InstInfo{"VPBROADCASTD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x59), 1, X86InstInfo{"VPBROADCASTQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x5A), 1, X86InstInfo{"VBROADCASTI128", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x58), 1, X86InstInfo{"VPBROADCASTD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x59), 1, X86InstInfo{"VPBROADCASTQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x5A), 1, X86InstInfo{"VBROADCASTI128", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_SF_MOD_MEM_ONLY | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
 
-    {OPD(2, 0b01, 0x78), 1, X86InstInfo{"VPBROADCASTB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x79), 1, X86InstInfo{"VPBROADCASTW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x78), 1, X86InstInfo{"VPBROADCASTB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x79), 1, X86InstInfo{"VPBROADCASTW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0, nullptr}},
 
     {OPD(2, 0b01, 0x8C), 1, X86InstInfo{"VPMASKMOV", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0x8E), 1, X86InstInfo{"VPMASKMOV", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -353,31 +353,31 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(2, 0b01, 0x97), 1, X86InstInfo{"VFMSUBADD132", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
 
     {OPD(2, 0b01, 0x98), 1, X86InstInfo{"VFMADD132", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x99), 1, X86InstInfo{"VFMADD132_S", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x99), 1, X86InstInfo{"VFMADD132_S", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
     {OPD(2, 0b01, 0x9A), 1, X86InstInfo{"VFMSUB132", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x9B), 1, X86InstInfo{"VFMSUB132_S", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x9B), 1, X86InstInfo{"VFMSUB132_S", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
     {OPD(2, 0b01, 0x9C), 1, X86InstInfo{"VFNMADD132", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x9D), 1, X86InstInfo{"VFNMADD132_S", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x9D), 1, X86InstInfo{"VFNMADD132_S", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
     {OPD(2, 0b01, 0x9E), 1, X86InstInfo{"VFNMSUB132", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0x9F), 1, X86InstInfo{"VFNMSUB132_S", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0x9F), 1, X86InstInfo{"VFNMSUB132_S", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(2, 0b01, 0xA8), 1, X86InstInfo{"VFMADD213", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0xA9), 1, X86InstInfo{"VFMADD213_S", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0xA9), 1, X86InstInfo{"VFMADD213_S", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
     {OPD(2, 0b01, 0xAA), 1, X86InstInfo{"VFMSUB213", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0xAB), 1, X86InstInfo{"VFMSUB213_S", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0xAB), 1, X86InstInfo{"VFMSUB213_S", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
     {OPD(2, 0b01, 0xAC), 1, X86InstInfo{"VFNMADD213", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0xAD), 1, X86InstInfo{"VFNMADD213_S", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0xAD), 1, X86InstInfo{"VFNMADD213_S", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
     {OPD(2, 0b01, 0xAE), 1, X86InstInfo{"VFNMSUB213", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0xAF), 1, X86InstInfo{"VFNMSUB213_S", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0xAF), 1, X86InstInfo{"VFNMSUB213_S", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(2, 0b01, 0xB8), 1, X86InstInfo{"VFMADD231", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0xB9), 1, X86InstInfo{"VFMADD231_S", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0xB9), 1, X86InstInfo{"VFMADD231_S", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
     {OPD(2, 0b01, 0xBA), 1, X86InstInfo{"VFMSUB231", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0xBB), 1, X86InstInfo{"VFMSUB231_S", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0xBB), 1, X86InstInfo{"VFMSUB231_S", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
     {OPD(2, 0b01, 0xBC), 1, X86InstInfo{"VFNMADD231", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0xBD), 1, X86InstInfo{"VFNMADD231_S", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0xBD), 1, X86InstInfo{"VFNMADD231_S", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
     {OPD(2, 0b01, 0xBE), 1, X86InstInfo{"VFNMSUB231", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
-    {OPD(2, 0b01, 0xBF), 1, X86InstInfo{"VFNMSUB231_S", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
+    {OPD(2, 0b01, 0xBF), 1, X86InstInfo{"VFNMSUB231_S", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_IGNORE, 0, nullptr}},
 
     {OPD(2, 0b01, 0xA6), 1, X86InstInfo{"VFMADDSUB213", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
     {OPD(2, 0b01, 0xA7), 1, X86InstInfo{"VFMSUBADD213", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -398,25 +398,25 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(2, 0b10, 0xF3), 1, X86InstInfo{"", TYPE_VEX_GROUP_17, FLAGS_NONE, 0, nullptr}}, // VEX Group 17
     {OPD(2, 0b11, 0xF3), 1, X86InstInfo{"", TYPE_VEX_GROUP_17, FLAGS_NONE, 0, nullptr}}, // VEX Group 17
 
-    {OPD(2, 0b00, 0xF5), 1, X86InstInfo{"BZHI", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_2ND_SRC, 0, nullptr}},
+    {OPD(2, 0b00, 0xF5), 1, X86InstInfo{"BZHI", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_2ND_SRC, 0, nullptr}},
     // AMD reference manual is incorrect. PEXT actually maps to 0b10, not 0b01.
-    {OPD(2, 0b10, 0xF5), 1, X86InstInfo{"PEXT", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_1ST_SRC, 0, nullptr}},
-    {OPD(2, 0b11, 0xF5), 1, X86InstInfo{"PDEP", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_1ST_SRC, 0, nullptr}},
+    {OPD(2, 0b10, 0xF5), 1, X86InstInfo{"PEXT", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_1ST_SRC, 0, nullptr}},
+    {OPD(2, 0b11, 0xF5), 1, X86InstInfo{"PDEP", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_1ST_SRC, 0, nullptr}},
 
-    {OPD(2, 0b11, 0xF6), 1, X86InstInfo{"MULX", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_1ST_SRC, 0, nullptr}},
+    {OPD(2, 0b11, 0xF6), 1, X86InstInfo{"MULX", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_1ST_SRC, 0, nullptr}},
 
-    {OPD(2, 0b00, 0xF7), 1, X86InstInfo{"BEXTR", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_2ND_SRC, 0, nullptr}},
-    {OPD(2, 0b01, 0xF7), 1, X86InstInfo{"SHLX", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_2ND_SRC, 0, nullptr}},
-    {OPD(2, 0b10, 0xF7), 1, X86InstInfo{"SARX", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_2ND_SRC, 0, nullptr}},
-    {OPD(2, 0b11, 0xF7), 1, X86InstInfo{"SHRX", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_2ND_SRC, 0, nullptr}},
+    {OPD(2, 0b00, 0xF7), 1, X86InstInfo{"BEXTR", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_2ND_SRC, 0, nullptr}},
+    {OPD(2, 0b01, 0xF7), 1, X86InstInfo{"SHLX", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_2ND_SRC, 0, nullptr}},
+    {OPD(2, 0b10, 0xF7), 1, X86InstInfo{"SARX", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_2ND_SRC, 0, nullptr}},
+    {OPD(2, 0b11, 0xF7), 1, X86InstInfo{"SHRX", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_2ND_SRC, 0, nullptr}},
 
     // VEX Map 3
-    {OPD(3, 0b01, 0x00), 1, X86InstInfo{"VPERMQ", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x01), 1, X86InstInfo{"VPERMPD", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x02), 1, X86InstInfo{"VPBLENDD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x04), 1, X86InstInfo{"VPERMILPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x05), 1, X86InstInfo{"VPERMILPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x06), 1, X86InstInfo{"VPERM2F128", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x00), 1, X86InstInfo{"VPERMQ", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_1 | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x01), 1, X86InstInfo{"VPERMPD", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_1 | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x02), 1, X86InstInfo{"VPBLENDD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x04), 1, X86InstInfo{"VPERMILPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x05), 1, X86InstInfo{"VPERMILPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x06), 1, X86InstInfo{"VPERM2F128", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS | FLAGS_VEX_L_1, 1, nullptr}},
 
     {OPD(3, 0b01, 0x08), 1, X86InstInfo{"VROUNDPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
     {OPD(3, 0b01, 0x09), 1, X86InstInfo{"VROUNDPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
@@ -427,41 +427,41 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(3, 0b01, 0x0E), 1, X86InstInfo{"VPBLENDW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
     {OPD(3, 0b01, 0x0F), 1, X86InstInfo{"VPALIGNR", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
 
-    {OPD(3, 0b01, 0x14), 1, X86InstInfo{"VPEXTRB", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x15), 1, X86InstInfo{"VPEXTRW", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x16), 1, X86InstInfo{"VPEXTRD", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x17), 1, X86InstInfo{"VEXTRACTPS", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x14), 1, X86InstInfo{"VPEXTRB", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x15), 1, X86InstInfo{"VPEXTRW", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x16), 1, X86InstInfo{"VPEXTRD", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x17), 1, X86InstInfo{"VEXTRACTPS", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
 
-    {OPD(3, 0b01, 0x18), 1, X86InstInfo{"VINSERTF128", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x19), 1, X86InstInfo{"VEXTRACTF128", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_256BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x1D), 1, X86InstInfo{"VCVTPS2PH", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x18), 1, X86InstInfo{"VINSERTF128", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x19), 1, X86InstInfo{"VEXTRACTF128", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_0 | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x1D), 1, X86InstInfo{"VCVTPS2PH", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
 
-    {OPD(3, 0b01, 0x20), 1, X86InstInfo{"VPINSRB", TYPE_INST, GenFlagsDstSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1, nullptr}},
-    {OPD(3, 0b01, 0x21), 1, X86InstInfo{"VINSERTPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x22), 1, X86InstInfo{"VPINSR{D,Q}", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1, nullptr}},
+    {OPD(3, 0b01, 0x20), 1, X86InstInfo{"VPINSRB", TYPE_INST, GenFlagsDstSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1, nullptr}},
+    {OPD(3, 0b01, 0x21), 1, X86InstInfo{"VINSERTPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x22), 1, X86InstInfo{"VPINSR{D,Q}", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1, nullptr}},
 
-    {OPD(3, 0b01, 0x38), 1, X86InstInfo{"VINSERTI128", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x39), 1, X86InstInfo{"VEXTRACTI128", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_256BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x38), 1, X86InstInfo{"VINSERTI128", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x39), 1, X86InstInfo{"VEXTRACTI128", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_0 | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
 
     {OPD(3, 0b01, 0x40), 1, X86InstInfo{"VDPPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x41), 1, X86InstInfo{"VDPPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x41), 1, X86InstInfo{"VDPPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1, nullptr}},
     {OPD(3, 0b01, 0x42), 1, X86InstInfo{"VMPSADBW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
     {OPD(3, 0b01, 0x44), 1, X86InstInfo{"VPCLMULQDQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x46), 1, X86InstInfo{"VPERM2I128", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x46), 1, X86InstInfo{"VPERM2I128", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_REX_W_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
 
-    {OPD(3, 0b01, 0x4A), 1, X86InstInfo{"VBLENDVPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x4B), 1, X86InstInfo{"VBLENDVPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x4C), 1, X86InstInfo{"VPBLENDVB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x4A), 1, X86InstInfo{"VBLENDVPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x4B), 1, X86InstInfo{"VBLENDVPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x4C), 1, X86InstInfo{"VPBLENDVB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
 
     {OPD(3, 0b01, 0x5C), 1, X86InstInfo{"VFMADDSUBPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}}, ///< FMA4
     {OPD(3, 0b01, 0x5D), 1, X86InstInfo{"VFMADDSUBPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}}, ///< FMA4
     {OPD(3, 0b01, 0x5E), 1, X86InstInfo{"VFMSUBADDPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}}, ///< FMA4
     {OPD(3, 0b01, 0x5F), 1, X86InstInfo{"VFMSUBADDPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}}, ///< FMA4
 
-    {OPD(3, 0b01, 0x60), 1, X86InstInfo{"VPCMPESTRM", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x61), 1, X86InstInfo{"VPCMPESTRI", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x62), 1, X86InstInfo{"VPCMPISTRM", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(3, 0b01, 0x63), 1, X86InstInfo{"VPCMPISTRI", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0x60), 1, X86InstInfo{"VPCMPESTRM", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1, nullptr}},
+    {OPD(3, 0b01, 0x61), 1, X86InstInfo{"VPCMPESTRI", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1, nullptr}},
+    {OPD(3, 0b01, 0x62), 1, X86InstInfo{"VPCMPISTRM", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1, nullptr}},
+    {OPD(3, 0b01, 0x63), 1, X86InstInfo{"VPCMPISTRI", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_VEX_L_0, 1, nullptr}},
 
     {OPD(3, 0b01, 0x68), 1, X86InstInfo{"VFMADDPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}}, ///< FMA4
     {OPD(3, 0b01, 0x69), 1, X86InstInfo{"VFMADDPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}}, ///< FMA4
@@ -481,9 +481,9 @@ std::array<X86InstInfo, MAX_VEX_TABLE_SIZE> VEXTableOps = []() consteval {
     {OPD(3, 0b01, 0x7E), 1, X86InstInfo{"VFNMSUBSS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}}, ///< FMA4
     {OPD(3, 0b01, 0x7F), 1, X86InstInfo{"VFNMSUBSD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}}, ///< FMA4
 
-    {OPD(3, 0b01, 0xDF), 1, X86InstInfo{"VAESKEYGENASSIST", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(3, 0b01, 0xDF), 1, X86InstInfo{"VAESKEYGENASSIST", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_0 | FLAGS_XMM_FLAGS, 1, nullptr}},
 
-    {OPD(3, 0b11, 0xF0), 1, X86InstInfo{"RORX", TYPE_INST, FLAGS_MODRM, 1, nullptr}},
+    {OPD(3, 0b11, 0xF0), 1, X86InstInfo{"RORX", TYPE_INST, FLAGS_MODRM | FLAGS_VEX_L_0, 1, nullptr}},
 
     // VEX Map 4 - 31 (Reserved)
   };
@@ -500,21 +500,21 @@ std::array<X86InstInfo, MAX_VEX_GROUP_TABLE_SIZE> VEXTableGroupOps = []() conste
 
 #define OPD(group, pp, opcode) (((group - TYPE_VEX_GROUP_12) << 4) | (pp << 3) | (opcode))
   constexpr U8U8InfoStruct VEXGroupTable[] = {
-    {OPD(TYPE_VEX_GROUP_12, 1, 0b010), 1, X86InstInfo{"VPSRLW",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(TYPE_VEX_GROUP_12, 1, 0b100), 1, X86InstInfo{"VPSRAW",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(TYPE_VEX_GROUP_12, 1, 0b110), 1, X86InstInfo{"VPSLLW",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(TYPE_VEX_GROUP_12, 1, 0b010), 1, X86InstInfo{"VPSRLW",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(TYPE_VEX_GROUP_12, 1, 0b100), 1, X86InstInfo{"VPSRAW",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(TYPE_VEX_GROUP_12, 1, 0b110), 1, X86InstInfo{"VPSLLW",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
 
-    {OPD(TYPE_VEX_GROUP_13, 1, 0b010), 1, X86InstInfo{"VPSRLD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(TYPE_VEX_GROUP_13, 1, 0b100), 1, X86InstInfo{"VPSRAD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(TYPE_VEX_GROUP_13, 1, 0b110), 1, X86InstInfo{"VPSLLD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(TYPE_VEX_GROUP_13, 1, 0b010), 1, X86InstInfo{"VPSRLD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(TYPE_VEX_GROUP_13, 1, 0b100), 1, X86InstInfo{"VPSRAD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(TYPE_VEX_GROUP_13, 1, 0b110), 1, X86InstInfo{"VPSLLD",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
 
-    {OPD(TYPE_VEX_GROUP_14, 1, 0b010), 1, X86InstInfo{"VPSRLQ",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(TYPE_VEX_GROUP_14, 1, 0b011), 1, X86InstInfo{"VPSRLDQ",  TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(TYPE_VEX_GROUP_14, 1, 0b110), 1, X86InstInfo{"VPSLLQ",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
-    {OPD(TYPE_VEX_GROUP_14, 1, 0b111), 1, X86InstInfo{"VPSLLDQ",  TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(TYPE_VEX_GROUP_14, 1, 0b010), 1, X86InstInfo{"VPSRLQ",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(TYPE_VEX_GROUP_14, 1, 0b011), 1, X86InstInfo{"VPSRLDQ",  TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(TYPE_VEX_GROUP_14, 1, 0b110), 1, X86InstInfo{"VPSLLQ",   TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
+    {OPD(TYPE_VEX_GROUP_14, 1, 0b111), 1, X86InstInfo{"VPSLLDQ",  TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_VEX_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
 
-    {OPD(TYPE_VEX_GROUP_15, 0, 0b010), 1, X86InstInfo{"VLDMXCSR", TYPE_INST, GenFlagsSameSize(SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY, 0, nullptr}},
-    {OPD(TYPE_VEX_GROUP_15, 0, 0b011), 1, X86InstInfo{"VSTMXCSR", TYPE_INST, GenFlagsSameSize(SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY, 0, nullptr}},
+    {OPD(TYPE_VEX_GROUP_15, 0, 0b010), 1, X86InstInfo{"VLDMXCSR", TYPE_INST, GenFlagsSameSize(SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_L_0 | FLAGS_SF_MOD_MEM_ONLY, 0, nullptr}},
+    {OPD(TYPE_VEX_GROUP_15, 0, 0b011), 1, X86InstInfo{"VSTMXCSR", TYPE_INST, GenFlagsSameSize(SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_L_0 | FLAGS_SF_MOD_MEM_ONLY, 0, nullptr}},
 
     {OPD(TYPE_VEX_GROUP_17, 0, 0b001), 1, X86InstInfo{"BLSR",     TYPE_INST, FLAGS_MODRM | FLAGS_VEX_DST, 0, nullptr}},
     {OPD(TYPE_VEX_GROUP_17, 0, 0b010), 1, X86InstInfo{"BLSMSK",   TYPE_INST, FLAGS_MODRM | FLAGS_VEX_DST, 0, nullptr}},

FEXCore/Source/Interface/Core/X86Tables/X86Tables.h

@@ -32,13 +32,15 @@ constexpr uint32_t FLAG_REX_WIDENING  = (1 << 7);
 constexpr uint32_t FLAG_REX_XGPR_B    = (1 << 8);
 constexpr uint32_t FLAG_REX_XGPR_X    = (1 << 9);
 constexpr uint32_t FLAG_REX_XGPR_R    = (1 << 10);
-constexpr uint32_t FLAG_ES_PREFIX     = (1 << 11);
-constexpr uint32_t FLAG_CS_PREFIX     = (1 << 12);
-constexpr uint32_t FLAG_SS_PREFIX     = (1 << 13);
-constexpr uint32_t FLAG_DS_PREFIX     = (1 << 14);
-constexpr uint32_t FLAG_FS_PREFIX     = (1 << 15);
-constexpr uint32_t FLAG_GS_PREFIX     = (1 << 16);
-constexpr uint32_t FLAG_SEGMENTS      = (0b11'1111 << 11);
+constexpr uint32_t FLAG_NO_PREFIX     = (0b000 << 11);
+constexpr uint32_t FLAG_ES_PREFIX     = (0b001 << 11);
+constexpr uint32_t FLAG_CS_PREFIX     = (0b010 << 11);
+constexpr uint32_t FLAG_SS_PREFIX     = (0b011 << 11);
+constexpr uint32_t FLAG_DS_PREFIX     = (0b100 << 11);
+constexpr uint32_t FLAG_FS_PREFIX     = (0b101 << 11);
+constexpr uint32_t FLAG_GS_PREFIX     = (0b110 << 11);
+constexpr uint32_t FLAG_SEGMENTS      = (0b111 << 11);
+// Bits 14, 15, 16 - Unused
 
 constexpr uint32_t FLAG_REP_PREFIX    = (1 << 17);
 constexpr uint32_t FLAG_REPNE_PREFIX  = (1 << 18);
@@ -353,6 +355,14 @@ constexpr InstFlagType FLAGS_VEX_1ST_SRC          = (0b10ULL << 22);
 constexpr InstFlagType FLAGS_VEX_2ND_SRC          = (0b11ULL << 22);
 // Whether or not the instruction has a VSIB byte
 constexpr InstFlagType FLAGS_VEX_VSIB             = (1ULL << 24);
+constexpr InstFlagType FLAGS_VEX_L_IGNORE         = (1ULL << 25);
+constexpr InstFlagType FLAGS_VEX_L_0             = (1ULL << 26);
+constexpr InstFlagType FLAGS_VEX_L_1             = (1ULL << 27);
+
+constexpr InstFlagType FLAGS_REX_W_0             = (1ULL << 28);
+constexpr InstFlagType FLAGS_REX_W_1             = (1ULL << 29);
+
+constexpr InstFlagType FLAGS_CALL             = (1ULL << 30);
 
 constexpr InstFlagType FLAGS_SIZE_DST_OFF = 58;
 constexpr InstFlagType FLAGS_SIZE_SRC_OFF = FLAGS_SIZE_DST_OFF + 3;

FEXCore/Source/Interface/GDBJIT/GDBJIT.cpp

@@ -113,8 +113,7 @@ void GDBJITRegister(FEXCore::IR::AOTIRCacheEntry* Entry, uintptr_t VAFileStart,
 } // namespace FEXCore
 #else
 namespace FEXCore {
-void GDBJITRegister([[maybe_unused]] FEXCore::IR::AOTIRCacheEntry* Entry, [[maybe_unused]] uintptr_t VAFileStart, [[maybe_unused]] uint64_t GuestRIP,
-                    [[maybe_unused]] uintptr_t HostEntry, [[maybe_unused]] FEXCore::Core::DebugData* DebugData) {
+void GDBJITRegister(FEXCore::IR::AOTIRCacheEntry*, uintptr_t, uint64_t, uintptr_t, FEXCore::Core::DebugData*) {
   ERROR_AND_DIE_FMT("GDBSymbols support not compiled in");
 }
 } // namespace FEXCore

FEXCore/Source/Interface/IR/IR.h

@@ -54,23 +54,19 @@ struct NodeID final {
   [[nodiscard]] friend constexpr bool operator==(NodeID, NodeID) noexcept = default;
 
   [[nodiscard]]
-  friend constexpr bool
-  operator<(NodeID lhs, NodeID rhs) noexcept {
+  friend constexpr bool operator<(NodeID lhs, NodeID rhs) noexcept {
     return lhs.Value < rhs.Value;
   }
   [[nodiscard]]
-  friend constexpr bool
-  operator>(NodeID lhs, NodeID rhs) noexcept {
+  friend constexpr bool operator>(NodeID lhs, NodeID rhs) noexcept {
     return operator<(rhs, lhs);
   }
   [[nodiscard]]
-  friend constexpr bool
-  operator<=(NodeID lhs, NodeID rhs) noexcept {
+  friend constexpr bool operator<=(NodeID lhs, NodeID rhs) noexcept {
     return !operator>(lhs, rhs);
   }
   [[nodiscard]]
-  friend constexpr bool
-  operator>=(NodeID lhs, NodeID rhs) noexcept {
+  friend constexpr bool operator>=(NodeID lhs, NodeID rhs) noexcept {
     return !operator<(lhs, rhs);
   }
 
@@ -144,9 +140,22 @@ struct FEX_PACKED NodeWrapperBase final {
     return NodeOffset & (1u << 31);
   }
 
+  [[nodiscard]]
+  bool HasKill() const {
+    return NodeOffset & (1u << 30);
+  }
+
+  void ClearKill() {
+    NodeOffset &= ~(1u << 30);
+  }
+
+  void SetKill() {
+    NodeOffset |= (1u << 30);
+  }
+
   [[nodiscard]]
   bool IsPointer() const {
-    return !IsImmediate();
+    return !IsImmediate() && !HasKill();
   }
 
   [[nodiscard]]
@@ -183,8 +192,7 @@ struct FEX_PACKED NodeWrapperBase final {
   }
 
   [[nodiscard]]
-  friend constexpr bool
-  operator==(const NodeWrapperBase<Type>&, const NodeWrapperBase<Type>&) = default;
+  friend constexpr bool operator==(const NodeWrapperBase<Type>&, const NodeWrapperBase<Type>&) = default;
 
   [[nodiscard]]
   static NodeWrapperBase<Type> FromImmediate(uint32_t Immediate) {
@@ -423,8 +431,7 @@ struct FEX_PACKED RegisterClassType final {
     return Val;
   }
   [[nodiscard]]
-  friend constexpr bool
-  operator==(const RegisterClassType&, const RegisterClassType&) = default;
+  friend constexpr bool operator==(const RegisterClassType&, const RegisterClassType&) = default;
 };
 
 struct FEX_PACKED CondClassType final {
@@ -433,8 +440,7 @@ struct FEX_PACKED CondClassType final {
     return Val;
   }
   [[nodiscard]]
-  friend constexpr bool
-  operator==(const CondClassType&, const CondClassType&) = default;
+  friend constexpr bool operator==(const CondClassType&, const CondClassType&) = default;
 };
 
 struct FEX_PACKED MemOffsetType final {
@@ -443,8 +449,7 @@ struct FEX_PACKED MemOffsetType final {
     return Val;
   }
   [[nodiscard]]
-  friend constexpr bool
-  operator==(const MemOffsetType&, const MemOffsetType&) = default;
+  friend constexpr bool operator==(const MemOffsetType&, const MemOffsetType&) = default;
 };
 
 struct FEX_PACKED TypeDefinition final {
@@ -479,8 +484,7 @@ struct FEX_PACKED TypeDefinition final {
   }
 
   [[nodiscard]]
-  friend constexpr bool
-  operator==(const TypeDefinition&, const TypeDefinition&) = default;
+  friend constexpr bool operator==(const TypeDefinition&, const TypeDefinition&) = default;
 };
 
 static_assert(std::is_trivially_copyable_v<TypeDefinition>);
@@ -493,8 +497,7 @@ struct FEX_PACKED FenceType final {
     return Val;
   }
   [[nodiscard]]
-  friend constexpr bool
-  operator==(const FenceType&, const FenceType&) = default;
+  friend constexpr bool operator==(const FenceType&, const FenceType&) = default;
 };
 
 struct FEX_PACKED RoundType final {
@@ -503,8 +506,7 @@ struct FEX_PACKED RoundType final {
     return Val;
   }
   [[nodiscard]]
-  friend constexpr bool
-  operator==(const RoundType&, const RoundType&) = default;
+  friend constexpr bool operator==(const RoundType&, const RoundType&) = default;
 };
 
 class NodeIterator;
@@ -515,7 +517,10 @@ class NodeIterator;
  */
 class NodeIterator {
 public:
-  using value_type = std::tuple<OrderedNode*, IROp_Header*>;
+  struct value_type final {
+    OrderedNode *Node;
+    IROp_Header *Header;
+  };
   using size_type = std::size_t;
   using difference_type = std::ptrdiff_t;
   using reference = value_type&;
@@ -537,14 +542,12 @@ public:
     , Node {Ptr} {}
 
   [[nodiscard]]
-  bool
-  operator==(const NodeIterator& rhs) const {
+  bool operator==(const NodeIterator& rhs) const {
     return Node.NodeOffset == rhs.Node.NodeOffset;
   }
 
   [[nodiscard]]
-  bool
-  operator!=(const NodeIterator& rhs) const {
+  bool operator!=(const NodeIterator& rhs) const {
     return !operator==(rhs);
   }
 
@@ -561,15 +564,13 @@ public:
   }
 
   [[nodiscard]]
-  value_type
-  operator*() {
+  value_type operator*() {
     OrderedNode* RealNode = Node.GetNode(BaseList);
     return {RealNode, RealNode->Op(IRList)};
   }
 
   [[nodiscard]]
-  value_type
-  operator()() {
+  value_type operator()() {
     OrderedNode* RealNode = Node.GetNode(BaseList);
     return {RealNode, RealNode->Op(IRList)};
   }
@@ -620,6 +621,12 @@ enum class ShiftType : uint8_t {
   ROR,
 };
 
+enum class BranchHint : uint8_t {
+  None = 0,
+  Call,
+  Return,
+};
+
 
 // Converts a size stored as an integer in to an OpSize enum.
 // This is a nop operation and will be eliminated by the compiler.

FEXCore/Source/Interface/IR/IR.json

@@ -158,7 +158,8 @@
     "FloatCompareOp": "FloatCompareOp",
     "NamedVectorConstant": "FEXCore::IR::NamedVectorConstant",
     "IndexNamedVectorConstant": "FEXCore::IR::IndexNamedVectorConstant",
-    "ShiftType": "FEXCore::IR::ShiftType"
+    "ShiftType": "FEXCore::IR::ShiftType",
+    "BranchHint": "FEXCore::IR::BranchHint"
   },
   "Ops": {
     "Misc": {
@@ -171,7 +172,7 @@
         "SwitchGen": false,
         "JITDispatchOverride": "NoOp"
       },
-      "CodeBlock SSA:$Begin, SSA:$Last, u32:$ID": {
+      "CodeBlock SSA:$Begin, SSA:$Last, u32:$ID, i1:$EntryPoint{false}, u32:$GuestEntryOffset{0}": {
         "SwitchGen": false,
         "RAOverride": "0",
         "JITDispatchOverride": "NoOp"
@@ -284,6 +285,13 @@
         "HasSideEffects": true,
         "Desc": ["This is a hint instruction that the CPU is likely to do a spin so it might want to pause to help out SMP",
                  "Can be implemented as a NOP if necessary"]
+      },
+      "WFET GPR:$Upper, GPR:$Lower": {
+        "HasSideEffects": true,
+        "Desc": [
+          "Implement a low power wait attempting to sleep until RDTSC >= Upper:Lower.",
+          "Will spuriously wake up."
+        ]
       }
     },
     "Branch": {
@@ -295,11 +303,12 @@
         "HasSideEffects": true,
         "RAOverride": "2"
       },
-      "ExitFunction OpSize:#Size, GPR:$NewRIP": {
+      "ExitFunction OpSize:#Size, GPR:$NewRIP, BranchHint:$Hint, GPR:$CallReturnAddress, SSA:$CallReturnBlock": {
         "Desc": ["Exits the current JIT function with a target RIP"
                 ],
         "HasSideEffects": true,
-        "DestSize": "Size"
+        "DestSize": "Size",
+        "RAOverride": "2"
       },
       "Break BreakDefinition:$Reason": {
         "HasSideEffects": true
@@ -817,56 +826,6 @@
           "Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit"
         ]
       },
-      "AtomicAdd OpSize:#Size, GPR:$Value, GPR:$Addr": {
-        "HasSideEffects": true,
-        "Desc": ["Atomic integer add",
-                 "IR layout must match Fetch-variant, otherwise DCE IR optimization breaks!"
-                ],
-        "DestSize": "Size",
-        "EmitValidation": [
-          "Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit"
-        ]
-      },
-      "AtomicSub OpSize:#Size, GPR:$Value, GPR:$Addr": {
-        "HasSideEffects": true,
-        "Desc": ["Atomic integer sub",
-                 "IR layout must match Fetch-variant, otherwise DCE IR optimization breaks!"
-                ],
-        "DestSize": "Size",
-        "EmitValidation": [
-          "Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit"
-        ]
-      },
-      "AtomicAnd OpSize:#Size, GPR:$Value, GPR:$Addr": {
-        "HasSideEffects": true,
-        "Desc": ["Atomic integer and",
-                 "IR layout must match Fetch-variant, otherwise DCE IR optimization breaks!"
-                ],
-        "DestSize": "Size",
-        "EmitValidation": [
-          "Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit"
-        ]
-      },
-      "AtomicCLR OpSize:#Size, GPR:$Value, GPR:$Addr": {
-        "HasSideEffects": true,
-        "Desc": ["Atomic integer binary clear",
-                 "IR layout must match Fetch-variant, otherwise DCE IR optimization breaks!"
-                ],
-        "DestSize": "Size",
-        "EmitValidation": [
-          "Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit"
-        ]
-      },
-      "AtomicOr OpSize:#Size, GPR:$Value, GPR:$Addr": {
-        "HasSideEffects": true,
-        "Desc": ["Atomic integer or",
-                 "IR layout must match Fetch-variant, otherwise DCE IR optimization breaks!"
-                ],
-        "DestSize": "Size",
-        "EmitValidation": [
-          "Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit"
-        ]
-      },
       "AtomicXor OpSize:#Size, GPR:$Value, GPR:$Addr": {
         "HasSideEffects": true,
         "Desc": ["Atomic integer xor",
@@ -877,16 +836,6 @@
           "Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit"
         ]
       },
-      "AtomicNeg OpSize:#Size, GPR:$Addr": {
-        "HasSideEffects": true,
-        "Desc": ["Atomic integer two's complement negate",
-                 "IR layout must match Fetch-variant, otherwise DCE IR optimization breaks!"
-                ],
-        "DestSize": "Size",
-        "EmitValidation": [
-          "Size == FEXCore::IR::OpSize::i8Bit || Size == FEXCore::IR::OpSize::i16Bit || Size == FEXCore::IR::OpSize::i32Bit || Size == FEXCore::IR::OpSize::i64Bit"
-        ]
-      },
       "GPR = AtomicSwap OpSize:#Size, GPR:$Value, GPR:$Addr": {
         "HasSideEffects": true,
         "Desc": ["Atomic integer swap"

FEXCore/Source/Interface/IR/IRDumper.cpp

@@ -29,23 +29,18 @@ namespace FEXCore::IR {
 
 #include <FEXCore/IR/IRDefines.inc>
 
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, const SHA256Sum& Arg) {
+static void PrintArg(fextl::stringstream* out, const IRListView*, const SHA256Sum& Arg) {
   *out << "sha256:";
   for (auto byte : Arg.data) {
     *out << std::hex << std::setfill('0') << std::setw(2) << (unsigned int)byte;
   }
 }
 
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, uint64_t Arg) {
+static void PrintArg(fextl::stringstream* out, const IRListView*, uint64_t Arg) {
   *out << "#0x" << std::hex << Arg << std::dec;
 }
 
-[[maybe_unused]]
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, const char* Arg) {
-  *out << Arg;
-}
-
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, CondClassType Arg) {
+static void PrintArg(fextl::stringstream* out, const IRListView*, CondClassType Arg) {
   if (Arg == COND_AL) {
     *out << "ALWAYS";
     return;
@@ -58,7 +53,7 @@ static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView
   *out << CondNames[Arg];
 }
 
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, MemOffsetType Arg) {
+static void PrintArg(fextl::stringstream* out, const IRListView*, MemOffsetType Arg) {
   static constexpr std::array<std::string_view, 3> Names = {
     "SXTX",
     "UXTW",
@@ -68,7 +63,7 @@ static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView
   *out << Names[Arg];
 }
 
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, RegisterClassType Arg) {
+static void PrintArg(fextl::stringstream* out, const IRListView*, RegisterClassType Arg) {
   if (Arg == GPRClass.Val) {
     *out << "GPR";
   } else if (Arg == GPRFixedClass.Val) {
@@ -131,7 +126,7 @@ static void PrintArg(fextl::stringstream* out, const IRListView* IR, OrderedNode
   }
 }
 
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, FEXCore::IR::FenceType Arg) {
+static void PrintArg(fextl::stringstream* out, const IRListView*, FEXCore::IR::FenceType Arg) {
   if (Arg == IR::Fence_Load) {
     *out << "Loads";
   } else if (Arg == IR::Fence_Store) {
@@ -143,7 +138,7 @@ static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView
   }
 }
 
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, FEXCore::IR::RoundType Arg) {
+static void PrintArg(fextl::stringstream* out, const IRListView*, FEXCore::IR::RoundType Arg) {
   switch (Arg) {
   case FEXCore::IR::Round_Nearest: *out << "Nearest"; break;
   case FEXCore::IR::Round_Negative_Infinity: *out << "-Inf"; break;
@@ -154,7 +149,7 @@ static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView
   }
 }
 
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, FEXCore::IR::SyscallFlags Arg) {
+static void PrintArg(fextl::stringstream* out, const IRListView*, FEXCore::IR::SyscallFlags Arg) {
   switch (Arg) {
   case FEXCore::IR::SyscallFlags::DEFAULT: *out << "Default"; break;
   case FEXCore::IR::SyscallFlags::OPTIMIZETHROUGH: *out << "Optimize Through"; break;
@@ -165,7 +160,7 @@ static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView
   }
 }
 
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, FEXCore::IR::NamedVectorConstant Arg) {
+static void PrintArg(fextl::stringstream* out, const IRListView*, FEXCore::IR::NamedVectorConstant Arg) {
   *out << [Arg] {
     // clang-format off
     switch (Arg) {
@@ -230,7 +225,7 @@ static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView
   }();
 }
 
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, FEXCore::IR::OpSize Arg) {
+static void PrintArg(fextl::stringstream* out, const IRListView*, FEXCore::IR::OpSize Arg) {
   switch (Arg) {
   case OpSize::i8Bit: *out << "i8"; break;
   case OpSize::i16Bit: *out << "i16"; break;
@@ -243,7 +238,7 @@ static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView
   }
 }
 
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, FEXCore::IR::FloatCompareOp Arg) {
+static void PrintArg(fextl::stringstream* out, const IRListView*, FEXCore::IR::FloatCompareOp Arg) {
   switch (Arg) {
   case FloatCompareOp::EQ: *out << "FEQ"; break;
   case FloatCompareOp::LT: *out << "FLT"; break;
@@ -255,14 +250,14 @@ static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView
   }
 }
 
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, FEXCore::IR::BreakDefinition Arg) {
+static void PrintArg(fextl::stringstream* out, const IRListView*, FEXCore::IR::BreakDefinition Arg) {
   *out << "{" << Arg.ErrorRegister << ".";
   *out << static_cast<uint32_t>(Arg.Signal) << ".";
   *out << static_cast<uint32_t>(Arg.TrapNumber) << ".";
   *out << static_cast<uint32_t>(Arg.si_code) << "}";
 }
 
-static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, FEXCore::IR::ShiftType Arg) {
+static void PrintArg(fextl::stringstream* out, const IRListView*, FEXCore::IR::ShiftType Arg) {
   switch (Arg) {
   case ShiftType::LSL: *out << "LSL"; break;
   case ShiftType::LSR: *out << "LSR"; break;
@@ -272,6 +267,15 @@ static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView
   }
 }
 
+static void PrintArg(fextl::stringstream* out, [[maybe_unused]] const IRListView* IR, FEXCore::IR::BranchHint Arg) {
+  switch (Arg) {
+  case BranchHint::None: *out << "None"; break;
+  case BranchHint::Call: *out << "Call"; break;
+  case BranchHint::Return: *out << "Return"; break;
+  default: *out << "<Unknown Branch Hint>"; break;
+  }
+}
+
 void Dump(fextl::stringstream* out, const IRListView* IR) {
   auto HeaderOp = IR->GetHeader();
 

FEXCore/Source/Interface/IR/IREmitter.cpp

@@ -187,6 +187,9 @@ void IREmitter::SetCurrentCodeBlock(Ref Node) {
   LOGMAN_THROW_A_FMT(Node->Op(DualListData.DataBegin())->Op == OP_CODEBLOCK, "Node wasn't codeblock. It was '{}'",
                      IR::GetName(Node->Op(DualListData.DataBegin())->Op));
   SetWriteCursor(Node->Op(DualListData.DataBegin())->CW<IROp_CodeBlock>()->Begin.GetNode(DualListData.ListBegin()));
+
+  // Constants are pooled only within a single block.
+  NrConstants = 0;
 }
 
 } // namespace FEXCore::IR

FEXCore/Source/Interface/IR/IREmitter.h

@@ -58,15 +58,6 @@ public:
 #define IROP_ALLOCATE_HELPERS
 #define IROP_DISPATCH_HELPERS
 #include <FEXCore/IR/IRDefines.inc>
-  IRPair<IROp_Constant> _Constant(IR::OpSize Size, uint64_t Constant) {
-    auto Op = AllocateOp<IROp_Constant, IROps::OP_CONSTANT>();
-    LOGMAN_THROW_A_FMT(Size >= IR::OpSize::i8Bit && Size <= IR::OpSize::i64Bit, "Invalid size");
-    uint64_t Mask = ~0ULL >> (64 - IR::OpSizeAsBits(Size));
-    Op.first->Constant = (Constant & Mask);
-    Op.first->Header.Size = Size;
-    Op.first->Header.ElementSize = Size;
-    return Op;
-  }
   IRPair<IROp_Jump> _Jump() {
     return _Jump(InvalidNode);
   }
@@ -87,16 +78,30 @@ public:
   IRPair<IROp_LoadMem> _LoadMem(FEXCore::IR::RegisterClassType Class, IR::OpSize Size, Ref ssa0, IR::OpSize Align = OpSize::i8Bit) {
     return _LoadMem(Class, Size, ssa0, Invalid(), Align, MEM_OFFSET_SXTX, 1);
   }
-  IRPair<IROp_LoadMemTSO> _LoadMemTSO(FEXCore::IR::RegisterClassType Class, IR::OpSize Size, Ref ssa0, IR::OpSize Align = OpSize::i8Bit) {
-    return _LoadMemTSO(Class, Size, ssa0, Invalid(), Align, MEM_OFFSET_SXTX, 1);
-  }
   IRPair<IROp_StoreMem> _StoreMem(FEXCore::IR::RegisterClassType Class, IR::OpSize Size, Ref Addr, Ref Value, IR::OpSize Align = OpSize::i8Bit) {
     return _StoreMem(Class, Size, Value, Addr, Invalid(), Align, MEM_OFFSET_SXTX, 1);
   }
-  IRPair<IROp_StoreMemTSO>
-  _StoreMemTSO(FEXCore::IR::RegisterClassType Class, IR::OpSize Size, Ref Addr, Ref Value, IR::OpSize Align = OpSize::i8Bit) {
-    return _StoreMemTSO(Class, Size, Value, Addr, Invalid(), Align, MEM_OFFSET_SXTX, 1);
+
+  int64_t Constants[32];
+  Ref ConstantRefs[32];
+  uint32_t NrConstants;
+
+  Ref Constant(int64_t Value) {
+    // Search for the constant in the pool.
+    for (unsigned i = 0; i < std::min(NrConstants, 32u); ++i) {
+      if (Constants[i] == Value) {
+        return ConstantRefs[i];
+      }
+    }
+
+    // Otherwise, materialize a fresh constant and pool it.
+    Ref R = _Constant(Value);
+    unsigned i = (NrConstants++) & 31;
+    Constants[i] = Value;
+    ConstantRefs[i] = R;
+    return R;
   }
+
   Ref Invalid() {
     return InvalidNode;
   }
@@ -228,7 +233,7 @@ public:
     auto Before = IR.at(Node);
     --Before;
 
-    SetWriteCursor(std::get<0>(*Before));
+    SetWriteCursor((*Before).Node);
   }
 
   Ref GetWriteCursor() {
@@ -248,11 +253,11 @@ public:
    *
    * @return OrderedNode
    */
-  IRPair<IROp_CodeBlock> CreateCodeNode() {
+  IRPair<IROp_CodeBlock> CreateCodeNode(bool EntryPoint = false, uint32_t GuestEntryOffset = 0) {
     SetWriteCursor(nullptr); // Orphan from any previous nodes
 
     auto ID = ViewIR().GetHeader()->BlockCount++;
-    auto CodeNode = _CodeBlock(InvalidNode, InvalidNode, ID);
+    auto CodeNode = _CodeBlock(InvalidNode, InvalidNode, ID, EntryPoint, GuestEntryOffset);
 
     CodeBlocks.emplace_back(CodeNode);
 

FEXCore/Source/Interface/IR/IntrusiveIRList.h

@@ -127,10 +127,6 @@ public:
     PoolObject.ReownOrClaimBuffer();
   }
 
-  ~DualIntrusiveAllocatorThreadPool() {
-    PoolObject.UnclaimBuffer();
-  }
-
   void ReownOrClaimBuffer() {
     Data = PoolObject.ReownOrClaimBuffer();
     List = Data + MemorySize;

FEXCore/Source/Interface/IR/PassManager.cpp

@@ -70,7 +70,7 @@ void PassManager::AddDefaultPasses(FEXCore::Context::ContextImpl* ctx) {
   FEX_CONFIG_OPT(DisablePasses, O0);
 
   if (!DisablePasses()) {
-    InsertPass(CreateX87StackOptimizationPass(ctx->HostFeatures, ctx->GetGPROpSize()));
+    InsertPass(CreateX87StackOptimizationPass(ctx->HostFeatures, ctx->Config.Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit));
     InsertPass(CreateConstProp(ctx->HostFeatures.SupportsTSOImm9));
     InsertPass(CreateDeadFlagCalculationEliminination());
   }

FEXCore/Source/Interface/IR/Passes/ConstProp.cpp

@@ -327,70 +327,10 @@ void ConstProp::Run(IREmitter* IREmit) {
   FEXCORE_PROFILE_SCOPED("PassManager::ConstProp");
 
   auto CurrentIR = IREmit->ViewIR();
-  const uint32_t SSACount = CurrentIR.GetSSACount();
-
-  // Allocation/initialization deferred until first use, since many multiblocks
-  // don't have constants leftover after all inlining.
-  fextl::vector<Ref> Remap {};
-
-  struct Entry {
-    int64_t Value;
-    Ref R;
-  };
-
-  fextl::vector<Entry> Pool {};
 
   for (auto [BlockNode, BlockIROp] : CurrentIR.GetBlocks()) {
-    Pool.clear();
-
     for (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {
-      if (IROp->Op == OP_CONSTANT) {
-        auto Op = IROp->C<IR::IROp_Constant>();
-        bool Found = false;
-
-        // Search for the constant. This is O(n^2) but n is small since it's
-        // local and most constants are inlined. In practice, it ends up much
-        // faster than a hash table.
-        for (auto K : Pool) {
-          if (K.Value == Op->Constant) {
-            uint32_t Value = CurrentIR.GetID(CodeNode).Value;
-            if (Value < SSACount) {
-              if (Remap.empty()) {
-                Remap.resize(SSACount, nullptr);
-              }
-
-              Remap[Value] = K.R;
-            }
-            Found = true;
-            break;
-          }
-        }
-
-        if (!Found) {
-          Pool.push_back({.Value = Op->Constant, .R = CodeNode});
-        }
-
-        continue;
-      }
-
       ConstantPropagation(IREmit, CurrentIR, CodeNode, IROp);
-
-      if (!Remap.empty()) {
-        const uint8_t NumArgs = IR::GetArgs(IROp->Op);
-        for (uint8_t i = 0; i < NumArgs; ++i) {
-          if (IROp->Args[i].IsInvalid()) {
-            continue;
-          }
-
-          uint32_t Value = IROp->Args[i].ID().Value;
-          if (Value < SSACount) {
-            Ref New = Remap[Value];
-            if (New) {
-              IREmit->ReplaceNodeArgument(CodeNode, i, New);
-            }
-          }
-        }
-      }
     }
   }
 }

FEXCore/Source/Interface/IR/Passes/RedundantFlagCalculationElimination.cpp

@@ -452,35 +452,6 @@ bool DeadFlagCalculationEliminination::EliminateDeadCode(IREmitter* IREmit, Ref
     return false;
   }
 
-  switch (IROp->Op) {
-  case OP_SYSCALL: {
-    auto Op = IROp->C<IR::IROp_Syscall>();
-    if ((Op->Flags & IR::SyscallFlags::NOSIDEEFFECTS) != IR::SyscallFlags::NOSIDEEFFECTS) {
-      return false;
-    }
-
-    break;
-  }
-  case OP_INLINESYSCALL: {
-    auto Op = IROp->C<IR::IROp_Syscall>();
-    if ((Op->Flags & IR::SyscallFlags::NOSIDEEFFECTS) != IR::SyscallFlags::NOSIDEEFFECTS) {
-      return false;
-    }
-
-    break;
-  }
-
-  // If the result of the atomic fetch is completely unused, convert it to a non-fetching atomic operation.
-  case OP_ATOMICFETCHADD: IROp->Op = OP_ATOMICADD; return true;
-  case OP_ATOMICFETCHSUB: IROp->Op = OP_ATOMICSUB; return true;
-  case OP_ATOMICFETCHAND: IROp->Op = OP_ATOMICAND; return true;
-  case OP_ATOMICFETCHCLR: IROp->Op = OP_ATOMICCLR; return true;
-  case OP_ATOMICFETCHOR: IROp->Op = OP_ATOMICOR; return true;
-  case OP_ATOMICFETCHXOR: IROp->Op = OP_ATOMICXOR; return true;
-  case OP_ATOMICFETCHNEG: IROp->Op = OP_ATOMICNEG; return true;
-  default: break;
-  }
-
   IREmit->Remove(CodeNode);
   return true;
 }

FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp

@@ -75,6 +75,15 @@ private:
   // Maps defs to their assigned spill slot + 1, or 0 if not spilled.
   fextl::vector<unsigned> SpillSlots;
 
+  // Next-use distance relative to the block end of each source, last first.
+  fextl::vector<uint32_t> SourcesNextUses;
+
+  // Sources that have been seen
+  fextl::vector<bool> Seen;
+
+  // SourcesNextUses is read backwards, this tracks the index
+  int64_t SourceIndex;
+
   bool Rematerializable(IROp_Header* IROp) {
     return IROp->Op == OP_CONSTANT;
   }
@@ -152,11 +161,13 @@ private:
     if (IROp->Op == OP_LOADREGISTER || IROp->Op == OP_LOADPF || IROp->Op == OP_LOADAF) {
       return Node;
     } else if (IROp->Op == OP_STOREREGISTER) {
-      const IROp_StoreRegister* Op = IROp->C<IR::IROp_StoreRegister>();
-      return IR->GetNode(Op->Value);
+      auto V = IROp->C<IR::IROp_StorePF>()->Value;
+      V.ClearKill();
+      return IR->GetNode(V);
     } else if (IROp->Op == OP_STOREPF || IROp->Op == OP_STOREAF) {
-      const IROp_StorePF* Op = IROp->C<IR::IROp_StorePF>();
-      return IR->GetNode(Op->Value);
+      auto V = IROp->C<IR::IROp_StorePF>()->Value;
+      V.ClearKill();
+      return IR->GetNode(V);
     }
 
     return nullptr;
@@ -199,7 +210,56 @@ private:
   // the next set bit and then clearing on each iteration.
 #define foreach_bit(b, x) for (uint32_t __x = (x), b; ((b) = __builtin_ffs(__x) - 1, __x); __x &= ~(1 << (b)))
 
-  void SpillReg(RegisterClass* Class, IROp_Header* Exclude) {
+  void CalculateNextUses(IROp_CodeBlock* BlockIROp, IROp_Header* Until) {
+    SourcesNextUses.clear();
+    NextUses.resize(IR->GetSSACount(), 0);
+
+    // IP relative to the end of the block.
+    uint32_t IP = 1;
+
+    // We grab these nodes this way so we can iterate easily
+    auto CodeBegin = IR->at(BlockIROp->Begin);
+    auto CodeLast = IR->at(BlockIROp->Last);
+
+    while (1) {
+      auto [CodeNode, IROp] = CodeLast();
+      if (IROp == Until) {
+        break;
+      }
+      // End of iteration gunk
+
+      const uint8_t NumArgs = IR::GetRAArgs(IROp->Op);
+      for (int i = NumArgs - 1; i >= 0; --i) {
+        auto& Arg = IROp->Args[i];
+        Arg.ClearKill();
+
+        if (!Arg.IsInvalid()) {
+          const uint32_t Index = Arg.ID().Value;
+
+          SourcesNextUses.push_back(NextUses[Index]);
+          NextUses[Index] = IP;
+        }
+      }
+
+      // IP is relative to block end and we iterate backwards, so increment.
+      ++IP;
+
+      // Rest is iteration gunk
+      if (CodeLast == CodeBegin) {
+        break;
+      }
+      --CodeLast;
+    }
+
+    SourceIndex = SourcesNextUses.size();
+  }
+
+  void SpillReg(RegisterClass* Class, IROp_CodeBlock* Block, IROp_Header* Exclude) {
+    // We're about to use next-use information, so calculate it.
+    if (!AnySpilled) {
+      CalculateNextUses(Block, Exclude);
+    }
+
     // Find the best node to spill according to the "furthest-first" heuristic.
     // Since we defined IPs relative to the end of the block, the furthest
     // next-use has the /smallest/ unsigned IP.
@@ -291,7 +351,7 @@ private:
   };
 
   // Assign a register for a given Node, spilling if necessary.
-  void AssignReg(IROp_Header* IROp, Ref CodeNode, IROp_Header* Pivot) {
+  void AssignReg(IROp_Header* IROp, IROp_CodeBlock* Block, Ref CodeNode, IROp_Header* Pivot) {
     const uint32_t Node = IR->GetID(CodeNode).Value;
 
     // Prioritize preferred registers.
@@ -352,7 +412,7 @@ private:
     // Spill to make room in the register file.
     if (!Class->Available) {
       IREmit->SetWriteCursorBefore(CodeNode);
-      SpillReg(Class, Pivot);
+      SpillReg(Class, Block, Pivot);
     }
 
     // Assign a free register in the appropriate class.
@@ -499,30 +559,22 @@ void ConstrainedRAPass::Run(IREmitter* IREmit_) {
 
   PreferredReg.resize(IR->GetSSACount(), PhysicalRegister::Invalid());
   SSAToReg.resize(IR->GetSSACount(), PhysicalRegister::Invalid());
-  NextUses.resize(IR->GetSSACount(), 0);
-  AnySpilled = false;
-
-  // Next-use distance relative to the block end of each source, last first.
-  fextl::vector<uint32_t> SourcesNextUses;
+  Seen.resize(IR->GetSSACount(), false);
 
   for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {
+    // Spilling is local, so reset this per-block
+    AnySpilled = false;
+
     // At the start of each block, all registers are available.
     for (auto& Class : Classes) {
       Class.Available = (1u << Class.Count) - 1;
     }
 
-    SourcesNextUses.clear();
+    auto BlockIROp = BlockHeader->CW<IR::IROp_CodeBlock>();
 
-    // IP relative to the end of the block.
-    uint32_t IP = 1;
-
-    // Backwards pass:
-    //  - analyze kill bits, next-use distances, and affinities
-    //  - insert moves for tied operands (TODO)
+    // Backwards pass: analyze kill bits and SRA affinities
     {
       // Reverse iteration is not yet working with the iterators
-      auto BlockIROp = BlockHeader->CW<IR::IROp_CodeBlock>();
-
       // We grab these nodes this way so we can iterate easily
       auto CodeBegin = IR->at(BlockIROp->Begin);
       auto CodeLast = IR->at(BlockIROp->Last);
@@ -531,20 +583,6 @@ void ConstrainedRAPass::Run(IREmitter* IREmit_) {
         auto [CodeNode, IROp] = CodeLast();
         // End of iteration gunk
 
-        // Iterate sources backwards, since we walk backwards. Ensures the order
-        // of SourcesNextUses is consistent. The forward pass can then iterate
-        // forwards and just flip the order.
-        const uint8_t NumArgs = IR::GetRAArgs(IROp->Op);
-        for (int i = NumArgs - 1; i >= 0; --i) {
-          const auto& Arg = IROp->Args[i];
-          if (!Arg.IsInvalid()) {
-            const uint32_t Index = Arg.ID().Value;
-
-            SourcesNextUses.push_back(NextUses[Index]);
-            NextUses[Index] = IP;
-          }
-        }
-
         // Record preferred registers for SRA. We also record the Node accessing
         // each register, used below. Since we initialized Class->Available,
         // RegToSSA is otherwise undefined so we can stash our temps there.
@@ -573,8 +611,17 @@ void ConstrainedRAPass::Run(IREmitter* IREmit_) {
           }
         }
 
-        // IP is relative to block end and we iterate backwards, so increment.
-        ++IP;
+        const uint8_t NumArgs = IR::GetRAArgs(IROp->Op);
+        for (int i = NumArgs - 1; i >= 0; --i) {
+          const auto& Arg = IROp->Args[i];
+          if (!Arg.IsInvalid()) {
+            const uint32_t Index = Arg.ID().Value;
+            if (!Seen[Index]) {
+              Seen[Index] = true;
+              IROp->Args[i].SetKill();
+            }
+          }
+        }
 
         // Rest is iteration gunk
         if (CodeLast == CodeBegin) {
@@ -587,14 +634,13 @@ void ConstrainedRAPass::Run(IREmitter* IREmit_) {
     // NextUses currently contains first use distances, the exact initialization
     // assumed by the forward pass. Do not reset it.
 
-    // SourcesNextUses is read backwards, this tracks the index
-    int64_t SourceIndex = SourcesNextUses.size();
-
     // Last nontrivial instruction, for merging as we go.
     Ref LastNode = nullptr;
 
     // Forward pass: Assign registers, spilling & optimizing as we go.
     for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {
+      bool AnySpilledBeforeThisInstruction = AnySpilled;
+
       // These do not read or write registers, and must be skipped for merging.
       // Since we'd be doing this check anyway for merging, do the check now so
       // we can skip the rest of the logic too.
@@ -628,7 +674,7 @@ void ConstrainedRAPass::Run(IREmitter* IREmit_) {
             }
 
             FreeReg(Reg);
-            AssignReg(IR->GetOp<IROp_Header>(Copy), Copy, IROp);
+            AssignReg(IR->GetOp<IROp_Header>(Copy), BlockIROp, Copy, IROp);
             RemapReg(Old, PhysicalRegister(Copy));
           }
         }
@@ -638,7 +684,7 @@ void ConstrainedRAPass::Run(IREmitter* IREmit_) {
       //
       // This happens before freeing killed sources, since we need all sources in
       // the register file simultaneously.
-      if (AnySpilled) {
+      if (AnySpilledBeforeThisInstruction) {
         for (auto s = 0; s < IR::GetRAArgs(IROp->Op); ++s) {
           if (!IsValidArg(IROp->Args[s])) {
             continue;
@@ -652,39 +698,60 @@ void ConstrainedRAPass::Run(IREmitter* IREmit_) {
 
             Ref Fill = InsertFill(Old);
 
-            AssignReg(IR->GetOp<IROp_Header>(Fill), Fill, IROp);
+            AssignReg(IR->GetOp<IROp_Header>(Fill), BlockIROp, Fill, IROp);
             RemapReg(Old, PhysicalRegister(Fill));
           }
         }
-      }
 
-      for (auto s = 0; s < IR::GetRAArgs(IROp->Op); ++s) {
-        if (IROp->Args[s].IsInvalid()) {
-          continue;
+        for (auto s = 0; s < IR::GetRAArgs(IROp->Op); ++s) {
+          if (IROp->Args[s].IsInvalid()) {
+            continue;
+          }
+
+          Ref Node = IR->GetNode(IROp->Args[s]);
+          auto ID = IR->GetID(Node).Value;
+          auto Reg = SSAToReg[ID];
+
+          SourceIndex--;
+          LOGMAN_THROW_A_FMT(SourceIndex >= 0, "Consistent source count");
+
+          if (!Reg.IsInvalid()) {
+            IROp->Args[s].SetImmediate(Reg.Raw);
+
+            if (!SourcesNextUses[SourceIndex]) {
+              LOGMAN_THROW_A_FMT(IsInRegisterFile(Node), "sources in file");
+              FreeReg(Reg);
+            }
+          }
+
+          NextUses[ID] = SourcesNextUses[SourceIndex];
         }
+      } else {
+        for (auto s = 0; s < IR::GetRAArgs(IROp->Op); ++s) {
+          if (IROp->Args[s].IsInvalid()) {
+            continue;
+          }
 
-        Ref Node = IR->GetNode(IROp->Args[s]);
-        auto ID = IR->GetID(Node).Value;
-        auto Reg = SSAToReg[ID];
+          bool Kill = IROp->Args[s].HasKill();
+          IROp->Args[s].ClearKill();
+          Ref Node = IR->GetNode(IROp->Args[s]);
+          auto ID = IR->GetID(Node).Value;
+          auto Reg = SSAToReg[ID];
 
-        SourceIndex--;
-        LOGMAN_THROW_A_FMT(SourceIndex >= 0, "Consistent source count");
+          if (!Reg.IsInvalid()) {
+            if (Kill) {
+              LOGMAN_THROW_A_FMT(IsInRegisterFile(Node), "sources in file");
+              FreeReg(Reg);
+            }
 
-        if (!Reg.IsInvalid()) {
-          IROp->Args[s].SetImmediate(Reg.Raw);
-
-          if (!SourcesNextUses[SourceIndex]) {
-            LOGMAN_THROW_A_FMT(IsInRegisterFile(Node), "sources in file");
-            FreeReg(Reg);
+            IROp->Args[s].SetImmediate(Reg.Raw);
           }
         }
-
-        NextUses[ID] = SourcesNextUses[SourceIndex];
       }
 
       // Assign destinations.
       if (GetHasDest(IROp->Op) && PhysicalRegister(CodeNode).IsInvalid()) {
-        AssignReg(IROp, CodeNode, IROp);
+        AssignReg(IROp, BlockIROp, CodeNode, IROp);
       }
 
       if (IsTrivial(CodeNode, IROp)) {
@@ -699,13 +766,16 @@ void ConstrainedRAPass::Run(IREmitter* IREmit_) {
       }
     }
 
-    LOGMAN_THROW_A_FMT(SourceIndex == 0, "Consistent source count in block");
+    if (AnySpilled) {
+      LOGMAN_THROW_A_FMT(SourceIndex == 0, "Consistent source count in block");
+    }
   }
 
   PreferredReg.clear();
   SSAToReg.clear();
   SpillSlots.clear();
   NextUses.clear();
+  Seen.clear();
 
   IR->GetHeader()->PostRA = true;
 }

FEXCore/Source/Interface/IR/Passes/x87StackOptimizationPass.cpp

@@ -52,6 +52,11 @@ enum class StackSlot { UNUSED, INVALID, VALID };
 template<typename T>
 class FixedSizeStack {
 public:
+  struct StackSlotEntry final {
+    StackSlot Type;
+    T Value;
+  };
+
   static constexpr uint8_t size = 8;
 
   // Real top as an offset from stored top value (or the one at the beginning of the block)
@@ -84,7 +89,7 @@ public:
     rotate(false);
   }
 
-  const std::pair<StackSlot, T>& top(size_t Offset = 0) const {
+  const StackSlotEntry& top(size_t Offset = 0) const {
     return buffer[Offset];
   }
 
@@ -118,14 +123,14 @@ public:
   }
 
   void setTagInvalid(size_t Index) {
-    buffer[Index].first = StackSlot::INVALID;
+    buffer[Index].Type = StackSlot::INVALID;
   }
 
   // Returns a mask to set in AbridgedTagWord
   uint8_t getValidMask() {
     uint8_t Mask = 0;
     for (size_t i = 0; i < buffer.size(); i++) {
-      if (buffer[i].first == StackSlot::VALID) {
+      if (buffer[i].Type == StackSlot::VALID) {
         Mask |= 1U << i;
       }
     }
@@ -136,7 +141,7 @@ public:
   uint8_t getInvalidMask() {
     uint8_t Mask = 0;
     for (size_t i = 0; i < buffer.size(); i++) {
-      if (buffer[i].first == StackSlot::INVALID) {
+      if (buffer[i].Type == StackSlot::INVALID) {
         Mask |= 1U << i;
       }
     }
@@ -144,7 +149,7 @@ public:
   }
 
 private:
-  fextl::vector<std::pair<StackSlot, T>> buffer;
+  fextl::vector<StackSlotEntry> buffer;
 };
 
 class X87StackOptimization final : public Pass {
@@ -305,9 +310,13 @@ private:
       , InterpretAsFloat(Float) {}
     Ref StackDataNode {}; // Reference to the data in the Stack.
                           // This is the source data node in the stack format, possibly converted to 64/80 bits.
+    struct StackMemberData final {
+      OpSize Size;
+      Ref Node;
+    };
     // Tuple is only valid if we have information about the Source of the Stack Data Node.
     // In it's valid then OpSize is the original source size and Ref is the original source node.
-    std::optional<std::pair<OpSize, Ref>> Source {};
+    std::optional<StackMemberData> Source {};
     bool InterpretAsFloat {false}; // True if this is a floating point value, false if integer
   };
 
@@ -916,8 +925,8 @@ void X87StackOptimization::Run(IREmitter* Emit) {
         // str w2, [x1]
         // or similar. As long as the source size and dest size are one and the same.
         // This will avoid any conversions between source and stack element size and conversion back.
-        if (!SlowPath && Value->Source && Value->Source->first == Op->StoreSize && Value->InterpretAsFloat) {
-          IREmit->_StoreMem(Value->InterpretAsFloat ? FPRClass : GPRClass, Op->StoreSize, Value->Source->second, AddrNode, Offset, Align,
+        if (!SlowPath && Value->Source && Value->Source->Size == Op->StoreSize && Value->InterpretAsFloat) {
+          IREmit->_StoreMem(Value->InterpretAsFloat ? FPRClass : GPRClass, Op->StoreSize, Value->Source->Node, AddrNode, Offset, Align,
                             OffsetType, OffsetScale);
           break;
         }

FEXCore/Source/Utils/Allocator/64BitAllocator.cpp

@@ -275,8 +275,13 @@ void* OSAllocator_64Bit::Mmap(void* addr, size_t length, int prot, int flags, in
 
 again:
 
+  struct RangeResult final {
+    LiveVMARegion *RegionInsertedInto;
+    void *Ptr;
+  };
+
   auto CheckIfRangeFits = [&AllocatedOffset](LiveVMARegion* Region, uint64_t length, int prot, int flags, int fd, off_t offset,
-                                             uint64_t StartingPosition = 0) -> std::pair<LiveVMARegion*, void*> {
+                                             uint64_t StartingPosition = 0) -> RangeResult {
     uint64_t AllocatedPage {~0ULL};
     uint64_t NumberOfPages = length >> FEXCore::Utils::FEX_PAGE_SHIFT;
 
@@ -314,13 +319,13 @@ again:
         void* MMapResult = ::mmap(reinterpret_cast<void*>(AllocatedOffset), length, prot, (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED, fd, offset);
 
         if (MMapResult == MAP_FAILED) {
-          return std::make_pair(Region, reinterpret_cast<void*>(-errno));
+          return RangeResult {Region, reinterpret_cast<void*>(-errno)};
         }
-        return std::make_pair(Region, MMapResult);
+        return RangeResult {Region, MMapResult};
       }
     }
 
-    return std::make_pair(nullptr, nullptr);
+    return {};
   };
 
   if (Fixed) {
@@ -329,7 +334,7 @@ again:
       // Found a slab that fits this
       if (flags & MAP_FIXED_NOREPLACE) {
         auto Fits = CheckIfRangeFits(LiveRegion, length, prot, flags, fd, offset, Addr);
-        if (Fits.first && Fits.second == reinterpret_cast<void*>(Addr)) {
+        if (Fits.RegionInsertedInto && Fits.Ptr == reinterpret_cast<void*>(Addr)) {
           // We fit correctly
           AllocatedOffset = Addr;
         } else {
@@ -355,7 +360,7 @@ again:
       // We found a LiveRegion that could hold this address. Let's try to place it
       // Check if this area is free
       auto Fits = CheckIfRangeFits(LiveRegion, length, prot, flags, fd, offset, Addr);
-      if (Fits.first && Fits.second == reinterpret_cast<void*>(Addr)) {
+      if (Fits.RegionInsertedInto && Fits.Ptr == reinterpret_cast<void*>(Addr)) {
         // We fit correctly
         AllocatedOffset = Addr;
       } else {
@@ -368,15 +373,15 @@ again:
     if (!LiveRegion) {
       for (auto it = LiveRegions->begin(); it != LiveRegions->end(); ++it) {
         auto Fits = CheckIfRangeFits(*it, length, prot, flags, fd, offset);
-        if (Fits.first && Fits.second == reinterpret_cast<void*>(AllocatedOffset)) {
+        if (Fits.RegionInsertedInto && Fits.Ptr == reinterpret_cast<void*>(AllocatedOffset)) {
           // We fit correctly
-          LiveRegion = Fits.first;
+          LiveRegion = Fits.RegionInsertedInto;
           break;
         }
 
         // Couldn't fit but mmap gave us an error
-        if (!Fits.first && Fits.second) {
-          return Fits.second;
+        if (!Fits.RegionInsertedInto && Fits.Ptr) {
+          return Fits.Ptr;
         }
 
         // nullptr on both means no error and couldn't fit

FEXCore/Source/Utils/AllocatorOverride.cpp

@@ -106,7 +106,7 @@ void EvaluateReturnAddress(void* Return) {
   // We don't know where we are when allocating. Make sure to be safe and generate the string on the stack.
   // Print an error message to let a developer know that an allocation faulted.
   char Tmp[512];
-  auto Res = fmt::format_to_n(Tmp, 512, "Allocation from 0x{:x}\n", reinterpret_cast<uint64_t>(Return));
+  auto Res = fmt::format_to_n(Tmp, 512, "ERROR: Requested memory using non-FEX allocator at 0x{:x}\n", reinterpret_cast<uint64_t>(Return));
   Tmp[Res.size] = 0;
   write(STDERR_FILENO, Tmp, Res.size);
 

FEXCore/Source/Utils/ArchHelpers/Arm64.cpp

@@ -309,7 +309,7 @@ static uint64_t DoLoad64(uint64_t Addr) {
   }
 }
 
-static std::pair<uint64_t, uint64_t> DoLoad128(uint64_t Addr) {
+static __uint128_t DoLoad128(uint64_t Addr) {
   // Any misalignment here means we cross a 16byte boundary
   // So we need two 128bit loads
   uint64_t Alignment = Addr & 0b1111;
@@ -330,10 +330,9 @@ static std::pair<uint64_t, uint64_t> DoLoad128(uint64_t Addr) {
   Data->Large.Upper = LoadAcquire128(AddrUpper);
   Data->Large.Lower = LoadAcquire128(Addr);
 
-  uint64_t ResultLower {}, ResultUpper {};
-  memcpy(&ResultLower, &Data->Bytes.Data[Alignment], sizeof(uint64_t));
-  memcpy(&ResultUpper, &Data->Bytes.Data[Alignment + sizeof(uint64_t)], sizeof(uint64_t));
-  return {ResultLower, ResultUpper};
+  __uint128_t Result {};
+  memcpy(&Result, &Data->Bytes.Data[Alignment], sizeof(Result));
+  return Result;
 }
 
 static bool RunCASPAL(uint64_t* GPRs, uint32_t Size, uint32_t DesiredReg1, uint32_t DesiredReg2, uint32_t ExpectedReg1,
@@ -580,10 +579,10 @@ static uint64_t HandleCASPAL_ARMv8(uint32_t Instr, uintptr_t ProgramCounter, uin
         auto Res = DoLoad128(Addr);
         // We set the result register if it isn't a zero register
         if (DataReg != 31) {
-          GPRs[DataReg] = std::get<0>(Res);
+          GPRs[DataReg] = Res;
         }
         if (DataReg2 != 31) {
-          GPRs[DataReg2] = std::get<1>(Res);
+          GPRs[DataReg2] = Res >> 64;
         }
 
         // Skip ldaxp and clrex
@@ -1953,7 +1952,7 @@ static uint64_t HandleAtomicLoadstoreExclusive(uintptr_t ProgramCounter, uint64_
 }
 
 [[nodiscard]]
-std::pair<bool, int32_t>
+std::optional<int32_t>
 HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandlerType HandleType, uintptr_t ProgramCounter, uint64_t* GPRs) {
 #ifdef _M_ARM_64
   constexpr bool is_arm64 = true;
@@ -1961,9 +1960,8 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandl
   constexpr bool is_arm64 = false;
 #endif
 
-  constexpr auto NotHandled = std::make_pair(false, 0);
   if constexpr (!is_arm64) {
-    return NotHandled;
+    return std::nullopt;
   }
 
   uint32_t* PC = (uint32_t*)ProgramCounter;
@@ -1985,38 +1983,38 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandl
         (Instr & LDAXR_MASK) == LDAPR_INST) { // LDAPR*
       if (ArchHelpers::Arm64::HandleAtomicLoad(Instr, GPRs, 0)) {
         // Skip this instruction now
-        return std::make_pair(true, 4);
+        return 4;
       } else {
         LogMan::Msg::EFmt("Unhandled JIT SIGBUS LDAR*: PC: 0x{:x} Instruction: 0x{:08x}\n", ProgramCounter, PC[0]);
-        return NotHandled;
+        return std::nullopt;
       }
     } else if ((Instr & LDAXR_MASK) == STLR_INST) { // STLR*
       if (ArchHelpers::Arm64::HandleAtomicStore(Instr, GPRs, 0, StrictSplitLockMutex)) {
         // Skip this instruction now
-        return std::make_pair(true, 4);
+        return 4;
       } else {
         LogMan::Msg::EFmt("Unhandled JIT SIGBUS STLR*: PC: 0x{:x} Instruction: 0x{:08x}\n", ProgramCounter, PC[0]);
-        return NotHandled;
+        return std::nullopt;
       }
     } else if ((Instr & RCPC2_MASK) == LDAPUR_INST) { // LDAPUR*
       // Extract the 9-bit offset from the instruction
       int32_t Offset = static_cast<int32_t>(Instr) << 11 >> 23;
       if (ArchHelpers::Arm64::HandleAtomicLoad(Instr, GPRs, Offset)) {
         // Skip this instruction now
-        return std::make_pair(true, 4);
+        return 4;
       } else {
         LogMan::Msg::EFmt("Unhandled JIT SIGBUS LDAPUR*: PC: 0x{:x} Instruction: 0x{:08x}\n", ProgramCounter, PC[0]);
-        return NotHandled;
+        return std::nullopt;
       }
     } else if ((Instr & RCPC2_MASK) == STLUR_INST) { // STLUR*
       // Extract the 9-bit offset from the instruction
       int32_t Offset = static_cast<int32_t>(Instr) << 11 >> 23;
       if (ArchHelpers::Arm64::HandleAtomicStore(Instr, GPRs, Offset, StrictSplitLockMutex)) {
         // Skip this instruction now
-        return std::make_pair(true, 4);
+        return 4;
       } else {
         LogMan::Msg::EFmt("Unhandled JIT SIGBUS LDLUR*: PC: 0x{:x} Instruction: 0x{:08x}\n", ProgramCounter, PC[0]);
-        return NotHandled;
+        return std::nullopt;
       }
     }
   }
@@ -2030,18 +2028,18 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandl
   if ((Instr & ArchHelpers::Arm64::CASPAL_MASK) == ArchHelpers::Arm64::CASPAL_INST) { // CASPAL
     if (ArchHelpers::Arm64::HandleCASPAL(Instr, GPRs, StrictSplitLockMutex)) {
       // Skip this instruction now
-      return std::make_pair(true, 4);
+      return 4;
     } else {
       LogMan::Msg::EFmt("Unhandled JIT SIGBUS CASPAL: PC: 0x{:x} Instruction: 0x{:08x}\n", ProgramCounter, PC[0]);
-      return NotHandled;
+      return std::nullopt;
     }
   } else if ((Instr & ArchHelpers::Arm64::CASAL_MASK) == ArchHelpers::Arm64::CASAL_INST) { // CASAL
     if (ArchHelpers::Arm64::HandleCASAL(GPRs, Instr, StrictSplitLockMutex)) {
       // Skip this instruction now
-      return std::make_pair(true, 4);
+      return 4;
     } else {
       LogMan::Msg::EFmt("Unhandled JIT SIGBUS CASAL: PC: 0x{:x} Instruction: 0x{:08x}\n", ProgramCounter, PC[0]);
-      return NotHandled;
+      return std::nullopt;
     }
   } else if ((Instr & LDAXR_MASK) == LDAR_INST ||  // LDAR*
              (Instr & LDAXR_MASK) == LDAPR_INST || // LDAPR*
@@ -2051,17 +2049,17 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandl
   } else if ((Instr & ArchHelpers::Arm64::ATOMIC_MEM_MASK) == ArchHelpers::Arm64::ATOMIC_MEM_INST) { // Atomic memory op
     if (ArchHelpers::Arm64::HandleAtomicMemOp(Instr, GPRs, StrictSplitLockMutex)) {
       // Skip this instruction now
-      return std::make_pair(true, 4);
+      return 4;
     } else {
       uint8_t Op = (PC[0] >> 12) & 0xF;
       LogMan::Msg::EFmt("Unhandled JIT SIGBUS Atomic mem op 0x{:02x}: PC: 0x{:x} Instruction: 0x{:08x}\n", Op, ProgramCounter, PC[0]);
-      return NotHandled;
+      return std::nullopt;
     }
   } else if ((Instr & ArchHelpers::Arm64::LDAXR_MASK) == ArchHelpers::Arm64::LDAXR_INST) { // LDAXR*
     uint64_t BytesToSkip = ArchHelpers::Arm64::HandleAtomicLoadstoreExclusive(ProgramCounter, GPRs, StrictSplitLockMutex);
     if (BytesToSkip) {
       // Skip this instruction now
-      return std::make_pair(true, BytesToSkip);
+      return BytesToSkip;
     }
     // Explicit fallthrough to the backpatch handler below!
   } else if ((Instr & ArchHelpers::Arm64::LDAXP_MASK) == ArchHelpers::Arm64::LDAXP_INST) { // LDAXP
@@ -2069,7 +2067,7 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandl
     uint64_t BytesToSkip = ArchHelpers::Arm64::HandleCASPAL_ARMv8(Instr, ProgramCounter, GPRs, StrictSplitLockMutex);
     if (BytesToSkip) {
       // Skip this instruction now
-      return std::make_pair(true, BytesToSkip);
+      return BytesToSkip;
     }
   }
 
@@ -2091,7 +2089,7 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandl
     std::atomic_ref<uint32_t>(PC[0]).store(LDR, std::memory_order_release);
     ClearICache(&PC[0], 8);
     // With the instruction modified, now execute again.
-    return std::make_pair(true, 0);
+    return 0;
   } else if ((Instr & LDAXR_MASK) == STLR_INST) { // STLR*
     uint32_t STR = STR_INST;
     STR |= Size << 30;
@@ -2103,7 +2101,7 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandl
     std::atomic_ref<uint32_t>(PC[0]).store(STR, std::memory_order_release);
     ClearICache(&PC[-1], 8);
     // Back up one instruction and have another go
-    return std::make_pair(true, -4);
+    return -4;
   } else if ((Instr & RCPC2_MASK) == LDAPUR_INST) { // LDAPUR*
     // Extract the 9-bit offset from the instruction
     uint32_t LDUR = LDUR_INST;
@@ -2118,7 +2116,7 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandl
     std::atomic_ref<uint32_t>(PC[0]).store(LDUR, std::memory_order_release);
     ClearICache(&PC[0], 8);
     // With the instruction modified, now execute again.
-    return std::make_pair(true, 0);
+    return 0;
   } else if ((Instr & RCPC2_MASK) == STLUR_INST) { // STLUR*
     uint32_t STUR = STUR_INST;
     STUR |= Size << 30;
@@ -2132,20 +2130,20 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandl
 
     ClearICache(&PC[-1], 8);
     // Back up one instruction and have another go
-    return std::make_pair(true, -4);
+    return -4;
   } else if ((Instr & ArchHelpers::Arm64::LDAXP_MASK) == ArchHelpers::Arm64::LDAXP_INST) { // LDAXP
     /// This is handling the case of paranoid ARMv8.0-a atomic stores.
     /// This backpatches the ldaxp+stlxp+cbnz if the previous `HandleCASPAL_ARMv8` didn't handle the case.
     if (ArchHelpers::Arm64::HandleAtomicVectorStore(Instr, ProgramCounter)) {
-      return std::make_pair(true, 0);
+      return 0;
     } else {
       LogMan::Msg::EFmt("Unhandled JIT SIGBUS LDAXP: PC: 0x{:x} Instruction: 0x{:08x}\n", ProgramCounter, PC[0]);
-      return NotHandled;
+      return std::nullopt;
     }
   } else if ((Instr & ArchHelpers::Arm64::STLXP_MASK) == ArchHelpers::Arm64::STLXP_INST) { // STLXP
     // Should not trigger - middle of an LDAXP/STAXP pair.
     LogMan::Msg::EFmt("Unhandled JIT SIGBUS STLXP: PC: 0x{:x} Instruction: 0x{:08x}\n", ProgramCounter, PC[0]);
-    return NotHandled;
+    return std::nullopt;
   }
 
   // Check if another thread backpatched this instruction before this thread got here
@@ -2159,11 +2157,11 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandl
       // Check if the next instruction is a DMB.
       auto DMBInst = std::atomic_ref<uint32_t>(PC[1]).load(std::memory_order_acquire);
       if (DMBInst == DMB_LD) {
-        return std::make_pair(true, 0);
+        return 0;
       }
     } else {
       // No DMB instruction with this HandleType.
-      return std::make_pair(true, 0);
+      return 0;
     }
   } else if ((AtomicInst & LDSTREGISTER_MASK) == STR_INST || (AtomicInst & LDSTUNSCALED_MASK) == STUR_INST) {
     if (HandleType != UnalignedHandlerType::NonAtomic) {
@@ -2171,11 +2169,11 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandl
       auto DMBInst = std::atomic_ref<uint32_t>(PC[-1]).load(std::memory_order_acquire);
       if (DMBInst == DMB) {
         // Return handled, make sure to adjust PC so we run the DMB.
-        return std::make_pair(true, -4);
+        return -4;
       }
     } else {
       // No DMB instruction with this HandleType.
-      return std::make_pair(true, 0);
+      return 0;
     }
   } else if (AtomicInst == DMB) {
     // ARMv8.0-a LDAXP backpatch handling. Will have turned in to the following:
@@ -2186,12 +2184,12 @@ HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandl
     auto DMBInst = std::atomic_ref<uint32_t>(PC[2]).load(std::memory_order_acquire);
     if ((STPInst & LDSTP_MASK) == STP_INST && DMBInst == DMB) {
       // Code that was backpatched is what was expected for ARMv8.0-a LDAXP.
-      return std::make_pair(true, 0);
+      return 0;
     }
   }
 
   LogMan::Msg::EFmt("Unhandled JIT SIGBUS: PC: 0x{:x} Instruction: 0x{:08x}\n", ProgramCounter, PC[0]);
-  return NotHandled;
+  return std::nullopt;
 }
 
 

FEXCore/Source/Utils/ArchHelpers/Arm64_stubs.cpp

@@ -11,7 +11,7 @@ namespace FEXCore::ArchHelpers::Arm64 {
 
 // Obvously such a configuration can't do the actual arm64-specific stuff
 
-std::pair<bool, int32_t>
+std::optional<int32_t>
 HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandlerType HandleType, uintptr_t ProgramCounter, uint64_t* GPRs) {
   ERROR_AND_DIE_FMT("HandleAtomicMemOp Not Implemented");
 }

FEXCore/Source/Utils/Profiler.cpp

@@ -48,7 +48,7 @@ static inline uint64_t GetTime() {
 #endif
 
 namespace FEXCore::Profiler {
-ProfilerBlock::ProfilerBlock(std::string_view const Format)
+ProfilerBlock::ProfilerBlock(const std::string_view Format)
   : DurationBegin {GetTime()}
   , Format {Format} {}
 
@@ -92,7 +92,7 @@ void Shutdown() {
   }
 }
 
-void TraceObject(std::string_view const Format, uint64_t Duration) {
+void TraceObject(const std::string_view Format, uint64_t Duration) {
   if (TraceFD != -1) {
     // Print the duration as something that began negative duration ago
     const auto StringSize = Format.size() + strlen(" (lduration=-)\n") + 22;
@@ -102,7 +102,7 @@ void TraceObject(std::string_view const Format, uint64_t Duration) {
   }
 }
 
-void TraceObject(std::string_view const Format) {
+void TraceObject(const std::string_view Format) {
   if (TraceFD != -1) {
     const auto StringSize = Format.size() + 1;
     auto Event = reinterpret_cast<char*>(alloca(StringSize));
@@ -164,9 +164,9 @@ void Shutdown() {
   }
 }
 
-void TraceObject(std::string_view const Format, uint64_t Duration) {}
+void TraceObject(const std::string_view Format, uint64_t Duration) {}
 
-void TraceObject(std::string_view const Format) {
+void TraceObject(const std::string_view Format) {
   if (Tracy::Enable) {
     TracyMessage(Format.data(), Format.size());
   }
@@ -212,7 +212,7 @@ void Shutdown() {
 #endif
 }
 
-void TraceObject(std::string_view const Format, uint64_t Duration) {
+void TraceObject(const std::string_view Format, uint64_t Duration) {
 #if FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_GPUVIS
   GPUVis::TraceObject(Format, Duration);
 #elif FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_TRACY
@@ -220,7 +220,7 @@ void TraceObject(std::string_view const Format, uint64_t Duration) {
 #endif
 }
 
-void TraceObject(std::string_view const Format) {
+void TraceObject(const std::string_view Format) {
 #if FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_GPUVIS
   GPUVis::TraceObject(Format);
 #elif FEXCORE_PROFILER_BACKEND == FEXCORE_PROFILER_BACKEND_TRACY

FEXCore/include/FEXCore/Config/Config.h

@@ -75,7 +75,7 @@ enum class LayerType {
 };
 
 template<typename PairTypes, typename ArrayPairType>
-static inline std::optional<fextl::string> EnumParser(const ArrayPairType& EnumPairs, std::string_view const View) {
+static inline std::optional<fextl::string> EnumParser(const ArrayPairType& EnumPairs, const std::string_view View) {
   uint64_t EnumMask {};
   auto Results = std::from_chars(View.data(), View.data() + View.size(), EnumMask);
   if (Results.ec == std::errc()) {

FEXCore/include/FEXCore/Core/Context.h

@@ -64,6 +64,9 @@ public:
 
 using CodeRangeInvalidationFn = std::function<void(uint64_t start, uint64_t Length)>;
 
+// Nested vector of guest block entrypoints
+using InvalidatedEntryAccumulator = fextl::vector<fextl::vector<uint64_t>>;
+
 using CustomIREntrypointHandler = std::function<void(uintptr_t Entrypoint, IR::IREmitter*)>;
 
 using ExitHandler = std::function<void(Core::InternalThreadState* Thread)>;
@@ -172,7 +175,8 @@ public:
   FEX_DEFAULT_VISIBILITY virtual void WriteFilesWithCode(AOTIRCodeFileWriterFn Writer) = 0;
 
   FEX_DEFAULT_VISIBILITY virtual void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) = 0;
-  FEX_DEFAULT_VISIBILITY virtual void InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) = 0;
+  FEX_DEFAULT_VISIBILITY virtual void InvalidateGuestCodeRange(
+    FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, uint64_t Start, uint64_t Length) = 0;
   FEX_DEFAULT_VISIBILITY virtual FEXCore::ForkableSharedMutex& GetCodeInvalidationMutex() = 0;
 
   FEX_DEFAULT_VISIBILITY virtual void MarkMemoryShared(FEXCore::Core::InternalThreadState* Thread) = 0;

FEXCore/include/FEXCore/Core/CoreState.h

@@ -116,18 +116,57 @@ struct CPUState {
   uint64_t gs_cached {};
   uint64_t fs_cached {};
   uint8_t flags[48] {};
+  uint64_t callret_sp {};
   uint64_t _pad1 {};
-  uint64_t _pad2 {};
   uint64_t mm[8][2] {};
 
   // 32bit x86 state
-  struct {
-    uint32_t base;
+  struct gdt_segment {
+    uint16_t Limit0;
+    uint16_t Base0;
+    uint16_t Base1  : 8;
+    uint16_t Type   : 4;
+    uint16_t S      : 1;
+    uint16_t DPL    : 2;
+    uint16_t P      : 1;
+    uint16_t Limit1 : 4;
+    uint16_t AVL    : 1;
+    uint16_t L      : 1;
+    uint16_t D      : 1;
+    uint16_t G      : 1;
+    uint16_t Base2  : 8;
   } gdt[32] {};
+
+  static uint32_t CalculateGDTBase(gdt_segment GDT) {
+    uint32_t Base{};
+    Base |= GDT.Base2 << 24;
+    Base |= GDT.Base1 << 16;
+    Base |= GDT.Base0;
+    return Base;
+  }
+
+  static uint32_t CalculateGDTLimit(gdt_segment GDT) {
+    uint32_t Limit{};
+    Limit |= GDT.Limit1 << 16;
+    Limit |= GDT.Limit0;
+    return Limit;
+  }
+
+  static void SetGDTBase(gdt_segment *GDT, uint32_t Base) {
+    GDT->Base0 = Base;
+    GDT->Base1 = Base >> 16;
+    GDT->Base2 = Base >> 24;
+  }
+
+  static void SetGDTLimit(gdt_segment *GDT, uint32_t Limit) {
+    GDT->Limit0 = Limit;
+    GDT->Limit1 = Limit >> 16;
+  }
+
   uint16_t FCW {0x37F};
   uint8_t AbridgedFTW {};
 
-  uint8_t _pad3[5];
+  uint8_t _pad2[5];
   // PF/AF are statically mapped as-if they were r16/r17 (which do not exist in
   // x86 otherwise). This allows a straightforward mapping for SRA.
   static constexpr uint8_t PF_AS_GREG = 16;
@@ -135,6 +174,7 @@ struct CPUState {
 
   static constexpr size_t FLAG_SIZE = sizeof(flags[0]);
   static constexpr size_t GDT_SIZE = sizeof(gdt[0]);
+  static_assert(GDT_SIZE == sizeof(uint64_t), "Segments required to be 8-byte in size.");
   static constexpr size_t GPR_REG_SIZE = sizeof(gregs[0]);
   static constexpr size_t XMM_AVX_REG_SIZE = sizeof(xmm.avx.data[0]);
   static constexpr size_t XMM_SSE_REG_SIZE = XMM_AVX_REG_SIZE / 2;

FEXCore/include/FEXCore/Core/HostFeatures.h

@@ -38,6 +38,7 @@ struct HostFeatures {
   bool SupportsCPUIndexInTPIDRRO {};
   bool SupportsFRINTTS {};
   bool SupportsECV {};
+  bool SupportsWFXT {};
 
   // Float exception behaviour
   bool SupportsAFP {};

FEXCore/include/FEXCore/Debug/InternalThreadState.h

@@ -36,7 +36,7 @@ class OpDispatchBuilder;
 class PassManager;
 } // namespace FEXCore::IR
 
-namespace FEXCore::Profiler {
+namespace FEXCore::SHMStats {
 struct ThreadStats;
 };
 
@@ -100,11 +100,16 @@ struct InternalThreadState : public FEXCore::Allocator::FEXAllocOperators {
   std::shared_mutex ObjectCacheRefCounter {};
 
   // This pointer is owned by the frontend.
-  FEXCore::Profiler::ThreadStats* ThreadStats {};
+  FEXCore::SHMStats::ThreadStats* ThreadStats {};
 
   ///< Data pointer for exclusive use by the frontend
   void* FrontendPtr;
 
+  static constexpr size_t CALLRET_STACK_SIZE {0x400000};
+
+  // The low address of the call-ret stack allocation (not including guard pages)
+  void* CallRetStackBase {};
+
   // BaseFrameState should always be at the end, directly before the interrupt fault page
   alignas(16) FEXCore::Core::CpuStateFrame BaseFrameState {};
 

FEXCore/include/FEXCore/HLE/SyscallHandler.h

@@ -42,6 +42,12 @@ enum class SyscallOSABI {
   OS_GENERIC, // No JIT-side argument handling, spill/fill all regs.
 };
 
+struct ExecutableRangeInfo {
+  uint64_t Base;
+  uint64_t Size;
+  bool Writable;
+};
+
 class SyscallHandler;
 class SourcecodeResolver;
 
@@ -74,6 +80,7 @@ public:
   virtual void MarkGuestExecutableRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) {}
   virtual void MarkOvercommitRange(uint64_t Start, uint64_t Length) {}
   virtual void UnmarkOvercommitRange(uint64_t Start, uint64_t Length) {}
+  virtual ExecutableRangeInfo QueryGuestExecutableRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Address) = 0;
   virtual AOTIRCacheEntryLookupResult LookupAOTIRCacheEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestAddr) = 0;
 
   virtual void PreCompile() {}

FEXCore/include/FEXCore/IR/IR.h

@@ -107,14 +107,12 @@ enum IndexNamedVectorConstant : uint8_t {
 struct SHA256Sum final {
   uint8_t data[32];
   [[nodiscard]]
-  bool
-  operator<(const SHA256Sum& rhs) const {
+  bool operator<(const SHA256Sum& rhs) const {
     return memcmp(data, rhs.data, sizeof(data)) < 0;
   }
 
   [[nodiscard]]
-  bool
-  operator==(const SHA256Sum& rhs) const {
+  bool operator==(const SHA256Sum& rhs) const {
     return memcmp(data, rhs.data, sizeof(data)) == 0;
   }
 };

FEXCore/include/FEXCore/Utils/ArchHelpers/Arm64.h

@@ -4,7 +4,7 @@
 #include <FEXCore/Utils/CompilerDefs.h>
 
 #include <stdint.h>
-#include <utility>
+#include <optional>
 
 namespace FEXCore::Core {
 struct InternalThreadState;
@@ -30,10 +30,10 @@ enum class UnalignedHandlerType {
  * @param ProgramCounter The location in memory for the instruction that did the access
  * @param GPRs The array of GPRs from the signal context. This will be modified and the host context needs to be updated on signal return.
  *
- * @return A pair where the first element is if the unaligned access has been handle and the second element is how many bytes to modify the host PC
+ * @return Returns a value if the unaligned access has been handled with how many bytes to modify the host PC
  * by. FEXCore will return a positive or negative offset depending on internal handling.
  */
 [[nodiscard]]
-FEX_DEFAULT_VISIBILITY std::pair<bool, int32_t>
+FEX_DEFAULT_VISIBILITY std::optional<int32_t>
 HandleUnalignedAccess(FEXCore::Core::InternalThreadState* Thread, UnalignedHandlerType HandleType, uintptr_t ProgramCounter, uint64_t* GPRs);
 } // namespace FEXCore::ArchHelpers::Arm64

FEXCore/include/FEXCore/Utils/EnumUtils.h

@@ -12,20 +12,17 @@ namespace FEXCore {
 // boilerplate.
 #define FEX_DECLARE_ENUM_FLAG_OPERATORS(type)                        \
   [[nodiscard]]                                                      \
-  constexpr type                                                     \
-  operator|(type a, type b) noexcept {                               \
+  constexpr type operator|(type a, type b) noexcept {                \
     using T = std::underlying_type_t<type>;                          \
     return static_cast<type>(static_cast<T>(a) | static_cast<T>(b)); \
   }                                                                  \
   [[nodiscard]]                                                      \
-  constexpr type                                                     \
-  operator&(type a, type b) noexcept {                               \
+  constexpr type operator&(type a, type b) noexcept {                \
     using T = std::underlying_type_t<type>;                          \
     return static_cast<type>(static_cast<T>(a) & static_cast<T>(b)); \
   }                                                                  \
   [[nodiscard]]                                                      \
-  constexpr type                                                     \
-  operator^(type a, type b) noexcept {                               \
+  constexpr type operator^(type a, type b) noexcept {                \
     using T = std::underlying_type_t<type>;                          \
     return static_cast<type>(static_cast<T>(a) ^ static_cast<T>(b)); \
   }                                                                  \
@@ -42,8 +39,7 @@ namespace FEXCore {
     return a;                                                        \
   }                                                                  \
   [[nodiscard]]                                                      \
-  constexpr type                                                     \
-  operator~(type key) noexcept {                                     \
+  constexpr type operator~(type key) noexcept {                      \
     using T = std::underlying_type_t<type>;                          \
     return static_cast<type>(~static_cast<T>(key));                  \
   }                                                                  \

FEXCore/include/FEXCore/Utils/File.h

@@ -82,7 +82,7 @@ public:
 #endif
   }
 
-  ssize_t Write(std::string_view const Data) {
+  ssize_t Write(const std::string_view Data) {
     return Write(Data.data(), Data.size());
   }
 

FEXCore/include/FEXCore/Utils/IntervalList.h

@@ -31,6 +31,7 @@ public:
     bool Enclosed;       ///< If the given offset was enclosed by an interval
     DifferenceType Size; ///< Size of the interval starting from the query offset, or distance to the next interval if
                          /// `Enclosed` is false (if there is no next interval, size is 0)
+    Interval Interval;   ///< The interval that the query offset is enclosed by, or the next interval if `Enclosed` is false
   };
 
   void Clear() {
@@ -134,11 +135,11 @@ public:
     }); // Lowest offset interval that (maybe) overlaps with the query offset
 
     if (It == Intervals.end()) { // No overlaps past offset
-      return {false, {}};
+      return {false, 0, {}};
     } else if (It->Offset > Offset) { // No overlap, return the distance to the next possible overlap
-      return {false, It->Offset - Offset};
+      return {false, It->Offset - Offset, *It};
     } else { // Overlap, return the distance to the end of the overlap
-      return {true, It->End - Offset};
+      return {true, It->End - Offset, *It};
     }
   }
 

FEXCore/include/FEXCore/Utils/LogManager.h

@@ -141,10 +141,10 @@ namespace Msg {
     }                                 \
   } while (0);
 
-#define ERROR_AND_DIE_FMT(...)      \
-  do {                              \
-    LogMan::Msg::EFmt(__VA_ARGS__); \
-    FEX_TRAP_EXECUTION;             \
+#define ERROR_AND_DIE_FMT(...)                      \
+  do {                                              \
+    LogMan::Msg::MFmt(LogMan::ASSERT, __VA_ARGS__); \
+    FEX_TRAP_EXECUTION;                             \
   } while (0)
 
 } // namespace Msg

FEXCore/include/FEXCore/Utils/Profiler.h

@@ -1,13 +1,8 @@
 // SPDX-License-Identifier: MIT
 #pragma once
-#include <atomic>
 #include <cstdint>
 #include <string_view>
 
-#ifdef _M_X86_64
-#include <x86intrin.h>
-#endif
-
 #include <FEXCore/Utils/CompilerDefs.h>
 
 #define FEXCORE_PROFILER_BACKEND_OFF 0
@@ -19,73 +14,17 @@
 #endif
 
 namespace FEXCore::Profiler {
-// FEXCore live-stats
-constexpr uint8_t STATS_VERSION = 2;
-enum class AppType : uint8_t {
-  LINUX_32,
-  LINUX_64,
-  WIN_ARM64EC,
-  WIN_WOW64,
-};
-
-struct ThreadStatsHeader {
-  uint8_t Version;
-  AppType app_type;
-  uint8_t _pad[2];
-  char fex_version[48];
-  std::atomic<uint32_t> Head;
-  std::atomic<uint32_t> Size;
-  uint32_t Pad;
-};
-
-struct ThreadStats {
-  std::atomic<uint32_t> Next;
-  std::atomic<uint32_t> TID;
-
-  // Accumulated time (In unscaled CPU cycles!)
-  uint64_t AccumulatedJITTime;
-  uint64_t AccumulatedSignalTime;
-
-  // Accumulated event counts
-  uint64_t AccumulatedSIGBUSCount;
-  uint64_t AccumulatedSMCCount;
-  uint64_t AccumulatedFloatFallbackCount;
-};
+#define UniqueScopeName2(name, line) name##line
+#define UniqueScopeName(name, line) UniqueScopeName2(name, line)
 
 #ifdef ENABLE_FEXCORE_PROFILER
 
-#ifdef _M_ARM_64
-/**
- * @brief Get the raw cycle counter with synchronizing isb.
- *
- * `CNTVCTSS_EL0` also does the same thing, but requires the FEAT_ECV feature.
- */
-static inline uint64_t GetCycleCounter() {
-  uint64_t Result {};
-  __asm volatile(R"(
-      isb;
-      mrs %[Res], CNTVCT_EL0;
-    )"
-                 : [Res] "=r"(Result));
-  return Result;
-}
-#else
-static inline uint64_t GetCycleCounter() {
-  unsigned dummy;
-  uint64_t tsc = __rdtscp(&dummy);
-  return tsc;
-}
-#endif
-
 FEX_DEFAULT_VISIBILITY void Init(std::string_view ProgramName, std::string_view ProgramPath);
 FEX_DEFAULT_VISIBILITY void PostForkAction(bool IsChild);
 FEX_DEFAULT_VISIBILITY bool IsActive();
 FEX_DEFAULT_VISIBILITY void Shutdown();
-FEX_DEFAULT_VISIBILITY void TraceObject(std::string_view const Format);
-FEX_DEFAULT_VISIBILITY void TraceObject(std::string_view const Format, uint64_t Duration);
-
-#define UniqueScopeName2(name, line) name##line
-#define UniqueScopeName(name, line) UniqueScopeName2(name, line)
+FEX_DEFAULT_VISIBILITY void TraceObject(const std::string_view Format);
+FEX_DEFAULT_VISIBILITY void TraceObject(const std::string_view Format, uint64_t Duration);
 
 // Declare an instantaneous profiler event.
 #define FEXCORE_PROFILE_INSTANT(name) FEXCore::Profiler::TraceObject(name)
@@ -97,49 +36,19 @@ FEX_DEFAULT_VISIBILITY void TraceObject(std::string_view const Format, uint64_t
 // A class that follows scoping rules to generate a profile duration block
 class ProfilerBlock final {
 public:
-  ProfilerBlock(std::string_view const Format);
+  ProfilerBlock(const std::string_view Format);
 
   ~ProfilerBlock();
 
 private:
   uint64_t DurationBegin;
-  std::string_view const Format;
+  const std::string_view Format;
 };
 
 // Declare a scoped profile block variable with a fixed name.
 #define FEXCORE_PROFILE_SCOPED(name) FEXCore::Profiler::ProfilerBlock UniqueScopeName(ScopedBlock_, __LINE__)(name)
 #endif
 
-template<typename T, size_t FlatOffset = 0>
-class AccumulationBlock final {
-public:
-  AccumulationBlock(T* Stat)
-    : Begin {GetCycleCounter()}
-    , Stat {Stat} {}
-
-  ~AccumulationBlock() {
-    const auto Duration = GetCycleCounter() - Begin + FlatOffset;
-    if (Stat) {
-      auto ref = std::atomic_ref<T>(*Stat);
-      ref.fetch_add(Duration, std::memory_order_relaxed);
-    }
-  }
-
-private:
-  uint64_t Begin;
-  T* Stat;
-};
-
-#define FEXCORE_PROFILE_ACCUMULATION(ThreadState, Stat)                                                                          \
-  FEXCore::Profiler::AccumulationBlock<decltype(ThreadState->ThreadStats->Stat)> UniqueScopeName(ScopedAccumulation_, __LINE__)( \
-    ThreadState->ThreadStats ? &ThreadState->ThreadStats->Stat : nullptr);
-#define FEXCORE_PROFILE_INSTANT_INCREMENT(ThreadState, Stat, value) \
-  do {                                                              \
-    if (ThreadState->ThreadStats) {                                 \
-      ThreadState->ThreadStats->Stat += value;                      \
-    }                                                               \
-  } while (0)
-
 #else
 [[maybe_unused]]
 static void Init(std::string_view ProgramName, std::string_view ProgramPath) {}
@@ -148,9 +57,9 @@ static void PostForkAction(bool IsChild) {}
 [[maybe_unused]]
 static void Shutdown() {}
 [[maybe_unused]]
-static void TraceObject(std::string_view const Format) {}
+static void TraceObject(const std::string_view Format) {}
 [[maybe_unused]]
-static void TraceObject(std::string_view const, uint64_t) {}
+static void TraceObject(const std::string_view, uint64_t) {}
 
 #define FEXCORE_PROFILE_INSTANT(...) \
   do {                               \
@@ -158,12 +67,6 @@ static void TraceObject(std::string_view const, uint64_t) {}
 #define FEXCORE_PROFILE_SCOPED(...) \
   do {                              \
   } while (0)
-#define FEXCORE_PROFILE_ACCUMULATION(...) \
-  do {                                    \
-  } while (0)
-#define FEXCORE_PROFILE_INSTANT_INCREMENT(...) \
-  do {                                         \
-  } while (0)
 
 #endif
 } // namespace FEXCore::Profiler

FEXCore/include/FEXCore/Utils/SHMStats.h

@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+#include <atomic>
+#include <cstdint>
+
+#ifdef _M_X86_64
+#include <x86intrin.h>
+#endif
+
+namespace FEXCore::SHMStats {
+#ifdef _M_ARM_64
+/**
+ * @brief Get the raw cycle counter with synchronizing isb.
+ *
+ * `CNTVCTSS_EL0` also does the same thing, but requires the FEAT_ECV feature.
+ */
+static inline uint64_t GetCycleCounter() {
+  uint64_t Result {};
+  __asm volatile(R"(
+      isb;
+      mrs %[Res], CNTVCT_EL0;
+    )"
+                 : [Res] "=r"(Result));
+  return Result;
+}
+#else
+static inline uint64_t GetCycleCounter() {
+  unsigned dummy;
+  uint64_t tsc = __rdtscp(&dummy);
+  return tsc;
+}
+#endif
+// FEXCore live-stats
+constexpr uint8_t STATS_VERSION = 2;
+enum class AppType : uint8_t {
+  LINUX_32,
+  LINUX_64,
+  WIN_ARM64EC,
+  WIN_WOW64,
+};
+
+struct ThreadStatsHeader {
+  uint8_t Version;
+  AppType app_type;
+  uint8_t _pad[2];
+  char fex_version[48];
+  std::atomic<uint32_t> Head;
+  std::atomic<uint32_t> Size;
+  uint32_t Pad;
+};
+
+struct ThreadStats {
+  std::atomic<uint32_t> Next;
+  std::atomic<uint32_t> TID;
+
+  // Accumulated time (In unscaled CPU cycles!)
+  uint64_t AccumulatedJITTime;
+  uint64_t AccumulatedSignalTime;
+
+  // Accumulated event counts
+  uint64_t AccumulatedSIGBUSCount;
+  uint64_t AccumulatedSMCCount;
+  uint64_t AccumulatedFloatFallbackCount;
+};
+
+template<typename T, size_t FlatOffset = 0>
+class AccumulationBlock final {
+public:
+  AccumulationBlock(T* Stat)
+    : Begin {GetCycleCounter()}
+    , Stat {Stat} {}
+
+  ~AccumulationBlock() {
+    const auto Duration = GetCycleCounter() - Begin + FlatOffset;
+    if (Stat) {
+      auto ref = std::atomic_ref<T>(*Stat);
+      ref.fetch_add(Duration, std::memory_order_relaxed);
+    }
+  }
+
+private:
+  uint64_t Begin;
+  T* Stat;
+};
+#define UniqueScopeName2(name, line) name##line
+#define UniqueScopeName(name, line) UniqueScopeName2(name, line)
+
+#define FEXCORE_PROFILE_ACCUMULATION(ThreadState, Stat)                                                                          \
+  FEXCore::SHMStats::AccumulationBlock<decltype(ThreadState->ThreadStats->Stat)> UniqueScopeName(ScopedAccumulation_, __LINE__)( \
+    ThreadState->ThreadStats ? &ThreadState->ThreadStats->Stat : nullptr);
+#define FEXCORE_PROFILE_INSTANT_INCREMENT(ThreadState, Stat, value) \
+  do {                                                              \
+    if (ThreadState->ThreadStats) {                                 \
+      ThreadState->ThreadStats->Stat += value;                      \
+    }                                                               \
+  } while (0)
+
+} // namespace FEXCore::SHMStats

FEXCore/include/FEXCore/Utils/ThreadPoolAllocator.h

@@ -445,6 +445,10 @@ public:
     : ThreadAllocator {Allocator}
     , Size {Size} {}
 
+  ~PoolBufferWithTimedRetirement() {
+    UnclaimBuffer();
+  }
+
   /**
    * @brief Return the owned buffer or allocate another one from the `Allocator`
    *

FEXCore/include/FEXCore/Utils/refcount_shared_mutex.h

@@ -1,248 +0,0 @@
-// SPDX-License-Identifier: MIT
-#pragma once
-#include <FEXCore/Utils/LogManager.h>
-
-#include <atomic>
-#include <linux/futex.h>
-#include <sys/syscall.h>
-#include <shared_mutex>
-#include <unistd.h>
-
-namespace FEXCore::Utils {
-/**
- * @brief This class is similar to std::shared_mutex but is safe to shared lock multiple times from the same thread.
- *
- * Just like std::shared_mutex, this has shared lock priority when a shared lock is already held.
- */
-class refcount_shared_mutex final {
-public:
-  void lock() {
-    auto UniqueResult = TryUniqueLock();
-    if (UniqueResult.second) {
-      // Managed to get the unique lock
-      return;
-    }
-
-    int Op = FUTEX_WAIT | FUTEX_PRIVATE_FLAG;
-
-    do {
-      ::syscall(SYS_futex, &Futex, Op,
-                UniqueResult.first, // Value
-                nullptr,            // Timeout
-                nullptr,            // Addr
-                0);
-
-      UniqueResult = TryUniqueLock();
-      // If Res == 0 then check the unique lock to see if unique is no longer owned
-      if (UniqueResult.second) {
-        // Unique lock succeeded
-        return;
-      }
-    } while (true);
-  }
-
-  bool try_lock() {
-    auto UniqueResult = TryUniqueLock();
-    return UniqueResult.second;
-  }
-
-  void unlock() {
-    LOGMAN_THROW_A_FMT(Futex.load() == UNIQUE_LOCK_VALUE, "Tried unlocking not locked mutex?");
-
-    auto TryUniqueUnlock = [this]() -> std::pair<uint32_t, bool> {
-      auto LocalFutex = Futex.load();
-
-      if (LocalFutex != UNIQUE_LOCK_VALUE) {
-        // Refcount must be zero if we are to attempt getting a unique lock
-      } else {
-        // Try locking now in userspace
-        while (LocalFutex == UNIQUE_LOCK_VALUE) {
-          auto Desired = LocalFutex;
-          Desired = 0;
-          if (Futex.compare_exchange_strong(LocalFutex, Desired)) {
-            // We have successfully unique locked
-            return std::make_pair(Desired, true);
-          } else {
-            if (LocalFutex == UNIQUE_LOCK_VALUE) {
-              // If another thread pulled the unique lock or the ref count incremented
-              // Then we need to wait, loop will end
-            }
-          }
-        }
-      }
-
-      return std::make_pair(LocalFutex, false);
-    };
-
-    [[maybe_unused]] auto UniqueResult = TryUniqueUnlock();
-    LOGMAN_THROW_A_FMT(UniqueResult.second, "Couldn't unlock mutex memory?");
-
-    // We've now unlocked, use the futex to wake up any shared waiters
-    int Op = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
-    ::syscall(SYS_futex, &Futex, Op,
-              INT_MAX, // Could be any number of shared waiters
-              nullptr, // timeout
-              nullptr, // addr
-              0);
-  }
-
-  bool try_lock_shared() {
-    return TryRefIncrement();
-  }
-
-  void lock_shared() {
-    if (TryRefIncrement()) {
-      return;
-    }
-
-    // Unique lock was held. Wait until it is no longer held using a system futex
-    int Op = FUTEX_WAIT | FUTEX_PRIVATE_FLAG;
-
-    auto Expected = UNIQUE_LOCK_VALUE;
-    do {
-      ::syscall(SYS_futex, &Futex, Op,
-                Expected, // Value
-                nullptr,  // Timeout,
-                nullptr,  // Addr
-                0);
-
-      Expected = Futex.load();
-      // If Res == 0 then check the unique lock to see if unique is no longer owned
-      if (Expected != UNIQUE_LOCK_VALUE) {
-        if (TryRefIncrement()) {
-          // Ref count succeeded
-          return;
-        }
-      }
-    } while (true);
-  }
-
-  // Returns the number of ref counts remaining once this leaves
-  uint32_t unlock_shared() {
-    auto DecrementResult = TryRefDecrement();
-
-    if (DecrementResult.second) {
-      if (DecrementResult.first == 0) {
-        // If we were the last shared value out then we need to do a futex to wake up any waiters
-        int Op = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
-        ::syscall(SYS_futex, &Futex, Op,
-                  1,       // Wake up only one thread if one is waiting. Which would be the unique waiter
-                  nullptr, // timeout
-                  nullptr, // addr
-                  0);
-      }
-      return DecrementResult.first;
-    }
-
-    LOGMAN_MSG_A_FMT("Managed to squeeze a unique lock between shared locks?");
-    return 0; // Error
-  }
-
-  // Get the raw futex ref count number
-  uint32_t GetNumRefCounts() const {
-    return Futex.load();
-  }
-
-  // Be careful with this. Only use when you know the mutex is dead
-  void Reset() {
-    Futex.store(0);
-
-    int Op = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
-    ::syscall(SYS_futex, &Futex, Op,
-              INT_MAX, // Wake up all threads if any waiting
-              nullptr, // timeout
-              nullptr, // addr
-              0);
-  }
-
-private:
-  bool TryRefIncrement() {
-    auto LocalFutex = Futex.load();
-
-    if (LocalFutex == UNIQUE_LOCK_VALUE) {
-      // Unique lock held
-    } else {
-      // Try to increment the counter if unique lock isn't held
-      while (LocalFutex != UNIQUE_LOCK_VALUE) {
-        auto Desired = LocalFutex;
-        Desired++;
-
-        // Try to increment the ref count
-        if (Futex.compare_exchange_strong(LocalFutex, Desired)) {
-          // We have successfully incremented the ref counting mutex in userspace
-          return true;
-        } else {
-          if (LocalFutex == UNIQUE_LOCK_VALUE) {
-            // Unique lock was held
-            // Nothing to do, loop will end
-          }
-
-          // Try again. Can happen in a race to increment the ref count
-        }
-      }
-    }
-
-    return false;
-  };
-
-  std::pair<uint32_t, bool> TryRefDecrement() {
-    auto LocalFutex = Futex.load();
-
-    if (LocalFutex == UNIQUE_LOCK_VALUE) {
-      // Unique lock held
-    } else {
-      // Try to increment the counter if unique lock isn't held
-      while (LocalFutex != UNIQUE_LOCK_VALUE) {
-        auto Desired = LocalFutex;
-        Desired--;
-
-        // Try to increment the ref count
-        if (Futex.compare_exchange_strong(LocalFutex, Desired)) {
-          // We have successfully incremented the ref counting mutex in userspace
-          return std::make_pair(Desired, true);
-        } else {
-          if (LocalFutex == UNIQUE_LOCK_VALUE) {
-            // Unique lock was held
-            // Nothing to do, loop will end
-          }
-
-          // Try again. Can happen in a race to increment the ref count
-        }
-      }
-    }
-
-    return std::make_pair(LocalFutex, false);
-  };
-
-  std::pair<uint32_t, bool> TryUniqueLock() {
-    auto LocalFutex = Futex.load();
-
-    if (LocalFutex) {
-      // Refcount must be zero if we are to attempt getting a unique lock
-    } else {
-      // Try locking now in userspace
-      while (LocalFutex == 0) {
-        auto Desired = LocalFutex;
-        Desired = UNIQUE_LOCK_VALUE;
-        if (Futex.compare_exchange_strong(LocalFutex, Desired)) {
-          // We have successfully unique locked
-          return std::make_pair(Desired, true);
-        } else {
-          if (LocalFutex == 0) {
-            // If another thread pulled the unique lock or the ref count incremented
-            // Then we need to wait, loop will end
-          }
-        }
-      }
-    }
-
-    return std::make_pair(LocalFutex, false);
-  };
-
-  constexpr static uint32_t UNIQUE_LOCK_VALUE = -4096U;
-  // -1 = unique_lock
-  // 0 = no shared
-  // >0 = shared waiters
-  std::atomic<uint32_t> Futex {};
-};
-} // namespace FEXCore::Utils

FEXCore/unittests/Emitter/ASIMD_Tests.cpp

@@ -70,109 +70,61 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ASIMD: Advanced SIMD table lookup")
 }
 TEST_CASE_METHOD(TestDisassembler, "Emitter: ASIMD: Advanced SIMD permute") {
   // Commented out lines showcase unallocated encodings.
-  TEST_SINGLE(uzp1<SubRegSize::i8Bit>(QReg::q30, QReg::q29, QReg::q28), "uzp1 v30.16b, v29.16b, v28.16b");
-  TEST_SINGLE(uzp1<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "uzp1 v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(uzp1<SubRegSize::i32Bit>(QReg::q30, QReg::q29, QReg::q28), "uzp1 v30.4s, v29.4s, v28.4s");
-  TEST_SINGLE(uzp1<SubRegSize::i64Bit>(QReg::q30, QReg::q29, QReg::q28), "uzp1 v30.2d, v29.2d, v28.2d");
   TEST_SINGLE(uzp1(SubRegSize::i8Bit, QReg::q30, QReg::q29, QReg::q28), "uzp1 v30.16b, v29.16b, v28.16b");
   TEST_SINGLE(uzp1(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "uzp1 v30.8h, v29.8h, v28.8h");
   TEST_SINGLE(uzp1(SubRegSize::i32Bit, QReg::q30, QReg::q29, QReg::q28), "uzp1 v30.4s, v29.4s, v28.4s");
   TEST_SINGLE(uzp1(SubRegSize::i64Bit, QReg::q30, QReg::q29, QReg::q28), "uzp1 v30.2d, v29.2d, v28.2d");
 
-  TEST_SINGLE(uzp1<SubRegSize::i8Bit>(DReg::d30, DReg::d29, DReg::d28), "uzp1 v30.8b, v29.8b, v28.8b");
-  TEST_SINGLE(uzp1<SubRegSize::i16Bit>(DReg::d30, DReg::d29, DReg::d28), "uzp1 v30.4h, v29.4h, v28.4h");
-  TEST_SINGLE(uzp1<SubRegSize::i32Bit>(DReg::d30, DReg::d29, DReg::d28), "uzp1 v30.2s, v29.2s, v28.2s");
-  // TEST_SINGLE(uzp1<SubRegSize::i64Bit>(DReg::d30, DReg::d29, DReg::d28), "uzp1 v30.1d, v29.1d, v28.1d");
   TEST_SINGLE(uzp1(SubRegSize::i8Bit, DReg::d30, DReg::d29, DReg::d28), "uzp1 v30.8b, v29.8b, v28.8b");
   TEST_SINGLE(uzp1(SubRegSize::i16Bit, DReg::d30, DReg::d29, DReg::d28), "uzp1 v30.4h, v29.4h, v28.4h");
   TEST_SINGLE(uzp1(SubRegSize::i32Bit, DReg::d30, DReg::d29, DReg::d28), "uzp1 v30.2s, v29.2s, v28.2s");
   // TEST_SINGLE(uzp1(SubRegSize::i64Bit, DReg::d30, DReg::d29, DReg::d28), "uzp1 v30.1d, v29.1d, v28.1d");
 
-  TEST_SINGLE(trn1<SubRegSize::i8Bit>(QReg::q30, QReg::q29, QReg::q28), "trn1 v30.16b, v29.16b, v28.16b");
-  TEST_SINGLE(trn1<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "trn1 v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(trn1<SubRegSize::i32Bit>(QReg::q30, QReg::q29, QReg::q28), "trn1 v30.4s, v29.4s, v28.4s");
-  TEST_SINGLE(trn1<SubRegSize::i64Bit>(QReg::q30, QReg::q29, QReg::q28), "trn1 v30.2d, v29.2d, v28.2d");
   TEST_SINGLE(trn1(SubRegSize::i8Bit, QReg::q30, QReg::q29, QReg::q28), "trn1 v30.16b, v29.16b, v28.16b");
   TEST_SINGLE(trn1(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "trn1 v30.8h, v29.8h, v28.8h");
   TEST_SINGLE(trn1(SubRegSize::i32Bit, QReg::q30, QReg::q29, QReg::q28), "trn1 v30.4s, v29.4s, v28.4s");
   TEST_SINGLE(trn1(SubRegSize::i64Bit, QReg::q30, QReg::q29, QReg::q28), "trn1 v30.2d, v29.2d, v28.2d");
 
-  TEST_SINGLE(trn1<SubRegSize::i8Bit>(DReg::d30, DReg::d29, DReg::d28), "trn1 v30.8b, v29.8b, v28.8b");
-  TEST_SINGLE(trn1<SubRegSize::i16Bit>(DReg::d30, DReg::d29, DReg::d28), "trn1 v30.4h, v29.4h, v28.4h");
-  TEST_SINGLE(trn1<SubRegSize::i32Bit>(DReg::d30, DReg::d29, DReg::d28), "trn1 v30.2s, v29.2s, v28.2s");
-  // TEST_SINGLE(trn1<SubRegSize::i64Bit>(DReg::d30, DReg::d29, DReg::d28), "trn1 v30.1d, v29.1d, v28.1d");
   TEST_SINGLE(trn1(SubRegSize::i8Bit, DReg::d30, DReg::d29, DReg::d28), "trn1 v30.8b, v29.8b, v28.8b");
   TEST_SINGLE(trn1(SubRegSize::i16Bit, DReg::d30, DReg::d29, DReg::d28), "trn1 v30.4h, v29.4h, v28.4h");
   TEST_SINGLE(trn1(SubRegSize::i32Bit, DReg::d30, DReg::d29, DReg::d28), "trn1 v30.2s, v29.2s, v28.2s");
   // TEST_SINGLE(trn1(SubRegSize::i64Bit, DReg::d30, DReg::d29, DReg::d28), "trn1 v30.1d, v29.1d, v28.1d");
 
-  TEST_SINGLE(zip1<SubRegSize::i8Bit>(QReg::q30, QReg::q29, QReg::q28), "zip1 v30.16b, v29.16b, v28.16b");
-  TEST_SINGLE(zip1<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "zip1 v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(zip1<SubRegSize::i32Bit>(QReg::q30, QReg::q29, QReg::q28), "zip1 v30.4s, v29.4s, v28.4s");
-  TEST_SINGLE(zip1<SubRegSize::i64Bit>(QReg::q30, QReg::q29, QReg::q28), "zip1 v30.2d, v29.2d, v28.2d");
   TEST_SINGLE(zip1(SubRegSize::i8Bit, QReg::q30, QReg::q29, QReg::q28), "zip1 v30.16b, v29.16b, v28.16b");
   TEST_SINGLE(zip1(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "zip1 v30.8h, v29.8h, v28.8h");
   TEST_SINGLE(zip1(SubRegSize::i32Bit, QReg::q30, QReg::q29, QReg::q28), "zip1 v30.4s, v29.4s, v28.4s");
   TEST_SINGLE(zip1(SubRegSize::i64Bit, QReg::q30, QReg::q29, QReg::q28), "zip1 v30.2d, v29.2d, v28.2d");
 
-  TEST_SINGLE(zip1<SubRegSize::i8Bit>(DReg::d30, DReg::d29, DReg::d28), "zip1 v30.8b, v29.8b, v28.8b");
-  TEST_SINGLE(zip1<SubRegSize::i16Bit>(DReg::d30, DReg::d29, DReg::d28), "zip1 v30.4h, v29.4h, v28.4h");
-  TEST_SINGLE(zip1<SubRegSize::i32Bit>(DReg::d30, DReg::d29, DReg::d28), "zip1 v30.2s, v29.2s, v28.2s");
-  // TEST_SINGLE(zip1<SubRegSize::i64Bit>(DReg::d30, DReg::d29, DReg::d28), "zip1 v30.1d, v29.1d, v28.1d");
   TEST_SINGLE(zip1(SubRegSize::i8Bit, DReg::d30, DReg::d29, DReg::d28), "zip1 v30.8b, v29.8b, v28.8b");
   TEST_SINGLE(zip1(SubRegSize::i16Bit, DReg::d30, DReg::d29, DReg::d28), "zip1 v30.4h, v29.4h, v28.4h");
   TEST_SINGLE(zip1(SubRegSize::i32Bit, DReg::d30, DReg::d29, DReg::d28), "zip1 v30.2s, v29.2s, v28.2s");
   // TEST_SINGLE(zip1(SubRegSize::i64Bit, DReg::d30, DReg::d29, DReg::d28), "zip1 v30.1d, v29.1d, v28.1d");
 
-  TEST_SINGLE(uzp2<SubRegSize::i8Bit>(QReg::q30, QReg::q29, QReg::q28), "uzp2 v30.16b, v29.16b, v28.16b");
-  TEST_SINGLE(uzp2<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "uzp2 v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(uzp2<SubRegSize::i32Bit>(QReg::q30, QReg::q29, QReg::q28), "uzp2 v30.4s, v29.4s, v28.4s");
-  TEST_SINGLE(uzp2<SubRegSize::i64Bit>(QReg::q30, QReg::q29, QReg::q28), "uzp2 v30.2d, v29.2d, v28.2d");
   TEST_SINGLE(uzp2(SubRegSize::i8Bit, QReg::q30, QReg::q29, QReg::q28), "uzp2 v30.16b, v29.16b, v28.16b");
   TEST_SINGLE(uzp2(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "uzp2 v30.8h, v29.8h, v28.8h");
   TEST_SINGLE(uzp2(SubRegSize::i32Bit, QReg::q30, QReg::q29, QReg::q28), "uzp2 v30.4s, v29.4s, v28.4s");
   TEST_SINGLE(uzp2(SubRegSize::i64Bit, QReg::q30, QReg::q29, QReg::q28), "uzp2 v30.2d, v29.2d, v28.2d");
 
-  TEST_SINGLE(uzp2<SubRegSize::i8Bit>(DReg::d30, DReg::d29, DReg::d28), "uzp2 v30.8b, v29.8b, v28.8b");
-  TEST_SINGLE(uzp2<SubRegSize::i16Bit>(DReg::d30, DReg::d29, DReg::d28), "uzp2 v30.4h, v29.4h, v28.4h");
-  TEST_SINGLE(uzp2<SubRegSize::i32Bit>(DReg::d30, DReg::d29, DReg::d28), "uzp2 v30.2s, v29.2s, v28.2s");
-  // TEST_SINGLE(uzp2<SubRegSize::i64Bit>(DReg::d30, DReg::d29, DReg::d28), "uzp2 v30.1d, v29.1d, v28.1d");
   TEST_SINGLE(uzp2(SubRegSize::i8Bit, DReg::d30, DReg::d29, DReg::d28), "uzp2 v30.8b, v29.8b, v28.8b");
   TEST_SINGLE(uzp2(SubRegSize::i16Bit, DReg::d30, DReg::d29, DReg::d28), "uzp2 v30.4h, v29.4h, v28.4h");
   TEST_SINGLE(uzp2(SubRegSize::i32Bit, DReg::d30, DReg::d29, DReg::d28), "uzp2 v30.2s, v29.2s, v28.2s");
   // TEST_SINGLE(uzp2(SubRegSize::i64Bit, DReg::d30, DReg::d29, DReg::d28), "uzp2 v30.1d, v29.1d, v28.1d");
 
-  TEST_SINGLE(trn2<SubRegSize::i8Bit>(QReg::q30, QReg::q29, QReg::q28), "trn2 v30.16b, v29.16b, v28.16b");
-  TEST_SINGLE(trn2<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "trn2 v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(trn2<SubRegSize::i32Bit>(QReg::q30, QReg::q29, QReg::q28), "trn2 v30.4s, v29.4s, v28.4s");
-  TEST_SINGLE(trn2<SubRegSize::i64Bit>(QReg::q30, QReg::q29, QReg::q28), "trn2 v30.2d, v29.2d, v28.2d");
   TEST_SINGLE(trn2(SubRegSize::i8Bit, QReg::q30, QReg::q29, QReg::q28), "trn2 v30.16b, v29.16b, v28.16b");
   TEST_SINGLE(trn2(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "trn2 v30.8h, v29.8h, v28.8h");
   TEST_SINGLE(trn2(SubRegSize::i32Bit, QReg::q30, QReg::q29, QReg::q28), "trn2 v30.4s, v29.4s, v28.4s");
   TEST_SINGLE(trn2(SubRegSize::i64Bit, QReg::q30, QReg::q29, QReg::q28), "trn2 v30.2d, v29.2d, v28.2d");
 
-  TEST_SINGLE(trn2<SubRegSize::i8Bit>(DReg::d30, DReg::d29, DReg::d28), "trn2 v30.8b, v29.8b, v28.8b");
-  TEST_SINGLE(trn2<SubRegSize::i16Bit>(DReg::d30, DReg::d29, DReg::d28), "trn2 v30.4h, v29.4h, v28.4h");
-  TEST_SINGLE(trn2<SubRegSize::i32Bit>(DReg::d30, DReg::d29, DReg::d28), "trn2 v30.2s, v29.2s, v28.2s");
-  // TEST_SINGLE(trn2<SubRegSize::i64Bit>(DReg::d30, DReg::d29, DReg::d28), "trn2 v30.1d, v29.1d, v28.1d");
   TEST_SINGLE(trn2(SubRegSize::i8Bit, DReg::d30, DReg::d29, DReg::d28), "trn2 v30.8b, v29.8b, v28.8b");
   TEST_SINGLE(trn2(SubRegSize::i16Bit, DReg::d30, DReg::d29, DReg::d28), "trn2 v30.4h, v29.4h, v28.4h");
   TEST_SINGLE(trn2(SubRegSize::i32Bit, DReg::d30, DReg::d29, DReg::d28), "trn2 v30.2s, v29.2s, v28.2s");
   // TEST_SINGLE(trn2(SubRegSize::i64Bit, DReg::d30, DReg::d29, DReg::d28), "trn2 v30.1d, v29.1d, v28.1d");
 
-  TEST_SINGLE(zip2<SubRegSize::i8Bit>(QReg::q30, QReg::q29, QReg::q28), "zip2 v30.16b, v29.16b, v28.16b");
-  TEST_SINGLE(zip2<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "zip2 v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(zip2<SubRegSize::i32Bit>(QReg::q30, QReg::q29, QReg::q28), "zip2 v30.4s, v29.4s, v28.4s");
-  TEST_SINGLE(zip2<SubRegSize::i64Bit>(QReg::q30, QReg::q29, QReg::q28), "zip2 v30.2d, v29.2d, v28.2d");
   TEST_SINGLE(zip2(SubRegSize::i8Bit, QReg::q30, QReg::q29, QReg::q28), "zip2 v30.16b, v29.16b, v28.16b");
   TEST_SINGLE(zip2(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "zip2 v30.8h, v29.8h, v28.8h");
   TEST_SINGLE(zip2(SubRegSize::i32Bit, QReg::q30, QReg::q29, QReg::q28), "zip2 v30.4s, v29.4s, v28.4s");
   TEST_SINGLE(zip2(SubRegSize::i64Bit, QReg::q30, QReg::q29, QReg::q28), "zip2 v30.2d, v29.2d, v28.2d");
 
-  TEST_SINGLE(zip2<SubRegSize::i8Bit>(DReg::d30, DReg::d29, DReg::d28), "zip2 v30.8b, v29.8b, v28.8b");
-  TEST_SINGLE(zip2<SubRegSize::i16Bit>(DReg::d30, DReg::d29, DReg::d28), "zip2 v30.4h, v29.4h, v28.4h");
-  TEST_SINGLE(zip2<SubRegSize::i32Bit>(DReg::d30, DReg::d29, DReg::d28), "zip2 v30.2s, v29.2s, v28.2s");
-  // TEST_SINGLE(zip2<SubRegSize::i64Bit>(DReg::d30, DReg::d29, DReg::d28), "zip2 v30.1d, v29.1d, v28.1d");
   TEST_SINGLE(zip2(SubRegSize::i8Bit, DReg::d30, DReg::d29, DReg::d28), "zip2 v30.8b, v29.8b, v28.8b");
   TEST_SINGLE(zip2(SubRegSize::i16Bit, DReg::d30, DReg::d29, DReg::d28), "zip2 v30.4h, v29.4h, v28.4h");
   TEST_SINGLE(zip2(SubRegSize::i32Bit, DReg::d30, DReg::d29, DReg::d28), "zip2 v30.2s, v29.2s, v28.2s");
@@ -234,15 +186,6 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ASIMD: Advanced SIMD copy") {
   TEST_SINGLE(umov<SubRegSize::i64Bit>(Reg::r29, VReg::v30, 0), "mov x29, v30.d[0]");
   TEST_SINGLE(umov<SubRegSize::i64Bit>(Reg::r29, VReg::v30, 1), "mov x29, v30.d[1]");
 
-  TEST_SINGLE(ins<SubRegSize::i8Bit>(VReg::v30, 0, Reg::r29), "mov v30.b[0], w29");
-  TEST_SINGLE(ins<SubRegSize::i16Bit>(VReg::v30, 0, Reg::r29), "mov v30.h[0], w29");
-  TEST_SINGLE(ins<SubRegSize::i32Bit>(VReg::v30, 0, Reg::r29), "mov v30.s[0], w29");
-  TEST_SINGLE(ins<SubRegSize::i64Bit>(VReg::v30, 0, Reg::r29), "mov v30.d[0], x29");
-  TEST_SINGLE(ins<SubRegSize::i8Bit>(VReg::v30, 15, Reg::r29), "mov v30.b[15], w29");
-  TEST_SINGLE(ins<SubRegSize::i16Bit>(VReg::v30, 7, Reg::r29), "mov v30.h[7], w29");
-  TEST_SINGLE(ins<SubRegSize::i32Bit>(VReg::v30, 3, Reg::r29), "mov v30.s[3], w29");
-  TEST_SINGLE(ins<SubRegSize::i64Bit>(VReg::v30, 1, Reg::r29), "mov v30.d[1], x29");
-
   TEST_SINGLE(ins(SubRegSize::i8Bit, VReg::v30, 0, Reg::r29), "mov v30.b[0], w29");
   TEST_SINGLE(ins(SubRegSize::i16Bit, VReg::v30, 0, Reg::r29), "mov v30.h[0], w29");
   TEST_SINGLE(ins(SubRegSize::i32Bit, VReg::v30, 0, Reg::r29), "mov v30.s[0], w29");
@@ -262,61 +205,61 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ASIMD: Advanced SIMD copy") {
   TEST_SINGLE(ins(SubRegSize::i64Bit, VReg::v30, 1, VReg::v29, 0), "mov v30.d[1], v29.d[0]");
 }
 TEST_CASE_METHOD(TestDisassembler, "Emitter: ASIMD: Advanced SIMD three same (FP16)") {
-  TEST_SINGLE(fmaxnm<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fmaxnm v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fmla<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fmla v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fadd<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fadd v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fmulx<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fmulx v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fcmeq<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fcmeq v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fmax<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fmax v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(frecps<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "frecps v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fminnm<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fminnm v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fmls<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fmls v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fsub<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fsub v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fmin<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fmin v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(frsqrts<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "frsqrts v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fmaxnmp<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fmaxnmp v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(faddp<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "faddp v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fmul<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fmul v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fcmge<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fcmge v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(facge<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "facge v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fmaxp<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fmaxp v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fdiv<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fdiv v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fminnmp<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fminnmp v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fabd<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fabd v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fcmgt<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fcmgt v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(facgt<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "facgt v30.8h, v29.8h, v28.8h");
-  TEST_SINGLE(fminp<SubRegSize::i16Bit>(QReg::q30, QReg::q29, QReg::q28), "fminp v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fmaxnm(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fmaxnm v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fmla(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fmla v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fadd(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fadd v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fmulx(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fmulx v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fcmeq(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fcmeq v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fmax(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fmax v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(frecps(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "frecps v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fminnm(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fminnm v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fmls(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fmls v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fsub(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fsub v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fmin(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fmin v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(frsqrts(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "frsqrts v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fmaxnmp(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fmaxnmp v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(faddp(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "faddp v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fmul(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fmul v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fcmge(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fcmge v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(facge(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "facge v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fmaxp(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fmaxp v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fdiv(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fdiv v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fminnmp(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fminnmp v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fabd(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fabd v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fcmgt(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fcmgt v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(facgt(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "facgt v30.8h, v29.8h, v28.8h");
+  TEST_SINGLE(fminp(SubRegSize::i16Bit, QReg::q30, QReg::q29, QReg::q28), "fminp v30.8h, v29.8h, v28.8h");
 }
 TEST_CASE_METHOD(TestDisassembler, "Emitter: ASIMD: Advanced SIMD two-register miscellaneous (FP16)") {
-  TEST_SINGLE(frintn<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "frintn v30.8h, v29.8h");
-  TEST_SINGLE(frintm<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "frintm v30.8h, v29.8h");
-  TEST_SINGLE(fcvtns<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcvtns v30.8h, v29.8h");
-  TEST_SINGLE(fcvtms<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcvtms v30.8h, v29.8h");
-  TEST_SINGLE(fcvtas<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcvtas v30.8h, v29.8h");
-  TEST_SINGLE(scvtf<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "scvtf v30.8h, v29.8h");
-  TEST_SINGLE(fcmgt<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcmgt v30.8h, v29.8h, #0.0");
-  TEST_SINGLE(fcmeq<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcmeq v30.8h, v29.8h, #0.0");
-  TEST_SINGLE(fcmlt<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcmlt v30.8h, v29.8h, #0.0");
-  TEST_SINGLE(fabs<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fabs v30.8h, v29.8h");
-  TEST_SINGLE(frintp<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "frintp v30.8h, v29.8h");
-  TEST_SINGLE(frintz<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "frintz v30.8h, v29.8h");
-  TEST_SINGLE(fcvtps<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcvtps v30.8h, v29.8h");
-  TEST_SINGLE(fcvtzs<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcvtzs v30.8h, v29.8h");
-  TEST_SINGLE(frecpe<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "frecpe v30.8h, v29.8h");
-  TEST_SINGLE(frinta<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "frinta v30.8h, v29.8h");
-  TEST_SINGLE(frintx<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "frintx v30.8h, v29.8h");
-  TEST_SINGLE(fcvtnu<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcvtnu v30.8h, v29.8h");
-  TEST_SINGLE(fcvtmu<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcvtmu v30.8h, v29.8h");
-  TEST_SINGLE(fcvtau<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcvtau v30.8h, v29.8h");
-  TEST_SINGLE(ucvtf<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "ucvtf v30.8h, v29.8h");
-  TEST_SINGLE(fcmge<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcmge v30.8h, v29.8h, #0.0");
-  TEST_SINGLE(fcmle<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcmle v30.8h, v29.8h, #0.0");
-  TEST_SINGLE(fneg<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fneg v30.8h, v29.8h");
-  TEST_SINGLE(frinti<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "frinti v30.8h, v29.8h");
-  TEST_SINGLE(fcvtpu<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcvtpu v30.8h, v29.8h");
-  TEST_SINGLE(fcvtzu<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fcvtzu v30.8h, v29.8h");
-  TEST_SINGLE(frsqrte<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "frsqrte v30.8h, v29.8h");
-  TEST_SINGLE(fsqrt<SubRegSize::i16Bit>(QReg::q30, QReg::q29), "fsqrt v30.8h, v29.8h");
+  TEST_SINGLE(frintn(SubRegSize::i16Bit, QReg::q30, QReg::q29), "frintn v30.8h, v29.8h");
+  TEST_SINGLE(frintm(SubRegSize::i16Bit, QReg::q30, QReg::q29), "frintm v30.8h, v29.8h");
+  TEST_SINGLE(fcvtns(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcvtns v30.8h, v29.8h");
+  TEST_SINGLE(fcvtms(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcvtms v30.8h, v29.8h");
+  TEST_SINGLE(fcvtas(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcvtas v30.8h, v29.8h");
+  TEST_SINGLE(scvtf(SubRegSize::i16Bit, QReg::q30, QReg::q29), "scvtf v30.8h, v29.8h");
+  TEST_SINGLE(fcmgt(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcmgt v30.8h, v29.8h, #0.0");
+  TEST_SINGLE(fcmeq(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcmeq v30.8h, v29.8h, #0.0");
+  TEST_SINGLE(fcmlt(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcmlt v30.8h, v29.8h, #0.0");
+  TEST_SINGLE(fabs(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fabs v30.8h, v29.8h");
+  TEST_SINGLE(frintp(SubRegSize::i16Bit, QReg::q30, QReg::q29), "frintp v30.8h, v29.8h");
+  TEST_SINGLE(frintz(SubRegSize::i16Bit, QReg::q30, QReg::q29), "frintz v30.8h, v29.8h");
+  TEST_SINGLE(fcvtps(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcvtps v30.8h, v29.8h");
+  TEST_SINGLE(fcvtzs(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcvtzs v30.8h, v29.8h");
+  TEST_SINGLE(frecpe(SubRegSize::i16Bit, QReg::q30, QReg::q29), "frecpe v30.8h, v29.8h");
+  TEST_SINGLE(frinta(SubRegSize::i16Bit, QReg::q30, QReg::q29), "frinta v30.8h, v29.8h");
+  TEST_SINGLE(frintx(SubRegSize::i16Bit, QReg::q30, QReg::q29), "frintx v30.8h, v29.8h");
+  TEST_SINGLE(fcvtnu(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcvtnu v30.8h, v29.8h");
+  TEST_SINGLE(fcvtmu(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcvtmu v30.8h, v29.8h");
+  TEST_SINGLE(fcvtau(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcvtau v30.8h, v29.8h");
+  TEST_SINGLE(ucvtf(SubRegSize::i16Bit, QReg::q30, QReg::q29), "ucvtf v30.8h, v29.8h");
+  TEST_SINGLE(fcmge(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcmge v30.8h, v29.8h, #0.0");
+  TEST_SINGLE(fcmle(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcmle v30.8h, v29.8h, #0.0");
+  TEST_SINGLE(fneg(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fneg v30.8h, v29.8h");
+  TEST_SINGLE(frinti(SubRegSize::i16Bit, QReg::q30, QReg::q29), "frinti v30.8h, v29.8h");
+  TEST_SINGLE(fcvtpu(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcvtpu v30.8h, v29.8h");
+  TEST_SINGLE(fcvtzu(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fcvtzu v30.8h, v29.8h");
+  TEST_SINGLE(frsqrte(SubRegSize::i16Bit, QReg::q30, QReg::q29), "frsqrte v30.8h, v29.8h");
+  TEST_SINGLE(fsqrt(SubRegSize::i16Bit, QReg::q30, QReg::q29), "fsqrt v30.8h, v29.8h");
 }
 TEST_CASE_METHOD(TestDisassembler, "Emitter: ASIMD: Advanced SIMD three-register extension") {
   TEST_SINGLE(sdot(SubRegSize::i32Bit, QReg::q30, QReg::q29, QReg::q28), "sdot v30.4s, v29.16b, v28.16b");

FEXHeaderUtils/FEXHeaderUtils/TodoDefines.h

@@ -1,22 +0,0 @@
-// SPDX-License-Identifier: MIT
-#pragma once
-
-#ifndef DO_PRAGMA
-#define DO_PRAGMA(x) _Pragma(#x)
-#endif
-
-#if FEX_WARN_TODO
-// FEX_TODO_ISSUE(github ticket number, "comment")
-#define FEX_TODO_ISSUE(github_ticket, comment) DO_PRAGMA(GCC warning "TODO: https://github.com/FEX-Emu/FEX/issues/" #github_ticket comment);
-// FEX_TODO("comment")
-#define FEX_TODO(comment) DO_PRAGMA(GCC warning "TODO: " comment);
-#else
-// FEX_TODO_ISSUE(github ticket number, "comment")
-#define FEX_TODO_ISSUE(github_ticket, comment)
-// FEX_TODO("comment")
-#define FEX_TODO(comment)
-#endif
-
-// For linking to tickets, non-todo
-// FEX_TICKET(github ticket number) or FEX_TICKET(github ticket number, "comment")
-#define FEX_TICKET(github_ticket, ...)