// SPDX-License-Identifier: MIT
#pragma once

#include "Interface/Core/Frontend.h"
#include "Interface/Core/X86Tables/X86Tables.h"
#include "Interface/Core/Addressing.h"
#include "Interface/Context/Context.h"
#include "Interface/IR/IR.h"
#include "Interface/IR/IREmitter.h"
#include "Interface/IR/RegisterAllocationData.h"

#include <FEXCore/Config/Config.h>
#include <FEXCore/Core/Context.h>
#include <FEXCore/Core/X86Enums.h>
#include <FEXCore/IR/IR.h>

#include <FEXCore/Utils/LogManager.h>
#include <FEXCore/Utils/MathUtils.h>
#include <FEXCore/fextl/map.h>
#include <FEXCore/fextl/vector.h>

#include <bit>
#include <cstdint>
#include <fmt/format.h>
#include <stddef.h>
#include <utility>
#include <xxhash.h>

namespace FEXCore::IR {
enum class VectorCompareType {
  // SSE comparisons.
  EQ_OQ = 0,
  LT_OS = 1,
  LE_OS = 2,
  UNORD_Q = 3,
  NEQ_UQ = 4,
  NLT_US = 5,
  NLE_US = 6,
  ORD_Q = 7,
  // AVX-only comparisons.
  EQ_UQ = 8,
  NGE_US = 9,
  NGT_US = 10,
  FALSE_OQ = 11,
  NEQ_OQ = 12,
  GE_OS = 13,
  GT_OS = 14,
  TRUE_UQ = 15,
  EQ_OS = 16,
  LT_OQ = 17,
  LE_OQ = 18,
  UNORD_S = 19,
  NEQ_US = 20,
  NLT_UQ = 21,
  NLE_UQ = 22,
  ORD_S = 23,
  EQ_US = 24,
  NGE_UQ = 25,
  NGT_UQ = 26,
  FALSE_OS = 27,
  NEQ_OS = 28,
  GE_OQ = 29,
  GT_OQ = 30,
  TRUE_US = 31,
};

enum class MemoryAccessType {
  // Choose TSO or Non-TSO depending on access type
  DEFAULT,
  // TSO access behaviour
  TSO,
  // Non-TSO access behaviour
  NONTSO,
  // Non-temporal streaming
  STREAM,
};

enum class BTAction {
  BTNone,
  BTClear,
  BTSet,
  BTComplement,
};

enum class ForceTSOMode {
  NoOverride,
  ForceDisabled,
  ForceEnabled,
};

struct LoadSourceOptions {
  // Alignment of the load in bytes. iInvalid signifies opsize aligned.
  IR::OpSize Align = OpSize::iInvalid;

  // Whether or not to load the data if a memory access occurs.
  // If set to false, then the address that would have been loaded from
  // will be returned instead.
  //
  // Note: If returning the address, make sure to apply the segment offset
  //       after with AppendSegmentOffset().
  //
  bool LoadData = true;

  // Use to force a load even if the underlying type isn't loadable.
  bool ForceLoad = false;

  // Specifies the access type of the load.
  MemoryAccessType AccessType = MemoryAccessType::DEFAULT;

  // Whether or not a zero extend should clear the upper bits
  // in the register (e.g. an 8-bit load would clear the upper 24 bits
  // or 56 bits depending on the operating mode).
  // If true, no zero-extension occurs.
  bool AllowUpperGarbage = false;
};

struct DispatchTableEntry {
  uint16_t Op;
  uint8_t Count;
  X86Tables::OpDispatchPtr Ptr;
};

class OpDispatchBuilder final : public IREmitter {
public:
  Ref GetNewJumpBlock(uint64_t RIP) {
    auto it = JumpTargets.find(RIP);
    LOGMAN_THROW_A_FMT(it != JumpTargets.end(), "Couldn't find block generated for 0x{:x}", RIP);
    return it->second.BlockEntry;
  }

  void SetNewBlockIfChanged(uint64_t RIP) {
    auto it = JumpTargets.find(RIP);
    if (it == JumpTargets.end()) {
      return;
    }

    it->second.HaveEmitted = true;

    if (CurrentCodeBlock->Wrapped(DualListData.ListBegin()).ID() == it->second.BlockEntry->Wrapped(DualListData.ListBegin()).ID()) {
      return;
    }

    // We have hit a RIP that is a jump target
    // Thus we need to end up in a new block
    SetCurrentCodeBlock(it->second.BlockEntry);
  }

  void StartNewBlock() {
    // If we loaded flags but didn't change them, invalidate the cached copy and move on.
    // Changes get stored out by CalculateDeferredFlags.
    CachedNZCV = nullptr;
    CFInverted = CFInvertedABI;

    FlushRegisterCache();

    // Start block in X87 state.
    // This is important to ensure that blocks always start with the same state independently of predecessors
    // which allows independent compilation of blocks.
    // Starting in the X87 state is better than starting in MMX state because
    // MMX state is more work to initialize.
    MMXState = MMXState_X87;

    // New block needs to reset segment telemetry.
    SegmentsNeedReadCheck = ~0U;

    // Need to clear any named constants that were cached.
    ClearCachedNamedConstants();
  }

  IRPair<IROp_Jump> Jump() {
    FlushRegisterCache();
    return _Jump();
  }
  IRPair<IROp_Jump> Jump(Ref _TargetBlock) {
    FlushRegisterCache();
    return _Jump(_TargetBlock);
  }
  IRPair<IROp_CondJump> CondJump(Ref _Cmp1, Ref _Cmp2, Ref _TrueBlock, Ref _FalseBlock, CondClass _Cond = CondClass::NEQ,
                                 IR::OpSize _CompareSize = OpSize::iInvalid) {
    FlushRegisterCache();
    return _CondJump(_Cmp1, _Cmp2, _TrueBlock, _FalseBlock, _Cond, _CompareSize);
  }
  IRPair<IROp_CondJump> CondJump(Ref ssa0, CondClass cond = CondClass::NEQ) {
    FlushRegisterCache();
    return _CondJump(ssa0, cond);
  }
  IRPair<IROp_CondJump> CondJump(Ref ssa0, Ref ssa1, Ref ssa2, CondClass cond = CondClass::NEQ) {
    FlushRegisterCache();
    return _CondJump(ssa0, ssa1, ssa2, cond);
  }
  IRPair<IROp_CondJump> CondJumpNZCV(CondClass Cond) {
    FlushRegisterCache();
    return _CondJump(InvalidNode, InvalidNode, InvalidNode, InvalidNode, Cond, OpSize::iInvalid, true);
  }
  IRPair<IROp_CondJump> CondJumpBit(Ref Src, unsigned Bit, bool Set) {
    FlushRegisterCache();
    auto InlineConst = _InlineConstant(Bit);
    auto Cond = Set ? CondClass::TSTNZ : CondClass::TSTZ;
    return _CondJump(Src, InlineConst, InvalidNode, InvalidNode, Cond, OpSize::iInvalid, false);
  }
  IRPair<IROp_ExitFunction> ExitFunction(Ref NewRIP, BranchHint Hint = BranchHint::None) {
    FlushRegisterCache();
    return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, InvalidNode, InvalidNode);
  }
  IRPair<IROp_ExitFunction> ExitFunction(Ref NewRIP, BranchHint Hint, Ref CallReturnAddress, Ref CallReturnBlock) {
    FlushRegisterCache();
    return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, CallReturnAddress, CallReturnBlock);
  }
  IRPair<IROp_Break> Break(BreakDefinition Reason) {
    FlushRegisterCache();
    return _Break(Reason);
  }
  IRPair<IROp_Thunk> Thunk(Ref ArgPtr, SHA256Sum ThunkNameHash) {
    FlushRegisterCache();
    return _Thunk(ArgPtr, ThunkNameHash);
  }

  bool FinishOp(uint64_t NextRIP, bool LastOp) {
    // If we are switching to a new block and this current block has yet to set a RIP
    // Then we need to insert an unconditional jump from the current block to the one we are going to
    // This happens most frequently when an instruction jumps backwards to another location
    // eg:
    //
    //  nop dword [rax], eax
    // .label:
    //  rdi, 0x8
    //  cmp qword [rdi-8], 0
    //  jne .label
    if (LastOp && !BlockSetRIP) {
      auto it = JumpTargets.find(NextRIP);
      if (it == JumpTargets.end()) {

        const auto GPRSize = GetGPROpSize();
        // If we don't have a jump target to a new block then we have to leave
        // Set the RIP to the next instruction and leave
        ExitFunction(_InlineEntrypointOffset(GPRSize, NextRIP - Entry));
      } else if (it != JumpTargets.end()) {
        Jump(it->second.BlockEntry);
        return true;
      }
    }

    BlockSetRIP = false;

    return false;
  }

  static bool CanHaveSideEffects(const FEXCore::X86Tables::X86InstInfo* TableInfo, FEXCore::X86Tables::DecodedOp Op) {
    if (TableInfo) {
      if (TableInfo->Flags & X86Tables::InstFlags::FLAGS_DEBUG_MEM_ACCESS) {
        // If it is marked as having memory access then always say it has a side-effect.
        // Not always true but better to be safe.
        return true;
      }

      if (TableInfo->Flags & (X86Tables::InstFlags::FLAGS_SETS_RIP | X86Tables::InstFlags::FLAGS_BLOCK_END)) {
        // Cooperative suspend interrupts can be triggered at any back-edge, the RIP must be reconstructed correctly in such cases
        return true;
      }
    }

    auto CanHaveSideEffects = false;

    auto HasPotentialMemoryAccess = [](const X86Tables::DecodedOperand& Operand) -> bool {
      if (Operand.IsNone()) {
        return false;
      }

      // This isn't guaranteed that all of these types will access memory, but be safe.
      return Operand.IsGPRDirect() || Operand.IsGPRIndirect() || Operand.IsRIPRelative() || Operand.IsSIB();
    };

    CanHaveSideEffects |= HasPotentialMemoryAccess(Op->Dest);
    CanHaveSideEffects |= HasPotentialMemoryAccess(Op->Src[0]);
    CanHaveSideEffects |= HasPotentialMemoryAccess(Op->Src[1]);
    CanHaveSideEffects |= HasPotentialMemoryAccess(Op->Src[2]);
    return CanHaveSideEffects;
  }

  template<typename F>
  void ForeachDirection(F&& Routine) {
    // Otherwise, prepare to branch.
    auto Zero = Constant(0);

    // If the shift is zero, do not touch the flags.
    auto ForwardBlock = CreateNewCodeBlockAfter(GetCurrentBlock());
    auto BackwardBlock = CreateNewCodeBlockAfter(ForwardBlock);
    auto ExitBlock = CreateNewCodeBlockAfter(BackwardBlock);

    auto DF = GetRFLAG(X86State::RFLAG_DF_RAW_LOC);
    CondJump(DF, Zero, ForwardBlock, BackwardBlock, CondClass::EQ);

    for (auto D = 0; D < 2; ++D) {
      SetCurrentCodeBlock(D ? BackwardBlock : ForwardBlock);
      StartNewBlock();
      {
        Routine(D ? -1 : 1);
        Jump(ExitBlock);
      }
    }

    SetCurrentCodeBlock(ExitBlock);
    StartNewBlock();
  }

  OpDispatchBuilder(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::InternalThreadState* Thread);

  // Should only be called at the start of IR Emission.
  void ResetWorkingList();

  void ResetDecodeFailure() {
    NeedsBlockEnd = DecodeFailure = false;
  }
  bool HadDecodeFailure() const {
    return DecodeFailure;
  }
  bool NeedsBlockEnder() const {
    return NeedsBlockEnd;
  }

  void ResetHandledLock() {
    HandledLock = false;
  }
  bool HasHandledLock() const {
    return HandledLock;
  }

  void SetForceTSO(ForceTSOMode Mode) {
    ForceTSO = Mode;
  }
  ForceTSOMode GetForceTSO() const {
    return ForceTSO;
  }

  void SetDumpIR(bool DumpIR) {
    ShouldDump = DumpIR;
  }
  bool ShouldDumpIR() const {
    return ShouldDump;
  }

  void BeginFunction(uint64_t RIP, const fextl::vector<FEXCore::Frontend::Decoder::DecodedBlocks>* Blocks, uint32_t NumInstructions,
                     bool Is64BitMode, bool MonoBackpatcherBlock);
  void Finalize();

  // Dispatch builder functions
#define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op

  /**
   * Binds a sequence of compile-time constants as arguments to another member function.
   * This allows to construct OpDispatchPtrs that are specialized for the given set of arguments.
   */
  template<auto Fn, auto... Args>
  void Bind(OpcodeArgs) {
    [[clang::noinline]] (this->*Fn)(Op, Args...);
  };

  void UnhandledOp(OpcodeArgs);
  void MOVGPROp(OpcodeArgs, uint32_t SrcIndex);
  void MOVGPRNTOp(OpcodeArgs);
  void MOVVectorAlignedOp(OpcodeArgs);
  void MOVVectorUnalignedOp(OpcodeArgs);
  void MOVVectorNTOp(OpcodeArgs, bool IsAVX);
  void ALUOp(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCore::IR::IROps AtomicFetchOp, unsigned SrcIdx);
  void LSLOp(OpcodeArgs);
  void INTOp(OpcodeArgs);
  void SyscallOp(OpcodeArgs, bool IsSyscallInst);
  void ThunkOp(OpcodeArgs);
  void LEAOp(OpcodeArgs);
  void NOPOp(OpcodeArgs);
  void RETOp(OpcodeArgs);
  void IRETOp(OpcodeArgs);
  void CallbackReturnOp(OpcodeArgs);
  void SecondaryALUOp(OpcodeArgs);
  void ADCOp(OpcodeArgs, uint32_t SrcIndex);
  void SBBOp(OpcodeArgs, uint32_t SrcIndex);
  void SALCOp(OpcodeArgs);
  void PUSHOp(OpcodeArgs);
  void PUSHREGOp(OpcodeArgs);
  void PUSHAOp(OpcodeArgs);
  void PUSHSegmentOp(OpcodeArgs, uint32_t SegmentReg);
  void POPOp(OpcodeArgs);
  void POPAOp(OpcodeArgs);
  void POPSegmentOp(OpcodeArgs, uint32_t SegmentReg);
  void LEAVEOp(OpcodeArgs);
  void CALLOp(OpcodeArgs);
  void CALLAbsoluteOp(OpcodeArgs);
  void CondJUMPOp(OpcodeArgs);
  void CondJUMPRCXOp(OpcodeArgs);
  void LoopOp(OpcodeArgs);
  void JUMPOp(OpcodeArgs);
  void JUMPAbsoluteOp(OpcodeArgs);
  void JUMPFARIndirectOp(OpcodeArgs);
  void CALLFARIndirectOp(OpcodeArgs);
  void RETFARIndirectOp(OpcodeArgs);
  void TESTOp(OpcodeArgs, uint32_t SrcIndex);
  void ARPLOp(OpcodeArgs);
  void MOVSXDOp(OpcodeArgs);
  void MOVSXOp(OpcodeArgs);
  void MOVZXOp(OpcodeArgs);
  void CMPOp(OpcodeArgs, uint32_t SrcIndex);
  void SETccOp(OpcodeArgs);
  void CQOOp(OpcodeArgs);
  void CDQOp(OpcodeArgs);
  void XCHGOp(OpcodeArgs);
  void SAHFOp(OpcodeArgs);
  void LAHFOp(OpcodeArgs);
  void MOVSegOp(OpcodeArgs, bool ToSeg);
  void FLAGControlOp(OpcodeArgs);
  void MOVOffsetOp(OpcodeArgs);
  void CMOVOp(OpcodeArgs);
  void CPUIDOp(OpcodeArgs);
  void XGetBVOp(OpcodeArgs);
  uint32_t GetConstantShift(X86Tables::DecodedOp Op, bool Is1Bit);
  void SHLOp(OpcodeArgs);
  void SHLImmediateOp(OpcodeArgs, bool SHL1Bit);
  void SHROp(OpcodeArgs);
  void SHRImmediateOp(OpcodeArgs, bool SHR1Bit);
  void SHLDOp(OpcodeArgs);
  void SHLDImmediateOp(OpcodeArgs);
  void SHRDOp(OpcodeArgs);
  void SHRDImmediateOp(OpcodeArgs);
  void ASHROp(OpcodeArgs, bool IsImmediate, bool Is1Bit);
  void RotateOp(OpcodeArgs, bool Left, bool IsImmediate, bool Is1Bit);
  void RCROp1Bit(OpcodeArgs);
  void RCROp8x1Bit(OpcodeArgs);
  void RCROp(OpcodeArgs);
  void RCRSmallerOp(OpcodeArgs);
  void RCLOp1Bit(OpcodeArgs);
  void RCLOp(OpcodeArgs);
  void RCLSmallerOp(OpcodeArgs);

  void BTOp(OpcodeArgs, uint32_t SrcIndex, enum BTAction Action);

  void IMUL1SrcOp(OpcodeArgs);
  void IMUL2SrcOp(OpcodeArgs);
  void IMULOp(OpcodeArgs);
  void STOSOp(OpcodeArgs);
  void MOVSOp(OpcodeArgs);
  void CMPSOp(OpcodeArgs);
  void LODSOp(OpcodeArgs);
  void SCASOp(OpcodeArgs);
  void BSWAPOp(OpcodeArgs);
  void PUSHFOp(OpcodeArgs);
  void POPFOp(OpcodeArgs);

  struct CycleCounterPair {
    Ref CounterLow;
    Ref CounterHigh;
  };
  CycleCounterPair CycleCounter(bool SelfSynchronizingLoads);
  void RDTSCOp(OpcodeArgs);
  void INCOp(OpcodeArgs);
  void DECOp(OpcodeArgs);
  void NEGOp(OpcodeArgs);
  void DIVOp(OpcodeArgs);
  void IDIVOp(OpcodeArgs);
  void BSFOp(OpcodeArgs);
  void BSROp(OpcodeArgs);
  void CMPXCHGOp(OpcodeArgs);
  void CMPXCHGPairOp(OpcodeArgs);
  void MULOp(OpcodeArgs);
  void NOTOp(OpcodeArgs);
  void XADDOp(OpcodeArgs);
  void PopcountOp(OpcodeArgs);
  void DAAOp(OpcodeArgs);
  void DASOp(OpcodeArgs);
  void AAAOp(OpcodeArgs);
  void AASOp(OpcodeArgs);
  void AAMOp(OpcodeArgs);
  void AADOp(OpcodeArgs);
  void XLATOp(OpcodeArgs);
  void RDRANDOp(OpcodeArgs, bool Reseed);

  enum class Segment {
    FS,
    GS,
  };
  void ReadSegmentReg(OpcodeArgs, Segment Seg);
  void WriteSegmentReg(OpcodeArgs, Segment Seg);
  void EnterOp(OpcodeArgs);

  void SGDTOp(OpcodeArgs);
  void SIDTOp(OpcodeArgs);
  void SMSWOp(OpcodeArgs);

  enum class VectorOpType {
    MMX,
    SSE,
    AVX,
  };
  // SSE
  void MOVLPOp(OpcodeArgs);
  void MOVHPDOp(OpcodeArgs);
  void MOVSDOp(OpcodeArgs);
  void MOVSSOp(OpcodeArgs);
  void VectorALUOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
  void VectorXOROp(OpcodeArgs);

  void VectorALUROp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
  void VectorUnaryOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
  void RSqrt3DNowOp(OpcodeArgs, bool Duplicate);
  void VectorUnaryDuplicateOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);

  void MOVQOp(OpcodeArgs, VectorOpType VectorType);
  void MOVQMMXOp(OpcodeArgs);
  void MOVMSKOp(OpcodeArgs, IR::OpSize ElementSize);
  void MOVMSKOpOne(OpcodeArgs);
  void PUNPCKLOp(OpcodeArgs, IR::OpSize ElementSize);
  void PUNPCKHOp(OpcodeArgs, IR::OpSize ElementSize);
  void PSHUFBOp(OpcodeArgs);
  Ref PShufWLane(IR::OpSize Size, FEXCore::IR::IndexNamedVectorConstant IndexConstant, bool LowLane, Ref IncomingLane, uint8_t Shuffle);
  void PSHUFWOp(OpcodeArgs, bool Low);
  void PSHUFW8ByteOp(OpcodeArgs);
  void PSHUFDOp(OpcodeArgs);
  void PSRLDOp(OpcodeArgs, IR::OpSize ElementSize);
  void PSRLI(OpcodeArgs, IR::OpSize ElementSize);
  void PSLLI(OpcodeArgs, IR::OpSize ElementSize);
  void PSLL(OpcodeArgs, IR::OpSize ElementSize);
  void PSRAOp(OpcodeArgs, IR::OpSize ElementSize);
  void PSRLDQ(OpcodeArgs);
  void PSLLDQ(OpcodeArgs);
  void PSRAIOp(OpcodeArgs, IR::OpSize ElementSize);
  void MOVDDUPOp(OpcodeArgs);
  void CVTFPR_To_GPR(OpcodeArgs, IR::OpSize SrcElementSize, bool HostRoundingMode);
  void Vector_CVT_Int_To_Float(OpcodeArgs, IR::OpSize SrcElementSize, bool Widen, bool IsAVX);
  void Vector_CVT_Float_To_Float(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize, bool IsAVX);
  void Vector_CVT_Float_To_Int(OpcodeArgs, IR::OpSize SrcElementSize, bool HostRoundingMode, bool IsAVX);
  void MMX_To_XMM_Vector_CVT_Int_To_Float(OpcodeArgs);
  void XMM_To_MMX_Vector_CVT_Float_To_Int(OpcodeArgs, IR::OpSize SrcElementSize, bool HostRoundingMode);
  void MASKMOVOp(OpcodeArgs);
  void MOVBetweenGPR_FPR(OpcodeArgs, VectorOpType VectorType);
  void TZCNT(OpcodeArgs);
  void LZCNT(OpcodeArgs);
  void VFCMPOp(OpcodeArgs, IR::OpSize ElementSize);
  void SHUFOp(OpcodeArgs, IR::OpSize ElementSize);
  void PINSROp(OpcodeArgs, IR::OpSize ElementSize);
  void InsertPSOp(OpcodeArgs);
  void PExtrOp(OpcodeArgs, IR::OpSize ElementSize);

  void PSIGN(OpcodeArgs, IR::OpSize ElementSize);
  void VPSIGN(OpcodeArgs, IR::OpSize ElementSize);

  // BMI1 Ops
  void ANDNBMIOp(OpcodeArgs);
  void BEXTRBMIOp(OpcodeArgs);
  void BLSIBMIOp(OpcodeArgs);
  void BLSMSKBMIOp(OpcodeArgs);
  void BLSRBMIOp(OpcodeArgs);

  // BMI2 Ops
  void BMI2Shift(OpcodeArgs);
  void BZHI(OpcodeArgs);
  void MULX(OpcodeArgs);
  void PDEP(OpcodeArgs);
  void PEXT(OpcodeArgs);
  void RORX(OpcodeArgs);

  // ADX Ops
  void ADXOp(OpcodeArgs);

  // AVX Ops
  void AVXVectorXOROp(OpcodeArgs);

  void AVXVectorRound(OpcodeArgs, IR::OpSize ElementSize);

  void VectorScalarInsertALUOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
  void AVXVectorScalarInsertALUOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);

  void VectorScalarUnaryInsertALUOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
  void AVXVectorScalarUnaryInsertALUOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);

  void InsertMMX_To_XMM_Vector_CVT_Int_To_Float(OpcodeArgs);
  void InsertCVTGPR_To_FPR(OpcodeArgs, IR::OpSize DstElementSize);
  void AVXInsertCVTGPR_To_FPR(OpcodeArgs, IR::OpSize DstElementSize);

  void InsertScalar_CVT_Float_To_Float(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize);
  void AVXInsertScalar_CVT_Float_To_Float(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize);

  RoundMode TranslateRoundType(uint8_t Mode);

  void InsertScalarRound(OpcodeArgs, IR::OpSize ElementSize);
  void AVXInsertScalarRound(OpcodeArgs, IR::OpSize ElementSize);

  void InsertScalarFCMPOp(OpcodeArgs, IR::OpSize ElementSize);
  void AVXInsertScalarFCMPOp(OpcodeArgs, IR::OpSize ElementSize);

  void AVXVFCMPOp(OpcodeArgs, IR::OpSize ElementSize);

  void VADDSUBPOp(OpcodeArgs, IR::OpSize ElementSize);

  void VAESDecOp(OpcodeArgs);
  void VAESDecLastOp(OpcodeArgs);
  void VAESEncOp(OpcodeArgs);
  void VAESEncLastOp(OpcodeArgs);

  void VANDNOp(OpcodeArgs);

  Ref VBLENDOpImpl(IR::OpSize VecSize, IR::OpSize ElementSize, Ref Src1, Ref Src2, uint64_t Selector);
  void VBLENDPDOp(OpcodeArgs);
  void VPBLENDDOp(OpcodeArgs);
  void VPBLENDWOp(OpcodeArgs);

  void VBROADCASTOp(OpcodeArgs, IR::OpSize ElementSize);

  void VDPPOp(OpcodeArgs, IR::OpSize ElementSize);

  void VEXTRACT128Op(OpcodeArgs);

  void VHADDPOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
  void VHSUBPOp(OpcodeArgs, IR::OpSize ElementSize);

  void VINSERTOp(OpcodeArgs);
  void VINSERTPSOp(OpcodeArgs);

  void VMASKMOVOp(OpcodeArgs, IR::OpSize ElementSize, bool IsStore);

  void VMOVHPOp(OpcodeArgs);
  void VMOVLPOp(OpcodeArgs);

  void VMOVDDUPOp(OpcodeArgs);
  void VMOVSHDUPOp(OpcodeArgs, bool IsAVX);
  void VMOVSLDUPOp(OpcodeArgs, bool IsAVX);

  void VMOVSDOp(OpcodeArgs);
  void VMOVSSOp(OpcodeArgs);

  void VMOVAPS_VMOVAPDOp(OpcodeArgs);
  void VMOVUPS_VMOVUPDOp(OpcodeArgs);

  void VMPSADBWOp(OpcodeArgs);

  void VPACKSSOp(OpcodeArgs, IR::OpSize ElementSize);
  void VPACKUSOp(OpcodeArgs, IR::OpSize ElementSize);

  void VPALIGNROp(OpcodeArgs);

  void VPCMPESTRIOp(OpcodeArgs, bool IsAVX);
  void VPCMPESTRMOp(OpcodeArgs, bool IsAVX);
  void VPCMPISTRIOp(OpcodeArgs, bool IsAVX);
  void VPCMPISTRMOp(OpcodeArgs, bool IsAVX);

  void VCVTPH2PSOp(OpcodeArgs);
  void VCVTPS2PHOp(OpcodeArgs);

  Ref VPERMDIndices(OpSize DstSize, Ref Indices, Ref IndexMask, Ref Repeating3210);
  void VPERM2Op(OpcodeArgs);
  void VPERMDOp(OpcodeArgs);
  void VPERMQOp(OpcodeArgs);

  void VPERMILImmOp(OpcodeArgs, IR::OpSize ElementSize);

  Ref VPERMILRegOpImpl(OpSize DstSize, IR::OpSize ElementSize, Ref Src, Ref Indices);
  void VPERMILRegOp(OpcodeArgs, IR::OpSize ElementSize);

  void VPHADDSWOp(OpcodeArgs);

  void VPHSUBOp(OpcodeArgs, IR::OpSize ElementSize);
  void VPHSUBSWOp(OpcodeArgs);

  void VPINSRBWOp(OpcodeArgs, IR::OpSize ElementSize);
  void VPINSRDQOp(OpcodeArgs);

  void VPMADDUBSWOp(OpcodeArgs);
  void VPMADDWDOp(OpcodeArgs);

  void VPMASKMOVOp(OpcodeArgs, bool IsStore);

  void VPMULHRSWOp(OpcodeArgs);
  void VPMULHWOp(OpcodeArgs, bool Signed);
  void VPMULLOp(OpcodeArgs, IR::OpSize ElementSize, bool Signed);

  void VPSADBWOp(OpcodeArgs);

  void VPSHUFBOp(OpcodeArgs);
  void VPSHUFWOp(OpcodeArgs, IR::OpSize ElementSize, bool Low);

  void VPSLLOp(OpcodeArgs, IR::OpSize ElementSize);
  void VPSLLDQOp(OpcodeArgs);
  void VPSLLIOp(OpcodeArgs, IR::OpSize ElementSize);
  void VPSLLVOp(OpcodeArgs);

  void VPSRAOp(OpcodeArgs, IR::OpSize ElementSize);
  void VPSRAIOp(OpcodeArgs, IR::OpSize ElementSize);

  void VPSRAVDOp(OpcodeArgs);
  void VPSRLVOp(OpcodeArgs);

  void VPSRLDOp(OpcodeArgs, IR::OpSize ElementSize);
  void VPSRLDQOp(OpcodeArgs);
  void VPSRLIOp(OpcodeArgs, IR::OpSize ElementSize);

  void VPUNPCKHOp(OpcodeArgs, IR::OpSize ElementSize);
  void VPUNPCKLOp(OpcodeArgs, IR::OpSize ElementSize);

  void VSHUFOp(OpcodeArgs, IR::OpSize ElementSize);

  void VTESTPOp(OpcodeArgs, IR::OpSize ElementSize);

  void VZEROOp(OpcodeArgs);

  // X87 Ops
  Ref ReconstructFSW_Helper(Ref T = nullptr);
  // Returns new x87 stack top from FSW.
  Ref ReconstructX87StateFromFSW_Helper(Ref FSW);
  void FLD(OpcodeArgs, IR::OpSize Width);
  void FLDFromStack(OpcodeArgs);
  void FLD_Const(OpcodeArgs, NamedVectorConstant K);

  void FBLD(OpcodeArgs);
  void FBSTP(OpcodeArgs);

  void FILD(OpcodeArgs);

  void FST(OpcodeArgs, IR::OpSize Width);
  void FSTToStack(OpcodeArgs);

  void FIST(OpcodeArgs, bool Truncate);

  // OpResult is used for Stack operations,
  // describes if the result of the operation is stored in ST(0) or ST(i),
  // where ST(i) is one of the arguments to the operation.
  enum class OpResult {
    RES_ST0,
    RES_STI,
  };

  void FADD(OpcodeArgs, IR::OpSize Width, bool Integer, OpResult ResInST0);
  void FDIV(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpResult ResInST0);
  void FMUL(OpcodeArgs, IR::OpSize Width, bool Integer, OpResult ResInST0);
  void FNCLEX(OpcodeArgs);
  void FNINIT(OpcodeArgs);
  void FSUB(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpResult ResInST0);
  void FTST(OpcodeArgs);
  void FXCH(OpcodeArgs);
  void X87EMMS(OpcodeArgs);
  void X87FCMOV(OpcodeArgs);
  void X87FFREE(OpcodeArgs);
  void X87FLDCW(OpcodeArgs);
  void X87FNSAVE(OpcodeArgs);
  void X87FNSTENV(OpcodeArgs);
  void X87FNSTSW(OpcodeArgs);
  void X87FRSTOR(OpcodeArgs);
  void X87FSTCW(OpcodeArgs);
  void X87FXAM(OpcodeArgs);
  void X87FXTRACT(OpcodeArgs);
  void X87FYL2X(OpcodeArgs, bool IsFYL2XP1);
  void X87LDENV(OpcodeArgs);
  void X87ModifySTP(OpcodeArgs, bool Inc);
  void X87OpHelper(OpcodeArgs, FEXCore::IR::IROps IROp, bool ZeroC2);

  enum class FCOMIFlags {
    FLAGS_X87,
    FLAGS_RFLAGS,
  };
  void FCOMI(OpcodeArgs, IR::OpSize Width, bool Integer, FCOMIFlags WhichFlags, bool PopTwice);

  // F64 X87 Ops
  void FADDF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpResult ResInST0);
  void FBLDF64(OpcodeArgs);
  void FBSTPF64(OpcodeArgs);
  void FCOMIF64(OpcodeArgs, IR::OpSize width, bool Integer, FCOMIFlags whichflags, bool poptwice);
  void FDIVF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpResult ResInST0);
  void FILDF64(OpcodeArgs);
  void FISTF64(OpcodeArgs, bool Truncate);
  void FLDF64_Const(OpcodeArgs, uint64_t Num);
  void FLDF64(OpcodeArgs, IR::OpSize Width);
  void FMULF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpResult ResInST0);
  void FSUBF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpResult ResInST0);
  void FTSTF64(OpcodeArgs);
  void X87FLDCWF64(OpcodeArgs);
  void X87FXTRACTF64(OpcodeArgs);
  void X87LDENVF64(OpcodeArgs);

  void FXSaveOp(OpcodeArgs);
  void FXRStoreOp(OpcodeArgs);

  Ref XSaveBase(X86Tables::DecodedOp Op);
  void XSaveOp(OpcodeArgs);

  void PAlignrOp(OpcodeArgs);
  void UCOMISxOp(OpcodeArgs, IR::OpSize ElementSize);
  void LDMXCSR(OpcodeArgs);
  void STMXCSR(OpcodeArgs);

  void PACKUSOp(OpcodeArgs, IR::OpSize ElementSize);
  void PACKSSOp(OpcodeArgs, IR::OpSize ElementSize);

  void PMULLOp(OpcodeArgs, IR::OpSize ElementSize, bool Signed);

  void MOVQ2DQ(OpcodeArgs, bool ToXMM);

  void ADDSUBPOp(OpcodeArgs, IR::OpSize ElementSize);

  void PFNACCOp(OpcodeArgs);
  void PFPNACCOp(OpcodeArgs);
  void PSWAPDOp(OpcodeArgs);

  void VPFCMPOp(OpcodeArgs, uint8_t CompType);
  void PI2FWOp(OpcodeArgs);
  void PF2IWOp(OpcodeArgs);
  void PF2IDOp(OpcodeArgs);

  void PMULHRWOp(OpcodeArgs);

  void PMADDWD(OpcodeArgs);
  void PMADDUBSW(OpcodeArgs);

  void PMULHW(OpcodeArgs, bool Signed);
  void PMULHRSW(OpcodeArgs);

  void MOVBEOp(OpcodeArgs);
  void HSUBP(OpcodeArgs, IR::OpSize ElementSize);
  void PHSUB(OpcodeArgs, IR::OpSize ElementSize);

  void PHADDS(OpcodeArgs);
  void PHSUBS(OpcodeArgs);

  void CLWBOrTPause(OpcodeArgs);
  void CLFLUSHOPT(OpcodeArgs);
  void LoadFenceOrXRSTOR(OpcodeArgs);
  void MemFenceOrXSAVEOPT(OpcodeArgs);
  void StoreFenceOrCLFlush(OpcodeArgs);
  void UMonitorOrCLRSSBSY(OpcodeArgs);
  void UMWaitOp(OpcodeArgs);
  void CLZeroOp(OpcodeArgs);
  void RDTSCPOp(OpcodeArgs);
  void RDPIDOp(OpcodeArgs);

  void Prefetch(OpcodeArgs, bool ForStore, bool Stream, uint8_t Level);

  void PSADBW(OpcodeArgs);

  void SHA1NEXTEOp(OpcodeArgs);
  void SHA1MSG1Op(OpcodeArgs);
  void SHA1MSG2Op(OpcodeArgs);
  void SHA1RNDS4Op(OpcodeArgs);

  void SHA256MSG1Op(OpcodeArgs);
  void SHA256MSG2Op(OpcodeArgs);
  void SHA256RNDS2Op(OpcodeArgs);

  void AESImcOp(OpcodeArgs, bool IsAVX);
  void AESEncOp(OpcodeArgs);
  void AESEncLastOp(OpcodeArgs);
  void AESDecOp(OpcodeArgs);
  void AESDecLastOp(OpcodeArgs);
  void AESKeyGenAssist(OpcodeArgs, bool IsAVX);

  void VFMAImpl(OpcodeArgs, IROps IROp, bool Scalar, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);
  void VFMAddSubImpl(OpcodeArgs, bool AddSub, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);

  struct RefVSIB {
    Ref Low, High;
    Ref BaseAddr;
    int32_t Displacement;
    uint8_t Scale;
  };

  RefVSIB LoadVSIB(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags);
  void VPGATHER(OpcodeArgs, OpSize AddrElementSize);

  void AVXExtendVectorElements(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DstElementSize, bool Signed);
  void ExtendVectorElements(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DstElementSize, bool Signed);

  void VectorRound(OpcodeArgs, IR::OpSize ElementSize);

  Ref VectorBlendImpl(OpSize Size, IR::OpSize ElementSize, Ref Src1, Ref Src2, uint8_t Selector);
  void VectorBlend(OpcodeArgs, IR::OpSize ElementSize);

  void VectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize);
  void PTestOpImpl(OpSize Size, Ref Dest, Ref Src);
  void PTestOp(OpcodeArgs);

  void AVXPHMINPOSUWOp(OpcodeArgs);
  void PHMINPOSUWOp(OpcodeArgs);

  void DPPOp(OpcodeArgs, IR::OpSize ElementSize);

  void MPSADBWOp(OpcodeArgs);
  void PCLMULQDQOp(OpcodeArgs);
  void VPCLMULQDQOp(OpcodeArgs);

  void CRC32(OpcodeArgs);
  void Extrq_imm(OpcodeArgs);
  void Insertq_imm(OpcodeArgs);
  void Extrq(OpcodeArgs);
  void Insertq(OpcodeArgs);

  void BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDefinition);
  void UnimplementedOp(OpcodeArgs);
  void PermissionRestrictedOp(OpcodeArgs);

  ///< Helper for PSHUD and VPERMILPS(imm) since they are the same instruction
  Ref Single128Bit4ByteVectorShuffle(Ref Src, uint8_t Shuffle);
  // AVX 128-bit operations
  Ref AVX128_LoadXMMRegister(uint32_t XMM, bool High);
  void AVX128_StoreXMMRegister(uint32_t XMM, const Ref Src, bool High);

  struct RefPair {
    Ref Low, High;
  };

  RefPair AVX128_Zext(Ref R) {
    RefPair Pair;
    Pair.Low = R;
    Pair.High = LoadZeroVector(OpSize::i128Bit);
    return Pair;
  }

  Ref SHADataShuffle(Ref Src) {
    // SHA data shuffle matches PSHUFD shuffle where elements are inverted.
    // Because this shuffle mask gets reused multiple times per instruction, it's always a win to load the mask once and reuse it.
    const uint32_t Shuffle = 0b00'01'10'11;
    auto LookupIndexes =
      LoadAndCacheIndexedNamedVectorConstant(OpSize::i128Bit, FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFD, Shuffle * 16);
    return _VTBL1(OpSize::i128Bit, Src, LookupIndexes);
  }

  RefPair AVX128_LoadSource_WithOpSize(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags,
                                       bool NeedsHigh, MemoryAccessType AccessType = MemoryAccessType::DEFAULT);

  RefVSIB AVX128_LoadVSIB(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags, bool NeedsHigh);
  void AVX128_StoreResult_WithOpSize(FEXCore::X86Tables::DecodedOp Op, const FEXCore::X86Tables::DecodedOperand& Operand, const RefPair Src,
                                     MemoryAccessType AccessType = MemoryAccessType::DEFAULT);
  void AVX128_VMOVScalarImpl(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_VectorALU(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
  void AVX128_VectorUnary(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
  void AVX128_VectorUnaryImpl(OpcodeArgs, IR::OpSize SrcSize, IR::OpSize ElementSize, std::function<Ref(IR::OpSize ElementSize, Ref Src)> Helper);
  void AVX128_VectorBinaryImpl(OpcodeArgs, IR::OpSize SrcSize, IR::OpSize ElementSize,
                               std::function<Ref(IR::OpSize ElementSize, Ref Src1, Ref Src2)> Helper);
  void AVX128_VectorShiftWideImpl(OpcodeArgs, IR::OpSize ElementSize, IROps IROp);
  void AVX128_VectorShiftImmImpl(OpcodeArgs, IR::OpSize ElementSize, IROps IROp);
  void AVX128_VectorTrinaryImpl(OpcodeArgs, IR::OpSize SrcSize, IR::OpSize ElementSize, Ref Src3,
                                std::function<Ref(IR::OpSize ElementSize, Ref Src1, Ref Src2, Ref Src3)> Helper);

  enum class ShiftDirection { RIGHT, LEFT };
  void AVX128_ShiftDoubleImm(OpcodeArgs, ShiftDirection Dir);

  void AVX128_VMOVAPS(OpcodeArgs);
  void AVX128_VMOVSD(OpcodeArgs);
  void AVX128_VMOVSS(OpcodeArgs);

  void AVX128_VectorXOR(OpcodeArgs);

  void AVX128_VZERO(OpcodeArgs);
  void AVX128_MOVVectorNT(OpcodeArgs);
  void AVX128_MOVQ(OpcodeArgs);
  void AVX128_VMOVLP(OpcodeArgs);
  void AVX128_VMOVHP(OpcodeArgs);
  void AVX128_VMOVDDUP(OpcodeArgs);
  void AVX128_VMOVSLDUP(OpcodeArgs);
  void AVX128_VMOVSHDUP(OpcodeArgs);
  void AVX128_VBROADCAST(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_VPUNPCKL(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_VPUNPCKH(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_MOVVectorUnaligned(OpcodeArgs);
  void AVX128_InsertCVTGPR_To_FPR(OpcodeArgs, IR::OpSize DstElementSize);
  void AVX128_CVTFPR_To_GPR(OpcodeArgs, IR::OpSize SrcElementSize, bool HostRoundingMode);
  void AVX128_VANDN(OpcodeArgs);
  void AVX128_VPACKSS(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_VPACKUS(OpcodeArgs, IR::OpSize ElementSize);
  Ref AVX128_PSIGNImpl(IR::OpSize ElementSize, Ref Src1, Ref Src2);
  void AVX128_VPSIGN(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_UCOMISx(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_VectorScalarInsertALU(OpcodeArgs, FEXCore::IR::IROps IROp, IR::OpSize ElementSize);
  void AVX128_VFCMP(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_InsertScalarFCMP(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_MOVBetweenGPR_FPR(OpcodeArgs);
  void AVX128_PExtr(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_ExtendVectorElements(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DstElementSize, bool Signed);
  void AVX128_MOVMSK(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_MOVMSKB(OpcodeArgs);
  void AVX128_PINSRImpl(OpcodeArgs, IR::OpSize ElementSize, const X86Tables::DecodedOperand& Src1Op,
                        const X86Tables::DecodedOperand& Src2Op, const X86Tables::DecodedOperand& Imm);
  void AVX128_VPINSRB(OpcodeArgs);
  void AVX128_VPINSRW(OpcodeArgs);
  void AVX128_VPINSRDQ(OpcodeArgs);

  void AVX128_VariableShiftImpl(OpcodeArgs, IROps IROp);

  void AVX128_VINSERT(OpcodeArgs);
  void AVX128_VINSERTPS(OpcodeArgs);

  void AVX128_VPHSUB(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_VPHSUBSW(OpcodeArgs);

  void AVX128_VADDSUBP(OpcodeArgs, IR::OpSize ElementSize);

  void AVX128_VPMULL(OpcodeArgs, IR::OpSize ElementSize, bool Signed);

  void AVX128_VPMULHRSW(OpcodeArgs);

  void AVX128_VPMULHW(OpcodeArgs, bool Signed);

  void AVX128_InsertScalar_CVT_Float_To_Float(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize);

  void AVX128_Vector_CVT_Float_To_Float(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize);

  void AVX128_Vector_CVT_Float_To_Int(OpcodeArgs, IR::OpSize SrcElementSize, bool HostRoundingMode);

  void AVX128_Vector_CVT_Int_To_Float(OpcodeArgs, IR::OpSize SrcElementSize, bool Widen);

  void AVX128_VEXTRACT128(OpcodeArgs);
  void AVX128_VAESImc(OpcodeArgs);
  void AVX128_VAESEnc(OpcodeArgs);
  void AVX128_VAESEncLast(OpcodeArgs);
  void AVX128_VAESDec(OpcodeArgs);
  void AVX128_VAESDecLast(OpcodeArgs);
  void AVX128_VAESKeyGenAssist(OpcodeArgs);

  void AVX128_VPCMPESTRI(OpcodeArgs);
  void AVX128_VPCMPESTRM(OpcodeArgs);
  void AVX128_VPCMPISTRI(OpcodeArgs);
  void AVX128_VPCMPISTRM(OpcodeArgs);

  void AVX128_PHMINPOSUW(OpcodeArgs);

  void AVX128_VectorRound(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_InsertScalarRound(OpcodeArgs, IR::OpSize ElementSize);

  void AVX128_VDPP(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_VPERMQ(OpcodeArgs);

  void AVX128_VPSHUFW(OpcodeArgs, bool Low);

  void AVX128_VSHUF(OpcodeArgs, IR::OpSize ElementSize);

  void AVX128_VPERMILImm(OpcodeArgs, IR::OpSize ElementSize);

  void AVX128_VHADDP(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);

  void AVX128_VPHADDSW(OpcodeArgs);

  void AVX128_VPMADDUBSW(OpcodeArgs);
  void AVX128_VPMADDWD(OpcodeArgs);

  void AVX128_VBLEND(OpcodeArgs, IR::OpSize ElementSize);

  void AVX128_VHSUBP(OpcodeArgs, IR::OpSize ElementSize);

  void AVX128_VPSHUFB(OpcodeArgs);
  void AVX128_VPSADBW(OpcodeArgs);

  void AVX128_VMPSADBW(OpcodeArgs);
  void AVX128_VPALIGNR(OpcodeArgs);

  void AVX128_VMASKMOVImpl(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DstSize, bool IsStore, const X86Tables::DecodedOperand& MaskOp,
                           const X86Tables::DecodedOperand& DataOp);

  void AVX128_VPMASKMOV(OpcodeArgs, bool IsStore);

  void AVX128_VMASKMOV(OpcodeArgs, IR::OpSize ElementSize, bool IsStore);

  void AVX128_MASKMOV(OpcodeArgs);

  void AVX128_VectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize);

  void AVX128_SaveAVXState(Ref MemBase);
  void AVX128_RestoreAVXState(Ref MemBase);
  void AVX128_DefaultAVXState();

  void AVX128_VPERM2(OpcodeArgs);
  void AVX128_VTESTP(OpcodeArgs, IR::OpSize ElementSize);
  void AVX128_PTest(OpcodeArgs);

  void AVX128_VPERMILReg(OpcodeArgs, IR::OpSize ElementSize);

  void AVX128_VPERMD(OpcodeArgs);

  void AVX128_VPCLMULQDQ(OpcodeArgs);

  void AVX128_VFMAImpl(OpcodeArgs, IROps IROp, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);
  void AVX128_VFMAScalarImpl(OpcodeArgs, IROps IROp, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);
  void AVX128_VFMAddSubImpl(OpcodeArgs, bool AddSub, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);

  RefPair AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB);
  RefPair AVX128_VPGatherImpl(OpcodeArgs, OpSize Size, OpSize ElementLoadSize, OpSize AddrElementSize, RefPair Dest, RefPair Mask, RefVSIB VSIB);

  void AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize);

  void AVX128_VCVTPH2PS(OpcodeArgs);
  void AVX128_VCVTPS2PH(OpcodeArgs);

  // End of AVX 128-bit implementation

  // AVX 256-bit operations
  void StoreResult_WithAVXInsert(VectorOpType Type, RegClass Class, FEXCore::X86Tables::DecodedOp Op, Ref Value,
                                 IR::OpSize Align = IR::OpSize::iInvalid, MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {
    if (Op->Dest.IsGPR() && Op->Dest.Data.GPR.GPR >= X86State::REG_XMM_0 && Op->Dest.Data.GPR.GPR <= X86State::REG_XMM_15 &&
        GetGuestVectorLength() == OpSize::i256Bit && Type == VectorOpType::SSE) {
      const auto gpr = Op->Dest.Data.GPR.GPR;
      const auto gprIndex = gpr - X86State::REG_XMM_0;
      auto DestVector = LoadXMMRegister(gprIndex);
      Value = _VInsElement(GetGuestVectorLength(), OpSize::i128Bit, 0, 0, DestVector, Value);
      StoreXMMRegister(gprIndex, Value);
      return;
    }

    StoreResult(Class, Op, Value, Align, AccessType);
  }

  void StoreXMMRegister_WithAVXInsert(VectorOpType Type, uint32_t XMM, Ref Value) {
    if (GetGuestVectorLength() == OpSize::i256Bit && Type == VectorOpType::SSE) {
      ///< SSE vector stores need to insert in the low 128-bit lane of the 256-bit register.
      auto DestVector = LoadXMMRegister(XMM);
      Value = _VInsElement(GetGuestVectorLength(), OpSize::i128Bit, 0, 0, DestVector, Value);
      StoreXMMRegister(XMM, Value);
      return;
    }
    StoreXMMRegister(XMM, Value);
  }

  void AVXVectorALUOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
  void AVXVectorUnaryOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
  void AVXVectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize);

  // End of AVX 256-bit implementation

  void InvalidOp(OpcodeArgs);
  void NoExecOp(OpcodeArgs);

  void SetPackedRFLAG(bool Lower8, Ref Src);
  Ref GetPackedRFLAG(uint32_t FlagsMask = ~0U);

  void SetMultiblock(bool _Multiblock) {
    Multiblock = _Multiblock;
  }

  static inline constexpr unsigned IndexNZCV(unsigned BitOffset) {
    switch (BitOffset) {
    case FEXCore::X86State::RFLAG_OF_RAW_LOC: return 28;
    case FEXCore::X86State::RFLAG_CF_RAW_LOC: return 29;
    case FEXCore::X86State::RFLAG_ZF_RAW_LOC: return 30;
    case FEXCore::X86State::RFLAG_SF_RAW_LOC: return 31;
    default: FEX_UNREACHABLE;
    }
  }

  void StoreContextHelper(IR::OpSize Size, RegClass Class, Ref Value, uint32_t Offset) {
    // For i128Bit, we won't see a normal Constant to inline, but as a special
    // case we can replace with a 2x64-bit store which can use inline zeroes.
    if (Size == OpSize::i128Bit) {
      auto Header = GetOpHeader(WrapNode(Value));
      const auto MAX_STP_OFFSET = (252 * 4);

      if (Offset <= MAX_STP_OFFSET && Header->Op == OP_LOADNAMEDVECTORCONSTANT) {
        auto Const = Header->C<IR::IROp_LoadNamedVectorConstant>();

        if (Const->Constant == IR::NamedVectorConstant::NAMED_VECTOR_ZERO) {
          Ref Zero = _Constant(0);
          Ref STP = _StoreContextPair(IR::OpSize::i64Bit, RegClass::GPR, Zero, Zero, Offset);

          // XXX: This works around InlineConstant not having an associated
          // register class, else we'd just do InlineConstant above.
          Ref InlineZero = _InlineConstant(0);
          ReplaceNodeArgument(STP, 0, InlineZero);
          ReplaceNodeArgument(STP, 1, InlineZero);
          return;
        }
      }
    }

    _StoreContext(Size, Class, Value, Offset);
  }

  void FlushRegisterCache(bool SRAOnly = false, bool MMXOnly = false) {
    // At block boundaries, fix up the carry flag.
    if (!SRAOnly) {
      RectifyCarryInvert(CFInvertedABI);
    }

    if (!MMXOnly) {
      CalculateDeferredFlags();
    }

    const auto GPRSize = GetGPROpSize();
    const auto VectorSize = GetGuestVectorLength();

    // Write backwards. This is a heuristic to improve coalescing, since we
    // often copy from (low) fixed GPRs to (high) PF/AF for celebrity
    // instructions like "add rax, 1". This hack will go away with clauses.
    uint64_t Bits = RegCache.Written;

    // We have an SRA only mode that exists as a hack to make register caching
    // less aggressive. We should get rid of this once RA can take it.
    uint64_t Mask = ~0ULL;

    if (SRAOnly) {
      const uint64_t GPRMask = ((1ull << (AFIndex - GPR0Index + 1)) - 1) << GPR0Index;
      const uint64_t FPRMask = ((1ull << (FPR15Index - FPR0Index + 1)) - 1) << FPR0Index;

      Mask &= (GPRMask | FPRMask);
      Bits &= Mask;
    }

    if (MMXOnly) {
      Mask &= ((1ull << (MM7Index - MM0Index + 1)) - 1) << MM0Index;
      Bits &= Mask;
    }

    while (Bits != 0) {
      uint32_t Index = 63 - std::countl_zero(Bits);
      Ref Value = RegCache.Value[Index];

      if (Index >= GPR0Index && Index <= GPR15Index) {
        Ref R = _StoreRegister(Value, GPRSize);
        R->Reg = PhysicalRegister(RegClass::GPRFixed, Index - GPR0Index).Raw;
      } else if (Index == PFIndex) {
        _StorePF(Value, GPRSize);
      } else if (Index == AFIndex) {
        _StoreAF(Value, GPRSize);
      } else if (Index >= FPR0Index && Index <= FPR15Index) {
        Ref R = _StoreRegister(Value, VectorSize);
        R->Reg = PhysicalRegister(RegClass::FPRFixed, Index - FPR0Index).Raw;
      } else if (Index == DFIndex) {
        _StoreContextGPR(OpSize::i8Bit, Value, offsetof(Core::CPUState, flags[X86State::RFLAG_DF_RAW_LOC]));
      } else {
        bool Partial = RegCache.Partial & (1ull << Index);
        auto Size = Partial ? OpSize::i64Bit : CacheIndexToOpSize(Index);
        uint64_t NextBit = (1ull << (Index - 1));
        uint32_t Offset = CacheIndexToContextOffset(Index);
        auto Class = CacheIndexClass(Index);
        LOGMAN_THROW_A_FMT(Offset != ~0U, "Invalid offset");

        // Use stp where possible to store multiple values at a time. This accelerates AVX.
        // TODO: this is all really confusing because of backwards iteration,
        // can we peel back that hack?
        const auto SizeInt = IR::OpSizeToSize(Size);
        if ((Bits & NextBit) && !Partial && Size >= OpSize::i32Bit && CacheIndexToContextOffset(Index - 1) == Offset - SizeInt &&
            (Offset - SizeInt) / SizeInt < 64) {
          LOGMAN_THROW_A_FMT(CacheIndexClass(Index - 1) == Class, "construction");
          LOGMAN_THROW_A_FMT((Offset % SizeInt) == 0, "construction");
          Ref ValueNext = RegCache.Value[Index - 1];

          _StoreContextPair(Size, Class, ValueNext, Value, Offset - SizeInt);
          Bits &= ~NextBit;
        } else {
          StoreContextHelper(Size, Class, Value, Offset);
          // If Partial and MMX register, then we need to store all 1s in bits 64-80
          if (Partial && Index >= MM0Index && Index <= MM7Index) {
            _StoreContextGPR(OpSize::i16Bit, Constant(0xFFFF), Offset + 8);
          }
        }
      }

      Bits &= ~(1ull << Index);
    }

    RegCache.Written &= ~Mask;
    RegCache.Cached &= ~Mask;
    RegCache.Partial &= ~Mask;
  }

  IR::OpSize GetGPROpSize() const {
    return Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;
  }

protected:
  void RecordX87Use() override {
    CurrentHeader->HasX87 = true;
  }

  void SaveNZCV(IROps Op = OP_DUMMY) override {
    /* Some opcodes are conservatively marked as clobbering flags, but in fact
     * do not clobber flags in certain conditions. Check for that here as an
     * optimization.
     */
    switch (Op) {
    case OP_VFMINSCALARINSERT:
    case OP_VFMAXSCALARINSERT:
      /* On AFP platforms, becomes fmin/fmax and preserves NZCV. Otherwise
       * becomes fcmp and clobbers.
       */
      if (CTX->HostFeatures.SupportsAFP) {
        return;
      }
      break;

    case OP_VLOADVECTORMASKED:
    case OP_VLOADVECTORGATHERMASKED:
    case OP_VLOADVECTORGATHERMASKEDQPS:
    case OP_VSTOREVECTORMASKED:
      /* On ASIMD platforms, the emulation happens to preserve NZCV, unlike the
       * more optimal SVE implementation that clobbers.
       */
      if (!CTX->HostFeatures.SupportsSVE128 && !CTX->HostFeatures.SupportsSVE256) {
        return;
      }

      break;
    default: break;
    }

    // Invariant: When executing instructions that clobber NZCV, the flags must
    // be resident in a GPR, which is equivalent to CachedNZCV != nullptr. Get
    // the NZCV which fills the cache if necessary.
    if (CachedNZCV == nullptr) {
      GetNZCV();
    }

    // Assume we'll need a reload.
    NZCVDirty = true;
  }

private:
  FEX_CONFIG_OPT(ReducedPrecisionMode, X87REDUCEDPRECISION);

  struct JumpTargetInfo {
    Ref BlockEntry;
    bool HaveEmitted;
    bool IsEntryPoint;
  };

  FEXCore::Context::ContextImpl* CTX {};
  FEXCore::Core::InternalThreadState* Thread;

  constexpr static unsigned FullNZCVMask = (1U << FEXCore::X86State::RFLAG_CF_RAW_LOC) | (1U << FEXCore::X86State::RFLAG_ZF_RAW_LOC) |
                                           (1U << FEXCore::X86State::RFLAG_SF_RAW_LOC) | (1U << FEXCore::X86State::RFLAG_OF_RAW_LOC);

  static bool ContainsNZCV(unsigned BitMask) {
    return (BitMask & FullNZCVMask) != 0;
  }

  static bool IsNZCV(unsigned BitOffset) {
    return BitOffset < 32 && ContainsNZCV(1U << BitOffset);
  }

  Ref CachedNZCV {};
  bool NZCVDirty {};

  // Set if the host carry is inverted from the guest carry. This is set after
  // subtraction, because arm64 and x86 have inverted borrow flags, but clear
  // after addition.
  //
  // All CF access needs to maintain this flag. cfinv may be inserted at the end
  // of a block to rectify to the FEX convention (current convention: NOT
  // INVERTED).
  bool CFInverted {};

  // FEX convention for CF at the end of blocks: INVERTED.
  const bool CFInvertedABI {true};

  fextl::map<uint64_t, JumpTargetInfo> JumpTargets;
  bool HandledLock {false};
  bool DecodeFailure {false};
  bool NeedsBlockEnd {false};
  ForceTSOMode ForceTSO {ForceTSOMode::NoOverride};
  // Used during new op bringup
  bool ShouldDump {false};

  using SaveStoreAVXStatePtr = void (OpDispatchBuilder::*)(Ref MemBase);
  using DefaultAVXStatePtr = void (OpDispatchBuilder::*)();
  SaveStoreAVXStatePtr SaveAVXStateFunc {&OpDispatchBuilder::SaveAVXState};
  SaveStoreAVXStatePtr RestoreAVXStateFunc {&OpDispatchBuilder::RestoreAVXState};
  DefaultAVXStatePtr DefaultAVXStateFunc {&OpDispatchBuilder::DefaultAVXState};

  // Opcode helpers for generalizing behavior across VEX and non-VEX variants.

  Ref ADDSUBPOpImpl(OpSize Size, IR::OpSize ElementSize, Ref Src1, Ref Src2);

  void AVXVariableShiftImpl(OpcodeArgs, IROps IROp);

  Ref AESKeyGenAssistImpl(OpcodeArgs);

  Ref CVTGPR_To_FPRImpl(OpcodeArgs, IR::OpSize DstElementSize, const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op);

  Ref DPPOpImpl(IR::OpSize DstSize, Ref Src1, Ref Src2, uint8_t Mask, IR::OpSize ElementSize);

  Ref VDPPSOpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1, const X86Tables::DecodedOperand& Src2, const X86Tables::DecodedOperand& Imm);

  Ref ExtendVectorElementsImpl(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DstElementSize, bool Signed);

  Ref HSUBPOpImpl(OpSize Size, IR::OpSize ElementSize, Ref Src1, Ref Src2);

  Ref InsertPSOpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1, const X86Tables::DecodedOperand& Src2,
                     const X86Tables::DecodedOperand& Imm);

  Ref MPSADBWOpImpl(IR::OpSize SrcSize, Ref Src1, Ref Src2, uint8_t Select);

  Ref PALIGNROpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1, const X86Tables::DecodedOperand& Src2,
                    const X86Tables::DecodedOperand& Imm, bool IsAVX);

  void PCMPXSTRXOpImpl(OpcodeArgs, bool IsExplicit, bool IsMask, bool IsAVX);

  Ref PHADDSOpImpl(OpSize Size, Ref Src1, Ref Src2);

  Ref PHMINPOSUWOpImpl(OpcodeArgs);

  Ref PHSUBOpImpl(OpSize Size, Ref Src1, Ref Src2, IR::OpSize ElementSize);

  Ref PHSUBSOpImpl(OpSize Size, Ref Src1, Ref Src2);

  Ref PINSROpImpl(OpcodeArgs, IR::OpSize ElementSize, const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op,
                  const X86Tables::DecodedOperand& Imm);

  Ref PMADDWDOpImpl(IR::OpSize Size, Ref Src1, Ref Src2);

  Ref PMADDUBSWOpImpl(IR::OpSize Size, Ref Src1, Ref Src2);

  Ref PMULHRSWOpImpl(OpSize Size, Ref Src1, Ref Src2);

  Ref PMULHWOpImpl(OpcodeArgs, bool Signed, Ref Src1, Ref Src2);

  Ref PMULLOpImpl(OpSize Size, IR::OpSize ElementSize, bool Signed, Ref Src1, Ref Src2);

  Ref PSADBWOpImpl(IR::OpSize Size, Ref Src1, Ref Src2);

  Ref GeneratePSHUFBMask(IR::OpSize SrcSize);
  Ref PSHUFBOpImpl(IR::OpSize SrcSize, Ref Src1, Ref Src2, Ref MaskVector);

  Ref PSIGNImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src1, Ref Src2);

  Ref PSLLIImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src, uint64_t Shift);

  Ref PSLLImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src, Ref ShiftVec);

  Ref PSRAOpImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src, Ref ShiftVec);

  Ref PSRLDOpImpl(OpcodeArgs, IR::OpSize ElementSize, Ref Src, Ref ShiftVec);

  Ref SHUFOpImpl(IR::OpSize DstSize, IR::OpSize ElementSize, Ref Src1, Ref Src2, uint8_t Shuffle);

  void VMASKMOVOpImpl(OpcodeArgs, IR::OpSize ElementSize, IR::OpSize DataSize, bool IsStore, const X86Tables::DecodedOperand& MaskOp,
                      const X86Tables::DecodedOperand& DataOp);

  void MOVScalarOpImpl(OpcodeArgs, IR::OpSize ElementSize);
  void VMOVScalarOpImpl(OpcodeArgs, IR::OpSize ElementSize);

  Ref VFCMPOpImpl(OpSize Size, IR::OpSize ElementSize, Ref Src1, Ref Src2, uint8_t CompType);

  void VTESTOpImpl(OpSize SrcSize, IR::OpSize ElementSize, Ref Src1, Ref Src2);

  void VectorUnaryDuplicateOpImpl(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);

  // x86 ALU scalar operations operate in three different ways
  // - AVX512: Writemask shenanigans that we don't care about.
  // - AVX/VEX: Two source
  //   - Example 32bit VADDSS Dest, Src1, Src2
  //   - Dest[31:0] = Src1[31:0] + Src2[31:0]
  //   - Dest[127:32] = Src1[127:32]
  // - SSE: Scalar operation inserts in to the low bits, upper bits completely unaffected.
  //   - Example 32bit ADDSS Dest, Src
  //   - Dest[31:0] = Dest[31:0] + Src[31:0]
  //   - Dest[{256,128}:32] = (Unmodified)
  Ref VectorScalarInsertALUOpImpl(OpcodeArgs, IROps IROp, IR::OpSize DstSize, IR::OpSize ElementSize,
                                  const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op, bool ZeroUpperBits);

  Ref VectorScalarUnaryInsertALUOpImpl(OpcodeArgs, IROps IROp, IR::OpSize DstSize, IR::OpSize ElementSize,
                                       const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op, bool ZeroUpperBits);

  Ref InsertCVTGPR_To_FPRImpl(OpcodeArgs, IR::OpSize DstSize, IR::OpSize DstElementSize, const X86Tables::DecodedOperand& Src1Op,
                              const X86Tables::DecodedOperand& Src2Op, bool ZeroUpperBits);

  Ref InsertScalar_CVT_Float_To_FloatImpl(OpcodeArgs, IR::OpSize DstSize, IR::OpSize DstElementSize, IR::OpSize SrcElementSize,
                                          const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op, bool ZeroUpperBits);
  Ref InsertScalarRoundImpl(OpcodeArgs, IR::OpSize DstSize, IR::OpSize ElementSize, const X86Tables::DecodedOperand& Src1Op,
                            const X86Tables::DecodedOperand& Src2Op, uint64_t Mode, bool ZeroUpperBits);

  Ref InsertScalarFCMPOpImpl(OpSize Size, IR::OpSize OpDstSize, IR::OpSize ElementSize, Ref Src1, Ref Src2, uint8_t CompType, bool ZeroUpperBits);

  Ref VectorRoundImpl(OpSize Size, IR::OpSize ElementSize, Ref Src, uint64_t Mode);

  Ref Scalar_CVT_Float_To_FloatImpl(OpcodeArgs, IR::OpSize DstElementSize, IR::OpSize SrcElementSize,
                                    const X86Tables::DecodedOperand& Src1Op, const X86Tables::DecodedOperand& Src2Op);

  Ref CVTFPR_To_GPRImpl(OpcodeArgs, Ref Src, IR::OpSize SrcElementSize, bool HostRoundingMode);

  Ref Vector_CVT_Float_To_Int32Impl(OpcodeArgs, IR::OpSize DstSize, Ref Src, IR::OpSize SrcSize, IR::OpSize SrcElementSize,
                                    bool HostRoundingMode, bool ZeroUpperHalf);

  Ref Vector_CVT_Int_To_FloatImpl(OpcodeArgs, IR::OpSize SrcElementSize, bool Widen);

  void XSaveOpImpl(OpcodeArgs);
  void SaveX87State(OpcodeArgs, Ref MemBase);
  void SaveSSEState(Ref MemBase);
  void SaveMXCSRState(Ref MemBase);
  void SaveAVXState(Ref MemBase);

  void XRstorOpImpl(OpcodeArgs);
  void RestoreX87State(Ref MemBase);
  void RestoreSSEState(Ref MemBase);
  void RestoreMXCSRState(Ref MXCSR);
  void RestoreAVXState(Ref MemBase);
  void DefaultX87State(OpcodeArgs);
  void DefaultSSEState();
  void DefaultAVXState();

  Ref GetMXCSR();

#undef OpcodeArgs

  Ref AppendSegmentOffset(Ref Value, uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false);
  Ref GetSegment(uint32_t Flags, uint32_t DefaultPrefix = FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX, bool Override = false);

  void UpdatePrefixFromSegment(Ref Segment, uint32_t SegmentReg);

  Ref LoadGPRRegister(uint32_t GPR, IR::OpSize Size = OpSize::iInvalid, uint8_t Offset = 0, bool AllowUpperGarbage = false);
  void StoreGPRRegister(uint32_t GPR, const Ref Src, IR::OpSize Size = OpSize::iInvalid, uint8_t Offset = 0);
  void StoreXMMRegister(uint32_t XMM, const Ref Src);

  Ref _GetRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset, bool Inline) {
    const auto GPRSize = GetGPROpSize();
    const auto Offs = Op->PC + Op->InstSize + Offset - Entry;
    return Inline ? _InlineEntrypointOffset(GPRSize, Offs) : _EntrypointOffset(GPRSize, Offs);
  }

  Ref GetRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset = 0) {
    return _GetRelocatedPC(Op, Offset, false);
  }

  void ExitRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset = 0) {
    ExitFunction(_GetRelocatedPC(Op, Offset, true /* Inline */));
  }

  void ExitRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset, BranchHint Hint, Ref CallReturnAddress, Ref CallReturnBlock) {
    ExitFunction(_GetRelocatedPC(Op, Offset, true /* Inline */), Hint, CallReturnAddress, CallReturnBlock);
  }

  [[nodiscard]]
  static bool IsOperandMem(const X86Tables::DecodedOperand& Operand, bool Load) {
    // Literals are immediates as sources but memory addresses as destinations.
    return !(Load && (Operand.IsLiteral() || Operand.IsLiteralRelocation())) && !Operand.IsGPR();
  }

  [[nodiscard]]
  static bool IsNonTSOReg(MemoryAccessType Access, uint8_t Reg) {
    return Access == MemoryAccessType::DEFAULT && Reg == X86State::REG_RSP;
  }

  AddressMode DecodeAddress(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, MemoryAccessType AccessType, bool IsLoad);
  uint64_t CalcAddress(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, bool IsLoad);

  Ref LoadSource(RegClass Class, const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags,
                 const LoadSourceOptions& Options = {});
  Ref LoadSourceGPR(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags,
                    const LoadSourceOptions& Options = {}) {
    return LoadSource(RegClass::GPR, Op, Operand, Flags, Options);
  }
  Ref LoadSourceFPR(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags,
                    const LoadSourceOptions& Options = {}) {
    return LoadSource(RegClass::FPR, Op, Operand, Flags, Options);
  }

  Ref LoadSource_WithOpSize(RegClass Class, const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, IR::OpSize OpSize,
                            uint32_t Flags, const LoadSourceOptions& Options = {});
  Ref LoadSourceGPR_WithOpSize(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, IR::OpSize OpSize, uint32_t Flags,
                               const LoadSourceOptions& Options = {}) {
    return LoadSource_WithOpSize(RegClass::GPR, Op, Operand, OpSize, Flags, Options);
  }
  Ref LoadSourceFPR_WithOpSize(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, IR::OpSize OpSize, uint32_t Flags,
                               const LoadSourceOptions& Options = {}) {
    return LoadSource_WithOpSize(RegClass::FPR, Op, Operand, OpSize, Flags, Options);
  }

  void StoreResult_WithOpSize(RegClass Class, X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src, IR::OpSize OpSize,
                              IR::OpSize Align, MemoryAccessType AccessType = MemoryAccessType::DEFAULT);
  void StoreResultGPR_WithOpSize(X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src, IR::OpSize OpSize,
                                 IR::OpSize Align = IR::OpSize::iInvalid, MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {
    StoreResult_WithOpSize(RegClass::GPR, Op, Operand, Src, OpSize, Align, AccessType);
  }
  void StoreResultFPR_WithOpSize(X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src, IR::OpSize OpSize,
                                 IR::OpSize Align = IR::OpSize::iInvalid, MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {
    StoreResult_WithOpSize(RegClass::FPR, Op, Operand, Src, OpSize, Align, AccessType);
  }

  void StoreResult(RegClass Class, X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src, OpSize Align,
                   MemoryAccessType AccessType = MemoryAccessType::DEFAULT);
  void StoreResultGPR(X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src, OpSize Align = OpSize::iInvalid,
                      MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {
    StoreResult(RegClass::GPR, Op, Operand, Src, Align, AccessType);
  }
  void StoreResultFPR(X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src, OpSize Align = OpSize::iInvalid,
                      MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {
    StoreResult(RegClass::FPR, Op, Operand, Src, Align, AccessType);
  }

  void StoreResult(RegClass Class, X86Tables::DecodedOp Op, Ref Src, OpSize Align, MemoryAccessType AccessType = MemoryAccessType::DEFAULT);
  void StoreResultGPR(X86Tables::DecodedOp Op, Ref Src, OpSize Align = OpSize::iInvalid, MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {
    StoreResult(RegClass::GPR, Op, Src, Align, AccessType);
  }
  void StoreResultFPR(X86Tables::DecodedOp Op, Ref Src, OpSize Align = OpSize::iInvalid, MemoryAccessType AccessType = MemoryAccessType::DEFAULT) {
    StoreResult(RegClass::FPR, Op, Src, Align, AccessType);
  }

  // In several instances, it's desirable to get a base address with the segment offset
  // applied to it. This pulls all the common-case appending into a single set of functions.
  [[nodiscard]]
  Ref MakeSegmentAddress(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, IR::OpSize OpSize) {
    Ref Mem = LoadSourceGPR_WithOpSize(Op, Operand, OpSize, Op->Flags, {.LoadData = false});
    return AppendSegmentOffset(Mem, Op->Flags);
  }
  [[nodiscard]]
  Ref MakeSegmentAddress(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand) {
    return MakeSegmentAddress(Op, Operand, OpSizeFromSrc(Op));
  }
  [[nodiscard]]
  Ref MakeSegmentAddress(X86State::X86Reg Reg, uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false) {
    Ref Address = LoadGPRRegister(Reg);
    return AppendSegmentOffset(Address, Flags, DefaultPrefix, Override);
  }

  constexpr OpSize GetGuestVectorLength() const {
    return (CTX->HostFeatures.SupportsSVE256 && CTX->HostFeatures.SupportsAVX) ? OpSize::i256Bit : OpSize::i128Bit;
  }

  [[nodiscard]]
  static uint32_t GPROffset(X86State::X86Reg reg) {
    LOGMAN_THROW_A_FMT(reg <= X86State::X86Reg::REG_R15, "Invalid reg used");
    return static_cast<uint32_t>(ARRAY_OFFSETOF(Core::CPUState, gregs, reg));
  }

  [[nodiscard]]
  static uint32_t MMBaseOffset() {
    return static_cast<uint32_t>(offsetof(Core::CPUState, mm[0][0]));
  }

  [[nodiscard]]
  uint8_t GetDstSize(X86Tables::DecodedOp Op) const;
  [[nodiscard]]
  uint8_t GetSrcSize(X86Tables::DecodedOp Op) const;
  [[nodiscard]]
  uint32_t GetDstBitSize(X86Tables::DecodedOp Op) const;
  [[nodiscard]]
  uint32_t GetSrcBitSize(X86Tables::DecodedOp Op) const;
  [[nodiscard]]
  IR::OpSize OpSizeFromDst(X86Tables::DecodedOp Op) const {
    return IR::SizeToOpSize(GetDstSize(Op));
  }
  [[nodiscard]]
  IR::OpSize OpSizeFromSrc(X86Tables::DecodedOp Op) const {
    return IR::SizeToOpSize(GetSrcSize(Op));
  }

  [[nodiscard]]
  IR::OpSize GetStringOpSize(X86Tables::DecodedOp Op) const;

  // Set flag tracking to prepare for an operation that directly writes NZCV.
  void HandleNZCVWrite() {
    CachedNZCV = nullptr;
    NZCVDirty = false;
  }

  // Set flag tracking to prepare for a read-modify-write operation on NZCV.
  void HandleNZCV_RMW() {
    CalculateDeferredFlags();

    if (NZCVDirty && CachedNZCV) {
      _StoreNZCV(CachedNZCV);
    }

    HandleNZCVWrite();
  }

  // Special case of the above where we are known to zero C/V
  void HandleNZ00Write() {
    HandleNZCVWrite();

    // Host carry will be implicitly zeroed, and we want guest carry zeroed as
    // well. So do not invert.
    CFInverted = false;
  }

  Ref GetNZCV() {
    if (!CachedNZCV) {
      CachedNZCV = _LoadNZCV();
    }

    return CachedNZCV;
  }

  void SetNZCV(Ref Value) {
    CachedNZCV = Value;
    NZCVDirty = true;
  }

  void ZeroNZCV() {
    CachedNZCV = Constant(0);
    NZCVDirty = true;
  }

  void SetNZ_ZeroCV(IR::OpSize SrcSize, Ref Res, bool SetPF = false) {
    HandleNZ00Write();

    // x - 0 = x. NZ set according to Res. C always set. V always unset. This
    // matches what we want since we want carry inverted.
    //
    // This is currently worse for 8/16-bit, but that should be optimized. TODO
    if (SrcSize >= OpSize::i32Bit) {
      if (SetPF) {
        CalculatePF(SubWithFlags(SrcSize, Res, (uint64_t)0));
      } else {
        _SubNZCV(SrcSize, Res, Constant(0));
      }

      CFInverted = true;
    } else {
      _TestNZ(SrcSize, Res, Res);
      CFInverted = false;

      if (SetPF) {
        CalculatePF(Res);
      }
    }
  }

  void SetNZP_ZeroCV(IR::OpSize SrcSize, Ref Res) {
    SetNZ_ZeroCV(SrcSize, Res, true);
  }

  void InsertNZCV(unsigned BitOffset, Ref Value, signed FlagOffset, bool MustMask) {
    signed Bit = IndexNZCV(BitOffset);

    // Heuristic to choose rmif vs msr.
    bool PreferRmif = !NZCVDirty || FlagOffset || MustMask;

    if (CTX->HostFeatures.SupportsFlagM && PreferRmif) {
      // Update NZCV
      if (NZCVDirty && CachedNZCV) {
        _StoreNZCV(CachedNZCV);
      }

      CachedNZCV = nullptr;
      NZCVDirty = false;

      // Insert as NZCV.
      signed RmifBit = Bit - 28;
      _RmifNZCV(Value, (64 + FlagOffset - RmifBit) % 64, 1u << RmifBit);
      CachedNZCV = nullptr;
    } else {
      // Insert as GPR
      if (FlagOffset || MustMask) {
        Value = _Bfe(OpSize::i64Bit, 1, FlagOffset, Value);
      }

      SetNZCV(_Bfi(OpSize::i32Bit, 1, Bit, GetNZCV(), Value));
    }
  }

  // If we don't care about N/C/V and just need Z, we can test with a simple
  // mask without any shifting.
  void SetZ_InvalidateNCV(IR::OpSize Size, Ref Src) {
    HandleNZCVWrite();
    CFInverted = true;

    if (Size < OpSize::i32Bit) {
      _TestNZ(OpSize::i32Bit, Src, _InlineConstant((1u << (IR::OpSizeAsBits(Size))) - 1));
    } else {
      _TestNZ(Size, Src, Src);
    }
  }

  // Ensure the carry invert flag matches the desired form. Used before an
  // operation reading carry or at the end of a block.
  void RectifyCarryInvert(bool RequiredInvert) {
    if (CFInverted != RequiredInvert) {
      if (CTX->HostFeatures.SupportsFlagM && !NZCVDirty) {
        // Invert as NZCV.
        _CarryInvert();
        CachedNZCV = nullptr;
      } else {
        // Invert as a GPR
        unsigned Bit = IndexNZCV(FEXCore::X86State::RFLAG_CF_RAW_LOC);
        SetNZCV(_Xor(OpSize::i32Bit, GetNZCV(), Constant(1u << Bit)));
        CalculateDeferredFlags();
      }

      CFInverted ^= true;
    }

    LOGMAN_THROW_A_FMT(CFInverted == RequiredInvert, "post condition");
  }

  void CarryInvert() {
    CFInverted ^= true;
  }

  template<unsigned BitOffset>
  void SetRFLAG(Ref Value, unsigned ValueOffset = 0, bool MustMask = false) {
    SetRFLAG(Value, BitOffset, ValueOffset, MustMask);
  }

  void SetCFDirect(Ref Value, unsigned ValueOffset = 0, bool MustMask = false) {
    Value = _Xor(OpSize::i64Bit, Value, _InlineConstant(1ull << ValueOffset));
    SetRFLAG(Value, X86State::RFLAG_CF_RAW_LOC, ValueOffset, MustMask);
    CFInverted = true;
  }

  // Set CF directly to the given 0/1 value. This needs to respect the
  // invert. We use a subtraction:
  //
  //     0 - x = 0 + (~x) + 1.
  //
  // If x = 0, then 0 + (~0) + 1 = 0x100000000 so hardware C is set.
  // If x = 1, then 0 + (~1) + 1 = 0x0ffffffff so hardware C is not set.
  void SetCFDirect_InvalidateNZV(Ref Value, unsigned ValueOffset = 0, bool MustMask = false) {
    if (ValueOffset || MustMask) {
      Value = _Bfe(OpSize::i64Bit, 1, ValueOffset, Value);
    }

    HandleNZCVWrite();
    _SubNZCV(OpSize::i32Bit, Constant(0), Value);
    CFInverted = true;
  }

  void SetCFInverted(Ref Value, unsigned ValueOffset = 0, bool MustMask = false) {
    SetRFLAG(Value, X86State::RFLAG_CF_RAW_LOC, ValueOffset, MustMask);
    CFInverted = true;
  }

  void SetRFLAG(Ref Value, unsigned BitOffset, unsigned ValueOffset = 0, bool MustMask = false) {
    if (IsNZCV(BitOffset)) {
      InsertNZCV(BitOffset, Value, ValueOffset, MustMask);
      return;
    }

    if (ValueOffset || MustMask) {
      Value = _Bfe(OpSize::i32Bit, 1, ValueOffset, Value);
    }

    if (BitOffset == FEXCore::X86State::RFLAG_PF_RAW_LOC) {
      StoreRegister(Core::CPUState::PF_AS_GREG, false, Value);
    } else if (BitOffset == FEXCore::X86State::RFLAG_AF_RAW_LOC) {
      StoreRegister(Core::CPUState::AF_AS_GREG, false, Value);
    } else if (BitOffset == FEXCore::X86State::RFLAG_DF_RAW_LOC) {
      // For DF, we need to transform 0/1 into 1/-1
      StoreDF(_SubShift(OpSize::i64Bit, Constant(1), Value, ShiftType::LSL, 1));
    } else if (BitOffset == FEXCore::X86State::RFLAG_TF_RAW_LOC) {
      auto PackedTF = _LoadContextGPR(OpSize::i8Bit, ARRAY_OFFSETOF(FEXCore::Core::CPUState, flags, BitOffset));
      // An exception should still be raised after an instruction that unsets TF, leave the unblocked bit set but unset
      // the TF bit to cause such behaviour. The handling code at the start of the next block will then unset the
      // unblocked bit before raising the exception.
      auto NewPackedTF =
        _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, Value, Constant(0), _And(OpSize::i32Bit, PackedTF, Constant(~1)), Constant(1));
      _StoreContextGPR(OpSize::i8Bit, NewPackedTF, ARRAY_OFFSETOF(FEXCore::Core::CPUState, flags, BitOffset));
    } else {
      _StoreContextGPR(OpSize::i8Bit, Value, ARRAY_OFFSETOF(FEXCore::Core::CPUState, flags, BitOffset));
    }
  }

  void SetAF(unsigned K) {
    // AF is stored in bit 4 of the AF flag byte, with garbage in the other
    // bits. This allows us to defer the extract in the usual case. When it is
    // read, bit 4 is extracted.  In order to write a constant value of AF, that
    // means we need to left-shift here to compensate.
    SetRFLAG<FEXCore::X86State::RFLAG_AF_RAW_LOC>(Constant(K << 4));
  }

  void ZeroPF_AF();

  void InvalidateAF() {
    _InvalidateFlags((1u << X86State::RFLAG_AF_RAW_LOC));
    InvalidateReg(Core::CPUState::AF_AS_GREG);
  }

  void InvalidatePF_AF() {
    _InvalidateFlags((1u << X86State::RFLAG_PF_RAW_LOC) | (1u << X86State::RFLAG_AF_RAW_LOC));
    InvalidateReg(Core::CPUState::PF_AS_GREG);
    InvalidateReg(Core::CPUState::AF_AS_GREG);
  }

  [[nodiscard]]
  static CondClass CondForNZCVBit(unsigned BitOffset, bool Invert) {
    switch (BitOffset) {
    case X86State::RFLAG_SF_RAW_LOC: return Invert ? CondClass::PL : CondClass::MI;
    case X86State::RFLAG_ZF_RAW_LOC: return Invert ? CondClass::NEQ : CondClass::EQ;
    case X86State::RFLAG_CF_RAW_LOC: return Invert ? CondClass::ULT : CondClass::UGE;
    case X86State::RFLAG_OF_RAW_LOC: return Invert ? CondClass::FNU : CondClass::FU;
    default: FEX_UNREACHABLE;
    }
  }

  /* Layout of cache indices. We use a single 64-bit bitmask for the cache */
  static const int GPR0Index = 0;
  static const int GPR15Index = 15;
  static const int PFIndex = 16;
  static const int AFIndex = 17;
  /* Gap 18..19 */
  /* Note this range is only valid if MMXState = MMXState_MMX */
  static const int MM0Index = 20;
  static const int MM7Index = 27;
  /* Gap 28..30 */
  static const int DFIndex = 31;
  static const int FPR0Index = 32;
  static const int FPR15Index = 47;
  static const int AVXHigh0Index = 48;
  static const int AVXHigh15Index = 63;

  [[nodiscard]]
  static uint32_t CacheIndexToContextOffset(int Index) {
    switch (Index) {
    case MM0Index ... MM7Index: return ARRAY_OFFSETOF(FEXCore::Core::CPUState, mm, Index - MM0Index);
    case AVXHigh0Index ... AVXHigh15Index: return ARRAY_OFFSETOF(FEXCore::Core::CPUState, avx_high, Index - AVXHigh0Index);
    default: return ~0U;
    }
  }

  [[nodiscard]]
  static RegClass CacheIndexClass(int Index) {
    if ((Index >= MM0Index && Index <= MM7Index) || Index >= FPR0Index) {
      return RegClass::FPR;
    } else {
      return RegClass::GPR;
    }
  }

  [[nodiscard]]
  static IR::OpSize CacheIndexToOpSize(int Index) {
    // MMX registers are rounded up to 128-bit since they are shared with 80-bit
    // x87 registers, even though MMX is logically only 64-bit.
    if (Index >= AVXHigh0Index || ((Index >= MM0Index && Index <= MM7Index))) {
      return OpSize::i128Bit;
    } else {
      return OpSize::i8Bit;
    }
  }

  struct {
    uint64_t Cached;
    uint64_t Written;

    // Indicates that Value contains only the lower 64-bit of the full 80-bit
    // register. Used for MMX/x87 optimization.
    uint64_t Partial;

    Ref Value[64];
  } RegCache {};

  void InvalidateReg(uint8_t Index) {
    uint64_t Bit = (1ull << (uint64_t)Index);
    RegCache.Cached &= ~Bit;
    RegCache.Written &= ~Bit;
  }

  Ref LoadRegCache(uint64_t Offset, uint8_t Index, RegClass Class, IR::OpSize Size) {
    LOGMAN_THROW_A_FMT(Index < 64, "valid index");
    uint64_t Bit = (1ull << (uint64_t)Index);

    if (Size == OpSize::i128Bit && (RegCache.Partial & Bit)) {
      // We need to load the full register extend if we previously did a partial access.
      Ref Value = RegCache.Value[Index];
      Ref Full = _LoadContext(Size, Class, Offset);

      // If we did a partial store, we're inserting into the full register
      if (RegCache.Written & Bit) {
        Full = _VInsElement(OpSize::i128Bit, OpSize::i64Bit, 0, 0, Full, Value);
      }

      RegCache.Value[Index] = Full;
    }

    if (!(RegCache.Cached & Bit)) {
      if (Index == DFIndex) {
        RegCache.Value[Index] = _LoadDF();
      } else if ((Index >= MM0Index && Index <= MM7Index) || Index >= AVXHigh0Index) {
        RegCache.Value[Index] = _LoadContext(Size, Class, Offset);

        // We may have done a partial load, this requires special handling.
        if (Size == OpSize::i64Bit) {
          RegCache.Partial |= Bit;
        }
      } else if (Index == PFIndex) {
        RegCache.Value[Index] = _LoadPF(Size);
      } else if (Index == AFIndex) {
        RegCache.Value[Index] = _LoadAF(Size);
      } else {
        RegCache.Value[Index] = _LoadRegister(Offset, Class, Size);
      }

      RegCache.Cached |= Bit;
    }

    return RegCache.Value[Index];
  }

  RefPair AllocatePair(RegClass Class, IR::OpSize Size) {
    if (Class == RegClass::FPR) {
      return {_AllocateFPR(Size, Size), _AllocateFPR(Size, Size)};
    } else {
      return {_AllocateGPR(false), _AllocateGPR(false)};
    }
  }

  RefPair LoadContextPair_Uncached(RegClass Class, IR::OpSize Size, unsigned Offset) {
    RefPair Values = AllocatePair(Class, Size);
    _LoadContextPair(Size, Class, Offset, Values.Low, Values.High);
    return Values;
  }

  RefPair LoadRegCachePair(uint64_t Offset, uint8_t Index, RegClass Class, IR::OpSize Size) {
    LOGMAN_THROW_A_FMT(Index != DFIndex, "must be pairable");
    LOGMAN_THROW_A_FMT(Size != IR::OpSize::iUnsized, "Invalid size!");

    // Try to load a pair into the cache
    uint64_t Bits = (3ull << (uint64_t)Index);
    const auto SizeInt = IR::OpSizeToSize(Size);
    if (((RegCache.Partial | RegCache.Cached) & Bits) == 0 && ((Offset / SizeInt) < 64)) {
      auto Values = LoadContextPair_Uncached(Class, Size, Offset);
      RegCache.Value[Index] = Values.Low;
      RegCache.Value[Index + 1] = Values.High;
      RegCache.Cached |= Bits;
      if (Size == OpSize::i64Bit) {
        RegCache.Partial |= Bits;
      }
      return Values;
    }

    // Fallback on a pair of loads
    return {
      .Low = LoadRegCache(Offset, Index, Class, Size),
      .High = LoadRegCache(Offset + SizeInt, Index + 1, Class, Size),
    };
  }

  Ref LoadGPR(uint8_t Reg) {
    return LoadRegCache(Reg, GPR0Index + Reg, RegClass::GPR, GetGPROpSize());
  }

  Ref LoadContext(IR::OpSize Size, uint8_t Index) {
    return LoadRegCache(CacheIndexToContextOffset(Index), Index, CacheIndexClass(Index), Size);
  }

  RefPair LoadContextPair(IR::OpSize Size, uint8_t Index) {
    return LoadRegCachePair(CacheIndexToContextOffset(Index), Index, CacheIndexClass(Index), Size);
  }

  Ref LoadContext(uint8_t Index) {
    return LoadContext(CacheIndexToOpSize(Index), Index);
  }

  Ref LoadXMMRegister(uint8_t Reg) {
    return LoadRegCache(Reg, FPR0Index + Reg, RegClass::FPR, GetGuestVectorLength());
  }

  Ref LoadDF() {
    return LoadGPR(DFIndex);
  }

  void StoreContext(uint8_t Index, Ref Value) {
    LOGMAN_THROW_A_FMT(Index < 64, "valid index");
    LOGMAN_THROW_A_FMT(Value != InvalidNode, "storing valid");

    uint64_t Bit = (1ull << (uint64_t)Index);

    RegCache.Value[Index] = Value;
    RegCache.Cached |= Bit;
    RegCache.Written |= Bit;
  }

  void InvalidateHighAVXRegisters() {
    for (size_t i = 0; i < 16; ++i) {
      InvalidateReg(AVXHigh0Index + i);
    }
  }

  void StoreRegister(uint8_t Reg, bool FPR, Ref Value) {
    StoreContext(Reg + (FPR ? FPR0Index : GPR0Index), Value);
  }

  void StoreDF(Ref Value) {
    StoreContext(DFIndex, Value);
  }

  Ref GetRFLAG(unsigned BitOffset, bool Invert = false) {
    if (IsNZCV(BitOffset)) {
      // Handle the CFInverted state internally so GetRFLAG is safe regardless
      // of the invert state. This simplifies the call sites.
      if (BitOffset == X86State::RFLAG_CF_RAW_LOC) {
        Invert ^= CFInverted;
      }

      if (NZCVDirty) {
        auto Value = _Bfe(OpSize::i32Bit, 1, IndexNZCV(BitOffset), GetNZCV());

        if (Invert) {
          return _Xor(OpSize::i32Bit, Value, Constant(1));
        } else {
          return Value;
        }
      } else {
        // Because we explicitly inverted for CF above, we use the unsafe
        // _NZCVSelect rather than the safe CF-aware version.
        return _NZCVSelect01(CondForNZCVBit(BitOffset, Invert));
      }
    } else if (BitOffset == FEXCore::X86State::RFLAG_PF_RAW_LOC) {
      return LoadGPR(Core::CPUState::PF_AS_GREG);
    } else if (BitOffset == FEXCore::X86State::RFLAG_AF_RAW_LOC) {
      return LoadGPR(Core::CPUState::AF_AS_GREG);
    } else if (BitOffset == FEXCore::X86State::RFLAG_DF_RAW_LOC) {
      // Recover the sign bit, it is the logical DF value
      return _Lshr(OpSize::i64Bit, LoadDF(), Constant(63));
    } else {
      return _LoadContextGPR(OpSize::i8Bit, ARRAY_OFFSETOF(Core::CPUState, flags, BitOffset));
    }
  }

  // Returns (DF ? -Size : Size)
  Ref LoadDir(const unsigned Size) {
    return ARef(LoadDF()).Lshl(FEXCore::ilog2(Size)).Ref();
  }

  // Returns DF ? (X - Size) : (X + Size)
  Ref OffsetByDir(Ref X, const unsigned Size) {
    auto Shift = FEXCore::ilog2(Size);

    return _AddShift(OpSize::i64Bit, X, LoadDF(), ShiftType::LSL, Shift);
  }

  // Safe version of NZCVSelect that handles inverted carries automatically.
  Ref NZCVSelect(OpSize OpSize, CondClass Cond, Ref TrueV, Ref FalseV, bool CarryIsInverted = false) {
    switch (Cond) {
    case CondClass::UGE: /* cs */
    case CondClass::ULT: /* cc */
      // Invert the condition to match our expectations.
      if (CarryIsInverted != CFInverted) {
        Cond = (Cond == CondClass::UGE) ? CondClass::ULT : CondClass::UGE;
      }
      break;

    case CondClass::UGT: /* hi */
    case CondClass::ULE: /* ls */
      // No clever optimization we can do here, rectify carry itself.
      RectifyCarryInvert(CarryIsInverted);
      break;

    default:
      // No other condition codes read carry so no need to rectify.
      break;
    }

    return _NZCVSelect(OpSize, Cond, TrueV, FalseV);
  }

  // Compares two floats and sets flags for a COMISS instruction
  void Comiss(IR::OpSize ElementSize, Ref Src1, Ref Src2) {
    // First, set flags according to Arm FCMP.
    HandleNZCVWrite();
    _FCmp(ElementSize, Src1, Src2);
    CFInverted = false;
    ComissFlags();
  }

  // Sets flags for a COMISS instruction
  void ComissFlags() {
    LOGMAN_THROW_A_FMT(!NZCVDirty, "only expected after fcmp");

    // We need to set PF according to the unordered flag. We'd rather do this
    // after axflag, since some impls fuse fcmp+axflag, so we want to do this
    // after. We can recover "unordered" after axflag as (Z && !C), but
    // there's no condition code for this so it would take 2 instructions
    // instead of one, which seems worse than doing 1 op before and breaking
    // the fusion.
    //
    // We set PF to unordered (V), but our PF representation is inverted so we
    // actually set to !V. This is one instruction with the VC cond code.
    Ref V_inv = GetRFLAG(FEXCore::X86State::RFLAG_OF_RAW_LOC, true);
    SetRFLAG<FEXCore::X86State::RFLAG_PF_RAW_LOC>(V_inv);

    // Intel: OF, SF, and AF set to zero
    // AMD:   no mention of OF, SF and AF but actual hardware seems to always zero
    //
    // Zero AF. Note that the comparison sets the raw PF to 0/1 above, so
    // PF[4] is 0 so the XOR with PF will have no effect, so setting the AF
    // byte to zero will indeed zero AF as intended.
    // OF and SF are zeroed:
    // _AXFLAG always produces N=0 (SF), V=0 (OF)
    SetRFLAG<FEXCore::X86State::RFLAG_AF_RAW_LOC>(Constant(0));

    // Convert NZCV from the Arm representation to an eXternal representation
    // that's totally not a euphemism for x86, nuh-uh. But maps to exactly we
    // need, what a coincidence!
    //
    // Our AXFlag emulation on FlagM2-less systems needs V_inv passed.
    _AXFlag(CTX->HostFeatures.SupportsFlagM2 ? Invalid() : V_inv);
    CFInverted = true;
  }

  // Set x87 comparison flags based on the result set by Arm FCMP. Clobbers
  // NZCV on flagm2 platforms.
  void ConvertNZCVToX87() {
    LOGMAN_THROW_A_FMT(NZCVDirty && CachedNZCV, "NZCV must be saved");

    Ref V = _NZCVSelect01(CondForNZCVBit(FEXCore::X86State::RFLAG_OF_RAW_LOC, false));

    if (CTX->HostFeatures.SupportsFlagM2) {
      // Convert to x86 flags, saves us from or'ing after.
      _AXFlag(Invalid());
    }

    // CF is inverted after FCMP
    Ref C = _NZCVSelect01(CondForNZCVBit(FEXCore::X86State::RFLAG_CF_RAW_LOC, true));
    Ref Z = _NZCVSelect01(CondForNZCVBit(FEXCore::X86State::RFLAG_ZF_RAW_LOC, false));

    if (!CTX->HostFeatures.SupportsFlagM2) {
      C = _Or(OpSize::i32Bit, C, V);
      Z = _Or(OpSize::i32Bit, Z, V);
    }

    SetRFLAG<FEXCore::X86State::X87FLAG_C0_LOC>(C);
    SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Constant(0));
    SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(V);
    SetRFLAG<FEXCore::X86State::X87FLAG_C3_LOC>(Z);
  }

  // Helper to store a variable shift and calculate its flags for a variable
  // shift, with correct PF handling.
  void HandleShift(X86Tables::DecodedOp Op, Ref Result, Ref Dest, ShiftType Shift, Ref Src) {

    auto OldPF = GetRFLAG(X86State::RFLAG_PF_RAW_LOC);

    HandleNZCV_RMW();
    CalculatePF(_ShiftFlags(OpSizeFromSrc(Op), Result, Dest, Shift, Src, OldPF, CFInverted));
    StoreResultGPR(Op, Result);
  }

  // Helper to derive Dest by a given builder-using Expression with the opcode
  // replaced with NewOp. Useful for generic building code. Not safe in general.
  // but does the right handling of ImplicitFlagClobber at least and must be
  // used instead of raw Op mutation.
#define DeriveOp(Dest, NewOp, Expr)                \
  if (ImplicitFlagClobber(NewOp)) SaveNZCV(NewOp); \
  auto Dest = (Expr);                              \
  Dest.first->Header.Op = (NewOp)

  // Named constant cache for the current block.
  // Different arrays for sizes 1,2,4,8,16,32.
  Ref CachedNamedVectorConstants[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_MAX][6] {};
  struct IndexNamedVectorMapKey {
    uint32_t Index {};
    FEXCore::IR::IndexNamedVectorConstant NamedIndexedConstant;
    uint8_t log2_size_in_bytes {};
    uint16_t _pad {};

    bool operator==(const IndexNamedVectorMapKey&) const = default;
  };
  struct IndexNamedVectorMapKeyHasher {
    std::size_t operator()(const IndexNamedVectorMapKey& k) const noexcept {
      return XXH3_64bits(&k, sizeof(k));
    }
  };
  fextl::unordered_map<IndexNamedVectorMapKey, Ref, IndexNamedVectorMapKeyHasher> CachedIndexedNamedVectorConstants;

  // Load and cache a named vector constant.
  Ref LoadAndCacheNamedVectorConstant(IR::OpSize Size, FEXCore::IR::NamedVectorConstant NamedConstant) {
    auto log2_size_bytes = FEXCore::ilog2(IR::OpSizeToSize(Size));
    if (CachedNamedVectorConstants[NamedConstant][log2_size_bytes]) {
      return CachedNamedVectorConstants[NamedConstant][log2_size_bytes];
    }

    auto K = _LoadNamedVectorConstant(Size, NamedConstant);
    CachedNamedVectorConstants[NamedConstant][log2_size_bytes] = K;
    return K;
  }
  Ref LoadAndCacheIndexedNamedVectorConstant(IR::OpSize Size, FEXCore::IR::IndexNamedVectorConstant NamedIndexedConstant, uint32_t Index) {
    IndexNamedVectorMapKey Key {
      .Index = Index,
      .NamedIndexedConstant = NamedIndexedConstant,
      .log2_size_in_bytes = FEXCore::ilog2(IR::OpSizeToSize(Size)),
    };
    auto it = CachedIndexedNamedVectorConstants.find(Key);

    if (it != CachedIndexedNamedVectorConstants.end()) {
      return it->second;
    }

    auto K = _LoadNamedVectorIndexedConstant(Size, NamedIndexedConstant, Index);
    CachedIndexedNamedVectorConstants.insert_or_assign(Key, K);
    return K;
  }

  Ref LoadUncachedZeroVector(IR::OpSize Size) {
    return _LoadNamedVectorConstant(Size, IR::NamedVectorConstant::NAMED_VECTOR_ZERO);
  }

  Ref LoadZeroVector(IR::OpSize Size) {
    return LoadAndCacheNamedVectorConstant(Size, IR::NamedVectorConstant::NAMED_VECTOR_ZERO);
  }

  // Reset the named vector constants cache array.
  // These are only cached per block.
  void ClearCachedNamedConstants() {
    memset(CachedNamedVectorConstants, 0, sizeof(CachedNamedVectorConstants));
    CachedIndexedNamedVectorConstants.clear();
  }

  std::optional<CondClass> DecodeNZCVCondition(uint8_t OP);
  Ref SelectCC0All1(uint8_t OP);

  /**
   * @brief Flushes NZCV. Mostly vestigial.
   */
  void CalculateDeferredFlags();

  void ZeroShiftResult(FEXCore::X86Tables::DecodedOp Op) {
    // In the case of zero-rotate, we need to store the destination still to deal with 32-bit semantics.
    const auto Size = OpSizeFromSrc(Op);
    if (Size != OpSize::i32Bit) {
      return;
    }
    auto Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);
    StoreResultGPR(Op, Dest);
  }

  using ZeroShiftFunctionPtr = void (OpDispatchBuilder::*)(FEXCore::X86Tables::DecodedOp Op);

  template<typename F>
  void Calculate_ShiftVariable(FEXCore::X86Tables::DecodedOp Op, Ref Shift, F&& Calculate,
                               std::optional<ZeroShiftFunctionPtr> ZeroShiftResult = std::nullopt) {
    // RCR can call this with constants, so handle that without branching.
    uint64_t Const;
    if (IsValueConstant(WrapNode(Shift), &Const)) {
      if (Const) {
        Calculate();
      } else if (ZeroShiftResult) {
        (this->*(*ZeroShiftResult))(Op);
      }

      return;
    }

    // Otherwise, prepare to branch.
    auto Zero = Constant(0);

    // If the shift is zero, do not touch the flags.
    auto SetBlock = CreateNewCodeBlockAfter(GetCurrentBlock());
    IRPair<IROp_CodeBlock> NextBlock = SetBlock;
    IRPair<IROp_CodeBlock> ZeroShiftBlock;
    if (ZeroShiftResult) {
      ZeroShiftBlock = CreateNewCodeBlockAfter(NextBlock);
      NextBlock = ZeroShiftBlock;
    }
    auto EndBlock = CreateNewCodeBlockAfter(NextBlock);

    ///< Jump to zeroshift block or end block depending on if it was provided.
    IRPair<IROp_CodeBlock> TailHandling = ZeroShiftResult ? ZeroShiftBlock : EndBlock;
    CondJump(Shift, Zero, TailHandling, SetBlock, CondClass::EQ);

    SetCurrentCodeBlock(SetBlock);
    StartNewBlock();
    {
      Calculate();
      Jump(EndBlock);
    }

    if (ZeroShiftResult) {
      SetCurrentCodeBlock(ZeroShiftBlock);
      StartNewBlock();
      {
        (this->*(*ZeroShiftResult))(Op);
        Jump(EndBlock);
      }
    }

    SetCurrentCodeBlock(EndBlock);
    StartNewBlock();
  }

  /**
   * @name These functions are used by the deferred flag handling while it is calculating and storing flags in to RFLAGs.
   * @{ */
  Ref LoadPFRaw(bool Mask, bool Invert);
  Ref LoadAF();
  void FixupAF();
  void SetAFAndFixup(Ref AF);
  Ref CalculateAFForDecimal(Ref A);
  void CalculatePF(Ref Res);
  void CalculateAF(Ref Src1, Ref Src2);

  Ref IncrementByCarry(OpSize OpSize, Ref Src);

  void CalculateOF(IR::OpSize SrcSize, Ref Res, Ref Src1, Ref Src2, bool Sub);
  Ref CalculateFlags_ADC(IR::OpSize SrcSize, Ref Src1, Ref Src2);
  Ref CalculateFlags_SBB(IR::OpSize SrcSize, Ref Src1, Ref Src2);
  Ref CalculateFlags_SUB(IR::OpSize SrcSize, Ref Src1, Ref Src2, bool UpdateCF = true);
  Ref CalculateFlags_ADD(IR::OpSize SrcSize, Ref Src1, Ref Src2, bool UpdateCF = true);
  void CalculateFlags_MUL(IR::OpSize SrcSize, Ref Res, Ref High);
  void CalculateFlags_UMUL(Ref High);
  void CalculateFlags_Logical(IR::OpSize SrcSize, Ref Res);
  void CalculateFlags_ShiftLeftImmediate(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift, bool DoubleWide = false);
  void CalculateFlags_ShiftRightImmediate(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift);
  void CalculateFlags_ShiftRightDoubleImmediate(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift);
  void CalculateFlags_ShiftRightImmediateCommon(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift);
  void CalculateFlags_SignShiftRightImmediate(IR::OpSize SrcSize, Ref Res, Ref Src1, uint64_t Shift);
  void CalculateFlags_ZCNT(IR::OpSize SrcSize, Ref Result);
  /**  @} */

  Ref GetX87Top();
  void SetX87FTW(Ref FTW);
  Ref GetX87FTW_Helper();
  void SetX87Top(Ref Value);

  void ChgStateX87_MMX() override {
    LOGMAN_THROW_A_FMT(MMXState == MMXState_X87, "Expected state to be x87");
    _StackForceSlow();
    SetX87Top(Constant(0)); // top reset to zero
    _StoreContextGPR(OpSize::i8Bit, Constant(0xFFFFUL), offsetof(FEXCore::Core::CPUState, AbridgedFTW));
    MMXState = MMXState_MMX;
  }

  void ChgStateMMX_X87() override {
    LOGMAN_THROW_A_FMT(MMXState == MMXState_MMX, "Expected state to be MMX");
    // The opcode dispatcher register cache is used for MMX, but the x87 pass register cache is used for x87, spill to
    // context to ensure coherence.
    FlushRegisterCache(false, true);
    // We explicitly initialize to x87 state in StartNewBlock.
    // So if we ever change this to do something else, we need to
    // make sure that we consider if we need to explicitly set it there.
    MMXState = MMXState_X87;
  }

  bool DestIsLockedMem(FEXCore::X86Tables::DecodedOp Op) const {
    return DestIsMem(Op) && (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;
  }

  bool DestIsMem(FEXCore::X86Tables::DecodedOp Op) const {
    return !Op->Dest.IsGPR();
  }

  void CreateJumpBlocks(const fextl::vector<FEXCore::Frontend::Decoder::DecodedBlocks>* Blocks);
  bool BlockSetRIP {false};

  bool Multiblock {};
  bool Is64BitMode {};
  uint64_t Entry {};

  // Set if mono hacks are enabled and the current block is the mono callsite backpatcher, in which case the
  // XCHG ops that would patch code are replaced with a hook that performs the write and manually invalidates
  // the target address.
  bool IsMonoBackpatcherBlock {false};
  IROp_IRHeader* CurrentHeader {};

  [[nodiscard]]
  bool IsTSOEnabled(RegClass Class) const {
    if (ForceTSO == ForceTSOMode::ForceEnabled) {
      return true;
    } else if (ForceTSO == ForceTSOMode::ForceDisabled) {
      return false;
    } else if (Class == RegClass::FPR) {
      return CTX->IsVectorAtomicTSOEnabled();
    } else {
      return CTX->IsAtomicTSOEnabled();
    }
  }

  Ref _StoreMemAutoTSO(RegClass Class, OpSize Size, Ref Addr, Ref Value, OpSize Align = OpSize::i8Bit) {
    if (IsTSOEnabled(Class)) {
      return _StoreMemTSO(Class, Size, Value, Addr, Invalid(), Align, MemOffsetType::SXTX, 1);
    } else {
      return _StoreMem(Class, Size, Value, Addr, Invalid(), Align, MemOffsetType::SXTX, 1);
    }
  }
  Ref _StoreMemGPRAutoTSO(OpSize Size, Ref Addr, Ref Value, OpSize Align = OpSize::i8Bit) {
    return _StoreMemAutoTSO(RegClass::GPR, Size, Addr, Value, Align);
  }
  Ref _StoreMemFPRAutoTSO(OpSize Size, Ref Addr, Ref Value, OpSize Align = OpSize::i8Bit) {
    return _StoreMemAutoTSO(RegClass::FPR, Size, Addr, Value, Align);
  }

  Ref _LoadMemAutoTSO(RegClass Class, OpSize Size, Ref ssa0, OpSize Align = OpSize::i8Bit) {
    if (IsTSOEnabled(Class)) {
      return _LoadMemTSO(Class, Size, ssa0, Invalid(), Align, MemOffsetType::SXTX, 1);
    } else {
      return _LoadMem(Class, Size, ssa0, Invalid(), Align, MemOffsetType::SXTX, 1);
    }
  }
  Ref _LoadMemGPRAutoTSO(OpSize Size, Ref ssa0, OpSize Align = OpSize::i8Bit) {
    return _LoadMemAutoTSO(RegClass::GPR, Size, ssa0, Align);
  }
  Ref _LoadMemFPRAutoTSO(OpSize Size, Ref ssa0, OpSize Align = OpSize::i8Bit) {
    return _LoadMemAutoTSO(RegClass::FPR, Size, ssa0, Align);
  }

  Ref _LoadMemAutoTSO(RegClass Class, OpSize Size, const AddressMode& A, OpSize Align = OpSize::i8Bit) {
    const bool AtomicTSO = IsTSOEnabled(Class) && !A.NonTSO;
    const auto B = SelectAddressMode(this, A, GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, AtomicTSO, Class != RegClass::GPR, Size);

    if (AtomicTSO) {
      return _LoadMemTSO(Class, Size, B.Base, B.Index, Align, B.IndexType, B.IndexScale);
    } else {
      return _LoadMem(Class, Size, B.Base, B.Index, Align, B.IndexType, B.IndexScale);
    }
  }
  Ref _LoadMemGPRAutoTSO(OpSize Size, const AddressMode& A, OpSize Align = OpSize::i8Bit) {
    return _LoadMemAutoTSO(RegClass::GPR, Size, A, Align);
  }
  Ref _LoadMemFPRAutoTSO(OpSize Size, const AddressMode& A, OpSize Align = OpSize::i8Bit) {
    return _LoadMemAutoTSO(RegClass::FPR, Size, A, Align);
  }

  AddressMode SelectPairAddressMode(AddressMode A, IR::OpSize Size) {
    LOGMAN_THROW_A_FMT(Size != IR::OpSize::iUnsized, "Invalid size!");
    const auto SizeInt = IR::OpSizeToSize(Size);
    AddressMode Out {};

    signed OffsetEl = A.Offset / SizeInt;
    if ((A.Offset % SizeInt) == 0 && OffsetEl >= -64 && OffsetEl < 64) {
      Out.Offset = A.Offset;
      A.Offset = 0;
    }

    Out.Base = LoadEffectiveAddress(this, A, GetGPROpSize(), true, false);
    return Out;
  }


  RefPair LoadMemPair(RegClass Class, OpSize Size, Ref Base, uint32_t Offset) {
    RefPair Values = AllocatePair(Class, Size);
    _LoadMemPair(Class, Size, Base, Offset, Values.Low, Values.High);
    return Values;
  }
  RefPair LoadMemPairFPR(OpSize Size, Ref Base, uint32_t Offset) {
    return LoadMemPair(RegClass::FPR, Size, Base, Offset);
  }

  RefPair _LoadMemPairAutoTSO(RegClass Class, OpSize Size, const AddressMode& A, OpSize Align = OpSize::i8Bit) {
    const bool AtomicTSO = IsTSOEnabled(Class) && !A.NonTSO;

    // Use ldp if possible, otherwise fallback on two loads.
    if (!AtomicTSO && !A.Segment && Size >= OpSize::i32Bit && Size <= OpSize::i128Bit) {
      const auto B = SelectPairAddressMode(A, Size);
      return LoadMemPair(Class, Size, B.Base, B.Offset);
    }

    AddressMode HighA = A;
    HighA.Offset += 16;

    return {
      .Low = _LoadMemAutoTSO(Class, Size, A, Align),
      .High = _LoadMemAutoTSO(Class, Size, HighA, Align),
    };
  }
  RefPair _LoadMemPairFPRAutoTSO(OpSize Size, const AddressMode& A, OpSize Align = OpSize::i8Bit) {
    return _LoadMemPairAutoTSO(RegClass::FPR, Size, A, Align);
  }

  Ref _StoreMemAutoTSO(RegClass Class, OpSize Size, const AddressMode& A, Ref Value, OpSize Align = OpSize::i8Bit) {
    const bool AtomicTSO = IsTSOEnabled(Class) && !A.NonTSO;
    const auto B = SelectAddressMode(this, A, GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, AtomicTSO, Class != RegClass::GPR, Size);

    if (AtomicTSO) {
      return _StoreMemTSO(Class, Size, Value, B.Base, B.Index, Align, B.IndexType, B.IndexScale);
    } else {
      return _StoreMem(Class, Size, Value, B.Base, B.Index, Align, B.IndexType, B.IndexScale);
    }
  }
  Ref _StoreMemGPRAutoTSO(OpSize Size, const AddressMode& A, Ref Value, OpSize Align = OpSize::i8Bit) {
    return _StoreMemAutoTSO(RegClass::GPR, Size, A, Value, Align);
  }
  Ref _StoreMemFPRAutoTSO(OpSize Size, const AddressMode& A, Ref Value, OpSize Align = OpSize::i8Bit) {
    return _StoreMemAutoTSO(RegClass::FPR, Size, A, Value, Align);
  }

  void _StoreMemPairAutoTSO(RegClass Class, OpSize Size, const AddressMode& A, Ref Value1, Ref Value2, OpSize Align = OpSize::i8Bit) {
    const auto SizeInt = IR::OpSizeToSize(Size);
    const bool AtomicTSO = IsTSOEnabled(Class) && !A.NonTSO;

    // Use stp if possible, otherwise fallback on two stores.
    if (!AtomicTSO && !A.Segment && Size >= OpSize::i32Bit && Size <= OpSize::i128Bit) {
      const auto B = SelectPairAddressMode(A, Size);
      _StoreMemPair(Class, Size, Value1, Value2, B.Base, B.Offset);
    } else {
      auto B = A;

      _StoreMemAutoTSO(Class, Size, B, Value1, OpSize::i8Bit);
      B.Offset += SizeInt;
      _StoreMemAutoTSO(Class, Size, B, Value2, OpSize::i8Bit);
    }
  }
  void _StoreMemPairFPRAutoTSO(OpSize Size, const AddressMode& A, Ref Value1, Ref Value2, OpSize Align = OpSize::i8Bit) {
    return _StoreMemPairAutoTSO(RegClass::FPR, Size, A, Value1, Value2, Align);
  }

  Ref Pop(IR::OpSize Size, Ref SP_RMW) {
    Ref Value = _AllocateGPR(false);
    _Pop(Size, SP_RMW, Value);
    return Value;
  }

  Ref Pop(IR::OpSize Size) {
    Ref SP = _RMWHandle(LoadGPRRegister(X86State::REG_RSP));
    Ref Value = _AllocateGPR(false);

    _Pop(Size, SP, Value);

    // Store the new stack pointer
    StoreGPRRegister(X86State::REG_RSP, SP);
    return Value;
  }

  Ref VZeroExtendOperand(OpSize Size, X86Tables::DecodedOperand Op, Ref Value) {
    bool IsMMX = Op.IsGPR() && Op.Data.GPR.GPR >= X86State::REG_MM_0;
    bool AlreadyExtended = Op.IsGPRDirect() || Op.IsGPRIndirect() || IsMMX;

    return AlreadyExtended ? Value : _VMov(Size, Value);
  }

  void Push(IR::OpSize Size, Ref Value) {
    auto OldSP = LoadGPRRegister(X86State::REG_RSP);
    auto NewSP = _Push(GetGPROpSize(), Size, Value, OldSP);
    StoreGPRRegister(X86State::REG_RSP, NewSP);
    FlushRegisterCache();
  }

  struct ArithRef {
    IREmitter* E {};
    bool IsConstant {};
    union {
      Ref R {};
      uint64_t C;
    };

    ArithRef() {}

    ArithRef(IREmitter* IREmit, Ref Reference)
      : E(IREmit)
      , IsConstant(false)
      , R(Reference) {}

    ArithRef(IREmitter* IREmit, uint64_t K)
      : E(IREmit)
      , IsConstant(true)
      , C(K) {}

    ArithRef Neg() {
      return IsConstant ? ArithRef(E, -C) : ArithRef(E, E->_Neg(OpSize::i64Bit, R));
    }

    ArithRef And(uint64_t K) {
      return IsConstant ? ArithRef(E, C & K) : ArithRef(E, E->_And(OpSize::i64Bit, R, E->Constant(K)));
    }

    ArithRef Presub(uint64_t K) {
      return IsConstant ? ArithRef(E, K - C) : ArithRef(E, E->Sub(OpSize::i64Bit, E->Constant(K), R));
    }

    ArithRef Lshl(uint64_t Shift) {
      if (Shift == 0) {
        return *this;
      } else if (IsConstant) {
        return ArithRef(E, C << Shift);
      } else {
        return ArithRef(E, E->_Lshl(OpSize::i64Bit, R, E->Constant(Shift)));
      }
    }

    ArithRef Bfe(unsigned Start, unsigned Size) {
      if (IsConstant) {
        return ArithRef(E, (C >> Start) & ((1ull << Size) - 1));
      } else {
        return ArithRef(E, E->_Bfe(OpSize::i64Bit, Size, Start, R));
      }
    }

    ArithRef Sbfe(unsigned Start, unsigned Size) {
      if (IsConstant) {
        uint64_t SourceMask = Size == 64 ? ~0ULL : ((1ULL << Size) - 1);
        SourceMask <<= Start;

        int64_t NewConstant = (C & SourceMask) >> Start;
        NewConstant <<= 64 - Size;
        NewConstant >>= 64 - Size;

        return ArithRef(E, NewConstant);
      } else {
        return ArithRef(E, E->_Sbfe(OpSize::i64Bit, Size, Start, R));
      }
    }

    Ref BfiInto(Ref Bitfield, unsigned Start, unsigned Size) {
      if (IsConstant && (Size > 0 && Size < 64)) {
        uint64_t SourceMask = (1ULL << Size) - 1;
        uint64_t SourceMaskShifted = SourceMask << Start;

        if (C == 0) {
          return E->_And(OpSize::i64Bit, Bitfield, E->_InlineConstant(~SourceMaskShifted));
        } else if (C == SourceMask) {
          return E->_Or(OpSize::i64Bit, Bitfield, E->_InlineConstant(SourceMaskShifted));
        }
      }

      if (IsConstant) {
        return E->_Bfi(OpSize::i64Bit, Size, Start, Bitfield, E->Constant(C));
      } else {
        return E->_Bfi(OpSize::i64Bit, Size, Start, Bitfield, R);
      }
    }

    ArithRef MaskBit(OpSize Size) {
      if (IsConstant) {
        uint64_t ShiftMask = Size == OpSize::i64Bit ? 63 : 31;
        uint64_t Result = 1ull << (C & ShiftMask);
        if (ShiftMask == 31) {
          Result &= ((1ull << 32) - 1);
        }

        return ArithRef(E, Result);
      } else {
        return ArithRef(E, E->_Lshl(Size, E->Constant(1), R));
      }
    }

    Ref Ref() {
      return IsConstant ? E->Constant(C) : R;
    }

    bool IsDefinitelyZero() const {
      return IsConstant && C == 0;
    }
  };

  ArithRef ARef(Ref R) {
    uint64_t C;

    if (IsValueConstant(WrapNode(R), &C)) {
      return ARef(C);
    } else {
      return ArithRef(this, R);
    }
  }

  ArithRef ARef(uint64_t K) {
    return ArithRef(this, K);
  }

  ///< Segment telemetry tracking
  uint32_t SegmentsNeedReadCheck {~0U};
  void CheckLegacySegmentWrite(Ref NewNode, uint32_t SegmentReg);
  void CheckLegacySegmentRead(Ref NewNode, uint32_t SegmentReg);
};

constexpr inline void InstallToTable(auto& FinalTable, const auto& LocalTable) {
  for (const auto& Op : LocalTable) {
    auto OpNum = Op.Op;
    auto Dispatcher = Op.Ptr;
    for (uint8_t i = 0; i < Op.Count; ++i) {
      auto& TableOp = FinalTable[OpNum + i];
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
      if (TableOp.OpcodeDispatcher.OpDispatch) {
        ERROR_AND_DIE_FMT("Duplicate Entry {}", TableOp.Name);
      }
#endif

      TableOp.OpcodeDispatcher.OpDispatch = Dispatcher;
    }
  }
}

} // namespace FEXCore::IR