steam-frame/FEX-Emu--FEX
1
0 Fork 0
mirror of https://github.com/FEX-Emu/FEX.git synced 2026-10-06 23:00:17 +02:00
Code Issues Packages Projects Releases 66 Wiki Activity
Files
151fc5e97fe540e355f8bb9d07597f9da6e2cf6b
FEX-Emu--FEX/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp
T
Paulo Matos 0bccb1ece5 Ensure predicate cache is reset when control flow leaves block 
Whenever the control float leaves the block, it might clobber the
predicate register so we reset the cache whenever that happens.

Fixes #4264
2025-01-27 20:12:43 +01:00

6008 lines
245 KiB
C++
Raw Blame History

// SPDX-License-Identifier: MIT
/*
$info$
tags: frontend|x86-to-ir, opcodes|dispatcher-implementations
desc: Handles x86/64 ops to IR, no-pf opt, local-flags opt
$end_info$
*/
#include "FEXCore/Core/HostFeatures.h"
#include "FEXCore/Utils/Telemetry.h"
#include "Interface/Context/Context.h"
#include "Interface/Core/OpcodeDispatcher.h"
#include "Interface/Core/X86Tables/X86Tables.h"
#include "Interface/IR/IR.h"
#include "Interface/IR/IREmitter.h"
#include <FEXCore/Config/Config.h>
#include <FEXCore/Core/Context.h>
#include <FEXCore/Core/CoreState.h>
#include <FEXCore/Core/X86Enums.h>
#include <FEXCore/HLE/SyscallHandler.h>
#include <FEXCore/IR/IR.h>
#include <FEXCore/Utils/EnumUtils.h>
#include <FEXCore/Utils/LogManager.h>
#include <FEXHeaderUtils/BitUtils.h>
#include <algorithm>
#include <array>
#include <bit>
#include <cstdint>
#include <tuple>
namespace FEXCore::IR {
using X86Tables::OpToIndex;
#define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op
void OpDispatchBuilder::SyscallOp(OpcodeArgs, bool IsSyscallInst) {
constexpr size_t SyscallArgs = 7;
using SyscallArray = std::array<uint64_t, SyscallArgs>;
size_t NumArguments {};
const SyscallArray* GPRIndexes {};
static constexpr SyscallArray GPRIndexes_64 = {
FEXCore::X86State::REG_RAX, FEXCore::X86State::REG_RDI, FEXCore::X86State::REG_RSI, FEXCore::X86State::REG_RDX,
FEXCore::X86State::REG_R10, FEXCore::X86State::REG_R8, FEXCore::X86State::REG_R9,
};
static constexpr SyscallArray GPRIndexes_32 = {
FEXCore::X86State::REG_RAX, FEXCore::X86State::REG_RBX, FEXCore::X86State::REG_RCX, FEXCore::X86State::REG_RDX,
FEXCore::X86State::REG_RSI, FEXCore::X86State::REG_RDI, FEXCore::X86State::REG_RBP,
};
SyscallFlags DefaultSyscallFlags = FEXCore::IR::SyscallFlags::DEFAULT;
const auto OSABI = CTX->SyscallHandler->GetOSABI();
if (OSABI == FEXCore::HLE::SyscallOSABI::OS_LINUX64) {
NumArguments = GPRIndexes_64.size();
GPRIndexes = &GPRIndexes_64;
} else if (OSABI == FEXCore::HLE::SyscallOSABI::OS_LINUX32) {
NumArguments = GPRIndexes_32.size();
GPRIndexes = &GPRIndexes_32;
} else if (OSABI == FEXCore::HLE::SyscallOSABI::OS_GENERIC) {
// All registers will be spilled before the syscall and filled afterwards so no JIT-side argument handling is necessary.
NumArguments = 0;
GPRIndexes = nullptr;
DefaultSyscallFlags = FEXCore::IR::SyscallFlags::NORETURNEDRESULT;
} else {
LogMan::Msg::DFmt("Unhandled OSABI syscall");
}
// Calculate flags early.
CalculateDeferredFlags();
const auto GPRSize = CTX->GetGPROpSize();
auto NewRIP = GetRelocatedPC(Op, -Op->InstSize);
_StoreContext(GPRSize, GPRClass, NewRIP, offsetof(FEXCore::Core::CPUState, rip));
Ref Arguments[SyscallArgs] {
InvalidNode, InvalidNode, InvalidNode, InvalidNode, InvalidNode, InvalidNode, InvalidNode,
};
for (size_t i = 0; i < NumArguments; ++i) {
Arguments[i] = LoadGPRRegister(GPRIndexes->at(i));
}
if (IsSyscallInst) {
// If this is the `Syscall` instruction rather than `int 0x80` then we need to do some additional work.
// RCX = RIP after this instruction
// R11 = EFlags
// Calculate flags.
CalculateDeferredFlags();
auto RFLAG = GetPackedRFLAG();
StoreGPRRegister(X86State::REG_R11, RFLAG, OpSize::i64Bit);
auto RIPAfterInst = GetRelocatedPC(Op);
StoreGPRRegister(X86State::REG_RCX, RIPAfterInst, OpSize::i64Bit);
}
FlushRegisterCache();
auto SyscallOp = _Syscall(Arguments[0], Arguments[1], Arguments[2], Arguments[3], Arguments[4], Arguments[5], Arguments[6], DefaultSyscallFlags);
if ((DefaultSyscallFlags & FEXCore::IR::SyscallFlags::NORETURNEDRESULT) != FEXCore::IR::SyscallFlags::NORETURNEDRESULT) {
StoreGPRRegister(X86State::REG_RAX, SyscallOp);
}
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_BLOCK_END) {
// RIP could have been updated after coming back from the Syscall.
NewRIP = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, rip));
ExitFunction(NewRIP);
}
}
void OpDispatchBuilder::ThunkOp(OpcodeArgs) {
const auto GPRSize = CTX->GetGPROpSize();
uint8_t* sha256 = (uint8_t*)(Op->PC + 2);
if (CTX->Config.Is64BitMode) {
// x86-64 ABI puts the function argument in RDI
Thunk(LoadGPRRegister(X86State::REG_RDI), *reinterpret_cast<SHA256Sum*>(sha256));
} else {
// x86 fastcall ABI puts the function argument in ECX
Thunk(LoadGPRRegister(X86State::REG_RCX), *reinterpret_cast<SHA256Sum*>(sha256));
}
auto NewRIP = Pop(GPRSize);
// Store the new RIP
ExitFunction(NewRIP);
BlockSetRIP = true;
}
void OpDispatchBuilder::LEAOp(OpcodeArgs) {
// LEA specifically ignores segment prefixes
const auto SrcSize = OpSizeFromSrc(Op);
if (CTX->Config.Is64BitMode) {
const auto DstSize = X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0) == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? OpSize::i16Bit :
X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0) == X86Tables::DecodeFlags::FLAG_WIDENING_SIZE_LAST ? OpSize::i64Bit :
OpSize::i32Bit;
auto Src = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], SrcSize, Op->Flags, {.LoadData = false, .AllowUpperGarbage = SrcSize > DstSize});
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Src, DstSize, OpSize::iInvalid);
} else {
const auto DstSize =
X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0) == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? OpSize::i16Bit : OpSize::i32Bit;
auto Src = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], SrcSize, Op->Flags, {.LoadData = false, .AllowUpperGarbage = SrcSize > DstSize});
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Src, DstSize, OpSize::iInvalid);
}
}
void OpDispatchBuilder::NOPOp(OpcodeArgs) {}
void OpDispatchBuilder::RETOp(OpcodeArgs) {
const auto GPRSize = CTX->GetGPROpSize();
// ABI Optimization: Flags don't survive calls or rets
if (CTX->Config.ABILocalFlags) {
_InvalidateFlags(~0UL); // all flags
InvalidatePF_AF();
}
Ref SP = _RMWHandle(LoadGPRRegister(X86State::REG_RSP));
Ref NewRIP = Pop(GPRSize, SP);
if (Op->OP == 0xC2) {
auto Offset = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
SP = _Add(GPRSize, SP, Offset);
}
// Store the new stack pointer
StoreGPRRegister(X86State::REG_RSP, SP);
// Store the new RIP
ExitFunction(NewRIP);
BlockSetRIP = true;
}
/*
stack contains:
Size of each member is 64-bit, 32-bit, or 16-bit depending on operating size
RIP
CS
EFLAGS
RSP
SS
*/
void OpDispatchBuilder::IRETOp(OpcodeArgs) {
// Operand Size override unsupported!
if ((Op->Flags & X86Tables::DecodeFlags::FLAG_OPERAND_SIZE) != 0) {
LogMan::Msg::EFmt("IRET only implemented for 64bit and 32bit sizes");
DecodeFailure = true;
return;
}
const auto GPRSize = CTX->GetGPROpSize();
Ref SP = _RMWHandle(LoadGPRRegister(X86State::REG_RSP));
// RIP (64/32/16 bits)
auto NewRIP = Pop(GPRSize, SP);
// CS (lower 16 used)
auto NewSegmentCS = Pop(GPRSize, SP);
_StoreContext(OpSize::i16Bit, GPRClass, NewSegmentCS, offsetof(FEXCore::Core::CPUState, cs_idx));
UpdatePrefixFromSegment(NewSegmentCS, FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX);
// eflags (lower 16 used)
SetPackedRFLAG(false, Pop(GPRSize, SP));
if (CTX->Config.Is64BitMode) {
// RSP and SS only happen in 64-bit mode or if this is a CPL mode jump!
// FEX doesn't support a CPL mode switch, so don't need to worry about this on 32-bit
StoreGPRRegister(X86State::REG_RSP, Pop(GPRSize, SP));
// ss
auto NewSegmentSS = Pop(GPRSize, SP);
_StoreContext(OpSize::i16Bit, GPRClass, NewSegmentSS, offsetof(FEXCore::Core::CPUState, ss_idx));
UpdatePrefixFromSegment(NewSegmentSS, FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX);
} else {
// Store the stack in 32-bit mode
StoreGPRRegister(X86State::REG_RSP, SP);
}
ExitFunction(NewRIP);
BlockSetRIP = true;
}
void OpDispatchBuilder::CallbackReturnOp(OpcodeArgs) {
const auto GPRSize = CTX->GetGPROpSize();
// Store the new RIP
_CallbackReturn();
auto NewRIP = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, rip));
// This ExitFunction won't actually get hit but needs to exist
ExitFunction(NewRIP);
BlockSetRIP = true;
}
void OpDispatchBuilder::SecondaryALUOp(OpcodeArgs) {
FEXCore::IR::IROps IROp, AtomicIROp;
#define OPD(group, prefix, Reg) (((group - FEXCore::X86Tables::TYPE_GROUP_1) << 6) | (prefix) << 3 | (Reg))
switch (Op->OP) {
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 0):
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 0):
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 0):
IROp = FEXCore::IR::IROps::OP_ADD;
AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHADD;
break;
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 1):
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 1):
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 1):
IROp = FEXCore::IR::IROps::OP_OR;
AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHOR;
break;
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 4):
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 4):
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 4):
IROp = FEXCore::IR::IROps::OP_ANDWITHFLAGS;
AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHAND;
break;
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 5):
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 5):
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 5):
IROp = FEXCore::IR::IROps::OP_SUB;
AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHSUB;
break;
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 6):
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 6):
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 6):
IROp = FEXCore::IR::IROps::OP_XOR;
AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHXOR;
break;
default:
IROp = FEXCore::IR::IROps::OP_LAST;
AtomicIROp = FEXCore::IR::IROps::OP_LAST;
LOGMAN_MSG_A_FMT("Unknown ALU Op: 0x{:x}", Op->OP);
break;
};
#undef OPD
ALUOp(Op, IROp, AtomicIROp, 1);
}
void OpDispatchBuilder::ADCOp(OpcodeArgs, uint32_t SrcIndex) {
// Calculate flags early.
CalculateDeferredFlags();
Ref Src = LoadSource(GPRClass, Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true});
const auto Size = OpSizeFromDst(Op);
const auto OpSize = std::max(OpSize::i32Bit, Size);
Ref Before {};
if (DestIsLockedMem(Op)) {
auto ALUOp = IncrementByCarry(OpSize, Src);
HandledLock = true;
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
Before = _AtomicFetchAdd(Size, ALUOp, DestMem);
} else {
Before = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
}
Ref Result;
if (!DestIsLockedMem(Op) && Op->Src[SrcIndex].IsLiteral() && Op->Src[SrcIndex].Literal() == 0 && Size >= OpSize::i32Bit) {
HandleNZCV_RMW();
RectifyCarryInvert(true);
Result = _AdcZeroWithFlags(OpSize, Before);
SetRFLAG<FEXCore::X86State::RFLAG_AF_RAW_LOC>(Before);
CalculatePF(Result);
CFInverted = false;
} else {
Result = CalculateFlags_ADC(Size, Before, Src);
}
if (!DestIsLockedMem(Op)) {
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
}
}
void OpDispatchBuilder::SBBOp(OpcodeArgs, uint32_t SrcIndex) {
// Calculate flags early.
CalculateDeferredFlags();
Ref Src = LoadSource(GPRClass, Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true});
const auto Size = OpSizeFromDst(Op);
const auto OpSize = std::max(OpSize::i32Bit, Size);
Ref Result {};
Ref Before {};
if (DestIsLockedMem(Op)) {
HandledLock = true;
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
auto SrcPlusCF = IncrementByCarry(OpSize, Src);
Before = _AtomicFetchSub(Size, SrcPlusCF, DestMem);
} else {
Before = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
}
Result = CalculateFlags_SBB(Size, Before, Src);
if (!DestIsLockedMem(Op)) {
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
}
}
void OpDispatchBuilder::SALCOp(OpcodeArgs) {
CalculateDeferredFlags();
auto Result = NZCVSelect(OpSize::i32Bit, {COND_UGE} /* CF = 1 */, _InlineConstant(0xffffffff), _InlineConstant(0));
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
}
void OpDispatchBuilder::PUSHOp(OpcodeArgs) {
const auto Size = OpSizeFromSrc(Op);
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
Push(Size, Src);
FlushRegisterCache();
}
void OpDispatchBuilder::PUSHREGOp(OpcodeArgs) {
const auto Size = OpSizeFromSrc(Op);
Ref Src = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
Push(Size, Src);
FlushRegisterCache();
}
void OpDispatchBuilder::PUSHAOp(OpcodeArgs) {
// 32bit only
const auto Size = OpSizeFromSrc(Op);
auto OldSP = LoadGPRRegister(X86State::REG_RSP);
// PUSHA order:
// Tmp = SP
// push EAX
// push ECX
// push EDX
// push EBX
// push Tmp
// push EBP
// push ESI
// push EDI
Ref Src {};
Ref NewSP = OldSP;
const auto GPRSize = CTX->GetGPROpSize();
Src = LoadGPRRegister(X86State::REG_RAX);
NewSP = _Push(GPRSize, Size, Src, NewSP);
Src = LoadGPRRegister(X86State::REG_RCX);
NewSP = _Push(GPRSize, Size, Src, NewSP);
Src = LoadGPRRegister(X86State::REG_RDX);
NewSP = _Push(GPRSize, Size, Src, NewSP);
Src = LoadGPRRegister(X86State::REG_RBX);
NewSP = _Push(GPRSize, Size, Src, NewSP);
// Push old-sp
NewSP = _Push(GPRSize, Size, OldSP, NewSP);
Src = LoadGPRRegister(X86State::REG_RBP);
NewSP = _Push(GPRSize, Size, Src, NewSP);
Src = LoadGPRRegister(X86State::REG_RSI);
NewSP = _Push(GPRSize, Size, Src, NewSP);
Src = LoadGPRRegister(X86State::REG_RDI);
NewSP = _Push(GPRSize, Size, Src, NewSP);
// Store the new stack pointer
StoreGPRRegister(X86State::REG_RSP, NewSP, OpSize::i32Bit);
FlushRegisterCache();
}
void OpDispatchBuilder::PUSHSegmentOp(OpcodeArgs, uint32_t SegmentReg) {
const auto SrcSize = OpSizeFromSrc(Op);
const auto DstSize = OpSizeFromDst(Op);
Ref Src {};
if (!CTX->Config.Is64BitMode()) {
switch (SegmentReg) {
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
Src = _LoadContext(SrcSize, GPRClass, offsetof(FEXCore::Core::CPUState, es_idx));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
Src = _LoadContext(SrcSize, GPRClass, offsetof(FEXCore::Core::CPUState, cs_idx));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
Src = _LoadContext(SrcSize, GPRClass, offsetof(FEXCore::Core::CPUState, ss_idx));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
Src = _LoadContext(SrcSize, GPRClass, offsetof(FEXCore::Core::CPUState, ds_idx));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:
Src = _LoadContext(SrcSize, GPRClass, offsetof(FEXCore::Core::CPUState, fs_idx));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:
Src = _LoadContext(SrcSize, GPRClass, offsetof(FEXCore::Core::CPUState, gs_idx));
break;
default: FEX_UNREACHABLE;
}
} else {
switch (SegmentReg) {
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
Src = _LoadContext(SrcSize, GPRClass, offsetof(FEXCore::Core::CPUState, es_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
Src = _LoadContext(SrcSize, GPRClass, offsetof(FEXCore::Core::CPUState, cs_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
Src = _LoadContext(SrcSize, GPRClass, offsetof(FEXCore::Core::CPUState, ss_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
Src = _LoadContext(SrcSize, GPRClass, offsetof(FEXCore::Core::CPUState, ds_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:
Src = _LoadContext(SrcSize, GPRClass, offsetof(FEXCore::Core::CPUState, fs_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:
Src = _LoadContext(SrcSize, GPRClass, offsetof(FEXCore::Core::CPUState, gs_cached));
break;
default: FEX_UNREACHABLE;
}
}
// Store our value to the new stack location
// AMD hardware zexts segment selector to 32bit
// Intel hardware inserts segment selector
Push(DstSize, Src);
}
void OpDispatchBuilder::POPOp(OpcodeArgs) {
Ref Value = Pop(OpSizeFromSrc(Op));
StoreResult(GPRClass, Op, Value, OpSize::iInvalid);
}
void OpDispatchBuilder::POPAOp(OpcodeArgs) {
// 32bit only
const auto Size = OpSizeFromSrc(Op);
Ref SP = _RMWHandle(LoadGPRRegister(X86State::REG_RSP));
StoreGPRRegister(X86State::REG_RDI, Pop(Size, SP), Size);
StoreGPRRegister(X86State::REG_RSI, Pop(Size, SP), Size);
StoreGPRRegister(X86State::REG_RBP, Pop(Size, SP), Size);
// Skip loading RSP because it'll be correct at the end
SP = _RMWHandle(_Add(OpSize::i64Bit, SP, _InlineConstant(IR::OpSizeToSize(Size))));
StoreGPRRegister(X86State::REG_RBX, Pop(Size, SP), Size);
StoreGPRRegister(X86State::REG_RDX, Pop(Size, SP), Size);
StoreGPRRegister(X86State::REG_RCX, Pop(Size, SP), Size);
StoreGPRRegister(X86State::REG_RAX, Pop(Size, SP), Size);
// Store the new stack pointer
StoreGPRRegister(X86State::REG_RSP, SP);
}
void OpDispatchBuilder::POPSegmentOp(OpcodeArgs, uint32_t SegmentReg) {
const auto SrcSize = OpSizeFromSrc(Op);
const auto DstSize = OpSizeFromDst(Op);
auto NewSegment = Pop(SrcSize);
switch (SegmentReg) {
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
_StoreContext(DstSize, GPRClass, NewSegment, offsetof(FEXCore::Core::CPUState, es_idx));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
_StoreContext(DstSize, GPRClass, NewSegment, offsetof(FEXCore::Core::CPUState, cs_idx));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
// Unset the 'active' bit in the packed TF, skipping the single step exception after this instruction
SetRFLAG<FEXCore::X86State::RFLAG_TF_RAW_LOC>(_And(OpSize::i32Bit, GetRFLAG(FEXCore::X86State::RFLAG_TF_RAW_LOC), _Constant(1)));
_StoreContext(DstSize, GPRClass, NewSegment, offsetof(FEXCore::Core::CPUState, ss_idx));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
_StoreContext(DstSize, GPRClass, NewSegment, offsetof(FEXCore::Core::CPUState, ds_idx));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:
_StoreContext(DstSize, GPRClass, NewSegment, offsetof(FEXCore::Core::CPUState, fs_idx));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:
_StoreContext(DstSize, GPRClass, NewSegment, offsetof(FEXCore::Core::CPUState, gs_idx));
break;
default: break; // Do nothing
}
UpdatePrefixFromSegment(NewSegment, SegmentReg);
}
void OpDispatchBuilder::LEAVEOp(OpcodeArgs) {
// First we move RBP in to RSP and then behave effectively like a pop
auto SP = _RMWHandle(LoadGPRRegister(X86State::REG_RBP));
auto NewGPR = Pop(OpSizeFromSrc(Op), SP);
// Store the new stack pointer
StoreGPRRegister(X86State::REG_RSP, SP);
// Store what we loaded to RBP
StoreGPRRegister(X86State::REG_RBP, NewGPR);
}
void OpDispatchBuilder::CALLOp(OpcodeArgs) {
const auto GPRSize = CTX->GetGPROpSize();
BlockSetRIP = true;
// ABI Optimization: Flags don't survive calls or rets
if (CTX->Config.ABILocalFlags) {
_InvalidateFlags(~0UL); // all flags
InvalidatePF_AF();
}
auto ConstantPC = GetRelocatedPC(Op);
// Call instruction only uses up to 32-bit signed displacement
int64_t TargetOffset = Op->Src[0].Literal();
uint64_t InstRIP = Op->PC + Op->InstSize;
uint64_t TargetRIP = InstRIP + TargetOffset;
Ref NewRIP = _Add(GPRSize, ConstantPC, _Constant(TargetOffset));
// Push the return address.
Push(GPRSize, ConstantPC);
const uint64_t NextRIP = Op->PC + Op->InstSize;
if (NextRIP != TargetRIP) {
// Store the RIP
ExitFunction(NewRIP); // If we get here then leave the function now
} else {
NeedsBlockEnd = true;
}
}
void OpDispatchBuilder::CALLAbsoluteOp(OpcodeArgs) {
BlockSetRIP = true;
const auto Size = OpSizeFromSrc(Op);
Ref JMPPCOffset = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
// Push the return address.
Push(Size, GetRelocatedPC(Op));
// Store the RIP
ExitFunction(JMPPCOffset); // If we get here then leave the function now
}
Ref OpDispatchBuilder::SelectPF(bool Invert, IR::OpSize ResultSize, Ref TrueValue, Ref FalseValue) {
uint64_t TrueConst, FalseConst;
if (IsValueConstant(WrapNode(TrueValue), &TrueConst) && IsValueConstant(WrapNode(FalseValue), &FalseConst) && FalseConst == 0) {
if (TrueConst == 1) {
return LoadPFRaw(true, Invert);
} else if (TrueConst == 0xffffffff) {
return _Sbfe(OpSize::i32Bit, 1, 0, LoadPFRaw(false, Invert));
} else if (TrueConst == 0xffffffffffffffffull) {
return _Sbfe(OpSize::i64Bit, 1, 0, LoadPFRaw(false, Invert));
}
}
Ref Cmp = LoadPFRaw(false, Invert);
SaveNZCV();
// Because we're only clobbering NZCV internally, we ignore all carry flag
// shenanigans and just use the raw test and raw select.
_TestNZ(OpSize::i32Bit, Cmp, _InlineConstant(1));
return _NZCVSelect(ResultSize, {COND_NEQ}, TrueValue, FalseValue);
}
std::pair<bool, CondClassType> OpDispatchBuilder::DecodeNZCVCondition(uint8_t OP) {
switch (OP) {
case 0x0: { // JO - Jump if OF == 1
return {false, CondClassType {COND_FU}};
}
case 0x1: { // JNO - Jump if OF == 0
return {false, CondClassType {COND_FNU}};
}
case 0x2: { // JC - Jump if CF == 1
return {false, CondClassType {CFInverted ? COND_ULT : COND_UGE}};
}
case 0x3: { // JNC - Jump if CF == 0
return {false, CondClassType {CFInverted ? COND_UGE : COND_ULT}};
}
case 0x4: { // JE - Jump if ZF == 1
return {false, CondClassType {COND_EQ}};
}
case 0x5: { // JNE - Jump if ZF == 0
return {false, CondClassType {COND_NEQ}};
}
case 0x6: { // JNA - Jump if CF == 1 || ZF == 1
// With CF, we want (C == 0 || Z == 1). By De Morgan's, that's
// equivalent to !(C == 1 && Z == 0). That's .ls
RectifyCarryInvert(true);
return {false, CondClassType {COND_ULE}};
}
case 0x7: { // JA - Jump if CF == 0 && ZF == 0
// With CF inverted, we want (C == 1 && Z == 0). That's .hi
RectifyCarryInvert(true);
return {false, CondClassType {COND_UGT}};
}
case 0x8: { // JS - Jump if SF == 1
return {false, CondClassType {COND_MI}};
}
case 0x9: { // JNS - Jump if SF == 0
return {false, CondClassType {COND_PL}};
}
case 0xC: { // SF <> OF
return {false, CondClassType {COND_SLT}};
}
case 0xD: { // SF = OF
return {false, CondClassType {COND_SGE}};
}
case 0xE: { // ZF = 1 || SF <> OF
return {false, CondClassType {COND_SLE}};
}
case 0xF: { // ZF = 0 && SF = OF
return {false, CondClassType {COND_SGT}};
}
default:
// Other conditions do not map directly, caller gets to deal with it.
return {true, CondClassType {0}};
}
}
Ref OpDispatchBuilder::SelectCC(uint8_t OP, IR::OpSize ResultSize, Ref TrueValue, Ref FalseValue) {
auto [Complex, Cond] = DecodeNZCVCondition(OP);
if (!Complex) {
// Use raw select since DecodeNZCVCondition handles the carry invert
return _NZCVSelect(ResultSize, Cond, TrueValue, FalseValue);
}
switch (OP) {
case 0xA: // JP - Jump if PF == 1
case 0xB: { // JNP - Jump if PF == 0
// Raw value contains inverted PF in bottom bit
return SelectPF(OP == 0xA, ResultSize, TrueValue, FalseValue);
}
default: LOGMAN_MSG_A_FMT("Unknown CC Op: 0x{:x}\n", OP); return nullptr;
}
}
void OpDispatchBuilder::SETccOp(OpcodeArgs) {
// Calculate flags early.
CalculateDeferredFlags();
auto ZeroConst = _Constant(0);
auto OneConst = _Constant(1);
auto SrcCond = SelectCC(Op->OP & 0xF, OpSize::i64Bit, OneConst, ZeroConst);
StoreResult(GPRClass, Op, SrcCond, OpSize::iInvalid);
}
void OpDispatchBuilder::CMOVOp(OpcodeArgs) {
const auto GPRSize = CTX->GetGPROpSize();
// Calculate flags early.
CalculateDeferredFlags();
// Destination is always a GPR.
Ref Dest = LoadSource_WithOpSize(GPRClass, Op, Op->Dest, GPRSize, Op->Flags);
Ref Src {};
if (Op->Src[0].IsGPR()) {
Src = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], GPRSize, Op->Flags);
} else {
Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
}
auto SrcCond = SelectCC(Op->OP & 0xF, std::max(OpSize::i32Bit, OpSizeFromSrc(Op)), Src, Dest);
StoreResult(GPRClass, Op, SrcCond, OpSize::iInvalid);
}
void OpDispatchBuilder::CondJUMPOp(OpcodeArgs) {
// Calculate flags early.
CalculateDeferredFlags();
BlockSetRIP = true;
// Jump instruction only uses up to 32-bit signed displacement
int64_t TargetOffset = Op->Src[0].Literal();
uint64_t InstRIP = Op->PC + Op->InstSize;
uint64_t Target = InstRIP + TargetOffset;
if (CTX->GetGPROpSize() == OpSize::i32Bit) {
// If the GPRSize is 4 then we need to be careful about PC wrapping
if (TargetOffset < 0 && -TargetOffset > InstRIP) {
// Invert the signed value if we are underflowing
TargetOffset = 0x1'0000'0000ULL + TargetOffset;
} else if (TargetOffset >= 0 && Target >= 0x1'0000'0000ULL) {
// We are overflowing, wrap around
TargetOffset = TargetOffset - 0x1'0000'0000ULL;
}
Target &= 0xFFFFFFFFU;
}
FlushRegisterCache();
auto TrueBlock = JumpTargets.find(Target);
auto FalseBlock = JumpTargets.find(Op->PC + Op->InstSize);
auto CurrentBlock = GetCurrentBlock();
{
IRPair<IR::IROp_CondJump> CondJump_;
auto OP = Op->OP & 0xF;
auto [Complex, SimpleCond] = DecodeNZCVCondition(OP);
if (Complex) {
LOGMAN_THROW_A_FMT(OP == 0xA || OP == 0xB, "only PF left");
CondJump_ = CondJumpBit(LoadPFRaw(false, false), 0, OP == 0xB);
} else {
CondJump_ = CondJumpNZCV(SimpleCond);
}
// Taking branch block
if (TrueBlock != JumpTargets.end()) {
SetTrueJumpTarget(CondJump_, TrueBlock->second.BlockEntry);
} else {
// Make sure to start a new block after ending this one
auto JumpTarget = CreateNewCodeBlockAtEnd();
SetTrueJumpTarget(CondJump_, JumpTarget);
SetCurrentCodeBlock(JumpTarget);
StartNewBlock();
auto NewRIP = GetRelocatedPC(Op, TargetOffset);
// Store the new RIP
ExitFunction(NewRIP);
}
// Failure to take branch
if (FalseBlock != JumpTargets.end()) {
SetFalseJumpTarget(CondJump_, FalseBlock->second.BlockEntry);
} else {
// Make sure to start a new block after ending this one
// Place it after this block for fallthrough optimization
auto JumpTarget = CreateNewCodeBlockAfter(CurrentBlock);
SetFalseJumpTarget(CondJump_, JumpTarget);
SetCurrentCodeBlock(JumpTarget);
StartNewBlock();
// Leave block
auto RIPTargetConst = GetRelocatedPC(Op);
// Store the new RIP
ExitFunction(RIPTargetConst);
}
}
}
void OpDispatchBuilder::CondJUMPRCXOp(OpcodeArgs) {
// Calculate flags early.
CalculateDeferredFlags();
BlockSetRIP = true;
auto JcxGPRSize = CTX->GetGPROpSize();
JcxGPRSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) ? (JcxGPRSize >> 1) : JcxGPRSize;
uint64_t Target = Op->PC + Op->InstSize + Op->Src[0].Literal();
Ref CondReg = LoadGPRRegister(X86State::REG_RCX, JcxGPRSize);
auto TrueBlock = JumpTargets.find(Target);
auto FalseBlock = JumpTargets.find(Op->PC + Op->InstSize);
auto CurrentBlock = GetCurrentBlock();
{
auto CondJump_ = CondJump(CondReg, {COND_EQ});
// Taking branch block
if (TrueBlock != JumpTargets.end()) {
SetTrueJumpTarget(CondJump_, TrueBlock->second.BlockEntry);
} else {
// Make sure to start a new block after ending this one
auto JumpTarget = CreateNewCodeBlockAtEnd();
SetTrueJumpTarget(CondJump_, JumpTarget);
SetCurrentCodeBlock(JumpTarget);
StartNewBlock();
auto NewRIP = GetRelocatedPC(Op, Op->Src[0].Data.Literal.Value);
// Store the new RIP
ExitFunction(NewRIP);
}
// Failure to take branch
if (FalseBlock != JumpTargets.end()) {
SetFalseJumpTarget(CondJump_, FalseBlock->second.BlockEntry);
} else {
// Make sure to start a new block after ending this one
// Place it after the current block for fallthrough behavior
auto JumpTarget = CreateNewCodeBlockAfter(CurrentBlock);
SetFalseJumpTarget(CondJump_, JumpTarget);
SetCurrentCodeBlock(JumpTarget);
StartNewBlock();
// Leave block
auto RIPTargetConst = GetRelocatedPC(Op);
// Store the new RIP
ExitFunction(RIPTargetConst);
}
}
}
void OpDispatchBuilder::LoopOp(OpcodeArgs) {
// Calculate flags early.
CalculateDeferredFlags();
bool CheckZF = Op->OP != 0xE2;
bool ZFTrue = Op->OP == 0xE1;
BlockSetRIP = true;
auto SrcSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) ? OpSize::i32Bit : OpSize::i64Bit;
auto OpSize = SrcSize == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit;
if (!CTX->Config.Is64BitMode) {
// RCX size is 32-bit or 16-bit when executing in 32-bit mode.
SrcSize = IR::SizeToOpSize(IR::OpSizeToSize(SrcSize) >> 1);
OpSize = OpSize::i32Bit;
}
uint64_t Target = Op->PC + Op->InstSize + Op->Src[1].Literal();
Ref CondReg = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], SrcSize, Op->Flags);
CondReg = _Sub(OpSize, CondReg, _InlineConstant(1));
StoreResult(GPRClass, Op, Op->Src[0], CondReg, OpSize::iInvalid);
// If LOOPE then jumps to target if RCX != 0 && ZF == 1
// If LOOPNE then jumps to target if RCX != 0 && ZF == 0
//
// To handle efficiently, smash RCX to zero if ZF is wrong (1 csel).
if (CheckZF) {
CondReg = NZCVSelect(OpSize, {ZFTrue ? COND_EQ : COND_NEQ}, CondReg, _InlineConstant(0));
}
CalculateDeferredFlags();
auto TrueBlock = JumpTargets.find(Target);
auto FalseBlock = JumpTargets.find(Op->PC + Op->InstSize);
{
auto CondJump_ = CondJump(CondReg);
// Taking branch block
if (TrueBlock != JumpTargets.end()) {
SetTrueJumpTarget(CondJump_, TrueBlock->second.BlockEntry);
} else {
// Make sure to start a new block after ending this one
auto JumpTarget = CreateNewCodeBlockAtEnd();
SetTrueJumpTarget(CondJump_, JumpTarget);
SetCurrentCodeBlock(JumpTarget);
StartNewBlock();
auto NewRIP = GetRelocatedPC(Op, Op->Src[1].Data.Literal.Value);
// Store the new RIP
ExitFunction(NewRIP);
}
// Failure to take branch
if (FalseBlock != JumpTargets.end()) {
SetFalseJumpTarget(CondJump_, FalseBlock->second.BlockEntry);
} else {
// Make sure to start a new block after ending this one
// Place after this block for fallthrough behavior
auto JumpTarget = CreateNewCodeBlockAfter(GetCurrentBlock());
SetFalseJumpTarget(CondJump_, JumpTarget);
SetCurrentCodeBlock(JumpTarget);
StartNewBlock();
// Leave block
auto RIPTargetConst = GetRelocatedPC(Op);
// Store the new RIP
ExitFunction(RIPTargetConst);
}
}
}
void OpDispatchBuilder::JUMPOp(OpcodeArgs) {
// Calculate flags early.
CalculateDeferredFlags();
BlockSetRIP = true;
// Jump instruction only uses up to 32-bit signed displacement
int64_t TargetOffset = Op->Src[0].Literal();
uint64_t InstRIP = Op->PC + Op->InstSize;
uint64_t TargetRIP = InstRIP + TargetOffset;
if (CTX->GetGPROpSize() == OpSize::i32Bit) {
// If the GPRSize is 4 then we need to be careful about PC wrapping
if (TargetOffset < 0 && -TargetOffset > InstRIP) {
// Invert the signed value if we are underflowing
TargetOffset = 0x1'0000'0000ULL + TargetOffset;
} else if (TargetOffset >= 0 && TargetRIP >= 0x1'0000'0000ULL) {
// We are overflowing, wrap around
TargetOffset = TargetOffset - 0x1'0000'0000ULL;
}
TargetRIP &= 0xFFFFFFFFU;
}
CalculateDeferredFlags();
// This is just an unconditional relative literal jump
if (Multiblock) {
auto JumpBlock = JumpTargets.find(TargetRIP);
if (JumpBlock != JumpTargets.end()) {
Jump(GetNewJumpBlock(TargetRIP));
} else {
// If the block isn't a jump target then we need to create an exit block
auto Jump_ = Jump();
// Place after this block for fallthrough behavior
auto JumpTarget = CreateNewCodeBlockAfter(GetCurrentBlock());
SetJumpTarget(Jump_, JumpTarget);
SetCurrentCodeBlock(JumpTarget);
StartNewBlock();
ExitFunction(GetRelocatedPC(Op, TargetOffset));
}
return;
}
// Fallback
{
auto RIPTargetConst = GetRelocatedPC(Op);
auto NewRIP = _Add(OpSize::i64Bit, _Constant(TargetOffset), RIPTargetConst);
// Store the new RIP
ExitFunction(NewRIP);
}
}
void OpDispatchBuilder::JUMPAbsoluteOp(OpcodeArgs) {
// Calculate flags early.
CalculateDeferredFlags();
BlockSetRIP = true;
// This is just an unconditional jump
// This uses ModRM to determine its location
// No way to use this effectively in multiblock
auto RIPOffset = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
// Store the new RIP
ExitFunction(RIPOffset);
}
void OpDispatchBuilder::TESTOp(OpcodeArgs, uint32_t SrcIndex) {
// TEST is an instruction that does an AND between the sources
// Result isn't stored in result, only writes to flags
Ref Src = LoadSource(GPRClass, Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true});
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
const auto Size = OpSizeFromDst(Op);
uint64_t Const;
bool AlwaysNonnegative = false;
if (IsValueConstant(WrapNode(Src), &Const)) {
// Optimize out masking constants
if (Const == (Size == OpSize::i64Bit ? ~0ULL : ((1ull << IR::OpSizeAsBits(Size)) - 1))) {
Src = Dest;
}
// Optimize test with non-sign bits
AlwaysNonnegative = (Const & (1ull << (IR::OpSizeAsBits(Size) - 1))) == 0;
}
if (Dest == Src) {
// Optimize out the AND.
SetNZP_ZeroCV(Size, Src);
} else if (Size < OpSize::i32Bit && AlwaysNonnegative) {
// If we know the result is always nonnegative, we can use a 32-bit test.
auto Res = _And(OpSize::i32Bit, Dest, Src);
CalculatePF(Res);
SetNZ_ZeroCV(OpSize::i32Bit, Res);
} else {
HandleNZ00Write();
CalculatePF(_AndWithFlags(Size, Dest, Src));
}
InvalidateAF();
}
void OpDispatchBuilder::MOVSXDOp(OpcodeArgs) {
// This instruction is a bit special
// if SrcSize == 2
// Then lower 16 bits of destination is written without changing the upper 48 bits
// else /* Size == 4 */
// if REX_WIDENING:
// Sext(32, Src)
// else
// Zext(32, Src)
//
auto Size = std::min<IR::OpSize>(OpSize::i32Bit, OpSizeFromSrc(Op));
bool Sext = (Size != OpSize::i16Bit) && Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING;
Ref Src = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], Size, Op->Flags, {.AllowUpperGarbage = Sext});
if (Size == OpSize::i16Bit) {
// This'll make sure to insert in to the lower 16bits without modifying upper bits
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Src, Size, OpSize::iInvalid);
} else if (Sext) {
// With REX.W then Sext
Src = _Sbfe(OpSize::i64Bit, IR::OpSizeAsBits(Size), 0, Src);
StoreResult(GPRClass, Op, Src, OpSize::iInvalid);
} else {
// Without REX.W then Zext (store result implicitly zero extends)
StoreResult(GPRClass, Op, Src, OpSize::iInvalid);
}
}
void OpDispatchBuilder::MOVSXOp(OpcodeArgs) {
// Load garbage in upper bits, since we're sign extending anyway
const auto Size = OpSizeFromSrc(Op);
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
// Sign-extend to DstSize and zero-extend to the register size, using a fast
// path for 32-bit dests where the native 32-bit Sbfe zero extends the top.
const auto DstSize = OpSizeFromDst(Op);
Src = _Sbfe(DstSize == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, Src);
StoreResult(GPRClass, Op, Op->Dest, Src, OpSize::iInvalid);
}
void OpDispatchBuilder::MOVZXOp(OpcodeArgs) {
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
// Store result implicitly zero extends
StoreResult(GPRClass, Op, Src, OpSize::iInvalid);
}
void OpDispatchBuilder::CMPOp(OpcodeArgs, uint32_t SrcIndex) {
// CMP is an instruction that does a SUB between the sources
// Result isn't stored in result, only writes to flags
Ref Src = LoadSource(GPRClass, Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true});
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
CalculateFlags_SUB(OpSizeFromSrc(Op), Dest, Src);
}
void OpDispatchBuilder::CQOOp(OpcodeArgs) {
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
auto Size = GetSrcSize(Op);
Ref Upper = _Sbfe(OpSize::i64Bit, 1, Size * 8 - 1, Src);
StoreResult(GPRClass, Op, Upper, OpSize::iInvalid);
}
void OpDispatchBuilder::XCHGOp(OpcodeArgs) {
// Load both the source and the destination
if (Op->OP == 0x90 && GetSrcSize(Op) >= 4 && Op->Src[0].IsGPR() && Op->Src[0].Data.GPR.GPR == FEXCore::X86State::REG_RAX &&
Op->Dest.IsGPR() && Op->Dest.Data.GPR.GPR == FEXCore::X86State::REG_RAX) {
// This is one heck of a sucky special case
// If we are the 0x90 XCHG opcode (Meaning source is GPR RAX)
// and destination register is ALSO RAX
// and in this very specific case we are 32bit or above
// Then this is a no-op
// This is because 0x90 without a prefix is technically `xchg eax, eax`
// But this would result in a zext on 64bit, which would ruin the no-op nature of the instruction
// So x86-64 spec mandates this special case that even though it is a 32bit instruction and
// is supposed to zext the result, it is a true no-op
if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX) {
// If this instruction has a REP prefix then this is architecturally
// defined to be a `PAUSE` instruction. On older processors this ends up
// being a true `REP NOP` which is why they stuck this here.
_Yield();
}
return;
}
// AllowUpperGarbage: OK to allow as it will be overwritten by StoreResult.
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
if (DestIsMem(Op)) {
HandledLock = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;
Ref Dest = MakeSegmentAddress(Op, Op->Dest);
auto Result = _AtomicSwap(OpSizeFromSrc(Op), Src, Dest);
StoreResult(GPRClass, Op, Op->Src[0], Result, OpSize::iInvalid);
} else {
// AllowUpperGarbage: OK to allow as it will be overwritten by StoreResult.
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
// Swap the contents
// Order matters here since we don't want to swap context contents for one that effects the other
StoreResult(GPRClass, Op, Op->Dest, Src, OpSize::iInvalid);
StoreResult(GPRClass, Op, Op->Src[0], Dest, OpSize::iInvalid);
}
}
void OpDispatchBuilder::CDQOp(OpcodeArgs) {
const auto DstSize = OpSizeFromDst(Op);
const auto SrcSize = DstSize / 2;
Ref Src = LoadGPRRegister(X86State::REG_RAX, SrcSize, 0, true);
Src = _Sbfe(DstSize <= OpSize::i32Bit ? OpSize::i32Bit : OpSize::i64Bit, IR::OpSizeAsBits(SrcSize), 0, Src);
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Src, DstSize, OpSize::iInvalid);
}
void OpDispatchBuilder::SAHFOp(OpcodeArgs) {
// Extract AH
Ref Src = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit, 8);
// Clear bits that aren't supposed to be set
Src = _Andn(OpSize::i64Bit, Src, _Constant(0b101000));
// Set the bit that is always set here
Src = _Or(OpSize::i64Bit, Src, _InlineConstant(0b10));
// Store the lower 8 bits in to RFLAGS
SetPackedRFLAG(true, Src);
}
void OpDispatchBuilder::LAHFOp(OpcodeArgs) {
// Load the lower 8 bits of the Rflags register
auto RFLAG = GetPackedRFLAG(0xFF);
// Store the lower 8 bits of the rflags register in to AH
StoreGPRRegister(X86State::REG_RAX, RFLAG, OpSize::i8Bit, 8);
}
void OpDispatchBuilder::FLAGControlOp(OpcodeArgs) {
// Calculate flags early.
CalculateDeferredFlags();
switch (Op->OP) {
case 0xF5: // CMC
CarryInvert();
break;
case 0xF8: // CLC
SetCFInverted(_Constant(1));
break;
case 0xF9: // STC
SetCFInverted(_Constant(0));
break;
case 0xFC: // CLD
SetRFLAG(_Constant(0), FEXCore::X86State::RFLAG_DF_RAW_LOC);
break;
case 0xFD: // STD
SetRFLAG(_Constant(1), FEXCore::X86State::RFLAG_DF_RAW_LOC);
break;
}
}
void OpDispatchBuilder::MOVSegOp(OpcodeArgs, bool ToSeg) {
// In x86-64 mode the accesses to the segment registers end up being constant zero moves
// Aside from FS/GS
// In x86-64 mode the accesses to segment registers can actually still touch the segments
// These write to the selector portion of the register
//
// FS and GS are specially handled here though
// AMD documentation is /wrong/ in this regard
// AMD documentation claims that the MOV to SReg and POP SReg registers will load a 32bit
// value in to the HIDDEN portions of the FS and GS registers /OR/ ignored if a null selector is
// selected for the registers
// This statement is actually untrue, the instructions will /actually/ load 16bits in to the selector portion of the register!
// Tested on a Zen+ CPU, the selector is the portion that is modified!
// We don't currently support FS/GS selector modifying, so this needs to be asserted out
// The loads here also load the selector, NOT the base
if (ToSeg) {
Ref Src = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], OpSize::i16Bit, Op->Flags);
switch (Op->Dest.Data.GPR.GPR) {
case FEXCore::X86State::REG_RAX: // ES
case FEXCore::X86State::REG_R8: // ES
_StoreContext(OpSize::i16Bit, GPRClass, Src, offsetof(FEXCore::Core::CPUState, es_idx));
UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX);
break;
case FEXCore::X86State::REG_RBX: // DS
case FEXCore::X86State::REG_R11: // DS
_StoreContext(OpSize::i16Bit, GPRClass, Src, offsetof(FEXCore::Core::CPUState, ds_idx));
UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX);
break;
case FEXCore::X86State::REG_RCX: // CS
case FEXCore::X86State::REG_R9: // CS
// CPL3 can't write to this
Break(FEXCore::IR::BreakDefinition {
.ErrorRegister = 0,
.Signal = SIGILL,
.TrapNumber = 0,
.si_code = 0,
});
break;
case FEXCore::X86State::REG_RDX: // SS
case FEXCore::X86State::REG_R10: // SS
_StoreContext(OpSize::i16Bit, GPRClass, Src, offsetof(FEXCore::Core::CPUState, ss_idx));
UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX);
break;
case FEXCore::X86State::REG_RBP: // GS
case FEXCore::X86State::REG_R13: // GS
if (!CTX->Config.Is64BitMode) {
_StoreContext(OpSize::i16Bit, GPRClass, Src, offsetof(FEXCore::Core::CPUState, gs_idx));
UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX);
} else {
LogMan::Msg::EFmt("We don't support modifying GS selector in 64bit mode!");
DecodeFailure = true;
}
break;
case FEXCore::X86State::REG_RSP: // FS
case FEXCore::X86State::REG_R12: // FS
if (!CTX->Config.Is64BitMode) {
_StoreContext(OpSize::i16Bit, GPRClass, Src, offsetof(FEXCore::Core::CPUState, fs_idx));
UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX);
} else {
LogMan::Msg::EFmt("We don't support modifying FS selector in 64bit mode!");
DecodeFailure = true;
}
break;
default:
LogMan::Msg::EFmt("Unknown segment register: {}", Op->Dest.Data.GPR.GPR);
DecodeFailure = true;
break;
}
} else {
Ref Segment {};
switch (Op->Src[0].Data.GPR.GPR) {
case FEXCore::X86State::REG_RAX: // ES
case FEXCore::X86State::REG_R8: // ES
Segment = _LoadContext(OpSize::i16Bit, GPRClass, offsetof(FEXCore::Core::CPUState, es_idx));
break;
case FEXCore::X86State::REG_RBX: // DS
case FEXCore::X86State::REG_R11: // DS
Segment = _LoadContext(OpSize::i16Bit, GPRClass, offsetof(FEXCore::Core::CPUState, ds_idx));
break;
case FEXCore::X86State::REG_RCX: // CS
case FEXCore::X86State::REG_R9: // CS
Segment = _LoadContext(OpSize::i16Bit, GPRClass, offsetof(FEXCore::Core::CPUState, cs_idx));
break;
case FEXCore::X86State::REG_RDX: // SS
case FEXCore::X86State::REG_R10: // SS
Segment = _LoadContext(OpSize::i16Bit, GPRClass, offsetof(FEXCore::Core::CPUState, ss_idx));
break;
case FEXCore::X86State::REG_RBP: // GS
case FEXCore::X86State::REG_R13: // GS
if (CTX->Config.Is64BitMode) {
Segment = _Constant(0);
} else {
Segment = _LoadContext(OpSize::i16Bit, GPRClass, offsetof(FEXCore::Core::CPUState, gs_idx));
}
break;
case FEXCore::X86State::REG_RSP: // FS
case FEXCore::X86State::REG_R12: // FS
if (CTX->Config.Is64BitMode) {
Segment = _Constant(0);
} else {
Segment = _LoadContext(OpSize::i16Bit, GPRClass, offsetof(FEXCore::Core::CPUState, fs_idx));
}
break;
default:
LogMan::Msg::EFmt("Unknown segment register: {}", Op->Dest.Data.GPR.GPR);
DecodeFailure = true;
return;
}
if (DestIsMem(Op)) {
// If the destination is memory then we always store 16-bits only
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Segment, OpSize::i16Bit, OpSize::iInvalid);
} else {
// If the destination is a GPR then we follow register storing rules
StoreResult(GPRClass, Op, Segment, OpSize::iInvalid);
}
}
}
void OpDispatchBuilder::MOVOffsetOp(OpcodeArgs) {
Ref Src;
switch (Op->OP) {
case 0xA0:
case 0xA1:
// Source is memory(literal)
// Dest is GPR
Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.ForceLoad = true});
StoreResult(GPRClass, Op, Op->Dest, Src, OpSize::iInvalid);
break;
case 0xA2:
case 0xA3:
// Source is GPR
// Dest is memory(literal)
Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
// This one is a bit special since the destination is a literal
// So the destination gets stored in Src[1]
StoreResult(GPRClass, Op, Op->Src[1], Src, OpSize::iInvalid);
break;
}
}
void OpDispatchBuilder::CPUIDOp(OpcodeArgs) {
const auto GPRSize = CTX->GetGPROpSize();
Ref Src = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], GPRSize, Op->Flags);
Ref Leaf = LoadGPRRegister(X86State::REG_RCX);
Ref RAX = _AllocateGPR(false);
Ref RBX = _AllocateGPR(false);
Ref RCX = _AllocateGPR(false);
Ref RDX = _AllocateGPR(false);
_CPUID(Src, Leaf, RAX, RBX, RCX, RDX);
StoreGPRRegister(X86State::REG_RAX, RAX);
StoreGPRRegister(X86State::REG_RBX, RBX);
StoreGPRRegister(X86State::REG_RCX, RCX);
StoreGPRRegister(X86State::REG_RDX, RDX);
}
uint32_t OpDispatchBuilder::LoadConstantShift(X86Tables::DecodedOp Op, bool Is1Bit) {
if (Is1Bit) {
return 1;
} else {
// x86 masks the shift by 0x3F or 0x1F depending on size of op
const auto Size = OpSizeFromSrc(Op);
uint64_t Mask = Size == OpSize::i64Bit ? 0x3F : 0x1F;
return Op->Src[1].Literal() & Mask;
}
}
void OpDispatchBuilder::XGetBVOp(OpcodeArgs) {
Ref Function = LoadGPRRegister(X86State::REG_RCX);
auto RAX = _AllocateGPR(false);
auto RDX = _AllocateGPR(false);
_XGetBV(Function, RAX, RDX);
StoreGPRRegister(X86State::REG_RAX, RAX);
StoreGPRRegister(X86State::REG_RDX, RDX);
}
void OpDispatchBuilder::SHLOp(OpcodeArgs) {
const auto Size = OpSizeFromSrc(Op);
auto Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
auto Src = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
Ref Result = _Lshl(Size == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit, Dest, Src);
HandleShift(Op, Result, Dest, ShiftType::LSL, Src);
}
void OpDispatchBuilder::SHLImmediateOp(OpcodeArgs, bool SHL1Bit) {
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
uint64_t Shift = LoadConstantShift(Op, SHL1Bit);
const auto Size = GetSrcBitSize(Op);
Ref Src = _Constant(OpSizeFromSrc(Op), Shift);
Ref Result = _Lshl(Size == 64 ? OpSize::i64Bit : OpSize::i32Bit, Dest, Src);
CalculateFlags_ShiftLeftImmediate(OpSizeFromSrc(Op), Result, Dest, Shift);
CalculateDeferredFlags();
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
}
void OpDispatchBuilder::SHROp(OpcodeArgs) {
const auto Size = OpSizeFromSrc(Op);
auto Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= OpSize::i32Bit});
auto Src = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
auto ALUOp = _Lshr(std::max(OpSize::i32Bit, Size), Dest, Src);
HandleShift(Op, ALUOp, Dest, ShiftType::LSR, Src);
}
void OpDispatchBuilder::SHRImmediateOp(OpcodeArgs, bool SHR1Bit) {
const auto Size = GetSrcBitSize(Op);
auto Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= 32});
uint64_t Shift = LoadConstantShift(Op, SHR1Bit);
Ref Src = _Constant(OpSizeFromSrc(Op), Shift);
auto ALUOp = _Lshr(Size == 64 ? OpSize::i64Bit : OpSize::i32Bit, Dest, Src);
CalculateFlags_ShiftRightImmediate(OpSizeFromSrc(Op), ALUOp, Dest, Shift);
CalculateDeferredFlags();
StoreResult(GPRClass, Op, ALUOp, OpSize::iInvalid);
}
void OpDispatchBuilder::SHLDOp(OpcodeArgs) {
// Calculate flags early.
CalculateDeferredFlags();
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags);
// Allow garbage on the shift, we're masking it anyway.
Ref Shift = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
const auto Size = GetSrcBitSize(Op);
// x86 masks the shift by 0x3F or 0x1F depending on size of op.
if (Size == 64) {
Shift = _And(OpSize::i64Bit, Shift, _InlineConstant(0x3F));
} else {
Shift = _And(OpSize::i64Bit, Shift, _InlineConstant(0x1F));
}
// a64 masks the bottom bits, so if we're using a native 32/64-bit shift, we
// can negate to do the subtract (it's congruent), which saves a constant.
auto ShiftRight = Size >= 32 ? _Neg(OpSize::i64Bit, Shift) : _Sub(OpSize::i64Bit, _Constant(Size), Shift);
auto Tmp1 = _Lshl(OpSize::i64Bit, Dest, Shift);
auto Tmp2 = _Lshr(Size == 64 ? OpSize::i64Bit : OpSize::i32Bit, Src, ShiftRight);
Ref Res = _Or(OpSize::i64Bit, Tmp1, Tmp2);
// If shift count was zero then output doesn't change
// Needs to be checked for the 32bit operand case
// where shift = 0 and the source register still gets Zext
//
// TODO: With a backwards pass ahead-of-time, we could stick this in the
// if(shift) used for flags.
//
// TODO: This whole function wants to be wrapped in the if. Maybe b/w pass is
// a good idea after all.
Res = _Select(FEXCore::IR::COND_EQ, Shift, _Constant(0), Dest, Res);
HandleShift(Op, Res, Dest, ShiftType::LSL, Shift);
}
void OpDispatchBuilder::SHLDImmediateOp(OpcodeArgs) {
uint64_t Shift = LoadConstantShift(Op, false);
const auto Size = GetSrcBitSize(Op);
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = Size >= 32});
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= 32});
if (Shift != 0) {
Ref Res {};
if (Size < 32) {
Ref ShiftLeft = _Constant(Shift);
auto ShiftRight = _Constant(Size - Shift);
auto Tmp1 = _Lshl(OpSize::i64Bit, Dest, ShiftLeft);
auto Tmp2 = _Lshr(OpSize::i32Bit, Src, ShiftRight);
Res = _Or(OpSize::i64Bit, Tmp1, Tmp2);
} else {
// 32-bit and 64-bit SHLD behaves like an EXTR where the lower bits are filled from the source.
Res = _Extr(OpSizeFromSrc(Op), Dest, Src, Size - Shift);
}
CalculateFlags_ShiftLeftImmediate(OpSizeFromSrc(Op), Res, Dest, Shift);
CalculateDeferredFlags();
StoreResult(GPRClass, Op, Res, OpSize::iInvalid);
} else if (Shift == 0 && Size == 32) {
// Ensure Zext still occurs
StoreResult(GPRClass, Op, Dest, OpSize::iInvalid);
}
}
void OpDispatchBuilder::SHRDOp(OpcodeArgs) {
// Calculate flags early.
// This instruction conditionally generates flags so we need to insure sane state going in.
CalculateDeferredFlags();
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags);
Ref Shift = LoadGPRRegister(X86State::REG_RCX);
const auto Size = GetDstBitSize(Op);
// x86 masks the shift by 0x3F or 0x1F depending on size of op
if (Size == 64) {
Shift = _And(OpSize::i64Bit, Shift, _InlineConstant(0x3F));
} else {
Shift = _And(OpSize::i64Bit, Shift, _InlineConstant(0x1F));
}
auto ShiftLeft = _Sub(OpSize::i64Bit, _Constant(Size), Shift);
auto Tmp1 = _Lshr(Size == 64 ? OpSize::i64Bit : OpSize::i32Bit, Dest, Shift);
auto Tmp2 = _Lshl(OpSize::i64Bit, Src, ShiftLeft);
Ref Res = _Or(OpSize::i64Bit, Tmp1, Tmp2);
// If shift count was zero then output doesn't change
// Needs to be checked for the 32bit operand case
// where shift = 0 and the source register still gets Zext
Res = _Select(FEXCore::IR::COND_EQ, Shift, _Constant(0), Dest, Res);
HandleShift(Op, Res, Dest, ShiftType::LSR, Shift);
}
void OpDispatchBuilder::SHRDImmediateOp(OpcodeArgs) {
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags);
uint64_t Shift = LoadConstantShift(Op, false);
const auto Size = GetSrcBitSize(Op);
if (Shift != 0) {
Ref Res {};
if (Size < 32) {
Ref ShiftRight = _Constant(Shift);
auto ShiftLeft = _Constant(Size - Shift);
auto Tmp1 = _Lshr(Size == 64 ? OpSize::i64Bit : OpSize::i32Bit, Dest, ShiftRight);
auto Tmp2 = _Lshl(OpSize::i64Bit, Src, ShiftLeft);
Res = _Or(OpSize::i64Bit, Tmp1, Tmp2);
} else {
// 32-bit and 64-bit SHRD behaves like an EXTR where the upper bits are filled from the source.
Res = _Extr(OpSizeFromSrc(Op), Src, Dest, Shift);
}
StoreResult(GPRClass, Op, Res, OpSize::iInvalid);
CalculateFlags_ShiftRightDoubleImmediate(OpSizeFromSrc(Op), Res, Dest, Shift);
} else if (Shift == 0 && Size == 32) {
// Ensure Zext still occurs
StoreResult(GPRClass, Op, Dest, OpSize::iInvalid);
}
}
void OpDispatchBuilder::ASHROp(OpcodeArgs, bool Immediate, bool SHR1Bit) {
const auto Size = OpSizeFromSrc(Op);
const auto OpSize = std::max(OpSize::i32Bit, OpSizeFromDst(Op));
// If Size < 4, then we Sbfe the Dest so we can have garbage.
// Otherwise, if Size = Opsize, then both are 4 or 8 and match the a64
// semantics directly, so again we can have garbage. The only case where we
// need zero-extension here is when the sizes mismatch.
auto Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = (OpSize == Size) || (Size < OpSize::i32Bit)});
if (Size < OpSize::i32Bit) {
Dest = _Sbfe(OpSize::i64Bit, IR::OpSizeAsBits(Size), 0, Dest);
}
if (Immediate) {
uint64_t Shift = LoadConstantShift(Op, SHR1Bit);
Ref Result = _Ashr(OpSize, Dest, _Constant(Shift));
CalculateFlags_SignShiftRightImmediate(OpSizeFromSrc(Op), Result, Dest, Shift);
CalculateDeferredFlags();
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
} else {
auto Src = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
Ref Result = _Ashr(OpSize, Dest, Src);
HandleShift(Op, Result, Dest, ShiftType::ASR, Src);
}
}
void OpDispatchBuilder::RotateOp(OpcodeArgs, bool Left, bool IsImmediate, bool Is1Bit) {
CalculateDeferredFlags();
const uint32_t Size = GetSrcBitSize(Op);
const auto OpSize = Size == 64 ? OpSize::i64Bit : OpSize::i32Bit;
uint64_t UnmaskedConst;
// x86 masks the shift by 0x3F or 0x1F depending on size of op. But it's
// equivalent to mask to the actual size of the op, that way we can bound
// things tighter for 8-bit later in the function.
uint64_t Mask = Size == 8 ? 7 : (Size == 64 ? 0x3F : 0x1F);
Ref Src, UnmaskedSrc;
if (Is1Bit || IsImmediate) {
UnmaskedConst = LoadConstantShift(Op, Is1Bit);
UnmaskedSrc = _Constant(UnmaskedConst);
Src = _Constant(UnmaskedConst & Mask);
} else {
UnmaskedSrc = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
Src = _And(OpSize::i64Bit, UnmaskedSrc, _InlineConstant(Mask));
}
// We fill the upper bits so we allow garbage on load.
auto Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
if (Size < 32) {
// ARM doesn't support 8/16bit rotates. Emulate with an insert
// StoreResult truncates back to a 8/16 bit value
Dest = _Bfi(OpSize, Size, Left ? (32 - Size) : Size, Dest, Dest);
}
// To rotate 64-bits left, right-rotate by (64 - Shift) = -Shift mod 64.
auto Res = _Ror(OpSize, Dest, Left ? _Neg(OpSize, Src) : Src);
StoreResult(GPRClass, Op, Res, OpSize::iInvalid);
if (Is1Bit || IsImmediate) {
if (UnmaskedConst) {
// Extract the last bit shifted in to CF
SetCFDirect(Res, Left ? 0 : Size - 1, true);
// For ROR, OF is the XOR of the new CF bit and the most significant bit of the result.
// For ROL, OF is the LSB and MSB XOR'd together.
// OF is architecturally only defined for 1-bit rotate.
if (UnmaskedConst == 1) {
auto NewOF = _XorShift(OpSize, Res, Res, ShiftType::LSR, Left ? Size - 1 : 1);
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(NewOF, Left ? 0 : Size - 2, true);
}
}
} else {
HandleNZCVWrite();
RectifyCarryInvert(true);
// We deferred the masking for 8-bit to the flag section, do it here.
if (Size == 8) {
Src = _And(OpSize::i64Bit, UnmaskedSrc, _InlineConstant(0x1F));
}
_RotateFlags(OpSizeFromSrc(Op), Res, Src, Left);
}
}
void OpDispatchBuilder::ANDNBMIOp(OpcodeArgs) {
auto* Src1 = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
auto* Src2 = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
auto Dest = _Andn(OpSizeFromSrc(Op), Src2, Src1);
StoreResult(GPRClass, Op, Dest, OpSize::iInvalid);
CalculateFlags_Logical(OpSizeFromSrc(Op), Dest, Src1, Src2);
}
void OpDispatchBuilder::BEXTRBMIOp(OpcodeArgs) {
// Essentially (Src1 >> Start) & ((1 << Length) - 1)
// along with some edge-case handling and flag setting.
LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed");
auto* Src1 = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
auto* Src2 = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
const auto Size = OpSizeFromSrc(Op);
const auto SrcSize = IR::OpSizeAsBits(Size);
const auto MaxSrcBit = SrcSize - 1;
auto MaxSrcBitOp = _Constant(Size, MaxSrcBit);
// Shift the operand down to the starting bit
auto Start = _Bfe(OpSizeFromSrc(Op), 8, 0, Src2);
auto Shifted = _Lshr(Size, Src1, Start);
// Shifts larger than operand size need to be set to zero.
auto SanitizedShifted = _Select(IR::COND_ULE, Start, MaxSrcBitOp, Shifted, _Constant(Size, 0));
// Now handle the length specifier.
auto Length = _Bfe(Size, 8, 8, Src2);
// Now build up the mask
// (1 << Length) - 1 = ~(~0 << Length)
auto AllOnes = _Constant(~0ull);
auto InvertedMask = _Lshl(Size, AllOnes, Length);
// Now put it all together and make the result.
auto Masked = _Andn(Size, SanitizedShifted, InvertedMask);
// Sanitize the length. If it is above the max, we don't do the masking.
auto Dest = _Select(IR::COND_ULE, Length, MaxSrcBitOp, Masked, SanitizedShifted);
// Finally store the result.
StoreResult(GPRClass, Op, Dest, OpSize::iInvalid);
// ZF is set properly. CF and OF are defined as being set to zero. SF, PF, and
// AF are undefined.
SetNZ_ZeroCV(GetOpSize(Dest), Dest);
InvalidatePF_AF();
}
void OpDispatchBuilder::BLSIBMIOp(OpcodeArgs) {
// Equivalent to performing: SRC & -SRC
LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed");
const auto Size = OpSizeFromSrc(Op);
auto* Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
auto NegatedSrc = _Neg(Size, Src);
auto Result = _And(Size, Src, NegatedSrc);
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
// CF is cleared if Src is zero, otherwise it's set. However, Src is zero iff
// Result is zero, so we can test the result instead. So, CF is just the
// inverted ZF.
//
// ZF/SF/OF set as usual.
SetNZ_ZeroCV(Size, Result);
InvalidatePF_AF();
SetCFInverted(GetRFLAG(X86State::RFLAG_ZF_RAW_LOC));
}
void OpDispatchBuilder::BLSMSKBMIOp(OpcodeArgs) {
// Equivalent to: (Src - 1) ^ Src
LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed");
const auto Size = OpSizeFromSrc(Op);
auto* Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
auto Result = _Xor(Size, _Sub(Size, Src, _InlineConstant(1)), Src);
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
InvalidatePF_AF();
// CF set according to the Src
auto Zero = _Constant(0);
auto One = _Constant(1);
auto CFInv = _Select(IR::COND_NEQ, Src, Zero, One, Zero);
// The output of BLSMSK is always nonzero, so TST will clear Z (along with C
// and O) while setting S.
SetNZ_ZeroCV(Size, Result);
SetCFInverted(CFInv);
}
void OpDispatchBuilder::BLSRBMIOp(OpcodeArgs) {
// Equivalent to: (Src - 1) & Src
LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed");
const auto Size = OpSizeFromSrc(Op);
auto* Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
auto Result = _And(Size, _Sub(Size, Src, _InlineConstant(1)), Src);
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
auto Zero = _Constant(0);
auto One = _Constant(1);
auto CFInv = _Select(IR::COND_NEQ, Src, Zero, One, Zero);
SetNZ_ZeroCV(Size, Result);
SetCFInverted(CFInv);
InvalidatePF_AF();
}
// Handles SARX, SHLX, and SHRX
void OpDispatchBuilder::BMI2Shift(OpcodeArgs) {
// In the event the source is a memory operand, use the
// exact width instead of the GPR size.
const auto GPRSize = CTX->GetGPROpSize();
const auto Size = OpSizeFromSrc(Op);
const auto SrcSize = Op->Src[0].IsGPR() ? GPRSize : Size;
auto* Src = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], SrcSize, Op->Flags);
auto* Shift = LoadSource_WithOpSize(GPRClass, Op, Op->Src[1], GPRSize, Op->Flags, {.AllowUpperGarbage = true});
Ref Result;
if (Op->OP == 0x6F7) {
// SARX
Result = _Ashr(Size, Src, Shift);
} else if (Op->OP == 0x5F7) {
// SHLX
Result = _Lshl(Size, Src, Shift);
} else {
// SHRX
Result = _Lshr(Size, Src, Shift);
}
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
}
void OpDispatchBuilder::BZHI(OpcodeArgs) {
const auto Size = OpSizeFromSrc(Op);
const auto OperandSize = IR::OpSizeAsBits(Size);
// In 32-bit mode we only look at bottom 32-bit, no 8 or 16-bit BZHI so no
// need to zero-extend sources
auto* Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
auto* Index = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
// Clear the high bits specified by the index. A64 only considers bottom bits
// of the shift, so we don't need to mask bottom 8-bits ourselves.
// Out-of-bounds results ignored after.
auto NegOne = _Constant(Size, -1);
auto Mask = _Lshl(Size, NegOne, Index);
auto MaskResult = _Andn(Size, Src, Mask);
// If the index is above OperandSize, we don't clear anything. BZHI only
// considers the bottom 8-bits, so we really want to know if the bottom 8-bits
// have their top bits set. Test exactly that.
//
// Because we're clobbering flags internally we ignore all carry invert
// shenanigans and use the raw versions here.
_TestNZ(OpSize::i64Bit, Index, _Constant(0xFF & ~(OperandSize - 1)));
auto Result = _NZCVSelect(Size, {COND_NEQ}, Src, MaskResult);
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
auto Zero = _InlineConstant(0);
auto One = _InlineConstant(1);
auto CFInv = _NZCVSelect(OpSize::i32Bit, {COND_EQ}, One, Zero);
InvalidatePF_AF();
SetNZ_ZeroCV(Size, Result);
SetCFInverted(CFInv);
}
void OpDispatchBuilder::RORX(OpcodeArgs) {
const auto SrcSize = OpSizeFromSrc(Op);
const auto SrcSizeBits = IR::OpSizeAsBits(SrcSize);
const auto Amount = Op->Src[1].Literal() & (SrcSizeBits - 1);
const auto GPRSize = CTX->GetGPROpSize();
const auto DoRotation = Amount != 0 && Amount < SrcSizeBits;
const auto IsSameGPR = Op->Src[0].IsGPR() && Op->Dest.IsGPR() && Op->Src[0].Data.GPR.GPR == Op->Dest.Data.GPR.GPR;
const auto SrcSizeIsGPRSize = SrcSize == GPRSize;
// If we don't need to rotate and our source is the same as the destination
// then we don't need to do anything at all. We still need to be careful,
// since 32-bit operations on 64-bit mode still need to zero-extend the
// destination register. So also compare source size and GPR size.
//
// Very unlikely, but hey, we can do nothing faster.
if (!DoRotation && IsSameGPR && SrcSizeIsGPRSize) [[unlikely]] {
return;
}
auto* Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
auto* Result = Src;
if (DoRotation) [[likely]] {
Result = _Ror(OpSizeFromSrc(Op), Src, _InlineConstant(Amount));
}
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
}
void OpDispatchBuilder::MULX(OpcodeArgs) {
// RDX is the implied source operand in the instruction
const auto OpSize = OpSizeFromSrc(Op);
// Src1 can be a memory operand, so ensure we constrain to the
// absolute width of the access in that scenario.
const auto GPRSize = CTX->GetGPROpSize();
const auto Src1Size = Op->Src[1].IsGPR() ? GPRSize : OpSize;
Ref Src1 = LoadSource_WithOpSize(GPRClass, Op, Op->Src[1], Src1Size, Op->Flags);
Ref Src2 = LoadGPRRegister(X86State::REG_RDX, GPRSize);
// As per the Intel Software Development Manual, if the destination and
// first operand correspond to the same register, then the result
// will be the high half of the multiplication result.
if (Op->Dest.Data.GPR.GPR == Op->Src[0].Data.GPR.GPR) {
Ref ResultHi = _UMulH(OpSize, Src1, Src2);
StoreResult(GPRClass, Op, Op->Dest, ResultHi, OpSize::iInvalid);
} else {
Ref ResultLo = _UMul(OpSize, Src1, Src2);
Ref ResultHi = _UMulH(OpSize, Src1, Src2);
StoreResult(GPRClass, Op, Op->Src[0], ResultLo, OpSize::iInvalid);
StoreResult(GPRClass, Op, Op->Dest, ResultHi, OpSize::iInvalid);
}
}
void OpDispatchBuilder::PDEP(OpcodeArgs) {
LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed");
auto* Input = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
auto* Mask = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
auto Result = _PDep(OpSizeFromSrc(Op), Input, Mask);
StoreResult(GPRClass, Op, Op->Dest, Result, OpSize::iInvalid);
}
void OpDispatchBuilder::PEXT(OpcodeArgs) {
LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed");
auto* Input = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
auto* Mask = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
auto Result = _PExt(OpSizeFromSrc(Op), Input, Mask);
StoreResult(GPRClass, Op, Op->Dest, Result, OpSize::iInvalid);
}
void OpDispatchBuilder::ADXOp(OpcodeArgs) {
const auto OpSize = OpSizeFromSrc(Op);
// Only 32/64-bit anyway so allow garbage, we use 32-bit ops.
auto* Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
auto* Before = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
// Handles ADCX and ADOX
const bool IsADCX = Op->OP == 0x1F6;
auto Zero = _Constant(0);
// Before we go trashing NZCV, save the current NZCV state.
Ref OldNZCV = GetNZCV();
// We want to use arm64 adc. For ADOX, copy the overflow flag into CF. For
// ADCX, we just rectify the carry.
if (IsADCX) {
RectifyCarryInvert(false);
} else {
// If overflow, 0 - 0 sets carry. Else, forces carry to 0.
_CondSubNZCV(OpSize::i32Bit, Zero, Zero, {COND_FU}, 0x0 /* nzcv */);
}
// Do the actual add.
HandleNZCV_RMW();
auto Result = _AdcWithFlags(OpSize, Src, Before);
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
// Now restore all flags except the one we're updating.
if (CTX->HostFeatures.SupportsFlagM) {
// For ADOX, we need to copy the new carry into the overflow flag. If carry is clear (ULT with uninverted
// carry), 0 - 0 clears overflow. Else, force overflow on.
if (!IsADCX) {
_CondSubNZCV(OpSize::i32Bit, Zero, Zero, {COND_ULT}, 0x1 /* nzcV */);
}
_RmifNZCV(OldNZCV, 28, IsADCX ? 0xd /* NzcV */ : 0xe /* NZCv */);
} else {
// For either operation, insert the new flag into the old NZCV.
bool SavedCFInvert = CFInverted;
CFInverted = false;
Ref OutputCF = GetRFLAG(X86State::RFLAG_CF_RAW_LOC, IsADCX);
CFInverted = IsADCX ? true : SavedCFInvert;
Ref NewNZCV = _Bfi(OpSize::i32Bit, 1, IsADCX ? 29 : 28, OldNZCV, OutputCF);
SetNZCV(NewNZCV);
}
}
void OpDispatchBuilder::RCROp1Bit(OpcodeArgs) {
// Calculate flags early.
CalculateDeferredFlags();
// We expliclty mask for <32-bit so allow garbage
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
const auto Size = GetSrcBitSize(Op);
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
Ref Res;
// Our new CF will be bit 0 of the source. Set upfront to avoid a move.
SetCFDirect(Dest, 0, true);
uint32_t Shift = 1;
if (Size == 32 || Size == 64) {
// Rotate and insert CF in the upper bit
Res = _Extr(OpSizeFromSrc(Op), CF, Dest, Shift);
} else {
// Res = Src >> Shift
Res = _Bfe(OpSize::i32Bit, Size - Shift, Shift, Dest);
// inject the CF
Res = _Orlshl(OpSize::i32Bit, Res, CF, Size - Shift);
}
StoreResult(GPRClass, Op, Res, OpSize::iInvalid);
// OF is the top two MSBs XOR'd together
// Only when Shift == 1, it is undefined otherwise
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(_XorShift(OpSize::i64Bit, Res, Res, ShiftType::LSR, 1), Size - 2, true);
}
void OpDispatchBuilder::RCROp8x1Bit(OpcodeArgs) {
// Calculate flags early.
CalculateDeferredFlags();
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags);
const auto SizeBit = GetSrcBitSize(Op);
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
// Our new CF will be bit (Shift - 1) of the source
SetCFDirect(Dest, 0, true);
// Rotate and insert CF in the upper bit
Ref Res = _Bfe(OpSize::i32Bit, 7, 1, Dest);
Res = _Bfi(OpSize::i32Bit, 1, 7, Res, CF);
StoreResult(GPRClass, Op, Res, OpSize::iInvalid);
// OF is the top two MSBs XOR'd together
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(_XorShift(OpSize::i32Bit, Res, Res, ShiftType::LSR, 1), SizeBit - 2, true);
}
void OpDispatchBuilder::RCROp(OpcodeArgs) {
const auto Size = GetSrcBitSize(Op);
if (Size == 8 || Size == 16) {
RCRSmallerOp(Op);
return;
}
const auto Mask = (Size == 64) ? 0x3F : 0x1F;
// Calculate flags early.
CalculateDeferredFlags();
const auto OpSize = OpSizeFromSrc(Op);
Ref Src = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
uint64_t Const;
if (IsValueConstant(WrapNode(Src), &Const)) {
Const &= Mask;
if (!Const) {
ZeroShiftResult(Op);
return;
}
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
// Res = Src >> Shift
Ref Res = _Lshr(OpSize, Dest, Src);
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
// Constant folded version of the above, with fused shifts.
if (Const > 1) {
Res = _Orlshl(OpSize, Res, Dest, Size + 1 - Const);
}
// Our new CF will be bit (Shift - 1) of the source.
SetCFDirect(Dest, Const - 1, true);
// Since shift != 0 we can inject the CF
Res = _Orlshl(OpSize, Res, CF, Size - Const);
// OF is the top two MSBs XOR'd together
// Only when Shift == 1, it is undefined otherwise
if (Const == 1) {
auto Xor = _XorShift(OpSize, Res, Res, ShiftType::LSR, 1);
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(Xor, Size - 2, true);
}
StoreResult(GPRClass, Op, Res, OpSize::iInvalid);
return;
}
Ref SrcMasked = _And(OpSize, Src, _InlineConstant(Mask));
Calculate_ShiftVariable(
Op, SrcMasked,
[this, Op, Size, OpSize]() {
// Rematerialize loads to avoid crossblock liveness
Ref Src = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
// Res = Src >> Shift
Ref Res = _Lshr(OpSize, Dest, Src);
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
auto One = _Constant(OpSizeFromSrc(Op), 1);
// Res |= (Dest << (Size - Shift + 1));
// Expressed as Res | ((Src << (Size - Shift)) << 1) to get correct
// behaviour for Shift without clobbering NZCV. Then observe that modulo
// Size, Size - Shift = -Shift so we can use a simple Neg.
//
// The masking of Lshl means we don't need mask the source, since:
//
// -(x & Mask) & Mask = (-x) & Mask
Ref NegSrc = _Neg(OpSize, Src);
Res = _Orlshl(OpSize, Res, _Lshl(OpSize, Dest, NegSrc), 1);
// Our new CF will be bit (Shift - 1) of the source. this is hoisted up to
// avoid the need to copy the source. Again, the Lshr absorbs the masking.
auto NewCF = _Lshr(OpSize, Dest, _Sub(OpSize, Src, One));
SetCFDirect(NewCF, 0, true);
// Since shift != 0 we can inject the CF
Res = _Or(OpSize, Res, _Lshl(OpSize, CF, NegSrc));
// OF is the top two MSBs XOR'd together
// Only when Shift == 1, it is undefined otherwise
auto Xor = _XorShift(OpSize, Res, Res, ShiftType::LSR, 1);
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(Xor, Size - 2, true);
StoreResult(GPRClass, Op, Res, OpSize::iInvalid);
},
OpSizeFromSrc(Op) == OpSize::i32Bit ? std::make_optional(&OpDispatchBuilder::ZeroShiftResult) : std::nullopt);
}
void OpDispatchBuilder::RCRSmallerOp(OpcodeArgs) {
CalculateDeferredFlags();
const auto Size = GetSrcBitSize(Op);
// x86 masks the shift by 0x3F or 0x1F depending on size of op
Ref Src = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
Src = AndConst(OpSize::i32Bit, Src, 0x1F);
// CF only changes if we actually shifted. OF undefined if we didn't shift.
// The result is unchanged if we didn't shift. So branch over the whole thing.
Calculate_ShiftVariable(Op, Src, [this, Op, Size]() {
// Rematerialized to avoid crossblock liveness
Ref Src = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
Src = AndConst(OpSize::i32Bit, Src, 0x1F);
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags);
Ref Tmp {};
// Insert the incoming value across the temporary 64bit source
// Make sure to insert at <BitSize> + 1 offsets
// We need to cover 32bits plus the amount that could rotate in
if (Size == 8) {
// 8-bit optimal cascade
// Cascade: 0
// Data: -> [7:0]
// CF: -> [8:8]
// Cascade: 1
// Data: -> [16:9]
// CF: -> [17:17]
// Cascade: 2
// Data: -> [25:18]
// CF: -> [26:26]
// Cascade: 3
// Data: -> [34:27]
// CF: -> [35:35]
// Cascade: 4
// Data: -> [43:36]
// CF: -> [44:44]
// Insert CF, Destination already at [7:0]
Tmp = _Bfi(OpSize::i64Bit, 1, 8, Dest, CF);
// First Cascade, copies 9 bits from itself.
Tmp = _Bfi(OpSize::i64Bit, 9, 9, Tmp, Tmp);
// Second cascade, copies 18 bits from itself.
Tmp = _Bfi(OpSize::i64Bit, 18, 18, Tmp, Tmp);
// Final cascade, copies 9 bits again from itself.
Tmp = _Bfi(OpSize::i64Bit, 9, 36, Tmp, Tmp);
} else {
// 16-bit optimal cascade
// Cascade: 0
// Data: -> [15:0]
// CF: -> [16:16]
// Cascade: 1
// Data: -> [32:17]
// CF: -> [33:33]
// Cascade: 2
// Data: -> [49:34]
// CF: -> [50:50]
// Insert CF, Destination already at [15:0]
Tmp = _Bfi(OpSize::i64Bit, 1, 16, Dest, CF);
// First Cascade, copies 17 bits from itself.
Tmp = _Bfi(OpSize::i64Bit, 17, 17, Tmp, Tmp);
// Final Cascade, copies 17 bits from itself again.
Tmp = _Bfi(OpSize::i64Bit, 17, 34, Tmp, Tmp);
}
// Entire bitfield has been setup. Just extract the 8 or 16bits we need.
// 64-bit shift used because we want to rotate in our cascaded upper bits
// rather than zeroes.
Ref Res = _Lshr(OpSize::i64Bit, Tmp, Src);
StoreResult(GPRClass, Op, Res, OpSize::iInvalid);
uint64_t SrcConst;
bool IsSrcConst = IsValueConstant(WrapNode(Src), &SrcConst);
SrcConst &= 0x1f;
// Our new CF will be bit (Shift - 1) of the source. 32-bit Lshr masks the
// same as x86, but if we constant fold we must mask ourselves.
if (IsSrcConst) {
SetCFDirect(Tmp, SrcConst - 1, true);
} else {
auto One = _Constant(OpSizeFromSrc(Op), 1);
auto NewCF = _Lshr(OpSize::i32Bit, Tmp, _Sub(OpSize::i32Bit, Src, One));
SetCFDirect(NewCF, 0, true);
}
// OF is the top two MSBs XOR'd together
// Only when Shift == 1, it is undefined otherwise
if (!IsSrcConst || SrcConst == 1) {
auto NewOF = _XorShift(OpSize::i32Bit, Res, Res, ShiftType::LSR, 1);
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(NewOF, Size - 2, true);
}
});
}
void OpDispatchBuilder::RCLOp1Bit(OpcodeArgs) {
// Calculate flags early.
CalculateDeferredFlags();
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags);
const auto Size = GetSrcBitSize(Op);
const auto OpSize = Size == 64 ? OpSize::i64Bit : OpSize::i32Bit;
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
// Rotate left and insert CF in to lowest bit
// TODO: Use `adc Res, xzr, Dest, lsl 1` to save an instruction
Ref Res = _Orlshl(OpSize, CF, Dest, 1);
// Our new CF will be the top bit of the source
SetCFDirect(Dest, Size - 1, true);
// OF is the top two MSBs XOR'd together
// Top two MSBs is CF and top bit of result
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(_Xor(OpSize, Res, Dest), Size - 1, true);
StoreResult(GPRClass, Op, Res, OpSize::iInvalid);
}
void OpDispatchBuilder::RCLOp(OpcodeArgs) {
const auto Size = GetSrcBitSize(Op);
if (Size == 8 || Size == 16) {
RCLSmallerOp(Op);
return;
}
const auto Mask = (Size == 64) ? 0x3F : 0x1F;
// Calculate flags early.
CalculateDeferredFlags();
Ref Src = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
const auto OpSize = OpSizeFromSrc(Op);
uint64_t Const;
if (IsValueConstant(WrapNode(Src), &Const)) {
Const &= Mask;
if (!Const) {
ZeroShiftResult(Op);
return;
}
// Res = Src << Shift
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
Ref Res = _Lshl(OpSize, Dest, Src);
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
// Res |= (Src << (Size - Shift + 1));
if (Const > 1) {
Res = _Orlshr(OpSize, Res, Dest, Size + 1 - Const);
}
// Our new CF will be bit (Shift - 1) of the source
SetCFDirect(Dest, Size - Const, true);
// Since Shift != 0 we can inject the CF
Res = _Orlshl(OpSize, Res, CF, Const - 1);
// OF is the top two MSBs XOR'd together
// Only when Shift == 1, it is undefined otherwise
if (Const == 1) {
auto NewOF = _Xor(OpSize, Res, Dest);
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(NewOF, Size - 1, true);
}
StoreResult(GPRClass, Op, Res, OpSize::iInvalid);
return;
}
Ref SrcMasked = _And(OpSize, Src, _InlineConstant(Mask));
Calculate_ShiftVariable(
Op, SrcMasked,
[this, Op, Size, OpSize]() {
// Rematerialized to avoid crossblock liveness
Ref Src = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
// Res = Src << Shift
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
Ref Res = _Lshl(OpSize, Dest, Src);
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
// Res |= (Dest >> (Size - Shift + 1)), expressed as
// Res | ((Dest >> (-Shift)) >> 1), since Size - Shift = -Shift mod
// Size. The shift aborbs the masking.
auto NegSrc = _Neg(OpSize, Src);
Res = _Orlshr(OpSize, Res, _Lshr(OpSize, Dest, NegSrc), 1);
// Our new CF will be bit (Shift - 1) of the source
auto NewCF = _Lshr(OpSize, Dest, NegSrc);
SetCFDirect(NewCF, 0, true);
// Since Shift != 0 we can inject the CF. Shift absorbs the masking.
Ref CFShl = _Sub(OpSize, Src, _InlineConstant(1));
auto TmpCF = _Lshl(OpSize, CF, CFShl);
Res = _Or(OpSize, Res, TmpCF);
// OF is the top two MSBs XOR'd together
// Only when Shift == 1, it is undefined otherwise
//
// Note that NewCF has garbage in the upper bits, but we ignore them here
// and mask as part of the set after.
auto NewOF = _XorShift(OpSize, Res, NewCF, ShiftType::LSL, Size - 1);
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(NewOF, Size - 1, true);
StoreResult(GPRClass, Op, Res, OpSize::iInvalid);
},
OpSizeFromSrc(Op) == OpSize::i32Bit ? std::make_optional(&OpDispatchBuilder::ZeroShiftResult) : std::nullopt);
}
void OpDispatchBuilder::RCLSmallerOp(OpcodeArgs) {
CalculateDeferredFlags();
const auto Size = GetSrcBitSize(Op);
// x86 masks the shift by 0x3F or 0x1F depending on size of op
Ref Src = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
Src = AndConst(OpSize::i32Bit, Src, 0x1F);
// CF only changes if we actually shifted. OF undefined if we didn't shift.
// The result is unchanged if we didn't shift. So branch over the whole thing.
Calculate_ShiftVariable(Op, Src, [this, Op, Size]() {
// Rematerialized to avoid crossblock liveness
Ref Src = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
Src = AndConst(OpSize::i32Bit, Src, 0x1F);
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags);
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
Ref Tmp = _Constant(OpSize::i64Bit, 0);
for (size_t i = 0; i < (32 + Size + 1); i += (Size + 1)) {
// Insert incoming value
Tmp = _Bfi(OpSize::i64Bit, Size, 63 - i - Size, Tmp, Dest);
// Insert CF
Tmp = _Bfi(OpSize::i64Bit, 1, 63 - i, Tmp, CF);
}
// Insert incoming value
Tmp = _Bfi(OpSize::i64Bit, Size, 0, Tmp, Dest);
// The data is now set up like this
// [Data][CF]:[Data][CF]:[Data][CF]:[Data][CF]
// Shift 1 more bit that expected to get our result
// Shifting to the right will now behave like a rotate to the left
// Which we emulate with a _Ror
Ref Res = _Ror(OpSize::i64Bit, Tmp, _Neg(OpSize::i32Bit, Src));
StoreResult(GPRClass, Op, Res, OpSize::iInvalid);
// Our new CF is now at the bit position that we are shifting
// Either 0 if CF hasn't changed (CF is living in bit 0)
// or higher
auto NewCF = _Ror(OpSize::i64Bit, Tmp, _Sub(OpSize::i64Bit, _Constant(63), Src));
SetCFDirect(NewCF, 0, true);
// OF is the XOR of the NewCF and the MSB of the result
// Only defined for 1-bit rotates.
uint64_t SrcConst;
if (!IsValueConstant(WrapNode(Src), &SrcConst) || SrcConst == 1) {
auto NewOF = _XorShift(OpSize::i64Bit, NewCF, Res, ShiftType::LSR, Size - 1);
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(NewOF, 0, true);
}
});
}
void OpDispatchBuilder::BTOp(OpcodeArgs, uint32_t SrcIndex, BTAction Action) {
Ref Value;
Ref Src {};
bool IsNonconstant = Op->Src[SrcIndex].IsGPR();
uint8_t ConstantShift = 0;
const uint32_t Size = GetDstBitSize(Op);
const uint32_t Mask = Size - 1;
if (IsNonconstant) {
// Because we mask explicitly with And/Bfe/Sbfe after, we can allow garbage here.
Src = LoadSource(GPRClass, Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true});
} else {
// Can only be an immediate
// Masked by operand size
ConstantShift = Op->Src[SrcIndex].Data.Literal.Value & Mask;
Src = _Constant(ConstantShift);
}
if (Op->Dest.IsGPR()) {
// When the destination is a GPR, we don't care about garbage in the upper bits.
// Load the full register.
auto Dest = LoadSource_WithOpSize(GPRClass, Op, Op->Dest, CTX->GetGPROpSize(), Op->Flags);
Value = Dest;
// Get the bit selection from the src. We need to mask for 8/16-bit, but
// rely on the implicit masking of Lshr for native sizes.
unsigned LshrSize = std::max<uint8_t>(IR::OpSizeToSize(OpSize::i32Bit), Size / 8);
auto BitSelect = (Size == (LshrSize * 8)) ? Src : _And(OpSize::i64Bit, Src, _Constant(Mask));
// OF/SF/ZF/AF/PF undefined.
// Set CF before the action to save a move, except for complements where we
// can reuse the invert.
if (Action != BTAction::BTComplement) {
if (IsNonconstant) {
Value = _Lshr(IR::SizeToOpSize(LshrSize), Value, BitSelect);
}
SetCFDirect_InvalidateNZV(Value, ConstantShift, Value);
}
switch (Action) {
case BTAction::BTNone: {
/* Nothing to do */
break;
}
case BTAction::BTClear: {
Ref BitMask = _Lshl(IR::SizeToOpSize(LshrSize), _Constant(1), BitSelect);
Dest = _Andn(IR::SizeToOpSize(LshrSize), Dest, BitMask);
StoreResult(GPRClass, Op, Dest, OpSize::iInvalid);
break;
}
case BTAction::BTSet: {
Ref BitMask = _Lshl(IR::SizeToOpSize(LshrSize), _Constant(1), BitSelect);
Dest = _Or(IR::SizeToOpSize(LshrSize), Dest, BitMask);
StoreResult(GPRClass, Op, Dest, OpSize::iInvalid);
break;
}
case BTAction::BTComplement: {
Ref BitMask = _Lshl(IR::SizeToOpSize(LshrSize), _Constant(1), BitSelect);
Dest = _Xor(IR::SizeToOpSize(LshrSize), Dest, BitMask);
if (IsNonconstant) {
Value = _Lshr(IR::SizeToOpSize(LshrSize), Dest, BitSelect);
} else {
Value = Dest;
}
SetCFInverted_InvalidateNZV(Value, ConstantShift, true);
StoreResult(GPRClass, Op, Dest, OpSize::iInvalid);
break;
}
}
} else {
// Load the address to the memory location
Ref Dest = MakeSegmentAddress(Op, Op->Dest);
// Get the bit selection from the src
Ref BitSelect = _Bfe(std::max(OpSize::i32Bit, GetOpSize(Src)), 3, 0, Src);
// Address is provided as bits we want BYTE offsets
// Extract Signed offset
Src = _Sbfe(OpSize::i64Bit, Size - 3, 3, Src);
// Get the address offset by shifting out the size of the op (To shift out the bit selection)
// Then use that to index in to the memory location by size of op
AddressMode Address = {.Base = Dest, .Index = Src, .AddrSize = OpSize::i64Bit};
ConstantShift = 0;
switch (Action) {
case BTAction::BTNone: {
Value = _LoadMemAutoTSO(GPRClass, OpSize::i8Bit, Address, OpSize::i8Bit);
break;
}
case BTAction::BTClear: {
Ref BitMask = _Lshl(OpSize::i64Bit, _Constant(1), BitSelect);
if (DestIsLockedMem(Op)) {
HandledLock = true;
Value = _AtomicFetchCLR(OpSize::i8Bit, BitMask, LoadEffectiveAddress(Address, true));
} else {
Value = _LoadMemAutoTSO(GPRClass, OpSize::i8Bit, Address, OpSize::i8Bit);
auto Modified = _Andn(OpSize::i64Bit, Value, BitMask);
_StoreMemAutoTSO(GPRClass, OpSize::i8Bit, Address, Modified, OpSize::i8Bit);
}
break;
}
case BTAction::BTSet: {
Ref BitMask = _Lshl(OpSize::i64Bit, _Constant(1), BitSelect);
if (DestIsLockedMem(Op)) {
HandledLock = true;
Value = _AtomicFetchOr(OpSize::i8Bit, BitMask, LoadEffectiveAddress(Address, true));
} else {
Value = _LoadMemAutoTSO(GPRClass, OpSize::i8Bit, Address, OpSize::i8Bit);
auto Modified = _Or(OpSize::i64Bit, Value, BitMask);
_StoreMemAutoTSO(GPRClass, OpSize::i8Bit, Address, Modified, OpSize::i8Bit);
}
break;
}
case BTAction::BTComplement: {
Ref BitMask = _Lshl(OpSize::i64Bit, _Constant(1), BitSelect);
if (DestIsLockedMem(Op)) {
HandledLock = true;
Value = _AtomicFetchXor(OpSize::i8Bit, BitMask, LoadEffectiveAddress(Address, true));
} else {
Value = _LoadMemAutoTSO(GPRClass, OpSize::i8Bit, Address, OpSize::i8Bit);
auto Modified = _Xor(OpSize::i64Bit, Value, BitMask);
_StoreMemAutoTSO(GPRClass, OpSize::i8Bit, Address, Modified, OpSize::i8Bit);
}
break;
}
}
// Now shift in to the correct bit location
Value = _Lshr(std::max(OpSize::i32Bit, GetOpSize(Value)), Value, BitSelect);
// OF/SF/ZF/AF/PF undefined.
SetCFDirect(Value, ConstantShift, true);
}
}
void OpDispatchBuilder::IMUL1SrcOp(OpcodeArgs) {
/* We're just going to sign-extend the non-garbage anyway.. */
Ref Src1 = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
Ref Src2 = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
const auto Size = OpSizeFromSrc(Op);
const auto SizeBits = IR::OpSizeAsBits(Size);
Ref Dest {};
Ref ResultHigh {};
switch (Size) {
case OpSize::i8Bit:
case OpSize::i16Bit: {
Src1 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src1);
Src2 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src2);
Dest = _Mul(OpSize::i64Bit, Src1, Src2);
ResultHigh = _Sbfe(OpSize::i64Bit, SizeBits, SizeBits, Dest);
break;
}
case OpSize::i32Bit: {
ResultHigh = _SMull(Src1, Src2);
ResultHigh = _Sbfe(OpSize::i64Bit, SizeBits, SizeBits, ResultHigh);
// Flipped order to save a move
Dest = _Mul(OpSize::i32Bit, Src1, Src2);
break;
}
case OpSize::i64Bit: {
ResultHigh = _MulH(OpSize::i64Bit, Src1, Src2);
// Flipped order to save a move
Dest = _Mul(OpSize::i64Bit, Src1, Src2);
break;
}
default: FEX_UNREACHABLE;
}
StoreResult(GPRClass, Op, Dest, OpSize::iInvalid);
CalculateFlags_MUL(Size, Dest, ResultHigh);
}
void OpDispatchBuilder::IMUL2SrcOp(OpcodeArgs) {
Ref Src1 = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
Ref Src2 = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
const auto Size = OpSizeFromSrc(Op);
const auto SizeBits = IR::OpSizeAsBits(Size);
Ref Dest {};
Ref ResultHigh {};
switch (Size) {
case OpSize::i8Bit:
case OpSize::i16Bit: {
Src1 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src1);
Src2 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src2);
Dest = _Mul(OpSize::i64Bit, Src1, Src2);
ResultHigh = _Sbfe(OpSize::i64Bit, SizeBits, SizeBits, Dest);
break;
}
case OpSize::i32Bit: {
ResultHigh = _SMull(Src1, Src2);
ResultHigh = _Sbfe(OpSize::i64Bit, SizeBits, SizeBits, ResultHigh);
// Flipped order to save a move
Dest = _Mul(OpSize::i32Bit, Src1, Src2);
break;
}
case OpSize::i64Bit: {
ResultHigh = _MulH(OpSize::i64Bit, Src1, Src2);
// Flipped order to save a move
Dest = _Mul(OpSize::i64Bit, Src1, Src2);
break;
}
default: FEX_UNREACHABLE;
}
StoreResult(GPRClass, Op, Dest, OpSize::iInvalid);
CalculateFlags_MUL(Size, Dest, ResultHigh);
}
void OpDispatchBuilder::IMULOp(OpcodeArgs) {
const auto Size = OpSizeFromSrc(Op);
const auto SizeBits = IR::OpSizeAsBits(Size);
Ref Src1 = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
Ref Src2 = LoadGPRRegister(X86State::REG_RAX);
if (Size != OpSize::i64Bit) {
Src1 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src1);
Src2 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src2);
}
// 64-bit special cased to save a move
Ref Result = Size < OpSize::i64Bit ? _Mul(OpSize::i64Bit, Src1, Src2) : nullptr;
Ref ResultHigh {};
if (Size == OpSize::i8Bit) {
// Result is stored in AX
StoreGPRRegister(X86State::REG_RAX, Result, OpSize::i16Bit);
ResultHigh = _Sbfe(OpSize::i64Bit, 8, 8, Result);
} else if (Size == OpSize::i16Bit) {
// 16bits stored in AX
// 16bits stored in DX
StoreGPRRegister(X86State::REG_RAX, Result, Size);
ResultHigh = _Sbfe(OpSize::i64Bit, 16, 16, Result);
StoreGPRRegister(X86State::REG_RDX, ResultHigh, Size);
} else if (Size == OpSize::i32Bit) {
// 32bits stored in EAX
// 32bits stored in EDX
// Make sure they get Zext correctly
auto LocalResult = _Bfe(OpSize::i64Bit, 32, 0, Result);
auto LocalResultHigh = _Bfe(OpSize::i64Bit, 32, 32, Result);
ResultHigh = _Sbfe(OpSize::i64Bit, 32, 32, Result);
Result = _Sbfe(OpSize::i64Bit, 32, 0, Result);
StoreGPRRegister(X86State::REG_RAX, LocalResult);
StoreGPRRegister(X86State::REG_RDX, LocalResultHigh);
} else if (Size == OpSize::i64Bit) {
if (!CTX->Config.Is64BitMode) {
LogMan::Msg::EFmt("Doesn't exist in 32bit mode");
DecodeFailure = true;
return;
}
// 64bits stored in RAX
// 64bits stored in RDX
ResultHigh = _MulH(OpSize::i64Bit, Src1, Src2);
Result = _Mul(OpSize::i64Bit, Src1, Src2);
StoreGPRRegister(X86State::REG_RAX, Result);
StoreGPRRegister(X86State::REG_RDX, ResultHigh);
}
CalculateFlags_MUL(Size, Result, ResultHigh);
}
void OpDispatchBuilder::MULOp(OpcodeArgs) {
const auto Size = OpSizeFromSrc(Op);
const auto SizeBits = IR::OpSizeAsBits(Size);
Ref Src1 = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
Ref Src2 = LoadGPRRegister(X86State::REG_RAX);
Ref Result;
if (Size != OpSize::i64Bit) {
Src1 = _Bfe(OpSize::i64Bit, SizeBits, 0, Src1);
Src2 = _Bfe(OpSize::i64Bit, SizeBits, 0, Src2);
Result = _UMul(OpSize::i64Bit, Src1, Src2);
}
Ref ResultHigh {};
if (Size == OpSize::i8Bit) {
// Result is stored in AX
StoreGPRRegister(X86State::REG_RAX, Result, OpSize::i16Bit);
ResultHigh = _Bfe(OpSize::i64Bit, 8, 8, Result);
} else if (Size == OpSize::i16Bit) {
// 16bits stored in AX
// 16bits stored in DX
StoreGPRRegister(X86State::REG_RAX, Result, Size);
ResultHigh = _Bfe(OpSize::i64Bit, 16, 16, Result);
StoreGPRRegister(X86State::REG_RDX, ResultHigh, Size);
} else if (Size == OpSize::i32Bit) {
// 32bits stored in EAX
// 32bits stored in EDX
Ref ResultLow = _Bfe(OpSize::i64Bit, 32, 0, Result);
ResultHigh = _Bfe(OpSize::i64Bit, 32, 32, Result);
StoreGPRRegister(X86State::REG_RAX, ResultLow);
StoreGPRRegister(X86State::REG_RDX, ResultHigh);
} else if (Size == OpSize::i64Bit) {
if (!CTX->Config.Is64BitMode) {
LogMan::Msg::EFmt("Doesn't exist in 32bit mode");
DecodeFailure = true;
return;
}
// 64bits stored in RAX
// 64bits stored in RDX
//
// Calculate high first to allow better RA.
ResultHigh = _UMulH(OpSize::i64Bit, Src1, Src2);
Result = _UMul(OpSize::i64Bit, Src1, Src2);
StoreGPRRegister(X86State::REG_RAX, Result);
StoreGPRRegister(X86State::REG_RDX, ResultHigh);
}
CalculateFlags_UMUL(ResultHigh);
}
void OpDispatchBuilder::NOTOp(OpcodeArgs) {
const auto Size = OpSizeFromSrc(Op);
const auto SizeBits = IR::OpSizeAsBits(Size);
Ref MaskConst {};
if (Size == OpSize::i64Bit) {
MaskConst = _Constant(~0ULL);
} else {
MaskConst = _Constant((1ULL << SizeBits) - 1);
}
if (DestIsLockedMem(Op)) {
HandledLock = true;
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
_AtomicXor(Size, MaskConst, DestMem);
} else if (!Op->Dest.IsGPR()) {
// GPR version plays fast and loose with sizes, be safe for memory tho.
Ref Src = LoadSource(GPRClass, Op, Op->Dest, Op->Flags);
Src = _Xor(OpSize::i64Bit, Src, MaskConst);
StoreResult(GPRClass, Op, Src, OpSize::iInvalid);
} else {
// Specially handle high bits so we can invert in place with the correct
// mask and a larger type.
auto Dest = Op->Dest;
if (Dest.Data.GPR.HighBits) {
LOGMAN_THROW_A_FMT(Size == OpSize::i8Bit, "Only 8-bit GPRs get high bits");
MaskConst = _Constant(0xFF00);
Dest.Data.GPR.HighBits = false;
}
// Always load full size, we explicitly want the upper bits to get the
// insert behaviour for free/implicitly.
const auto GPRSize = CTX->GetGPROpSize();
Ref Src = LoadSource_WithOpSize(GPRClass, Op, Dest, GPRSize, Op->Flags);
// For 8/16-bit, use 64-bit invert so we invert in place, while getting
// insert behaviour. For 32-bit, use 32-bit invert to zero the upper bits.
const auto EffectiveSize = Size == OpSize::i32Bit ? OpSize::i32Bit : GPRSize;
// If we're inverting the whole thing, use Not instead of Xor to save a constant.
if (Size >= OpSize::i32Bit) {
Src = _Not(EffectiveSize, Src);
} else {
Src = _Xor(EffectiveSize, Src, MaskConst);
}
// Always store 64-bit, the Not/Xor correctly handle the upper bits and this
// way we can delete the store.
StoreResult_WithOpSize(GPRClass, Op, Dest, Src, GPRSize, OpSize::iInvalid);
}
}
void OpDispatchBuilder::XADDOp(OpcodeArgs) {
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.LoadData = false});
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
Ref Result;
if (Op->Dest.IsGPR()) {
// If this is a GPR then we can just do an Add
Result = CalculateFlags_ADD(OpSizeFromSrc(Op), Dest, Src);
// Previous value in dest gets stored in src
StoreResult(GPRClass, Op, Op->Src[0], Dest, OpSize::iInvalid);
// Calculated value gets stored in dst (order is important if dst is same as src)
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
} else {
HandledLock = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK;
Dest = AppendSegmentOffset(Dest, Op->Flags);
auto Before = _AtomicFetchAdd(OpSizeFromSrc(Op), Src, Dest);
CalculateFlags_ADD(OpSizeFromSrc(Op), Before, Src);
StoreResult(GPRClass, Op, Op->Src[0], Before, OpSize::iInvalid);
}
}
void OpDispatchBuilder::PopcountOp(OpcodeArgs) {
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = GetSrcSize(Op) >= 4});
Src = _Popcount(OpSizeFromSrc(Op), Src);
StoreResult(GPRClass, Op, Src, OpSize::iInvalid);
// We need to set ZF while clearing the rest of NZCV. The result of a popcount
// is in the range [0, 63]. In particular, it is always positive. So a
// combined NZ test will correctly zero SF/CF/OF while setting ZF.
SetNZ_ZeroCV(OpSize::i32Bit, Src);
ZeroPF_AF();
}
Ref OpDispatchBuilder::CalculateAFForDecimal(Ref A) {
auto Nibble = _And(OpSize::i64Bit, A, _Constant(0xF));
auto Greater = _Select(FEXCore::IR::COND_UGT, Nibble, _Constant(9), _Constant(1), _Constant(0));
return _Or(OpSize::i64Bit, LoadAF(), Greater);
}
void OpDispatchBuilder::DAAOp(OpcodeArgs) {
CalculateDeferredFlags();
auto AL = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit);
auto CFInv = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC, true);
auto AF = CalculateAFForDecimal(AL);
// CF |= (AL > 0x99);
CFInv = _And(OpSize::i64Bit, CFInv, _Select(FEXCore::IR::COND_ULE, AL, _Constant(0x99), _Constant(1), _Constant(0)));
// AL = AF ? (AL + 0x6) : AL;
AL = _Select(FEXCore::IR::COND_NEQ, AF, _Constant(0), _Add(OpSize::i64Bit, AL, _Constant(0x6)), AL);
// AL = CF ? (AL + 0x60) : AL;
AL = _Select(FEXCore::IR::COND_EQ, CFInv, _Constant(0), _Add(OpSize::i64Bit, AL, _Constant(0x60)), AL);
// SF, ZF, PF set according to result. CF set per above. OF undefined.
StoreGPRRegister(X86State::REG_RAX, AL, OpSize::i8Bit);
SetNZ_ZeroCV(OpSize::i8Bit, AL);
SetCFInverted(CFInv);
CalculatePF(AL);
SetAFAndFixup(AF);
}
void OpDispatchBuilder::DASOp(OpcodeArgs) {
CalculateDeferredFlags();
auto AL = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit);
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
auto AF = CalculateAFForDecimal(AL);
// CF |= (AL > 0x99);
CF = _Or(OpSize::i64Bit, CF, _Select(FEXCore::IR::COND_UGT, AL, _Constant(0x99), _Constant(1), _Constant(0)));
// NewCF = CF | (AF && (Borrow from AL - 6))
auto NewCF = _Or(OpSize::i32Bit, CF, _Select(FEXCore::IR::COND_ULT, AL, _Constant(6), AF, CF));
// AL = AF ? (AL - 0x6) : AL;
AL = _Select(FEXCore::IR::COND_NEQ, AF, _Constant(0), _Sub(OpSize::i64Bit, AL, _Constant(0x6)), AL);
// AL = CF ? (AL - 0x60) : AL;
AL = _Select(FEXCore::IR::COND_NEQ, CF, _Constant(0), _Sub(OpSize::i64Bit, AL, _Constant(0x60)), AL);
// SF, ZF, PF set according to result. CF set per above. OF undefined.
StoreGPRRegister(X86State::REG_RAX, AL, OpSize::i8Bit);
SetNZ_ZeroCV(OpSize::i8Bit, AL);
SetCFDirect(NewCF);
CalculatePF(AL);
SetAFAndFixup(AF);
}
void OpDispatchBuilder::AAAOp(OpcodeArgs) {
auto A = LoadGPRRegister(X86State::REG_RAX);
auto AF = CalculateAFForDecimal(A);
// CF = AF, OF/SF/ZF/PF undefined
SetCFDirect_InvalidateNZV(AF);
SetAFAndFixup(AF);
CalculateDeferredFlags();
// AX = CF ? (AX + 0x106) : 0
A = NZCVSelect(OpSize::i32Bit, {COND_UGE} /* CF = 1 */, _Add(OpSize::i32Bit, A, _Constant(0x106)), A);
// AL = AL & 0x0F
A = _And(OpSize::i32Bit, A, _Constant(0xFF0F));
StoreGPRRegister(X86State::REG_RAX, A, OpSize::i16Bit);
}
void OpDispatchBuilder::AASOp(OpcodeArgs) {
auto A = LoadGPRRegister(X86State::REG_RAX);
auto AF = CalculateAFForDecimal(A);
// CF = AF, OF/SF/ZF/PF undefined
SetCFDirect_InvalidateNZV(AF);
SetAFAndFixup(AF);
CalculateDeferredFlags();
// AX = CF ? (AX - 0x106) : 0
A = NZCVSelect(OpSize::i32Bit, {COND_UGE} /* CF = 1 */, _Sub(OpSize::i32Bit, A, _Constant(0x106)), A);
// AL = AL & 0x0F
A = _And(OpSize::i32Bit, A, _Constant(0xFF0F));
StoreGPRRegister(X86State::REG_RAX, A, OpSize::i16Bit);
}
void OpDispatchBuilder::AAMOp(OpcodeArgs) {
auto AL = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit);
auto Imm8 = _Constant(Op->Src[0].Data.Literal.Value & 0xFF);
auto UDivOp = _UDiv(OpSize::i64Bit, AL, Imm8);
auto URemOp = _URem(OpSize::i64Bit, AL, Imm8);
auto Res = _AddShift(OpSize::i64Bit, URemOp, UDivOp, ShiftType::LSL, 8);
StoreGPRRegister(X86State::REG_RAX, Res, OpSize::i16Bit);
SetNZ_ZeroCV(OpSize::i8Bit, Res);
CalculatePF(Res);
InvalidateAF();
}
void OpDispatchBuilder::AADOp(OpcodeArgs) {
auto A = LoadGPRRegister(X86State::REG_RAX);
auto AH = _Lshr(OpSize::i32Bit, A, _Constant(8));
auto Imm8 = _Constant(Op->Src[0].Data.Literal.Value & 0xFF);
auto NewAL = _Add(OpSize::i64Bit, A, _Mul(OpSize::i64Bit, AH, Imm8));
auto Result = _And(OpSize::i64Bit, NewAL, _Constant(0xFF));
StoreGPRRegister(X86State::REG_RAX, Result, OpSize::i16Bit);
SetNZ_ZeroCV(OpSize::i8Bit, Result);
CalculatePF(Result);
InvalidateAF();
}
void OpDispatchBuilder::XLATOp(OpcodeArgs) {
Ref Src = MakeSegmentAddress(X86State::REG_RBX, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);
Ref Offset = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit);
AddressMode A = {.Base = Src, .Index = Offset, .AddrSize = OpSize::i64Bit};
auto Res = _LoadMemAutoTSO(GPRClass, OpSize::i8Bit, A, OpSize::i8Bit);
StoreGPRRegister(X86State::REG_RAX, Res, OpSize::i8Bit);
}
void OpDispatchBuilder::ReadSegmentReg(OpcodeArgs, OpDispatchBuilder::Segment Seg) {
// 64-bit only
// Doesn't hit the segment register optimization
const auto Size = OpSizeFromSrc(Op);
Ref Src {};
if (Seg == Segment::FS) {
Src = _LoadContext(Size, GPRClass, offsetof(FEXCore::Core::CPUState, fs_cached));
} else {
Src = _LoadContext(Size, GPRClass, offsetof(FEXCore::Core::CPUState, gs_cached));
}
StoreResult(GPRClass, Op, Src, OpSize::iInvalid);
}
void OpDispatchBuilder::WriteSegmentReg(OpcodeArgs, OpDispatchBuilder::Segment Seg) {
// Documentation claims that the 32-bit version of this instruction inserts in to the lower 32-bits of the segment
// This is incorrect and it instead zero extends the 32-bit value to 64-bit
const auto Size = OpSizeFromDst(Op);
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
if (Seg == Segment::FS) {
_StoreContext(Size, GPRClass, Src, offsetof(FEXCore::Core::CPUState, fs_cached));
} else {
_StoreContext(Size, GPRClass, Src, offsetof(FEXCore::Core::CPUState, gs_cached));
}
}
void OpDispatchBuilder::EnterOp(OpcodeArgs) {
const auto GPRSize = CTX->GetGPROpSize();
const uint64_t Value = Op->Src[0].Literal();
const uint16_t AllocSpace = Value & 0xFFFF;
const uint8_t Level = (Value >> 16) & 0x1F;
const auto PushValue = [&](IR::OpSize Size, Ref Src) -> Ref {
const auto GPRSize = CTX->GetGPROpSize();
auto OldSP = LoadGPRRegister(X86State::REG_RSP);
auto NewSP = _Push(GPRSize, Size, Src, OldSP);
// Store the new stack pointer
StoreGPRRegister(X86State::REG_RSP, NewSP);
return NewSP;
};
auto OldBP = LoadGPRRegister(X86State::REG_RBP);
auto NewSP = PushValue(GPRSize, OldBP);
auto temp_RBP = NewSP;
if (Level > 0) {
for (uint8_t i = 1; i < Level; ++i) {
auto Offset = _Constant(i * IR::OpSizeToSize(GPRSize));
auto MemLoc = _Sub(GPRSize, OldBP, Offset);
auto Mem = _LoadMem(GPRClass, GPRSize, MemLoc, GPRSize);
NewSP = PushValue(GPRSize, Mem);
}
NewSP = PushValue(GPRSize, temp_RBP);
}
NewSP = _Sub(GPRSize, NewSP, _Constant(AllocSpace));
StoreGPRRegister(X86State::REG_RSP, NewSP);
StoreGPRRegister(X86State::REG_RBP, temp_RBP);
}
void OpDispatchBuilder::SGDTOp(OpcodeArgs) {
auto DestAddress = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.LoadData = false});
// Store an emulated value in the format of:
// uint16_t Limit;
// {uint32_t,uint64_t} Base;
//
// Limit is always 0
// Base is always in kernel space at: 0xFFFFFFFFFFFE0000ULL
//
// Operand size prefix is ignored on this instruction, size purely depends on operating mode.
uint64_t GDTAddress = 0xFFFFFFFFFFFE0000ULL;
auto GDTStoreSize = OpSize::i64Bit;
if (!CTX->Config.Is64BitMode) {
// Mask off upper bits if 32-bit result.
GDTAddress &= ~0U;
GDTStoreSize = OpSize::i32Bit;
}
_StoreMemAutoTSO(GPRClass, OpSize::i16Bit, DestAddress, _Constant(0));
_StoreMemAutoTSO(GPRClass, GDTStoreSize, AddressMode {.Base = DestAddress, .Offset = 2, .AddrSize = OpSize::i64Bit}, _Constant(GDTAddress));
}
void OpDispatchBuilder::SMSWOp(OpcodeArgs) {
const bool IsMemDst = DestIsMem(Op);
IR::OpSize DstSize {OpSize::iInvalid};
Ref Const = _Constant((1U << 31) | ///< PG - Paging
(0U << 30) | ///< CD - Cache Disable
(0U << 29) | ///< NW - Not Writethrough (Legacy, now ignored)
///< [28:19] - Reserved
(1U << 18) | ///< AM - Alignment Mask
///< 17 - Reserved
(1U << 16) | ///< WP - Write Protect
///< [15:6] - Reserved
(1U << 5) | ///< NE - Numeric Error
(1U << 4) | ///< ET - Extension Type (Legacy, now reserved and 1)
(0U << 3) | ///< TS - Task Switched
(0U << 2) | ///< EM - Emulation
(1U << 1) | ///< MP - Monitor Coprocessor
(1U << 0)); ///< PE - Protection Enabled
if (CTX->Config.Is64BitMode) {
DstSize = X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0) == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? OpSize::i16Bit :
X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0) == X86Tables::DecodeFlags::FLAG_WIDENING_SIZE_LAST ? OpSize::i64Bit :
OpSize::i32Bit;
if (!IsMemDst && DstSize == OpSize::i32Bit) {
// Special-case version of `smsw ebx`. This instruction does an insert in to the lower 32-bits on 64-bit hosts.
// Override and insert.
auto Dest = LoadSource_WithOpSize(GPRClass, Op, Op->Dest, CTX->GetGPROpSize(), Op->Flags);
Const = _Bfi(OpSize::i64Bit, 32, 0, Dest, Const);
DstSize = OpSize::i64Bit;
}
} else {
DstSize = X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0) == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? OpSize::i16Bit : OpSize::i32Bit;
}
if (IsMemDst) {
// Memory destinatino always writes only 16-bits.
DstSize = OpSize::i16Bit;
}
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Const, DstSize, OpSize::iInvalid);
}
OpDispatchBuilder::CycleCounterPair OpDispatchBuilder::CycleCounter() {
Ref CounterLow {};
Ref CounterHigh {};
auto Counter = _CycleCounter();
if (CTX->Config.TSCScale) {
CounterLow = _Lshl(OpSize::i32Bit, Counter, _Constant(CTX->Config.TSCScale));
CounterHigh = _Lshr(OpSize::i64Bit, Counter, _Constant(32 - CTX->Config.TSCScale));
} else {
CounterLow = _Bfe(OpSize::i64Bit, 32, 0, Counter);
CounterHigh = _Bfe(OpSize::i64Bit, 32, 32, Counter);
}
return {
.CounterLow = CounterLow,
.CounterHigh = CounterHigh,
};
}
void OpDispatchBuilder::RDTSCOp(OpcodeArgs) {
auto Counter = CycleCounter();
StoreGPRRegister(X86State::REG_RAX, Counter.CounterLow);
StoreGPRRegister(X86State::REG_RDX, Counter.CounterHigh);
}
void OpDispatchBuilder::INCOp(OpcodeArgs) {
Ref Dest;
Ref Result;
const auto Size = GetSrcBitSize(Op);
auto OneConst = _Constant(OpSizeFromSrc(Op), 1);
const bool IsLocked = DestIsLockedMem(Op);
if (IsLocked) {
HandledLock = true;
Ref DestAddress = MakeSegmentAddress(Op, Op->Dest);
Dest = _AtomicFetchAdd(OpSizeFromSrc(Op), OneConst, DestAddress);
} else {
Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= 32});
}
CalculateDeferredFlags();
if (Size < 32 && CTX->HostFeatures.SupportsFlagM) {
// Addition producing upper garbage
Result = _Add(OpSize::i32Bit, Dest, OneConst);
CalculatePF(Result);
CalculateAF(Dest, OneConst);
// Correctly set NZ flags, preserving C
HandleNZCV_RMW();
_SetSmallNZV(OpSizeFromSrc(Op), Result);
// Fix up V flag. INC overflows only when incrementing a positive and
// getting a negative. So compare the sign bits to calculate V.
_RmifNZCV(_Andn(OpSize::i32Bit, Result, Dest), Size - 1, 1);
} else {
Result = CalculateFlags_ADD(OpSizeFromSrc(Op), Dest, OneConst, false);
}
if (!IsLocked) {
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
}
}
void OpDispatchBuilder::DECOp(OpcodeArgs) {
Ref Dest;
Ref Result;
const auto Size = GetSrcBitSize(Op);
auto OneConst = _Constant(OpSizeFromSrc(Op), 1);
const bool IsLocked = DestIsLockedMem(Op);
if (IsLocked) {
HandledLock = true;
Ref DestAddress = MakeSegmentAddress(Op, Op->Dest);
// Use Add instead of Sub to avoid a NEG
Dest = _AtomicFetchAdd(OpSizeFromSrc(Op), _Constant(OpSizeFromSrc(Op), -1), DestAddress);
} else {
Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= 32});
}
CalculateDeferredFlags();
if (Size < 32 && CTX->HostFeatures.SupportsFlagM) {
// Subtraction producing upper garbage
Result = _Sub(OpSize::i32Bit, Dest, OneConst);
CalculatePF(Result);
CalculateAF(Dest, OneConst);
// Correctly set NZ flags, preserving C
HandleNZCV_RMW();
_SetSmallNZV(OpSizeFromSrc(Op), Result);
// Fix up V flag. DEC overflows only when decrementing a negative and
// getting a positive. So compare the sign bits to calculate V.
_RmifNZCV(_Andn(OpSize::i32Bit, Dest, Result), Size - 1, 1);
} else {
Result = CalculateFlags_SUB(OpSizeFromSrc(Op), Dest, OneConst, false);
}
if (!IsLocked) {
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
}
}
void OpDispatchBuilder::STOSOp(OpcodeArgs) {
if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) {
LogMan::Msg::EFmt("Can't handle adddress size");
DecodeFailure = true;
return;
}
const auto Size = OpSizeFromSrc(Op);
const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) != 0;
if (!Repeat) {
// Src is used only for a store of the same size so allow garbage
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
// Only ES prefix
Ref Dest = MakeSegmentAddress(X86State::REG_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
// Store to memory where RDI points
_StoreMemAutoTSO(GPRClass, Size, Dest, Src, Size);
// Offset the pointer
Ref TailDest = LoadGPRRegister(X86State::REG_RDI);
StoreGPRRegister(X86State::REG_RDI, OffsetByDir(TailDest, IR::OpSizeToSize(Size)));
} else {
// FEX doesn't support partial faulting REP instructions.
// Converting this to a `MemSet` IR op optimizes this quite significantly in our codegen.
// If FEX is to gain support for faulting REP instructions, then this implementation needs to change significantly.
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
Ref Dest = LoadGPRRegister(X86State::REG_RDI);
// Only ES prefix
auto Segment = GetSegment(0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
Ref Counter = LoadGPRRegister(X86State::REG_RCX);
auto Result = _MemSet(CTX->IsAtomicTSOEnabled(), Size, Segment ?: InvalidNode, Dest, Src, Counter, LoadDir(1));
StoreGPRRegister(X86State::REG_RCX, _Constant(0));
StoreGPRRegister(X86State::REG_RDI, Result);
}
}
void OpDispatchBuilder::MOVSOp(OpcodeArgs) {
if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) {
LogMan::Msg::EFmt("Can't handle adddress size");
DecodeFailure = true;
return;
}
// RA now can handle these to be here, to avoid DF accesses
const auto Size = OpSizeFromSrc(Op);
if (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) {
auto SrcAddr = LoadGPRRegister(X86State::REG_RSI);
auto DstAddr = LoadGPRRegister(X86State::REG_RDI);
auto Counter = LoadGPRRegister(X86State::REG_RCX);
auto DstSegment = GetSegment(0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
auto SrcSegment = GetSegment(Op->Flags, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX);
if (DstSegment) {
DstAddr = _Add(OpSize::i64Bit, DstAddr, DstSegment);
}
if (SrcSegment) {
SrcAddr = _Add(OpSize::i64Bit, SrcAddr, SrcSegment);
}
Ref Result_Src = _AllocateGPR(false);
Ref Result_Dst = _AllocateGPR(false);
_MemCpy(CTX->IsAtomicTSOEnabled(), Size, DstAddr, SrcAddr, Counter, LoadDir(1), Result_Dst, Result_Src);
if (DstSegment) {
Result_Dst = _Sub(OpSize::i64Bit, Result_Dst, DstSegment);
}
if (SrcSegment) {
Result_Src = _Sub(OpSize::i64Bit, Result_Src, SrcSegment);
}
StoreGPRRegister(X86State::REG_RCX, _Constant(0));
StoreGPRRegister(X86State::REG_RDI, Result_Dst);
StoreGPRRegister(X86State::REG_RSI, Result_Src);
} else {
Ref RSI = MakeSegmentAddress(X86State::REG_RSI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);
Ref RDI = MakeSegmentAddress(X86State::REG_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
auto Src = _LoadMemAutoTSO(GPRClass, Size, RSI, Size);
// Store to memory where RDI points
_StoreMemAutoTSO(GPRClass, Size, RDI, Src, Size);
auto PtrDir = LoadDir(IR::OpSizeToSize(Size));
RSI = _Add(OpSize::i64Bit, RSI, PtrDir);
RDI = _Add(OpSize::i64Bit, RDI, PtrDir);
StoreGPRRegister(X86State::REG_RSI, RSI);
StoreGPRRegister(X86State::REG_RDI, RDI);
}
}
void OpDispatchBuilder::CMPSOp(OpcodeArgs) {
if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) {
LogMan::Msg::EFmt("Can't handle adddress size");
DecodeFailure = true;
return;
}
const auto Size = OpSizeFromSrc(Op);
bool Repeat = Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX);
if (!Repeat) {
// Default DS prefix
Ref Dest_RSI = MakeSegmentAddress(X86State::REG_RSI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);
// Only ES prefix
Ref Dest_RDI = MakeSegmentAddress(X86State::REG_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
auto Src1 = _LoadMemAutoTSO(GPRClass, Size, Dest_RDI, Size);
auto Src2 = _LoadMemAutoTSO(GPRClass, Size, Dest_RSI, Size);
CalculateFlags_SUB(OpSizeFromSrc(Op), Src2, Src1);
auto PtrDir = LoadDir(IR::OpSizeToSize(Size));
// Offset the pointer
Dest_RDI = _Add(OpSize::i64Bit, Dest_RDI, PtrDir);
StoreGPRRegister(X86State::REG_RDI, Dest_RDI);
// Offset second pointer
Dest_RSI = _Add(OpSize::i64Bit, Dest_RSI, PtrDir);
StoreGPRRegister(X86State::REG_RSI, Dest_RSI);
} else {
// Calculate flags early.
CalculateDeferredFlags();
bool REPE = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX;
// If rcx = 0, skip the whole loop.
Ref Counter = LoadGPRRegister(X86State::REG_RCX);
auto OuterJump = CondJump(Counter, {COND_EQ});
auto BeforeLoop = CreateNewCodeBlockAfter(GetCurrentBlock());
SetFalseJumpTarget(OuterJump, BeforeLoop);
SetCurrentCodeBlock(BeforeLoop);
StartNewBlock();
ForeachDirection([this, Op, Size, REPE](int PtrDir) {
IRPair<IROp_CondJump> InnerJump;
auto JumpIntoLoop = Jump();
// Setup for the loop
auto LoopHeader = CreateNewCodeBlockAfter(GetCurrentBlock());
SetCurrentCodeBlock(LoopHeader);
StartNewBlock();
SetJumpTarget(JumpIntoLoop, LoopHeader);
// Working loop
{
// Default DS prefix
Ref Dest_RSI = MakeSegmentAddress(X86State::REG_RSI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);
// Only ES prefix
Ref Dest_RDI = MakeSegmentAddress(X86State::REG_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
auto Src1 = _LoadMemAutoTSO(GPRClass, Size, Dest_RDI, Size);
auto Src2 = _LoadMem(GPRClass, Size, Dest_RSI, Size);
// We'll calculate PF/AF after the loop, so use them as temporaries here.
StoreRegister(Core::CPUState::PF_AS_GREG, false, Src1);
StoreRegister(Core::CPUState::AF_AS_GREG, false, Src2);
Ref TailCounter = LoadGPRRegister(X86State::REG_RCX);
// Decrement counter
TailCounter = _SubWithFlags(OpSize::i64Bit, TailCounter, _Constant(1));
// Store the counter since we don't have phis
StoreGPRRegister(X86State::REG_RCX, TailCounter);
// Offset the pointer
Dest_RDI = _Add(OpSize::i64Bit, Dest_RDI, _Constant(PtrDir * IR::OpSizeToSize(Size)));
StoreGPRRegister(X86State::REG_RDI, Dest_RDI);
// Offset second pointer
Dest_RSI = _Add(OpSize::i64Bit, Dest_RSI, _Constant(PtrDir * IR::OpSizeToSize(Size)));
StoreGPRRegister(X86State::REG_RSI, Dest_RSI);
// If TailCounter != 0, compare sources.
// If TailCounter == 0, set ZF iff that would break.
_CondSubNZCV(OpSize::i64Bit, Src2, Src1, {COND_NEQ}, REPE ? 0 : (1 << 2) /* Z */);
CachedNZCV = nullptr;
NZCVDirty = false;
InnerJump = CondJumpNZCV({REPE ? COND_EQ : COND_NEQ});
// Jump back to the start if we have more work to do
SetTrueJumpTarget(InnerJump, LoopHeader);
}
// Make sure to start a new block after ending this one
auto LoopEnd = CreateNewCodeBlockAfter(GetCurrentBlock());
SetFalseJumpTarget(InnerJump, LoopEnd);
SetCurrentCodeBlock(LoopEnd);
StartNewBlock();
});
// Make sure to start a new block after ending this one
{
// Grab the sources from the last iteration so we can set flags.
auto Src1 = LoadGPR(Core::CPUState::PF_AS_GREG);
auto Src2 = LoadGPR(Core::CPUState::AF_AS_GREG);
CalculateFlags_SUB(OpSizeFromSrc(Op), Src2, Src1);
}
auto Jump_ = Jump();
auto Exit = CreateNewCodeBlockAfter(GetCurrentBlock());
SetJumpTarget(Jump_, Exit);
SetTrueJumpTarget(OuterJump, Exit);
SetCurrentCodeBlock(Exit);
StartNewBlock();
}
}
void OpDispatchBuilder::LODSOp(OpcodeArgs) {
if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) {
LogMan::Msg::EFmt("Can't handle adddress size");
DecodeFailure = true;
return;
}
const auto Size = OpSizeFromSrc(Op);
const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) != 0;
if (!Repeat) {
Ref Dest_RSI = MakeSegmentAddress(X86State::REG_RSI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);
auto Src = _LoadMemAutoTSO(GPRClass, Size, Dest_RSI, Size);
StoreResult(GPRClass, Op, Src, OpSize::iInvalid);
// Offset the pointer
Ref TailDest_RSI = LoadGPRRegister(X86State::REG_RSI);
StoreGPRRegister(X86State::REG_RSI, OffsetByDir(TailDest_RSI, IR::OpSizeToSize(Size)));
} else {
// Calculate flags early. because end of block
CalculateDeferredFlags();
ForeachDirection([this, Op, Size](int PtrDir) {
// XXX: Theoretically LODS could be optimized to
// RSI += {-}(RCX * Size)
// RAX = [RSI - Size]
// But this might violate the case of an application scanning pages for read permission and catching the fault
// May or may not matter
auto JumpStart = Jump();
// Make sure to start a new block after ending this one
auto LoopStart = CreateNewCodeBlockAfter(GetCurrentBlock());
SetJumpTarget(JumpStart, LoopStart);
SetCurrentCodeBlock(LoopStart);
StartNewBlock();
Ref Counter = LoadGPRRegister(X86State::REG_RCX);
// Can we end the block?
// We leave if RCX = 0
auto CondJump_ = CondJump(Counter, {COND_EQ});
auto LoopTail = CreateNewCodeBlockAfter(LoopStart);
SetFalseJumpTarget(CondJump_, LoopTail);
SetCurrentCodeBlock(LoopTail);
StartNewBlock();
// Working loop
{
Ref Dest_RSI = MakeSegmentAddress(X86State::REG_RSI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);
auto Src = _LoadMemAutoTSO(GPRClass, Size, Dest_RSI, Size);
StoreResult(GPRClass, Op, Src, OpSize::iInvalid);
Ref TailCounter = LoadGPRRegister(X86State::REG_RCX);
Ref TailDest_RSI = LoadGPRRegister(X86State::REG_RSI);
// Decrement counter
TailCounter = _Sub(OpSize::i64Bit, TailCounter, _Constant(1));
// Store the counter since we don't have phis
StoreGPRRegister(X86State::REG_RCX, TailCounter);
// Offset the pointer
TailDest_RSI = _Add(OpSize::i64Bit, TailDest_RSI, _Constant(PtrDir * IR::OpSizeToSize(Size)));
StoreGPRRegister(X86State::REG_RSI, TailDest_RSI);
// Jump back to the start, we have more work to do
Jump(LoopStart);
}
// Make sure to start a new block after ending this one
auto LoopEnd = CreateNewCodeBlockAfter(LoopTail);
SetTrueJumpTarget(CondJump_, LoopEnd);
SetCurrentCodeBlock(LoopEnd);
StartNewBlock();
});
}
}
void OpDispatchBuilder::SCASOp(OpcodeArgs) {
if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) {
LogMan::Msg::EFmt("Can't handle adddress size");
DecodeFailure = true;
return;
}
const auto Size = OpSizeFromSrc(Op);
const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX)) != 0;
if (!Repeat) {
Ref Dest_RDI = MakeSegmentAddress(X86State::REG_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
auto Src1 = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
auto Src2 = _LoadMemAutoTSO(GPRClass, Size, Dest_RDI, Size);
CalculateFlags_SUB(OpSizeFromSrc(Op), Src1, Src2);
// Offset the pointer
Ref TailDest_RDI = LoadGPRRegister(X86State::REG_RDI);
StoreGPRRegister(X86State::REG_RDI, OffsetByDir(TailDest_RDI, IR::OpSizeToSize(Size)));
} else {
// Calculate flags early. because end of block
CalculateDeferredFlags();
ForeachDirection([this, Op, Size](int Dir) {
bool REPE = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX;
auto JumpStart = Jump();
// Make sure to start a new block after ending this one
auto LoopStart = CreateNewCodeBlockAfter(GetCurrentBlock());
SetJumpTarget(JumpStart, LoopStart);
SetCurrentCodeBlock(LoopStart);
StartNewBlock();
Ref Counter = LoadGPRRegister(X86State::REG_RCX);
// Can we end the block?
// We leave if RCX = 0
auto CondJump_ = CondJump(Counter, {COND_EQ});
IRPair<IROp_CondJump> InternalCondJump;
auto LoopTail = CreateNewCodeBlockAfter(LoopStart);
SetFalseJumpTarget(CondJump_, LoopTail);
SetCurrentCodeBlock(LoopTail);
StartNewBlock();
// Working loop
{
Ref Dest_RDI = MakeSegmentAddress(X86State::REG_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
auto Src1 = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
auto Src2 = _LoadMemAutoTSO(GPRClass, Size, Dest_RDI, Size);
CalculateFlags_SUB(OpSizeFromSrc(Op), Src1, Src2);
// Calculate flags early.
CalculateDeferredFlags();
Ref TailCounter = LoadGPRRegister(X86State::REG_RCX);
Ref TailDest_RDI = LoadGPRRegister(X86State::REG_RDI);
// Decrement counter
TailCounter = _Sub(OpSize::i64Bit, TailCounter, _Constant(1));
// Store the counter since we don't have phis
StoreGPRRegister(X86State::REG_RCX, TailCounter);
// Offset the pointer
TailDest_RDI = _Add(OpSize::i64Bit, TailDest_RDI, _Constant(Dir * IR::OpSizeToSize(Size)));
StoreGPRRegister(X86State::REG_RDI, TailDest_RDI);
CalculateDeferredFlags();
InternalCondJump = CondJumpNZCV({REPE ? COND_EQ : COND_NEQ});
// Jump back to the start if we have more work to do
SetTrueJumpTarget(InternalCondJump, LoopStart);
}
// Make sure to start a new block after ending this one
auto LoopEnd = CreateNewCodeBlockAfter(LoopTail);
SetTrueJumpTarget(CondJump_, LoopEnd);
SetFalseJumpTarget(InternalCondJump, LoopEnd);
SetCurrentCodeBlock(LoopEnd);
StartNewBlock();
});
}
}
void OpDispatchBuilder::BSWAPOp(OpcodeArgs) {
Ref Dest;
const auto Size = OpSizeFromSrc(Op);
if (Size == OpSize::i16Bit) {
// BSWAP of 16bit is undef. ZEN+ causes the lower 16bits to get zero'd
Dest = _Constant(0);
} else {
Dest = LoadSource_WithOpSize(GPRClass, Op, Op->Dest, CTX->GetGPROpSize(), Op->Flags);
Dest = _Rev(Size, Dest);
}
StoreResult(GPRClass, Op, Dest, OpSize::iInvalid);
}
void OpDispatchBuilder::PUSHFOp(OpcodeArgs) {
const auto Size = OpSizeFromSrc(Op);
Ref Src = GetPackedRFLAG();
Push(Size, Src);
}
void OpDispatchBuilder::POPFOp(OpcodeArgs) {
const auto Size = OpSizeFromSrc(Op);
Ref Src = Pop(Size);
// Add back our flag constants
// Bit 1 is always 1
// Bit 9 is always 1 because we always have interrupts enabled
Src = _Or(OpSize::i64Bit, Src, _Constant(Size, 0x202));
SetPackedRFLAG(false, Src);
}
void OpDispatchBuilder::NEGOp(OpcodeArgs) {
HandledLock = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;
const auto Size = OpSizeFromSrc(Op);
auto ZeroConst = _Constant(0);
if (DestIsLockedMem(Op)) {
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
Ref Dest = _AtomicFetchNeg(Size, DestMem);
CalculateFlags_SUB(Size, ZeroConst, Dest);
} else {
Ref Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
Ref Result = CalculateFlags_SUB(Size, ZeroConst, Dest);
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
}
}
void OpDispatchBuilder::DIVOp(OpcodeArgs) {
// This loads the divisor
Ref Divisor = LoadSource(GPRClass, Op, Op->Dest, Op->Flags);
const auto GPRSize = CTX->GetGPROpSize();
const auto Size = OpSizeFromSrc(Op);
if (Size == OpSize::i8Bit) {
Ref Src1 = LoadGPRRegister(X86State::REG_RAX, OpSize::i16Bit);
auto UDivOp = _UDiv(OpSize::i16Bit, Src1, Divisor);
auto URemOp = _URem(OpSize::i16Bit, Src1, Divisor);
// AX[15:0] = concat<URem[7:0]:UDiv[7:0]>
auto ResultAX = _Bfi(GPRSize, 8, 8, UDivOp, URemOp);
StoreGPRRegister(X86State::REG_RAX, ResultAX, OpSize::i16Bit);
} else if (Size == OpSize::i16Bit) {
Ref Src1 = LoadGPRRegister(X86State::REG_RAX);
Ref Src2 = LoadGPRRegister(X86State::REG_RDX);
auto UDivOp = _LUDiv(OpSize::i16Bit, Src1, Src2, Divisor);
auto URemOp = _LURem(OpSize::i16Bit, Src1, Src2, Divisor);
StoreGPRRegister(X86State::REG_RAX, UDivOp, Size);
StoreGPRRegister(X86State::REG_RDX, URemOp, Size);
} else if (Size == OpSize::i32Bit) {
Ref Src1 = LoadGPRRegister(X86State::REG_RAX);
Ref Src2 = LoadGPRRegister(X86State::REG_RDX);
Ref UDivOp = _Bfe(OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, _LUDiv(OpSize::i32Bit, Src1, Src2, Divisor));
Ref URemOp = _Bfe(OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, _LURem(OpSize::i32Bit, Src1, Src2, Divisor));
StoreGPRRegister(X86State::REG_RAX, UDivOp);
StoreGPRRegister(X86State::REG_RDX, URemOp);
} else if (Size == OpSize::i64Bit) {
if (!CTX->Config.Is64BitMode) {
LogMan::Msg::EFmt("Doesn't exist in 32bit mode");
DecodeFailure = true;
return;
}
Ref Src1 = LoadGPRRegister(X86State::REG_RAX);
Ref Src2 = LoadGPRRegister(X86State::REG_RDX);
auto UDivOp = _LUDiv(OpSize::i64Bit, Src1, Src2, Divisor);
auto URemOp = _LURem(OpSize::i64Bit, Src1, Src2, Divisor);
StoreGPRRegister(X86State::REG_RAX, UDivOp);
StoreGPRRegister(X86State::REG_RDX, URemOp);
}
}
void OpDispatchBuilder::IDIVOp(OpcodeArgs) {
// This loads the divisor
Ref Divisor = LoadSource(GPRClass, Op, Op->Dest, Op->Flags);
const auto GPRSize = CTX->GetGPROpSize();
const auto Size = OpSizeFromSrc(Op);
if (Size == OpSize::i8Bit) {
Ref Src1 = LoadGPRRegister(X86State::REG_RAX);
Src1 = _Sbfe(OpSize::i64Bit, 16, 0, Src1);
Divisor = _Sbfe(OpSize::i64Bit, 8, 0, Divisor);
auto UDivOp = _Div(OpSize::i64Bit, Src1, Divisor);
auto URemOp = _Rem(OpSize::i64Bit, Src1, Divisor);
// AX[15:0] = concat<URem[7:0]:UDiv[7:0]>
auto ResultAX = _Bfi(GPRSize, 8, 8, UDivOp, URemOp);
StoreGPRRegister(X86State::REG_RAX, ResultAX, OpSize::i16Bit);
} else if (Size == OpSize::i16Bit) {
Ref Src1 = LoadGPRRegister(X86State::REG_RAX);
Ref Src2 = LoadGPRRegister(X86State::REG_RDX);
auto UDivOp = _LDiv(OpSize::i16Bit, Src1, Src2, Divisor);
auto URemOp = _LRem(OpSize::i16Bit, Src1, Src2, Divisor);
StoreGPRRegister(X86State::REG_RAX, UDivOp, Size);
StoreGPRRegister(X86State::REG_RDX, URemOp, Size);
} else if (Size == OpSize::i32Bit) {
Ref Src1 = LoadGPRRegister(X86State::REG_RAX);
Ref Src2 = LoadGPRRegister(X86State::REG_RDX);
Ref UDivOp = _Bfe(OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, _LDiv(OpSize::i32Bit, Src1, Src2, Divisor));
Ref URemOp = _Bfe(OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, _LRem(OpSize::i32Bit, Src1, Src2, Divisor));
StoreGPRRegister(X86State::REG_RAX, UDivOp);
StoreGPRRegister(X86State::REG_RDX, URemOp);
} else if (Size == OpSize::i64Bit) {
if (!CTX->Config.Is64BitMode) {
LogMan::Msg::EFmt("Doesn't exist in 32bit mode");
DecodeFailure = true;
return;
}
Ref Src1 = LoadGPRRegister(X86State::REG_RAX);
Ref Src2 = LoadGPRRegister(X86State::REG_RDX);
auto UDivOp = _LDiv(OpSize::i64Bit, Src1, Src2, Divisor);
auto URemOp = _LRem(OpSize::i64Bit, Src1, Src2, Divisor);
StoreGPRRegister(X86State::REG_RAX, UDivOp);
StoreGPRRegister(X86State::REG_RDX, URemOp);
}
}
void OpDispatchBuilder::BSFOp(OpcodeArgs) {
const auto GPRSize = CTX->GetGPROpSize();
const auto DstSize = OpSizeFromDst(Op) == OpSize::i16Bit ? OpSize::i16Bit : GPRSize;
Ref Dest = LoadSource_WithOpSize(GPRClass, Op, Op->Dest, DstSize, Op->Flags, {.AllowUpperGarbage = true});
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
// Find the LSB of this source
auto Result = _FindLSB(OpSizeFromSrc(Op), Src);
// OF, SF, AF, PF, CF all undefined
// ZF is set to 1 if the source was zero
SetZ_InvalidateNCV(OpSizeFromSrc(Op), Src);
// If Src was zero then the destination doesn't get modified.
//
// Although Intel does not guarantee that semantic, AMD does and Intel
// hardware satisfies it. We provide the stronger AMD behaviour as
// applications might rely on that in the wild.
auto SelectOp = NZCVSelect(GPRSize, {COND_EQ}, Dest, Result);
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, SelectOp, DstSize, OpSize::iInvalid);
}
void OpDispatchBuilder::BSROp(OpcodeArgs) {
const auto GPRSize = CTX->GetGPROpSize();
const auto DstSize = OpSizeFromDst(Op) == OpSize::i16Bit ? OpSize::i16Bit : GPRSize;
Ref Dest = LoadSource_WithOpSize(GPRClass, Op, Op->Dest, DstSize, Op->Flags, {.AllowUpperGarbage = true});
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
// Find the MSB of this source
auto Result = _FindMSB(OpSizeFromSrc(Op), Src);
// OF, SF, AF, PF, CF all undefined
// ZF is set to 1 if the source was zero
SetZ_InvalidateNCV(OpSizeFromSrc(Op), Src);
// If Src was zero then the destination doesn't get modified
auto SelectOp = NZCVSelect(GPRSize, {COND_EQ}, Dest, Result);
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, SelectOp, DstSize, OpSize::iInvalid);
}
void OpDispatchBuilder::CMPXCHGOp(OpcodeArgs) {
// CMPXCHG ModRM, reg, {RAX}
// MemData = *ModRM.dest
// if (RAX == MemData)
// modRM.dest = reg;
// ZF = 1
// else
// ZF = 0
// RAX = MemData
//
// CASL Xs, Xt, Xn
// MemData = *Xn
// if (MemData == Xs)
// *Xn = Xt
// Xs = MemData
const auto GPRSize = CTX->GetGPROpSize();
auto Size = OpSizeFromSrc(Op);
if (Op->Dest.IsGPR()) {
// This is our source register
Ref Src2 = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
Ref Src3 = LoadGPRRegister(X86State::REG_RAX);
// If the destination is also the accumulator, we get some algebraic
// simplifications. Not sure if this is actually hit but it's in
// InstCountCI.
bool Trivial = Op->Dest.Data.GPR.GPR == X86State::REG_RAX && !Op->Dest.IsGPRDirect() && !Op->Dest.Data.GPR.HighBits;
Ref Src1 {};
Ref Src1Lower {};
if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {
Src1 = LoadSource_WithOpSize(GPRClass, Op, Op->Dest, GPRSize, Op->Flags, {.AllowUpperGarbage = true});
Src1Lower = _Bfe(GPRSize, IR::OpSizeAsBits(Size), 0, Src1);
} else {
Src1 = LoadSource_WithOpSize(GPRClass, Op, Op->Dest, Size, Op->Flags, {.AllowUpperGarbage = true});
Src1Lower = Src1;
}
// Compare RAX with the destination, setting flags accordingly.
CalculateFlags_SUB(OpSizeFromSrc(Op), Src3, Src1Lower);
CalculateDeferredFlags();
if (!Trivial) {
if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {
// This allows us to only hit the ZEXT case on failure
Ref RAXResult = NZCVSelect(OpSize::i64Bit, {COND_EQ}, Src3, Src1Lower);
// When the size is 4 we need to make sure not zext the GPR when the comparison fails
StoreGPRRegister(X86State::REG_RAX, RAXResult);
} else {
StoreGPRRegister(X86State::REG_RAX, Src1Lower, Size);
}
}
// Op1 = RAX == Op1 ? Op2 : Op1
// If they match then set the rm operand to the input
// else don't set the rm operand
Ref DestResult = Trivial ? Src2 : NZCVSelect(OpSize::i64Bit, CondClassType {COND_EQ}, Src2, Src1);
// Store in to GPR Dest
if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, DestResult, GPRSize, OpSize::iInvalid);
} else {
StoreResult(GPRClass, Op, DestResult, OpSize::iInvalid);
}
} else {
Ref Src2 = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags);
HandledLock = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK;
Ref Src3 {};
Ref Src3Lower {};
if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {
Src3 = LoadGPRRegister(X86State::REG_RAX);
Src3Lower = _Bfe(OpSize::i32Bit, 32, 0, Src3);
} else {
Src3 = LoadGPRRegister(X86State::REG_RAX, Size);
Src3Lower = Src3;
}
// If this is a memory location then we want the pointer to it
Ref Src1 = MakeSegmentAddress(Op, Op->Dest);
// DataSrc = *Src1
// if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc
// This will write to memory! Careful!
// Third operand must be a calculated guest memory address
Ref CASResult = _CAS(Size, Src3Lower, Src2, Src1);
Ref RAXResult = CASResult;
CalculateFlags_SUB(OpSizeFromSrc(Op), Src3Lower, CASResult);
CalculateDeferredFlags();
if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {
// This allows us to only hit the ZEXT case on failure
RAXResult = _NZCVSelect(OpSize::i64Bit, {COND_EQ}, Src3, CASResult);
Size = OpSize::i64Bit;
}
// RAX gets the result of the CAS op
StoreGPRRegister(X86State::REG_RAX, RAXResult, Size);
}
}
void OpDispatchBuilder::CMPXCHGPairOp(OpcodeArgs) {
// Calculate flags early.
CalculateDeferredFlags();
// REX.W used to determine if it is 16byte or 8byte
// Unlike CMPXCHG, the destination can only be a memory location
const auto Size = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;
HandledLock = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;
// If this is a memory location then we want the pointer to it
Ref Src1 = MakeSegmentAddress(Op, Op->Dest);
// Load the full 64-bit registers, all the users ignore the upper 32-bits for
// 32-bit only cmpxchg. This saves some zero extension.
Ref Expected_Lower = LoadGPRRegister(X86State::REG_RAX);
Ref Expected_Upper = LoadGPRRegister(X86State::REG_RDX);
Ref Desired_Lower = LoadGPRRegister(X86State::REG_RBX);
Ref Desired_Upper = LoadGPRRegister(X86State::REG_RCX);
// ssa0 = Expected
// ssa1 = Desired
// ssa2 = MemoryLocation
// DataSrc = *MemSrc
// if (DataSrc == Expected) { *MemSrc == Desired; } Expected = DataSrc
// This will write to memory! Careful!
// Third operand must be a calculated guest memory address
Ref Result_Lower = _AllocateGPR(true);
Ref Result_Upper = _AllocateGPRAfter(Result_Lower);
_CASPair(Size, Expected_Lower, Expected_Upper, Desired_Lower, Desired_Upper, Src1, Result_Lower, Result_Upper);
HandleNZCV_RMW();
_CmpPairZ(Size, Result_Lower, Result_Upper, Expected_Lower, Expected_Upper);
CalculateDeferredFlags();
auto UpdateIfNotZF = [this](auto Reg, auto Value) {
// Always use 64-bit csel to preserve existing upper bits. If we have a
// 32-bit cmpxchg in a 64-bit context, Value will be zeroed in upper bits.
StoreGPRRegister(Reg, NZCVSelect(OpSize::i64Bit, {COND_NEQ}, Value, LoadGPRRegister(Reg)));
};
UpdateIfNotZF(X86State::REG_RAX, Result_Lower);
UpdateIfNotZF(X86State::REG_RDX, Result_Upper);
}
void OpDispatchBuilder::CreateJumpBlocks(const fextl::vector<FEXCore::Frontend::Decoder::DecodedBlocks>* Blocks) {
Ref PrevCodeBlock {};
for (auto& Target : *Blocks) {
auto CodeNode = CreateCodeNode();
JumpTargets.try_emplace(Target.Entry, JumpTargetInfo {CodeNode, false});
if (PrevCodeBlock) {
LinkCodeBlocks(PrevCodeBlock, CodeNode);
}
PrevCodeBlock = CodeNode;
}
}
void OpDispatchBuilder::BeginFunction(uint64_t RIP, const fextl::vector<FEXCore::Frontend::Decoder::DecodedBlocks>* Blocks, uint32_t NumInstructions) {
Entry = RIP;
auto IRHeader = _IRHeader(InvalidNode, RIP, 0, NumInstructions);
CreateJumpBlocks(Blocks);
auto Block = GetNewJumpBlock(RIP);
SetCurrentCodeBlock(Block);
IRHeader.first->Blocks = Block->Wrapped(DualListData.ListBegin());
CurrentHeader = IRHeader.first;
}
void OpDispatchBuilder::Finalize() {
// This usually doesn't emit any IR but in the case of hitting the block instruction limit it will
FlushRegisterCache();
const auto GPRSize = CTX->GetGPROpSize();
// Node 0 is invalid node
Ref RealNode = reinterpret_cast<Ref>(GetNode(1));
[[maybe_unused]] const FEXCore::IR::IROp_Header* IROp = RealNode->Op(DualListData.DataBegin());
LOGMAN_THROW_A_FMT(IROp->Op == OP_IRHEADER, "First op in function must be our header");
// Let's walk the jump blocks and see if we have handled every block target
for (auto& Handler : JumpTargets) {
if (Handler.second.HaveEmitted) {
continue;
}
// We haven't emitted. Dump out to the dispatcher
SetCurrentCodeBlock(Handler.second.BlockEntry);
ExitFunction(_EntrypointOffset(GPRSize, Handler.first - Entry));
}
}
uint8_t OpDispatchBuilder::GetDstSize(X86Tables::DecodedOp Op) const {
const uint32_t DstSizeFlag = X86Tables::DecodeFlags::GetSizeDstFlags(Op->Flags);
LOGMAN_THROW_A_FMT(DstSizeFlag != 0 && DstSizeFlag != X86Tables::DecodeFlags::SIZE_MASK, "Invalid destination size for op");
return 1u << (DstSizeFlag - 1);
}
uint8_t OpDispatchBuilder::GetSrcSize(X86Tables::DecodedOp Op) const {
const uint32_t SrcSizeFlag = X86Tables::DecodeFlags::GetSizeSrcFlags(Op->Flags);
LOGMAN_THROW_A_FMT(SrcSizeFlag != 0 && SrcSizeFlag != X86Tables::DecodeFlags::SIZE_MASK, "Invalid destination size for op");
return 1u << (SrcSizeFlag - 1);
}
uint32_t OpDispatchBuilder::GetSrcBitSize(X86Tables::DecodedOp Op) const {
return GetSrcSize(Op) * 8;
}
uint32_t OpDispatchBuilder::GetDstBitSize(X86Tables::DecodedOp Op) const {
return GetDstSize(Op) * 8;
}
Ref OpDispatchBuilder::GetSegment(uint32_t Flags, uint32_t DefaultPrefix, bool Override) {
const auto GPRSize = CTX->GetGPROpSize();
if (CTX->Config.Is64BitMode) {
if (Flags & FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX) {
return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, fs_cached));
} else if (Flags & FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {
return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, gs_cached));
}
// If there was any other segment in 64bit then it is ignored
} else {
uint32_t Prefix = Flags & FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS;
if (!Prefix || Override) {
// If there was no prefix then use the default one if available
// Or the argument only uses a specific prefix (with override set)
Prefix = DefaultPrefix;
}
// With the segment register optimization we store the GDT bases directly in the segment register to remove indexed loads
Ref SegmentResult {};
switch (Prefix) {
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
SegmentResult = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, es_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
SegmentResult = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, cs_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
SegmentResult = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, ss_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
SegmentResult = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, ds_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:
SegmentResult = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, fs_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:
SegmentResult = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, gs_cached));
break;
default: break; // Do nothing
}
CheckLegacySegmentRead(SegmentResult, Prefix);
return SegmentResult;
}
return nullptr;
}
Ref OpDispatchBuilder::AppendSegmentOffset(Ref Value, uint32_t Flags, uint32_t DefaultPrefix, bool Override) {
auto Segment = GetSegment(Flags, DefaultPrefix, Override);
if (Segment) {
Value = _Add(std::max(OpSize::i32Bit, std::max(GetOpSize(Value), GetOpSize(Segment))), Value, Segment);
}
return Value;
}
void OpDispatchBuilder::CheckLegacySegmentRead(Ref NewNode, uint32_t SegmentReg) {
#ifndef FEX_DISABLE_TELEMETRY
if (SegmentReg == FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX || SegmentReg == FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {
// FS and GS segments aren't considered legacy.
return;
}
if (!(SegmentsNeedReadCheck & SegmentReg)) {
// If the block has done multiple reads of a segment register then skip redundant read checks.
// Segment write will cause another read check.
return;
}
if (CTX->Config.DisableTelemetry()) {
// Telemetry disabled at runtime.
return;
}
FEXCore::Telemetry::TelemetryType TelemIndex {};
switch (SegmentReg) {
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_ES;
SegmentsNeedReadCheck &= ~FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX;
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_CS;
SegmentsNeedReadCheck &= ~FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX;
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_SS;
SegmentsNeedReadCheck &= ~FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX;
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_DS;
SegmentsNeedReadCheck &= ~FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX;
break;
default: FEX_UNREACHABLE;
}
// Will set the telemetry value if NewNode is != 0
_TelemetrySetValue(NewNode, TelemIndex);
#endif
}
void OpDispatchBuilder::CheckLegacySegmentWrite(Ref NewNode, uint32_t SegmentReg) {
#ifndef FEX_DISABLE_TELEMETRY
if (SegmentReg == FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX || SegmentReg == FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {
// FS and GS segments aren't considered legacy.
return;
}
if (CTX->Config.DisableTelemetry()) {
// Telemetry disabled at runtime.
return;
}
FEXCore::Telemetry::TelemetryType TelemIndex {};
switch (SegmentReg) {
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_ES;
SegmentsNeedReadCheck |= FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX;
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_CS;
SegmentsNeedReadCheck |= FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX;
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_SS;
SegmentsNeedReadCheck |= FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX;
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_DS;
SegmentsNeedReadCheck |= FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX;
break;
default: FEX_UNREACHABLE;
}
// Will set the telemetry value if NewNode is != 0
_TelemetrySetValue(NewNode, TelemIndex);
#endif
}
void OpDispatchBuilder::UpdatePrefixFromSegment(Ref Segment, uint32_t SegmentReg) {
// Use BFE to extract the selector index in bits [15,3] of the segment register.
// In some cases the upper 16-bits of the 32-bit GPR contain garbage to ignore.
Segment = _Bfe(OpSize::i32Bit, 16 - 3, 3, Segment);
auto NewSegment = _LoadContextIndexed(Segment, OpSize::i32Bit, offsetof(FEXCore::Core::CPUState, gdt[0]), 4, GPRClass);
CheckLegacySegmentWrite(NewSegment, SegmentReg);
switch (SegmentReg) {
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
_StoreContext(OpSize::i32Bit, GPRClass, NewSegment, offsetof(FEXCore::Core::CPUState, es_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
_StoreContext(OpSize::i32Bit, GPRClass, NewSegment, offsetof(FEXCore::Core::CPUState, cs_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
_StoreContext(OpSize::i32Bit, GPRClass, NewSegment, offsetof(FEXCore::Core::CPUState, ss_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
_StoreContext(OpSize::i32Bit, GPRClass, NewSegment, offsetof(FEXCore::Core::CPUState, ds_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:
_StoreContext(OpSize::i32Bit, GPRClass, NewSegment, offsetof(FEXCore::Core::CPUState, fs_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:
_StoreContext(OpSize::i32Bit, GPRClass, NewSegment, offsetof(FEXCore::Core::CPUState, gs_cached));
break;
default: break; // Do nothing
}
}
Ref OpDispatchBuilder::LoadEffectiveAddress(AddressMode A, bool AddSegmentBase, bool AllowUpperGarbage) {
const auto GPRSize = CTX->GetGPROpSize();
Ref Tmp = A.Base;
if (A.Offset) {
Ref Offset = _Constant(A.Offset);
Tmp = Tmp ? _Add(GPRSize, Tmp, Offset) : Offset;
}
if (A.Index) {
if (A.IndexScale != 1) {
LOGMAN_THROW_A_FMT((A.IndexScale & (A.IndexScale - 1)) == 0, "power of two");
uint32_t Log2 = FEXCore::ilog2(A.IndexScale);
if (Tmp) {
Tmp = _AddShift(GPRSize, Tmp, A.Index, ShiftType::LSL, Log2);
} else {
Tmp = _Lshl(GPRSize, A.Index, _Constant(Log2));
}
} else {
Tmp = Tmp ? _Add(GPRSize, Tmp, A.Index) : A.Index;
}
}
// For 64-bit AddrSize can be 32-bit or 64-bit
// For 32-bit AddrSize can be 32-bit or 16-bit
//
// If the AddrSize is not the GPRSize then we need to clear the upper bits.
if ((A.AddrSize < GPRSize) && !AllowUpperGarbage && Tmp) {
Tmp = _Bfe(GPRSize, IR::OpSizeAsBits(A.AddrSize), 0, Tmp);
}
if (A.Segment && AddSegmentBase) {
Tmp = Tmp ? _Add(GPRSize, Tmp, A.Segment) : A.Segment;
}
return Tmp ?: _Constant(0);
}
AddressMode OpDispatchBuilder::SelectAddressMode(AddressMode A, bool AtomicTSO, bool Vector, IR::OpSize AccessSize) {
const auto GPRSize = CTX->GetGPROpSize();
// In the future this also needs to account for LRCPC3.
bool SupportsRegIndex = Vector || !AtomicTSO;
// Try a constant offset. For 64-bit, this maps directly. For 32-bit, this
// works only for displacements with magnitude < 16KB, since those bottom
// addresses are reserved and therefore wrap around is invalid.
//
// TODO: Also handle GPR TSO if we can guarantee the constant inlines.
if (SupportsRegIndex) {
if ((A.Base || A.Segment) && A.Offset) {
const bool Const_16K = A.Offset > -16384 && A.Offset < 16384 && A.AddrSize == OpSize::i32Bit && GPRSize == OpSize::i32Bit;
if ((A.AddrSize == OpSize::i64Bit) || Const_16K) {
// Peel off the offset
AddressMode B = A;
B.Offset = 0;
return {
.Base = LoadEffectiveAddress(B, true /* AddSegmentBase */, false),
.Index = _Constant(A.Offset),
.IndexType = MEM_OFFSET_SXTX,
.IndexScale = 1,
};
}
}
// Try a (possibly scaled) register index.
if (A.AddrSize == OpSize::i64Bit && A.Base && (A.Index || A.Segment) && !A.Offset &&
(A.IndexScale == 1 || A.IndexScale == IR::OpSizeToSize(AccessSize))) {
if (A.Index && A.Segment) {
A.Base = _Add(GPRSize, A.Base, A.Segment);
} else if (A.Segment) {
A.Index = A.Segment;
A.IndexScale = 1;
}
return A;
}
}
// Fallback on software address calculation
return {
.Base = LoadEffectiveAddress(A, true),
.Index = InvalidNode,
};
}
AddressMode OpDispatchBuilder::DecodeAddress(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand,
MemoryAccessType AccessType, bool IsLoad) {
const auto GPRSize = CTX->GetGPROpSize();
AddressMode A {};
A.Segment = GetSegment(Op->Flags);
A.AddrSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) != 0 ? (GPRSize >> 1) : GPRSize;
A.NonTSO = AccessType == MemoryAccessType::NONTSO || AccessType == MemoryAccessType::STREAM;
if (Operand.IsLiteral()) {
A.Offset = Operand.Literal();
if (Operand.Data.Literal.Size != 8 && IsLoad) {
// zero extend
uint64_t width = Operand.Data.Literal.Size * 8;
A.Offset &= ((1ULL << width) - 1);
}
} else if (Operand.IsGPR()) {
// Not an address, let the caller deal with it
A.AddrSize = GPRSize;
} else if (Operand.IsGPRDirect()) {
A.Base = LoadGPRRegister(Operand.Data.GPR.GPR, GPRSize);
A.NonTSO |= IsNonTSOReg(AccessType, Operand.Data.GPR.GPR);
} else if (Operand.IsGPRIndirect()) {
A.Base = LoadGPRRegister(Operand.Data.GPRIndirect.GPR, GPRSize);
A.Offset = Operand.Data.GPRIndirect.Displacement;
A.NonTSO |= IsNonTSOReg(AccessType, Operand.Data.GPRIndirect.GPR);
} else if (Operand.IsRIPRelative()) {
if (CTX->Config.Is64BitMode) {
A.Base = GetRelocatedPC(Op, Operand.Data.RIPLiteral.Value.s);
} else {
// 32bit this isn't RIP relative but instead absolute
A.Offset = Operand.Data.RIPLiteral.Value.u;
}
} else if (Operand.IsSIB()) {
const bool IsVSIB = IsLoad && ((Op->Flags & X86Tables::DecodeFlags::FLAG_VSIB_BYTE) != 0);
if (Operand.Data.SIB.Base != FEXCore::X86State::REG_INVALID) {
A.Base = LoadGPRRegister(Operand.Data.SIB.Base, GPRSize);
}
// NOTE: VSIB cannot have the index * scale portion calculated ahead of time,
// since the index in this case is a vector. So, we can't just apply the scale
// to it, since this needs to be applied to each element in the index register
// after said element has been sign extended. So, we pass this through for the
// instruction implementation to handle.
//
// What we do handle though, is the applying the displacement value to
// the base register (if a base register is provided), since this is a
// part of the address calculation that can be done ahead of time.
if (!IsVSIB && Operand.Data.SIB.Index != FEXCore::X86State::REG_INVALID) {
A.Index = LoadGPRRegister(Operand.Data.SIB.Index, GPRSize);
A.IndexScale = Operand.Data.SIB.Scale;
}
A.Offset = Operand.Data.SIB.Offset;
A.NonTSO |= IsNonTSOReg(AccessType, Operand.Data.SIB.Base) || IsNonTSOReg(AccessType, Operand.Data.SIB.Index);
} else {
LOGMAN_MSG_A_FMT("Unknown Src Type: {}\n", Operand.Type);
}
return A;
}
Ref OpDispatchBuilder::LoadSource_WithOpSize(RegisterClassType Class, const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand,
IR::OpSize OpSize, uint32_t Flags, const LoadSourceOptions& Options) {
auto [Align, LoadData, ForceLoad, AccessType, AllowUpperGarbage] = Options;
AddressMode A = DecodeAddress(Op, Operand, AccessType, true /* IsLoad */);
if (Operand.IsGPR()) {
const auto gpr = Operand.Data.GPR.GPR;
const auto highIndex = Operand.Data.GPR.HighBits ? 1 : 0;
if (gpr >= FEXCore::X86State::REG_MM_0) {
LOGMAN_THROW_A_FMT(OpSize == OpSize::i64Bit, "full");
A.Base = LoadContext(OpSize::i64Bit, MM0Index + gpr - FEXCore::X86State::REG_MM_0);
} else if (gpr >= FEXCore::X86State::REG_XMM_0) {
const auto gprIndex = gpr - X86State::REG_XMM_0;
// Load the full register size if it is a XMM register source.
A.Base = LoadXMMRegister(gprIndex);
// Now extract the subregister if it was a partial load /smaller/ than SSE size
// TODO: Instead of doing the VMov implicitly on load, hunt down all use cases that require partial loads and do it after load.
// We don't have information here to know if the operation needs zero upper bits or can contain data.
if (!AllowUpperGarbage && OpSize < OpSize::i128Bit) {
A.Base = _VMov(OpSize, A.Base);
}
} else {
A.Base = LoadGPRRegister(gpr, OpSize, highIndex ? 8 : 0, AllowUpperGarbage);
}
}
if ((IsOperandMem(Operand, true) && LoadData) || ForceLoad) {
if (OpSize == OpSize::f80Bit) {
Ref MemSrc = LoadEffectiveAddress(A, true);
if (CTX->HostFeatures.SupportsSVE128 || CTX->HostFeatures.SupportsSVE256) {
return _LoadMemX87SVEOptPredicate(OpSize::i128Bit, OpSize::i16Bit, MemSrc);
} else {
// For X87 extended doubles, Split the load.
auto Res = _LoadMem(Class, OpSize::i64Bit, MemSrc, Align == OpSize::iInvalid ? OpSize : Align);
return _VLoadVectorElement(OpSize::i128Bit, OpSize::i16Bit, Res, 4, _Add(OpSize::i64Bit, MemSrc, _InlineConstant(8)));
}
}
return _LoadMemAutoTSO(Class, OpSize, A, Align == OpSize::iInvalid ? OpSize : Align);
} else {
return LoadEffectiveAddress(A, false, AllowUpperGarbage);
}
}
Ref OpDispatchBuilder::GetRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset) {
const auto GPRSize = CTX->GetGPROpSize();
return _EntrypointOffset(GPRSize, Op->PC + Op->InstSize + Offset - Entry);
}
Ref OpDispatchBuilder::LoadGPRRegister(uint32_t GPR, IR::OpSize Size, uint8_t Offset, bool AllowUpperGarbage) {
const auto GPRSize = CTX->GetGPROpSize();
if (Size == OpSize::iInvalid) {
Size = GPRSize;
}
Ref Reg = LoadGPR(GPR);
if ((!AllowUpperGarbage && (Size != GPRSize)) || Offset != 0) {
// Extract the subregister if requested.
const auto OpSize = std::max(OpSize::i32Bit, Size);
if (AllowUpperGarbage) {
Reg = _Lshr(OpSize, Reg, _Constant(Offset));
} else {
Reg = _Bfe(OpSize, IR::OpSizeAsBits(Size), Offset, Reg);
}
}
return Reg;
}
void OpDispatchBuilder::StoreGPRRegister(uint32_t GPR, const Ref Src, IR::OpSize Size, uint8_t Offset) {
const auto GPRSize = CTX->GetGPROpSize();
if (Size == OpSize::iInvalid) {
Size = GPRSize;
}
Ref Reg = Src;
if (Size != GPRSize || Offset != 0) {
// Need to do an insert if not automatic size or zero offset.
Reg = _Bfi(GPRSize, IR::OpSizeAsBits(Size), Offset, LoadGPRRegister(GPR), Src);
}
StoreRegister(GPR, false, Reg);
}
void OpDispatchBuilder::StoreXMMRegister(uint32_t XMM, const Ref Src) {
StoreRegister(XMM, true, Src);
}
Ref OpDispatchBuilder::LoadSource(RegisterClassType Class, const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand,
uint32_t Flags, const LoadSourceOptions& Options) {
const auto OpSize = OpSizeFromSrc(Op);
return LoadSource_WithOpSize(Class, Op, Operand, OpSize, Flags, Options);
}
void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op,
const FEXCore::X86Tables::DecodedOperand& Operand, const Ref Src, IR::OpSize OpSize,
IR::OpSize Align, MemoryAccessType AccessType) {
if (Operand.IsGPR()) {
// 8Bit and 16bit destination types store their result without effecting the upper bits
// 32bit ops ZEXT the result to 64bit
const auto GPRSize = CTX->GetGPROpSize();
const auto gpr = Operand.Data.GPR.GPR;
if (gpr >= FEXCore::X86State::REG_MM_0) {
LOGMAN_THROW_A_FMT(OpSize == OpSize::i64Bit, "full");
LOGMAN_THROW_A_FMT(Class == FPRClass, "MMX is floaty");
if (MMXState != MMXState_MMX) {
ChgStateX87_MMX();
}
uint8_t Index = MM0Index + gpr - FEXCore::X86State::REG_MM_0;
StoreContext(Index, Src);
RegCache.Partial |= (1ull << (uint64_t)Index);
} else if (gpr >= FEXCore::X86State::REG_XMM_0) {
const auto gprIndex = gpr - X86State::REG_XMM_0;
const auto VectorSize = GetGuestVectorLength();
auto Result = Src;
if (OpSize != VectorSize) {
// Partial writes can come from FPRs.
// TODO: Fix the instructions doing partial writes rather than dealing with it here.
LOGMAN_THROW_A_FMT(Class != IR::GPRClass, "Partial writes from GPR not allowed. Instruction: {}", Op->TableInfo->Name);
// XMM-size is handled in implementations.
if (VectorSize != OpSize::i256Bit || OpSize != OpSize::i128Bit) {
auto SrcVector = LoadXMMRegister(gprIndex);
Result = _VInsElement(VectorSize, OpSize, 0, 0, SrcVector, Src);
}
}
StoreXMMRegister(gprIndex, Result);
} else {
if (GPRSize == OpSize::i64Bit && OpSize == OpSize::i32Bit) {
// If the Source IR op is 64 bits, we need to zext the upper bits
// For all other sizes, the upper bits are guaranteed to already be zero
Ref Value = GetOpSize(Src) == OpSize::i64Bit ? _Bfe(OpSize::i32Bit, 32, 0, Src) : Src;
StoreGPRRegister(gpr, Value, GPRSize);
LOGMAN_THROW_A_FMT(!Operand.Data.GPR.HighBits, "Can't handle 32bit store to high 8bit register");
} else {
LOGMAN_THROW_A_FMT(!(GPRSize == OpSize::i32Bit && OpSize > OpSize::i32Bit), "Oops had a {} GPR load", OpSize);
if (GPRSize != OpSize) {
// if the GPR isn't the full size then we need to insert.
// eg:
// mov al, 2 ; Move in to lower 8-bits.
// mov ah, 2 ; Move in to upper 8-bits of 16-bit reg.
// mov ax, 2 ; Move in to lower 16-bits of reg.
StoreGPRRegister(gpr, Src, OpSize, Operand.Data.GPR.HighBits * 8);
} else {
StoreGPRRegister(gpr, Src, std::min(GPRSize, OpSize));
}
}
}
return;
}
AddressMode A = DecodeAddress(Op, Operand, AccessType, false /* IsLoad */);
if (OpSize == OpSize::f80Bit) {
Ref MemStoreDst = LoadEffectiveAddress(A, true);
if (CTX->HostFeatures.SupportsSVE128 || CTX->HostFeatures.SupportsSVE256) {
_StoreMemX87SVEOptPredicate(OpSize::i128Bit, OpSize::i16Bit, Src, MemStoreDst);
} else {
// For X87 extended doubles, split before storing
_StoreMem(FPRClass, OpSize::i64Bit, MemStoreDst, Src, Align);
auto Upper = _VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, Src, 1);
_StoreMem(GPRClass, OpSize::i16Bit, Upper, MemStoreDst, _Constant(8), std::min(Align, OpSize::i64Bit), MEM_OFFSET_SXTX, 1);
}
} else {
_StoreMemAutoTSO(Class, OpSize, A, Src, Align == OpSize::iInvalid ? OpSize : Align);
}
}
void OpDispatchBuilder::StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op,
const FEXCore::X86Tables::DecodedOperand& Operand, const Ref Src, IR::OpSize Align,
MemoryAccessType AccessType) {
StoreResult_WithOpSize(Class, Op, Operand, Src, OpSizeFromDst(Op), Align, AccessType);
}
void OpDispatchBuilder::StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, const Ref Src, IR::OpSize Align,
MemoryAccessType AccessType) {
StoreResult(Class, Op, Op->Dest, Src, Align, AccessType);
}
OpDispatchBuilder::OpDispatchBuilder(FEXCore::Context::ContextImpl* ctx)
: IREmitter {ctx->OpDispatcherAllocator}
, CTX {ctx} {
ResetWorkingList();
InstallHostSpecificOpcodeHandlers();
if (CTX->HostFeatures.SupportsAVX && CTX->HostFeatures.SupportsSVE256) {
SaveAVXStateFunc = &OpDispatchBuilder::SaveAVXState;
RestoreAVXStateFunc = &OpDispatchBuilder::RestoreAVXState;
DefaultAVXStateFunc = &OpDispatchBuilder::DefaultAVXState;
} else if (CTX->HostFeatures.SupportsAVX) {
SaveAVXStateFunc = &OpDispatchBuilder::AVX128_SaveAVXState;
RestoreAVXStateFunc = &OpDispatchBuilder::AVX128_RestoreAVXState;
DefaultAVXStateFunc = &OpDispatchBuilder::AVX128_DefaultAVXState;
}
}
OpDispatchBuilder::OpDispatchBuilder(FEXCore::Utils::IntrusivePooledAllocator& Allocator)
: IREmitter {Allocator}
, CTX {nullptr} {}
void OpDispatchBuilder::ResetWorkingList() {
IREmitter::ResetWorkingList();
JumpTargets.clear();
BlockSetRIP = false;
DecodeFailure = false;
ShouldDump = false;
CurrentCodeBlock = nullptr;
RegCache.Written = 0;
RegCache.Cached = 0;
}
void OpDispatchBuilder::UnhandledOp(OpcodeArgs) {
DecodeFailure = true;
}
void OpDispatchBuilder::MOVGPROp(OpcodeArgs, uint32_t SrcIndex) {
// StoreResult will store with the same size as the input, so we allow upper
// garbage on the input. The zero extension would be pointless.
Ref Src = LoadSource(GPRClass, Op, Op->Src[SrcIndex], Op->Flags, {.Align = OpSize::i8Bit, .AllowUpperGarbage = true});
StoreResult(GPRClass, Op, Src, OpSize::i8Bit);
}
void OpDispatchBuilder::MOVGPRNTOp(OpcodeArgs) {
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit});
StoreResult(GPRClass, Op, Src, OpSize::i8Bit, MemoryAccessType::STREAM);
}
void OpDispatchBuilder::ALUOp(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCore::IR::IROps AtomicFetchOp, unsigned SrcIdx) {
/* On x86, the canonical way to zero a register is XOR with itself... because
* modern x86 detects this pattern in hardware. arm64 does not detect this
* pattern, we should do it like the x86 hardware would. On arm64, "mov x0,
* #0" is faster than "eor x0, x0, x0". Additionally this lets more constant
* folding kick in for flags.
*/
if (!DestIsLockedMem(Op) && ALUIROp == FEXCore::IR::IROps::OP_XOR && Op->Dest.IsGPR() && Op->Src[SrcIdx].IsGPR() &&
Op->Dest.Data.GPR == Op->Src[SrcIdx].Data.GPR) {
auto Result = _Constant(0);
StoreResult(GPRClass, Op, Result, OpSize::iInvalid);
CalculateFlags_Logical(OpSizeFromSrc(Op), Result, Result, Result);
return;
}
auto Size = OpSizeFromDst(Op);
auto ResultSize = Size;
auto RoundedSize = Size;
if (ALUIROp != FEXCore::IR::IROps::OP_ANDWITHFLAGS) {
RoundedSize = std::max(OpSize::i32Bit, RoundedSize);
}
// X86 basic ALU ops just do the operation between the destination and a single source
Ref Src = LoadSource(GPRClass, Op, Op->Src[SrcIdx], Op->Flags, {.AllowUpperGarbage = true});
// Try to eliminate the masking after 8/16-bit operations with constants, by
// promoting to a full size operation that preserves the upper bits.
uint64_t Const;
if (Size < OpSize::i32Bit && !DestIsLockedMem(Op) && Op->Dest.IsGPR() && !Op->Dest.Data.GPR.HighBits && IsValueConstant(WrapNode(Src), &Const) &&
(ALUIROp == IR::IROps::OP_XOR || ALUIROp == IR::IROps::OP_OR || ALUIROp == IR::IROps::OP_ANDWITHFLAGS)) {
RoundedSize = ResultSize = CTX->GetGPROpSize();
LOGMAN_THROW_A_FMT(Const < (1ull << IR::OpSizeAsBits(Size)), "does not clobber");
// For AND, we can play the same trick but we instead need the upper bits of
// the constant to be all-1s instead of all-0s to preserve. We also can't
// use andwithflags in this case, since we've promoted to 64-bit so the
// negate flag would be wrong, but using the regular logical operation path
// instead still ends up a net win for uops.
//
// In the common case where the constant is of the form (1 << x) - 1, the
// adjusted constant here will inline into the arm64 and instruction, so if
// flags are not needed, we save an instruction overall.
if (ALUIROp == IR::IROps::OP_ANDWITHFLAGS) {
Src = _Constant(Const | ~((1ull << IR::OpSizeAsBits(Size)) - 1));
ALUIROp = IR::IROps::OP_AND;
}
}
Ref Result {};
Ref Dest {};
if (DestIsLockedMem(Op)) {
HandledLock = true;
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
DeriveOp(FetchOp, AtomicFetchOp, _AtomicFetchAdd(Size, Src, DestMem));
Dest = FetchOp;
} else {
Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
}
const auto OpSize = RoundedSize;
DeriveOp(ALUOp, ALUIROp, _AndWithFlags(OpSize, Dest, Src));
Result = ALUOp;
// Flags set
switch (ALUIROp) {
case FEXCore::IR::IROps::OP_ADD: Result = CalculateFlags_ADD(Size, Dest, Src); break;
case FEXCore::IR::IROps::OP_SUB: Result = CalculateFlags_SUB(Size, Dest, Src); break;
case FEXCore::IR::IROps::OP_XOR:
case FEXCore::IR::IROps::OP_AND:
case FEXCore::IR::IROps::OP_OR: {
CalculateFlags_Logical(Size, Result, Dest, Src);
break;
}
case FEXCore::IR::IROps::OP_ANDWITHFLAGS: {
HandleNZ00Write();
CalculatePF(Result);
InvalidateAF();
break;
}
default: break;
}
if (!DestIsLockedMem(Op)) {
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Result, ResultSize, OpSize::iInvalid, MemoryAccessType::DEFAULT);
}
}
void OpDispatchBuilder::INTOp(OpcodeArgs) {
IR::BreakDefinition Reason;
bool SetRIPToNext = false;
switch (Op->OP) {
case 0xCD: { // INT imm8
uint8_t Literal = Op->Src[0].Data.Literal.Value;
#ifndef _WIN32
constexpr uint8_t SYSCALL_LITERAL = 0x80;
if (Literal == SYSCALL_LITERAL) {
if (CTX->Config.Is64BitMode()) [[unlikely]] {
ERROR_AND_DIE_FMT("[Unsupported] Trying to execute 32-bit syscall from a 64-bit process.");
}
// Syscall on linux
SyscallOp(Op, false);
return;
}
#else
constexpr uint8_t SYSCALL_LITERAL = 0x2E;
if (Literal == SYSCALL_LITERAL) {
// Can be used for both 64-bit and 32-bit syscalls on windows
SyscallOp(Op, false);
return;
}
#endif
#ifdef _M_ARM_64EC
// This is used when QueryPerformanceCounter is called on recent Windows versions, it causes CNTVCT to be written into RAX.
constexpr uint8_t GET_CNTVCT_LITERAL = 0x81;
if (Literal == GET_CNTVCT_LITERAL) {
StoreGPRRegister(X86State::REG_RAX, _CycleCounter());
return;
}
#endif
Reason.ErrorRegister = Literal << 3 | (0b010);
Reason.Signal = Core::FAULT_SIGSEGV;
// GP is raised when task-gate isn't setup to be valid
Reason.TrapNumber = X86State::X86_TRAPNO_GP;
Reason.si_code = 0x80;
break;
}
case 0xCE: // INTO
Reason.ErrorRegister = 0;
Reason.Signal = Core::FAULT_SIGSEGV;
Reason.TrapNumber = X86State::X86_TRAPNO_OF;
Reason.si_code = 0x80;
break;
case 0xF1: // INT1
Reason.ErrorRegister = 0;
Reason.Signal = Core::FAULT_SIGTRAP;
Reason.TrapNumber = X86State::X86_TRAPNO_DB;
Reason.si_code = 1;
SetRIPToNext = true;
break;
case 0xF4: { // HLT
Reason.ErrorRegister = 0;
Reason.Signal = Core::FAULT_SIGSEGV;
Reason.TrapNumber = X86State::X86_TRAPNO_GP;
Reason.si_code = 0x80;
break;
}
case 0x0B: // UD2
Reason.ErrorRegister = 0;
Reason.Signal = Core::FAULT_SIGILL;
Reason.TrapNumber = X86State::X86_TRAPNO_UD;
Reason.si_code = 2;
break;
case 0xCC: // INT3
Reason.ErrorRegister = 0;
Reason.Signal = Core::FAULT_SIGTRAP;
Reason.TrapNumber = X86State::X86_TRAPNO_BP;
Reason.si_code = 0x80;
SetRIPToNext = true;
break;
default: FEX_UNREACHABLE;
}
// Calculate flags early.
FlushRegisterCache();
const auto GPRSize = CTX->GetGPROpSize();
if (SetRIPToNext) {
BlockSetRIP = SetRIPToNext;
// We want to set RIP to the next instruction after INT3/INT1
auto NewRIP = GetRelocatedPC(Op);
_StoreContext(GPRSize, GPRClass, NewRIP, offsetof(FEXCore::Core::CPUState, rip));
} else if (Op->OP != 0xCE) {
auto NewRIP = GetRelocatedPC(Op, -Op->InstSize);
_StoreContext(GPRSize, GPRClass, NewRIP, offsetof(FEXCore::Core::CPUState, rip));
}
if (Op->OP == 0xCE) { // Conditional to only break if Overflow == 1
CalculateDeferredFlags();
// If condition doesn't hold then keep going
// COND_FNU means OF == 0
auto CondJump_ = CondJumpNZCV({COND_FNU});
auto FalseBlock = CreateNewCodeBlockAfter(GetCurrentBlock());
SetFalseJumpTarget(CondJump_, FalseBlock);
SetCurrentCodeBlock(FalseBlock);
StartNewBlock();
auto NewRIP = GetRelocatedPC(Op);
_StoreContext(GPRSize, GPRClass, NewRIP, offsetof(FEXCore::Core::CPUState, rip));
Break(Reason);
// Make sure to start a new block after ending this one
auto JumpTarget = CreateNewCodeBlockAfter(FalseBlock);
SetTrueJumpTarget(CondJump_, JumpTarget);
SetCurrentCodeBlock(JumpTarget);
StartNewBlock();
} else {
BlockSetRIP = true;
Break(Reason);
}
}
void OpDispatchBuilder::TZCNT(OpcodeArgs) {
// _FindTrailingZeroes ignores upper garbage so we don't need to mask
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
Src = _FindTrailingZeroes(OpSizeFromSrc(Op), Src);
StoreResult(GPRClass, Op, Src, OpSize::iInvalid);
CalculateFlags_ZCNT(OpSizeFromSrc(Op), Src);
}
void OpDispatchBuilder::LZCNT(OpcodeArgs) {
// _CountLeadingZeroes clears upper garbage so we don't need to mask
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
auto Res = _CountLeadingZeroes(OpSizeFromSrc(Op), Src);
StoreResult(GPRClass, Op, Res, OpSize::iInvalid);
CalculateFlags_ZCNT(OpSizeFromSrc(Op), Res);
}
void OpDispatchBuilder::MOVBEOp(OpcodeArgs) {
const auto GPRSize = CTX->GetGPROpSize();
const auto SrcSize = OpSizeFromSrc(Op);
Ref Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit});
Src = _Rev(std::max(OpSize::i32Bit, SrcSize), Src);
if (SrcSize == OpSize::i16Bit) {
// 16-bit does an insert.
// Rev of 16-bit value as 32-bit replaces the result in the upper 16-bits of the result.
// bfxil the 16-bit result in to the GPR.
Ref Dest = LoadSource_WithOpSize(GPRClass, Op, Op->Dest, GPRSize, Op->Flags);
auto Result = _Bfxil(GPRSize, 16, 16, Dest, Src);
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Result, GPRSize, OpSize::iInvalid);
} else {
// 32-bit does regular zext
StoreResult(GPRClass, Op, Op->Dest, Src, OpSize::iInvalid);
}
}
void OpDispatchBuilder::CLWB(OpcodeArgs) {
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
_CacheLineClean(DestMem);
}
void OpDispatchBuilder::CLFLUSHOPT(OpcodeArgs) {
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
_CacheLineClear(DestMem, false);
}
void OpDispatchBuilder::LoadFenceOrXRSTOR(OpcodeArgs) {
// 0xE8 signifies LFENCE
if (Op->ModRM == 0xE8) {
_Fence(IR::Fence_Load);
} else {
XRstorOpImpl(Op);
}
}
void OpDispatchBuilder::MemFenceOrXSAVEOPT(OpcodeArgs) {
if (Op->ModRM == 0xF0) {
// 0xF0 is MFENCE
_Fence(FEXCore::IR::Fence_LoadStore);
} else {
LogMan::Msg::EFmt("Application tried using XSAVEOPT");
UnimplementedOp(Op);
}
}
void OpDispatchBuilder::StoreFenceOrCLFlush(OpcodeArgs) {
if (Op->ModRM == 0xF8) {
// 0xF8 is SFENCE
_Fence({FEXCore::IR::Fence_Store});
} else {
// This is a CLFlush
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
_CacheLineClear(DestMem, true);
}
}
void OpDispatchBuilder::CLZeroOp(OpcodeArgs) {
Ref DestMem = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.LoadData = false});
_CacheLineZero(DestMem);
}
void OpDispatchBuilder::Prefetch(OpcodeArgs, bool ForStore, bool Stream, uint8_t Level) {
Ref DestMem = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.LoadData = false});
_Prefetch(ForStore, Stream, Level, DestMem, Invalid(), MEM_OFFSET_SXTX, 1);
}
void OpDispatchBuilder::RDTSCPOp(OpcodeArgs) {
// RDTSCP is slightly different than RDTSC
// IA32_TSC_AUX is returned in RCX
// All previous loads are globally visible
// - Explicitly does not wait for stores to be globally visible
// - Explicitly use an MFENCE before this instruction if you want this behaviour
// This instruction is not an execution fence, so subsequent instructions can execute after this
// - Explicitly use an LFENCE after RDTSCP if you want to block this behaviour
_Fence({FEXCore::IR::Fence_Load});
auto Counter = CycleCounter();
auto ID = _ProcessorID();
StoreGPRRegister(X86State::REG_RAX, Counter.CounterLow);
StoreGPRRegister(X86State::REG_RCX, ID);
StoreGPRRegister(X86State::REG_RDX, Counter.CounterHigh);
}
void OpDispatchBuilder::RDPIDOp(OpcodeArgs) {
StoreResult(GPRClass, Op, _ProcessorID(), OpSize::iInvalid);
}
void OpDispatchBuilder::CRC32(OpcodeArgs) {
const auto GPRSize = CTX->GetGPROpSize();
// Destination GPR size is always 4 or 8 bytes depending on widening
const auto DstSize = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;
Ref Dest = LoadSource_WithOpSize(GPRClass, Op, Op->Dest, GPRSize, Op->Flags);
// Incoming memory is 8, 16, 32, or 64
Ref Src {};
if (Op->Src[0].IsGPR()) {
Src = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], GPRSize, Op->Flags);
} else {
Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit});
}
auto Result = _CRC32(Dest, Src, OpSizeFromSrc(Op));
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Result, DstSize, OpSize::iInvalid);
}
template<bool Reseed>
void OpDispatchBuilder::RDRANDOp(OpcodeArgs) {
StoreResult(GPRClass, Op, _RDRAND(Reseed), OpSize::iInvalid);
// If the rng number is valid then NZCV is 0b0000, otherwise NZCV is 0b0100
auto CF_inv = GetRFLAG(X86State::RFLAG_ZF_RAW_LOC);
// OF, SF, ZF, AF, PF all zero. CF indicates if valid.
ZeroPF_AF();
if (!CTX->HostFeatures.SupportsFlagM) {
ZeroNZCV();
SetCFInverted(CF_inv);
} else {
// Accelerated path. Invalid is 0 or 1, so set NZCV with a single rmif.
HandleNZCVWrite();
_RmifNZCV(CF_inv, (64 - 1) /* rotate bit 0 into bit 1 = C */, 0xf);
CFInverted = true;
}
}
void OpDispatchBuilder::BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDefinition) {
const auto GPRSize = CTX->GetGPROpSize();
// We don't actually support this instruction
// Multiblock may hit it though
_StoreContext(GPRSize, GPRClass, GetRelocatedPC(Op, -Op->InstSize), offsetof(FEXCore::Core::CPUState, rip));
Break(BreakDefinition);
BlockSetRIP = true;
if (Multiblock) {
auto NextBlock = CreateNewCodeBlockAfter(GetCurrentBlock());
SetCurrentCodeBlock(NextBlock);
StartNewBlock();
}
}
void OpDispatchBuilder::UnimplementedOp(OpcodeArgs) {
BreakOp(Op, FEXCore::IR::BreakDefinition {
.ErrorRegister = 0,
.Signal = SIGILL,
.TrapNumber = X86State::X86_TRAPNO_UD,
.si_code = 2, ///< ILL_ILLOPN
});
}
void OpDispatchBuilder::PermissionRestrictedOp(OpcodeArgs) {
BreakOp(Op, FEXCore::IR::BreakDefinition {
.ErrorRegister = 0,
.Signal = SIGSEGV,
.TrapNumber = X86State::X86_TRAPNO_GP,
.si_code = 0x80,
});
}
void OpDispatchBuilder::InvalidOp(OpcodeArgs) {
BreakOp(Op, FEXCore::IR::BreakDefinition {
.ErrorRegister = 0,
.Signal = SIGILL,
.TrapNumber = 0,
.si_code = 0,
});
}
#undef OpcodeArgs
void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() {
static bool Initialized = false;
if (!CTX || Initialized) {
return;
}
#define OPD(prefix, opcode) (((prefix) << 8) | opcode)
constexpr uint16_t PF_38_NONE = 0;
constexpr uint16_t PF_38_66 = (1U << 0);
constexpr uint16_t PF_38_F2 = (1U << 1);
constexpr static std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> H0F38_SHA[] = {
{OPD(PF_38_NONE, 0xC8), 1, &OpDispatchBuilder::SHA1NEXTEOp}, {OPD(PF_38_NONE, 0xC9), 1, &OpDispatchBuilder::SHA1MSG1Op},
{OPD(PF_38_NONE, 0xCA), 1, &OpDispatchBuilder::SHA1MSG2Op}, {OPD(PF_38_NONE, 0xCB), 1, &OpDispatchBuilder::SHA256RNDS2Op},
{OPD(PF_38_NONE, 0xCC), 1, &OpDispatchBuilder::SHA256MSG1Op}, {OPD(PF_38_NONE, 0xCD), 1, &OpDispatchBuilder::SHA256MSG2Op},
};
constexpr static std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> H0F38_AES[] = {
{OPD(PF_38_66, 0xDB), 1, &OpDispatchBuilder::AESImcOp}, {OPD(PF_38_66, 0xDC), 1, &OpDispatchBuilder::AESEncOp},
{OPD(PF_38_66, 0xDD), 1, &OpDispatchBuilder::AESEncLastOp}, {OPD(PF_38_66, 0xDE), 1, &OpDispatchBuilder::AESDecOp},
{OPD(PF_38_66, 0xDF), 1, &OpDispatchBuilder::AESDecLastOp},
};
constexpr static std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> H0F38_CRC[] = {
{OPD(PF_38_F2, 0xF0), 1, &OpDispatchBuilder::CRC32},
{OPD(PF_38_F2, 0xF1), 1, &OpDispatchBuilder::CRC32},
{OPD(PF_38_66 | PF_38_F2, 0xF0), 1, &OpDispatchBuilder::CRC32},
{OPD(PF_38_66 | PF_38_F2, 0xF1), 1, &OpDispatchBuilder::CRC32},
};
#undef OPD
#define OPD(REX, prefix, opcode) ((REX << 9) | (prefix << 8) | opcode)
#define PF_3A_NONE 0
#define PF_3A_66 1
constexpr static std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> H0F3A_AES[] = {
{OPD(0, PF_3A_66, 0xDF), 1, &OpDispatchBuilder::AESKeyGenAssist},
{OPD(1, PF_3A_66, 0xDF), 1, &OpDispatchBuilder::AESKeyGenAssist},
};
constexpr static std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> H0F3A_PCLMUL[] = {
{OPD(0, PF_3A_66, 0x44), 1, &OpDispatchBuilder::PCLMULQDQOp},
{OPD(1, PF_3A_66, 0x44), 1, &OpDispatchBuilder::PCLMULQDQOp},
};
#undef PF_3A_NONE
#undef PF_3A_66
#undef OPD
#define OPD(map_select, pp, opcode) (((map_select - 1) << 10) | (pp << 8) | (opcode))
constexpr static std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> VEX_PCLMUL[] = {
{OPD(3, 0b01, 0x44), 1, &OpDispatchBuilder::VPCLMULQDQOp},
};
#undef OPD
#define OPD(group, prefix, Reg) (((group - FEXCore::X86Tables::TYPE_GROUP_6) << 5) | (prefix) << 3 | (Reg))
constexpr uint16_t PF_NONE = 0;
constexpr uint16_t PF_66 = 2;
constexpr static std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> SecondaryExtensionOp_RDRAND[] = {
// GROUP 9
{OPD(FEXCore::X86Tables::TYPE_GROUP_9, PF_NONE, 6), 1, &OpDispatchBuilder::RDRANDOp<false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_9, PF_NONE, 7), 1, &OpDispatchBuilder::RDRANDOp<true>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_9, PF_66, 6), 1, &OpDispatchBuilder::RDRANDOp<false>},
{OPD(FEXCore::X86Tables::TYPE_GROUP_9, PF_66, 7), 1, &OpDispatchBuilder::RDRANDOp<true>},
};
#undef OPD
constexpr static std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> SecondaryModRMExtensionOp_CLZero[] = {
{((3 << 3) | 4), 1, &OpDispatchBuilder::CLZeroOp},
};
#define OPD(map_select, pp, opcode) (((map_select - 1) << 10) | (pp << 8) | (opcode))
static constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> AVXTable[] = {
{OPD(1, 0b00, 0x10), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPDOp},
{OPD(1, 0b01, 0x10), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPDOp},
{OPD(1, 0b10, 0x10), 1, &OpDispatchBuilder::VMOVSSOp},
{OPD(1, 0b11, 0x10), 1, &OpDispatchBuilder::VMOVSDOp},
{OPD(1, 0b00, 0x11), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPDOp},
{OPD(1, 0b01, 0x11), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPDOp},
{OPD(1, 0b10, 0x11), 1, &OpDispatchBuilder::VMOVSSOp},
{OPD(1, 0b11, 0x11), 1, &OpDispatchBuilder::VMOVSDOp},
{OPD(1, 0b00, 0x12), 1, &OpDispatchBuilder::VMOVLPOp},
{OPD(1, 0b01, 0x12), 1, &OpDispatchBuilder::VMOVLPOp},
{OPD(1, 0b10, 0x12), 1, &OpDispatchBuilder::VMOVSLDUPOp},
{OPD(1, 0b11, 0x12), 1, &OpDispatchBuilder::VMOVDDUPOp},
{OPD(1, 0b00, 0x13), 1, &OpDispatchBuilder::VMOVLPOp},
{OPD(1, 0b01, 0x13), 1, &OpDispatchBuilder::VMOVLPOp},
{OPD(1, 0b00, 0x14), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKLOp, OpSize::i32Bit>},
{OPD(1, 0b01, 0x14), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKLOp, OpSize::i64Bit>},
{OPD(1, 0b00, 0x15), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKHOp, OpSize::i32Bit>},
{OPD(1, 0b01, 0x15), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKHOp, OpSize::i64Bit>},
{OPD(1, 0b00, 0x16), 1, &OpDispatchBuilder::VMOVHPOp},
{OPD(1, 0b01, 0x16), 1, &OpDispatchBuilder::VMOVHPOp},
{OPD(1, 0b10, 0x16), 1, &OpDispatchBuilder::VMOVSHDUPOp},
{OPD(1, 0b00, 0x17), 1, &OpDispatchBuilder::VMOVHPOp},
{OPD(1, 0b01, 0x17), 1, &OpDispatchBuilder::VMOVHPOp},
{OPD(1, 0b00, 0x28), 1, &OpDispatchBuilder::VMOVAPS_VMOVAPDOp},
{OPD(1, 0b01, 0x28), 1, &OpDispatchBuilder::VMOVAPS_VMOVAPDOp},
{OPD(1, 0b00, 0x29), 1, &OpDispatchBuilder::VMOVAPS_VMOVAPDOp},
{OPD(1, 0b01, 0x29), 1, &OpDispatchBuilder::VMOVAPS_VMOVAPDOp},
{OPD(1, 0b10, 0x2A), 1, &OpDispatchBuilder::AVXInsertCVTGPR_To_FPR<OpSize::i32Bit>},
{OPD(1, 0b11, 0x2A), 1, &OpDispatchBuilder::AVXInsertCVTGPR_To_FPR<OpSize::i64Bit>},
{OPD(1, 0b00, 0x2B), 1, &OpDispatchBuilder::MOVVectorNTOp},
{OPD(1, 0b01, 0x2B), 1, &OpDispatchBuilder::MOVVectorNTOp},
{OPD(1, 0b10, 0x2C), 1, &OpDispatchBuilder::CVTFPR_To_GPR<OpSize::i32Bit, false>},
{OPD(1, 0b11, 0x2C), 1, &OpDispatchBuilder::CVTFPR_To_GPR<OpSize::i64Bit, false>},
{OPD(1, 0b10, 0x2D), 1, &OpDispatchBuilder::CVTFPR_To_GPR<OpSize::i32Bit, true>},
{OPD(1, 0b11, 0x2D), 1, &OpDispatchBuilder::CVTFPR_To_GPR<OpSize::i64Bit, true>},
{OPD(1, 0b00, 0x2E), 1, &OpDispatchBuilder::UCOMISxOp<OpSize::i32Bit>},
{OPD(1, 0b01, 0x2E), 1, &OpDispatchBuilder::UCOMISxOp<OpSize::i64Bit>},
{OPD(1, 0b00, 0x2F), 1, &OpDispatchBuilder::UCOMISxOp<OpSize::i32Bit>},
{OPD(1, 0b01, 0x2F), 1, &OpDispatchBuilder::UCOMISxOp<OpSize::i64Bit>},
{OPD(1, 0b00, 0x50), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVMSKOp, OpSize::i32Bit>},
{OPD(1, 0b01, 0x50), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVMSKOp, OpSize::i64Bit>},
{OPD(1, 0b00, 0x51), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VFSQRT, OpSize::i32Bit>},
{OPD(1, 0b01, 0x51), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VFSQRT, OpSize::i64Bit>},
{OPD(1, 0b10, 0x51), 1, &OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp<IR::OP_VFSQRTSCALARINSERT, OpSize::i32Bit>},
{OPD(1, 0b11, 0x51), 1, &OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp<IR::OP_VFSQRTSCALARINSERT, OpSize::i64Bit>},
{OPD(1, 0b00, 0x52), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VFRSQRT, OpSize::i32Bit>},
{OPD(1, 0b10, 0x52), 1, &OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp<IR::OP_VFRSQRTSCALARINSERT, OpSize::i32Bit>},
{OPD(1, 0b00, 0x53), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VFRECP, OpSize::i32Bit>},
{OPD(1, 0b10, 0x53), 1, &OpDispatchBuilder::AVXVectorScalarUnaryInsertALUOp<IR::OP_VFRECPSCALARINSERT, OpSize::i32Bit>},
{OPD(1, 0b00, 0x54), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VAND, OpSize::i128Bit>},
{OPD(1, 0b01, 0x54), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VAND, OpSize::i128Bit>},
{OPD(1, 0b00, 0x55), 1, &OpDispatchBuilder::VANDNOp},
{OPD(1, 0b01, 0x55), 1, &OpDispatchBuilder::VANDNOp},
{OPD(1, 0b00, 0x56), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VOR, OpSize::i128Bit>},
{OPD(1, 0b01, 0x56), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VOR, OpSize::i128Bit>},
{OPD(1, 0b00, 0x57), 1, &OpDispatchBuilder::AVXVectorXOROp},
{OPD(1, 0b01, 0x57), 1, &OpDispatchBuilder::AVXVectorXOROp},
{OPD(1, 0b00, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFADD, OpSize::i32Bit>},
{OPD(1, 0b01, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFADD, OpSize::i64Bit>},
{OPD(1, 0b10, 0x58), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp<IR::OP_VFADDSCALARINSERT, OpSize::i32Bit>},
{OPD(1, 0b11, 0x58), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp<IR::OP_VFADDSCALARINSERT, OpSize::i64Bit>},
{OPD(1, 0b00, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFMUL, OpSize::i32Bit>},
{OPD(1, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFMUL, OpSize::i64Bit>},
{OPD(1, 0b10, 0x59), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp<IR::OP_VFMULSCALARINSERT, OpSize::i32Bit>},
{OPD(1, 0b11, 0x59), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp<IR::OP_VFMULSCALARINSERT, OpSize::i64Bit>},
{OPD(1, 0b00, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Vector_CVT_Float_To_Float, OpSize::i64Bit, OpSize::i32Bit, true>},
{OPD(1, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Vector_CVT_Float_To_Float, OpSize::i32Bit, OpSize::i64Bit, true>},
{OPD(1, 0b10, 0x5A), 1, &OpDispatchBuilder::AVXInsertScalar_CVT_Float_To_Float<OpSize::i64Bit, OpSize::i32Bit>},
{OPD(1, 0b11, 0x5A), 1, &OpDispatchBuilder::AVXInsertScalar_CVT_Float_To_Float<OpSize::i32Bit, OpSize::i64Bit>},
{OPD(1, 0b00, 0x5B), 1, &OpDispatchBuilder::Vector_CVT_Int_To_Float<OpSize::i32Bit, false>},
{OPD(1, 0b01, 0x5B), 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int<OpSize::i32Bit, true>},
{OPD(1, 0b10, 0x5B), 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int<OpSize::i32Bit, false>},
{OPD(1, 0b00, 0x5C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFSUB, OpSize::i32Bit>},
{OPD(1, 0b01, 0x5C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFSUB, OpSize::i64Bit>},
{OPD(1, 0b10, 0x5C), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp<IR::OP_VFSUBSCALARINSERT, OpSize::i32Bit>},
{OPD(1, 0b11, 0x5C), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp<IR::OP_VFSUBSCALARINSERT, OpSize::i64Bit>},
{OPD(1, 0b00, 0x5D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFMIN, OpSize::i32Bit>},
{OPD(1, 0b01, 0x5D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFMIN, OpSize::i64Bit>},
{OPD(1, 0b10, 0x5D), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp<IR::OP_VFMINSCALARINSERT, OpSize::i32Bit>},
{OPD(1, 0b11, 0x5D), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp<IR::OP_VFMINSCALARINSERT, OpSize::i64Bit>},
{OPD(1, 0b00, 0x5E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFDIV, OpSize::i32Bit>},
{OPD(1, 0b01, 0x5E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFDIV, OpSize::i64Bit>},
{OPD(1, 0b10, 0x5E), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp<IR::OP_VFDIVSCALARINSERT, OpSize::i32Bit>},
{OPD(1, 0b11, 0x5E), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp<IR::OP_VFDIVSCALARINSERT, OpSize::i64Bit>},
{OPD(1, 0b00, 0x5F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFMAX, OpSize::i32Bit>},
{OPD(1, 0b01, 0x5F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VFMAX, OpSize::i64Bit>},
{OPD(1, 0b10, 0x5F), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp<IR::OP_VFMAXSCALARINSERT, OpSize::i32Bit>},
{OPD(1, 0b11, 0x5F), 1, &OpDispatchBuilder::AVXVectorScalarInsertALUOp<IR::OP_VFMAXSCALARINSERT, OpSize::i64Bit>},
{OPD(1, 0b01, 0x60), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKLOp, OpSize::i8Bit>},
{OPD(1, 0b01, 0x61), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKLOp, OpSize::i16Bit>},
{OPD(1, 0b01, 0x62), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKLOp, OpSize::i32Bit>},
{OPD(1, 0b01, 0x63), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPACKSSOp, OpSize::i16Bit>},
{OPD(1, 0b01, 0x64), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPGT, OpSize::i8Bit>},
{OPD(1, 0b01, 0x65), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPGT, OpSize::i16Bit>},
{OPD(1, 0b01, 0x66), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPGT, OpSize::i32Bit>},
{OPD(1, 0b01, 0x67), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPACKUSOp, OpSize::i16Bit>},
{OPD(1, 0b01, 0x68), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKHOp, OpSize::i8Bit>},
{OPD(1, 0b01, 0x69), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKHOp, OpSize::i16Bit>},
{OPD(1, 0b01, 0x6A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKHOp, OpSize::i32Bit>},
{OPD(1, 0b01, 0x6B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPACKSSOp, OpSize::i32Bit>},
{OPD(1, 0b01, 0x6C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKLOp, OpSize::i64Bit>},
{OPD(1, 0b01, 0x6D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPUNPCKHOp, OpSize::i64Bit>},
{OPD(1, 0b01, 0x6E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVBetweenGPR_FPR, OpDispatchBuilder::VectorOpType::AVX>},
{OPD(1, 0b01, 0x6F), 1, &OpDispatchBuilder::VMOVAPS_VMOVAPDOp},
{OPD(1, 0b10, 0x6F), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPDOp},
{OPD(1, 0b01, 0x70), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSHUFWOp, OpSize::i32Bit, true>},
{OPD(1, 0b10, 0x70), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSHUFWOp, OpSize::i16Bit, false>},
{OPD(1, 0b11, 0x70), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSHUFWOp, OpSize::i16Bit, true>},
{OPD(1, 0b01, 0x74), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPEQ, OpSize::i8Bit>},
{OPD(1, 0b01, 0x75), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPEQ, OpSize::i16Bit>},
{OPD(1, 0b01, 0x76), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPEQ, OpSize::i32Bit>},
{OPD(1, 0b00, 0x77), 1, &OpDispatchBuilder::VZEROOp},
{OPD(1, 0b01, 0x7C), 1, &OpDispatchBuilder::VHADDPOp<IR::OP_VFADDP, OpSize::i64Bit>},
{OPD(1, 0b11, 0x7C), 1, &OpDispatchBuilder::VHADDPOp<IR::OP_VFADDP, OpSize::i32Bit>},
{OPD(1, 0b01, 0x7D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VHSUBPOp, OpSize::i64Bit>},
{OPD(1, 0b11, 0x7D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VHSUBPOp, OpSize::i32Bit>},
{OPD(1, 0b01, 0x7E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVBetweenGPR_FPR, OpDispatchBuilder::VectorOpType::AVX>},
{OPD(1, 0b10, 0x7E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVQOp, OpDispatchBuilder::VectorOpType::AVX>},
{OPD(1, 0b01, 0x7F), 1, &OpDispatchBuilder::VMOVAPS_VMOVAPDOp},
{OPD(1, 0b10, 0x7F), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPDOp},
{OPD(1, 0b00, 0xC2), 1, &OpDispatchBuilder::AVXVFCMPOp<OpSize::i32Bit>},
{OPD(1, 0b01, 0xC2), 1, &OpDispatchBuilder::AVXVFCMPOp<OpSize::i64Bit>},
{OPD(1, 0b10, 0xC2), 1, &OpDispatchBuilder::AVXInsertScalarFCMPOp<OpSize::i32Bit>},
{OPD(1, 0b11, 0xC2), 1, &OpDispatchBuilder::AVXInsertScalarFCMPOp<OpSize::i64Bit>},
{OPD(1, 0b01, 0xC4), 1, &OpDispatchBuilder::VPINSRWOp},
{OPD(1, 0b01, 0xC5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i16Bit>},
{OPD(1, 0b00, 0xC6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VSHUFOp, OpSize::i32Bit>},
{OPD(1, 0b01, 0xC6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VSHUFOp, OpSize::i64Bit>},
{OPD(1, 0b01, 0xD0), 1, &OpDispatchBuilder::VADDSUBPOp<OpSize::i64Bit>},
{OPD(1, 0b11, 0xD0), 1, &OpDispatchBuilder::VADDSUBPOp<OpSize::i32Bit>},
{OPD(1, 0b01, 0xD1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRLDOp, OpSize::i16Bit>},
{OPD(1, 0b01, 0xD2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRLDOp, OpSize::i32Bit>},
{OPD(1, 0b01, 0xD3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRLDOp, OpSize::i64Bit>},
{OPD(1, 0b01, 0xD4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VADD, OpSize::i64Bit>},
{OPD(1, 0b01, 0xD5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VMUL, OpSize::i16Bit>},
{OPD(1, 0b01, 0xD6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVQOp, OpDispatchBuilder::VectorOpType::AVX>},
{OPD(1, 0b01, 0xD7), 1, &OpDispatchBuilder::MOVMSKOpOne},
{OPD(1, 0b01, 0xD8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUQSUB, OpSize::i8Bit>},
{OPD(1, 0b01, 0xD9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUQSUB, OpSize::i16Bit>},
{OPD(1, 0b01, 0xDA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUMIN, OpSize::i8Bit>},
{OPD(1, 0b01, 0xDB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VAND, OpSize::i128Bit>},
{OPD(1, 0b01, 0xDC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUQADD, OpSize::i8Bit>},
{OPD(1, 0b01, 0xDD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUQADD, OpSize::i16Bit>},
{OPD(1, 0b01, 0xDE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUMAX, OpSize::i8Bit>},
{OPD(1, 0b01, 0xDF), 1, &OpDispatchBuilder::VANDNOp},
{OPD(1, 0b01, 0xE0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VURAVG, OpSize::i8Bit>},
{OPD(1, 0b01, 0xE1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRAOp, OpSize::i16Bit>},
{OPD(1, 0b01, 0xE2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRAOp, OpSize::i32Bit>},
{OPD(1, 0b01, 0xE3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VURAVG, OpSize::i16Bit>},
{OPD(1, 0b01, 0xE4), 1, &OpDispatchBuilder::VPMULHWOp<false>},
{OPD(1, 0b01, 0xE5), 1, &OpDispatchBuilder::VPMULHWOp<true>},
{OPD(1, 0b01, 0xE6), 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int<OpSize::i64Bit, false>},
{OPD(1, 0b10, 0xE6), 1, &OpDispatchBuilder::Vector_CVT_Int_To_Float<OpSize::i32Bit, true>},
{OPD(1, 0b11, 0xE6), 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int<OpSize::i64Bit, true>},
{OPD(1, 0b01, 0xE7), 1, &OpDispatchBuilder::MOVVectorNTOp},
{OPD(1, 0b01, 0xE8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSQSUB, OpSize::i8Bit>},
{OPD(1, 0b01, 0xE9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSQSUB, OpSize::i16Bit>},
{OPD(1, 0b01, 0xEA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSMIN, OpSize::i16Bit>},
{OPD(1, 0b01, 0xEB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VOR, OpSize::i128Bit>},
{OPD(1, 0b01, 0xEC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSQADD, OpSize::i8Bit>},
{OPD(1, 0b01, 0xED), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSQADD, OpSize::i16Bit>},
{OPD(1, 0b01, 0xEE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSMAX, OpSize::i16Bit>},
{OPD(1, 0b01, 0xEF), 1, &OpDispatchBuilder::AVXVectorXOROp},
{OPD(1, 0b11, 0xF0), 1, &OpDispatchBuilder::MOVVectorUnalignedOp},
{OPD(1, 0b01, 0xF1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSLLOp, OpSize::i16Bit>},
{OPD(1, 0b01, 0xF2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSLLOp, OpSize::i32Bit>},
{OPD(1, 0b01, 0xF3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSLLOp, OpSize::i64Bit>},
{OPD(1, 0b01, 0xF4), 1, &OpDispatchBuilder::VPMULLOp<OpSize::i32Bit, false>},
{OPD(1, 0b01, 0xF5), 1, &OpDispatchBuilder::VPMADDWDOp},
{OPD(1, 0b01, 0xF6), 1, &OpDispatchBuilder::VPSADBWOp},
{OPD(1, 0b01, 0xF7), 1, &OpDispatchBuilder::MASKMOVOp},
{OPD(1, 0b01, 0xF8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSUB, OpSize::i8Bit>},
{OPD(1, 0b01, 0xF9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSUB, OpSize::i16Bit>},
{OPD(1, 0b01, 0xFA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSUB, OpSize::i32Bit>},
{OPD(1, 0b01, 0xFB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSUB, OpSize::i64Bit>},
{OPD(1, 0b01, 0xFC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VADD, OpSize::i8Bit>},
{OPD(1, 0b01, 0xFD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VADD, OpSize::i16Bit>},
{OPD(1, 0b01, 0xFE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VADD, OpSize::i32Bit>},
{OPD(2, 0b01, 0x00), 1, &OpDispatchBuilder::VPSHUFBOp},
{OPD(2, 0b01, 0x01), 1, &OpDispatchBuilder::VHADDPOp<IR::OP_VADDP, OpSize::i16Bit>},
{OPD(2, 0b01, 0x02), 1, &OpDispatchBuilder::VHADDPOp<IR::OP_VADDP, OpSize::i32Bit>},
{OPD(2, 0b01, 0x03), 1, &OpDispatchBuilder::VPHADDSWOp},
{OPD(2, 0b01, 0x04), 1, &OpDispatchBuilder::VPMADDUBSWOp},
{OPD(2, 0b01, 0x05), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPHSUBOp, OpSize::i16Bit>},
{OPD(2, 0b01, 0x06), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPHSUBOp, OpSize::i32Bit>},
{OPD(2, 0b01, 0x07), 1, &OpDispatchBuilder::VPHSUBSWOp},
{OPD(2, 0b01, 0x08), 1, &OpDispatchBuilder::VPSIGN<OpSize::i8Bit>},
{OPD(2, 0b01, 0x09), 1, &OpDispatchBuilder::VPSIGN<OpSize::i16Bit>},
{OPD(2, 0b01, 0x0A), 1, &OpDispatchBuilder::VPSIGN<OpSize::i32Bit>},
{OPD(2, 0b01, 0x0B), 1, &OpDispatchBuilder::VPMULHRSWOp},
{OPD(2, 0b01, 0x0C), 1, &OpDispatchBuilder::VPERMILRegOp<OpSize::i32Bit>},
{OPD(2, 0b01, 0x0D), 1, &OpDispatchBuilder::VPERMILRegOp<OpSize::i64Bit>},
{OPD(2, 0b01, 0x0E), 1, &OpDispatchBuilder::VTESTPOp<OpSize::i32Bit>},
{OPD(2, 0b01, 0x0F), 1, &OpDispatchBuilder::VTESTPOp<OpSize::i64Bit>},
{OPD(2, 0b01, 0x13), 1, &OpDispatchBuilder::VCVTPH2PSOp},
{OPD(2, 0b01, 0x16), 1, &OpDispatchBuilder::VPERMDOp},
{OPD(2, 0b01, 0x17), 1, &OpDispatchBuilder::PTestOp},
{OPD(2, 0b01, 0x18), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i32Bit>},
{OPD(2, 0b01, 0x19), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i64Bit>},
{OPD(2, 0b01, 0x1A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i128Bit>},
{OPD(2, 0b01, 0x1C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VABS, OpSize::i8Bit>},
{OPD(2, 0b01, 0x1D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VABS, OpSize::i16Bit>},
{OPD(2, 0b01, 0x1E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorUnaryOp, IR::OP_VABS, OpSize::i32Bit>},
{OPD(2, 0b01, 0x20), 1, &OpDispatchBuilder::ExtendVectorElements<OpSize::i8Bit, OpSize::i16Bit, true>},
{OPD(2, 0b01, 0x21), 1, &OpDispatchBuilder::ExtendVectorElements<OpSize::i8Bit, OpSize::i32Bit, true>},
{OPD(2, 0b01, 0x22), 1, &OpDispatchBuilder::ExtendVectorElements<OpSize::i8Bit, OpSize::i64Bit, true>},
{OPD(2, 0b01, 0x23), 1, &OpDispatchBuilder::ExtendVectorElements<OpSize::i16Bit, OpSize::i32Bit, true>},
{OPD(2, 0b01, 0x24), 1, &OpDispatchBuilder::ExtendVectorElements<OpSize::i16Bit, OpSize::i64Bit, true>},
{OPD(2, 0b01, 0x25), 1, &OpDispatchBuilder::ExtendVectorElements<OpSize::i32Bit, OpSize::i64Bit, true>},
{OPD(2, 0b01, 0x28), 1, &OpDispatchBuilder::VPMULLOp<OpSize::i32Bit, true>},
{OPD(2, 0b01, 0x29), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPEQ, OpSize::i64Bit>},
{OPD(2, 0b01, 0x2A), 1, &OpDispatchBuilder::MOVVectorNTOp},
{OPD(2, 0b01, 0x2B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPACKUSOp, OpSize::i32Bit>},
{OPD(2, 0b01, 0x2C), 1, &OpDispatchBuilder::VMASKMOVOp<OpSize::i32Bit, false>},
{OPD(2, 0b01, 0x2D), 1, &OpDispatchBuilder::VMASKMOVOp<OpSize::i64Bit, false>},
{OPD(2, 0b01, 0x2E), 1, &OpDispatchBuilder::VMASKMOVOp<OpSize::i32Bit, true>},
{OPD(2, 0b01, 0x2F), 1, &OpDispatchBuilder::VMASKMOVOp<OpSize::i64Bit, true>},
{OPD(2, 0b01, 0x30), 1, &OpDispatchBuilder::ExtendVectorElements<OpSize::i8Bit, OpSize::i16Bit, false>},
{OPD(2, 0b01, 0x31), 1, &OpDispatchBuilder::ExtendVectorElements<OpSize::i8Bit, OpSize::i32Bit, false>},
{OPD(2, 0b01, 0x32), 1, &OpDispatchBuilder::ExtendVectorElements<OpSize::i8Bit, OpSize::i64Bit, false>},
{OPD(2, 0b01, 0x33), 1, &OpDispatchBuilder::ExtendVectorElements<OpSize::i16Bit, OpSize::i32Bit, false>},
{OPD(2, 0b01, 0x34), 1, &OpDispatchBuilder::ExtendVectorElements<OpSize::i16Bit, OpSize::i64Bit, false>},
{OPD(2, 0b01, 0x35), 1, &OpDispatchBuilder::ExtendVectorElements<OpSize::i32Bit, OpSize::i64Bit, false>},
{OPD(2, 0b01, 0x36), 1, &OpDispatchBuilder::VPERMDOp},
{OPD(2, 0b01, 0x37), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VCMPGT, OpSize::i64Bit>},
{OPD(2, 0b01, 0x38), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSMIN, OpSize::i8Bit>},
{OPD(2, 0b01, 0x39), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSMIN, OpSize::i32Bit>},
{OPD(2, 0b01, 0x3A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUMIN, OpSize::i16Bit>},
{OPD(2, 0b01, 0x3B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUMIN, OpSize::i32Bit>},
{OPD(2, 0b01, 0x3C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSMAX, OpSize::i8Bit>},
{OPD(2, 0b01, 0x3D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VSMAX, OpSize::i32Bit>},
{OPD(2, 0b01, 0x3E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUMAX, OpSize::i16Bit>},
{OPD(2, 0b01, 0x3F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VUMAX, OpSize::i32Bit>},
{OPD(2, 0b01, 0x40), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorALUOp, IR::OP_VMUL, OpSize::i32Bit>},
{OPD(2, 0b01, 0x41), 1, &OpDispatchBuilder::PHMINPOSUWOp},
{OPD(2, 0b01, 0x45), 1, &OpDispatchBuilder::VPSRLVOp},
{OPD(2, 0b01, 0x46), 1, &OpDispatchBuilder::VPSRAVDOp},
{OPD(2, 0b01, 0x47), 1, &OpDispatchBuilder::VPSLLVOp},
{OPD(2, 0b01, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i32Bit>},
{OPD(2, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i64Bit>},
{OPD(2, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i128Bit>},
{OPD(2, 0b01, 0x78), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i8Bit>},
{OPD(2, 0b01, 0x79), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i16Bit>},
{OPD(2, 0b01, 0x8C), 1, &OpDispatchBuilder::VPMASKMOVOp<false>},
{OPD(2, 0b01, 0x8E), 1, &OpDispatchBuilder::VPMASKMOVOp<true>},
{OPD(2, 0b01, 0x90), 1, &OpDispatchBuilder::VPGATHER<OpSize::i32Bit>},
{OPD(2, 0b01, 0x91), 1, &OpDispatchBuilder::VPGATHER<OpSize::i64Bit>},
{OPD(2, 0b01, 0x92), 1, &OpDispatchBuilder::VPGATHER<OpSize::i32Bit>},
{OPD(2, 0b01, 0x93), 1, &OpDispatchBuilder::VPGATHER<OpSize::i64Bit>},
{OPD(2, 0b01, 0x96), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 1, 3, 2>}, // VFMADDSUB
{OPD(2, 0b01, 0x97), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 1, 3, 2>}, // VFMSUBADD
{OPD(2, 0b01, 0x98), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLA, false, 1, 3, 2>}, // VFMADD
{OPD(2, 0b01, 0x99), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLA, true, 1, 3, 2>}, // VFMADD
{OPD(2, 0b01, 0x9A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLS, false, 1, 3, 2>}, // VFMSUB
{OPD(2, 0b01, 0x9B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLS, true, 1, 3, 2>}, // VFMSUB
{OPD(2, 0b01, 0x9C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLA, false, 1, 3, 2>}, // VFNMADD
{OPD(2, 0b01, 0x9D), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLA, true, 1, 3, 2>}, // VFNMADD
{OPD(2, 0b01, 0x9E), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLS, false, 1, 3, 2>}, // VFNMSUB
{OPD(2, 0b01, 0x9F), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLS, true, 1, 3, 2>}, // VFNMSUB
{OPD(2, 0b01, 0xA8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLA, false, 2, 1, 3>}, // VFMADD
{OPD(2, 0b01, 0xA9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLA, true, 2, 1, 3>}, // VFMADD
{OPD(2, 0b01, 0xAA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLS, false, 2, 1, 3>}, // VFMSUB
{OPD(2, 0b01, 0xAB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLS, true, 2, 1, 3>}, // VFMSUB
{OPD(2, 0b01, 0xAC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLA, false, 2, 1, 3>}, // VFNMADD
{OPD(2, 0b01, 0xAD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLA, true, 2, 1, 3>}, // VFNMADD
{OPD(2, 0b01, 0xAE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLS, false, 2, 1, 3>}, // VFNMSUB
{OPD(2, 0b01, 0xAF), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLS, true, 2, 1, 3>}, // VFNMSUB
{OPD(2, 0b01, 0xB8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLA, false, 2, 3, 1>}, // VFMADD
{OPD(2, 0b01, 0xB9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLA, true, 2, 3, 1>}, // VFMADD
{OPD(2, 0b01, 0xBA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLS, false, 2, 3, 1>}, // VFMSUB
{OPD(2, 0b01, 0xBB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFMLS, true, 2, 3, 1>}, // VFMSUB
{OPD(2, 0b01, 0xBC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLA, false, 2, 3, 1>}, // VFNMADD
{OPD(2, 0b01, 0xBD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLA, true, 2, 3, 1>}, // VFNMADD
{OPD(2, 0b01, 0xBE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLS, false, 2, 3, 1>}, // VFNMSUB
{OPD(2, 0b01, 0xBF), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAImpl, IR::OP_VFNMLS, true, 2, 3, 1>}, // VFNMSUB
{OPD(2, 0b01, 0xA6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 2, 1, 3>}, // VFMADDSUB
{OPD(2, 0b01, 0xA7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 2, 1, 3>}, // VFMSUBADD
{OPD(2, 0b01, 0xB6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 2, 3, 1>}, // VFMADDSUB
{OPD(2, 0b01, 0xB7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 2, 3, 1>}, // VFMSUBADD
{OPD(2, 0b01, 0xDB), 1, &OpDispatchBuilder::AESImcOp},
{OPD(2, 0b01, 0xDC), 1, &OpDispatchBuilder::VAESEncOp},
{OPD(2, 0b01, 0xDD), 1, &OpDispatchBuilder::VAESEncLastOp},
{OPD(2, 0b01, 0xDE), 1, &OpDispatchBuilder::VAESDecOp},
{OPD(2, 0b01, 0xDF), 1, &OpDispatchBuilder::VAESDecLastOp},
{OPD(3, 0b01, 0x00), 1, &OpDispatchBuilder::VPERMQOp},
{OPD(3, 0b01, 0x01), 1, &OpDispatchBuilder::VPERMQOp},
{OPD(3, 0b01, 0x02), 1, &OpDispatchBuilder::VPBLENDDOp},
{OPD(3, 0b01, 0x04), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPERMILImmOp, OpSize::i32Bit>},
{OPD(3, 0b01, 0x05), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPERMILImmOp, OpSize::i64Bit>},
{OPD(3, 0b01, 0x06), 1, &OpDispatchBuilder::VPERM2Op},
{OPD(3, 0b01, 0x08), 1, &OpDispatchBuilder::AVXVectorRound<OpSize::i32Bit>},
{OPD(3, 0b01, 0x09), 1, &OpDispatchBuilder::AVXVectorRound<OpSize::i64Bit>},
{OPD(3, 0b01, 0x0A), 1, &OpDispatchBuilder::AVXInsertScalarRound<OpSize::i32Bit>},
{OPD(3, 0b01, 0x0B), 1, &OpDispatchBuilder::AVXInsertScalarRound<OpSize::i64Bit>},
{OPD(3, 0b01, 0x0C), 1, &OpDispatchBuilder::VPBLENDDOp},
{OPD(3, 0b01, 0x0D), 1, &OpDispatchBuilder::VBLENDPDOp},
{OPD(3, 0b01, 0x0E), 1, &OpDispatchBuilder::VPBLENDWOp},
{OPD(3, 0b01, 0x0F), 1, &OpDispatchBuilder::VPALIGNROp},
{OPD(3, 0b01, 0x14), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i8Bit>},
{OPD(3, 0b01, 0x15), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i16Bit>},
{OPD(3, 0b01, 0x16), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i32Bit>},
{OPD(3, 0b01, 0x17), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::PExtrOp, OpSize::i32Bit>},
{OPD(3, 0b01, 0x18), 1, &OpDispatchBuilder::VINSERTOp},
{OPD(3, 0b01, 0x19), 1, &OpDispatchBuilder::VEXTRACT128Op},
{OPD(3, 0b01, 0x1D), 1, &OpDispatchBuilder::VCVTPS2PHOp},
{OPD(3, 0b01, 0x20), 1, &OpDispatchBuilder::VPINSRBOp},
{OPD(3, 0b01, 0x21), 1, &OpDispatchBuilder::VINSERTPSOp},
{OPD(3, 0b01, 0x22), 1, &OpDispatchBuilder::VPINSRDQOp},
{OPD(3, 0b01, 0x38), 1, &OpDispatchBuilder::VINSERTOp},
{OPD(3, 0b01, 0x39), 1, &OpDispatchBuilder::VEXTRACT128Op},
{OPD(3, 0b01, 0x40), 1, &OpDispatchBuilder::VDPPOp<OpSize::i32Bit>},
{OPD(3, 0b01, 0x41), 1, &OpDispatchBuilder::VDPPOp<OpSize::i64Bit>},
{OPD(3, 0b01, 0x42), 1, &OpDispatchBuilder::VMPSADBWOp},
{OPD(3, 0b01, 0x46), 1, &OpDispatchBuilder::VPERM2Op},
{OPD(3, 0b01, 0x4A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorVariableBlend, OpSize::i32Bit>},
{OPD(3, 0b01, 0x4B), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorVariableBlend, OpSize::i64Bit>},
{OPD(3, 0b01, 0x4C), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVXVectorVariableBlend, OpSize::i8Bit>},
{OPD(3, 0b01, 0x60), 1, &OpDispatchBuilder::VPCMPESTRMOp},
{OPD(3, 0b01, 0x61), 1, &OpDispatchBuilder::VPCMPESTRIOp},
{OPD(3, 0b01, 0x62), 1, &OpDispatchBuilder::VPCMPISTRMOp},
{OPD(3, 0b01, 0x63), 1, &OpDispatchBuilder::VPCMPISTRIOp},
{OPD(3, 0b01, 0xDF), 1, &OpDispatchBuilder::AESKeyGenAssist},
};
#undef OPD
#define OPD(group, pp, opcode) (((group - X86Tables::TYPE_VEX_GROUP_12) << 4) | (pp << 3) | (opcode))
static constexpr std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr> VEXTableGroupOps[] {
{OPD(X86Tables::TYPE_VEX_GROUP_12, 1, 0b010), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRLIOp, OpSize::i16Bit>},
{OPD(X86Tables::TYPE_VEX_GROUP_12, 1, 0b110), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSLLIOp, OpSize::i16Bit>},
{OPD(X86Tables::TYPE_VEX_GROUP_12, 1, 0b100), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRAIOp, OpSize::i16Bit>},
{OPD(X86Tables::TYPE_VEX_GROUP_13, 1, 0b010), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRLIOp, OpSize::i32Bit>},
{OPD(X86Tables::TYPE_VEX_GROUP_13, 1, 0b110), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSLLIOp, OpSize::i32Bit>},
{OPD(X86Tables::TYPE_VEX_GROUP_13, 1, 0b100), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRAIOp, OpSize::i32Bit>},
{OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b010), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSRLIOp, OpSize::i64Bit>},
{OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b011), 1, &OpDispatchBuilder::VPSRLDQOp},
{OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b110), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPSLLIOp, OpSize::i64Bit>},
{OPD(X86Tables::TYPE_VEX_GROUP_14, 1, 0b111), 1, &OpDispatchBuilder::VPSLLDQOp},
{OPD(X86Tables::TYPE_VEX_GROUP_15, 0, 0b010), 1, &OpDispatchBuilder::LDMXCSR},
{OPD(X86Tables::TYPE_VEX_GROUP_15, 0, 0b011), 1, &OpDispatchBuilder::STMXCSR},
};
#undef OPD
if (CTX->HostFeatures.SupportsCRC) {
InstallToTable(FEXCore::X86Tables::H0F38TableOps, H0F38_CRC);
}
InstallToTable(FEXCore::X86Tables::H0F38TableOps, H0F38_SHA);
if (CTX->HostFeatures.SupportsAES) {
InstallToTable(FEXCore::X86Tables::H0F38TableOps, H0F38_AES);
InstallToTable(FEXCore::X86Tables::H0F3ATableOps, H0F3A_AES);
}
if (CTX->HostFeatures.SupportsCLZERO) {
InstallToTable(FEXCore::X86Tables::SecondModRMTableOps, SecondaryModRMExtensionOp_CLZero);
}
if (CTX->HostFeatures.SupportsRAND) {
InstallToTable(FEXCore::X86Tables::SecondInstGroupOps, SecondaryExtensionOp_RDRAND);
}
if (CTX->HostFeatures.SupportsAVX && CTX->HostFeatures.SupportsSVE256) {
InstallToTable(FEXCore::X86Tables::VEXTableOps, AVXTable);
InstallToTable(FEXCore::X86Tables::VEXTableGroupOps, VEXTableGroupOps);
if (CTX->HostFeatures.SupportsPMULL_128Bit) {
InstallToTable(FEXCore::X86Tables::VEXTableOps, VEX_PCLMUL);
}
} else if (CTX->HostFeatures.SupportsAVX) {
InstallAVX128Handlers();
}
if (CTX->HostFeatures.SupportsPMULL_128Bit) {
InstallToTable(FEXCore::X86Tables::H0F3ATableOps, H0F3A_PCLMUL);
}
Initialized = true;
}
void InstallOpcodeHandlers(Context::OperatingMode Mode) {
// Top bit indicating if it needs to be repeated with {0x40, 0x80} or'd in
// All OPDReg versions need it
#define OPDReg(op, reg) ((1 << 15) | ((op - 0xD8) << 8) | (reg << 3))
#define OPD(op, modrmop) (((op - 0xD8) << 8) | modrmop)
constexpr static std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> X87F64OpTable[] = {
{OPDReg(0xD8, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::i32Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xD8, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::i32Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xD8, 2) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i32Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xD8, 3) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i32Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xD8, 4) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i32Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xD8, 5) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i32Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xD8, 6) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i32Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xD8, 7) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i32Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xD8, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xD8, 0xC8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xD8, 0xD0), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPD(0xD8, 0xD8), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPD(0xD8, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xD8, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xD8, 0xF0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xD8, 0xF8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xD9, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64, OpSize::i32Bit>},
// 1 = Invalid
{OPDReg(0xD9, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTF64, OpSize::i32Bit>},
{OPDReg(0xD9, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTF64, OpSize::i32Bit>},
{OPDReg(0xD9, 4) | 0x00, 8, &OpDispatchBuilder::X87LDENVF64},
{OPDReg(0xD9, 5) | 0x00, 8, &OpDispatchBuilder::X87FLDCWF64},
{OPDReg(0xD9, 6) | 0x00, 8, &OpDispatchBuilder::X87FNSTENV},
{OPDReg(0xD9, 7) | 0x00, 8, &OpDispatchBuilder::X87FSTCW},
{OPD(0xD9, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDFromStack>},
{OPD(0xD9, 0xC8), 8, &OpDispatchBuilder::FXCH},
{OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::NOPOp}, // FNOP
// D1 = Invalid
// D8 = Invalid
{OPD(0xD9, 0xE0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKCHANGESIGN, false>},
{OPD(0xD9, 0xE1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKABS, false>},
// E2 = Invalid
{OPD(0xD9, 0xE4), 1, &OpDispatchBuilder::FTSTF64},
{OPD(0xD9, 0xE5), 1, &OpDispatchBuilder::X87FXAM},
// E6 = Invalid
{OPD(0xD9, 0xE8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0x3FF0000000000000>}, // 1.0
{OPD(0xD9, 0xE9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0x400A934F0979A372>}, // log2l(10)
{OPD(0xD9, 0xEA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0x3FF71547652B82FE>}, // log2l(e)
{OPD(0xD9, 0xEB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0x400921FB54442D18>}, // pi
{OPD(0xD9, 0xEC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0x3FD34413509F79FF>}, // log10l(2)
{OPD(0xD9, 0xED), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0x3FE62E42FEFA39EF>}, // log(2)
{OPD(0xD9, 0xEE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64_Const, 0>}, // 0.0
// EF = Invalid
{OPD(0xD9, 0xF0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80F2XM1STACK, false>},
{OPD(0xD9, 0xF1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FYL2X, false>},
{OPD(0xD9, 0xF2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80PTANSTACK, true>},
{OPD(0xD9, 0xF3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80ATANSTACK, false>},
{OPD(0xD9, 0xF4), 1, &OpDispatchBuilder::X87FXTRACTF64},
{OPD(0xD9, 0xF5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80FPREM1STACK, true>},
{OPD(0xD9, 0xF6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87ModifySTP, false>},
{OPD(0xD9, 0xF7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87ModifySTP, true>},
{OPD(0xD9, 0xF8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80FPREMSTACK, true>},
{OPD(0xD9, 0xF9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FYL2X, true>},
{OPD(0xD9, 0xFA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SQRTSTACK, false>},
{OPD(0xD9, 0xFB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SINCOSSTACK, true>},
{OPD(0xD9, 0xFC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80ROUNDSTACK, false>},
{OPD(0xD9, 0xFD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SCALESTACK, false>},
{OPD(0xD9, 0xFE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SINSTACK, true>},
{OPD(0xD9, 0xFF), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80COSSTACK, true>},
{OPDReg(0xDA, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::i32Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDA, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::i32Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDA, 2) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i32Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDA, 3) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i32Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDA, 4) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i32Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDA, 5) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i32Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDA, 6) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i32Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDA, 7) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i32Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xDA, 0xC0), 8, &OpDispatchBuilder::X87FCMOV},
{OPD(0xDA, 0xC8), 8, &OpDispatchBuilder::X87FCMOV},
{OPD(0xDA, 0xD0), 8, &OpDispatchBuilder::X87FCMOV},
{OPD(0xDA, 0xD8), 8, &OpDispatchBuilder::X87FCMOV},
// E0 = Invalid
// E8 = Invalid
{OPD(0xDA, 0xE9), 1,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, true>},
// EA = Invalid
// F0 = Invalid
// F8 = Invalid
{OPDReg(0xDB, 0) | 0x00, 8, &OpDispatchBuilder::FILDF64},
{OPDReg(0xDB, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, true>},
{OPDReg(0xDB, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, false>},
{OPDReg(0xDB, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, false>},
// 4 = Invalid
{OPDReg(0xDB, 5) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64, OpSize::f80Bit>},
// 6 = Invalid
{OPDReg(0xDB, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTF64, OpSize::f80Bit>},
{OPD(0xDB, 0xC0), 8, &OpDispatchBuilder::X87FCMOV},
{OPD(0xDB, 0xC8), 8, &OpDispatchBuilder::X87FCMOV},
{OPD(0xDB, 0xD0), 8, &OpDispatchBuilder::X87FCMOV},
{OPD(0xDB, 0xD8), 8, &OpDispatchBuilder::X87FCMOV},
// E0 = Invalid
{OPD(0xDB, 0xE2), 1, &OpDispatchBuilder::NOPOp}, // FNCLEX
{OPD(0xDB, 0xE3), 1, &OpDispatchBuilder::FNINIT},
// E4 = Invalid
{OPD(0xDB, 0xE8), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},
{OPD(0xDB, 0xF0), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},
// F8 = Invalid
{OPDReg(0xDC, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::i64Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDC, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::i64Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDC, 2) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i64Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDC, 3) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i64Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDC, 4) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i64Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDC, 5) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i64Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDC, 6) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i64Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDC, 7) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i64Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xDC, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDC, 0xC8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDC, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDC, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDC, 0xF0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDC, 0xF8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPDReg(0xDD, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDF64, OpSize::i64Bit>},
{OPDReg(0xDD, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, true>},
{OPDReg(0xDD, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTF64, OpSize::i64Bit>},
{OPDReg(0xDD, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTF64, OpSize::i64Bit>},
{OPDReg(0xDD, 4) | 0x00, 8, &OpDispatchBuilder::X87FRSTOR},
// 5 = Invalid
{OPDReg(0xDD, 6) | 0x00, 8, &OpDispatchBuilder::X87FNSAVE},
{OPDReg(0xDD, 7) | 0x00, 8, &OpDispatchBuilder::X87FNSTSW},
{OPD(0xDD, 0xC0), 8, &OpDispatchBuilder::X87FFREE},
{OPD(0xDD, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>}, // register-register from regular X87
{OPD(0xDD, 0xD8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>}, //^
{OPD(0xDD, 0xE0), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPD(0xDD, 0xE8), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDE, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::i16Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDE, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::i16Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDE, 2) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i16Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDE, 3) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::i16Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDE, 4) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i16Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDE, 5) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::i16Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDE, 6) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i16Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDE, 7) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::i16Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xDE, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADDF64, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDE, 0xC8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMULF64, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDE, 0xD9), 1,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, true>},
{OPD(0xDE, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDE, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUBF64, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDE, 0xF0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDE, 0xF8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIVF64, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPDReg(0xDF, 0) | 0x00, 8, &OpDispatchBuilder::FILDF64},
{OPDReg(0xDF, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, true>},
{OPDReg(0xDF, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, false>},
{OPDReg(0xDF, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, false>},
{OPDReg(0xDF, 4) | 0x00, 8, &OpDispatchBuilder::FBLDF64},
{OPDReg(0xDF, 5) | 0x00, 8, &OpDispatchBuilder::FILDF64},
{OPDReg(0xDF, 6) | 0x00, 8, &OpDispatchBuilder::FBSTPF64},
{OPDReg(0xDF, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FISTF64, false>},
// XXX: This should also set the x87 tag bits to empty
// We don't support this currently, so just pop the stack
{OPD(0xDF, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87ModifySTP, true>},
{OPD(0xDF, 0xE0), 8, &OpDispatchBuilder::X87FNSTSW},
{OPD(0xDF, 0xE8), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},
{OPD(0xDF, 0xF0), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},
};
constexpr static std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> X87OpTable[] = {
{OPDReg(0xD8, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::i32Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xD8, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::i32Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xD8, 2) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i32Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xD8, 3) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i32Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xD8, 4) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i32Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xD8, 5) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i32Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xD8, 6) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i32Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xD8, 7) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i32Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xD8, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xD8, 0xC8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xD8, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPD(0xD8, 0xD8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPD(0xD8, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xD8, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xD8, 0xF0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xD8, 0xF8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xD9, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD, OpSize::i32Bit>},
// 1 = Invalid
{OPDReg(0xD9, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::i32Bit>},
{OPDReg(0xD9, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::i32Bit>},
{OPDReg(0xD9, 4) | 0x00, 8, &OpDispatchBuilder::X87LDENV},
{OPDReg(0xD9, 5) | 0x00, 8, &OpDispatchBuilder::X87FLDCW}, // XXX: stubbed FLDCW
{OPDReg(0xD9, 6) | 0x00, 8, &OpDispatchBuilder::X87FNSTENV},
{OPDReg(0xD9, 7) | 0x00, 8, &OpDispatchBuilder::X87FSTCW},
{OPD(0xD9, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDFromStack>},
{OPD(0xD9, 0xC8), 8, &OpDispatchBuilder::FXCH},
{OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::NOPOp}, // FNOP
// D1 = Invalid
// D8 = Invalid
{OPD(0xD9, 0xE0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKCHANGESIGN, false>},
{OPD(0xD9, 0xE1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKABS, false>},
// E2 = Invalid
{OPD(0xD9, 0xE4), 1, &OpDispatchBuilder::FTST},
{OPD(0xD9, 0xE5), 1, &OpDispatchBuilder::X87FXAM},
// E6 = Invalid
{OPD(0xD9, 0xE8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_X87_ONE>}, // 1.0
{OPD(0xD9, 0xE9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_X87_LOG2_10>}, // log2l(10)
{OPD(0xD9, 0xEA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_X87_LOG2_E>}, // log2l(e)
{OPD(0xD9, 0xEB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_X87_PI>}, // pi
{OPD(0xD9, 0xEC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_X87_LOG10_2>}, // log10l(2)
{OPD(0xD9, 0xED), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_X87_LOG_2>}, // log(2)
{OPD(0xD9, 0xEE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD_Const, NamedVectorConstant::NAMED_VECTOR_ZERO>}, // 0.0
// EF = Invalid
{OPD(0xD9, 0xF0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80F2XM1STACK, false>},
{OPD(0xD9, 0xF1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FYL2X, false>},
{OPD(0xD9, 0xF2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80PTANSTACK, true>},
{OPD(0xD9, 0xF3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80ATANSTACK, false>},
{OPD(0xD9, 0xF4), 1, &OpDispatchBuilder::X87FXTRACT},
{OPD(0xD9, 0xF5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80FPREM1STACK, true>},
{OPD(0xD9, 0xF6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87ModifySTP, false>},
{OPD(0xD9, 0xF7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87ModifySTP, true>},
{OPD(0xD9, 0xF8), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80FPREMSTACK, true>},
{OPD(0xD9, 0xF9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FYL2X, true>},
{OPD(0xD9, 0xFA), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SQRTSTACK, false>},
{OPD(0xD9, 0xFB), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SINCOSSTACK, true>},
{OPD(0xD9, 0xFC), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80ROUNDSTACK, false>},
{OPD(0xD9, 0xFD), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SCALESTACK, false>},
{OPD(0xD9, 0xFE), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80SINSTACK, true>},
{OPD(0xD9, 0xFF), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80COSSTACK, true>},
{OPDReg(0xDA, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::i32Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDA, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::i32Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDA, 2) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i32Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDA, 3) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i32Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDA, 4) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i32Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDA, 5) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i32Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDA, 6) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i32Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDA, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i32Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xDA, 0xC0), 8, &OpDispatchBuilder::X87FCMOV},
{OPD(0xDA, 0xC8), 8, &OpDispatchBuilder::X87FCMOV},
{OPD(0xDA, 0xD0), 8, &OpDispatchBuilder::X87FCMOV},
{OPD(0xDA, 0xD8), 8, &OpDispatchBuilder::X87FCMOV},
// E0 = Invalid
// E8 = Invalid
{OPD(0xDA, 0xE9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, true>},
// EA = Invalid
// F0 = Invalid
// F8 = Invalid
{OPDReg(0xDB, 0) | 0x00, 8, &OpDispatchBuilder::FILD},
{OPDReg(0xDB, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, true>},
{OPDReg(0xDB, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, false>},
{OPDReg(0xDB, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, false>},
// 4 = Invalid
{OPDReg(0xDB, 5) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD, OpSize::f80Bit>},
// 6 = Invalid
{OPDReg(0xDB, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::f80Bit>},
{OPD(0xDB, 0xC0), 8, &OpDispatchBuilder::X87FCMOV},
{OPD(0xDB, 0xC8), 8, &OpDispatchBuilder::X87FCMOV},
{OPD(0xDB, 0xD0), 8, &OpDispatchBuilder::X87FCMOV},
{OPD(0xDB, 0xD8), 8, &OpDispatchBuilder::X87FCMOV},
// E0 = Invalid
{OPD(0xDB, 0xE2), 1, &OpDispatchBuilder::NOPOp}, // FNCLEX
{OPD(0xDB, 0xE3), 1, &OpDispatchBuilder::FNINIT},
// E4 = Invalid
{OPD(0xDB, 0xE8), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},
{OPD(0xDB, 0xF0), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},
// F8 = Invalid
{OPDReg(0xDC, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::i64Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDC, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::i64Bit, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDC, 2) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i64Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDC, 3) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i64Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDC, 4) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i64Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDC, 5) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i64Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDC, 6) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i64Bit, false, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDC, 7) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i64Bit, false, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xDC, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDC, 0xC8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDC, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDC, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDC, 0xF0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDC, 0xF8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPDReg(0xDD, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLD, OpSize::i64Bit>},
{OPDReg(0xDD, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, true>},
{OPDReg(0xDD, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::i64Bit>},
{OPDReg(0xDD, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FST, OpSize::i64Bit>},
{OPDReg(0xDD, 4) | 0x00, 8, &OpDispatchBuilder::X87FRSTOR},
// 5 = Invalid
{OPDReg(0xDD, 6) | 0x00, 8, &OpDispatchBuilder::X87FNSAVE},
{OPDReg(0xDD, 7) | 0x00, 8, &OpDispatchBuilder::X87FNSTSW},
{OPD(0xDD, 0xC0), 8, &OpDispatchBuilder::X87FFREE},
{OPD(0xDD, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},
{OPD(0xDD, 0xD8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},
{OPD(0xDD, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPD(0xDD, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDE, 0) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::i16Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDE, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::i16Bit, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDE, 2) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i16Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDE, 3) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::i16Bit, true, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, false>},
{OPDReg(0xDE, 4) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i16Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDE, 5) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::i16Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDE, 6) | 0x00, 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i16Bit, true, false, OpDispatchBuilder::OpResult::RES_ST0>},
{OPDReg(0xDE, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::i16Bit, true, true, OpDispatchBuilder::OpResult::RES_ST0>},
{OPD(0xDE, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FADD, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDE, 0xC8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FMUL, OpSize::f80Bit, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDE, 0xD9), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_X87, true>},
{OPD(0xDE, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDE, 0xE8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSUB, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDE, 0xF0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::f80Bit, false, true, OpDispatchBuilder::OpResult::RES_STI>},
{OPD(0xDE, 0xF8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FDIV, OpSize::f80Bit, false, false, OpDispatchBuilder::OpResult::RES_STI>},
{OPDReg(0xDF, 0) | 0x00, 8, &OpDispatchBuilder::FILD},
{OPDReg(0xDF, 1) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, true>},
{OPDReg(0xDF, 2) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, false>},
{OPDReg(0xDF, 3) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, false>},
{OPDReg(0xDF, 4) | 0x00, 8, &OpDispatchBuilder::FBLD},
{OPDReg(0xDF, 5) | 0x00, 8, &OpDispatchBuilder::FILD},
{OPDReg(0xDF, 6) | 0x00, 8, &OpDispatchBuilder::FBSTP},
{OPDReg(0xDF, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FIST, false>},
// XXX: This should also set the x87 tag bits to empty
// We don't support this currently, so just pop the stack
{OPD(0xDF, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87ModifySTP, true>},
{OPD(0xDF, 0xE0), 8, &OpDispatchBuilder::X87FNSTSW},
{OPD(0xDF, 0xE8), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},
{OPD(0xDF, 0xF0), 8,
&OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},
};
#undef OPD
#undef OPDReg
auto InstallToX87Table = [](auto& FinalTable, auto& LocalTable) {
for (auto Op : LocalTable) {
auto OpNum = std::get<0>(Op);
bool Repeat = (OpNum & 0x8000) != 0;
OpNum = OpNum & 0x7FF;
auto Dispatcher = std::get<2>(Op);
for (uint8_t i = 0; i < std::get<1>(Op); ++i) {
LOGMAN_THROW_A_FMT(FinalTable[OpNum + i].OpcodeDispatcher == nullptr, "Duplicate Entry");
FinalTable[OpNum + i].OpcodeDispatcher = Dispatcher;
// Flag to indicate if we need to repeat this op in {0x40, 0x80} ranges
if (Repeat) {
FinalTable[(OpNum | 0x40) + i].OpcodeDispatcher = Dispatcher;
FinalTable[(OpNum | 0x80) + i].OpcodeDispatcher = Dispatcher;
}
}
}
};
FEX_CONFIG_OPT(ReducedPrecision, X87REDUCEDPRECISION);
if (ReducedPrecision) {
InstallToX87Table(FEXCore::X86Tables::X87Ops, X87F64OpTable);
} else {
InstallToX87Table(FEXCore::X86Tables::X87Ops, X87OpTable);
}
}
} // namespace FEXCore::IR
Reference in new issue View Git Blame Copy Permalink