mirror of
https://github.com/FEX-Emu/FEX.git
synced 2026-10-06 22:00:16 +02:00
Pierre-Loup A. Griffais
c92805475d
Only detect certain kinds of mov reg,immediate so far, which was the bulk of Mono JIT activity.
5123 lines
182 KiB
C++
5123 lines
182 KiB
C++
// SPDX-License-Identifier: MIT
|
|
/*
|
|
$info$
|
|
tags: frontend|x86-to-ir, opcodes|dispatcher-implementations
|
|
desc: Handles x86/64 ops to IR, no-pf opt, local-flags opt
|
|
$end_info$
|
|
*/
|
|
|
|
#include "FEXCore/Core/HostFeatures.h"
|
|
#include "FEXCore/Utils/Telemetry.h"
|
|
#include "Interface/Context/Context.h"
|
|
#include "Interface/Core/OpcodeDispatcher.h"
|
|
#include "Interface/Core/X86Tables/X86Tables.h"
|
|
#include "Interface/IR/IR.h"
|
|
#include "Interface/IR/IREmitter.h"
|
|
|
|
#include <FEXCore/Config/Config.h>
|
|
#include <FEXCore/Core/Context.h>
|
|
#include <FEXCore/Core/CoreState.h>
|
|
#include <FEXCore/Core/X86Enums.h>
|
|
#include <FEXCore/HLE/SyscallHandler.h>
|
|
#include <FEXCore/IR/IR.h>
|
|
#include <FEXCore/Utils/EnumUtils.h>
|
|
#include <FEXCore/Utils/LogManager.h>
|
|
|
|
#include <FEXHeaderUtils/BitUtils.h>
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <cstdint>
|
|
|
|
namespace FEXCore::IR {
|
|
|
|
using X86Tables::OpToIndex;
|
|
|
|
#define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op
|
|
|
|
void OpDispatchBuilder::SyscallOp(OpcodeArgs, bool IsSyscallInst) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
const auto GPRSize = GetGPROpSize();
|
|
auto NewRIP = GetRelocatedPC(Op, -Op->InstSize);
|
|
_StoreContextGPR(GPRSize, NewRIP, offsetof(FEXCore::Core::CPUState, rip));
|
|
|
|
if (IsSyscallInst) {
|
|
// If this is the `Syscall` instruction rather than `int 0x80` then we need to do some additional work.
|
|
// RCX = RIP after this instruction
|
|
// R11 = EFlags
|
|
// Calculate flags.
|
|
CalculateDeferredFlags();
|
|
|
|
auto RFLAG = GetPackedRFLAG();
|
|
StoreGPRRegister(X86State::REG_R11, RFLAG, OpSize::i64Bit);
|
|
|
|
auto RIPAfterInst = GetRelocatedPC(Op);
|
|
StoreGPRRegister(X86State::REG_RCX, RIPAfterInst, OpSize::i64Bit);
|
|
}
|
|
|
|
FlushRegisterCache();
|
|
_Syscall();
|
|
|
|
if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_BLOCK_END) {
|
|
// RIP could have been updated after coming back from the Syscall.
|
|
NewRIP = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, rip));
|
|
ExitFunction(NewRIP);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::ThunkOp(OpcodeArgs) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
uint8_t* sha256 = (uint8_t*)(Op->PC + 2);
|
|
|
|
if (Is64BitMode) {
|
|
// x86-64 ABI puts the function argument in RDI
|
|
Thunk(LoadGPRRegister(X86State::REG_RDI), *reinterpret_cast<SHA256Sum*>(sha256));
|
|
} else {
|
|
// x86 fastcall ABI puts the function argument in ECX
|
|
Thunk(LoadGPRRegister(X86State::REG_RCX), *reinterpret_cast<SHA256Sum*>(sha256));
|
|
}
|
|
|
|
auto NewRIP = Pop(GPRSize);
|
|
|
|
// Store the new RIP
|
|
ExitFunction(NewRIP, BranchHint::Return);
|
|
BlockSetRIP = true;
|
|
}
|
|
|
|
void OpDispatchBuilder::LEAOp(OpcodeArgs) {
|
|
// LEA specifically ignores segment prefixes
|
|
const auto SrcSize = OpSizeFromSrc(Op);
|
|
const auto OpAddr = X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0);
|
|
OpSize DstSize {};
|
|
|
|
if (Is64BitMode) {
|
|
DstSize = OpAddr == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? OpSize::i16Bit :
|
|
OpAddr == X86Tables::DecodeFlags::FLAG_WIDENING_SIZE_LAST ? OpSize::i64Bit :
|
|
OpSize::i32Bit;
|
|
} else {
|
|
DstSize = OpAddr == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? OpSize::i16Bit : OpSize::i32Bit;
|
|
}
|
|
|
|
auto Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags, {.LoadData = false, .AllowUpperGarbage = SrcSize > DstSize});
|
|
StoreResultGPR_WithOpSize(Op, Op->Dest, Src, DstSize);
|
|
}
|
|
|
|
void OpDispatchBuilder::NOPOp(OpcodeArgs) {}
|
|
|
|
void OpDispatchBuilder::RETOp(OpcodeArgs) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
|
|
Ref SP = _RMWHandle(LoadGPRRegister(X86State::REG_RSP));
|
|
Ref NewRIP = Pop(GPRSize, SP);
|
|
|
|
if (Op->OP == 0xC2) {
|
|
auto Offset = LoadSourceGPR(Op, Op->Src[0], Op->Flags);
|
|
SP = Add(GPRSize, SP, Offset);
|
|
}
|
|
|
|
// Store the new stack pointer
|
|
StoreGPRRegister(X86State::REG_RSP, SP);
|
|
|
|
// Store the new RIP
|
|
ExitFunction(NewRIP, BranchHint::Return);
|
|
BlockSetRIP = true;
|
|
}
|
|
|
|
/*
|
|
stack contains:
|
|
Size of each member is 64-bit, 32-bit, or 16-bit depending on operating size
|
|
RIP
|
|
CS
|
|
EFLAGS
|
|
RSP
|
|
SS
|
|
*/
|
|
void OpDispatchBuilder::IRETOp(OpcodeArgs) {
|
|
// Operand Size override unsupported!
|
|
if ((Op->Flags & X86Tables::DecodeFlags::FLAG_OPERAND_SIZE) != 0) {
|
|
LogMan::Msg::EFmt("IRET only implemented for 64bit and 32bit sizes");
|
|
DecodeFailure = true;
|
|
return;
|
|
}
|
|
|
|
const auto GPRSize = GetGPROpSize();
|
|
|
|
Ref SP = _RMWHandle(LoadGPRRegister(X86State::REG_RSP));
|
|
|
|
// RIP (64/32/16 bits)
|
|
auto NewRIP = Pop(GPRSize, SP);
|
|
// CS (lower 16 used)
|
|
auto NewSegmentCS = Pop(GPRSize, SP);
|
|
_StoreContextGPR(OpSize::i16Bit, NewSegmentCS, offsetof(FEXCore::Core::CPUState, cs_idx));
|
|
UpdatePrefixFromSegment(NewSegmentCS, FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX);
|
|
|
|
// eflags (lower 16 used)
|
|
SetPackedRFLAG(false, Pop(GPRSize, SP));
|
|
|
|
if (Is64BitMode) {
|
|
// RSP and SS only happen in 64-bit mode or if this is a CPL mode jump!
|
|
// FEX doesn't support a CPL mode switch, so don't need to worry about this on 32-bit
|
|
StoreGPRRegister(X86State::REG_RSP, Pop(GPRSize, SP));
|
|
|
|
// ss
|
|
auto NewSegmentSS = Pop(GPRSize, SP);
|
|
_StoreContextGPR(OpSize::i16Bit, NewSegmentSS, offsetof(FEXCore::Core::CPUState, ss_idx));
|
|
UpdatePrefixFromSegment(NewSegmentSS, FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX);
|
|
} else {
|
|
// Store the stack in 32-bit mode
|
|
StoreGPRRegister(X86State::REG_RSP, SP);
|
|
}
|
|
|
|
ExitFunction(NewRIP);
|
|
BlockSetRIP = true;
|
|
}
|
|
|
|
void OpDispatchBuilder::CallbackReturnOp(OpcodeArgs) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
// Store the new RIP
|
|
_CallbackReturn();
|
|
auto NewRIP = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, rip));
|
|
// This ExitFunction won't actually get hit but needs to exist
|
|
ExitFunction(NewRIP);
|
|
BlockSetRIP = true;
|
|
}
|
|
|
|
void OpDispatchBuilder::SecondaryALUOp(OpcodeArgs) {
|
|
FEXCore::IR::IROps IROp, AtomicIROp;
|
|
#define OPD(group, prefix, Reg) (((group - FEXCore::X86Tables::TYPE_GROUP_1) << 6) | (prefix) << 3 | (Reg))
|
|
switch (Op->OP) {
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 0):
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 0):
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 0):
|
|
IROp = FEXCore::IR::IROps::OP_ADD;
|
|
AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHADD;
|
|
break;
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 1):
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 1):
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 1):
|
|
IROp = FEXCore::IR::IROps::OP_OR;
|
|
AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHOR;
|
|
break;
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 4):
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 4):
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 4):
|
|
IROp = FEXCore::IR::IROps::OP_ANDWITHFLAGS;
|
|
AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHAND;
|
|
break;
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 5):
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 5):
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 5):
|
|
IROp = FEXCore::IR::IROps::OP_SUB;
|
|
AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHSUB;
|
|
break;
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 6):
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 6):
|
|
case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 6):
|
|
IROp = FEXCore::IR::IROps::OP_XOR;
|
|
AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHXOR;
|
|
break;
|
|
default:
|
|
IROp = FEXCore::IR::IROps::OP_LAST;
|
|
AtomicIROp = FEXCore::IR::IROps::OP_LAST;
|
|
LogMan::Msg::EFmt("Unknown ALU Op: 0x{:x}", Op->OP);
|
|
DecodeFailure = true;
|
|
return;
|
|
};
|
|
#undef OPD
|
|
|
|
ALUOp(Op, IROp, AtomicIROp, 1);
|
|
}
|
|
|
|
void OpDispatchBuilder::ADCOp(OpcodeArgs, uint32_t SrcIndex) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true});
|
|
const auto Size = OpSizeFromDst(Op);
|
|
const auto OpSize = std::max(OpSize::i32Bit, Size);
|
|
|
|
Ref Before {};
|
|
if (DestIsLockedMem(Op)) {
|
|
auto ALUOp = IncrementByCarry(OpSize, Src);
|
|
HandledLock = true;
|
|
|
|
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
|
|
Before = _AtomicFetchAdd(Size, ALUOp, DestMem);
|
|
} else {
|
|
Before = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
}
|
|
|
|
Ref Result;
|
|
if (!DestIsLockedMem(Op) && Op->Src[SrcIndex].IsLiteral() && Op->Src[SrcIndex].Literal() == 0 && Size >= OpSize::i32Bit) {
|
|
HandleNZCV_RMW();
|
|
RectifyCarryInvert(true);
|
|
Result = _AdcZeroWithFlags(OpSize, Before);
|
|
SetRFLAG<FEXCore::X86State::RFLAG_AF_RAW_LOC>(Before);
|
|
CalculatePF(Result);
|
|
CFInverted = false;
|
|
} else {
|
|
Result = CalculateFlags_ADC(Size, Before, Src);
|
|
}
|
|
|
|
if (!DestIsLockedMem(Op)) {
|
|
StoreResultGPR(Op, Result);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::SBBOp(OpcodeArgs, uint32_t SrcIndex) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true});
|
|
const auto Size = OpSizeFromDst(Op);
|
|
const auto OpSize = std::max(OpSize::i32Bit, Size);
|
|
|
|
Ref Result {};
|
|
Ref Before {};
|
|
if (DestIsLockedMem(Op)) {
|
|
HandledLock = true;
|
|
|
|
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
|
|
auto SrcPlusCF = IncrementByCarry(OpSize, Src);
|
|
Before = _AtomicFetchSub(Size, SrcPlusCF, DestMem);
|
|
} else {
|
|
Before = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
}
|
|
|
|
Result = CalculateFlags_SBB(Size, Before, Src);
|
|
|
|
if (!DestIsLockedMem(Op)) {
|
|
StoreResultGPR(Op, Result);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::SALCOp(OpcodeArgs) {
|
|
CalculateDeferredFlags();
|
|
|
|
auto Result = NZCVSelect(OpSize::i32Bit, CondClass::UGE /* CF = 1 */, _InlineConstant(0xffffffff), _InlineConstant(0));
|
|
|
|
StoreResultGPR(Op, Result);
|
|
}
|
|
|
|
void OpDispatchBuilder::PUSHOp(OpcodeArgs) {
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
|
|
Push(Size, LoadSourceGPR(Op, Op->Src[0], Op->Flags));
|
|
}
|
|
|
|
void OpDispatchBuilder::PUSHREGOp(OpcodeArgs) {
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
|
|
Push(Size, LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true}));
|
|
}
|
|
|
|
void OpDispatchBuilder::PUSHAOp(OpcodeArgs) {
|
|
// 32bit only
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
|
|
Ref OldSP = _Copy(LoadGPRRegister(X86State::REG_RSP));
|
|
|
|
Push(Size, LoadGPRRegister(X86State::REG_RAX));
|
|
Push(Size, LoadGPRRegister(X86State::REG_RCX));
|
|
Push(Size, LoadGPRRegister(X86State::REG_RDX));
|
|
Push(Size, LoadGPRRegister(X86State::REG_RBX));
|
|
Push(Size, OldSP);
|
|
Push(Size, LoadGPRRegister(X86State::REG_RBP));
|
|
Push(Size, LoadGPRRegister(X86State::REG_RSI));
|
|
Push(Size, LoadGPRRegister(X86State::REG_RDI));
|
|
}
|
|
|
|
void OpDispatchBuilder::PUSHSegmentOp(OpcodeArgs, uint32_t SegmentReg) {
|
|
const auto SrcSize = OpSizeFromSrc(Op);
|
|
const auto DstSize = OpSizeFromDst(Op);
|
|
|
|
Ref Src {};
|
|
if (!Is64BitMode) {
|
|
switch (SegmentReg) {
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX: {
|
|
Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, es_idx));
|
|
break;
|
|
}
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX: {
|
|
Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, cs_idx));
|
|
break;
|
|
}
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX: {
|
|
Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, ss_idx));
|
|
break;
|
|
}
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX: {
|
|
Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, ds_idx));
|
|
break;
|
|
}
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX: {
|
|
Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, fs_idx));
|
|
break;
|
|
}
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX: {
|
|
Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, gs_idx));
|
|
break;
|
|
}
|
|
default: FEX_UNREACHABLE;
|
|
}
|
|
} else {
|
|
switch (SegmentReg) {
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
|
|
Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, es_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
|
|
Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, cs_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
|
|
Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, ss_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
|
|
Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, ds_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:
|
|
Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, fs_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:
|
|
Src = _LoadContextGPR(SrcSize, offsetof(FEXCore::Core::CPUState, gs_cached));
|
|
break;
|
|
default: FEX_UNREACHABLE;
|
|
}
|
|
}
|
|
|
|
// Store our value to the new stack location
|
|
// AMD hardware zexts segment selector to 32bit
|
|
// Intel hardware inserts segment selector
|
|
Push(DstSize, Src);
|
|
}
|
|
|
|
void OpDispatchBuilder::POPOp(OpcodeArgs) {
|
|
Ref Value = Pop(OpSizeFromSrc(Op));
|
|
StoreResultGPR(Op, Value);
|
|
}
|
|
|
|
void OpDispatchBuilder::POPAOp(OpcodeArgs) {
|
|
// 32bit only
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
|
|
Ref SP = _RMWHandle(LoadGPRRegister(X86State::REG_RSP));
|
|
|
|
StoreGPRRegister(X86State::REG_RDI, Pop(Size, SP), Size);
|
|
StoreGPRRegister(X86State::REG_RSI, Pop(Size, SP), Size);
|
|
StoreGPRRegister(X86State::REG_RBP, Pop(Size, SP), Size);
|
|
|
|
// Skip loading RSP because it'll be correct at the end
|
|
SP = _RMWHandle(Add(OpSize::i64Bit, SP, IR::OpSizeToSize(Size)));
|
|
|
|
StoreGPRRegister(X86State::REG_RBX, Pop(Size, SP), Size);
|
|
StoreGPRRegister(X86State::REG_RDX, Pop(Size, SP), Size);
|
|
StoreGPRRegister(X86State::REG_RCX, Pop(Size, SP), Size);
|
|
StoreGPRRegister(X86State::REG_RAX, Pop(Size, SP), Size);
|
|
|
|
// Store the new stack pointer
|
|
StoreGPRRegister(X86State::REG_RSP, SP);
|
|
}
|
|
|
|
void OpDispatchBuilder::POPSegmentOp(OpcodeArgs, uint32_t SegmentReg) {
|
|
const auto SrcSize = OpSizeFromSrc(Op);
|
|
const auto DstSize = OpSizeFromDst(Op);
|
|
|
|
auto NewSegment = Pop(SrcSize);
|
|
|
|
switch (SegmentReg) {
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
|
|
_StoreContextGPR(DstSize, NewSegment, offsetof(FEXCore::Core::CPUState, es_idx));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
|
|
_StoreContextGPR(DstSize, NewSegment, offsetof(FEXCore::Core::CPUState, cs_idx));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
|
|
// Unset the 'active' bit in the packed TF, skipping the single step exception after this instruction
|
|
SetRFLAG<FEXCore::X86State::RFLAG_TF_RAW_LOC>(_And(OpSize::i32Bit, GetRFLAG(FEXCore::X86State::RFLAG_TF_RAW_LOC), Constant(1)));
|
|
_StoreContextGPR(DstSize, NewSegment, offsetof(FEXCore::Core::CPUState, ss_idx));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
|
|
_StoreContextGPR(DstSize, NewSegment, offsetof(FEXCore::Core::CPUState, ds_idx));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:
|
|
_StoreContextGPR(DstSize, NewSegment, offsetof(FEXCore::Core::CPUState, fs_idx));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:
|
|
_StoreContextGPR(DstSize, NewSegment, offsetof(FEXCore::Core::CPUState, gs_idx));
|
|
break;
|
|
default: break; // Do nothing
|
|
}
|
|
|
|
UpdatePrefixFromSegment(NewSegment, SegmentReg);
|
|
}
|
|
|
|
void OpDispatchBuilder::LEAVEOp(OpcodeArgs) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
const auto OperandSize = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_OPERAND_SIZE) ? OpSize::i16Bit : GPRSize;
|
|
|
|
// First we move RBP in to RSP and then behave effectively like a pop
|
|
auto SP = _RMWHandle(LoadGPRRegister(X86State::REG_RBP));
|
|
auto NewGPR = Pop(OperandSize, SP);
|
|
|
|
// Store the new stack pointer
|
|
StoreGPRRegister(X86State::REG_RSP, SP, GPRSize);
|
|
|
|
// Store what we loaded to RBP
|
|
StoreGPRRegister(X86State::REG_RBP, NewGPR, OperandSize);
|
|
}
|
|
|
|
void OpDispatchBuilder::CALLOp(OpcodeArgs) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
|
|
BlockSetRIP = true;
|
|
|
|
// Call instruction only uses up to 32-bit signed displacement
|
|
const int64_t TargetOffset = Op->Src[0].Literal();
|
|
|
|
const auto ConstantPC = GetRelocatedPC(Op);
|
|
|
|
// Push the return address.
|
|
Push(GPRSize, ConstantPC);
|
|
|
|
if (TargetOffset != 0) {
|
|
// Store the RIP
|
|
const uint64_t NextRIP = Op->PC + Op->InstSize;
|
|
|
|
ExitRelocatedPC(Op, TargetOffset, BranchHint::Call, ConstantPC, [&]() {
|
|
auto CallReturnJumpTarget = JumpTargets.find(NextRIP);
|
|
if (CallReturnJumpTarget != JumpTargets.end() && CallReturnJumpTarget->second.IsEntryPoint) {
|
|
return CallReturnJumpTarget->second.BlockEntry;
|
|
}
|
|
return InvalidNode;
|
|
}());
|
|
} else {
|
|
NeedsBlockEnd = true;
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::CALLAbsoluteOp(OpcodeArgs) {
|
|
BlockSetRIP = true;
|
|
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
Ref JMPPCOffset = LoadSourceGPR(Op, Op->Src[0], Op->Flags);
|
|
|
|
// Push the return address.
|
|
auto ConstantPC = GetRelocatedPC(Op);
|
|
Push(Size, ConstantPC);
|
|
|
|
// Store the RIP
|
|
const uint64_t NextRIP = Op->PC + Op->InstSize;
|
|
ExitFunction(JMPPCOffset, BranchHint::Call, ConstantPC, [&]() {
|
|
auto CallReturnJumpTarget = JumpTargets.find(NextRIP);
|
|
if (CallReturnJumpTarget != JumpTargets.end() && CallReturnJumpTarget->second.IsEntryPoint) {
|
|
return CallReturnJumpTarget->second.BlockEntry;
|
|
}
|
|
return InvalidNode;
|
|
}());
|
|
}
|
|
|
|
std::optional<CondClass> OpDispatchBuilder::DecodeNZCVCondition(uint8_t OP) {
|
|
switch (OP) {
|
|
case 0x0: { // JO - Jump if OF == 1
|
|
return CondClass::FU;
|
|
}
|
|
case 0x1: { // JNO - Jump if OF == 0
|
|
return CondClass::FNU;
|
|
}
|
|
case 0x2: { // JC - Jump if CF == 1
|
|
return CFInverted ? CondClass::ULT : CondClass::UGE;
|
|
}
|
|
case 0x3: { // JNC - Jump if CF == 0
|
|
return CFInverted ? CondClass::UGE : CondClass::ULT;
|
|
}
|
|
case 0x4: { // JE - Jump if ZF == 1
|
|
return CondClass::EQ;
|
|
}
|
|
case 0x5: { // JNE - Jump if ZF == 0
|
|
return CondClass::NEQ;
|
|
}
|
|
case 0x6: { // JNA - Jump if CF == 1 || ZF == 1
|
|
// With CF, we want (C == 0 || Z == 1). By De Morgan's, that's
|
|
// equivalent to !(C == 1 && Z == 0). That's .ls
|
|
RectifyCarryInvert(true);
|
|
return CondClass::ULE;
|
|
}
|
|
case 0x7: { // JA - Jump if CF == 0 && ZF == 0
|
|
// With CF inverted, we want (C == 1 && Z == 0). That's .hi
|
|
RectifyCarryInvert(true);
|
|
return CondClass::UGT;
|
|
}
|
|
case 0x8: { // JS - Jump if SF == 1
|
|
return CondClass::MI;
|
|
}
|
|
case 0x9: { // JNS - Jump if SF == 0
|
|
return CondClass::PL;
|
|
}
|
|
case 0xC: { // SF <> OF
|
|
return CondClass::SLT;
|
|
}
|
|
case 0xD: { // SF = OF
|
|
return CondClass::SGE;
|
|
}
|
|
case 0xE: { // ZF = 1 || SF <> OF
|
|
return CondClass::SLE;
|
|
}
|
|
case 0xF: { // ZF = 0 && SF = OF
|
|
return CondClass::SGT;
|
|
}
|
|
default:
|
|
// Other conditions do not map directly, caller gets to deal with it.
|
|
return std::nullopt;
|
|
}
|
|
}
|
|
|
|
static bool ParityJumpIsJP(uint8_t OP) {
|
|
LOGMAN_THROW_A_FMT(OP == 0xA || OP == 0xB, "JP or JNP");
|
|
return OP == 0xA;
|
|
}
|
|
|
|
Ref OpDispatchBuilder::SelectCC0All1(uint8_t OP) {
|
|
if (auto Cond = DecodeNZCVCondition(OP); Cond) {
|
|
// Use raw select since DecodeNZCVCondition handles the carry invert
|
|
return _NZCVSelect(OpSize::i64Bit, *Cond, _InlineConstant(~0ULL), _InlineConstant(0));
|
|
} else {
|
|
// Raw value contains inverted PF in bottom bit
|
|
return _Sbfe(OpSize::i64Bit, 1, 0, LoadPFRaw(false, ParityJumpIsJP(OP)));
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::SETccOp(OpcodeArgs) {
|
|
CalculateDeferredFlags();
|
|
|
|
Ref SrcCond;
|
|
if (auto Cond = DecodeNZCVCondition(Op->OP & 0xf); Cond) {
|
|
// Use raw select since DecodeNZCVCondition handles the carry invert
|
|
SrcCond = _NZCVSelect01(*Cond);
|
|
} else {
|
|
SrcCond = LoadPFRaw(true, ParityJumpIsJP(Op->OP & 0xf));
|
|
}
|
|
|
|
StoreResultGPR(Op, SrcCond);
|
|
}
|
|
|
|
void OpDispatchBuilder::CMOVOp(OpcodeArgs) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
const auto OP = Op->OP & 0xF;
|
|
const auto ResultSize = std::max(OpSize::i32Bit, OpSizeFromSrc(Op));
|
|
|
|
CalculateDeferredFlags();
|
|
|
|
// Destination is always a GPR.
|
|
Ref Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, GPRSize, Op->Flags);
|
|
Ref Src {}, SrcCond {};
|
|
if (Op->Src[0].IsGPR()) {
|
|
Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], GPRSize, Op->Flags);
|
|
} else {
|
|
Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);
|
|
}
|
|
|
|
if (auto Cond = DecodeNZCVCondition(OP); Cond) {
|
|
// Use raw select since DecodeNZCVCondition handles the carry invert
|
|
SrcCond = _NZCVSelect(ResultSize, *Cond, Src, Dest);
|
|
} else {
|
|
// Raw value contains inverted PF in bottom bit
|
|
Ref Cmp = LoadPFRaw(false, ParityJumpIsJP(OP));
|
|
SaveNZCV();
|
|
|
|
// Because we're only clobbering NZCV internally, we ignore all carry flag
|
|
// shenanigans and just use the raw test and raw select.
|
|
_TestNZ(OpSize::i32Bit, Cmp, _InlineConstant(1));
|
|
SrcCond = _NZCVSelect(ResultSize, CondClass::NEQ, Src, Dest);
|
|
}
|
|
|
|
StoreResultGPR(Op, SrcCond);
|
|
}
|
|
|
|
void OpDispatchBuilder::CondJUMPOp(OpcodeArgs) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
BlockSetRIP = true;
|
|
|
|
// Jump instruction only uses up to 32-bit signed displacement
|
|
int64_t TargetOffset = Op->Src[0].Literal();
|
|
uint64_t InstRIP = Op->PC + Op->InstSize;
|
|
uint64_t Target = InstRIP + TargetOffset;
|
|
|
|
if (GetGPROpSize() == OpSize::i32Bit) {
|
|
// If the GPRSize is 4 then we need to be careful about PC wrapping
|
|
if (TargetOffset < 0 && -TargetOffset > InstRIP) {
|
|
// Invert the signed value if we are underflowing
|
|
TargetOffset = 0x1'0000'0000ULL + TargetOffset;
|
|
} else if (TargetOffset >= 0 && Target >= 0x1'0000'0000ULL) {
|
|
// We are overflowing, wrap around
|
|
TargetOffset = TargetOffset - 0x1'0000'0000ULL;
|
|
}
|
|
Target &= 0xFFFFFFFFU;
|
|
}
|
|
|
|
FlushRegisterCache();
|
|
auto TrueBlock = JumpTargets.find(Target);
|
|
auto FalseBlock = JumpTargets.find(Op->PC + Op->InstSize);
|
|
|
|
auto CurrentBlock = GetCurrentBlock();
|
|
|
|
{
|
|
IRPair<IR::IROp_CondJump> CondJump_;
|
|
auto OP = Op->OP & 0xF;
|
|
auto Cond = DecodeNZCVCondition(OP);
|
|
if (Cond) {
|
|
CondJump_ = CondJumpNZCV(*Cond);
|
|
} else {
|
|
LOGMAN_THROW_A_FMT(OP == 0xA || OP == 0xB, "only PF left");
|
|
CondJump_ = CondJumpBit(LoadPFRaw(false, false), 0, OP == 0xB);
|
|
}
|
|
|
|
// Taking branch block
|
|
if (TrueBlock != JumpTargets.end()) {
|
|
SetTrueJumpTarget(CondJump_, TrueBlock->second.BlockEntry);
|
|
} else {
|
|
// Make sure to start a new block after ending this one
|
|
auto JumpTarget = CreateNewCodeBlockAtEnd();
|
|
SetTrueJumpTarget(CondJump_, JumpTarget);
|
|
SetCurrentCodeBlock(JumpTarget);
|
|
StartNewBlock();
|
|
|
|
// Store the new RIP
|
|
ExitRelocatedPC(Op, TargetOffset);
|
|
}
|
|
|
|
// Failure to take branch
|
|
if (FalseBlock != JumpTargets.end()) {
|
|
SetFalseJumpTarget(CondJump_, FalseBlock->second.BlockEntry);
|
|
} else {
|
|
// Make sure to start a new block after ending this one
|
|
// Place it after this block for fallthrough optimization
|
|
auto JumpTarget = CreateNewCodeBlockAfter(CurrentBlock);
|
|
SetFalseJumpTarget(CondJump_, JumpTarget);
|
|
SetCurrentCodeBlock(JumpTarget);
|
|
StartNewBlock();
|
|
|
|
// Leave block & store the new RIP
|
|
ExitRelocatedPC(Op);
|
|
}
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::CondJUMPRCXOp(OpcodeArgs) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
BlockSetRIP = true;
|
|
auto JcxGPRSize = GetGPROpSize();
|
|
JcxGPRSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) ? (JcxGPRSize >> 1) : JcxGPRSize;
|
|
|
|
uint64_t Target = Op->PC + Op->InstSize + Op->Src[0].Literal();
|
|
|
|
Ref CondReg = LoadGPRRegister(X86State::REG_RCX, JcxGPRSize);
|
|
|
|
auto TrueBlock = JumpTargets.find(Target);
|
|
auto FalseBlock = JumpTargets.find(Op->PC + Op->InstSize);
|
|
|
|
auto CurrentBlock = GetCurrentBlock();
|
|
|
|
{
|
|
auto CondJump_ = CondJump(CondReg, CondClass::EQ);
|
|
|
|
// Taking branch block
|
|
if (TrueBlock != JumpTargets.end()) {
|
|
SetTrueJumpTarget(CondJump_, TrueBlock->second.BlockEntry);
|
|
} else {
|
|
// Make sure to start a new block after ending this one
|
|
auto JumpTarget = CreateNewCodeBlockAtEnd();
|
|
SetTrueJumpTarget(CondJump_, JumpTarget);
|
|
SetCurrentCodeBlock(JumpTarget);
|
|
StartNewBlock();
|
|
|
|
// Store the new RIP
|
|
ExitRelocatedPC(Op, Op->Src[0].Literal());
|
|
}
|
|
|
|
// Failure to take branch
|
|
if (FalseBlock != JumpTargets.end()) {
|
|
SetFalseJumpTarget(CondJump_, FalseBlock->second.BlockEntry);
|
|
} else {
|
|
// Make sure to start a new block after ending this one
|
|
// Place it after the current block for fallthrough behavior
|
|
auto JumpTarget = CreateNewCodeBlockAfter(CurrentBlock);
|
|
SetFalseJumpTarget(CondJump_, JumpTarget);
|
|
SetCurrentCodeBlock(JumpTarget);
|
|
StartNewBlock();
|
|
|
|
// Leave block & store the new RIP
|
|
ExitRelocatedPC(Op);
|
|
}
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::LoopOp(OpcodeArgs) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
bool CheckZF = Op->OP != 0xE2;
|
|
bool ZFTrue = Op->OP == 0xE1;
|
|
|
|
BlockSetRIP = true;
|
|
auto SrcSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) ? OpSize::i32Bit : OpSize::i64Bit;
|
|
auto OpSize = SrcSize == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit;
|
|
|
|
if (!Is64BitMode) {
|
|
// RCX size is 32-bit or 16-bit when executing in 32-bit mode.
|
|
SrcSize = IR::SizeToOpSize(IR::OpSizeToSize(SrcSize) >> 1);
|
|
OpSize = OpSize::i32Bit;
|
|
}
|
|
|
|
uint64_t Target = Op->PC + Op->InstSize + Op->Src[1].Literal();
|
|
|
|
Ref CondReg = LoadSourceGPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);
|
|
CondReg = Sub(OpSize, CondReg, 1);
|
|
StoreResultGPR(Op, Op->Src[0], CondReg);
|
|
|
|
// If LOOPE then jumps to target if RCX != 0 && ZF == 1
|
|
// If LOOPNE then jumps to target if RCX != 0 && ZF == 0
|
|
//
|
|
// To handle efficiently, smash RCX to zero if ZF is wrong (1 csel).
|
|
if (CheckZF) {
|
|
const auto cond = ZFTrue ? CondClass::EQ : CondClass::NEQ;
|
|
CondReg = NZCVSelect(OpSize, cond, CondReg, _InlineConstant(0));
|
|
}
|
|
|
|
CalculateDeferredFlags();
|
|
auto TrueBlock = JumpTargets.find(Target);
|
|
auto FalseBlock = JumpTargets.find(Op->PC + Op->InstSize);
|
|
|
|
{
|
|
auto CondJump_ = CondJump(CondReg);
|
|
|
|
// Taking branch block
|
|
if (TrueBlock != JumpTargets.end()) {
|
|
SetTrueJumpTarget(CondJump_, TrueBlock->second.BlockEntry);
|
|
} else {
|
|
// Make sure to start a new block after ending this one
|
|
auto JumpTarget = CreateNewCodeBlockAtEnd();
|
|
SetTrueJumpTarget(CondJump_, JumpTarget);
|
|
SetCurrentCodeBlock(JumpTarget);
|
|
StartNewBlock();
|
|
|
|
// Store the new RIP
|
|
ExitRelocatedPC(Op, Op->Src[1].Literal());
|
|
}
|
|
|
|
// Failure to take branch
|
|
if (FalseBlock != JumpTargets.end()) {
|
|
SetFalseJumpTarget(CondJump_, FalseBlock->second.BlockEntry);
|
|
} else {
|
|
// Make sure to start a new block after ending this one
|
|
// Place after this block for fallthrough behavior
|
|
auto JumpTarget = CreateNewCodeBlockAfter(GetCurrentBlock());
|
|
SetFalseJumpTarget(CondJump_, JumpTarget);
|
|
SetCurrentCodeBlock(JumpTarget);
|
|
StartNewBlock();
|
|
|
|
// Leave block & store the new RIP
|
|
ExitRelocatedPC(Op);
|
|
}
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::JUMPOp(OpcodeArgs) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
BlockSetRIP = true;
|
|
|
|
// Jump instruction only uses up to 32-bit signed displacement
|
|
int64_t TargetOffset = Op->Src[0].Literal();
|
|
uint64_t InstRIP = Op->PC + Op->InstSize;
|
|
uint64_t TargetRIP = InstRIP + TargetOffset;
|
|
|
|
if (GetGPROpSize() == OpSize::i32Bit) {
|
|
// If the GPRSize is 4 then we need to be careful about PC wrapping
|
|
if (TargetOffset < 0 && -TargetOffset > InstRIP) {
|
|
// Invert the signed value if we are underflowing
|
|
TargetOffset = 0x1'0000'0000ULL + TargetOffset;
|
|
} else if (TargetOffset >= 0 && TargetRIP >= 0x1'0000'0000ULL) {
|
|
// We are overflowing, wrap around
|
|
TargetOffset = TargetOffset - 0x1'0000'0000ULL;
|
|
}
|
|
|
|
TargetRIP &= 0xFFFFFFFFU;
|
|
}
|
|
|
|
CalculateDeferredFlags();
|
|
// This is just an unconditional relative literal jump
|
|
if (Multiblock) {
|
|
auto JumpBlock = JumpTargets.find(TargetRIP);
|
|
if (JumpBlock != JumpTargets.end()) {
|
|
Jump(GetNewJumpBlock(TargetRIP));
|
|
} else {
|
|
// If the block isn't a jump target then we need to create an exit block
|
|
auto Jump_ = Jump();
|
|
|
|
// Place after this block for fallthrough behavior
|
|
auto JumpTarget = CreateNewCodeBlockAfter(GetCurrentBlock());
|
|
SetJumpTarget(Jump_, JumpTarget);
|
|
SetCurrentCodeBlock(JumpTarget);
|
|
StartNewBlock();
|
|
ExitRelocatedPC(Op, TargetOffset);
|
|
}
|
|
} else {
|
|
ExitRelocatedPC(Op, TargetOffset);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::JUMPAbsoluteOp(OpcodeArgs) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
BlockSetRIP = true;
|
|
// This is just an unconditional jump
|
|
// This uses ModRM to determine its location
|
|
// No way to use this effectively in multiblock
|
|
auto RIPOffset = LoadSourceGPR(Op, Op->Src[0], Op->Flags);
|
|
|
|
// Store the new RIP
|
|
ExitFunction(RIPOffset);
|
|
}
|
|
|
|
void OpDispatchBuilder::JUMPFARIndirectOp(OpcodeArgs) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
BlockSetRIP = true;
|
|
// This is just an unconditional jump
|
|
// This uses ModRM to determine its location
|
|
// No way to use this effectively in multiblock
|
|
Ref Src = MakeSegmentAddress(Op, Op->Dest);
|
|
AddressMode SrcCS = {.Base = Src, .Offset = 4, .AddrSize = OpSize::i64Bit};
|
|
auto RIPOffset = _LoadMemGPRAutoTSO(OpSize::i32Bit, Src, OpSize::i8Bit);
|
|
auto NewSegmentCS = _LoadMemGPRAutoTSO(OpSize::i16Bit, SrcCS, OpSize::i8Bit);
|
|
|
|
// Set up the new CSSegment.
|
|
_StoreContextGPR(OpSize::i16Bit, NewSegmentCS, offsetof(FEXCore::Core::CPUState, cs_idx));
|
|
UpdatePrefixFromSegment(NewSegmentCS, FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX);
|
|
|
|
// Store the new RIP
|
|
ExitFunction(RIPOffset);
|
|
}
|
|
|
|
void OpDispatchBuilder::CALLFARIndirectOp(OpcodeArgs) {
|
|
const auto SrcSize = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;
|
|
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
BlockSetRIP = true;
|
|
|
|
Ref Src = MakeSegmentAddress(Op, Op->Dest);
|
|
AddressMode SrcCS = {.Base = Src, .Offset = 4, .AddrSize = OpSize::i64Bit};
|
|
auto RIPOffset = _LoadMemGPRAutoTSO(OpSize::i32Bit, Src, OpSize::i8Bit);
|
|
auto NewSegmentCS = _LoadMemGPRAutoTSO(OpSize::i16Bit, SrcCS, OpSize::i8Bit);
|
|
auto CurrentCS = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, cs_idx));
|
|
|
|
auto NewRIP = GetRelocatedPC(Op);
|
|
|
|
// Push the current CS
|
|
Push(SrcSize, CurrentCS);
|
|
|
|
// Push the return address.
|
|
Push(SrcSize, NewRIP);
|
|
|
|
// Set up the new CSSegment.
|
|
_StoreContextGPR(OpSize::i16Bit, NewSegmentCS, offsetof(FEXCore::Core::CPUState, cs_idx));
|
|
UpdatePrefixFromSegment(NewSegmentCS, FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX);
|
|
|
|
// Store the new RIP
|
|
ExitFunction(RIPOffset);
|
|
}
|
|
|
|
void OpDispatchBuilder::RETFARIndirectOp(OpcodeArgs) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
const auto SrcSize = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;
|
|
|
|
Ref SP = _RMWHandle(LoadGPRRegister(X86State::REG_RSP));
|
|
Ref NewRIP = Pop(SrcSize, SP);
|
|
Ref NewSegmentCS = Pop(SrcSize, SP);
|
|
|
|
// Optional SP offset.
|
|
if (Op->Src[0].IsLiteral()) {
|
|
SP = Add(GPRSize, SP, Op->Src[0].Literal());
|
|
}
|
|
|
|
// Store the new stack pointer
|
|
StoreGPRRegister(X86State::REG_RSP, SP);
|
|
|
|
_StoreContextGPR(OpSize::i16Bit, NewSegmentCS, offsetof(FEXCore::Core::CPUState, cs_idx));
|
|
UpdatePrefixFromSegment(NewSegmentCS, FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX);
|
|
|
|
// Store the new RIP
|
|
ExitFunction(NewRIP);
|
|
BlockSetRIP = true;
|
|
}
|
|
|
|
void OpDispatchBuilder::TESTOp(OpcodeArgs, uint32_t SrcIndex) {
|
|
// TEST is an instruction that does an AND between the sources
|
|
// Result isn't stored in result, only writes to flags
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
const auto Size = OpSizeFromDst(Op);
|
|
LOGMAN_THROW_A_FMT(Size >= IR::OpSize::i8Bit && Size <= IR::OpSize::i64Bit, "Invalid size");
|
|
|
|
uint64_t Const;
|
|
bool AlwaysNonnegative = false;
|
|
if (IsValueConstant(WrapNode(Src), &Const)) {
|
|
// Optimize out masking constants
|
|
if (Const == (Size == OpSize::i64Bit ? ~0ULL : ((1ull << IR::OpSizeAsBits(Size)) - 1))) {
|
|
Src = Dest;
|
|
}
|
|
|
|
// Optimize test with non-sign bits
|
|
AlwaysNonnegative = (Const & (1ull << (IR::OpSizeAsBits(Size) - 1))) == 0;
|
|
}
|
|
|
|
if (Dest == Src) {
|
|
// Optimize out the AND.
|
|
SetNZP_ZeroCV(Size, Src);
|
|
} else if (Size < OpSize::i32Bit && AlwaysNonnegative) {
|
|
// If we know the result is always nonnegative, we can use a 32-bit test.
|
|
auto Res = _And(OpSize::i32Bit, Dest, Src);
|
|
CalculatePF(Res);
|
|
SetNZ_ZeroCV(OpSize::i32Bit, Res);
|
|
} else {
|
|
HandleNZ00Write();
|
|
CalculatePF(_AndWithFlags(Size, Dest, Src));
|
|
}
|
|
|
|
InvalidateAF();
|
|
}
|
|
|
|
void OpDispatchBuilder::ARPLOp(OpcodeArgs) {
|
|
// ARPL r/m16, r16
|
|
// If the RPL field in the destination selector is less privileged than the
|
|
// RPL field in the source selector, then adjust destination RPL to match
|
|
// source RPL and set ZF=1. Otherwise ZF=0 and destination is unchanged.
|
|
//
|
|
// Only ZF is modified by ARPL.
|
|
constexpr auto Size = OpSize::i16Bit;
|
|
|
|
Ref Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, Size, Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], Size, Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// RPL is the low two bits of the selector.
|
|
Ref DestRPL = _Bfe(OpSize::i32Bit, 2, 0, Dest);
|
|
Ref SrcRPL = _Bfe(OpSize::i32Bit, 2, 0, Src);
|
|
|
|
// NeedUpdate is 1 when DestRPL < SrcRPL, else 0.
|
|
Ref NeedUpdate = _Select(OpSize::i32Bit, OpSize::i32Bit, CondClass::ULT, DestRPL, SrcRPL, Constant(1), Constant(0));
|
|
SetRFLAG<FEXCore::X86State::RFLAG_ZF_RAW_LOC>(NeedUpdate);
|
|
|
|
// Compute adjusted destination selector: (Dest & ~3) | SrcRPL.
|
|
auto NewDest = _Bfxil(OpSize::i32Bit, 2, 0, Dest, SrcRPL);
|
|
|
|
// Conditionally select updated selector based on NeedUpdate.
|
|
Ref FinalDest = _Select(OpSize::i32Bit, OpSize::i32Bit, CondClass::NEQ, NeedUpdate, Constant(0), NewDest, Dest);
|
|
StoreResultGPR_WithOpSize(Op, Op->Dest, FinalDest, Size);
|
|
}
|
|
|
|
void OpDispatchBuilder::MOVSXDOp(OpcodeArgs) {
|
|
// This instruction is a bit special
|
|
// if SrcSize == 2
|
|
// Then lower 16 bits of destination is written without changing the upper 48 bits
|
|
// else /* Size == 4 */
|
|
// if REX_WIDENING:
|
|
// Sext(32, Src)
|
|
// else
|
|
// Zext(32, Src)
|
|
//
|
|
auto Size = std::min<IR::OpSize>(OpSize::i32Bit, OpSizeFromSrc(Op));
|
|
bool Sext = (Size != OpSize::i16Bit) && Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING;
|
|
|
|
Ref Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], Size, Op->Flags, {.AllowUpperGarbage = Sext});
|
|
if (Size == OpSize::i16Bit) {
|
|
// This'll make sure to insert in to the lower 16bits without modifying upper bits
|
|
StoreResultGPR_WithOpSize(Op, Op->Dest, Src, Size);
|
|
} else if (Sext) {
|
|
// With REX.W then Sext
|
|
Src = _Sbfe(OpSize::i64Bit, IR::OpSizeAsBits(Size), 0, Src);
|
|
StoreResultGPR(Op, Src);
|
|
} else {
|
|
// Without REX.W then Zext (store result implicitly zero extends)
|
|
StoreResultGPR(Op, Src);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::MOVSXOp(OpcodeArgs) {
|
|
// Load garbage in upper bits, since we're sign extending anyway
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// Sign-extend to DstSize and zero-extend to the register size, using a fast
|
|
// path for 32-bit dests where the native 32-bit Sbfe zero extends the top.
|
|
const auto DstSize = OpSizeFromDst(Op);
|
|
Src = _Sbfe(DstSize == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, Src);
|
|
StoreResultGPR(Op, Op->Dest, Src);
|
|
}
|
|
|
|
void OpDispatchBuilder::MOVZXOp(OpcodeArgs) {
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);
|
|
// Store result implicitly zero extends
|
|
StoreResultGPR(Op, Src);
|
|
}
|
|
|
|
void OpDispatchBuilder::CMPOp(OpcodeArgs, uint32_t SrcIndex) {
|
|
// CMP is an instruction that does a SUB between the sources
|
|
// Result isn't stored in result, only writes to flags
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
CalculateFlags_SUB(OpSizeFromSrc(Op), Dest, Src);
|
|
}
|
|
|
|
void OpDispatchBuilder::CQOOp(OpcodeArgs) {
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Size = OpSizeFromSrc(Op);
|
|
Ref Upper = _Sbfe(std::max(OpSize::i32Bit, Size), 1, GetSrcBitSize(Op) - 1, Src);
|
|
|
|
StoreResultGPR(Op, Upper);
|
|
}
|
|
|
|
void OpDispatchBuilder::XCHGOp(OpcodeArgs) {
|
|
// Load both the source and the destination
|
|
if (Op->OP == 0x90 && Op->Src[0].IsGPR() && Op->Src[0].Data.GPR.GPR == FEXCore::X86State::REG_RAX && Op->Dest.IsGPR() &&
|
|
Op->Dest.Data.GPR.GPR == FEXCore::X86State::REG_RAX) {
|
|
// This is one heck of a sucky special case
|
|
// If we are the 0x90 XCHG opcode (Meaning source is GPR RAX)
|
|
// and destination register is ALSO RAX
|
|
// and in this very specific case we are 32bit or above
|
|
// Then this is a no-op
|
|
// This is because 0x90 without a prefix is technically `xchg eax, eax`
|
|
// But this would result in a zext on 64bit, which would ruin the no-op nature of the instruction
|
|
// So x86-64 spec mandates this special case that even though it is a 32bit instruction and
|
|
// is supposed to zext the result, it is a true no-op
|
|
//
|
|
// x86 spec text here:
|
|
//
|
|
// XCHG (E)AX, (E)AX (encoded instruction byte is 90H) is an alias for
|
|
// NOP regardless of data size prefixes, including REX.W.
|
|
//
|
|
// Note that also includes 16-bit so we don't gate this on size. The
|
|
// sequence (66 90) is a valid two-byte nop that we also ignore.
|
|
if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX) {
|
|
// If this instruction has a REP prefix then this is architecturally
|
|
// defined to be a `PAUSE` instruction. On older processors this ends up
|
|
// being a true `REP NOP` which is why they stuck this here.
|
|
_Yield();
|
|
}
|
|
return;
|
|
}
|
|
|
|
// AllowUpperGarbage: OK to allow as it will be overwritten by StoreResult.
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
if (DestIsMem(Op)) {
|
|
HandledLock = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;
|
|
|
|
Ref Dest = MakeSegmentAddress(Op, Op->Dest);
|
|
if (IsMonoBackpatcherBlock) {
|
|
_MonoBackpatcherWrite(OpSizeFromSrc(Op), Src, Dest);
|
|
} else {
|
|
auto Result = _AtomicSwap(OpSizeFromSrc(Op), Src, Dest);
|
|
StoreResultGPR(Op, Op->Src[0], Result);
|
|
}
|
|
} else {
|
|
// AllowUpperGarbage: OK to allow as it will be overwritten by StoreResult.
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// Swap the contents
|
|
// Order matters here since we don't want to swap context contents for one that effects the other
|
|
StoreResultGPR(Op, Op->Dest, Src);
|
|
StoreResultGPR(Op, Op->Src[0], Dest);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::CDQOp(OpcodeArgs) {
|
|
const auto DstSize = OpSizeFromDst(Op);
|
|
const auto SrcSize = DstSize / 2;
|
|
Ref Src = LoadGPRRegister(X86State::REG_RAX, SrcSize, 0, true);
|
|
|
|
Src = _Sbfe(DstSize <= OpSize::i32Bit ? OpSize::i32Bit : OpSize::i64Bit, IR::OpSizeAsBits(SrcSize), 0, Src);
|
|
|
|
StoreResultGPR_WithOpSize(Op, Op->Dest, Src, DstSize);
|
|
}
|
|
|
|
void OpDispatchBuilder::SAHFOp(OpcodeArgs) {
|
|
// Extract AH
|
|
Ref Src = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit, 8);
|
|
|
|
// Clear bits that aren't supposed to be set
|
|
Src = _Andn(OpSize::i64Bit, Src, Constant(0b101000));
|
|
|
|
// Set the bit that is always set here
|
|
Src = _Or(OpSize::i64Bit, Src, _InlineConstant(0b10));
|
|
|
|
// Store the lower 8 bits in to RFLAGS
|
|
SetPackedRFLAG(true, Src);
|
|
}
|
|
void OpDispatchBuilder::LAHFOp(OpcodeArgs) {
|
|
// Load the lower 8 bits of the Rflags register
|
|
auto RFLAG = GetPackedRFLAG(0xFF);
|
|
|
|
// Store the lower 8 bits of the rflags register in to AH
|
|
StoreGPRRegister(X86State::REG_RAX, RFLAG, OpSize::i8Bit, 8);
|
|
}
|
|
|
|
void OpDispatchBuilder::FLAGControlOp(OpcodeArgs) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
switch (Op->OP) {
|
|
case 0xF5: // CMC
|
|
CarryInvert();
|
|
break;
|
|
case 0xF8: // CLC
|
|
SetCFInverted(Constant(1));
|
|
break;
|
|
case 0xF9: // STC
|
|
SetCFInverted(Constant(0));
|
|
break;
|
|
case 0xFC: // CLD
|
|
// Transformed
|
|
StoreDF(Constant(1));
|
|
break;
|
|
case 0xFD: // STD
|
|
StoreDF(Constant(-1));
|
|
break;
|
|
}
|
|
}
|
|
|
|
|
|
void OpDispatchBuilder::MOVSegOp(OpcodeArgs, bool ToSeg) {
|
|
// In x86-64 mode the accesses to the segment registers end up being constant zero moves
|
|
// Aside from FS/GS
|
|
// In x86-64 mode the accesses to segment registers can actually still touch the segments
|
|
// These write to the selector portion of the register
|
|
//
|
|
// FS and GS are specially handled here though
|
|
// AMD documentation is /wrong/ in this regard
|
|
// AMD documentation claims that the MOV to SReg and POP SReg registers will load a 32bit
|
|
// value in to the HIDDEN portions of the FS and GS registers /OR/ ignored if a null selector is
|
|
// selected for the registers
|
|
// This statement is actually untrue, the instructions will /actually/ load 16bits in to the selector portion of the register!
|
|
// Tested on a Zen+ CPU, the selector is the portion that is modified!
|
|
// We don't currently support FS/GS selector modifying, so this needs to be asserted out
|
|
// The loads here also load the selector, NOT the base
|
|
|
|
if (ToSeg) {
|
|
Ref Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], OpSize::i16Bit, Op->Flags);
|
|
|
|
switch (Op->Dest.Data.GPR.GPR) {
|
|
case FEXCore::X86State::REG_RAX: // ES
|
|
case FEXCore::X86State::REG_R8: // ES
|
|
_StoreContextGPR(OpSize::i16Bit, Src, offsetof(FEXCore::Core::CPUState, es_idx));
|
|
UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX);
|
|
break;
|
|
case FEXCore::X86State::REG_RBX: // DS
|
|
case FEXCore::X86State::REG_R11: // DS
|
|
_StoreContextGPR(OpSize::i16Bit, Src, offsetof(FEXCore::Core::CPUState, ds_idx));
|
|
UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX);
|
|
break;
|
|
case FEXCore::X86State::REG_RCX: // CS
|
|
case FEXCore::X86State::REG_R9: // CS
|
|
// CPL3 can't write to this
|
|
Break(FEXCore::IR::BreakDefinition {
|
|
.ErrorRegister = 0,
|
|
.Signal = SIGILL,
|
|
.TrapNumber = 0,
|
|
.si_code = 0,
|
|
});
|
|
break;
|
|
case FEXCore::X86State::REG_RDX: // SS
|
|
case FEXCore::X86State::REG_R10: // SS
|
|
_StoreContextGPR(OpSize::i16Bit, Src, offsetof(FEXCore::Core::CPUState, ss_idx));
|
|
UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX);
|
|
break;
|
|
case FEXCore::X86State::REG_RBP: // GS
|
|
case FEXCore::X86State::REG_R13: // GS
|
|
if (!Is64BitMode) {
|
|
_StoreContextGPR(OpSize::i16Bit, Src, offsetof(FEXCore::Core::CPUState, gs_idx));
|
|
UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX);
|
|
} else {
|
|
LogMan::Msg::EFmt("We don't support modifying GS selector in 64bit mode!");
|
|
DecodeFailure = true;
|
|
}
|
|
break;
|
|
case FEXCore::X86State::REG_RSP: // FS
|
|
case FEXCore::X86State::REG_R12: // FS
|
|
if (!Is64BitMode) {
|
|
_StoreContextGPR(OpSize::i16Bit, Src, offsetof(FEXCore::Core::CPUState, fs_idx));
|
|
UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX);
|
|
} else {
|
|
LogMan::Msg::EFmt("We don't support modifying FS selector in 64bit mode!");
|
|
DecodeFailure = true;
|
|
}
|
|
break;
|
|
default: UnimplementedOp(Op); return;
|
|
}
|
|
} else {
|
|
Ref Segment {};
|
|
|
|
switch (Op->Src[0].Data.GPR.GPR) {
|
|
case FEXCore::X86State::REG_RAX: // ES
|
|
case FEXCore::X86State::REG_R8: // ES
|
|
Segment = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, es_idx));
|
|
break;
|
|
case FEXCore::X86State::REG_RBX: // DS
|
|
case FEXCore::X86State::REG_R11: // DS
|
|
Segment = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, ds_idx));
|
|
break;
|
|
case FEXCore::X86State::REG_RCX: // CS
|
|
case FEXCore::X86State::REG_R9: // CS
|
|
Segment = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, cs_idx));
|
|
break;
|
|
case FEXCore::X86State::REG_RDX: // SS
|
|
case FEXCore::X86State::REG_R10: // SS
|
|
Segment = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, ss_idx));
|
|
break;
|
|
case FEXCore::X86State::REG_RBP: // GS
|
|
case FEXCore::X86State::REG_R13: // GS
|
|
if (Is64BitMode) {
|
|
Segment = Constant(0);
|
|
} else {
|
|
Segment = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, gs_idx));
|
|
}
|
|
break;
|
|
case FEXCore::X86State::REG_RSP: // FS
|
|
case FEXCore::X86State::REG_R12: // FS
|
|
if (Is64BitMode) {
|
|
Segment = Constant(0);
|
|
} else {
|
|
Segment = _LoadContextGPR(OpSize::i16Bit, offsetof(FEXCore::Core::CPUState, fs_idx));
|
|
}
|
|
break;
|
|
default: UnimplementedOp(Op); return;
|
|
}
|
|
if (DestIsMem(Op)) {
|
|
// If the destination is memory then we always store 16-bits only
|
|
StoreResultGPR_WithOpSize(Op, Op->Dest, Segment, OpSize::i16Bit);
|
|
} else {
|
|
// If the destination is a GPR then we follow register storing rules
|
|
StoreResultGPR(Op, Segment);
|
|
}
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::MOVOffsetOp(OpcodeArgs) {
|
|
switch (Op->OP) {
|
|
case 0xA0:
|
|
case 0xA1: {
|
|
// Source is memory(literal)
|
|
// Dest is GPR
|
|
auto Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.ForceLoad = true});
|
|
StoreResultGPR(Op, Op->Dest, Src);
|
|
break;
|
|
}
|
|
case 0xA2:
|
|
case 0xA3: {
|
|
// Source is GPR
|
|
// Dest is memory(literal)
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// This one is a bit special since the destination is a literal
|
|
// So the destination gets stored in Src[1]
|
|
StoreResultGPR(Op, Op->Src[1], Src);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::CPUIDOp(OpcodeArgs) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
|
|
Ref Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], GPRSize, Op->Flags);
|
|
Ref Leaf = LoadGPRRegister(X86State::REG_RCX);
|
|
|
|
Ref RAX = _AllocateGPR(false);
|
|
Ref RBX = _AllocateGPR(false);
|
|
Ref RCX = _AllocateGPR(false);
|
|
Ref RDX = _AllocateGPR(false);
|
|
|
|
_CPUID(Src, Leaf, RAX, RBX, RCX, RDX);
|
|
|
|
StoreGPRRegister(X86State::REG_RAX, RAX);
|
|
StoreGPRRegister(X86State::REG_RBX, RBX);
|
|
StoreGPRRegister(X86State::REG_RCX, RCX);
|
|
StoreGPRRegister(X86State::REG_RDX, RDX);
|
|
}
|
|
|
|
uint32_t OpDispatchBuilder::GetConstantShift(X86Tables::DecodedOp Op, bool Is1Bit) {
|
|
if (Is1Bit) {
|
|
return 1;
|
|
} else {
|
|
// x86 masks the shift by 0x3F or 0x1F depending on size of op
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
uint64_t Mask = Size == OpSize::i64Bit ? 0x3F : 0x1F;
|
|
|
|
return Op->Src[1].Literal() & Mask;
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::XGetBVOp(OpcodeArgs) {
|
|
Ref Function = LoadGPRRegister(X86State::REG_RCX);
|
|
|
|
auto RAX = _AllocateGPR(false);
|
|
auto RDX = _AllocateGPR(false);
|
|
_XGetBV(Function, RAX, RDX);
|
|
|
|
StoreGPRRegister(X86State::REG_RAX, RAX);
|
|
StoreGPRRegister(X86State::REG_RDX, RDX);
|
|
}
|
|
|
|
void OpDispatchBuilder::SHLOp(OpcodeArgs) {
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
auto Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
Ref Result = _Lshl(Size == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit, Dest, Src);
|
|
HandleShift(Op, Result, Dest, ShiftType::LSL, Src);
|
|
}
|
|
|
|
void OpDispatchBuilder::SHLImmediateOp(OpcodeArgs, bool SHL1Bit) {
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
uint64_t Shift = GetConstantShift(Op, SHL1Bit);
|
|
const auto Size = GetSrcBitSize(Op);
|
|
|
|
Ref Result = _Lshl(Size == 64 ? OpSize::i64Bit : OpSize::i32Bit, Dest, Constant(Shift));
|
|
|
|
CalculateFlags_ShiftLeftImmediate(OpSizeFromSrc(Op), Result, Dest, Shift);
|
|
CalculateDeferredFlags();
|
|
StoreResultGPR(Op, Result);
|
|
}
|
|
|
|
void OpDispatchBuilder::SHROp(OpcodeArgs) {
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
auto Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= OpSize::i32Bit});
|
|
auto Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
auto ALUOp = _Lshr(std::max(OpSize::i32Bit, Size), Dest, Src);
|
|
HandleShift(Op, ALUOp, Dest, ShiftType::LSR, Src);
|
|
}
|
|
|
|
void OpDispatchBuilder::SHRImmediateOp(OpcodeArgs, bool SHR1Bit) {
|
|
const auto Size = GetSrcBitSize(Op);
|
|
auto Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= 32});
|
|
|
|
uint64_t Shift = GetConstantShift(Op, SHR1Bit);
|
|
auto ALUOp = _Lshr(Size == 64 ? OpSize::i64Bit : OpSize::i32Bit, Dest, Constant(Shift));
|
|
|
|
CalculateFlags_ShiftRightImmediate(OpSizeFromSrc(Op), ALUOp, Dest, Shift);
|
|
CalculateDeferredFlags();
|
|
StoreResultGPR(Op, ALUOp);
|
|
}
|
|
|
|
void OpDispatchBuilder::SHLDOp(OpcodeArgs) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
const auto Size = GetSrcBitSize(Op);
|
|
|
|
// Allow garbage on the Src if it will be ignored by the Lshr below
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = Size >= 32});
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);
|
|
|
|
// Allow garbage on the shift, we're masking it anyway.
|
|
Ref Shift = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// x86 masks the shift by 0x3F or 0x1F depending on size of op.
|
|
if (Size == 64) {
|
|
Shift = _And(OpSize::i64Bit, Shift, _InlineConstant(0x3F));
|
|
} else {
|
|
Shift = _And(OpSize::i64Bit, Shift, _InlineConstant(0x1F));
|
|
}
|
|
|
|
// a64 masks the bottom bits, so if we're using a native 32/64-bit shift, we
|
|
// can negate to do the subtract (it's congruent), which saves a constant.
|
|
auto ShiftRight = Size >= 32 ? _Neg(OpSize::i64Bit, Shift) : Sub(OpSize::i64Bit, Constant(Size), Shift);
|
|
|
|
auto Tmp1 = _Lshl(OpSize::i64Bit, Dest, Shift);
|
|
auto Tmp2 = _Lshr(Size == 64 ? OpSize::i64Bit : OpSize::i32Bit, Src, ShiftRight);
|
|
|
|
Ref Res = _Or(OpSize::i64Bit, Tmp1, Tmp2);
|
|
|
|
// If shift count was zero then output doesn't change
|
|
// Needs to be checked for the 32bit operand case
|
|
// where shift = 0 and the source register still gets Zext
|
|
//
|
|
// TODO: With a backwards pass ahead-of-time, we could stick this in the
|
|
// if(shift) used for flags.
|
|
//
|
|
// TODO: This whole function wants to be wrapped in the if. Maybe b/w pass is
|
|
// a good idea after all.
|
|
Res = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, Shift, Constant(0), Dest, Res);
|
|
|
|
HandleShift(Op, Res, Dest, ShiftType::LSL, Shift);
|
|
}
|
|
|
|
void OpDispatchBuilder::SHLDImmediateOp(OpcodeArgs) {
|
|
uint64_t Shift = GetConstantShift(Op, false);
|
|
const auto Size = GetSrcBitSize(Op);
|
|
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = Size >= 32});
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= 32});
|
|
|
|
if (Shift != 0) {
|
|
Ref Res {};
|
|
if (Size < 32) {
|
|
Ref ShiftLeft = Constant(Shift);
|
|
auto ShiftRight = Size - Shift;
|
|
|
|
auto Tmp1 = _Lshl(OpSize::i64Bit, Dest, ShiftLeft);
|
|
Ref Tmp2 = ShiftRight ? _Lshr(OpSize::i32Bit, Src, Constant(ShiftRight)) : Src;
|
|
|
|
Res = _Or(OpSize::i64Bit, Tmp1, Tmp2);
|
|
} else {
|
|
// 32-bit and 64-bit SHLD behaves like an EXTR where the lower bits are filled from the source.
|
|
Res = _Extr(OpSizeFromSrc(Op), Dest, Src, Size - Shift);
|
|
}
|
|
|
|
CalculateFlags_ShiftLeftImmediate(OpSizeFromSrc(Op), Res, Dest, Shift, true);
|
|
CalculateDeferredFlags();
|
|
StoreResultGPR(Op, Res);
|
|
} else if (Shift == 0 && Size == 32) {
|
|
// Ensure Zext still occurs
|
|
StoreResultGPR(Op, Dest);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::SHRDOp(OpcodeArgs) {
|
|
// Calculate flags early.
|
|
// This instruction conditionally generates flags so we need to insure sane state going in.
|
|
CalculateDeferredFlags();
|
|
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);
|
|
|
|
Ref Shift = LoadGPRRegister(X86State::REG_RCX);
|
|
|
|
const auto Size = GetDstBitSize(Op);
|
|
|
|
// x86 masks the shift by 0x3F or 0x1F depending on size of op
|
|
if (Size == 64) {
|
|
Shift = _And(OpSize::i64Bit, Shift, _InlineConstant(0x3F));
|
|
} else {
|
|
Shift = _And(OpSize::i64Bit, Shift, _InlineConstant(0x1F));
|
|
}
|
|
|
|
auto ShiftLeft = Sub(OpSize::i64Bit, Constant(Size), Shift);
|
|
|
|
auto Tmp1 = _Lshr(Size == 64 ? OpSize::i64Bit : OpSize::i32Bit, Dest, Shift);
|
|
auto Tmp2 = _Lshl(OpSize::i64Bit, Src, ShiftLeft);
|
|
|
|
Ref Res = _Or(OpSize::i64Bit, Tmp1, Tmp2);
|
|
|
|
// If shift count was zero then output doesn't change
|
|
// Needs to be checked for the 32bit operand case
|
|
// where shift = 0 and the source register still gets Zext
|
|
Res = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, Shift, Constant(0), Dest, Res);
|
|
|
|
HandleShift(Op, Res, Dest, ShiftType::LSR, Shift);
|
|
}
|
|
|
|
void OpDispatchBuilder::SHRDImmediateOp(OpcodeArgs) {
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);
|
|
|
|
uint64_t Shift = GetConstantShift(Op, false);
|
|
const auto Size = GetSrcBitSize(Op);
|
|
|
|
if (Shift != 0) {
|
|
Ref Res {};
|
|
if (Size < 32) {
|
|
Ref ShiftRight = Constant(Shift);
|
|
auto ShiftLeft = Constant(Size - Shift);
|
|
|
|
auto Tmp1 = _Lshr(OpSize::i32Bit, Dest, ShiftRight);
|
|
auto Tmp2 = _Lshl(OpSize::i64Bit, Src, ShiftLeft);
|
|
|
|
Res = _Or(OpSize::i64Bit, Tmp1, Tmp2);
|
|
} else {
|
|
// 32-bit and 64-bit SHRD behaves like an EXTR where the upper bits are filled from the source.
|
|
Res = _Extr(OpSizeFromSrc(Op), Src, Dest, Shift);
|
|
}
|
|
|
|
StoreResultGPR(Op, Res);
|
|
CalculateFlags_ShiftRightDoubleImmediate(OpSizeFromSrc(Op), Res, Dest, Shift);
|
|
} else if (Shift == 0 && Size == 32) {
|
|
// Ensure Zext still occurs
|
|
StoreResultGPR(Op, Dest);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::ASHROp(OpcodeArgs, bool Immediate, bool SHR1Bit) {
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
const auto OpSize = std::max(OpSize::i32Bit, OpSizeFromDst(Op));
|
|
|
|
// If Size < 4, then we Sbfe the Dest so we can have garbage.
|
|
// Otherwise, if Size = Opsize, then both are 4 or 8 and match the a64
|
|
// semantics directly, so again we can have garbage. The only case where we
|
|
// need zero-extension here is when the sizes mismatch.
|
|
auto Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = (OpSize == Size) || (Size < OpSize::i32Bit)});
|
|
|
|
if (Size < OpSize::i32Bit) {
|
|
Dest = _Sbfe(OpSize::i64Bit, IR::OpSizeAsBits(Size), 0, Dest);
|
|
}
|
|
|
|
if (Immediate) {
|
|
uint64_t Shift = GetConstantShift(Op, SHR1Bit);
|
|
Ref Result = _Ashr(OpSize, Dest, Constant(Shift));
|
|
|
|
CalculateFlags_SignShiftRightImmediate(OpSizeFromSrc(Op), Result, Dest, Shift);
|
|
CalculateDeferredFlags();
|
|
StoreResultGPR(Op, Result);
|
|
} else {
|
|
auto Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Result = _Ashr(OpSize, Dest, Src);
|
|
|
|
HandleShift(Op, Result, Dest, ShiftType::ASR, Src);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::RotateOp(OpcodeArgs, bool Left, bool IsImmediate, bool Is1Bit) {
|
|
CalculateDeferredFlags();
|
|
|
|
const uint32_t Size = GetSrcBitSize(Op);
|
|
const auto OpSize = Size == 64 ? OpSize::i64Bit : OpSize::i32Bit;
|
|
uint64_t UnmaskedConst {};
|
|
|
|
// x86 masks the shift by 0x3F or 0x1F depending on size of op. But it's
|
|
// equivalent to mask to the actual size of the op, that way we can bound
|
|
// things tighter for 8-bit later in the function.
|
|
uint64_t Mask = Size == 8 ? 7 : (Size == 64 ? 0x3F : 0x1F);
|
|
|
|
ArithRef UnmaskedSrc;
|
|
if (Is1Bit || IsImmediate) {
|
|
UnmaskedConst = GetConstantShift(Op, Is1Bit);
|
|
UnmaskedSrc = ARef(UnmaskedConst);
|
|
} else {
|
|
UnmaskedSrc = ARef(LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true}));
|
|
}
|
|
auto Src = UnmaskedSrc.And(Mask);
|
|
|
|
// We fill the upper bits so we allow garbage on load.
|
|
auto Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
if (Size < 32) {
|
|
// ARM doesn't support 8/16bit rotates. Emulate with an insert
|
|
// StoreResult truncates back to a 8/16 bit value
|
|
Dest = _Bfi(OpSize, Size, Left ? (32 - Size) : Size, Dest, Dest);
|
|
}
|
|
|
|
// To rotate 64-bits left, right-rotate by (64 - Shift) = -Shift mod 64.
|
|
auto Res = _Ror(OpSize, Dest, (Left ? Src.Neg() : Src).Ref());
|
|
StoreResultGPR(Op, Res);
|
|
|
|
if (Is1Bit || IsImmediate) {
|
|
if (UnmaskedSrc.C) {
|
|
// Extract the last bit shifted in to CF
|
|
SetCFDirect(Res, Left ? 0 : Size - 1, true);
|
|
|
|
// For ROR, OF is the XOR of the new CF bit and the most significant bit of the result.
|
|
// For ROL, OF is the LSB and MSB XOR'd together.
|
|
// OF is architecturally only defined for 1-bit rotate.
|
|
if (UnmaskedSrc.C == 1) {
|
|
auto NewOF = _XorShift(OpSize, Res, Res, ShiftType::LSR, Left ? Size - 1 : 1);
|
|
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(NewOF, Left ? 0 : Size - 2, true);
|
|
}
|
|
}
|
|
} else {
|
|
HandleNZCVWrite();
|
|
RectifyCarryInvert(true);
|
|
|
|
// We deferred the masking for 8-bit to the flag section, do it here.
|
|
if (Size == 8) {
|
|
Src = UnmaskedSrc.And(0x1F);
|
|
}
|
|
|
|
_RotateFlags(OpSizeFromSrc(Op), Res, Src.Ref(), Left);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::ANDNBMIOp(OpcodeArgs) {
|
|
auto Src1 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Src2 = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
auto Dest = _Andn(OpSizeFromSrc(Op), Src2, Src1);
|
|
|
|
StoreResultGPR(Op, Dest);
|
|
CalculateFlags_Logical(OpSizeFromSrc(Op), Dest);
|
|
}
|
|
|
|
void OpDispatchBuilder::BEXTRBMIOp(OpcodeArgs) {
|
|
// Essentially (Src1 >> Start) & ((1 << Length) - 1)
|
|
// along with some edge-case handling and flag setting.
|
|
|
|
LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed");
|
|
auto Src1 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Src2 = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
const auto SrcSize = IR::OpSizeAsBits(Size);
|
|
const auto MaxSrcBit = SrcSize - 1;
|
|
auto MaxSrcBitOp = Constant(MaxSrcBit);
|
|
|
|
// Shift the operand down to the starting bit
|
|
auto Start = _Bfe(OpSizeFromSrc(Op), 8, 0, Src2);
|
|
auto Shifted = _Lshr(Size, Src1, Start);
|
|
|
|
// Shifts larger than operand size need to be set to zero.
|
|
auto SanitizedShifted = _Select(Size, Size, CondClass::ULE, Start, MaxSrcBitOp, Shifted, Constant(0));
|
|
|
|
// Now handle the length specifier.
|
|
auto Length = _Bfe(Size, 8, 8, Src2);
|
|
|
|
// Now build up the mask
|
|
// (1 << Length) - 1 = ~(~0 << Length)
|
|
auto AllOnes = Constant(~0ull);
|
|
auto InvertedMask = _Lshl(Size, AllOnes, Length);
|
|
|
|
// Now put it all together and make the result.
|
|
auto Masked = _Andn(Size, SanitizedShifted, InvertedMask);
|
|
|
|
// Sanitize the length. If it is above the max, we don't do the masking.
|
|
auto Dest = _Select(Size, Size, CondClass::ULE, Length, MaxSrcBitOp, Masked, SanitizedShifted);
|
|
|
|
// Finally store the result.
|
|
StoreResultGPR(Op, Dest);
|
|
|
|
// ZF is set properly. CF and OF are defined as being set to zero. SF, PF, and
|
|
// AF are undefined.
|
|
SetNZ_ZeroCV(GetOpSize(Dest), Dest);
|
|
InvalidatePF_AF();
|
|
}
|
|
|
|
void OpDispatchBuilder::BLSIBMIOp(OpcodeArgs) {
|
|
// Equivalent to performing: SRC & -SRC
|
|
LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed");
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
|
|
auto Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto NegatedSrc = _Neg(Size, Src);
|
|
auto Result = _And(Size, Src, NegatedSrc);
|
|
|
|
StoreResultGPR(Op, Result);
|
|
|
|
// CF is cleared if Src is zero, otherwise it's set. However, Src is zero iff
|
|
// Result is zero, so we can test the result instead. So, CF is just the
|
|
// inverted ZF.
|
|
//
|
|
// ZF/SF/OF set as usual.
|
|
SetNZ_ZeroCV(Size, Result);
|
|
InvalidatePF_AF();
|
|
SetCFInverted(GetRFLAG(X86State::RFLAG_ZF_RAW_LOC));
|
|
}
|
|
|
|
void OpDispatchBuilder::BLSMSKBMIOp(OpcodeArgs) {
|
|
// Equivalent to: (Src - 1) ^ Src
|
|
LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed");
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
|
|
auto Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Result = _Xor(Size, Sub(Size, Src, 1), Src);
|
|
|
|
StoreResultGPR(Op, Result);
|
|
InvalidatePF_AF();
|
|
|
|
// CF set according to the Src
|
|
auto CFInv = To01(Size, Src);
|
|
|
|
// The output of BLSMSK is always nonzero, so TST will clear Z (along with C
|
|
// and O) while setting S.
|
|
SetNZ_ZeroCV(Size, Result);
|
|
SetCFInverted(CFInv);
|
|
}
|
|
|
|
void OpDispatchBuilder::BLSRBMIOp(OpcodeArgs) {
|
|
// Equivalent to: (Src - 1) & Src
|
|
LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed");
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
|
|
auto Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Result = _And(Size, Sub(Size, Src, 1), Src);
|
|
|
|
StoreResultGPR(Op, Result);
|
|
|
|
auto CFInv = To01(Size, Src);
|
|
|
|
SetNZ_ZeroCV(Size, Result);
|
|
SetCFInverted(CFInv);
|
|
InvalidatePF_AF();
|
|
}
|
|
|
|
// Handles SARX, SHLX, and SHRX
|
|
void OpDispatchBuilder::BMI2Shift(OpcodeArgs) {
|
|
// In the event the source is a memory operand, use the
|
|
// exact width instead of the GPR size.
|
|
const auto GPRSize = GetGPROpSize();
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
const auto SrcSize = Op->Src[0].IsGPR() ? GPRSize : Size;
|
|
|
|
auto Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags);
|
|
auto Shift = LoadSourceGPR_WithOpSize(Op, Op->Src[1], GPRSize, Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
Ref Result;
|
|
if (Op->OP == 0x6F7) {
|
|
// SARX
|
|
Result = _Ashr(Size, Src, Shift);
|
|
} else if (Op->OP == 0x5F7) {
|
|
// SHLX
|
|
Result = _Lshl(Size, Src, Shift);
|
|
} else {
|
|
// SHRX
|
|
Result = _Lshr(Size, Src, Shift);
|
|
}
|
|
|
|
StoreResultGPR(Op, Result);
|
|
}
|
|
|
|
void OpDispatchBuilder::BZHI(OpcodeArgs) {
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
const auto OperandSize = IR::OpSizeAsBits(Size);
|
|
|
|
// In 32-bit mode we only look at bottom 32-bit, no 8 or 16-bit BZHI so no
|
|
// need to zero-extend sources
|
|
auto Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
auto Index = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// Clear the high bits specified by the index. A64 only considers bottom bits
|
|
// of the shift, so we don't need to mask bottom 8-bits ourselves.
|
|
// Out-of-bounds results ignored after.
|
|
auto Mask = _Lshl(Size, Constant(-1), Index);
|
|
auto MaskResult = _Andn(Size, Src, Mask);
|
|
|
|
// If the index is above OperandSize, we don't clear anything. BZHI only
|
|
// considers the bottom 8-bits, so we really want to know if the bottom 8-bits
|
|
// have their top bits set. Test exactly that.
|
|
//
|
|
// Because we're clobbering flags internally we ignore all carry invert
|
|
// shenanigans and use the raw versions here.
|
|
_TestNZ(OpSize::i64Bit, Index, Constant(0xFF & ~(OperandSize - 1)));
|
|
auto Result = _NZCVSelect(Size, CondClass::NEQ, Src, MaskResult);
|
|
StoreResultGPR(Op, Result);
|
|
|
|
auto CFInv = _NZCVSelect01(CondClass::EQ);
|
|
|
|
InvalidatePF_AF();
|
|
SetNZ_ZeroCV(Size, Result);
|
|
SetCFInverted(CFInv);
|
|
}
|
|
|
|
void OpDispatchBuilder::RORX(OpcodeArgs) {
|
|
const auto SrcSize = OpSizeFromSrc(Op);
|
|
const auto SrcSizeBits = IR::OpSizeAsBits(SrcSize);
|
|
const auto Amount = Op->Src[1].Literal() & (SrcSizeBits - 1);
|
|
const auto GPRSize = GetGPROpSize();
|
|
|
|
const auto DoRotation = Amount != 0 && Amount < SrcSizeBits;
|
|
const auto IsSameGPR = Op->Src[0].IsGPR() && Op->Dest.IsGPR() && Op->Src[0].Data.GPR.GPR == Op->Dest.Data.GPR.GPR;
|
|
const auto SrcSizeIsGPRSize = SrcSize == GPRSize;
|
|
|
|
// If we don't need to rotate and our source is the same as the destination
|
|
// then we don't need to do anything at all. We still need to be careful,
|
|
// since 32-bit operations on 64-bit mode still need to zero-extend the
|
|
// destination register. So also compare source size and GPR size.
|
|
//
|
|
// Very unlikely, but hey, we can do nothing faster.
|
|
if (!DoRotation && IsSameGPR && SrcSizeIsGPRSize) [[unlikely]] {
|
|
return;
|
|
}
|
|
|
|
auto Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Result = Src;
|
|
if (DoRotation) [[likely]] {
|
|
Result = _Ror(OpSizeFromSrc(Op), Src, _InlineConstant(Amount));
|
|
}
|
|
|
|
StoreResultGPR(Op, Result);
|
|
}
|
|
|
|
void OpDispatchBuilder::MULX(OpcodeArgs) {
|
|
// RDX is the implied source operand in the instruction
|
|
const auto OpSize = OpSizeFromSrc(Op);
|
|
|
|
// Src1 can be a memory operand, so ensure we constrain to the
|
|
// absolute width of the access in that scenario.
|
|
const auto GPRSize = GetGPROpSize();
|
|
const auto Src1Size = Op->Src[1].IsGPR() ? GPRSize : OpSize;
|
|
|
|
Ref Src1 = LoadSourceGPR_WithOpSize(Op, Op->Src[1], Src1Size, Op->Flags);
|
|
Ref Src2 = LoadGPRRegister(X86State::REG_RDX, GPRSize);
|
|
|
|
// As per the Intel Software Development Manual, if the destination and
|
|
// first operand correspond to the same register, then the result
|
|
// will be the high half of the multiplication result.
|
|
if (Op->Dest.Data.GPR.GPR == Op->Src[0].Data.GPR.GPR) {
|
|
Ref ResultHi = _UMulH(OpSize, Src1, Src2);
|
|
StoreResultGPR(Op, Op->Dest, ResultHi);
|
|
} else {
|
|
Ref ResultLo = _UMul(OpSize, Src1, Src2);
|
|
Ref ResultHi = _UMulH(OpSize, Src1, Src2);
|
|
|
|
StoreResultGPR(Op, Op->Src[0], ResultLo);
|
|
StoreResultGPR(Op, Op->Dest, ResultHi);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::PDEP(OpcodeArgs) {
|
|
LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed");
|
|
auto Input = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Mask = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Result = _PDep(OpSizeFromSrc(Op), Input, Mask);
|
|
|
|
StoreResultGPR(Op, Op->Dest, Result);
|
|
}
|
|
|
|
void OpDispatchBuilder::PEXT(OpcodeArgs) {
|
|
LOGMAN_THROW_A_FMT(Op->InstSize >= 4, "No masking needed");
|
|
auto Input = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Mask = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Result = _PExt(OpSizeFromSrc(Op), Input, Mask);
|
|
|
|
StoreResultGPR(Op, Op->Dest, Result);
|
|
}
|
|
|
|
void OpDispatchBuilder::ADXOp(OpcodeArgs) {
|
|
const auto OpSize = OpSizeFromSrc(Op);
|
|
|
|
// Only 32/64-bit anyway so allow garbage, we use 32-bit ops.
|
|
auto Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Before = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// Handles ADCX and ADOX
|
|
const bool IsADCX = Op->OP == 0x1F6;
|
|
auto Zero = Constant(0);
|
|
|
|
// Before we go trashing NZCV, save the current NZCV state.
|
|
Ref OldNZCV = GetNZCV();
|
|
|
|
// We want to use arm64 adc. For ADOX, copy the overflow flag into CF. For
|
|
// ADCX, we just rectify the carry.
|
|
if (IsADCX) {
|
|
RectifyCarryInvert(false);
|
|
} else {
|
|
// If overflow, 0 - 0 sets carry. Else, forces carry to 0.
|
|
_CondSubNZCV(OpSize::i32Bit, Zero, Zero, CondClass::FU, 0x0 /* nzcv */);
|
|
}
|
|
|
|
// Do the actual add.
|
|
HandleNZCV_RMW();
|
|
auto Result = _AdcWithFlags(OpSize, Src, Before);
|
|
StoreResultGPR(Op, Result);
|
|
|
|
// Now restore all flags except the one we're updating.
|
|
if (CTX->HostFeatures.SupportsFlagM) {
|
|
// For ADOX, we need to copy the new carry into the overflow flag. If carry is clear (ULT with uninverted
|
|
// carry), 0 - 0 clears overflow. Else, force overflow on.
|
|
if (!IsADCX) {
|
|
_CondSubNZCV(OpSize::i32Bit, Zero, Zero, CondClass::ULT, 0x1 /* nzcV */);
|
|
}
|
|
|
|
_RmifNZCV(OldNZCV, 28, IsADCX ? 0xd /* NzcV */ : 0xe /* NZCv */);
|
|
} else {
|
|
// For either operation, insert the new flag into the old NZCV.
|
|
bool SavedCFInvert = CFInverted;
|
|
CFInverted = false;
|
|
Ref OutputCF = GetRFLAG(X86State::RFLAG_CF_RAW_LOC, IsADCX);
|
|
CFInverted = IsADCX ? true : SavedCFInvert;
|
|
|
|
Ref NewNZCV = _Bfi(OpSize::i32Bit, 1, IsADCX ? 29 : 28, OldNZCV, OutputCF);
|
|
SetNZCV(NewNZCV);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::RCROp1Bit(OpcodeArgs) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
// We expliclty mask for <32-bit so allow garbage
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
const auto Size = GetSrcBitSize(Op);
|
|
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
|
|
Ref Res;
|
|
|
|
// Our new CF will be bit 0 of the source. Set upfront to avoid a move.
|
|
SetCFDirect(Dest, 0, true);
|
|
|
|
uint32_t Shift = 1;
|
|
|
|
if (Size == 32 || Size == 64) {
|
|
// Rotate and insert CF in the upper bit
|
|
Res = _Extr(OpSizeFromSrc(Op), CF, Dest, Shift);
|
|
} else {
|
|
// Res = Src >> Shift
|
|
Res = _Bfe(OpSize::i32Bit, Size - Shift, Shift, Dest);
|
|
|
|
// inject the CF
|
|
Res = _Orlshl(OpSize::i32Bit, Res, CF, Size - Shift);
|
|
}
|
|
|
|
StoreResultGPR(Op, Res);
|
|
|
|
// OF is the top two MSBs XOR'd together
|
|
// Only when Shift == 1, it is undefined otherwise
|
|
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(_XorShift(OpSize::i64Bit, Res, Res, ShiftType::LSR, 1), Size - 2, true);
|
|
}
|
|
|
|
void OpDispatchBuilder::RCROp8x1Bit(OpcodeArgs) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);
|
|
const auto SizeBit = GetSrcBitSize(Op);
|
|
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
|
|
|
|
// Our new CF will be bit (Shift - 1) of the source
|
|
SetCFDirect(Dest, 0, true);
|
|
|
|
// Rotate and insert CF in the upper bit
|
|
Ref Res = _Bfe(OpSize::i32Bit, 7, 1, Dest);
|
|
Res = _Bfi(OpSize::i32Bit, 1, 7, Res, CF);
|
|
|
|
StoreResultGPR(Op, Res);
|
|
|
|
// OF is the top two MSBs XOR'd together
|
|
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(_XorShift(OpSize::i32Bit, Res, Res, ShiftType::LSR, 1), SizeBit - 2, true);
|
|
}
|
|
|
|
void OpDispatchBuilder::RCROp(OpcodeArgs) {
|
|
const auto Size = GetSrcBitSize(Op);
|
|
|
|
if (Size == 8 || Size == 16) {
|
|
RCRSmallerOp(Op);
|
|
return;
|
|
}
|
|
|
|
const auto Mask = (Size == 64) ? 0x3F : 0x1F;
|
|
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
const auto OpSize = OpSizeFromSrc(Op);
|
|
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
uint64_t Const;
|
|
if (IsValueConstant(WrapNode(Src), &Const)) {
|
|
Const &= Mask;
|
|
if (!Const) {
|
|
ZeroShiftResult(Op);
|
|
return;
|
|
}
|
|
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// Res = Src >> Shift
|
|
Ref Res = _Lshr(OpSize, Dest, Src);
|
|
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
|
|
|
|
// Constant folded version of the above, with fused shifts.
|
|
if (Const > 1) {
|
|
Res = _Orlshl(OpSize, Res, Dest, Size + 1 - Const);
|
|
}
|
|
|
|
// Our new CF will be bit (Shift - 1) of the source.
|
|
SetCFDirect(Dest, Const - 1, true);
|
|
|
|
// Since shift != 0 we can inject the CF
|
|
Res = _Orlshl(OpSize, Res, CF, Size - Const);
|
|
|
|
// OF is the top two MSBs XOR'd together
|
|
// Only when Shift == 1, it is undefined otherwise
|
|
if (Const == 1) {
|
|
auto Xor = _XorShift(OpSize, Res, Res, ShiftType::LSR, 1);
|
|
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(Xor, Size - 2, true);
|
|
}
|
|
|
|
StoreResultGPR(Op, Res);
|
|
return;
|
|
}
|
|
|
|
Ref SrcMasked = _And(OpSize, Src, _InlineConstant(Mask));
|
|
Calculate_ShiftVariable(
|
|
Op, SrcMasked,
|
|
[this, Op, Size, OpSize]() {
|
|
// Rematerialize loads to avoid crossblock liveness
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// Res = Src >> Shift
|
|
Ref Res = _Lshr(OpSize, Dest, Src);
|
|
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
|
|
|
|
// Res |= (Dest << (Size - Shift + 1));
|
|
// Expressed as Res | ((Src << (Size - Shift)) << 1) to get correct
|
|
// behaviour for Shift without clobbering NZCV. Then observe that modulo
|
|
// Size, Size - Shift = -Shift so we can use a simple Neg.
|
|
//
|
|
// The masking of Lshl means we don't need mask the source, since:
|
|
//
|
|
// -(x & Mask) & Mask = (-x) & Mask
|
|
Ref NegSrc = _Neg(OpSize, Src);
|
|
Res = _Orlshl(OpSize, Res, _Lshl(OpSize, Dest, NegSrc), 1);
|
|
|
|
// Our new CF will be bit (Shift - 1) of the source. this is hoisted up to
|
|
// avoid the need to copy the source. Again, the Lshr absorbs the masking.
|
|
auto NewCF = _Lshr(OpSize, Dest, Sub(OpSize, Src, 1));
|
|
SetCFDirect(NewCF, 0, true);
|
|
|
|
// Since shift != 0 we can inject the CF
|
|
Res = _Or(OpSize, Res, _Lshl(OpSize, CF, NegSrc));
|
|
|
|
// OF is the top two MSBs XOR'd together
|
|
// Only when Shift == 1, it is undefined otherwise
|
|
auto Xor = _XorShift(OpSize, Res, Res, ShiftType::LSR, 1);
|
|
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(Xor, Size - 2, true);
|
|
|
|
StoreResultGPR(Op, Res);
|
|
},
|
|
OpSizeFromSrc(Op) == OpSize::i32Bit ? std::make_optional(&OpDispatchBuilder::ZeroShiftResult) : std::nullopt);
|
|
}
|
|
|
|
void OpDispatchBuilder::RCRSmallerOp(OpcodeArgs) {
|
|
CalculateDeferredFlags();
|
|
|
|
const auto Size = GetSrcBitSize(Op);
|
|
|
|
// x86 masks the shift by 0x3F or 0x1F depending on size of op
|
|
auto Src = ARef(LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true}));
|
|
Src = Src.And(0x1F);
|
|
|
|
// CF only changes if we actually shifted. OF undefined if we didn't shift.
|
|
// The result is unchanged if we didn't shift. So branch over the whole thing.
|
|
Calculate_ShiftVariable(Op, Src.Ref(), [this, Op, Size]() {
|
|
// Rematerialized to avoid crossblock liveness
|
|
auto Src = ARef(LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true}));
|
|
Src = Src.And(0x1F);
|
|
|
|
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
|
|
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);
|
|
Ref Tmp {};
|
|
|
|
// Insert the incoming value across the temporary 64bit source
|
|
// Make sure to insert at <BitSize> + 1 offsets
|
|
// We need to cover 32bits plus the amount that could rotate in
|
|
|
|
if (Size == 8) {
|
|
// 8-bit optimal cascade
|
|
// Cascade: 0
|
|
// Data: -> [7:0]
|
|
// CF: -> [8:8]
|
|
// Cascade: 1
|
|
// Data: -> [16:9]
|
|
// CF: -> [17:17]
|
|
// Cascade: 2
|
|
// Data: -> [25:18]
|
|
// CF: -> [26:26]
|
|
// Cascade: 3
|
|
// Data: -> [34:27]
|
|
// CF: -> [35:35]
|
|
// Cascade: 4
|
|
// Data: -> [43:36]
|
|
// CF: -> [44:44]
|
|
|
|
// Insert CF, Destination already at [7:0]
|
|
Tmp = _Bfi(OpSize::i64Bit, 1, 8, Dest, CF);
|
|
|
|
// First Cascade, copies 9 bits from itself.
|
|
Tmp = _Bfi(OpSize::i64Bit, 9, 9, Tmp, Tmp);
|
|
|
|
// Second cascade, copies 18 bits from itself.
|
|
Tmp = _Bfi(OpSize::i64Bit, 18, 18, Tmp, Tmp);
|
|
|
|
// Final cascade, copies 9 bits again from itself.
|
|
Tmp = _Bfi(OpSize::i64Bit, 9, 36, Tmp, Tmp);
|
|
} else {
|
|
// 16-bit optimal cascade
|
|
// Cascade: 0
|
|
// Data: -> [15:0]
|
|
// CF: -> [16:16]
|
|
// Cascade: 1
|
|
// Data: -> [32:17]
|
|
// CF: -> [33:33]
|
|
// Cascade: 2
|
|
// Data: -> [49:34]
|
|
// CF: -> [50:50]
|
|
|
|
// Insert CF, Destination already at [15:0]
|
|
Tmp = _Bfi(OpSize::i64Bit, 1, 16, Dest, CF);
|
|
|
|
// First Cascade, copies 17 bits from itself.
|
|
Tmp = _Bfi(OpSize::i64Bit, 17, 17, Tmp, Tmp);
|
|
|
|
// Final Cascade, copies 17 bits from itself again.
|
|
Tmp = _Bfi(OpSize::i64Bit, 17, 34, Tmp, Tmp);
|
|
}
|
|
|
|
// Entire bitfield has been setup. Just extract the 8 or 16bits we need.
|
|
// 64-bit shift used because we want to rotate in our cascaded upper bits
|
|
// rather than zeroes.
|
|
Ref Res = _Lshr(OpSize::i64Bit, Tmp, Src.Ref());
|
|
|
|
StoreResultGPR(Op, Res);
|
|
|
|
// Our new CF will be bit (Shift - 1) of the source. 32-bit Lshr masks the
|
|
// same as x86, but if we constant fold we must mask ourselves.
|
|
if (Src.IsConstant) {
|
|
SetCFDirect(Tmp, (Src.C & 0x1f) - 1, true);
|
|
} else {
|
|
auto NewCF = _Lshr(OpSize::i32Bit, Tmp, Sub(OpSize::i32Bit, Src.Ref(), 1));
|
|
SetCFDirect(NewCF, 0, true);
|
|
}
|
|
|
|
// OF is the top two MSBs XOR'd together
|
|
// Only when Shift == 1, it is undefined otherwise
|
|
if (!Src.IsConstant || Src.C == 1) {
|
|
auto NewOF = _XorShift(OpSize::i32Bit, Res, Res, ShiftType::LSR, 1);
|
|
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(NewOF, Size - 2, true);
|
|
}
|
|
});
|
|
}
|
|
|
|
void OpDispatchBuilder::RCLOp1Bit(OpcodeArgs) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);
|
|
const auto Size = GetSrcBitSize(Op);
|
|
const auto OpSize = Size == 64 ? OpSize::i64Bit : OpSize::i32Bit;
|
|
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
|
|
|
|
// Rotate left and insert CF in to lowest bit
|
|
// TODO: Use `adc Res, xzr, Dest, lsl 1` to save an instruction
|
|
Ref Res = _Orlshl(OpSize, CF, Dest, 1);
|
|
|
|
// Our new CF will be the top bit of the source
|
|
SetCFDirect(Dest, Size - 1, true);
|
|
|
|
// OF is the top two MSBs XOR'd together
|
|
// Top two MSBs is CF and top bit of result
|
|
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(_Xor(OpSize, Res, Dest), Size - 1, true);
|
|
|
|
StoreResultGPR(Op, Res);
|
|
}
|
|
|
|
void OpDispatchBuilder::RCLOp(OpcodeArgs) {
|
|
const auto Size = GetSrcBitSize(Op);
|
|
|
|
if (Size == 8 || Size == 16) {
|
|
RCLSmallerOp(Op);
|
|
return;
|
|
}
|
|
|
|
const auto Mask = (Size == 64) ? 0x3F : 0x1F;
|
|
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
const auto OpSize = OpSizeFromSrc(Op);
|
|
|
|
uint64_t Const;
|
|
if (IsValueConstant(WrapNode(Src), &Const)) {
|
|
Const &= Mask;
|
|
if (!Const) {
|
|
ZeroShiftResult(Op);
|
|
return;
|
|
}
|
|
|
|
// Res = Src << Shift
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Res = _Lshl(OpSize, Dest, Src);
|
|
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
|
|
|
|
// Res |= (Src << (Size - Shift + 1));
|
|
if (Const > 1) {
|
|
Res = _Orlshr(OpSize, Res, Dest, Size + 1 - Const);
|
|
}
|
|
|
|
// Our new CF will be bit (Shift - 1) of the source
|
|
SetCFDirect(Dest, Size - Const, true);
|
|
|
|
// Since Shift != 0 we can inject the CF
|
|
Res = _Orlshl(OpSize, Res, CF, Const - 1);
|
|
|
|
// OF is the top two MSBs XOR'd together
|
|
// Only when Shift == 1, it is undefined otherwise
|
|
if (Const == 1) {
|
|
auto NewOF = _Xor(OpSize, Res, Dest);
|
|
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(NewOF, Size - 1, true);
|
|
}
|
|
|
|
StoreResultGPR(Op, Res);
|
|
return;
|
|
}
|
|
|
|
Ref SrcMasked = _And(OpSize, Src, _InlineConstant(Mask));
|
|
Calculate_ShiftVariable(
|
|
Op, SrcMasked,
|
|
[this, Op, Size, OpSize]() {
|
|
// Rematerialized to avoid crossblock liveness
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// Res = Src << Shift
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Res = _Lshl(OpSize, Dest, Src);
|
|
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
|
|
|
|
// Res |= (Dest >> (Size - Shift + 1)), expressed as
|
|
// Res | ((Dest >> (-Shift)) >> 1), since Size - Shift = -Shift mod
|
|
// Size. The shift aborbs the masking.
|
|
auto NegSrc = _Neg(OpSize, Src);
|
|
Res = _Orlshr(OpSize, Res, _Lshr(OpSize, Dest, NegSrc), 1);
|
|
|
|
// Our new CF will be bit (Shift - 1) of the source
|
|
auto NewCF = _Lshr(OpSize, Dest, NegSrc);
|
|
SetCFDirect(NewCF, 0, true);
|
|
|
|
// Since Shift != 0 we can inject the CF. Shift absorbs the masking.
|
|
Ref CFShl = Sub(OpSize, Src, 1);
|
|
auto TmpCF = _Lshl(OpSize, CF, CFShl);
|
|
Res = _Or(OpSize, Res, TmpCF);
|
|
|
|
// OF is the top two MSBs XOR'd together
|
|
// Only when Shift == 1, it is undefined otherwise
|
|
//
|
|
// Note that NewCF has garbage in the upper bits, but we ignore them here
|
|
// and mask as part of the set after.
|
|
auto NewOF = _XorShift(OpSize, Res, NewCF, ShiftType::LSL, Size - 1);
|
|
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(NewOF, Size - 1, true);
|
|
|
|
StoreResultGPR(Op, Res);
|
|
},
|
|
OpSizeFromSrc(Op) == OpSize::i32Bit ? std::make_optional(&OpDispatchBuilder::ZeroShiftResult) : std::nullopt);
|
|
}
|
|
|
|
void OpDispatchBuilder::RCLSmallerOp(OpcodeArgs) {
|
|
CalculateDeferredFlags();
|
|
|
|
const auto Size = GetSrcBitSize(Op);
|
|
|
|
// x86 masks the shift by 0x3F or 0x1F depending on size of op
|
|
auto Src = ARef(LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true}));
|
|
Src = Src.And(0x1F);
|
|
|
|
// CF only changes if we actually shifted. OF undefined if we didn't shift.
|
|
// The result is unchanged if we didn't shift. So branch over the whole thing.
|
|
Calculate_ShiftVariable(Op, Src.Ref(), [this, Op, Size]() {
|
|
// Rematerialized to avoid crossblock liveness
|
|
auto Src = ARef(LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true}));
|
|
Src = Src.And(0x1F);
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags);
|
|
|
|
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
|
|
|
|
Ref Tmp = Constant(0);
|
|
|
|
for (size_t i = 0; i < (32 + Size + 1); i += (Size + 1)) {
|
|
// Insert incoming value
|
|
Tmp = _Bfi(OpSize::i64Bit, Size, 63 - i - Size, Tmp, Dest);
|
|
|
|
// Insert CF
|
|
Tmp = _Bfi(OpSize::i64Bit, 1, 63 - i, Tmp, CF);
|
|
}
|
|
|
|
// Insert incoming value
|
|
Tmp = _Bfi(OpSize::i64Bit, Size, 0, Tmp, Dest);
|
|
|
|
// The data is now set up like this
|
|
// [Data][CF]:[Data][CF]:[Data][CF]:[Data][CF]
|
|
// Shift 1 more bit that expected to get our result
|
|
// Shifting to the right will now behave like a rotate to the left
|
|
// Which we emulate with a _Ror
|
|
Ref Res = _Ror(OpSize::i64Bit, Tmp, Src.Neg().Ref());
|
|
|
|
StoreResultGPR(Op, Res);
|
|
|
|
// Our new CF is now at the bit position that we are shifting
|
|
// Either 0 if CF hasn't changed (CF is living in bit 0)
|
|
// or higher
|
|
auto NewCF = _Ror(OpSize::i64Bit, Tmp, Src.Presub(63).Ref());
|
|
SetCFDirect(NewCF, 0, true);
|
|
|
|
// OF is the XOR of the NewCF and the MSB of the result
|
|
// Only defined for 1-bit rotates.
|
|
if (!Src.IsConstant || Src.C == 1) {
|
|
auto NewOF = _XorShift(OpSize::i64Bit, NewCF, Res, ShiftType::LSR, Size - 1);
|
|
SetRFLAG<FEXCore::X86State::RFLAG_OF_RAW_LOC>(NewOF, 0, true);
|
|
}
|
|
});
|
|
}
|
|
|
|
void OpDispatchBuilder::BTOp(OpcodeArgs, uint32_t SrcIndex, BTAction Action) {
|
|
Ref Value;
|
|
ArithRef Src;
|
|
bool IsNonconstant = Op->Src[SrcIndex].IsGPR();
|
|
|
|
const uint32_t Size = GetDstBitSize(Op);
|
|
const uint32_t Mask = Size - 1;
|
|
|
|
if (IsNonconstant) {
|
|
// Because we mask explicitly with And/Bfe/Sbfe after, we can allow garbage here.
|
|
Src = ARef(LoadSourceGPR(Op, Op->Src[SrcIndex], Op->Flags, {.AllowUpperGarbage = true}));
|
|
} else {
|
|
// Can only be an immediate
|
|
// Masked by operand size
|
|
Src = ARef(Op->Src[SrcIndex].Literal() & Mask);
|
|
}
|
|
|
|
if (Op->Dest.IsGPR()) {
|
|
// When the destination is a GPR, we don't care about garbage in the upper bits.
|
|
// Load the full register.
|
|
auto Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, GetGPROpSize(), Op->Flags);
|
|
Value = Dest;
|
|
|
|
// Get the bit selection from the src. We need to mask for 8/16-bit, but
|
|
// rely on the implicit masking of Lshr for native sizes.
|
|
unsigned LshrSize = std::max<uint8_t>(IR::OpSizeToSize(OpSize::i32Bit), Size / 8);
|
|
auto BitSelect = (Size == (LshrSize * 8)) ? Src : Src.And(Mask);
|
|
auto LshrOpSize = IR::SizeToOpSize(LshrSize);
|
|
|
|
// AMD: OF/SF/ZF/AF/PF undefined.
|
|
// Intel: OF/SF/AF/PF undefined. ZF must be preserved.
|
|
// We choose to preserve ZF/OF/SF since we just use an rmif
|
|
// to insert into CF directly. We could optimize perhaps.
|
|
//
|
|
// Set CF before the action to save a move, except for complements where we
|
|
// can reuse the invert.
|
|
if (Action != BTAction::BTComplement) {
|
|
if (IsNonconstant) {
|
|
Value = _Lshr(IR::SizeToOpSize(LshrSize), Value, BitSelect.Ref());
|
|
}
|
|
|
|
SetRFLAG(Value, X86State::RFLAG_CF_RAW_LOC, Src.IsConstant ? Src.C : 0, true);
|
|
CFInverted = false;
|
|
}
|
|
|
|
switch (Action) {
|
|
case BTAction::BTNone: {
|
|
/* Nothing to do */
|
|
break;
|
|
}
|
|
|
|
case BTAction::BTClear: {
|
|
Dest = _Andn(LshrOpSize, Dest, BitSelect.MaskBit(LshrOpSize).Ref());
|
|
StoreResultGPR(Op, Dest);
|
|
break;
|
|
}
|
|
|
|
case BTAction::BTSet: {
|
|
Dest = _Or(LshrOpSize, Dest, BitSelect.MaskBit(LshrOpSize).Ref());
|
|
StoreResultGPR(Op, Dest);
|
|
break;
|
|
}
|
|
|
|
case BTAction::BTComplement: {
|
|
Dest = _Xor(LshrOpSize, Dest, BitSelect.MaskBit(LshrOpSize).Ref());
|
|
|
|
if (IsNonconstant) {
|
|
Value = _Lshr(LshrOpSize, Dest, BitSelect.Ref());
|
|
} else {
|
|
Value = Dest;
|
|
}
|
|
|
|
SetRFLAG(Value, X86State::RFLAG_CF_RAW_LOC, Src.IsConstant ? Src.C : 0, true);
|
|
CFInverted = true;
|
|
|
|
StoreResultGPR(Op, Dest);
|
|
break;
|
|
}
|
|
}
|
|
} else {
|
|
// Load the address to the memory location
|
|
Ref Dest = MakeSegmentAddress(Op, Op->Dest);
|
|
// Get the bit selection from the src
|
|
auto BitSelect = Src.Bfe(0, 3);
|
|
|
|
// Address is provided as bits we want BYTE offsets
|
|
// Extract Signed offset
|
|
Src = Src.Sbfe(3, Size - 3);
|
|
|
|
// Get the address offset by shifting out the size of the op (To shift out the bit selection)
|
|
// Then use that to index in to the memory location by size of op
|
|
AddressMode Address = {.Base = Dest, .Index = Src.Ref(), .AddrSize = OpSize::i64Bit};
|
|
|
|
switch (Action) {
|
|
case BTAction::BTNone: {
|
|
Value = _LoadMemGPRAutoTSO(OpSize::i8Bit, Address, OpSize::i8Bit);
|
|
break;
|
|
}
|
|
|
|
case BTAction::BTClear: {
|
|
Ref BitMask = BitSelect.MaskBit(OpSize::i64Bit).Ref();
|
|
|
|
if (DestIsLockedMem(Op)) {
|
|
HandledLock = true;
|
|
Value = _AtomicFetchCLR(OpSize::i8Bit, BitMask, LoadEffectiveAddress(this, Address, GetGPROpSize(), true));
|
|
} else {
|
|
Value = _LoadMemGPRAutoTSO(OpSize::i8Bit, Address, OpSize::i8Bit);
|
|
|
|
auto Modified = _Andn(OpSize::i64Bit, Value, BitMask);
|
|
_StoreMemGPRAutoTSO(OpSize::i8Bit, Address, Modified, OpSize::i8Bit);
|
|
}
|
|
break;
|
|
}
|
|
|
|
case BTAction::BTSet: {
|
|
Ref BitMask = BitSelect.MaskBit(OpSize::i64Bit).Ref();
|
|
|
|
if (DestIsLockedMem(Op)) {
|
|
HandledLock = true;
|
|
Value = _AtomicFetchOr(OpSize::i8Bit, BitMask, LoadEffectiveAddress(this, Address, GetGPROpSize(), true));
|
|
} else {
|
|
Value = _LoadMemGPRAutoTSO(OpSize::i8Bit, Address, OpSize::i8Bit);
|
|
|
|
auto Modified = _Or(OpSize::i64Bit, Value, BitMask);
|
|
_StoreMemGPRAutoTSO(OpSize::i8Bit, Address, Modified, OpSize::i8Bit);
|
|
}
|
|
break;
|
|
}
|
|
|
|
case BTAction::BTComplement: {
|
|
Ref BitMask = BitSelect.MaskBit(OpSize::i64Bit).Ref();
|
|
|
|
if (DestIsLockedMem(Op)) {
|
|
HandledLock = true;
|
|
Value = _AtomicFetchXor(OpSize::i8Bit, BitMask, LoadEffectiveAddress(this, Address, GetGPROpSize(), true));
|
|
} else {
|
|
Value = _LoadMemGPRAutoTSO(OpSize::i8Bit, Address, OpSize::i8Bit);
|
|
|
|
auto Modified = _Xor(OpSize::i64Bit, Value, BitMask);
|
|
_StoreMemGPRAutoTSO(OpSize::i8Bit, Address, Modified, OpSize::i8Bit);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Now shift in to the correct bit location
|
|
if (!BitSelect.IsDefinitelyZero()) {
|
|
Value = _Lshr(std::max(OpSize::i32Bit, GetOpSize(Value)), Value, BitSelect.Ref());
|
|
}
|
|
|
|
// AMD: OF/SF/ZF/AF/PF undefined.
|
|
// Intel: OF/SF/AF/PF undefined. ZF must be preserved.
|
|
// We choose to preserve ZF/OF/SF since we just use an rmif
|
|
// to insert into CF directly. We could optimize perhaps.
|
|
SetCFDirect(Value, 0, true);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::IMUL1SrcOp(OpcodeArgs) {
|
|
/* We're just going to sign-extend the non-garbage anyway.. */
|
|
Ref Src1 = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Src2 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
const auto SizeBits = IR::OpSizeAsBits(Size);
|
|
|
|
Ref Dest {};
|
|
Ref ResultHigh {};
|
|
switch (Size) {
|
|
case OpSize::i8Bit:
|
|
case OpSize::i16Bit: {
|
|
Src1 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src1);
|
|
Src2 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src2);
|
|
Dest = _Mul(OpSize::i64Bit, Src1, Src2);
|
|
ResultHigh = _Sbfe(OpSize::i64Bit, SizeBits, SizeBits, Dest);
|
|
break;
|
|
}
|
|
case OpSize::i32Bit: {
|
|
ResultHigh = _SMull(Src1, Src2);
|
|
ResultHigh = _Sbfe(OpSize::i64Bit, SizeBits, SizeBits, ResultHigh);
|
|
// Flipped order to save a move
|
|
Dest = _Mul(OpSize::i32Bit, Src1, Src2);
|
|
break;
|
|
}
|
|
case OpSize::i64Bit: {
|
|
ResultHigh = _MulH(OpSize::i64Bit, Src1, Src2);
|
|
// Flipped order to save a move
|
|
Dest = _Mul(OpSize::i64Bit, Src1, Src2);
|
|
break;
|
|
}
|
|
default: FEX_UNREACHABLE;
|
|
}
|
|
|
|
StoreResultGPR(Op, Dest);
|
|
CalculateFlags_MUL(Size, Dest, ResultHigh);
|
|
}
|
|
|
|
void OpDispatchBuilder::IMUL2SrcOp(OpcodeArgs) {
|
|
Ref Src1 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Src2 = LoadSourceGPR(Op, Op->Src[1], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
const auto SizeBits = IR::OpSizeAsBits(Size);
|
|
|
|
Ref Dest {};
|
|
Ref ResultHigh {};
|
|
|
|
switch (Size) {
|
|
case OpSize::i8Bit:
|
|
case OpSize::i16Bit: {
|
|
Src1 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src1);
|
|
Src2 = ARef(Src2).Sbfe(0, SizeBits).Ref();
|
|
Dest = _Mul(OpSize::i64Bit, Src1, Src2);
|
|
ResultHigh = _Sbfe(OpSize::i64Bit, SizeBits, SizeBits, Dest);
|
|
break;
|
|
}
|
|
case OpSize::i32Bit: {
|
|
ResultHigh = _SMull(Src1, Src2);
|
|
ResultHigh = _Sbfe(OpSize::i64Bit, SizeBits, SizeBits, ResultHigh);
|
|
// Flipped order to save a move
|
|
Dest = _Mul(OpSize::i32Bit, Src1, Src2);
|
|
break;
|
|
}
|
|
case OpSize::i64Bit: {
|
|
ResultHigh = _MulH(OpSize::i64Bit, Src1, Src2);
|
|
// Flipped order to save a move
|
|
Dest = _Mul(OpSize::i64Bit, Src1, Src2);
|
|
break;
|
|
}
|
|
default: FEX_UNREACHABLE;
|
|
}
|
|
|
|
StoreResultGPR(Op, Dest);
|
|
CalculateFlags_MUL(Size, Dest, ResultHigh);
|
|
}
|
|
|
|
void OpDispatchBuilder::IMULOp(OpcodeArgs) {
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
const auto SizeBits = IR::OpSizeAsBits(Size);
|
|
|
|
Ref Src1 = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Src2 = LoadGPRRegister(X86State::REG_RAX);
|
|
|
|
if (Size != OpSize::i64Bit) {
|
|
Src1 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src1);
|
|
Src2 = _Sbfe(OpSize::i64Bit, SizeBits, 0, Src2);
|
|
}
|
|
|
|
// 64-bit special cased to save a move
|
|
Ref Result {};
|
|
if (Size < OpSize::i64Bit) {
|
|
Result = _Mul(OpSize::i64Bit, Src1, Src2);
|
|
}
|
|
Ref ResultHigh {};
|
|
if (Size == OpSize::i8Bit) {
|
|
// Result is stored in AX
|
|
StoreGPRRegister(X86State::REG_RAX, Result, OpSize::i16Bit);
|
|
ResultHigh = _Sbfe(OpSize::i64Bit, 8, 8, Result);
|
|
} else if (Size == OpSize::i16Bit) {
|
|
// 16bits stored in AX
|
|
// 16bits stored in DX
|
|
StoreGPRRegister(X86State::REG_RAX, Result, Size);
|
|
ResultHigh = _Sbfe(OpSize::i64Bit, 16, 16, Result);
|
|
StoreGPRRegister(X86State::REG_RDX, ResultHigh, Size);
|
|
} else if (Size == OpSize::i32Bit) {
|
|
// 32bits stored in EAX
|
|
// 32bits stored in EDX
|
|
// Make sure they get Zext correctly
|
|
auto LocalResult = _Bfe(OpSize::i64Bit, 32, 0, Result);
|
|
auto LocalResultHigh = _Bfe(OpSize::i64Bit, 32, 32, Result);
|
|
ResultHigh = _Sbfe(OpSize::i64Bit, 32, 32, Result);
|
|
Result = _Sbfe(OpSize::i64Bit, 32, 0, Result);
|
|
StoreGPRRegister(X86State::REG_RAX, LocalResult);
|
|
StoreGPRRegister(X86State::REG_RDX, LocalResultHigh);
|
|
} else if (Size == OpSize::i64Bit) {
|
|
if (!Is64BitMode) {
|
|
LogMan::Msg::EFmt("Doesn't exist in 32bit mode");
|
|
DecodeFailure = true;
|
|
return;
|
|
}
|
|
// 64bits stored in RAX
|
|
// 64bits stored in RDX
|
|
ResultHigh = _MulH(OpSize::i64Bit, Src1, Src2);
|
|
Result = _Mul(OpSize::i64Bit, Src1, Src2);
|
|
StoreGPRRegister(X86State::REG_RAX, Result);
|
|
StoreGPRRegister(X86State::REG_RDX, ResultHigh);
|
|
}
|
|
|
|
CalculateFlags_MUL(Size, Result, ResultHigh);
|
|
}
|
|
|
|
void OpDispatchBuilder::MULOp(OpcodeArgs) {
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
const auto SizeBits = IR::OpSizeAsBits(Size);
|
|
|
|
Ref Src1 = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Src2 = LoadGPRRegister(X86State::REG_RAX);
|
|
Ref Result {};
|
|
|
|
if (Size != OpSize::i64Bit) {
|
|
Src1 = _Bfe(OpSize::i64Bit, SizeBits, 0, Src1);
|
|
Src2 = _Bfe(OpSize::i64Bit, SizeBits, 0, Src2);
|
|
Result = _UMul(OpSize::i64Bit, Src1, Src2);
|
|
}
|
|
Ref ResultHigh {};
|
|
|
|
if (Size == OpSize::i8Bit) {
|
|
// Result is stored in AX
|
|
StoreGPRRegister(X86State::REG_RAX, Result, OpSize::i16Bit);
|
|
ResultHigh = _Bfe(OpSize::i64Bit, 8, 8, Result);
|
|
} else if (Size == OpSize::i16Bit) {
|
|
// 16bits stored in AX
|
|
// 16bits stored in DX
|
|
StoreGPRRegister(X86State::REG_RAX, Result, Size);
|
|
ResultHigh = _Bfe(OpSize::i64Bit, 16, 16, Result);
|
|
StoreGPRRegister(X86State::REG_RDX, ResultHigh, Size);
|
|
} else if (Size == OpSize::i32Bit) {
|
|
// 32bits stored in EAX
|
|
// 32bits stored in EDX
|
|
Ref ResultLow = _Bfe(OpSize::i64Bit, 32, 0, Result);
|
|
ResultHigh = _Bfe(OpSize::i64Bit, 32, 32, Result);
|
|
StoreGPRRegister(X86State::REG_RAX, ResultLow);
|
|
StoreGPRRegister(X86State::REG_RDX, ResultHigh);
|
|
} else if (Size == OpSize::i64Bit) {
|
|
if (!Is64BitMode) {
|
|
LogMan::Msg::EFmt("Doesn't exist in 32bit mode");
|
|
DecodeFailure = true;
|
|
return;
|
|
}
|
|
// 64bits stored in RAX
|
|
// 64bits stored in RDX
|
|
//
|
|
// Calculate high first to allow better RA.
|
|
ResultHigh = _UMulH(OpSize::i64Bit, Src1, Src2);
|
|
Result = _UMul(OpSize::i64Bit, Src1, Src2);
|
|
StoreGPRRegister(X86State::REG_RAX, Result);
|
|
StoreGPRRegister(X86State::REG_RDX, ResultHigh);
|
|
}
|
|
|
|
CalculateFlags_UMUL(ResultHigh);
|
|
}
|
|
|
|
void OpDispatchBuilder::NOTOp(OpcodeArgs) {
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
const auto SizeBits = IR::OpSizeAsBits(Size);
|
|
LOGMAN_THROW_A_FMT(Size >= IR::OpSize::i8Bit && Size <= IR::OpSize::i64Bit, "Invalid size");
|
|
|
|
Ref MaskConst {};
|
|
if (Size == OpSize::i64Bit) {
|
|
MaskConst = Constant(~0ULL);
|
|
} else {
|
|
MaskConst = Constant((1ULL << SizeBits) - 1);
|
|
}
|
|
|
|
if (DestIsLockedMem(Op)) {
|
|
HandledLock = true;
|
|
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
|
|
// Result unused
|
|
_AtomicFetchXor(Size, MaskConst, DestMem);
|
|
} else if (!Op->Dest.IsGPR()) {
|
|
// GPR version plays fast and loose with sizes, be safe for memory tho.
|
|
Ref Src = LoadSourceGPR(Op, Op->Dest, Op->Flags);
|
|
Src = _Xor(OpSize::i64Bit, Src, MaskConst);
|
|
StoreResultGPR(Op, Src);
|
|
} else {
|
|
// Specially handle high bits so we can invert in place with the correct
|
|
// mask and a larger type.
|
|
auto Dest = Op->Dest;
|
|
if (Dest.Data.GPR.HighBits) {
|
|
LOGMAN_THROW_A_FMT(Size == OpSize::i8Bit, "Only 8-bit GPRs get high bits");
|
|
MaskConst = Constant(0xFF00);
|
|
Dest.Data.GPR.HighBits = false;
|
|
}
|
|
|
|
// Always load full size, we explicitly want the upper bits to get the
|
|
// insert behaviour for free/implicitly.
|
|
const auto GPRSize = GetGPROpSize();
|
|
Ref Src = LoadSourceGPR_WithOpSize(Op, Dest, GPRSize, Op->Flags);
|
|
|
|
// For 8/16-bit, use 64-bit invert so we invert in place, while getting
|
|
// insert behaviour. For 32-bit, use 32-bit invert to zero the upper bits.
|
|
const auto EffectiveSize = Size == OpSize::i32Bit ? OpSize::i32Bit : GPRSize;
|
|
|
|
// If we're inverting the whole thing, use Not instead of Xor to save a constant.
|
|
if (Size >= OpSize::i32Bit) {
|
|
Src = _Not(EffectiveSize, Src);
|
|
} else {
|
|
Src = _Xor(EffectiveSize, Src, MaskConst);
|
|
}
|
|
|
|
// Always store 64-bit, the Not/Xor correctly handle the upper bits and this
|
|
// way we can delete the store.
|
|
StoreResultGPR_WithOpSize(Op, Dest, Src, GPRSize);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::XADDOp(OpcodeArgs) {
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.LoadData = false});
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);
|
|
Ref Result;
|
|
|
|
if (Op->Dest.IsGPR()) {
|
|
// If this is a GPR then we can just do an Add
|
|
Result = CalculateFlags_ADD(OpSizeFromSrc(Op), Dest, Src);
|
|
|
|
// Previous value in dest gets stored in src
|
|
StoreResultGPR(Op, Op->Src[0], Dest);
|
|
|
|
// Calculated value gets stored in dst (order is important if dst is same as src)
|
|
StoreResultGPR(Op, Result);
|
|
} else {
|
|
HandledLock = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK;
|
|
Dest = AppendSegmentOffset(Dest, Op->Flags);
|
|
auto Before = _AtomicFetchAdd(OpSizeFromSrc(Op), Src, Dest);
|
|
CalculateFlags_ADD(OpSizeFromSrc(Op), Before, Src);
|
|
StoreResultGPR(Op, Op->Src[0], Before);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::PopcountOp(OpcodeArgs) {
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = CTX->HostFeatures.SupportsCSSC || GetSrcSize(Op) >= 4});
|
|
Src = _Popcount(OpSizeFromSrc(Op), Src);
|
|
StoreResultGPR(Op, Src);
|
|
|
|
// We need to set ZF while clearing the rest of NZCV. The result of a popcount
|
|
// is in the range [0, 63]. In particular, it is always positive. So a
|
|
// combined NZ test will correctly zero SF/CF/OF while setting ZF.
|
|
SetNZ_ZeroCV(OpSize::i32Bit, Src);
|
|
ZeroPF_AF();
|
|
}
|
|
|
|
Ref OpDispatchBuilder::CalculateAFForDecimal(Ref A) {
|
|
auto Nibble = _And(OpSize::i64Bit, A, Constant(0xF));
|
|
auto Greater = Select01(OpSize::i64Bit, CondClass::UGT, Nibble, Constant(9));
|
|
|
|
return _Or(OpSize::i64Bit, LoadAF(), Greater);
|
|
}
|
|
|
|
void OpDispatchBuilder::DAAOp(OpcodeArgs) {
|
|
CalculateDeferredFlags();
|
|
auto AL = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit);
|
|
auto CFInv = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC, true);
|
|
auto AF = CalculateAFForDecimal(AL);
|
|
|
|
// CF |= (AL > 0x99);
|
|
CFInv = _And(OpSize::i64Bit, CFInv, Select01(OpSize::i64Bit, CondClass::ULE, AL, Constant(0x99)));
|
|
|
|
// AL = AF ? (AL + 0x6) : AL;
|
|
AL = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::NEQ, AF, Constant(0), Add(OpSize::i64Bit, AL, 0x6), AL);
|
|
|
|
// AL = CF ? (AL + 0x60) : AL;
|
|
AL = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, CFInv, Constant(0), Add(OpSize::i64Bit, AL, 0x60), AL);
|
|
|
|
// SF, ZF, PF set according to result. CF set per above. OF undefined.
|
|
StoreGPRRegister(X86State::REG_RAX, AL, OpSize::i8Bit);
|
|
SetNZ_ZeroCV(OpSize::i8Bit, AL);
|
|
SetCFInverted(CFInv);
|
|
CalculatePF(AL);
|
|
SetAFAndFixup(AF);
|
|
}
|
|
|
|
void OpDispatchBuilder::DASOp(OpcodeArgs) {
|
|
CalculateDeferredFlags();
|
|
auto AL = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit);
|
|
auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_RAW_LOC);
|
|
auto AF = CalculateAFForDecimal(AL);
|
|
|
|
// CF |= (AL > 0x99);
|
|
CF = _Or(OpSize::i64Bit, CF, Select01(OpSize::i64Bit, CondClass::UGT, AL, Constant(0x99)));
|
|
|
|
// NewCF = CF | (AF && (Borrow from AL - 6))
|
|
auto NewCF = _Or(OpSize::i32Bit, CF, _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::ULT, AL, Constant(6), AF, CF));
|
|
|
|
// AL = AF ? (AL - 0x6) : AL;
|
|
AL = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::NEQ, AF, Constant(0), Sub(OpSize::i64Bit, AL, 0x6), AL);
|
|
|
|
// AL = CF ? (AL - 0x60) : AL;
|
|
AL = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::NEQ, CF, Constant(0), Sub(OpSize::i64Bit, AL, 0x60), AL);
|
|
|
|
// SF, ZF, PF set according to result. CF set per above. OF undefined.
|
|
StoreGPRRegister(X86State::REG_RAX, AL, OpSize::i8Bit);
|
|
SetNZ_ZeroCV(OpSize::i8Bit, AL);
|
|
SetCFDirect(NewCF);
|
|
CalculatePF(AL);
|
|
SetAFAndFixup(AF);
|
|
}
|
|
|
|
void OpDispatchBuilder::AAAOp(OpcodeArgs) {
|
|
auto A = LoadGPRRegister(X86State::REG_RAX);
|
|
auto AF = CalculateAFForDecimal(A);
|
|
|
|
// CF = AF, OF/SF/ZF/PF undefined
|
|
SetCFDirect_InvalidateNZV(AF);
|
|
SetAFAndFixup(AF);
|
|
CalculateDeferredFlags();
|
|
|
|
// AX = CF ? (AX + 0x106) : 0
|
|
A = NZCVSelect(OpSize::i32Bit, CondClass::UGE /* CF = 1 */, Add(OpSize::i32Bit, A, 0x106), A);
|
|
|
|
// AL = AL & 0x0F
|
|
A = _And(OpSize::i32Bit, A, Constant(0xFF0F));
|
|
StoreGPRRegister(X86State::REG_RAX, A, OpSize::i16Bit);
|
|
}
|
|
|
|
void OpDispatchBuilder::AASOp(OpcodeArgs) {
|
|
auto A = LoadGPRRegister(X86State::REG_RAX);
|
|
auto AF = CalculateAFForDecimal(A);
|
|
|
|
// CF = AF, OF/SF/ZF/PF undefined
|
|
SetCFDirect_InvalidateNZV(AF);
|
|
SetAFAndFixup(AF);
|
|
CalculateDeferredFlags();
|
|
|
|
// AX = CF ? (AX - 0x106) : 0
|
|
A = NZCVSelect(OpSize::i32Bit, CondClass::UGE /* CF = 1 */, Sub(OpSize::i32Bit, A, 0x106), A);
|
|
|
|
// AL = AL & 0x0F
|
|
A = _And(OpSize::i32Bit, A, Constant(0xFF0F));
|
|
StoreGPRRegister(X86State::REG_RAX, A, OpSize::i16Bit);
|
|
}
|
|
|
|
void OpDispatchBuilder::AAMOp(OpcodeArgs) {
|
|
auto AL = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit);
|
|
auto Imm8 = Constant(Op->Src[0].Literal() & 0xFF);
|
|
Ref Quotient = _AllocateGPR(true);
|
|
Ref Remainder = _AllocateGPR(true);
|
|
_UDiv(OpSize::i64Bit, AL, Invalid(), Imm8, Quotient, Remainder);
|
|
auto Res = _AddShift(OpSize::i64Bit, Remainder, Quotient, ShiftType::LSL, 8);
|
|
StoreGPRRegister(X86State::REG_RAX, Res, OpSize::i16Bit);
|
|
|
|
SetNZ_ZeroCV(OpSize::i8Bit, Res);
|
|
CalculatePF(Res);
|
|
InvalidateAF();
|
|
}
|
|
|
|
void OpDispatchBuilder::AADOp(OpcodeArgs) {
|
|
auto A = LoadGPRRegister(X86State::REG_RAX);
|
|
auto AH = _Lshr(OpSize::i32Bit, A, Constant(8));
|
|
auto Imm8 = Constant(Op->Src[0].Literal() & 0xFF);
|
|
auto NewAL = Add(OpSize::i64Bit, A, _Mul(OpSize::i64Bit, AH, Imm8));
|
|
auto Result = _And(OpSize::i64Bit, NewAL, Constant(0xFF));
|
|
StoreGPRRegister(X86State::REG_RAX, Result, OpSize::i16Bit);
|
|
|
|
SetNZ_ZeroCV(OpSize::i8Bit, Result);
|
|
CalculatePF(Result);
|
|
InvalidateAF();
|
|
}
|
|
|
|
void OpDispatchBuilder::XLATOp(OpcodeArgs) {
|
|
Ref Src = MakeSegmentAddress(X86State::REG_RBX, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);
|
|
Ref Offset = LoadGPRRegister(X86State::REG_RAX, OpSize::i8Bit);
|
|
|
|
AddressMode A = {.Base = Src, .Index = Offset, .AddrSize = OpSize::i64Bit};
|
|
auto Res = _LoadMemGPRAutoTSO(OpSize::i8Bit, A, OpSize::i8Bit);
|
|
|
|
StoreGPRRegister(X86State::REG_RAX, Res, OpSize::i8Bit);
|
|
}
|
|
|
|
void OpDispatchBuilder::ReadSegmentReg(OpcodeArgs, OpDispatchBuilder::Segment Seg) {
|
|
// 64-bit only
|
|
// Doesn't hit the segment register optimization
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
Ref Src {};
|
|
if (Seg == Segment::FS) {
|
|
Src = _LoadContextGPR(Size, offsetof(FEXCore::Core::CPUState, fs_cached));
|
|
} else {
|
|
Src = _LoadContextGPR(Size, offsetof(FEXCore::Core::CPUState, gs_cached));
|
|
}
|
|
|
|
StoreResultGPR(Op, Src);
|
|
}
|
|
|
|
void OpDispatchBuilder::WriteSegmentReg(OpcodeArgs, OpDispatchBuilder::Segment Seg) {
|
|
// Documentation claims that the 32-bit version of this instruction inserts in to the lower 32-bits of the segment
|
|
// This is incorrect and it instead zero extends the 32-bit value to 64-bit
|
|
const auto Size = OpSizeFromDst(Op);
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);
|
|
if (Seg == Segment::FS) {
|
|
_StoreContextGPR(Size, Src, offsetof(FEXCore::Core::CPUState, fs_cached));
|
|
} else {
|
|
_StoreContextGPR(Size, Src, offsetof(FEXCore::Core::CPUState, gs_cached));
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::EnterOp(OpcodeArgs) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
const auto OperandSize = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_OPERAND_SIZE) ? OpSize::i16Bit : GPRSize;
|
|
const uint64_t Value = Op->Src[0].Literal();
|
|
|
|
const uint16_t AllocSpace = Value & 0xFFFF;
|
|
const uint8_t Level = (Value >> 16) & 0x1F;
|
|
|
|
const auto PushValue = [&](IR::OpSize Size, Ref Src) -> Ref {
|
|
auto OldSP = LoadGPRRegister(X86State::REG_RSP);
|
|
auto NewSP = _Push(GPRSize, Size, Src, OldSP);
|
|
|
|
// Store the new stack pointer
|
|
StoreGPRRegister(X86State::REG_RSP, NewSP);
|
|
return NewSP;
|
|
};
|
|
|
|
auto OldBP = LoadGPRRegister(X86State::REG_RBP);
|
|
auto NewSP = PushValue(OperandSize, OldBP);
|
|
auto temp_RBP = NewSP;
|
|
|
|
if (Level > 0) {
|
|
for (uint8_t i = 1; i < Level; ++i) {
|
|
auto MemLoc = Sub(GPRSize, OldBP, i * IR::OpSizeToSize(OperandSize));
|
|
auto Mem = _LoadMemGPR(OperandSize, MemLoc, OperandSize);
|
|
NewSP = PushValue(OperandSize, Mem);
|
|
}
|
|
NewSP = PushValue(OperandSize, temp_RBP);
|
|
}
|
|
NewSP = Sub(GPRSize, NewSP, AllocSpace);
|
|
StoreGPRRegister(X86State::REG_RSP, NewSP);
|
|
StoreGPRRegister(X86State::REG_RBP, temp_RBP);
|
|
}
|
|
|
|
void OpDispatchBuilder::SGDTOp(OpcodeArgs) {
|
|
auto DestAddress = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.LoadData = false});
|
|
|
|
// Store an emulated value in the format of:
|
|
// uint16_t Limit;
|
|
// {uint32_t,uint64_t} Base;
|
|
//
|
|
// Limit is always 0
|
|
// Base is always in kernel space at: 0xFFFFFFFFFFFE0000ULL
|
|
//
|
|
// Operand size prefix is ignored on this instruction, size purely depends on operating mode.
|
|
uint64_t GDTAddress = 0xFFFFFFFFFFFE0000ULL;
|
|
auto GDTStoreSize = OpSize::i64Bit;
|
|
if (!Is64BitMode) {
|
|
// Mask off upper bits if 32-bit result.
|
|
GDTAddress &= ~0U;
|
|
GDTStoreSize = OpSize::i32Bit;
|
|
}
|
|
|
|
_StoreMemGPRAutoTSO(OpSize::i16Bit, DestAddress, Constant(0));
|
|
_StoreMemGPRAutoTSO(GDTStoreSize, AddressMode {.Base = DestAddress, .Offset = 2, .AddrSize = OpSize::i64Bit}, Constant(GDTAddress));
|
|
}
|
|
|
|
void OpDispatchBuilder::SIDTOp(OpcodeArgs) {
|
|
auto DestAddress = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.LoadData = false});
|
|
|
|
// See SGDTOp, matches Linux in reported values
|
|
uint64_t IDTAddress = 0xFFFFFE0000000000ULL;
|
|
auto IDTStoreSize = OpSize::i64Bit;
|
|
if (!Is64BitMode) {
|
|
// Mask off upper bits if 32-bit result.
|
|
IDTAddress &= ~0U;
|
|
IDTStoreSize = OpSize::i32Bit;
|
|
}
|
|
|
|
_StoreMemGPRAutoTSO(OpSize::i16Bit, DestAddress, Constant(0xfff));
|
|
_StoreMemGPRAutoTSO(IDTStoreSize, AddressMode {.Base = DestAddress, .Offset = 2, .AddrSize = OpSize::i64Bit}, Constant(IDTAddress));
|
|
}
|
|
|
|
void OpDispatchBuilder::SMSWOp(OpcodeArgs) {
|
|
const bool IsMemDst = DestIsMem(Op);
|
|
|
|
IR::OpSize DstSize {OpSize::iInvalid};
|
|
Ref Const = Constant((1U << 31) | ///< PG - Paging
|
|
(0U << 30) | ///< CD - Cache Disable
|
|
(0U << 29) | ///< NW - Not Writethrough (Legacy, now ignored)
|
|
///< [28:19] - Reserved
|
|
(1U << 18) | ///< AM - Alignment Mask
|
|
///< 17 - Reserved
|
|
(1U << 16) | ///< WP - Write Protect
|
|
///< [15:6] - Reserved
|
|
(1U << 5) | ///< NE - Numeric Error
|
|
(1U << 4) | ///< ET - Extension Type (Legacy, now reserved and 1)
|
|
(0U << 3) | ///< TS - Task Switched
|
|
(0U << 2) | ///< EM - Emulation
|
|
(1U << 1) | ///< MP - Monitor Coprocessor
|
|
(1U << 0)); ///< PE - Protection Enabled
|
|
const auto OpAddr = X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0);
|
|
if (Is64BitMode) {
|
|
DstSize = OpAddr == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? OpSize::i16Bit :
|
|
OpAddr == X86Tables::DecodeFlags::FLAG_WIDENING_SIZE_LAST ? OpSize::i64Bit :
|
|
OpSize::i32Bit;
|
|
|
|
if (!IsMemDst && DstSize == OpSize::i32Bit) {
|
|
// Special-case version of `smsw ebx`. This instruction does an insert in to the lower 32-bits on 64-bit hosts.
|
|
// Override and insert.
|
|
auto Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, GetGPROpSize(), Op->Flags);
|
|
Const = _Bfi(OpSize::i64Bit, 32, 0, Dest, Const);
|
|
DstSize = OpSize::i64Bit;
|
|
}
|
|
} else {
|
|
DstSize = OpAddr == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? OpSize::i16Bit : OpSize::i32Bit;
|
|
}
|
|
|
|
if (IsMemDst) {
|
|
// Memory destinatino always writes only 16-bits.
|
|
DstSize = OpSize::i16Bit;
|
|
}
|
|
|
|
StoreResultGPR_WithOpSize(Op, Op->Dest, Const, DstSize);
|
|
}
|
|
|
|
OpDispatchBuilder::CycleCounterPair OpDispatchBuilder::CycleCounter(bool SelfSynchronizingLoads) {
|
|
Ref CounterLow {};
|
|
Ref CounterHigh {};
|
|
auto Counter = _CycleCounter(SelfSynchronizingLoads);
|
|
if (CTX->Config.TSCScale) {
|
|
CounterLow = _Lshl(OpSize::i32Bit, Counter, Constant(CTX->Config.TSCScale));
|
|
CounterHigh = _Lshr(OpSize::i64Bit, Counter, Constant(32 - CTX->Config.TSCScale));
|
|
} else {
|
|
CounterLow = _Bfe(OpSize::i64Bit, 32, 0, Counter);
|
|
CounterHigh = _Bfe(OpSize::i64Bit, 32, 32, Counter);
|
|
}
|
|
|
|
return {
|
|
.CounterLow = CounterLow,
|
|
.CounterHigh = CounterHigh,
|
|
};
|
|
}
|
|
|
|
void OpDispatchBuilder::RDTSCOp(OpcodeArgs) {
|
|
auto Counter = CycleCounter(false);
|
|
StoreGPRRegister(X86State::REG_RAX, Counter.CounterLow);
|
|
StoreGPRRegister(X86State::REG_RDX, Counter.CounterHigh);
|
|
}
|
|
|
|
void OpDispatchBuilder::INCOp(OpcodeArgs) {
|
|
Ref Dest;
|
|
Ref Result;
|
|
const auto Size = GetSrcBitSize(Op);
|
|
const bool IsLocked = DestIsLockedMem(Op);
|
|
|
|
if (IsLocked) {
|
|
HandledLock = true;
|
|
|
|
Ref DestAddress = MakeSegmentAddress(Op, Op->Dest);
|
|
Dest = _AtomicFetchAdd(OpSizeFromSrc(Op), Constant(1), DestAddress);
|
|
} else {
|
|
Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= 32});
|
|
}
|
|
|
|
CalculateDeferredFlags();
|
|
|
|
if (Size < 32 && CTX->HostFeatures.SupportsFlagM) {
|
|
// Addition producing upper garbage
|
|
Result = Add(OpSize::i32Bit, Dest, 1);
|
|
CalculatePF(Result);
|
|
CalculateAF(Dest, Constant(1));
|
|
|
|
// Correctly set NZ flags, preserving C
|
|
HandleNZCV_RMW();
|
|
_SetSmallNZV(OpSizeFromSrc(Op), Result);
|
|
|
|
// Fix up V flag. INC overflows only when incrementing a positive and
|
|
// getting a negative. So compare the sign bits to calculate V.
|
|
_RmifNZCV(_Andn(OpSize::i32Bit, Result, Dest), Size - 1, 1);
|
|
} else {
|
|
Result = CalculateFlags_ADD(OpSizeFromSrc(Op), Dest, Constant(1), false);
|
|
}
|
|
|
|
if (!IsLocked) {
|
|
StoreResultGPR(Op, Result);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::DECOp(OpcodeArgs) {
|
|
Ref Dest;
|
|
Ref Result;
|
|
const auto Size = GetSrcBitSize(Op);
|
|
const bool IsLocked = DestIsLockedMem(Op);
|
|
|
|
if (IsLocked) {
|
|
HandledLock = true;
|
|
|
|
Ref DestAddress = MakeSegmentAddress(Op, Op->Dest);
|
|
|
|
// Use Add instead of Sub to avoid a NEG
|
|
Dest = _AtomicFetchAdd(OpSizeFromSrc(Op), Constant(Size == 64 ? -1 : ((1ULL << Size) - 1)), DestAddress);
|
|
} else {
|
|
Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= 32});
|
|
}
|
|
|
|
CalculateDeferredFlags();
|
|
|
|
if (Size < 32 && CTX->HostFeatures.SupportsFlagM) {
|
|
// Subtraction producing upper garbage
|
|
Result = Sub(OpSize::i32Bit, Dest, 1);
|
|
CalculatePF(Result);
|
|
CalculateAF(Dest, Constant(1));
|
|
|
|
// Correctly set NZ flags, preserving C
|
|
HandleNZCV_RMW();
|
|
_SetSmallNZV(OpSizeFromSrc(Op), Result);
|
|
|
|
// Fix up V flag. DEC overflows only when decrementing a negative and
|
|
// getting a positive. So compare the sign bits to calculate V.
|
|
_RmifNZCV(_Andn(OpSize::i32Bit, Dest, Result), Size - 1, 1);
|
|
} else {
|
|
Result = CalculateFlags_SUB(OpSizeFromSrc(Op), Dest, Constant(1), false);
|
|
}
|
|
|
|
if (!IsLocked) {
|
|
StoreResultGPR(Op, Result);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::STOSOp(OpcodeArgs) {
|
|
if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) {
|
|
LogMan::Msg::EFmt("STOSOp: Can't handle address size override (OP: 0x{:04X}, Flags: 0x{:08X})", Op->OP, Op->Flags);
|
|
DecodeFailure = true;
|
|
return;
|
|
}
|
|
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) != 0;
|
|
|
|
if (!Repeat) {
|
|
// Src is used only for a store of the same size so allow garbage
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// Only ES prefix
|
|
Ref Dest = MakeSegmentAddress(X86State::REG_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
|
|
|
|
// Store to memory where RDI points
|
|
if (CTX->IsMemcpyAtomicTSOEnabled()) {
|
|
_StoreMemGPRAutoTSO(Size, Dest, Src, Size);
|
|
} else {
|
|
_StoreMem(RegClass::GPR, Size, Src, Dest, Invalid(), OpSize::i8Bit, MemOffsetType::SXTX, 1);
|
|
}
|
|
|
|
// Offset the pointer
|
|
Ref TailDest = LoadGPRRegister(X86State::REG_RDI);
|
|
StoreGPRRegister(X86State::REG_RDI, OffsetByDir(TailDest, IR::OpSizeToSize(Size)));
|
|
} else {
|
|
// FEX doesn't support partial faulting REP instructions.
|
|
// Converting this to a `MemSet` IR op optimizes this quite significantly in our codegen.
|
|
// If FEX is to gain support for faulting REP instructions, then this implementation needs to change significantly.
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags);
|
|
Ref Dest = LoadGPRRegister(X86State::REG_RDI);
|
|
|
|
// Only ES prefix
|
|
auto Segment = GetSegment(0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
|
|
|
|
Ref Counter = LoadGPRRegister(X86State::REG_RCX);
|
|
|
|
auto Result = _MemSet(CTX->IsAtomicTSOEnabled(), Size, Segment ?: InvalidNode, Dest, Src, Counter, LoadDir(1));
|
|
StoreGPRRegister(X86State::REG_RCX, Constant(0));
|
|
StoreGPRRegister(X86State::REG_RDI, Result);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::MOVSOp(OpcodeArgs) {
|
|
if (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) {
|
|
LogMan::Msg::EFmt("MOVSOp: Can't handle address size override (OP: 0x{:04X}, Flags: 0x{:08X})", Op->OP, Op->Flags);
|
|
DecodeFailure = true;
|
|
return;
|
|
}
|
|
|
|
// RA now can handle these to be here, to avoid DF accesses
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
|
|
if (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) {
|
|
auto SrcAddr = LoadGPRRegister(X86State::REG_RSI);
|
|
auto DstAddr = LoadGPRRegister(X86State::REG_RDI);
|
|
auto Counter = LoadGPRRegister(X86State::REG_RCX);
|
|
|
|
auto DstSegment = GetSegment(0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
|
|
auto SrcSegment = GetSegment(Op->Flags, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX);
|
|
|
|
if (DstSegment) {
|
|
DstAddr = Add(OpSize::i64Bit, DstAddr, DstSegment);
|
|
}
|
|
|
|
if (SrcSegment) {
|
|
SrcAddr = Add(OpSize::i64Bit, SrcAddr, SrcSegment);
|
|
}
|
|
|
|
Ref Result_Src = _AllocateGPR(false);
|
|
Ref Result_Dst = _AllocateGPR(false);
|
|
_MemCpy(CTX->IsAtomicTSOEnabled(), Size, DstAddr, SrcAddr, Counter, LoadDir(1), Result_Dst, Result_Src);
|
|
|
|
if (DstSegment) {
|
|
Result_Dst = Sub(OpSize::i64Bit, Result_Dst, DstSegment);
|
|
}
|
|
|
|
if (SrcSegment) {
|
|
Result_Src = Sub(OpSize::i64Bit, Result_Src, SrcSegment);
|
|
}
|
|
|
|
StoreGPRRegister(X86State::REG_RCX, Constant(0));
|
|
StoreGPRRegister(X86State::REG_RDI, Result_Dst);
|
|
StoreGPRRegister(X86State::REG_RSI, Result_Src);
|
|
} else {
|
|
Ref RSI = MakeSegmentAddress(X86State::REG_RSI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);
|
|
Ref RDI = MakeSegmentAddress(X86State::REG_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
|
|
|
|
if (CTX->IsMemcpyAtomicTSOEnabled()) {
|
|
auto Src = _LoadMemGPRAutoTSO(Size, RSI, Size);
|
|
|
|
// Store to memory where RDI points
|
|
_StoreMemGPRAutoTSO(Size, RDI, Src, Size);
|
|
} else {
|
|
auto Src = _LoadMem(RegClass::GPR, Size, RSI, Invalid(), OpSize::i8Bit, MemOffsetType::SXTX, 1);
|
|
_StoreMem(RegClass::GPR, Size, Src, RDI, Invalid(), OpSize::i8Bit, MemOffsetType::SXTX, 1);
|
|
}
|
|
|
|
RSI = OffsetByDir(RSI, IR::OpSizeToSize(Size));
|
|
RDI = OffsetByDir(RDI, IR::OpSizeToSize(Size));
|
|
|
|
StoreGPRRegister(X86State::REG_RSI, RSI);
|
|
StoreGPRRegister(X86State::REG_RDI, RDI);
|
|
}
|
|
}
|
|
|
|
IR::OpSize OpDispatchBuilder::GetStringOpSize(X86Tables::DecodedOp Op) const {
|
|
LOGMAN_THROW_A_FMT(Is64BitMode || !(Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE), "Invalid modifier on 32bit address");
|
|
return !Is64BitMode || (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) ? OpSize::i32Bit : OpSize::i64Bit;
|
|
}
|
|
|
|
void OpDispatchBuilder::CMPSOp(OpcodeArgs) {
|
|
if (!Is64BitMode && (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE)) {
|
|
LogMan::Msg::EFmt("CMPSOp: Address size override (0x67) not supported in 32-bit mode (OP: 0x{:04X}).", Op->OP);
|
|
DecodeFailure = true;
|
|
return;
|
|
}
|
|
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
OpSize AddrSize = GetStringOpSize(Op);
|
|
|
|
bool Repeat = Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX);
|
|
if (!Repeat) {
|
|
Ref Src_RSI = LoadGPRRegister(X86State::REG_RSI, AddrSize);
|
|
Ref Src_RDI = LoadGPRRegister(X86State::REG_RDI, AddrSize);
|
|
|
|
Ref Dest_RSI = AppendSegmentOffset(Src_RSI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);
|
|
Ref Dest_RDI = AppendSegmentOffset(Src_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
|
|
|
|
auto Src1 = _LoadMemGPRAutoTSO(Size, Dest_RDI, Size);
|
|
auto Src2 = _LoadMemGPRAutoTSO(Size, Dest_RSI, Size);
|
|
|
|
CalculateFlags_SUB(OpSizeFromSrc(Op), Src2, Src1);
|
|
|
|
Dest_RDI = OffsetByDir(Src_RDI, IR::OpSizeToSize(Size));
|
|
if (Is64BitMode && AddrSize == OpSize::i32Bit) {
|
|
Dest_RDI = _Bfe(OpSize::i64Bit, 32, 0, Dest_RDI);
|
|
StoreGPRRegister(X86State::REG_RDI, Dest_RDI);
|
|
} else {
|
|
StoreGPRRegister(X86State::REG_RDI, Dest_RDI, AddrSize);
|
|
}
|
|
|
|
Dest_RSI = OffsetByDir(Src_RSI, IR::OpSizeToSize(Size));
|
|
if (Is64BitMode && AddrSize == OpSize::i32Bit) {
|
|
Dest_RSI = _Bfe(OpSize::i64Bit, 32, 0, Dest_RSI);
|
|
StoreGPRRegister(X86State::REG_RSI, Dest_RSI);
|
|
} else {
|
|
StoreGPRRegister(X86State::REG_RSI, Dest_RSI, AddrSize);
|
|
}
|
|
} else {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
bool REPE = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX;
|
|
|
|
// If rcx = 0, skip the whole loop.
|
|
Ref Counter = LoadGPRRegister(X86State::REG_RCX);
|
|
auto OuterJump = CondJump(Counter, CondClass::EQ);
|
|
|
|
auto BeforeLoop = CreateNewCodeBlockAfter(GetCurrentBlock());
|
|
SetFalseJumpTarget(OuterJump, BeforeLoop);
|
|
SetCurrentCodeBlock(BeforeLoop);
|
|
StartNewBlock();
|
|
|
|
ForeachDirection([this, Op, Size, AddrSize, REPE](int32_t PtrDir) {
|
|
IRPair<IROp_CondJump> InnerJump;
|
|
auto JumpIntoLoop = Jump();
|
|
|
|
// Setup for the loop
|
|
auto LoopHeader = CreateNewCodeBlockAfter(GetCurrentBlock());
|
|
SetCurrentCodeBlock(LoopHeader);
|
|
StartNewBlock();
|
|
SetJumpTarget(JumpIntoLoop, LoopHeader);
|
|
|
|
// Working loop
|
|
{
|
|
Ref Src_RSI = LoadGPRRegister(X86State::REG_RSI, AddrSize);
|
|
Ref Src_RDI = LoadGPRRegister(X86State::REG_RDI, AddrSize);
|
|
|
|
Ref Dest_RSI = AppendSegmentOffset(Src_RSI, Op->Flags, X86Tables::DecodeFlags::FLAG_DS_PREFIX);
|
|
Ref Dest_RDI = AppendSegmentOffset(Src_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
|
|
|
|
auto Src1 = _LoadMemGPRAutoTSO(Size, Dest_RDI, Size);
|
|
auto Src2 = _LoadMemGPR(Size, Dest_RSI, Size);
|
|
|
|
// We'll calculate PF/AF after the loop, so use them as temporaries here.
|
|
StoreRegister(Core::CPUState::PF_AS_GREG, false, Src1);
|
|
StoreRegister(Core::CPUState::AF_AS_GREG, false, Src2);
|
|
|
|
Ref TailCounter = LoadGPRRegister(X86State::REG_RCX);
|
|
|
|
// Decrement counter
|
|
TailCounter = SubWithFlags(OpSize::i64Bit, TailCounter, 1);
|
|
|
|
// Store the counter since we don't have phis
|
|
StoreGPRRegister(X86State::REG_RCX, TailCounter);
|
|
|
|
Dest_RDI = Add(AddrSize, Src_RDI, PtrDir * static_cast<int32_t>(IR::OpSizeToSize(Size)));
|
|
if (Is64BitMode && AddrSize == OpSize::i32Bit) {
|
|
Dest_RDI = _Bfe(OpSize::i64Bit, 32, 0, Dest_RDI);
|
|
StoreGPRRegister(X86State::REG_RDI, Dest_RDI);
|
|
} else {
|
|
StoreGPRRegister(X86State::REG_RDI, Dest_RDI, AddrSize);
|
|
}
|
|
|
|
Dest_RSI = Add(AddrSize, Src_RSI, PtrDir * static_cast<int32_t>(IR::OpSizeToSize(Size)));
|
|
if (Is64BitMode && AddrSize == OpSize::i32Bit) {
|
|
Dest_RSI = _Bfe(OpSize::i64Bit, 32, 0, Dest_RSI);
|
|
StoreGPRRegister(X86State::REG_RSI, Dest_RSI);
|
|
} else {
|
|
StoreGPRRegister(X86State::REG_RSI, Dest_RSI, AddrSize);
|
|
}
|
|
|
|
// If TailCounter != 0, compare sources.
|
|
// If TailCounter == 0, set ZF iff that would break.
|
|
_CondSubNZCV(OpSize::i64Bit, Src2, Src1, CondClass::NEQ, REPE ? 0 : (1 << 2) /* Z */);
|
|
CachedNZCV = nullptr;
|
|
NZCVDirty = false;
|
|
InnerJump = CondJumpNZCV(REPE ? CondClass::EQ : CondClass::NEQ);
|
|
|
|
// Jump back to the start if we have more work to do
|
|
SetTrueJumpTarget(InnerJump, LoopHeader);
|
|
}
|
|
|
|
// Make sure to start a new block after ending this one
|
|
auto LoopEnd = CreateNewCodeBlockAfter(GetCurrentBlock());
|
|
SetFalseJumpTarget(InnerJump, LoopEnd);
|
|
SetCurrentCodeBlock(LoopEnd);
|
|
StartNewBlock();
|
|
});
|
|
|
|
// Make sure to start a new block after ending this one
|
|
{
|
|
// Grab the sources from the last iteration so we can set flags.
|
|
auto Src1 = LoadGPR(Core::CPUState::PF_AS_GREG);
|
|
auto Src2 = LoadGPR(Core::CPUState::AF_AS_GREG);
|
|
CalculateFlags_SUB(OpSizeFromSrc(Op), Src2, Src1);
|
|
}
|
|
auto Jump_ = Jump();
|
|
|
|
auto Exit = CreateNewCodeBlockAfter(GetCurrentBlock());
|
|
SetJumpTarget(Jump_, Exit);
|
|
SetTrueJumpTarget(OuterJump, Exit);
|
|
SetCurrentCodeBlock(Exit);
|
|
StartNewBlock();
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::LODSOp(OpcodeArgs) {
|
|
if (!Is64BitMode && (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE)) {
|
|
LogMan::Msg::EFmt("LODSOp: Address size override (0x67) not supported in 32-bit mode (OP: 0x{:04X}).", Op->OP);
|
|
DecodeFailure = true;
|
|
return;
|
|
}
|
|
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
OpSize AddrSize = GetStringOpSize(Op);
|
|
|
|
const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) != 0;
|
|
|
|
if (!Repeat) {
|
|
Ref Src_RSI = LoadGPRRegister(X86State::REG_RSI, AddrSize);
|
|
Ref Dest_RSI = AppendSegmentOffset(Src_RSI, 0, X86Tables::DecodeFlags::FLAG_DS_PREFIX, true);
|
|
|
|
auto Src = _LoadMemGPRAutoTSO(Size, Dest_RSI, Size);
|
|
|
|
StoreResultGPR(Op, Src);
|
|
|
|
// Offset the pointer
|
|
Ref TailDest_RSI = OffsetByDir(Src_RSI, IR::OpSizeToSize(Size));
|
|
if (Is64BitMode && AddrSize == OpSize::i32Bit) {
|
|
TailDest_RSI = _Bfe(OpSize::i64Bit, 32, 0, TailDest_RSI);
|
|
StoreGPRRegister(X86State::REG_RSI, TailDest_RSI);
|
|
} else {
|
|
StoreGPRRegister(X86State::REG_RSI, TailDest_RSI, AddrSize);
|
|
}
|
|
} else {
|
|
// Calculate flags early. because end of block
|
|
CalculateDeferredFlags();
|
|
|
|
ForeachDirection([this, Op, Size, AddrSize](int32_t PtrDir) {
|
|
// XXX: Theoretically LODS could be optimized to
|
|
// RSI += {-}(RCX * Size)
|
|
// RAX = [RSI - Size]
|
|
// But this might violate the case of an application scanning pages for read permission and catching the fault
|
|
// May or may not matter
|
|
|
|
auto JumpStart = Jump();
|
|
// Make sure to start a new block after ending this one
|
|
auto LoopStart = CreateNewCodeBlockAfter(GetCurrentBlock());
|
|
SetJumpTarget(JumpStart, LoopStart);
|
|
SetCurrentCodeBlock(LoopStart);
|
|
StartNewBlock();
|
|
|
|
Ref Counter = LoadGPRRegister(X86State::REG_RCX);
|
|
|
|
// Can we end the block?
|
|
|
|
// We leave if RCX = 0
|
|
auto CondJump_ = CondJump(Counter, CondClass::EQ);
|
|
|
|
auto LoopTail = CreateNewCodeBlockAfter(LoopStart);
|
|
SetFalseJumpTarget(CondJump_, LoopTail);
|
|
SetCurrentCodeBlock(LoopTail);
|
|
StartNewBlock();
|
|
|
|
// Working loop
|
|
{
|
|
Ref Src_RSI = LoadGPRRegister(X86State::REG_RSI, AddrSize);
|
|
Ref Dest_RSI = AppendSegmentOffset(Src_RSI, 0, X86Tables::DecodeFlags::FLAG_DS_PREFIX, true);
|
|
|
|
auto Src = _LoadMemGPRAutoTSO(Size, Dest_RSI, Size);
|
|
|
|
StoreResultGPR(Op, Src);
|
|
|
|
Ref TailCounter = LoadGPRRegister(X86State::REG_RCX);
|
|
Ref TailDest_RSI = LoadGPRRegister(X86State::REG_RSI);
|
|
|
|
// Decrement counter
|
|
TailCounter = Sub(OpSize::i64Bit, TailCounter, 1);
|
|
|
|
// Store the counter since we don't have phis
|
|
StoreGPRRegister(X86State::REG_RCX, TailCounter);
|
|
|
|
// Offset the pointer
|
|
TailDest_RSI = Add(AddrSize, TailDest_RSI, PtrDir * static_cast<int32_t>(IR::OpSizeToSize(Size)));
|
|
if (Is64BitMode && AddrSize == OpSize::i32Bit) {
|
|
TailDest_RSI = _Bfe(OpSize::i64Bit, 32, 0, TailDest_RSI);
|
|
StoreGPRRegister(X86State::REG_RSI, TailDest_RSI);
|
|
} else {
|
|
StoreGPRRegister(X86State::REG_RSI, TailDest_RSI, AddrSize);
|
|
}
|
|
|
|
// Jump back to the start, we have more work to do
|
|
Jump(LoopStart);
|
|
}
|
|
// Make sure to start a new block after ending this one
|
|
auto LoopEnd = CreateNewCodeBlockAfter(LoopTail);
|
|
SetTrueJumpTarget(CondJump_, LoopEnd);
|
|
SetCurrentCodeBlock(LoopEnd);
|
|
StartNewBlock();
|
|
});
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::SCASOp(OpcodeArgs) {
|
|
if (!Is64BitMode && (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE)) {
|
|
LogMan::Msg::EFmt("SCASOp: Address size override (0x67) not supported in 32-bit mode (OP: 0x{:04X}).", Op->OP);
|
|
DecodeFailure = true;
|
|
return;
|
|
}
|
|
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
OpSize AddrSize = GetStringOpSize(Op);
|
|
const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX)) != 0;
|
|
|
|
if (!Repeat) {
|
|
Ref Src_RDI = LoadGPRRegister(X86State::REG_RDI, AddrSize);
|
|
Ref Dest_RDI = AppendSegmentOffset(Src_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
|
|
|
|
auto Src1 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Src2 = _LoadMemGPRAutoTSO(Size, Dest_RDI, Size);
|
|
|
|
CalculateFlags_SUB(OpSizeFromSrc(Op), Src1, Src2);
|
|
|
|
Ref TailDest_RDI = OffsetByDir(Src_RDI, IR::OpSizeToSize(Size));
|
|
if (Is64BitMode && AddrSize == OpSize::i32Bit) {
|
|
TailDest_RDI = _Bfe(OpSize::i64Bit, 32, 0, TailDest_RDI);
|
|
StoreGPRRegister(X86State::REG_RDI, TailDest_RDI);
|
|
} else {
|
|
StoreGPRRegister(X86State::REG_RDI, TailDest_RDI, AddrSize);
|
|
}
|
|
} else {
|
|
// Calculate flags early. because end of block
|
|
CalculateDeferredFlags();
|
|
|
|
ForeachDirection([this, Op, Size, AddrSize](int32_t Dir) {
|
|
bool REPE = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX;
|
|
|
|
auto JumpStart = Jump();
|
|
// Make sure to start a new block after ending this one
|
|
auto LoopStart = CreateNewCodeBlockAfter(GetCurrentBlock());
|
|
SetJumpTarget(JumpStart, LoopStart);
|
|
SetCurrentCodeBlock(LoopStart);
|
|
StartNewBlock();
|
|
|
|
Ref Counter = LoadGPRRegister(X86State::REG_RCX);
|
|
|
|
// Can we end the block?
|
|
// We leave if RCX = 0
|
|
auto CondJump_ = CondJump(Counter, CondClass::EQ);
|
|
IRPair<IROp_CondJump> InternalCondJump;
|
|
|
|
auto LoopTail = CreateNewCodeBlockAfter(LoopStart);
|
|
SetFalseJumpTarget(CondJump_, LoopTail);
|
|
SetCurrentCodeBlock(LoopTail);
|
|
StartNewBlock();
|
|
|
|
// Working loop
|
|
{
|
|
Ref Src_RDI = LoadGPRRegister(X86State::REG_RDI, AddrSize);
|
|
Ref Dest_RDI = AppendSegmentOffset(Src_RDI, 0, X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
|
|
|
|
auto Src1 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
auto Src2 = _LoadMemGPRAutoTSO(Size, Dest_RDI, Size);
|
|
|
|
CalculateFlags_SUB(OpSizeFromSrc(Op), Src1, Src2);
|
|
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
Ref TailCounter = LoadGPRRegister(X86State::REG_RCX);
|
|
Ref Src_RDI_Tail = LoadGPRRegister(X86State::REG_RDI, AddrSize);
|
|
|
|
// Decrement counter
|
|
TailCounter = Sub(OpSize::i64Bit, TailCounter, 1);
|
|
|
|
// Store the counter since we don't have phis
|
|
StoreGPRRegister(X86State::REG_RCX, TailCounter);
|
|
|
|
Ref TailDest_RDI = Add(AddrSize, Src_RDI_Tail, Dir * static_cast<int32_t>(IR::OpSizeToSize(Size)));
|
|
if (Is64BitMode && AddrSize == OpSize::i32Bit) {
|
|
TailDest_RDI = _Bfe(OpSize::i64Bit, 32, 0, TailDest_RDI);
|
|
StoreGPRRegister(X86State::REG_RDI, TailDest_RDI);
|
|
} else {
|
|
StoreGPRRegister(X86State::REG_RDI, TailDest_RDI, AddrSize);
|
|
}
|
|
|
|
CalculateDeferredFlags();
|
|
InternalCondJump = CondJumpNZCV(REPE ? CondClass::EQ : CondClass::NEQ);
|
|
|
|
// Jump back to the start if we have more work to do
|
|
SetTrueJumpTarget(InternalCondJump, LoopStart);
|
|
}
|
|
// Make sure to start a new block after ending this one
|
|
auto LoopEnd = CreateNewCodeBlockAfter(LoopTail);
|
|
SetTrueJumpTarget(CondJump_, LoopEnd);
|
|
|
|
SetFalseJumpTarget(InternalCondJump, LoopEnd);
|
|
|
|
SetCurrentCodeBlock(LoopEnd);
|
|
StartNewBlock();
|
|
});
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::BSWAPOp(OpcodeArgs) {
|
|
Ref Dest;
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
if (Size == OpSize::i16Bit) {
|
|
// BSWAP of 16bit is undef. ZEN+ causes the lower 16bits to get zero'd
|
|
Dest = Constant(0);
|
|
} else {
|
|
Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, GetGPROpSize(), Op->Flags);
|
|
Dest = _Rev(Size, Dest);
|
|
}
|
|
StoreResultGPR(Op, Dest);
|
|
}
|
|
|
|
void OpDispatchBuilder::PUSHFOp(OpcodeArgs) {
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
|
|
Push(Size, GetPackedRFLAG());
|
|
}
|
|
|
|
void OpDispatchBuilder::POPFOp(OpcodeArgs) {
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
Ref Src = Pop(Size);
|
|
|
|
// Add back our flag constants
|
|
// Bit 1 is always 1
|
|
// Bit 9 is always 1 because we always have interrupts enabled
|
|
|
|
Src = _Or(OpSize::i64Bit, Src, Constant(0x202));
|
|
|
|
SetPackedRFLAG(false, Src);
|
|
|
|
auto NewRIP = GetRelocatedPC(Op);
|
|
ExitFunction(NewRIP, BranchHint::CheckTF);
|
|
BlockSetRIP = true;
|
|
}
|
|
|
|
void OpDispatchBuilder::NEGOp(OpcodeArgs) {
|
|
HandledLock = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;
|
|
|
|
const auto Size = OpSizeFromSrc(Op);
|
|
auto ZeroConst = Constant(0);
|
|
|
|
if (DestIsLockedMem(Op)) {
|
|
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
|
|
Ref Dest = _AtomicFetchNeg(Size, DestMem);
|
|
CalculateFlags_SUB(Size, ZeroConst, Dest);
|
|
} else {
|
|
Ref Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Result = CalculateFlags_SUB(Size, ZeroConst, Dest);
|
|
|
|
StoreResultGPR(Op, Result);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::DIVOp(OpcodeArgs) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
auto Size = OpSizeFromSrc(Op);
|
|
|
|
// This loads the divisor. 32-bit/64-bit paths mask inside the JIT, 8/16 do not.
|
|
Ref Divisor = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = Size >= OpSize::i32Bit});
|
|
|
|
if (Size == OpSize::i64Bit && !Is64BitMode) {
|
|
LogMan::Msg::EFmt("Doesn't exist in 32bit mode");
|
|
DecodeFailure = true;
|
|
return;
|
|
}
|
|
|
|
Ref Quotient = _AllocateGPR(true);
|
|
Ref Remainder = _AllocateGPR(true);
|
|
|
|
if (Size == OpSize::i8Bit) {
|
|
Ref Src1 = LoadGPRRegister(X86State::REG_RAX, OpSize::i16Bit);
|
|
|
|
_UDiv(OpSize::i16Bit, Src1, Invalid(), Divisor, Quotient, Remainder);
|
|
|
|
// AX[15:0] = concat<URem[7:0]:UDiv[7:0]>
|
|
auto ResultAX = _Bfi(GPRSize, 8, 8, Quotient, Remainder);
|
|
StoreGPRRegister(X86State::REG_RAX, ResultAX, OpSize::i16Bit);
|
|
} else {
|
|
Ref Src1 = LoadGPRRegister(X86State::REG_RAX);
|
|
Ref Src2 = LoadGPRRegister(X86State::REG_RDX);
|
|
|
|
_UDiv(Size, Src1, Src2, Divisor, Quotient, Remainder);
|
|
|
|
if (Size == OpSize::i32Bit) {
|
|
Quotient = _Bfe(OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, Quotient);
|
|
Remainder = _Bfe(OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, Remainder);
|
|
Size = OpSize::iInvalid;
|
|
}
|
|
|
|
StoreGPRRegister(X86State::REG_RAX, Quotient, Size);
|
|
StoreGPRRegister(X86State::REG_RDX, Remainder, Size);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::IDIVOp(OpcodeArgs) {
|
|
// This loads the divisor
|
|
Ref Divisor = LoadSourceGPR(Op, Op->Dest, Op->Flags);
|
|
|
|
const auto GPRSize = GetGPROpSize();
|
|
auto Size = OpSizeFromSrc(Op);
|
|
|
|
if (Size == OpSize::i64Bit && !Is64BitMode) {
|
|
LogMan::Msg::EFmt("Doesn't exist in 32bit mode");
|
|
DecodeFailure = true;
|
|
return;
|
|
}
|
|
|
|
Ref Quotient = _AllocateGPR(true);
|
|
Ref Remainder = _AllocateGPR(true);
|
|
|
|
if (Size == OpSize::i8Bit) {
|
|
Ref Src1 = LoadGPRRegister(X86State::REG_RAX);
|
|
Src1 = _Sbfe(OpSize::i64Bit, 16, 0, Src1);
|
|
Divisor = _Sbfe(OpSize::i64Bit, 8, 0, Divisor);
|
|
|
|
_Div(OpSize::i64Bit, Src1, Invalid(), Divisor, Quotient, Remainder);
|
|
|
|
// AX[15:0] = concat<URem[7:0]:UDiv[7:0]>
|
|
auto ResultAX = _Bfi(GPRSize, 8, 8, Quotient, Remainder);
|
|
StoreGPRRegister(X86State::REG_RAX, ResultAX, OpSize::i16Bit);
|
|
} else {
|
|
Ref Src1 = LoadGPRRegister(X86State::REG_RAX);
|
|
Ref Src2 = LoadGPRRegister(X86State::REG_RDX);
|
|
|
|
_Div(Size, Src1, Src2, Divisor, Quotient, Remainder);
|
|
|
|
if (Size == OpSize::i32Bit) {
|
|
Quotient = _Bfe(OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, Quotient);
|
|
Remainder = _Bfe(OpSize::i32Bit, IR::OpSizeAsBits(Size), 0, Remainder);
|
|
Size = OpSize::iInvalid;
|
|
}
|
|
|
|
StoreGPRRegister(X86State::REG_RAX, Quotient, Size);
|
|
StoreGPRRegister(X86State::REG_RDX, Remainder, Size);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::BSFOp(OpcodeArgs) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
const auto DstSize = OpSizeFromDst(Op) == OpSize::i16Bit ? OpSize::i16Bit : GPRSize;
|
|
Ref Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, DstSize, Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// Find the LSB of this source
|
|
auto Result = _FindLSB(OpSizeFromSrc(Op), Src);
|
|
|
|
// OF, SF, AF, PF, CF all undefined
|
|
// ZF is set to 1 if the source was zero
|
|
SetZ_InvalidateNCV(OpSizeFromSrc(Op), Src);
|
|
|
|
// If Src was zero then the destination doesn't get modified.
|
|
//
|
|
// Although Intel does not guarantee that semantic, AMD does and Intel
|
|
// hardware satisfies it. We provide the stronger AMD behaviour as
|
|
// applications might rely on that in the wild.
|
|
auto SelectOp = NZCVSelect(GPRSize, CondClass::EQ, Dest, Result);
|
|
StoreResultGPR_WithOpSize(Op, Op->Dest, SelectOp, DstSize);
|
|
}
|
|
|
|
void OpDispatchBuilder::BSROp(OpcodeArgs) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
const auto DstSize = OpSizeFromDst(Op) == OpSize::i16Bit ? OpSize::i16Bit : GPRSize;
|
|
Ref Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, DstSize, Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// Find the MSB of this source
|
|
auto Result = _FindMSB(OpSizeFromSrc(Op), Src);
|
|
|
|
// OF, SF, AF, PF, CF all undefined
|
|
// ZF is set to 1 if the source was zero
|
|
SetZ_InvalidateNCV(OpSizeFromSrc(Op), Src);
|
|
|
|
// If Src was zero then the destination doesn't get modified
|
|
auto SelectOp = NZCVSelect(GPRSize, CondClass::EQ, Dest, Result);
|
|
StoreResultGPR_WithOpSize(Op, Op->Dest, SelectOp, DstSize);
|
|
}
|
|
|
|
void OpDispatchBuilder::CMPXCHGOp(OpcodeArgs) {
|
|
// CMPXCHG ModRM, reg, {RAX}
|
|
// MemData = *ModRM.dest
|
|
// if (RAX == MemData)
|
|
// modRM.dest = reg;
|
|
// ZF = 1
|
|
// else
|
|
// ZF = 0
|
|
// RAX = MemData
|
|
//
|
|
// CASL Xs, Xt, Xn
|
|
// MemData = *Xn
|
|
// if (MemData == Xs)
|
|
// *Xn = Xt
|
|
// Xs = MemData
|
|
|
|
const auto GPRSize = GetGPROpSize();
|
|
auto Size = OpSizeFromSrc(Op);
|
|
|
|
if (Op->Dest.IsGPR()) {
|
|
// This is our source register
|
|
Ref Src2 = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
Ref Src3 = LoadGPRRegister(X86State::REG_RAX);
|
|
|
|
// If the destination is also the accumulator, we get some algebraic
|
|
// simplifications. Not sure if this is actually hit but it's in
|
|
// InstCountCI.
|
|
bool Trivial = Op->Dest.Data.GPR.GPR == X86State::REG_RAX && !Op->Dest.IsGPRDirect() && !Op->Dest.Data.GPR.HighBits;
|
|
|
|
Ref Src1 {};
|
|
Ref Src1Lower {};
|
|
|
|
if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {
|
|
Src1 = LoadSourceGPR_WithOpSize(Op, Op->Dest, GPRSize, Op->Flags, {.AllowUpperGarbage = true});
|
|
Src1Lower = Trivial ? Src1 : _Bfe(GPRSize, IR::OpSizeAsBits(Size), 0, Src1);
|
|
} else {
|
|
Src1 = LoadSourceGPR_WithOpSize(Op, Op->Dest, Size, Op->Flags, {.AllowUpperGarbage = true});
|
|
Src1Lower = Src1;
|
|
}
|
|
|
|
// Compare RAX with the destination, setting flags accordingly.
|
|
CalculateFlags_SUB(OpSizeFromSrc(Op), Src3, Src1Lower);
|
|
CalculateDeferredFlags();
|
|
|
|
if (!Trivial) {
|
|
if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {
|
|
// This allows us to only hit the ZEXT case on failure
|
|
Ref RAXResult = NZCVSelect(OpSize::i64Bit, CondClass::EQ, Src3, Src1Lower);
|
|
|
|
StoreGPRRegister(X86State::REG_RAX, RAXResult);
|
|
} else {
|
|
StoreGPRRegister(X86State::REG_RAX, Src1Lower, Size);
|
|
}
|
|
}
|
|
|
|
// Op1 = RAX == Op1 ? Op2 : Op1
|
|
// If they match then set the rm operand to the input
|
|
// else don't set the rm operand
|
|
Ref Src2Lower = Src2;
|
|
if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {
|
|
Src2Lower = _Bfe(GPRSize, IR::OpSizeAsBits(Size), 0, Src2);
|
|
}
|
|
Ref DestResult = Trivial ? Src2Lower : NZCVSelect(OpSize::i64Bit, CondClass::EQ, Src2Lower, Src1);
|
|
|
|
// Store in to GPR Dest
|
|
if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {
|
|
StoreResultGPR_WithOpSize(Op, Op->Dest, DestResult, GPRSize);
|
|
} else {
|
|
StoreResultGPR(Op, DestResult);
|
|
}
|
|
} else {
|
|
Ref Src2 = LoadSourceGPR(Op, Op->Src[0], Op->Flags);
|
|
HandledLock = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK;
|
|
|
|
auto Src3 = LoadGPRRegister(X86State::REG_RAX);
|
|
auto Src3Lower = _Bfe(OpSize::i64Bit, OpSizeAsBits(Size), 0, Src3);
|
|
|
|
// If this is a memory location then we want the pointer to it
|
|
Ref Src1 = MakeSegmentAddress(Op, Op->Dest);
|
|
|
|
// DataSrc = *Src1
|
|
// if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc
|
|
// This will write to memory! Careful!
|
|
// Third operand must be a calculated guest memory address
|
|
Ref CASResult = _CAS(Size, Src3, Src2, Src1);
|
|
Ref RAXResult = CASResult;
|
|
|
|
CalculateFlags_SUB(OpSizeFromSrc(Op), Src3Lower, CASResult);
|
|
CalculateDeferredFlags();
|
|
|
|
if (GPRSize == OpSize::i64Bit && Size == OpSize::i32Bit) {
|
|
// This allows us to only hit the ZEXT case on failure
|
|
RAXResult = _NZCVSelect(OpSize::i64Bit, CondClass::EQ, Src3, CASResult);
|
|
Size = OpSize::i64Bit;
|
|
}
|
|
|
|
// RAX gets the result of the CAS op
|
|
StoreGPRRegister(X86State::REG_RAX, RAXResult, Size);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::CMPXCHGPairOp(OpcodeArgs) {
|
|
// Calculate flags early.
|
|
CalculateDeferredFlags();
|
|
|
|
// REX.W used to determine if it is 16byte or 8byte
|
|
// Unlike CMPXCHG, the destination can only be a memory location
|
|
const auto Size = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;
|
|
|
|
HandledLock = (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;
|
|
|
|
// If this is a memory location then we want the pointer to it
|
|
Ref Src1 = MakeSegmentAddress(Op, Op->Dest);
|
|
|
|
// Load the full 64-bit registers, all the users ignore the upper 32-bits for
|
|
// 32-bit only cmpxchg. This saves some zero extension.
|
|
Ref Expected_Lower = LoadGPRRegister(X86State::REG_RAX);
|
|
Ref Expected_Upper = LoadGPRRegister(X86State::REG_RDX);
|
|
Ref Desired_Lower = LoadGPRRegister(X86State::REG_RBX);
|
|
Ref Desired_Upper = LoadGPRRegister(X86State::REG_RCX);
|
|
|
|
// ssa0 = Expected
|
|
// ssa1 = Desired
|
|
// ssa2 = MemoryLocation
|
|
|
|
// DataSrc = *MemSrc
|
|
// if (DataSrc == Expected) { *MemSrc == Desired; } Expected = DataSrc
|
|
// This will write to memory! Careful!
|
|
// Third operand must be a calculated guest memory address
|
|
|
|
Ref Result_Lower = _AllocateGPR(true);
|
|
Ref Result_Upper = _AllocateGPRAfter(Result_Lower);
|
|
_CASPair(Size, Expected_Lower, Expected_Upper, Desired_Lower, Desired_Upper, Src1, Result_Lower, Result_Upper);
|
|
|
|
HandleNZCV_RMW();
|
|
_CmpPairZ(Size, Result_Lower, Result_Upper, Expected_Lower, Expected_Upper);
|
|
CalculateDeferredFlags();
|
|
|
|
auto UpdateIfNotZF = [this](auto Reg, auto Value) {
|
|
// Always use 64-bit csel to preserve existing upper bits. If we have a
|
|
// 32-bit cmpxchg in a 64-bit context, Value will be zeroed in upper bits.
|
|
StoreGPRRegister(Reg, NZCVSelect(OpSize::i64Bit, CondClass::NEQ, Value, LoadGPRRegister(Reg)));
|
|
};
|
|
|
|
UpdateIfNotZF(X86State::REG_RAX, Result_Lower);
|
|
UpdateIfNotZF(X86State::REG_RDX, Result_Upper);
|
|
}
|
|
|
|
void OpDispatchBuilder::CreateJumpBlocks(const fextl::vector<FEXCore::Frontend::Decoder::DecodedBlocks>* Blocks) {
|
|
Ref PrevCodeBlock {};
|
|
for (auto& Target : *Blocks) {
|
|
auto CodeNode = CreateCodeNode(Target.IsEntryPoint, Target.Entry - Entry);
|
|
|
|
JumpTargets.try_emplace(Target.Entry, JumpTargetInfo {CodeNode, false, Target.IsEntryPoint});
|
|
|
|
if (PrevCodeBlock) {
|
|
LinkCodeBlocks(PrevCodeBlock, CodeNode);
|
|
}
|
|
|
|
PrevCodeBlock = CodeNode;
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::BeginFunction(uint64_t RIP, const fextl::vector<FEXCore::Frontend::Decoder::DecodedBlocks>* Blocks,
|
|
uint32_t NumInstructions, bool _Is64BitMode, bool MonoBackpatcherBlock) {
|
|
Entry = RIP;
|
|
Is64BitMode = _Is64BitMode;
|
|
LOGMAN_THROW_A_FMT(Is64BitMode == CTX->Config.Is64BitMode, "Expected operating mode to not change at runtime!");
|
|
IsMonoBackpatcherBlock = MonoBackpatcherBlock;
|
|
auto IRHeader = _IRHeader(InvalidNode, RIP, 0, NumInstructions, 0, 0);
|
|
CreateJumpBlocks(Blocks);
|
|
|
|
auto Block = GetNewJumpBlock(RIP);
|
|
SetCurrentCodeBlock(Block);
|
|
IRHeader.first->Blocks = Block->Wrapped(DualListData.ListBegin());
|
|
CurrentHeader = IRHeader.first;
|
|
}
|
|
|
|
void OpDispatchBuilder::Finalize() {
|
|
// This usually doesn't emit any IR but in the case of hitting the block instruction limit it will
|
|
FlushRegisterCache();
|
|
const auto GPRSize = GetGPROpSize();
|
|
|
|
// Node 0 is invalid node
|
|
Ref RealNode = reinterpret_cast<Ref>(GetNode(1));
|
|
|
|
const FEXCore::IR::IROp_Header* IROp = RealNode->Op(DualListData.DataBegin());
|
|
LOGMAN_THROW_A_FMT(IROp->Op == OP_IRHEADER, "First op in function must be our header");
|
|
|
|
// Let's walk the jump blocks and see if we have handled every block target
|
|
for (auto& Handler : JumpTargets) {
|
|
if (Handler.second.HaveEmitted) {
|
|
continue;
|
|
}
|
|
|
|
// We haven't emitted. Dump out to the dispatcher
|
|
SetCurrentCodeBlock(Handler.second.BlockEntry);
|
|
ExitFunction(_InlineEntrypointOffset(GPRSize, Handler.first - Entry));
|
|
}
|
|
}
|
|
|
|
uint8_t OpDispatchBuilder::GetDstSize(X86Tables::DecodedOp Op) const {
|
|
const uint32_t DstSizeFlag = X86Tables::DecodeFlags::GetSizeDstFlags(Op->Flags);
|
|
LOGMAN_THROW_A_FMT(DstSizeFlag != 0 && DstSizeFlag != X86Tables::DecodeFlags::SIZE_MASK, "Invalid destination size for op");
|
|
return 1u << (DstSizeFlag - 1);
|
|
}
|
|
|
|
uint8_t OpDispatchBuilder::GetSrcSize(X86Tables::DecodedOp Op) const {
|
|
const uint32_t SrcSizeFlag = X86Tables::DecodeFlags::GetSizeSrcFlags(Op->Flags);
|
|
LOGMAN_THROW_A_FMT(SrcSizeFlag != 0 && SrcSizeFlag != X86Tables::DecodeFlags::SIZE_MASK, "Invalid destination size for op");
|
|
return 1u << (SrcSizeFlag - 1);
|
|
}
|
|
|
|
uint32_t OpDispatchBuilder::GetSrcBitSize(X86Tables::DecodedOp Op) const {
|
|
return GetSrcSize(Op) * 8;
|
|
}
|
|
|
|
uint32_t OpDispatchBuilder::GetDstBitSize(X86Tables::DecodedOp Op) const {
|
|
return GetDstSize(Op) * 8;
|
|
}
|
|
|
|
Ref OpDispatchBuilder::GetSegment(uint32_t Flags, uint32_t DefaultPrefix, bool Override) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
uint32_t Prefix = Flags & FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS;
|
|
|
|
if (Is64BitMode) {
|
|
if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX) {
|
|
return _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, fs_cached));
|
|
} else if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {
|
|
return _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, gs_cached));
|
|
}
|
|
// If there was any other segment in 64bit then it is ignored
|
|
} else {
|
|
if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX || Override) {
|
|
// If there was no prefix then use the default one if available
|
|
// Or the argument only uses a specific prefix (with override set)
|
|
Prefix = DefaultPrefix;
|
|
}
|
|
// With the segment register optimization we store the GDT bases directly in the segment register to remove indexed loads
|
|
Ref SegmentResult {};
|
|
switch (Prefix) {
|
|
[[likely]] case FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX:
|
|
return nullptr;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
|
|
SegmentResult = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, es_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
|
|
SegmentResult = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, cs_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
|
|
SegmentResult = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, ss_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
|
|
SegmentResult = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, ds_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:
|
|
SegmentResult = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, fs_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:
|
|
SegmentResult = _LoadContextGPR(GPRSize, offsetof(FEXCore::Core::CPUState, gs_cached));
|
|
break;
|
|
default: FEX_UNREACHABLE;
|
|
}
|
|
|
|
CheckLegacySegmentRead(SegmentResult, Prefix);
|
|
return SegmentResult;
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
Ref OpDispatchBuilder::AppendSegmentOffset(Ref Value, uint32_t Flags, uint32_t DefaultPrefix, bool Override) {
|
|
auto Segment = GetSegment(Flags, DefaultPrefix, Override);
|
|
if (Segment) {
|
|
Value = Add(std::max(OpSize::i32Bit, std::max(GetOpSize(Value), GetOpSize(Segment))), Value, Segment);
|
|
}
|
|
|
|
return Value;
|
|
}
|
|
|
|
|
|
void OpDispatchBuilder::CheckLegacySegmentRead(Ref NewNode, uint32_t SegmentReg) {
|
|
#ifndef FEX_DISABLE_TELEMETRY
|
|
if (SegmentReg == FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX || SegmentReg == FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {
|
|
// FS and GS segments aren't considered legacy.
|
|
return;
|
|
}
|
|
|
|
if (!(SegmentsNeedReadCheck & SegmentReg)) {
|
|
// If the block has done multiple reads of a segment register then skip redundant read checks.
|
|
// Segment write will cause another read check.
|
|
return;
|
|
}
|
|
|
|
if (CTX->Config.DisableTelemetry()) {
|
|
// Telemetry disabled at runtime.
|
|
return;
|
|
}
|
|
|
|
FEXCore::Telemetry::TelemetryType TelemIndex {};
|
|
switch (SegmentReg) {
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
|
|
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_USES_32BIT_SEGMENT_ES;
|
|
SegmentsNeedReadCheck &= ~FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX;
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
|
|
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_USES_32BIT_SEGMENT_CS;
|
|
SegmentsNeedReadCheck &= ~FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX;
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
|
|
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_USES_32BIT_SEGMENT_SS;
|
|
SegmentsNeedReadCheck &= ~FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX;
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
|
|
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_USES_32BIT_SEGMENT_DS;
|
|
SegmentsNeedReadCheck &= ~FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX;
|
|
break;
|
|
default: FEX_UNREACHABLE;
|
|
}
|
|
|
|
// Will set the telemetry value if NewNode is != 0
|
|
_TelemetrySetValue(NewNode, TelemIndex);
|
|
// Telemetry will dirty flags, and user code does not expect LoadSource to clobber flags, fix that up here as this is an edge case.
|
|
CalculateDeferredFlags();
|
|
#endif
|
|
}
|
|
|
|
void OpDispatchBuilder::CheckLegacySegmentWrite(Ref NewNode, uint32_t SegmentReg) {
|
|
#ifndef FEX_DISABLE_TELEMETRY
|
|
if (SegmentReg == FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX || SegmentReg == FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {
|
|
// FS and GS segments aren't considered legacy.
|
|
return;
|
|
}
|
|
|
|
if (CTX->Config.DisableTelemetry()) {
|
|
// Telemetry disabled at runtime.
|
|
return;
|
|
}
|
|
|
|
FEXCore::Telemetry::TelemetryType TelemIndex {};
|
|
switch (SegmentReg) {
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
|
|
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_ES;
|
|
SegmentsNeedReadCheck |= FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX;
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
|
|
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_CS;
|
|
SegmentsNeedReadCheck |= FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX;
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
|
|
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_SS;
|
|
SegmentsNeedReadCheck |= FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX;
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
|
|
TelemIndex = FEXCore::Telemetry::TelemetryType::TYPE_WRITES_32BIT_SEGMENT_DS;
|
|
SegmentsNeedReadCheck |= FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX;
|
|
break;
|
|
default: FEX_UNREACHABLE;
|
|
}
|
|
|
|
// Will set the telemetry value if NewNode is != 0
|
|
_TelemetrySetValue(NewNode, TelemIndex);
|
|
// Telemetry will dirty flags, and user code does not expect LoadSource to clobber flags, fix that up here as this is an edge case.
|
|
CalculateDeferredFlags();
|
|
#endif
|
|
}
|
|
|
|
void OpDispatchBuilder::UpdatePrefixFromSegment(Ref Segment, uint32_t SegmentReg) {
|
|
// Use BFE to extract the selector index in bits [15,3] of the segment register.
|
|
// In some cases the upper 16-bits of the 32-bit GPR contain garbage to ignore.
|
|
auto GDT = _Bfe(OpSize::i32Bit, 1, 2, Segment);
|
|
// Fun quirk, if we mask the selector then it is premultiplied by 8 which we need to do for accessing anyway.
|
|
auto SegmentOffset = _And(OpSize::i32Bit, Segment, _Constant(0xfff8));
|
|
Ref SegmentBase = _LoadContextGPRIndexed(GDT, OpSize::i64Bit, offsetof(FEXCore::Core::CPUState, segment_arrays[0]), 8);
|
|
Ref NewSegment = _LoadMemGPR(OpSize::i64Bit, SegmentBase, SegmentOffset, OpSize::i8Bit, MemOffsetType::UXTW, 1);
|
|
CheckLegacySegmentWrite(NewSegment, SegmentReg);
|
|
|
|
// Extract the 32-bit base from the GDT segment.
|
|
auto Upper32 = _Lshr(OpSize::i64Bit, NewSegment, _Constant(32));
|
|
auto Masked = _And(OpSize::i32Bit, Upper32, _Constant(0xFF00'0000));
|
|
Ref Merged = _Orlshr(OpSize::i32Bit, Masked, NewSegment, 16);
|
|
NewSegment = _Bfi(OpSize::i32Bit, 8, 16, Merged, Upper32);
|
|
|
|
switch (SegmentReg) {
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
|
|
_StoreContextGPR(OpSize::i32Bit, NewSegment, offsetof(FEXCore::Core::CPUState, es_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
|
|
_StoreContextGPR(OpSize::i32Bit, NewSegment, offsetof(FEXCore::Core::CPUState, cs_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
|
|
_StoreContextGPR(OpSize::i32Bit, NewSegment, offsetof(FEXCore::Core::CPUState, ss_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
|
|
_StoreContextGPR(OpSize::i32Bit, NewSegment, offsetof(FEXCore::Core::CPUState, ds_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:
|
|
_StoreContextGPR(OpSize::i32Bit, NewSegment, offsetof(FEXCore::Core::CPUState, fs_cached));
|
|
break;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:
|
|
_StoreContextGPR(OpSize::i32Bit, NewSegment, offsetof(FEXCore::Core::CPUState, gs_cached));
|
|
break;
|
|
default: break; // Do nothing
|
|
}
|
|
}
|
|
|
|
uint64_t OpDispatchBuilder::CalcAddress(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, bool IsLoad) {
|
|
if constexpr (!Context::BLOCK_DEBUGGING) {
|
|
LOGMAN_MSG_A_FMT("Tried to calculate address without block debugging enabled!");
|
|
FEX_UNREACHABLE;
|
|
}
|
|
|
|
const auto GPRSize = GetGPROpSize();
|
|
const auto GPRMask = GPRSize == OpSize::i64Bit ? ~0ULL : ~0U;
|
|
|
|
// This makes the assumption that InternalThreadState is synchronized at the point of call!
|
|
uint64_t Ptr {};
|
|
if (Operand.IsLiteral()) {
|
|
Ptr = Operand.Literal();
|
|
|
|
if (Operand.Data.Literal.Size != 8 && IsLoad) {
|
|
// zero extend
|
|
uint64_t width = Operand.Data.Literal.Size * 8;
|
|
Ptr &= ((1ULL << width) - 1);
|
|
}
|
|
} else if (Operand.IsGPR()) {
|
|
// Not a memory source.
|
|
return ~0ULL;
|
|
} else if (Operand.IsGPRDirect()) {
|
|
Ptr = Thread->CurrentFrame->State.gregs[Operand.Data.GPR.GPR] & GPRMask;
|
|
} else if (Operand.IsGPRIndirect() || Operand.IsGPRIndirectRelocation()) {
|
|
Ptr = Thread->CurrentFrame->State.gregs[Operand.Data.GPR.GPR] & GPRMask;
|
|
Ptr += static_cast<int32_t>(Operand.Data.GPRIndirect.Displacement);
|
|
} else if (Operand.IsRIPRelative() || Operand.IsRIPRelativeRelocation()) {
|
|
// 64-bit is RIP relative, while 32-bit is absolute.
|
|
if (Is64BitMode) {
|
|
Ptr = Op->PC + Op->InstSize + static_cast<int32_t>(Operand.Data.RIPLiteral.Value) - Entry;
|
|
} else {
|
|
Ptr = Operand.Data.RIPLiteral.Value;
|
|
}
|
|
} else if (Operand.IsSIB() || Operand.IsSIBRelocation()) {
|
|
const bool IsVSIB = IsLoad && ((Op->Flags & X86Tables::DecodeFlags::FLAG_VSIB_BYTE) != 0);
|
|
if (IsVSIB) {
|
|
// TODO: Unhandled.
|
|
return ~0ULL;
|
|
}
|
|
if (Operand.Data.SIB.Base != FEXCore::X86State::REG_INVALID) {
|
|
Ptr = Thread->CurrentFrame->State.gregs[Operand.Data.SIB.Base] & GPRMask;
|
|
}
|
|
|
|
if (Operand.Data.SIB.Index != FEXCore::X86State::REG_INVALID) {
|
|
Ptr += (Thread->CurrentFrame->State.gregs[Operand.Data.SIB.Index] * Operand.Data.SIB.Scale) & GPRMask;
|
|
}
|
|
|
|
Ptr += static_cast<int32_t>(Operand.Data.SIB.Offset);
|
|
}
|
|
|
|
auto AppendSegment = [&](uint64_t Ptr, uint32_t Flags, uint32_t DefaultPrefix = FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX,
|
|
bool Override = false) -> uint64_t {
|
|
uint32_t Prefix = Flags & FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS;
|
|
|
|
if (Is64BitMode) {
|
|
if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX) {
|
|
return Ptr + Thread->CurrentFrame->State.fs_cached;
|
|
} else if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {
|
|
return Ptr + Thread->CurrentFrame->State.gs_cached;
|
|
}
|
|
// If there was any other segment in 64bit then it is ignored
|
|
} else {
|
|
if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX || Override) {
|
|
// If there was no prefix then use the default one if available
|
|
// Or the argument only uses a specific prefix (with override set)
|
|
Prefix = DefaultPrefix;
|
|
}
|
|
// With the segment register optimization we store the GDT bases directly in the segment register to remove indexed loads
|
|
switch (Prefix) {
|
|
[[likely]] case FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX:
|
|
return Ptr;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX: return Ptr + Thread->CurrentFrame->State.es_cached;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX: return Ptr + Thread->CurrentFrame->State.cs_cached;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX: return Ptr + Thread->CurrentFrame->State.ss_cached;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX: return Ptr + Thread->CurrentFrame->State.ds_cached;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX: return Ptr + Thread->CurrentFrame->State.fs_cached;
|
|
case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX: return Ptr + Thread->CurrentFrame->State.gs_cached;
|
|
default: FEX_UNREACHABLE;
|
|
}
|
|
}
|
|
|
|
return Ptr;
|
|
};
|
|
|
|
return AppendSegment(Ptr, Op->Flags);
|
|
};
|
|
|
|
AddressMode OpDispatchBuilder::DecodeAddress(const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand,
|
|
MemoryAccessType AccessType, bool IsLoad) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
|
|
AddressMode A {};
|
|
A.Segment = GetSegment(Op->Flags);
|
|
A.AddrSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) != 0 ? (GPRSize >> 1) : GPRSize;
|
|
A.NonTSO = AccessType == MemoryAccessType::NONTSO || AccessType == MemoryAccessType::STREAM;
|
|
|
|
if (Operand.IsLiteral()) {
|
|
A.Offset = Operand.Literal();
|
|
|
|
if (Operand.Data.Literal.Size != 8 && IsLoad) {
|
|
// zero extend
|
|
uint64_t width = Operand.Data.Literal.Size * 8;
|
|
A.Offset &= ((1ULL << width) - 1);
|
|
}
|
|
} else if (Operand.IsGPR()) {
|
|
// Not an address, let the caller deal with it
|
|
A.AddrSize = GPRSize;
|
|
} else if (Operand.IsGPRDirect()) {
|
|
A.Base = LoadGPRRegister(Operand.Data.GPR.GPR, GPRSize);
|
|
A.NonTSO |= IsNonTSOReg(AccessType, Operand.Data.GPR.GPR);
|
|
} else if (Operand.IsGPRIndirect() || Operand.IsGPRIndirectRelocation()) {
|
|
A.Base = LoadGPRRegister(Operand.Data.GPRIndirect.GPR, GPRSize);
|
|
if (Operand.IsGPRIndirectRelocation()) {
|
|
A.Base = Add(GPRSize, _EntrypointOffset(GPRSize, Operand.Data.GPRIndirect.Displacement), A.Base);
|
|
} else {
|
|
A.Offset = static_cast<int32_t>(Operand.Data.GPRIndirect.Displacement);
|
|
}
|
|
A.NonTSO |= IsNonTSOReg(AccessType, Operand.Data.GPRIndirect.GPR);
|
|
} else if (Operand.IsRIPRelative() || Operand.IsRIPRelativeRelocation()) {
|
|
if (Is64BitMode) {
|
|
A.Base = GetRelocatedPC(Op, static_cast<int32_t>(Operand.Data.RIPLiteral.Value));
|
|
} else {
|
|
// 32bit this isn't RIP relative but instead absolute
|
|
if (Operand.IsRIPRelativeRelocation()) {
|
|
A.Base = _EntrypointOffset(GPRSize, Operand.Data.RIPLiteral.Value);
|
|
} else {
|
|
A.Offset = Operand.Data.RIPLiteral.Value;
|
|
}
|
|
}
|
|
} else if (Operand.IsSIB() || Operand.IsSIBRelocation()) {
|
|
const bool IsVSIB = IsLoad && ((Op->Flags & X86Tables::DecodeFlags::FLAG_VSIB_BYTE) != 0);
|
|
|
|
if (Operand.Data.SIB.Base != FEXCore::X86State::REG_INVALID) {
|
|
A.Base = LoadGPRRegister(Operand.Data.SIB.Base, GPRSize);
|
|
}
|
|
|
|
// NOTE: VSIB cannot have the index * scale portion calculated ahead of time,
|
|
// since the index in this case is a vector. So, we can't just apply the scale
|
|
// to it, since this needs to be applied to each element in the index register
|
|
// after said element has been sign extended. So, we pass this through for the
|
|
// instruction implementation to handle.
|
|
//
|
|
// What we do handle though, is the applying the displacement value to
|
|
// the base register (if a base register is provided), since this is a
|
|
// part of the address calculation that can be done ahead of time.
|
|
if (!IsVSIB && Operand.Data.SIB.Index != FEXCore::X86State::REG_INVALID) {
|
|
A.Index = LoadGPRRegister(Operand.Data.SIB.Index, GPRSize);
|
|
A.IndexScale = Operand.Data.SIB.Scale;
|
|
}
|
|
|
|
if (Operand.IsSIBRelocation()) {
|
|
auto EPOffset = _EntrypointOffset(GPRSize, Operand.Data.SIB.Offset);
|
|
if (A.Base) {
|
|
A.Base = Add(GPRSize, EPOffset, A.Base);
|
|
} else {
|
|
A.Base = EPOffset;
|
|
}
|
|
} else {
|
|
A.Offset = static_cast<int32_t>(Operand.Data.SIB.Offset);
|
|
}
|
|
|
|
A.NonTSO |= IsNonTSOReg(AccessType, Operand.Data.SIB.Base) || IsNonTSOReg(AccessType, Operand.Data.SIB.Index);
|
|
} else if (Operand.IsLiteralRelocation()) {
|
|
A.Base = _EntrypointOffset(GPRSize, Operand.Data.LiteralRelocation.EntrypointOffset);
|
|
} else if (Operand.IsLiteralPatchable()) {
|
|
A.Base = _PatchableGuestData(OpSize::i64Bit, Operand.Data.LiteralPatchable.Value, Op->PC + Operand.Data.LiteralPatchable.FieldOffset,
|
|
static_cast<uint64_t>(Operand.Data.LiteralPatchable.Width));
|
|
} else {
|
|
LOGMAN_MSG_A_FMT("Unknown Src Type: {}\n", Operand.Type);
|
|
}
|
|
|
|
return A;
|
|
}
|
|
|
|
|
|
Ref OpDispatchBuilder::LoadSource_WithOpSize(RegClass Class, const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand,
|
|
IR::OpSize OpSize, uint32_t Flags, const LoadSourceOptions& Options) {
|
|
auto [Align, LoadData, ForceLoad, AccessType, AllowUpperGarbage] = Options;
|
|
AddressMode A = DecodeAddress(Op, Operand, AccessType, true /* IsLoad */);
|
|
|
|
Ref Result {};
|
|
if (Operand.IsGPR()) {
|
|
const auto gpr = Operand.Data.GPR.GPR;
|
|
const auto highIndex = Operand.Data.GPR.HighBits ? 1 : 0;
|
|
|
|
if (gpr >= FEXCore::X86State::REG_MM_0) {
|
|
LOGMAN_THROW_A_FMT(OpSize == OpSize::i64Bit, "full");
|
|
|
|
if (MMXState != MMXState_MMX) {
|
|
ChgStateX87_MMX();
|
|
}
|
|
|
|
A.Base = LoadContext(OpSize::i64Bit, MM0Index + gpr - FEXCore::X86State::REG_MM_0);
|
|
} else if (gpr >= FEXCore::X86State::REG_XMM_0) {
|
|
const auto gprIndex = gpr - X86State::REG_XMM_0;
|
|
|
|
// Load the full register size if it is a XMM register source.
|
|
A.Base = LoadXMMRegister(gprIndex);
|
|
|
|
// Now extract the subregister if it was a partial load /smaller/ than SSE size
|
|
// TODO: Instead of doing the VMov implicitly on load, hunt down all use cases that require partial loads and do it after load.
|
|
// We don't have information here to know if the operation needs zero upper bits or can contain data.
|
|
if (!AllowUpperGarbage && OpSize < OpSize::i128Bit) {
|
|
A.Base = _VMov(OpSize, A.Base);
|
|
}
|
|
} else {
|
|
A.Base = LoadGPRRegister(gpr, OpSize, highIndex ? 8 : 0, AllowUpperGarbage);
|
|
}
|
|
}
|
|
|
|
const bool ShouldLoad = (IsOperandMem(Operand, true) && LoadData) || ForceLoad;
|
|
if (ShouldLoad) {
|
|
if (OpSize == OpSize::f80Bit) {
|
|
Ref MemSrc = LoadEffectiveAddress(this, A, GetGPROpSize(), true);
|
|
if (CTX->HostFeatures.SupportsSVE()) {
|
|
Result = _LoadMemX87SVEOptPredicate(OpSize::i128Bit, OpSize::i16Bit, MemSrc);
|
|
} else {
|
|
// For X87 extended doubles, Split the load.
|
|
auto Res = _LoadMem(Class, OpSize::i64Bit, MemSrc, Align == OpSize::iInvalid ? OpSize : Align);
|
|
Result = _VLoadVectorElement(OpSize::i128Bit, OpSize::i16Bit, Res, 4, Add(OpSize::i64Bit, MemSrc, 8));
|
|
}
|
|
} else {
|
|
Result = _LoadMemAutoTSO(Class, OpSize, A, Align == OpSize::iInvalid ? OpSize : Align);
|
|
}
|
|
} else {
|
|
Result = LoadEffectiveAddress(this, A, GetGPROpSize(), false, AllowUpperGarbage);
|
|
}
|
|
|
|
if constexpr (Context::BLOCK_DEBUGGING) {
|
|
if (ShouldLoad && CTX->BlockDebuggerTracker.IsSingleStepTarget(Entry)) {
|
|
uint64_t Ptr = CalcAddress(Op, Operand, true);
|
|
if (CTX->BlockDebuggerTracker.ContainsReadWatchPoint(Ptr, OpSizeToSize(OpSize))) {
|
|
// It's up to the developer if they want more advanced debugging logic here.
|
|
LogMan::Msg::IFmt("Entrypoint 0x{:x} will hit read watch: [0x{:x}, 0x{:x})", Entry, Ptr, Ptr + OpSizeToSize(OpSize));
|
|
}
|
|
}
|
|
}
|
|
|
|
return Result;
|
|
}
|
|
|
|
Ref OpDispatchBuilder::LoadGPRRegister(uint32_t GPR, IR::OpSize Size, uint8_t Offset, bool AllowUpperGarbage) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
if (Size == OpSize::iInvalid) {
|
|
Size = GPRSize;
|
|
}
|
|
Ref Reg = LoadGPR(GPR);
|
|
|
|
if ((!AllowUpperGarbage && (Size != GPRSize)) || Offset != 0) {
|
|
// Extract the subregister if requested.
|
|
const auto OpSize = std::max(OpSize::i32Bit, Size);
|
|
if (AllowUpperGarbage) {
|
|
Reg = _Lshr(OpSize, Reg, Constant(Offset));
|
|
} else {
|
|
Reg = _Bfe(OpSize, IR::OpSizeAsBits(Size), Offset, Reg);
|
|
}
|
|
}
|
|
return Reg;
|
|
}
|
|
|
|
void OpDispatchBuilder::StoreGPRRegister(uint32_t GPR, const Ref Src, IR::OpSize Size, uint8_t Offset) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
if (Size == OpSize::iInvalid) {
|
|
Size = GPRSize;
|
|
}
|
|
|
|
Ref Reg = Src;
|
|
if (Size != GPRSize || Offset != 0) {
|
|
// Need to do an insert if not automatic size or zero offset.
|
|
Reg = ARef(Reg).BfiInto(LoadGPRRegister(GPR), Offset, IR::OpSizeAsBits(Size));
|
|
}
|
|
|
|
StoreRegister(GPR, false, Reg);
|
|
}
|
|
|
|
void OpDispatchBuilder::StoreXMMRegister(uint32_t XMM, const Ref Src) {
|
|
StoreRegister(XMM, true, Src);
|
|
}
|
|
|
|
Ref OpDispatchBuilder::LoadSource(RegClass Class, const X86Tables::DecodedOp& Op, const X86Tables::DecodedOperand& Operand, uint32_t Flags,
|
|
const LoadSourceOptions& Options) {
|
|
const auto OpSize = OpSizeFromSrc(Op);
|
|
return LoadSource_WithOpSize(Class, Op, Operand, OpSize, Flags, Options);
|
|
}
|
|
|
|
void OpDispatchBuilder::StoreResult_WithOpSize(RegClass Class, FEXCore::X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand,
|
|
Ref Src, IR::OpSize OpSize, IR::OpSize Align, MemoryAccessType AccessType) {
|
|
if (Operand.IsGPR()) {
|
|
// 8Bit and 16bit destination types store their result without effecting the upper bits
|
|
// 32bit ops ZEXT the result to 64bit
|
|
const auto GPRSize = GetGPROpSize();
|
|
|
|
const auto gpr = Operand.Data.GPR.GPR;
|
|
if (gpr >= FEXCore::X86State::REG_MM_0) {
|
|
LOGMAN_THROW_A_FMT(OpSize == OpSize::i64Bit, "full");
|
|
LOGMAN_THROW_A_FMT(Class == RegClass::FPR, "MMX is floaty");
|
|
|
|
if (MMXState != MMXState_MMX) {
|
|
ChgStateX87_MMX();
|
|
}
|
|
|
|
uint8_t Index = MM0Index + gpr - FEXCore::X86State::REG_MM_0;
|
|
StoreContext(Index, Src);
|
|
RegCache.Partial |= (1ull << (uint64_t)Index);
|
|
} else if (gpr >= FEXCore::X86State::REG_XMM_0) {
|
|
const auto gprIndex = gpr - X86State::REG_XMM_0;
|
|
const auto VectorSize = GetGuestVectorLength();
|
|
|
|
auto Result = Src;
|
|
if (OpSize != VectorSize) {
|
|
// Partial writes can come from FPRs.
|
|
// TODO: Fix the instructions doing partial writes rather than dealing with it here.
|
|
|
|
LOGMAN_THROW_A_FMT(Class != RegClass::GPR, "Partial writes from GPR not allowed. Instruction: {}", Op->TableInfo->Name);
|
|
|
|
// XMM-size is handled in implementations.
|
|
if (VectorSize != OpSize::i256Bit || OpSize != OpSize::i128Bit) {
|
|
auto SrcVector = LoadXMMRegister(gprIndex);
|
|
Result = _VInsElement(VectorSize, OpSize, 0, 0, SrcVector, Src);
|
|
}
|
|
}
|
|
|
|
StoreXMMRegister(gprIndex, Result);
|
|
} else {
|
|
if (GPRSize == OpSize::i64Bit && OpSize == OpSize::i32Bit) {
|
|
// If the Source IR op is 64 bits, we need to zext the upper bits
|
|
// For all other sizes, the upper bits are guaranteed to already be zero
|
|
Ref Value = GetOpSize(Src) == OpSize::i64Bit ? ARef(Src).Bfe(0, 32).Ref() : Src;
|
|
StoreGPRRegister(gpr, Value, GPRSize);
|
|
|
|
LOGMAN_THROW_A_FMT(!Operand.Data.GPR.HighBits, "Can't handle 32bit store to high 8bit register");
|
|
} else {
|
|
LOGMAN_THROW_A_FMT(!(GPRSize == OpSize::i32Bit && OpSize > OpSize::i32Bit), "Oops had a {} GPR load", OpSize);
|
|
|
|
if (GPRSize != OpSize) {
|
|
// if the GPR isn't the full size then we need to insert.
|
|
// eg:
|
|
// mov al, 2 ; Move in to lower 8-bits.
|
|
// mov ah, 2 ; Move in to upper 8-bits of 16-bit reg.
|
|
// mov ax, 2 ; Move in to lower 16-bits of reg.
|
|
StoreGPRRegister(gpr, Src, OpSize, Operand.Data.GPR.HighBits * 8);
|
|
} else {
|
|
StoreGPRRegister(gpr, Src, std::min(GPRSize, OpSize));
|
|
}
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
|
|
AddressMode A = DecodeAddress(Op, Operand, AccessType, false /* IsLoad */);
|
|
|
|
if (OpSize == OpSize::f80Bit) {
|
|
Ref MemStoreDst = LoadEffectiveAddress(this, A, GetGPROpSize(), true);
|
|
if (CTX->HostFeatures.SupportsSVE()) {
|
|
_StoreMemX87SVEOptPredicate(OpSize::i128Bit, OpSize::i16Bit, Src, MemStoreDst);
|
|
} else {
|
|
// For X87 extended doubles, split before storing
|
|
_StoreMemFPR(OpSize::i64Bit, MemStoreDst, Src, Align);
|
|
auto Upper = _VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, Src, 1);
|
|
_StoreMemGPR(OpSize::i16Bit, Upper, MemStoreDst, Constant(8), std::min(Align, OpSize::i64Bit), MemOffsetType::SXTX, 1);
|
|
}
|
|
} else {
|
|
_StoreMemAutoTSO(Class, OpSize, A, Src, Align == OpSize::iInvalid ? OpSize : Align);
|
|
}
|
|
|
|
if constexpr (Context::BLOCK_DEBUGGING) {
|
|
if (CTX->BlockDebuggerTracker.IsSingleStepTarget(Entry)) {
|
|
uint64_t Ptr = CalcAddress(Op, Operand, false);
|
|
if (CTX->BlockDebuggerTracker.ContainsWriteWatchPoint(Ptr, OpSizeToSize(OpSize))) {
|
|
// It's up to the developer if they want more advanced debugging logic here.
|
|
LogMan::Msg::IFmt("Entrypoint 0x{:x} will hit write watch: [0x{:x}, 0x{:x})", Entry, Ptr, Ptr + OpSizeToSize(OpSize));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::StoreResult(RegClass Class, X86Tables::DecodedOp Op, const X86Tables::DecodedOperand& Operand, Ref Src,
|
|
IR::OpSize Align, MemoryAccessType AccessType) {
|
|
StoreResult_WithOpSize(Class, Op, Operand, Src, OpSizeFromDst(Op), Align, AccessType);
|
|
}
|
|
|
|
void OpDispatchBuilder::StoreResult(RegClass Class, X86Tables::DecodedOp Op, Ref Src, IR::OpSize Align, MemoryAccessType AccessType) {
|
|
StoreResult(Class, Op, Op->Dest, Src, Align, AccessType);
|
|
}
|
|
|
|
OpDispatchBuilder::OpDispatchBuilder(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::InternalThreadState* Thread)
|
|
: IREmitter {ctx->OpDispatcherAllocator, ctx->HostFeatures.SupportsTSOImm9 != 0}
|
|
, CTX {ctx}
|
|
, Thread {Thread} {
|
|
if (CTX->HostFeatures.SupportsAVX && CTX->HostFeatures.SupportsSVE256) {
|
|
SaveAVXStateFunc = &OpDispatchBuilder::SaveAVXState;
|
|
RestoreAVXStateFunc = &OpDispatchBuilder::RestoreAVXState;
|
|
DefaultAVXStateFunc = &OpDispatchBuilder::DefaultAVXState;
|
|
} else if (CTX->HostFeatures.SupportsAVX) {
|
|
SaveAVXStateFunc = &OpDispatchBuilder::AVX128_SaveAVXState;
|
|
RestoreAVXStateFunc = &OpDispatchBuilder::AVX128_RestoreAVXState;
|
|
DefaultAVXStateFunc = &OpDispatchBuilder::AVX128_DefaultAVXState;
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::ResetWorkingList() {
|
|
IREmitter::ReownOrClaimBuffer();
|
|
|
|
JumpTargets.clear();
|
|
BlockSetRIP = false;
|
|
DecodeFailure = false;
|
|
ShouldDump = false;
|
|
CurrentCodeBlock = nullptr;
|
|
RegCache.Written = 0;
|
|
RegCache.Cached = 0;
|
|
}
|
|
|
|
void OpDispatchBuilder::UnhandledOp(OpcodeArgs) {
|
|
DecodeFailure = true;
|
|
}
|
|
|
|
void OpDispatchBuilder::MOVGPROp(OpcodeArgs, uint32_t SrcIndex) {
|
|
// StoreResult will store with the same size as the input, so we allow upper
|
|
// garbage on the input. The zero extension would be pointless.
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[SrcIndex], Op->Flags, {.Align = OpSize::i8Bit, .AllowUpperGarbage = true});
|
|
StoreResultGPR(Op, Src, OpSize::i8Bit);
|
|
}
|
|
|
|
void OpDispatchBuilder::MOVGPRNTOp(OpcodeArgs) {
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit});
|
|
StoreResultGPR(Op, Src, OpSize::i8Bit, MemoryAccessType::STREAM);
|
|
}
|
|
|
|
void OpDispatchBuilder::ALUOp(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCore::IR::IROps AtomicFetchOp, unsigned SrcIdx) {
|
|
// On x86, the canonical way to zero a register is XOR with itself. Detect and
|
|
// emit optimal arm64 assembly.
|
|
if (!DestIsLockedMem(Op) && ALUIROp == FEXCore::IR::IROps::OP_XOR && Op->Dest.IsGPR() && Op->Src[SrcIdx].IsGPR() &&
|
|
Op->Dest.Data.GPR == Op->Src[SrcIdx].Data.GPR) {
|
|
|
|
// Set flags for zero result with inverted carry. We subtract an arbitrary
|
|
// register from itself to get the zero, since `subs wzr, #0` is not
|
|
// encodable. This is optimal and works regardless of the opsize.
|
|
auto Zero = LoadGPR(Op->Dest.Data.GPR.GPR);
|
|
HandleNZ00Write();
|
|
InvalidateAF();
|
|
CalculatePF(SubWithFlags(OpSize::i32Bit, Zero, Zero));
|
|
CFInverted = true;
|
|
FlushRegisterCache();
|
|
|
|
// Move 0 into the register
|
|
StoreResultGPR(Op, Constant(0));
|
|
return;
|
|
}
|
|
|
|
auto Size = OpSizeFromDst(Op);
|
|
auto ResultSize = Size;
|
|
|
|
auto RoundedSize = Size;
|
|
if (ALUIROp != FEXCore::IR::IROps::OP_ANDWITHFLAGS) {
|
|
RoundedSize = std::max(OpSize::i32Bit, RoundedSize);
|
|
}
|
|
|
|
// X86 basic ALU ops just do the operation between the destination and a single source
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[SrcIdx], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
// Try to eliminate the masking after 8/16-bit operations with constants, by
|
|
// promoting to a full size operation that preserves the upper bits.
|
|
uint64_t Const;
|
|
bool IsConst = IsValueConstant(WrapNode(Src), &Const);
|
|
if (Size < OpSize::i32Bit && !DestIsLockedMem(Op) && Op->Dest.IsGPR() && !Op->Dest.Data.GPR.HighBits && IsConst &&
|
|
(ALUIROp == IR::IROps::OP_XOR || ALUIROp == IR::IROps::OP_OR || ALUIROp == IR::IROps::OP_ANDWITHFLAGS)) {
|
|
|
|
RoundedSize = ResultSize = GetGPROpSize();
|
|
LOGMAN_THROW_A_FMT(Const < (1ull << IR::OpSizeAsBits(Size)), "does not clobber");
|
|
|
|
// For AND, we can play the same trick but we instead need the upper bits of
|
|
// the constant to be all-1s instead of all-0s to preserve. We also can't
|
|
// use andwithflags in this case, since we've promoted to 64-bit so the
|
|
// negate flag would be wrong, but using the regular logical operation path
|
|
// instead still ends up a net win for uops.
|
|
//
|
|
// In the common case where the constant is of the form (1 << x) - 1, the
|
|
// adjusted constant here will inline into the arm64 and instruction, so if
|
|
// flags are not needed, we save an instruction overall.
|
|
if (ALUIROp == IR::IROps::OP_ANDWITHFLAGS) {
|
|
Src = Constant(Const | ~((1ull << IR::OpSizeAsBits(Size)) - 1));
|
|
ALUIROp = IR::IROps::OP_AND;
|
|
}
|
|
}
|
|
|
|
Ref Result {};
|
|
Ref Dest {};
|
|
|
|
if (DestIsLockedMem(Op)) {
|
|
HandledLock = true;
|
|
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
|
|
DeriveOp(FetchOp, AtomicFetchOp, _AtomicFetchAdd(Size, Src, DestMem));
|
|
Dest = FetchOp;
|
|
} else {
|
|
Dest = LoadSourceGPR(Op, Op->Dest, Op->Flags, {.AllowUpperGarbage = true});
|
|
}
|
|
|
|
const auto OpSize = RoundedSize;
|
|
uint64_t Mask = Size == OpSize::i64Bit ? ~0ull : ((1ull << IR::OpSizeAsBits(Size)) - 1);
|
|
if (IsConst && Const == Mask && !DestIsLockedMem(Op) && ALUIROp == IR::IROps::OP_XOR && Size >= OpSize::i32Bit) {
|
|
Result = _Not(OpSize, Dest);
|
|
} else if (IsConst && Const == Mask && !DestIsLockedMem(Op) && ALUIROp == IR::IROps::OP_AND) {
|
|
Result = Dest;
|
|
} else {
|
|
DeriveOp(ALUOp, ALUIROp, _AndWithFlags(OpSize, Dest, Src));
|
|
Result = ALUOp;
|
|
}
|
|
|
|
// Flags set
|
|
switch (ALUIROp) {
|
|
case FEXCore::IR::IROps::OP_ADD: Result = CalculateFlags_ADD(Size, Dest, Src); break;
|
|
case FEXCore::IR::IROps::OP_SUB: Result = CalculateFlags_SUB(Size, Dest, Src); break;
|
|
case FEXCore::IR::IROps::OP_XOR:
|
|
case FEXCore::IR::IROps::OP_AND:
|
|
case FEXCore::IR::IROps::OP_OR: {
|
|
CalculateFlags_Logical(Size, Result);
|
|
break;
|
|
}
|
|
case FEXCore::IR::IROps::OP_ANDWITHFLAGS: {
|
|
HandleNZ00Write();
|
|
CalculatePF(Result);
|
|
InvalidateAF();
|
|
break;
|
|
}
|
|
default: break;
|
|
}
|
|
|
|
if (!DestIsLockedMem(Op)) {
|
|
StoreResultGPR_WithOpSize(Op, Op->Dest, Result, ResultSize, OpSize::iInvalid, MemoryAccessType::DEFAULT);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::LSLOp(OpcodeArgs) {
|
|
// Emulate by always returning failure, this deviates from both Linux and Windows but
|
|
// shouldn't be depended on by anything.
|
|
SetRFLAG<FEXCore::X86State::RFLAG_ZF_RAW_LOC>(Constant(0));
|
|
}
|
|
|
|
void OpDispatchBuilder::INTOp(OpcodeArgs) {
|
|
IR::BreakDefinition Reason;
|
|
bool SetRIPToNext = false;
|
|
|
|
switch (Op->OP) {
|
|
case 0xCD: { // INT imm8
|
|
uint8_t Literal = Op->Src[0].Literal();
|
|
|
|
if (CTX->HostFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Linux) {
|
|
constexpr uint8_t SYSCALL_LITERAL = 0x80;
|
|
if (Literal == SYSCALL_LITERAL) {
|
|
if (Is64BitMode) [[unlikely]] {
|
|
LogMan::Msg::EFmt("[Unsupported] Trying to execute 32-bit syscall from a 64-bit process.");
|
|
UnhandledOp(Op);
|
|
return;
|
|
}
|
|
// Syscall on linux
|
|
SyscallOp(Op, false);
|
|
return;
|
|
}
|
|
} else if (CTX->HostFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Wow64 ||
|
|
CTX->HostFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Arm64ec) {
|
|
constexpr uint8_t SYSCALL_LITERAL = 0x2E;
|
|
if (Literal == SYSCALL_LITERAL) {
|
|
// Can be used for both 64-bit and 32-bit syscalls on windows
|
|
SyscallOp(Op, false);
|
|
return;
|
|
}
|
|
|
|
if (CTX->HostFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Arm64ec) {
|
|
// This is used when QueryPerformanceCounter is called on recent Windows versions, it causes CNTVCT to be written into RAX.
|
|
constexpr uint8_t GET_CNTVCT_LITERAL = 0x81;
|
|
if (Literal == GET_CNTVCT_LITERAL) {
|
|
StoreGPRRegister(X86State::REG_RAX, _CycleCounter(false));
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
Reason.ErrorRegister = Literal << 3 | (0b010);
|
|
Reason.Signal = Core::FAULT_SIGSEGV;
|
|
// GP is raised when task-gate isn't setup to be valid
|
|
Reason.TrapNumber = X86State::X86_TRAPNO_GP;
|
|
Reason.si_code = 0x80;
|
|
break;
|
|
}
|
|
case 0xCE: // INTO
|
|
Reason.ErrorRegister = 0;
|
|
Reason.Signal = Core::FAULT_SIGSEGV;
|
|
Reason.TrapNumber = X86State::X86_TRAPNO_OF;
|
|
Reason.si_code = 0x80;
|
|
break;
|
|
case 0xF1: // INT1
|
|
Reason.ErrorRegister = 0;
|
|
Reason.Signal = Core::FAULT_SIGTRAP;
|
|
Reason.TrapNumber = X86State::X86_TRAPNO_DB;
|
|
Reason.si_code = 1;
|
|
SetRIPToNext = true;
|
|
break;
|
|
case 0xF4: { // HLT
|
|
Reason.ErrorRegister = 0;
|
|
Reason.Signal = Core::FAULT_SIGSEGV;
|
|
Reason.TrapNumber = X86State::X86_TRAPNO_GP;
|
|
Reason.si_code = 0x80;
|
|
break;
|
|
}
|
|
case 0x0B: // UD2
|
|
Reason.ErrorRegister = 0;
|
|
Reason.Signal = Core::FAULT_SIGILL;
|
|
Reason.TrapNumber = X86State::X86_TRAPNO_UD;
|
|
Reason.si_code = 2;
|
|
break;
|
|
case 0xCC: // INT3
|
|
Reason.ErrorRegister = 0;
|
|
Reason.Signal = Core::FAULT_SIGTRAP;
|
|
Reason.TrapNumber = X86State::X86_TRAPNO_BP;
|
|
Reason.si_code = 0x80;
|
|
SetRIPToNext = true;
|
|
break;
|
|
default: FEX_UNREACHABLE;
|
|
}
|
|
|
|
// Calculate flags early.
|
|
FlushRegisterCache();
|
|
|
|
const auto GPRSize = GetGPROpSize();
|
|
|
|
if (SetRIPToNext) {
|
|
BlockSetRIP = SetRIPToNext;
|
|
|
|
// We want to set RIP to the next instruction after INT3/INT1
|
|
auto NewRIP = GetRelocatedPC(Op);
|
|
_StoreContextGPR(GPRSize, NewRIP, offsetof(FEXCore::Core::CPUState, rip));
|
|
} else if (Op->OP != 0xCE) {
|
|
auto NewRIP = GetRelocatedPC(Op, -Op->InstSize);
|
|
_StoreContextGPR(GPRSize, NewRIP, offsetof(FEXCore::Core::CPUState, rip));
|
|
}
|
|
|
|
if (Op->OP == 0xCE) { // Conditional to only break if Overflow == 1
|
|
CalculateDeferredFlags();
|
|
|
|
// If condition doesn't hold then keep going
|
|
// CondClass::FNU means OF == 0
|
|
auto CondJump_ = CondJumpNZCV(CondClass::FNU);
|
|
auto FalseBlock = CreateNewCodeBlockAfter(GetCurrentBlock());
|
|
SetFalseJumpTarget(CondJump_, FalseBlock);
|
|
SetCurrentCodeBlock(FalseBlock);
|
|
StartNewBlock();
|
|
|
|
auto NewRIP = GetRelocatedPC(Op);
|
|
_StoreContextGPR(GPRSize, NewRIP, offsetof(FEXCore::Core::CPUState, rip));
|
|
Break(Reason);
|
|
|
|
// Make sure to start a new block after ending this one
|
|
auto JumpTarget = CreateNewCodeBlockAfter(FalseBlock);
|
|
SetTrueJumpTarget(CondJump_, JumpTarget);
|
|
SetCurrentCodeBlock(JumpTarget);
|
|
StartNewBlock();
|
|
} else {
|
|
BlockSetRIP = true;
|
|
Break(Reason);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::TZCNT(OpcodeArgs) {
|
|
// _FindTrailingZeroes ignores upper garbage so we don't need to mask
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
Src = _FindTrailingZeroes(OpSizeFromSrc(Op), Src);
|
|
StoreResultGPR(Op, Src);
|
|
|
|
CalculateFlags_ZCNT(OpSizeFromSrc(Op), Src);
|
|
}
|
|
|
|
void OpDispatchBuilder::LZCNT(OpcodeArgs) {
|
|
// _CountLeadingZeroes clears upper garbage so we don't need to mask
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.AllowUpperGarbage = true});
|
|
|
|
auto Res = _CountLeadingZeroes(OpSizeFromSrc(Op), Src);
|
|
StoreResultGPR(Op, Res);
|
|
CalculateFlags_ZCNT(OpSizeFromSrc(Op), Res);
|
|
}
|
|
|
|
void OpDispatchBuilder::MOVBEOp(OpcodeArgs) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
const auto SrcSize = OpSizeFromSrc(Op);
|
|
|
|
Ref Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit});
|
|
|
|
if (DestIsMem(Op) || SrcSize != OpSize::i16Bit) {
|
|
Src = _Rev(SrcSize, Src);
|
|
StoreResultGPR(Op, Op->Dest, Src);
|
|
} else {
|
|
Src = _Rev(std::max(OpSize::i32Bit, SrcSize), Src);
|
|
// 16-bit does an insert.
|
|
// Rev of 16-bit value as 32-bit replaces the result in the upper 16-bits of the result.
|
|
// bfxil the 16-bit result in to the GPR.
|
|
Ref Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, GPRSize, Op->Flags);
|
|
auto Result = _Bfxil(GPRSize, 16, 16, Dest, Src);
|
|
StoreResultGPR_WithOpSize(Op, Op->Dest, Result, GPRSize);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::CLWBOrTPause(OpcodeArgs) {
|
|
if (DestIsMem(Op)) {
|
|
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
|
|
_CacheLineClean(DestMem);
|
|
} else {
|
|
if (!CTX->HostFeatures.SupportsWFXT) {
|
|
UnimplementedOp(Op);
|
|
} else {
|
|
auto RAX = LoadGPRRegister(X86State::REG_RAX);
|
|
auto RDX = LoadGPRRegister(X86State::REG_RDX);
|
|
|
|
// Incoming source register is unused.
|
|
_WFET(RDX, RAX);
|
|
|
|
// OF, SF, ZF, AF, PF, CF all zero.
|
|
// CF is used if the OS deadline is set, which we don't do anything with.
|
|
ZeroPF_AF();
|
|
ZeroNZCV();
|
|
}
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::CLFLUSHOPT(OpcodeArgs) {
|
|
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
|
|
_CacheLineClear(DestMem, false);
|
|
}
|
|
|
|
void OpDispatchBuilder::LoadFenceOrXRSTOR(OpcodeArgs) {
|
|
// 0xE8 signifies LFENCE
|
|
if (Op->ModRM == 0xE8) {
|
|
_Fence(FenceType::Load);
|
|
} else {
|
|
XRstorOpImpl(Op);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::MemFenceOrXSAVEOPT(OpcodeArgs) {
|
|
if (Op->ModRM == 0xF0) {
|
|
// 0xF0 is MFENCE
|
|
_Fence(FenceType::LoadStore);
|
|
} else {
|
|
XSaveOp(Op);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::StoreFenceOrCLFlush(OpcodeArgs) {
|
|
if (Op->ModRM == 0xF8) {
|
|
// 0xF8 is SFENCE
|
|
_Fence(FenceType::Store);
|
|
} else {
|
|
// This is a CLFlush
|
|
Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
|
|
_CacheLineClear(DestMem, true);
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::UMonitorOrCLRSSBSY(OpcodeArgs) {
|
|
if (DestIsMem(Op) || !CTX->HostFeatures.SupportsWFXT) {
|
|
// CLRSSBSY
|
|
UnimplementedOp(Op);
|
|
} else {
|
|
// Explicit NOP implementation of umonitor.
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::UMWaitOp(OpcodeArgs) {
|
|
if (DestIsMem(Op) || !CTX->HostFeatures.SupportsWFXT) {
|
|
UnimplementedOp(Op);
|
|
} else {
|
|
// Explicit NOP implementation of umwait.
|
|
// Still zero flags.
|
|
//
|
|
// OF, SF, ZF, AF, PF, CF all zero.
|
|
ZeroPF_AF();
|
|
ZeroNZCV();
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::CLZeroOp(OpcodeArgs) {
|
|
if (!CTX->HostFeatures.SupportsCLZERO) {
|
|
UnimplementedOp(Op);
|
|
return;
|
|
}
|
|
Ref DestMem = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.LoadData = false});
|
|
_CacheLineZero(DestMem);
|
|
}
|
|
|
|
void OpDispatchBuilder::Prefetch(OpcodeArgs, bool ForStore, bool Stream, uint8_t Level) {
|
|
if (Op->Src[0].IsGPR()) {
|
|
// NOP instance.
|
|
return;
|
|
}
|
|
|
|
Ref DestMem = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.LoadData = false});
|
|
_Prefetch(ForStore, Stream, Level, DestMem, Invalid(), MemOffsetType::SXTX, 1);
|
|
}
|
|
|
|
void OpDispatchBuilder::RDTSCPOp(OpcodeArgs) {
|
|
// RDTSCP is slightly different than RDTSC
|
|
// IA32_TSC_AUX is returned in RCX
|
|
// All previous loads are globally visible
|
|
// - Explicitly does not wait for stores to be globally visible
|
|
// - Explicitly use an MFENCE before this instruction if you want this behaviour
|
|
// This instruction is not an execution fence, so subsequent instructions can execute after this
|
|
// - Explicitly use an LFENCE after RDTSCP if you want to block this behaviour
|
|
if (CTX->HostFeatures.HostType != FEXCore::HostFeatures::HostTypeEnum::Linux && !CTX->HostFeatures.SupportsCPUIndexInTPIDRRO) {
|
|
// RDTSCP is unsupported on Win32 platforms if TPIDRRO isn't supported.
|
|
UnimplementedOp(Op);
|
|
return;
|
|
}
|
|
auto Counter = CycleCounter(true);
|
|
|
|
auto ID = _ProcessorID();
|
|
StoreGPRRegister(X86State::REG_RAX, Counter.CounterLow);
|
|
StoreGPRRegister(X86State::REG_RCX, ID);
|
|
StoreGPRRegister(X86State::REG_RDX, Counter.CounterHigh);
|
|
}
|
|
|
|
void OpDispatchBuilder::RDPIDOp(OpcodeArgs) {
|
|
StoreResultGPR(Op, _ProcessorID());
|
|
}
|
|
|
|
void OpDispatchBuilder::CRC32(OpcodeArgs) {
|
|
if (!CTX->HostFeatures.SupportsCRC) {
|
|
UnimplementedOp(Op);
|
|
return;
|
|
}
|
|
const auto GPRSize = GetGPROpSize();
|
|
const auto SrcSize = OpSizeFromSrc(Op);
|
|
|
|
// Destination GPR size is always 4 or 8 bytes depending on widening
|
|
const auto DstSize = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;
|
|
Ref Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, GPRSize, Op->Flags);
|
|
|
|
// Incoming memory is 8, 16, 32, or 64
|
|
Ref Src {};
|
|
if (Op->Src[0].IsGPR()) {
|
|
Src = LoadSourceGPR_WithOpSize(Op, Op->Src[0], SrcSize, Op->Flags, {.AllowUpperGarbage = true});
|
|
} else {
|
|
Src = LoadSourceGPR(Op, Op->Src[0], Op->Flags, {.Align = OpSize::i8Bit});
|
|
}
|
|
auto Result = _CRC32(Dest, Src, SrcSize);
|
|
StoreResultGPR_WithOpSize(Op, Op->Dest, Result, DstSize);
|
|
}
|
|
|
|
void OpDispatchBuilder::RDRANDOp(OpcodeArgs, bool Reseed) {
|
|
if (!CTX->HostFeatures.SupportsRAND) {
|
|
UnimplementedOp(Op);
|
|
return;
|
|
}
|
|
|
|
StoreResultGPR(Op, _RDRAND(Reseed));
|
|
|
|
// If the rng number is valid then NZCV is 0b0000, otherwise NZCV is 0b0100
|
|
auto CF_inv = GetRFLAG(X86State::RFLAG_ZF_RAW_LOC);
|
|
|
|
// OF, SF, ZF, AF, PF all zero. CF indicates if valid.
|
|
ZeroPF_AF();
|
|
|
|
if (!CTX->HostFeatures.SupportsFlagM) {
|
|
ZeroNZCV();
|
|
SetCFInverted(CF_inv);
|
|
} else {
|
|
// Accelerated path. Invalid is 0 or 1, so set NZCV with a single rmif.
|
|
HandleNZCVWrite();
|
|
_RmifNZCV(CF_inv, (64 - 1) /* rotate bit 0 into bit 1 = C */, 0xf);
|
|
CFInverted = true;
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDefinition) {
|
|
const auto GPRSize = GetGPROpSize();
|
|
|
|
// We don't actually support this instruction
|
|
// Multiblock may hit it though
|
|
_StoreContextGPR(GPRSize, GetRelocatedPC(Op, -Op->InstSize), offsetof(FEXCore::Core::CPUState, rip));
|
|
Break(BreakDefinition);
|
|
|
|
|
|
if (Multiblock) {
|
|
auto NextBlock = CreateNewCodeBlockAfter(GetCurrentBlock());
|
|
SetCurrentCodeBlock(NextBlock);
|
|
StartNewBlock();
|
|
} else {
|
|
BlockSetRIP = true;
|
|
}
|
|
}
|
|
|
|
void OpDispatchBuilder::UnimplementedOp(OpcodeArgs) {
|
|
BreakOp(Op, FEXCore::IR::BreakDefinition {
|
|
.ErrorRegister = 0,
|
|
.Signal = SIGILL,
|
|
.TrapNumber = X86State::X86_TRAPNO_UD,
|
|
.si_code = 2, ///< ILL_ILLOPN
|
|
});
|
|
}
|
|
|
|
void OpDispatchBuilder::PermissionRestrictedOp(OpcodeArgs) {
|
|
BreakOp(Op, FEXCore::IR::BreakDefinition {
|
|
.ErrorRegister = 0,
|
|
.Signal = SIGSEGV,
|
|
.TrapNumber = X86State::X86_TRAPNO_GP,
|
|
.si_code = 0x80,
|
|
});
|
|
}
|
|
|
|
void OpDispatchBuilder::InvalidOp(OpcodeArgs) {
|
|
BreakOp(Op, FEXCore::IR::BreakDefinition {
|
|
.ErrorRegister = 0,
|
|
.Signal = SIGILL,
|
|
.TrapNumber = 0,
|
|
.si_code = 0,
|
|
});
|
|
}
|
|
|
|
void OpDispatchBuilder::NoExecOp(OpcodeArgs) {
|
|
BreakOp(Op, FEXCore::IR::BreakDefinition {
|
|
.ErrorRegister = X86State::X86_PF_PROT | X86State::X86_PF_USER | X86State::X86_PF_INSTR,
|
|
.Signal = Core::FAULT_SIGSEGV,
|
|
.TrapNumber = X86State::X86_TRAPNO_PF,
|
|
.si_code = 2, // SEGV_ACCERR
|
|
});
|
|
}
|
|
|
|
#undef OpcodeArgs
|
|
} // namespace FEXCore::IR
|