FEXCore: Implements support for shifted bitwise ops

This wasn't implemented initially for the interpreter and x86 JIT.

This meant we are maintaining two codepaths. Implement these operations
in the interpreter and x86 JIT so we no longer need to do that.

The emitted code in the x86 JIT is hot garbage, but it's only necessary
for correctness testing, not performance testing there.
This commit is contained in:
Ryan Houdek committed 2023-09-11 13:17:35 -07:00
1 parent 48521a4416
commit 863331b117
9 files changed
+407 -13

No files matched your search

@@ -82,6 +82,91 @@ DEF_OP(Add) {
}
}
DEF_OP(AddNZCV) {
auto Op = IROp->C<IR::IROp_AddNZCV>();
const uint8_t OpSize = Op->Size;
const uint64_t Src1 = *GetSrc<uint64_t*>(Data->SSAData, Op->Src1);
const uint64_t Src2 = *GetSrc<uint64_t*>(Data->SSAData, Op->Src2);
// Results returned in Arm64 NZCV format
// N = Sign bit
// Z = Is Zero
// C = Carry occured (Unsigned result can't fit within resulting register)
// V = Overflow occured (Signed result can't fit in to resulting register)
uint32_t NZCV{};
switch (OpSize) {
case 4: {
uint32_t Result = Src1 + Src2;
int32_t ResultSigned{};
if ((Result >> (sizeof(uint32_t) * 8 - 1)) & 1) {
NZCV |= 1U << 31;
}
if (Result == 0) {
NZCV |= 1U << 30;
}
if (__builtin_uadd_overflow(Src1, Src2, &Result)) {
NZCV |= 1U << 29;
}
if (__builtin_sadd_overflow(Src1, Src2, &ResultSigned)) {
NZCV |= 1U << 28;
}
break;
}
case 8: {
uint64_t Result = Src1 + Src2;
int64_t ResultSigned{};
if ((Result >> (sizeof(uint64_t) * 8 - 1)) & 1) {
NZCV |= 1U << 31;
}
if (Result == 0) {
NZCV |= 1U << 30;
}
if (__builtin_uaddl_overflow(Src1, Src2, &Result)) {
NZCV |= 1U << 29;
}
if (__builtin_saddl_overflow(Src1, Src2, &ResultSigned)) {
NZCV |= 1U << 28;
}
break;
}
default: LOGMAN_MSG_A_FMT("Unknown {} Size: {}\n", __func__, OpSize); break;
}
GD = NZCV;
}
DEF_OP(TestNZ) {
auto Op = IROp->C<IR::IROp_TestNZ>();
const uint8_t OpSize = Op->Size;
const uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Src1);
// Results returned in Arm64 NZCV format
// N = Sign bit
// Z = Is Zero
// CV = 00
uint32_t NZCV{};
switch (OpSize) {
case 4:
if ((Src >> (sizeof(uint32_t) * 8 - 1)) & 1) {
NZCV |= 1U << 31;
}
if (static_cast<uint32_t>(Src) == 0) {
NZCV |= 1U << 30;
}
break;
case 8:
if ((Src >> (sizeof(uint64_t) * 8 - 1)) & 1) {
NZCV |= 1U << 31;
}
if (Src == 0) {
NZCV |= 1U << 30;
}
break;
default: LOGMAN_MSG_A_FMT("Unknown {} Size: {}\n", __func__, OpSize); break;
}
GD = NZCV;
}
DEF_OP(Sub) {
auto Op = IROp->C<IR::IROp_Sub>();
const uint8_t OpSize = IROp->Size;
@@ -97,6 +182,59 @@ DEF_OP(Sub) {
}
}
DEF_OP(SubNZCV) {
auto Op = IROp->C<IR::IROp_SubNZCV>();
const uint8_t OpSize = Op->Size;
const uint64_t Src1 = *GetSrc<uint64_t*>(Data->SSAData, Op->Src1);
const uint64_t Src2 = *GetSrc<uint64_t*>(Data->SSAData, Op->Src2);
// Results returned in Arm64 NZCV format
// N = Sign bit
// Z = Is Zero
// C = Carry occured (Unsigned result can't fit within resulting register)
// V = Overflow occured (Signed result can't fit in to resulting register)
uint32_t NZCV{};
switch (OpSize) {
case 4: {
uint32_t Result = Src1 - Src2;
int32_t ResultSigned{};
if ((Result >> (sizeof(uint32_t) * 8 - 1)) & 1) {
NZCV |= 1U << 31;
}
if (Result == 0) {
NZCV |= 1U << 30;
}
if (__builtin_usub_overflow(Src1, Src2, &Result)) {
NZCV |= 1U << 29;
}
if (__builtin_ssub_overflow(Src1, Src2, &ResultSigned)) {
NZCV |= 1U << 28;
}
break;
}
case 8: {
uint64_t Result = Src1 - Src2;
int64_t ResultSigned{};
if ((Result >> (sizeof(uint64_t) * 8 - 1)) & 1) {
NZCV |= 1U << 31;
}
if (Result == 0) {
NZCV |= 1U << 30;
}
if (__builtin_usubl_overflow(Src1, Src2, &Result)) {
NZCV |= 1U << 29;
}
if (__builtin_ssubl_overflow(Src1, Src2, &ResultSigned)) {
NZCV |= 1U << 28;
}
break;
}
default: LOGMAN_MSG_A_FMT("Unknown {} Size: {}\n", __func__, OpSize); break;
}
GD = NZCV;
}
DEF_OP(Neg) {
auto Op = IROp->C<IR::IROp_Neg>();
const uint8_t OpSize = IROp->Size;
@@ -356,6 +494,44 @@ DEF_OP(Or) {
}
}
DEF_OP(Orlshl) {
auto Op = IROp->C<IR::IROp_Orlshl>();
const uint8_t OpSize = IROp->Size;
void *Src1 = GetSrc<void*>(Data->SSAData, Op->Src1);
void *Src2 = GetSrc<void*>(Data->SSAData, Op->Src2);
const auto BitShift = Op->BitShift;
const auto Func = [BitShift](auto a, auto b) { return a | (b << BitShift); };
switch (OpSize) {
DO_OP(1, uint8_t, Func)
DO_OP(2, uint16_t, Func)
DO_OP(4, uint32_t, Func)
DO_OP(8, uint64_t, Func)
DO_OP(16, __uint128_t, Func)
default: LOGMAN_MSG_A_FMT("Unknown size: {}", OpSize); break;
}
}
DEF_OP(Orlshr) {
auto Op = IROp->C<IR::IROp_Orlshr>();
const uint8_t OpSize = IROp->Size;
void *Src1 = GetSrc<void*>(Data->SSAData, Op->Src1);
void *Src2 = GetSrc<void*>(Data->SSAData, Op->Src2);
const auto BitShift = Op->BitShift;
const auto Func = [BitShift](auto a, auto b) { return a | (b >> BitShift); };
switch (OpSize) {
DO_OP(1, uint8_t, Func)
DO_OP(2, uint16_t, Func)
DO_OP(4, uint32_t, Func)
DO_OP(8, uint64_t, Func)
DO_OP(16, __uint128_t, Func)
default: LOGMAN_MSG_A_FMT("Unknown size: {}", OpSize); break;
}
}
DEF_OP(And) {
auto Op = IROp->C<IR::IROp_And>();
const uint8_t OpSize = IROp->Size;
@@ -50,7 +50,10 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
REGISTER_OP(INLINEENTRYPOINTOFFSET, InlineEntrypointOffset);
REGISTER_OP(CYCLECOUNTER, CycleCounter);
REGISTER_OP(ADD, Add);
REGISTER_OP(ADDNZCV, AddNZCV);
REGISTER_OP(TESTNZ, TestNZ);
REGISTER_OP(SUB, Sub);
REGISTER_OP(SUBNZCV, SubNZCV);
REGISTER_OP(NEG, Neg);
REGISTER_OP(ABS, Abs);
REGISTER_OP(MUL, Mul);
@@ -62,6 +65,8 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
REGISTER_OP(MULH, MulH);
REGISTER_OP(UMULH, UMulH);
REGISTER_OP(OR, Or);
REGISTER_OP(ORLSHL, Orlshl);
REGISTER_OP(ORLSHR, Orlshr);
REGISTER_OP(AND, And);
REGISTER_OP(ANDN, Andn);
REGISTER_OP(XOR, Xor);
@@ -85,7 +85,10 @@ namespace FEXCore::CPU {
DEF_OP(InlineEntrypointOffset);
DEF_OP(CycleCounter);
DEF_OP(Add);
DEF_OP(AddNZCV);
DEF_OP(TestNZ);
DEF_OP(Sub);
DEF_OP(SubNZCV);
DEF_OP(Neg);
DEF_OP(Abs);
DEF_OP(Mul);
@@ -97,6 +100,8 @@ namespace FEXCore::CPU {
DEF_OP(MulH);
DEF_OP(UMulH);
DEF_OP(Or);
DEF_OP(Orlshl);
DEF_OP(Orlshr);
DEF_OP(And);
DEF_OP(Andn);
DEF_OP(Xor);
@@ -1228,7 +1228,6 @@ fextl::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::ContextImpl *
CPUBackendFeatures GetArm64JITBackendFeatures() {
return CPUBackendFeatures {
.SupportsStaticRegisterAllocation = true,
.SupportsShiftedBitwise = true,
.SupportsFlags = true,
.SupportsSaturatingRoundingShifts = true,
};
@@ -106,6 +106,91 @@ DEF_OP(Add) {
mov(GetDst<RA_64>(Node), rax);
}
DEF_OP(AddNZCV) {
auto Op = IROp->C<IR::IROp_AddNZCV>();
const uint8_t OpSize = Op->Size;
// Results returned in Arm64 NZCV format
// N = Sign bit
// Z = Is Zero
// C = Carry occured (Unsigned result can't fit within resulting register)
// V = Overflow occured (Signed result can't fit in to resulting register)
Xbyak::Reg Src2 = TMP2;
uint64_t Const;
if (IsInlineConstant(Op->Src2, &Const)) {
mov(Src2, Const);
}
else {
Src2 = GetSrc<RA_64>(Op->Src2.ID());
}
switch (OpSize) {
case 4:
mov(TMP1.cvt32(), GetSrc<RA_32>(Op->Src1.ID()));
add(TMP1.cvt32(), Src2.cvt32());
break;
case 8:
mov(TMP1.cvt64(), GetSrc<RA_64>(Op->Src1.ID()));
add(TMP1.cvt64(), Src2.cvt64());
break;
default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize);
break;
}
mov(TMP1, 0);
mov(TMP2, 0);
mov(TMP3, 0);
mov(TMP4, 0);
sets(TMP1.cvt8());
setz(TMP2.cvt8());
setc(TMP3.cvt8());
seto(TMP4.cvt8());
// Flags NZCV in Tmps 1,2,3,4 respectively
shl(TMP1, 31);
shl(TMP2, 30);
shl(TMP3, 29);
shl(TMP4, 28);
or_(TMP1, TMP2);
or_(TMP1, TMP3);
or_(TMP1, TMP4);
mov(GetDst<RA_64>(Node), TMP1);
}
DEF_OP(TestNZ) {
auto Op = IROp->C<IR::IROp_TestNZ>();
const uint8_t OpSize = Op->Size;
// Results returned in Arm64 NZCV format
// N = Sign bit
// Z = Is Zero
// CV = 00
switch (OpSize) {
case 4:
mov(TMP1.cvt32(), GetSrc<RA_32>(Op->Src1.ID()));
shr(TMP1.cvt32(), OpSize * 8 - 1);
shl(TMP1.cvt32(), 31);
cmp(GetSrc<RA_32>(Op->Src1.ID()), 0);
mov(GetDst<RA_32>(Node), 0);
sete(GetDst<RA_32>(Node).cvt8());
shl(GetDst<RA_32>(Node), 30);
or_(GetDst<RA_32>(Node), TMP1.cvt32());
break;
case 8:
mov(TMP1, GetSrc<RA_64>(Op->Src1.ID()));
shr(TMP1, OpSize * 8 - 1);
shl(TMP1, 31);
cmp(GetSrc<RA_64>(Op->Src1.ID()), 0);
mov(GetDst<RA_64>(Node), 0);
sete(GetDst<RA_64>(Node).cvt8());
shl(GetDst<RA_64>(Node), 30);
or_(GetDst<RA_64>(Node), TMP1);
break;
default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize);
break;
}
}
DEF_OP(Sub) {
auto Op = IROp->C<IR::IROp_Sub>();
const uint8_t OpSize = IROp->Size;
@@ -139,6 +224,64 @@ DEF_OP(Sub) {
mov(GetDst<RA_64>(Node), rax);
}
DEF_OP(SubNZCV) {
auto Op = IROp->C<IR::IROp_SubNZCV>();
const uint8_t OpSize = Op->Size;
// Results returned in Arm64 NZCV format
// N = Sign bit
// Z = Is Zero
// C = Carry occured (Unsigned result can't fit within resulting register)
// V = Overflow occured (Signed result can't fit in to resulting register)
Xbyak::Reg Src1 = TMP1;
Xbyak::Reg Src2 = TMP2;
uint64_t Const;
if (IsInlineConstant(Op->Src1, &Const)) {
mov(Src1, Const);
}
else {
Src1 = GetSrc<RA_64>(Op->Src1.ID());
}
if (IsInlineConstant(Op->Src2, &Const)) {
mov(Src2, Const);
}
else {
Src2 = GetSrc<RA_64>(Op->Src2.ID());
}
switch (OpSize) {
case 4:
cmp(Src1.cvt32(), Src2.cvt32());
break;
case 8:
cmp(Src1.cvt64(), Src2.cvt64());
break;
default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize);
break;
}
mov(TMP1, 0);
mov(TMP2, 0);
mov(TMP3, 0);
mov(TMP4, 0);
sets(TMP1.cvt8());
setz(TMP2.cvt8());
setc(TMP3.cvt8());
seto(TMP4.cvt8());
// Flags NZCV in Tmps 1,2,3,4 respectively
shl(TMP1, 31);
shl(TMP2, 30);
shl(TMP3, 29);
shl(TMP4, 28);
or_(TMP1, TMP2);
or_(TMP1, TMP3);
or_(TMP1, TMP4);
mov(GetDst<RA_64>(Node), TMP1);
}
DEF_OP(Neg) {
auto Op = IROp->C<IR::IROp_Neg>();
const uint8_t OpSize = IROp->Size;
@@ -439,6 +582,68 @@ DEF_OP(Or) {
mov(Dst, rax);
}
DEF_OP(Orlshl) {
auto Op = IROp->C<IR::IROp_Orlshl>();
auto Dst = GetDst<RA_64>(Node);
const auto BitShift = Op->BitShift;
uint64_t Const;
if (IsInlineConstant(Op->Src2, &Const)) {
if (IROp->Size == 8) {
mov(Dst, GetSrc<RA_64>(Op->Src1.ID()));
or_(Dst, Const << BitShift);
}
else {
mov(Dst.cvt32(), GetSrc<RA_32>(Op->Src1.ID()));
or_(Dst.cvt32(), Const << BitShift);
}
} else {
if (IROp->Size == 8) {
mov(TMP2, GetSrc<RA_64>(Op->Src2.ID()));
mov(Dst, GetSrc<RA_64>(Op->Src1.ID()));
shl(TMP2, BitShift);
or_(Dst, TMP2);
}
else {
mov(TMP2.cvt32(), GetSrc<RA_32>(Op->Src2.ID()));
mov(Dst.cvt32(), GetSrc<RA_32>(Op->Src1.ID()));
shl(TMP2.cvt32(), BitShift);
or_(Dst.cvt32(), TMP2.cvt32());
}
}
}
DEF_OP(Orlshr) {
auto Op = IROp->C<IR::IROp_Orlshr>();
auto Dst = GetDst<RA_64>(Node);
const auto BitShift = Op->BitShift;
uint64_t Const;
if (IsInlineConstant(Op->Src2, &Const)) {
if (IROp->Size == 8) {
mov(Dst, GetSrc<RA_64>(Op->Src1.ID()));
or_(Dst, Const >> BitShift);
}
else {
mov(Dst.cvt32(), GetSrc<RA_32>(Op->Src1.ID()));
or_(Dst.cvt32(), Const >> BitShift);
}
} else {
if (IROp->Size == 8) {
mov(TMP2, GetSrc<RA_64>(Op->Src2.ID()));
mov(Dst, GetSrc<RA_64>(Op->Src1.ID()));
shr(TMP2, BitShift);
or_(Dst, TMP2);
}
else {
mov(TMP2.cvt32(), GetSrc<RA_32>(Op->Src2.ID()));
mov(Dst.cvt32(), GetSrc<RA_32>(Op->Src1.ID()));
shr(TMP2.cvt32(), BitShift);
or_(Dst.cvt32(), TMP2.cvt32());
}
}
}
DEF_OP(And) {
auto Op = IROp->C<IR::IROp_And>();
auto Dst = GetDst<RA_64>(Node);
@@ -1341,7 +1546,10 @@ void X86JITCore::RegisterALUHandlers() {
REGISTER_OP(INLINEENTRYPOINTOFFSET, InlineEntrypointOffset);
REGISTER_OP(CYCLECOUNTER, CycleCounter);
REGISTER_OP(ADD, Add);
REGISTER_OP(ADDNZCV, AddNZCV);
REGISTER_OP(TESTNZ, TestNZ);
REGISTER_OP(SUB, Sub);
REGISTER_OP(SUBNZCV, SubNZCV);
REGISTER_OP(NEG, Neg);
REGISTER_OP(ABS, Abs);
REGISTER_OP(MUL, Mul);
@@ -1353,6 +1561,8 @@ void X86JITCore::RegisterALUHandlers() {
REGISTER_OP(MULH, MulH);
REGISTER_OP(UMULH, UMulH);
REGISTER_OP(OR, Or);
REGISTER_OP(ORLSHL, Orlshl);
REGISTER_OP(ORLSHR, Orlshr);
REGISTER_OP(AND, And);
REGISTER_OP(ANDN, Andn);
REGISTER_OP(XOR, Xor);
@@ -244,7 +244,10 @@ private:
DEF_OP(InlineEntrypointOffset);
DEF_OP(CycleCounter);
DEF_OP(Add);
DEF_OP(AddNZCV);
DEF_OP(TestNZ);
DEF_OP(Sub);
DEF_OP(SubNZCV);
DEF_OP(Neg);
DEF_OP(Abs);
DEF_OP(Mul);
@@ -256,6 +259,8 @@ private:
DEF_OP(MulH);
DEF_OP(UMulH);
DEF_OP(Or);
DEF_OP(Orlshl);
DEF_OP(Orlshr);
DEF_OP(And);
DEF_OP(Andn);
DEF_OP(Xor);
@@ -1157,7 +1157,7 @@ private:
void SetNZ_ZeroCV(unsigned SrcSize, OrderedNode *Res) {
// The TestNZ opcode does this operation natively for 32-bit or 64-bit.
// Otherwise we can implement the functionality ourselves with some bit math.
if (CTX->BackendFeatures.SupportsFlags && SrcSize >= 4) {
if (SrcSize >= 4) {
CachedNZCV = _TestNZ(SrcSize, Res);
PossiblySetNZCVBits = (1u << 31) | (1u << 30);
} else {
@@ -1180,7 +1180,7 @@ private:
if (SetBits == 0)
return _Lshl(OpSize::i64Bit, Value, _Constant(Bit));
else if (CTX->BackendFeatures.SupportsShiftedBitwise && (SetBits & (1u << Bit)) == 0)
else if ((SetBits & (1u << Bit)) == 0)
return _Orlshl(OpSize::i32Bit, NZCV, Value, Bit);
else
return _Bfi(OpSize::i32Bit, 1, Bit, NZCV, Value);
@@ -143,8 +143,7 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) {
// SF/ZF and N/Z are together on both arm64 and x86_64, so we special case that.
bool GetNZ = (FlagsMask & (1 << FEXCore::X86State::RFLAG_SF_LOC)) &&
(FlagsMask & (1 << FEXCore::X86State::RFLAG_ZF_LOC)) &&
CTX->BackendFeatures.SupportsShiftedBitwise;
(FlagsMask & (1 << FEXCore::X86State::RFLAG_ZF_LOC));
// Handle CF first, since it's at bit 0 and hence doesn't need shift or OR.
if (FlagsMask & (1 << FEXCore::X86State::RFLAG_CF_LOC)) {
@@ -175,11 +174,7 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) {
else
Flag = GetRFLAG(FlagOffset);
if (CTX->BackendFeatures.SupportsShiftedBitwise) {
Original = _Orlshl(OpSize::i64Bit, Original, Flag, FlagOffset);
} else {
Original = _Bfi(OpSize::i32Bit, 1, FlagOffset, Original, Flag);
}
Original = _Orlshl(OpSize::i64Bit, Original, Flag, FlagOffset);
}
// OR in the SF/ZF flags at the end, allowing the lshr to fold with the OR
@@ -537,7 +532,7 @@ void OpDispatchBuilder::CalculateFlags_SUB(uint8_t SrcSize, OrderedNode *Res, Or
auto OldCF = UpdateCF ? nullptr : GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC);
// TODO: Could do this path for small sources if we have FEAT_FlagM
if (CTX->BackendFeatures.SupportsFlags && SrcSize >= 4) {
if (SrcSize >= 4) {
SetNZCV(_SubNZCV(OpSize, Src1, Src2));
} else {
// SF/ZF
@@ -580,7 +575,7 @@ void OpDispatchBuilder::CalculateFlags_ADD(uint8_t SrcSize, OrderedNode *Res, Or
auto OldCF = UpdateCF ? nullptr : GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC);
// TODO: Could do this path for small sources if we have FEAT_FlagM
if (CTX->BackendFeatures.SupportsFlags && SrcSize >= 4) {
if (SrcSize >= 4) {
SetNZCV(_AddNZCV(OpSize, Src1, Src2));
} else {
// SF/ZF
@@ -35,7 +35,6 @@ namespace CodeSerialize {
namespace CPU {
struct CPUBackendFeatures {
bool SupportsStaticRegisterAllocation = false;
bool SupportsShiftedBitwise = false;
bool SupportsFlags = false;
bool SupportsSaturatingRoundingShifts = false;
};