Merge pull request #3077 from Sonicadvance1/x86_shifted

FEXCore: Implements support for shifted bitwise ops
This commit is contained in:
Ryan Houdek authored and GitHub committed 2023-09-15 08:09:35 -07:00
commit d5782567e8
9 files changed
+407 -13

No files matched your search

@@ -82,6 +82,91 @@ DEF_OP(Add) {
}
}
DEF_OP(AddNZCV) {
auto Op = IROp->C<IR::IROp_AddNZCV>();
const uint8_t OpSize = Op->Size;
const uint64_t Src1 = *GetSrc<uint64_t*>(Data->SSAData, Op->Src1);
const uint64_t Src2 = *GetSrc<uint64_t*>(Data->SSAData, Op->Src2);
// Results returned in Arm64 NZCV format
// N = Sign bit
// Z = Is Zero
// C = Carry occured (Unsigned result can't fit within resulting register)
// V = Overflow occured (Signed result can't fit in to resulting register)
uint32_t NZCV{};
switch (OpSize) {
case 4: {
uint32_t Result = Src1 + Src2;
int32_t ResultSigned{};
if ((Result >> (sizeof(uint32_t) * 8 - 1)) & 1) {
NZCV |= 1U << 31;
}
if (Result == 0) {
NZCV |= 1U << 30;
}
if (__builtin_uadd_overflow(Src1, Src2, &Result)) {
NZCV |= 1U << 29;
}
if (__builtin_sadd_overflow(Src1, Src2, &ResultSigned)) {
NZCV |= 1U << 28;
}
break;
}
case 8: {
uint64_t Result = Src1 + Src2;
int64_t ResultSigned{};
if ((Result >> (sizeof(uint64_t) * 8 - 1)) & 1) {
NZCV |= 1U << 31;
}
if (Result == 0) {
NZCV |= 1U << 30;
}
if (__builtin_uaddl_overflow(Src1, Src2, &Result)) {
NZCV |= 1U << 29;
}
if (__builtin_saddl_overflow(Src1, Src2, &ResultSigned)) {
NZCV |= 1U << 28;
}
break;
}
default: LOGMAN_MSG_A_FMT("Unknown {} Size: {}\n", __func__, OpSize); break;
}
GD = NZCV;
}
DEF_OP(TestNZ) {
auto Op = IROp->C<IR::IROp_TestNZ>();
const uint8_t OpSize = Op->Size;
const uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Src1);
// Results returned in Arm64 NZCV format
// N = Sign bit
// Z = Is Zero
// CV = 00
uint32_t NZCV{};
switch (OpSize) {
case 4:
if ((Src >> (sizeof(uint32_t) * 8 - 1)) & 1) {
NZCV |= 1U << 31;
}
if (static_cast<uint32_t>(Src) == 0) {
NZCV |= 1U << 30;
}
break;
case 8:
if ((Src >> (sizeof(uint64_t) * 8 - 1)) & 1) {
NZCV |= 1U << 31;
}
if (Src == 0) {
NZCV |= 1U << 30;
}
break;
default: LOGMAN_MSG_A_FMT("Unknown {} Size: {}\n", __func__, OpSize); break;
}
GD = NZCV;
}
DEF_OP(Sub) {
auto Op = IROp->C<IR::IROp_Sub>();
const uint8_t OpSize = IROp->Size;
@@ -97,6 +182,59 @@ DEF_OP(Sub) {
}
}
DEF_OP(SubNZCV) {
auto Op = IROp->C<IR::IROp_SubNZCV>();
const uint8_t OpSize = Op->Size;
const uint64_t Src1 = *GetSrc<uint64_t*>(Data->SSAData, Op->Src1);
const uint64_t Src2 = *GetSrc<uint64_t*>(Data->SSAData, Op->Src2);
// Results returned in Arm64 NZCV format
// N = Sign bit
// Z = Is Zero
// C = Carry occured (Unsigned result can't fit within resulting register)
// V = Overflow occured (Signed result can't fit in to resulting register)
uint32_t NZCV{};
switch (OpSize) {
case 4: {
uint32_t Result = Src1 - Src2;
int32_t ResultSigned{};
if ((Result >> (sizeof(uint32_t) * 8 - 1)) & 1) {
NZCV |= 1U << 31;
}
if (Result == 0) {
NZCV |= 1U << 30;
}
if (__builtin_usub_overflow(Src1, Src2, &Result)) {
NZCV |= 1U << 29;
}
if (__builtin_ssub_overflow(Src1, Src2, &ResultSigned)) {
NZCV |= 1U << 28;
}
break;
}
case 8: {
uint64_t Result = Src1 - Src2;
int64_t ResultSigned{};
if ((Result >> (sizeof(uint64_t) * 8 - 1)) & 1) {
NZCV |= 1U << 31;
}
if (Result == 0) {
NZCV |= 1U << 30;
}
if (__builtin_usubl_overflow(Src1, Src2, &Result)) {
NZCV |= 1U << 29;
}
if (__builtin_ssubl_overflow(Src1, Src2, &ResultSigned)) {
NZCV |= 1U << 28;
}
break;
}
default: LOGMAN_MSG_A_FMT("Unknown {} Size: {}\n", __func__, OpSize); break;
}
GD = NZCV;
}
DEF_OP(Neg) {
auto Op = IROp->C<IR::IROp_Neg>();
const uint8_t OpSize = IROp->Size;
@@ -356,6 +494,44 @@ DEF_OP(Or) {
}
}
DEF_OP(Orlshl) {
auto Op = IROp->C<IR::IROp_Orlshl>();
const uint8_t OpSize = IROp->Size;
void *Src1 = GetSrc<void*>(Data->SSAData, Op->Src1);
void *Src2 = GetSrc<void*>(Data->SSAData, Op->Src2);
const auto BitShift = Op->BitShift;
const auto Func = [BitShift](auto a, auto b) { return a | (b << BitShift); };
switch (OpSize) {
DO_OP(1, uint8_t, Func)
DO_OP(2, uint16_t, Func)
DO_OP(4, uint32_t, Func)
DO_OP(8, uint64_t, Func)
DO_OP(16, __uint128_t, Func)
default: LOGMAN_MSG_A_FMT("Unknown size: {}", OpSize); break;
}
}
DEF_OP(Orlshr) {
auto Op = IROp->C<IR::IROp_Orlshr>();
const uint8_t OpSize = IROp->Size;
void *Src1 = GetSrc<void*>(Data->SSAData, Op->Src1);
void *Src2 = GetSrc<void*>(Data->SSAData, Op->Src2);
const auto BitShift = Op->BitShift;
const auto Func = [BitShift](auto a, auto b) { return a | (b >> BitShift); };
switch (OpSize) {
DO_OP(1, uint8_t, Func)
DO_OP(2, uint16_t, Func)
DO_OP(4, uint32_t, Func)
DO_OP(8, uint64_t, Func)
DO_OP(16, __uint128_t, Func)
default: LOGMAN_MSG_A_FMT("Unknown size: {}", OpSize); break;
}
}
DEF_OP(And) {
auto Op = IROp->C<IR::IROp_And>();
const uint8_t OpSize = IROp->Size;
@@ -50,7 +50,10 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
REGISTER_OP(INLINEENTRYPOINTOFFSET, InlineEntrypointOffset);
REGISTER_OP(CYCLECOUNTER, CycleCounter);
REGISTER_OP(ADD, Add);
REGISTER_OP(ADDNZCV, AddNZCV);
REGISTER_OP(TESTNZ, TestNZ);
REGISTER_OP(SUB, Sub);
REGISTER_OP(SUBNZCV, SubNZCV);
REGISTER_OP(NEG, Neg);
REGISTER_OP(ABS, Abs);
REGISTER_OP(MUL, Mul);
@@ -62,6 +65,8 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
REGISTER_OP(MULH, MulH);
REGISTER_OP(UMULH, UMulH);
REGISTER_OP(OR, Or);
REGISTER_OP(ORLSHL, Orlshl);
REGISTER_OP(ORLSHR, Orlshr);
REGISTER_OP(AND, And);
REGISTER_OP(ANDN, Andn);
REGISTER_OP(XOR, Xor);
@@ -84,7 +84,10 @@ namespace FEXCore::CPU {
DEF_OP(InlineEntrypointOffset);
DEF_OP(CycleCounter);
DEF_OP(Add);
DEF_OP(AddNZCV);
DEF_OP(TestNZ);
DEF_OP(Sub);
DEF_OP(SubNZCV);
DEF_OP(Neg);
DEF_OP(Abs);
DEF_OP(Mul);
@@ -96,6 +99,8 @@ namespace FEXCore::CPU {
DEF_OP(MulH);
DEF_OP(UMulH);
DEF_OP(Or);
DEF_OP(Orlshl);
DEF_OP(Orlshr);
DEF_OP(And);
DEF_OP(Andn);
DEF_OP(Xor);
@@ -1223,7 +1223,6 @@ fextl::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::ContextImpl *
CPUBackendFeatures GetArm64JITBackendFeatures() {
return CPUBackendFeatures {
.SupportsStaticRegisterAllocation = true,
.SupportsShiftedBitwise = true,
.SupportsFlags = true,
.SupportsSaturatingRoundingShifts = true,
.SupportsVTBL2 = true,
@@ -106,6 +106,91 @@ DEF_OP(Add) {
mov(GetDst<RA_64>(Node), rax);
}
DEF_OP(AddNZCV) {
auto Op = IROp->C<IR::IROp_AddNZCV>();
const uint8_t OpSize = Op->Size;
// Results returned in Arm64 NZCV format
// N = Sign bit
// Z = Is Zero
// C = Carry occured (Unsigned result can't fit within resulting register)
// V = Overflow occured (Signed result can't fit in to resulting register)
Xbyak::Reg Src2 = TMP2;
uint64_t Const;
if (IsInlineConstant(Op->Src2, &Const)) {
mov(Src2, Const);
}
else {
Src2 = GetSrc<RA_64>(Op->Src2.ID());
}
switch (OpSize) {
case 4:
mov(TMP1.cvt32(), GetSrc<RA_32>(Op->Src1.ID()));
add(TMP1.cvt32(), Src2.cvt32());
break;
case 8:
mov(TMP1.cvt64(), GetSrc<RA_64>(Op->Src1.ID()));
add(TMP1.cvt64(), Src2.cvt64());
break;
default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize);
break;
}
mov(TMP1, 0);
mov(TMP2, 0);
mov(TMP3, 0);
mov(TMP4, 0);
sets(TMP1.cvt8());
setz(TMP2.cvt8());
setc(TMP3.cvt8());
seto(TMP4.cvt8());
// Flags NZCV in Tmps 1,2,3,4 respectively
shl(TMP1, 31);
shl(TMP2, 30);
shl(TMP3, 29);
shl(TMP4, 28);
or_(TMP1, TMP2);
or_(TMP1, TMP3);
or_(TMP1, TMP4);
mov(GetDst<RA_64>(Node), TMP1);
}
DEF_OP(TestNZ) {
auto Op = IROp->C<IR::IROp_TestNZ>();
const uint8_t OpSize = Op->Size;
// Results returned in Arm64 NZCV format
// N = Sign bit
// Z = Is Zero
// CV = 00
switch (OpSize) {
case 4:
mov(TMP1.cvt32(), GetSrc<RA_32>(Op->Src1.ID()));
shr(TMP1.cvt32(), OpSize * 8 - 1);
shl(TMP1.cvt32(), 31);
cmp(GetSrc<RA_32>(Op->Src1.ID()), 0);
mov(GetDst<RA_32>(Node), 0);
sete(GetDst<RA_32>(Node).cvt8());
shl(GetDst<RA_32>(Node), 30);
or_(GetDst<RA_32>(Node), TMP1.cvt32());
break;
case 8:
mov(TMP1, GetSrc<RA_64>(Op->Src1.ID()));
shr(TMP1, OpSize * 8 - 1);
shl(TMP1, 31);
cmp(GetSrc<RA_64>(Op->Src1.ID()), 0);
mov(GetDst<RA_64>(Node), 0);
sete(GetDst<RA_64>(Node).cvt8());
shl(GetDst<RA_64>(Node), 30);
or_(GetDst<RA_64>(Node), TMP1);
break;
default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize);
break;
}
}
DEF_OP(Sub) {
auto Op = IROp->C<IR::IROp_Sub>();
const uint8_t OpSize = IROp->Size;
@@ -139,6 +224,64 @@ DEF_OP(Sub) {
mov(GetDst<RA_64>(Node), rax);
}
DEF_OP(SubNZCV) {
auto Op = IROp->C<IR::IROp_SubNZCV>();
const uint8_t OpSize = Op->Size;
// Results returned in Arm64 NZCV format
// N = Sign bit
// Z = Is Zero
// C = Carry occured (Unsigned result can't fit within resulting register)
// V = Overflow occured (Signed result can't fit in to resulting register)
Xbyak::Reg Src1 = TMP1;
Xbyak::Reg Src2 = TMP2;
uint64_t Const;
if (IsInlineConstant(Op->Src1, &Const)) {
mov(Src1, Const);
}
else {
Src1 = GetSrc<RA_64>(Op->Src1.ID());
}
if (IsInlineConstant(Op->Src2, &Const)) {
mov(Src2, Const);
}
else {
Src2 = GetSrc<RA_64>(Op->Src2.ID());
}
switch (OpSize) {
case 4:
cmp(Src1.cvt32(), Src2.cvt32());
break;
case 8:
cmp(Src1.cvt64(), Src2.cvt64());
break;
default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize);
break;
}
mov(TMP1, 0);
mov(TMP2, 0);
mov(TMP3, 0);
mov(TMP4, 0);
sets(TMP1.cvt8());
setz(TMP2.cvt8());
setc(TMP3.cvt8());
seto(TMP4.cvt8());
// Flags NZCV in Tmps 1,2,3,4 respectively
shl(TMP1, 31);
shl(TMP2, 30);
shl(TMP3, 29);
shl(TMP4, 28);
or_(TMP1, TMP2);
or_(TMP1, TMP3);
or_(TMP1, TMP4);
mov(GetDst<RA_64>(Node), TMP1);
}
DEF_OP(Neg) {
auto Op = IROp->C<IR::IROp_Neg>();
const uint8_t OpSize = IROp->Size;
@@ -439,6 +582,68 @@ DEF_OP(Or) {
mov(Dst, rax);
}
DEF_OP(Orlshl) {
auto Op = IROp->C<IR::IROp_Orlshl>();
auto Dst = GetDst<RA_64>(Node);
const auto BitShift = Op->BitShift;
uint64_t Const;
if (IsInlineConstant(Op->Src2, &Const)) {
if (IROp->Size == 8) {
mov(Dst, GetSrc<RA_64>(Op->Src1.ID()));
or_(Dst, Const << BitShift);
}
else {
mov(Dst.cvt32(), GetSrc<RA_32>(Op->Src1.ID()));
or_(Dst.cvt32(), Const << BitShift);
}
} else {
if (IROp->Size == 8) {
mov(TMP2, GetSrc<RA_64>(Op->Src2.ID()));
mov(Dst, GetSrc<RA_64>(Op->Src1.ID()));
shl(TMP2, BitShift);
or_(Dst, TMP2);
}
else {
mov(TMP2.cvt32(), GetSrc<RA_32>(Op->Src2.ID()));
mov(Dst.cvt32(), GetSrc<RA_32>(Op->Src1.ID()));
shl(TMP2.cvt32(), BitShift);
or_(Dst.cvt32(), TMP2.cvt32());
}
}
}
DEF_OP(Orlshr) {
auto Op = IROp->C<IR::IROp_Orlshr>();
auto Dst = GetDst<RA_64>(Node);
const auto BitShift = Op->BitShift;
uint64_t Const;
if (IsInlineConstant(Op->Src2, &Const)) {
if (IROp->Size == 8) {
mov(Dst, GetSrc<RA_64>(Op->Src1.ID()));
or_(Dst, Const >> BitShift);
}
else {
mov(Dst.cvt32(), GetSrc<RA_32>(Op->Src1.ID()));
or_(Dst.cvt32(), Const >> BitShift);
}
} else {
if (IROp->Size == 8) {
mov(TMP2, GetSrc<RA_64>(Op->Src2.ID()));
mov(Dst, GetSrc<RA_64>(Op->Src1.ID()));
shr(TMP2, BitShift);
or_(Dst, TMP2);
}
else {
mov(TMP2.cvt32(), GetSrc<RA_32>(Op->Src2.ID()));
mov(Dst.cvt32(), GetSrc<RA_32>(Op->Src1.ID()));
shr(TMP2.cvt32(), BitShift);
or_(Dst.cvt32(), TMP2.cvt32());
}
}
}
DEF_OP(And) {
auto Op = IROp->C<IR::IROp_And>();
auto Dst = GetDst<RA_64>(Node);
@@ -1369,7 +1574,10 @@ void X86JITCore::RegisterALUHandlers() {
REGISTER_OP(INLINEENTRYPOINTOFFSET, InlineEntrypointOffset);
REGISTER_OP(CYCLECOUNTER, CycleCounter);
REGISTER_OP(ADD, Add);
REGISTER_OP(ADDNZCV, AddNZCV);
REGISTER_OP(TESTNZ, TestNZ);
REGISTER_OP(SUB, Sub);
REGISTER_OP(SUBNZCV, SubNZCV);
REGISTER_OP(NEG, Neg);
REGISTER_OP(ABS, Abs);
REGISTER_OP(MUL, Mul);
@@ -1381,6 +1589,8 @@ void X86JITCore::RegisterALUHandlers() {
REGISTER_OP(MULH, MulH);
REGISTER_OP(UMULH, UMulH);
REGISTER_OP(OR, Or);
REGISTER_OP(ORLSHL, Orlshl);
REGISTER_OP(ORLSHR, Orlshr);
REGISTER_OP(AND, And);
REGISTER_OP(ANDN, Andn);
REGISTER_OP(XOR, Xor);
@@ -244,7 +244,10 @@ private:
DEF_OP(InlineEntrypointOffset);
DEF_OP(CycleCounter);
DEF_OP(Add);
DEF_OP(AddNZCV);
DEF_OP(TestNZ);
DEF_OP(Sub);
DEF_OP(SubNZCV);
DEF_OP(Neg);
DEF_OP(Abs);
DEF_OP(Mul);
@@ -256,6 +259,8 @@ private:
DEF_OP(MulH);
DEF_OP(UMulH);
DEF_OP(Or);
DEF_OP(Orlshl);
DEF_OP(Orlshr);
DEF_OP(And);
DEF_OP(Andn);
DEF_OP(Xor);
@@ -1157,7 +1157,7 @@ private:
void SetNZ_ZeroCV(unsigned SrcSize, OrderedNode *Res) {
// The TestNZ opcode does this operation natively for 32-bit or 64-bit.
// Otherwise we can implement the functionality ourselves with some bit math.
if (CTX->BackendFeatures.SupportsFlags && SrcSize >= 4) {
if (SrcSize >= 4) {
CachedNZCV = _TestNZ(SrcSize, Res);
PossiblySetNZCVBits = (1u << 31) | (1u << 30);
} else {
@@ -1180,7 +1180,7 @@ private:
if (SetBits == 0)
return _Lshl(OpSize::i64Bit, Value, _Constant(Bit));
else if (CTX->BackendFeatures.SupportsShiftedBitwise && (SetBits & (1u << Bit)) == 0)
else if ((SetBits & (1u << Bit)) == 0)
return _Orlshl(OpSize::i32Bit, NZCV, Value, Bit);
else
return _Bfi(OpSize::i32Bit, 1, Bit, NZCV, Value);
@@ -143,8 +143,7 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) {
// SF/ZF and N/Z are together on both arm64 and x86_64, so we special case that.
bool GetNZ = (FlagsMask & (1 << FEXCore::X86State::RFLAG_SF_LOC)) &&
(FlagsMask & (1 << FEXCore::X86State::RFLAG_ZF_LOC)) &&
CTX->BackendFeatures.SupportsShiftedBitwise;
(FlagsMask & (1 << FEXCore::X86State::RFLAG_ZF_LOC));
// Handle CF first, since it's at bit 0 and hence doesn't need shift or OR.
if (FlagsMask & (1 << FEXCore::X86State::RFLAG_CF_LOC)) {
@@ -175,11 +174,7 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) {
else
Flag = GetRFLAG(FlagOffset);
if (CTX->BackendFeatures.SupportsShiftedBitwise) {
Original = _Orlshl(OpSize::i64Bit, Original, Flag, FlagOffset);
} else {
Original = _Bfi(OpSize::i32Bit, 1, FlagOffset, Original, Flag);
}
Original = _Orlshl(OpSize::i64Bit, Original, Flag, FlagOffset);
}
// OR in the SF/ZF flags at the end, allowing the lshr to fold with the OR
@@ -537,7 +532,7 @@ void OpDispatchBuilder::CalculateFlags_SUB(uint8_t SrcSize, OrderedNode *Res, Or
auto OldCF = UpdateCF ? nullptr : GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC);
// TODO: Could do this path for small sources if we have FEAT_FlagM
if (CTX->BackendFeatures.SupportsFlags && SrcSize >= 4) {
if (SrcSize >= 4) {
SetNZCV(_SubNZCV(OpSize, Src1, Src2));
} else {
// SF/ZF
@@ -580,7 +575,7 @@ void OpDispatchBuilder::CalculateFlags_ADD(uint8_t SrcSize, OrderedNode *Res, Or
auto OldCF = UpdateCF ? nullptr : GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC);
// TODO: Could do this path for small sources if we have FEAT_FlagM
if (CTX->BackendFeatures.SupportsFlags && SrcSize >= 4) {
if (SrcSize >= 4) {
SetNZCV(_AddNZCV(OpSize, Src1, Src2));
} else {
// SF/ZF
@@ -35,7 +35,6 @@ namespace CodeSerialize {
namespace CPU {
struct CPUBackendFeatures {
bool SupportsStaticRegisterAllocation = false;
bool SupportsShiftedBitwise = false;
bool SupportsFlags = false;
bool SupportsSaturatingRoundingShifts = false;
bool SupportsVTBL2 = false;