diff --git a/FEXCore/Source/Interface/Core/Interpreter/ALUOps.cpp b/FEXCore/Source/Interface/Core/Interpreter/ALUOps.cpp index 7092a32e4..3d4469c0f 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/ALUOps.cpp +++ b/FEXCore/Source/Interface/Core/Interpreter/ALUOps.cpp @@ -82,6 +82,91 @@ DEF_OP(Add) { } } +DEF_OP(AddNZCV) { + auto Op = IROp->C(); + const uint8_t OpSize = Op->Size; + + const uint64_t Src1 = *GetSrc(Data->SSAData, Op->Src1); + const uint64_t Src2 = *GetSrc(Data->SSAData, Op->Src2); + + // Results returned in Arm64 NZCV format + // N = Sign bit + // Z = Is Zero + // C = Carry occured (Unsigned result can't fit within resulting register) + // V = Overflow occured (Signed result can't fit in to resulting register) + uint32_t NZCV{}; + switch (OpSize) { + case 4: { + uint32_t Result = Src1 + Src2; + int32_t ResultSigned{}; + if ((Result >> (sizeof(uint32_t) * 8 - 1)) & 1) { + NZCV |= 1U << 31; + } + if (Result == 0) { + NZCV |= 1U << 30; + } + if (__builtin_uadd_overflow(Src1, Src2, &Result)) { + NZCV |= 1U << 29; + } + if (__builtin_sadd_overflow(Src1, Src2, &ResultSigned)) { + NZCV |= 1U << 28; + } + break; + } + case 8: { + uint64_t Result = Src1 + Src2; + int64_t ResultSigned{}; + if ((Result >> (sizeof(uint64_t) * 8 - 1)) & 1) { + NZCV |= 1U << 31; + } + if (Result == 0) { + NZCV |= 1U << 30; + } + if (__builtin_uaddl_overflow(Src1, Src2, &Result)) { + NZCV |= 1U << 29; + } + if (__builtin_saddl_overflow(Src1, Src2, &ResultSigned)) { + NZCV |= 1U << 28; + } + break; + } + default: LOGMAN_MSG_A_FMT("Unknown {} Size: {}\n", __func__, OpSize); break; + } + GD = NZCV; +} + +DEF_OP(TestNZ) { + auto Op = IROp->C(); + const uint8_t OpSize = Op->Size; + + const uint64_t Src = *GetSrc(Data->SSAData, Op->Src1); + // Results returned in Arm64 NZCV format + // N = Sign bit + // Z = Is Zero + // CV = 00 + uint32_t NZCV{}; + switch (OpSize) { + case 4: + if ((Src >> (sizeof(uint32_t) * 8 - 1)) & 1) { + NZCV |= 1U << 31; + } + if (static_cast(Src) == 0) { + NZCV |= 1U << 30; + } + break; + case 8: + if ((Src >> (sizeof(uint64_t) * 8 - 1)) & 1) { + NZCV |= 1U << 31; + } + if (Src == 0) { + NZCV |= 1U << 30; + } + break; + default: LOGMAN_MSG_A_FMT("Unknown {} Size: {}\n", __func__, OpSize); break; + } + GD = NZCV; +} + DEF_OP(Sub) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; @@ -97,6 +182,59 @@ DEF_OP(Sub) { } } +DEF_OP(SubNZCV) { + auto Op = IROp->C(); + const uint8_t OpSize = Op->Size; + + const uint64_t Src1 = *GetSrc(Data->SSAData, Op->Src1); + const uint64_t Src2 = *GetSrc(Data->SSAData, Op->Src2); + + // Results returned in Arm64 NZCV format + // N = Sign bit + // Z = Is Zero + // C = Carry occured (Unsigned result can't fit within resulting register) + // V = Overflow occured (Signed result can't fit in to resulting register) + uint32_t NZCV{}; + switch (OpSize) { + case 4: { + uint32_t Result = Src1 - Src2; + int32_t ResultSigned{}; + if ((Result >> (sizeof(uint32_t) * 8 - 1)) & 1) { + NZCV |= 1U << 31; + } + if (Result == 0) { + NZCV |= 1U << 30; + } + if (__builtin_usub_overflow(Src1, Src2, &Result)) { + NZCV |= 1U << 29; + } + if (__builtin_ssub_overflow(Src1, Src2, &ResultSigned)) { + NZCV |= 1U << 28; + } + break; + } + case 8: { + uint64_t Result = Src1 - Src2; + int64_t ResultSigned{}; + if ((Result >> (sizeof(uint64_t) * 8 - 1)) & 1) { + NZCV |= 1U << 31; + } + if (Result == 0) { + NZCV |= 1U << 30; + } + if (__builtin_usubl_overflow(Src1, Src2, &Result)) { + NZCV |= 1U << 29; + } + if (__builtin_ssubl_overflow(Src1, Src2, &ResultSigned)) { + NZCV |= 1U << 28; + } + break; + } + default: LOGMAN_MSG_A_FMT("Unknown {} Size: {}\n", __func__, OpSize); break; + } + GD = NZCV; +} + DEF_OP(Neg) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; @@ -356,6 +494,44 @@ DEF_OP(Or) { } } +DEF_OP(Orlshl) { + auto Op = IROp->C(); + const uint8_t OpSize = IROp->Size; + + void *Src1 = GetSrc(Data->SSAData, Op->Src1); + void *Src2 = GetSrc(Data->SSAData, Op->Src2); + const auto BitShift = Op->BitShift; + const auto Func = [BitShift](auto a, auto b) { return a | (b << BitShift); }; + + switch (OpSize) { + DO_OP(1, uint8_t, Func) + DO_OP(2, uint16_t, Func) + DO_OP(4, uint32_t, Func) + DO_OP(8, uint64_t, Func) + DO_OP(16, __uint128_t, Func) + default: LOGMAN_MSG_A_FMT("Unknown size: {}", OpSize); break; + } +} + +DEF_OP(Orlshr) { + auto Op = IROp->C(); + const uint8_t OpSize = IROp->Size; + + void *Src1 = GetSrc(Data->SSAData, Op->Src1); + void *Src2 = GetSrc(Data->SSAData, Op->Src2); + const auto BitShift = Op->BitShift; + const auto Func = [BitShift](auto a, auto b) { return a | (b >> BitShift); }; + + switch (OpSize) { + DO_OP(1, uint8_t, Func) + DO_OP(2, uint16_t, Func) + DO_OP(4, uint32_t, Func) + DO_OP(8, uint64_t, Func) + DO_OP(16, __uint128_t, Func) + default: LOGMAN_MSG_A_FMT("Unknown size: {}", OpSize); break; + } +} + DEF_OP(And) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; diff --git a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp index c7094dce1..a8d8b12eb 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp +++ b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp @@ -50,7 +50,10 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] { REGISTER_OP(INLINEENTRYPOINTOFFSET, InlineEntrypointOffset); REGISTER_OP(CYCLECOUNTER, CycleCounter); REGISTER_OP(ADD, Add); + REGISTER_OP(ADDNZCV, AddNZCV); + REGISTER_OP(TESTNZ, TestNZ); REGISTER_OP(SUB, Sub); + REGISTER_OP(SUBNZCV, SubNZCV); REGISTER_OP(NEG, Neg); REGISTER_OP(ABS, Abs); REGISTER_OP(MUL, Mul); @@ -62,6 +65,8 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] { REGISTER_OP(MULH, MulH); REGISTER_OP(UMULH, UMulH); REGISTER_OP(OR, Or); + REGISTER_OP(ORLSHL, Orlshl); + REGISTER_OP(ORLSHR, Orlshr); REGISTER_OP(AND, And); REGISTER_OP(ANDN, Andn); REGISTER_OP(XOR, Xor); diff --git a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h index 5c18c127c..3ff425f60 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h +++ b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h @@ -84,7 +84,10 @@ namespace FEXCore::CPU { DEF_OP(InlineEntrypointOffset); DEF_OP(CycleCounter); DEF_OP(Add); + DEF_OP(AddNZCV); + DEF_OP(TestNZ); DEF_OP(Sub); + DEF_OP(SubNZCV); DEF_OP(Neg); DEF_OP(Abs); DEF_OP(Mul); @@ -96,6 +99,8 @@ namespace FEXCore::CPU { DEF_OP(MulH); DEF_OP(UMulH); DEF_OP(Or); + DEF_OP(Orlshl); + DEF_OP(Orlshr); DEF_OP(And); DEF_OP(Andn); DEF_OP(Xor); diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp index 0ea9e9f93..802a016d1 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp @@ -1223,7 +1223,6 @@ fextl::unique_ptr CreateArm64JITCore(FEXCore::Context::ContextImpl * CPUBackendFeatures GetArm64JITBackendFeatures() { return CPUBackendFeatures { .SupportsStaticRegisterAllocation = true, - .SupportsShiftedBitwise = true, .SupportsFlags = true, .SupportsSaturatingRoundingShifts = true, .SupportsVTBL2 = true, diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/ALUOps.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/ALUOps.cpp index 919d604f3..95ff9a61c 100644 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/ALUOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/x86_64/ALUOps.cpp @@ -106,6 +106,91 @@ DEF_OP(Add) { mov(GetDst(Node), rax); } +DEF_OP(AddNZCV) { + auto Op = IROp->C(); + const uint8_t OpSize = Op->Size; + + // Results returned in Arm64 NZCV format + // N = Sign bit + // Z = Is Zero + // C = Carry occured (Unsigned result can't fit within resulting register) + // V = Overflow occured (Signed result can't fit in to resulting register) + + Xbyak::Reg Src2 = TMP2; + uint64_t Const; + if (IsInlineConstant(Op->Src2, &Const)) { + mov(Src2, Const); + } + else { + Src2 = GetSrc(Op->Src2.ID()); + } + + switch (OpSize) { + case 4: + mov(TMP1.cvt32(), GetSrc(Op->Src1.ID())); + add(TMP1.cvt32(), Src2.cvt32()); + break; + case 8: + mov(TMP1.cvt64(), GetSrc(Op->Src1.ID())); + add(TMP1.cvt64(), Src2.cvt64()); + break; + default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize); + break; + } + + mov(TMP1, 0); + mov(TMP2, 0); + mov(TMP3, 0); + mov(TMP4, 0); + sets(TMP1.cvt8()); + setz(TMP2.cvt8()); + setc(TMP3.cvt8()); + seto(TMP4.cvt8()); + // Flags NZCV in Tmps 1,2,3,4 respectively + shl(TMP1, 31); + shl(TMP2, 30); + shl(TMP3, 29); + shl(TMP4, 28); + or_(TMP1, TMP2); + or_(TMP1, TMP3); + or_(TMP1, TMP4); + mov(GetDst(Node), TMP1); +} + +DEF_OP(TestNZ) { + auto Op = IROp->C(); + const uint8_t OpSize = Op->Size; + + // Results returned in Arm64 NZCV format + // N = Sign bit + // Z = Is Zero + // CV = 00 + switch (OpSize) { + case 4: + mov(TMP1.cvt32(), GetSrc(Op->Src1.ID())); + shr(TMP1.cvt32(), OpSize * 8 - 1); + shl(TMP1.cvt32(), 31); + cmp(GetSrc(Op->Src1.ID()), 0); + mov(GetDst(Node), 0); + sete(GetDst(Node).cvt8()); + shl(GetDst(Node), 30); + or_(GetDst(Node), TMP1.cvt32()); + break; + case 8: + mov(TMP1, GetSrc(Op->Src1.ID())); + shr(TMP1, OpSize * 8 - 1); + shl(TMP1, 31); + cmp(GetSrc(Op->Src1.ID()), 0); + mov(GetDst(Node), 0); + sete(GetDst(Node).cvt8()); + shl(GetDst(Node), 30); + or_(GetDst(Node), TMP1); + break; + default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize); + break; + } +} + DEF_OP(Sub) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; @@ -139,6 +224,64 @@ DEF_OP(Sub) { mov(GetDst(Node), rax); } +DEF_OP(SubNZCV) { + auto Op = IROp->C(); + const uint8_t OpSize = Op->Size; + + // Results returned in Arm64 NZCV format + // N = Sign bit + // Z = Is Zero + // C = Carry occured (Unsigned result can't fit within resulting register) + // V = Overflow occured (Signed result can't fit in to resulting register) + + Xbyak::Reg Src1 = TMP1; + Xbyak::Reg Src2 = TMP2; + uint64_t Const; + if (IsInlineConstant(Op->Src1, &Const)) { + mov(Src1, Const); + } + else { + Src1 = GetSrc(Op->Src1.ID()); + } + + if (IsInlineConstant(Op->Src2, &Const)) { + mov(Src2, Const); + } + else { + Src2 = GetSrc(Op->Src2.ID()); + } + + switch (OpSize) { + case 4: + cmp(Src1.cvt32(), Src2.cvt32()); + break; + case 8: + cmp(Src1.cvt64(), Src2.cvt64()); + break; + default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize); + break; + } + + mov(TMP1, 0); + mov(TMP2, 0); + mov(TMP3, 0); + mov(TMP4, 0); + sets(TMP1.cvt8()); + setz(TMP2.cvt8()); + setc(TMP3.cvt8()); + seto(TMP4.cvt8()); + // Flags NZCV in Tmps 1,2,3,4 respectively + shl(TMP1, 31); + shl(TMP2, 30); + shl(TMP3, 29); + shl(TMP4, 28); + or_(TMP1, TMP2); + or_(TMP1, TMP3); + or_(TMP1, TMP4); + mov(GetDst(Node), TMP1); +} + + DEF_OP(Neg) { auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; @@ -439,6 +582,68 @@ DEF_OP(Or) { mov(Dst, rax); } +DEF_OP(Orlshl) { + auto Op = IROp->C(); + auto Dst = GetDst(Node); + const auto BitShift = Op->BitShift; + + uint64_t Const; + if (IsInlineConstant(Op->Src2, &Const)) { + if (IROp->Size == 8) { + mov(Dst, GetSrc(Op->Src1.ID())); + or_(Dst, Const << BitShift); + } + else { + mov(Dst.cvt32(), GetSrc(Op->Src1.ID())); + or_(Dst.cvt32(), Const << BitShift); + } + } else { + if (IROp->Size == 8) { + mov(TMP2, GetSrc(Op->Src2.ID())); + mov(Dst, GetSrc(Op->Src1.ID())); + shl(TMP2, BitShift); + or_(Dst, TMP2); + } + else { + mov(TMP2.cvt32(), GetSrc(Op->Src2.ID())); + mov(Dst.cvt32(), GetSrc(Op->Src1.ID())); + shl(TMP2.cvt32(), BitShift); + or_(Dst.cvt32(), TMP2.cvt32()); + } + } +} + +DEF_OP(Orlshr) { + auto Op = IROp->C(); + auto Dst = GetDst(Node); + const auto BitShift = Op->BitShift; + + uint64_t Const; + if (IsInlineConstant(Op->Src2, &Const)) { + if (IROp->Size == 8) { + mov(Dst, GetSrc(Op->Src1.ID())); + or_(Dst, Const >> BitShift); + } + else { + mov(Dst.cvt32(), GetSrc(Op->Src1.ID())); + or_(Dst.cvt32(), Const >> BitShift); + } + } else { + if (IROp->Size == 8) { + mov(TMP2, GetSrc(Op->Src2.ID())); + mov(Dst, GetSrc(Op->Src1.ID())); + shr(TMP2, BitShift); + or_(Dst, TMP2); + } + else { + mov(TMP2.cvt32(), GetSrc(Op->Src2.ID())); + mov(Dst.cvt32(), GetSrc(Op->Src1.ID())); + shr(TMP2.cvt32(), BitShift); + or_(Dst.cvt32(), TMP2.cvt32()); + } + } +} + DEF_OP(And) { auto Op = IROp->C(); auto Dst = GetDst(Node); @@ -1369,7 +1574,10 @@ void X86JITCore::RegisterALUHandlers() { REGISTER_OP(INLINEENTRYPOINTOFFSET, InlineEntrypointOffset); REGISTER_OP(CYCLECOUNTER, CycleCounter); REGISTER_OP(ADD, Add); + REGISTER_OP(ADDNZCV, AddNZCV); + REGISTER_OP(TESTNZ, TestNZ); REGISTER_OP(SUB, Sub); + REGISTER_OP(SUBNZCV, SubNZCV); REGISTER_OP(NEG, Neg); REGISTER_OP(ABS, Abs); REGISTER_OP(MUL, Mul); @@ -1381,6 +1589,8 @@ void X86JITCore::RegisterALUHandlers() { REGISTER_OP(MULH, MulH); REGISTER_OP(UMULH, UMulH); REGISTER_OP(OR, Or); + REGISTER_OP(ORLSHL, Orlshl); + REGISTER_OP(ORLSHR, Orlshr); REGISTER_OP(AND, And); REGISTER_OP(ANDN, Andn); REGISTER_OP(XOR, Xor); diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h b/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h index b19bb97cf..502a93ae9 100644 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h +++ b/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h @@ -244,7 +244,10 @@ private: DEF_OP(InlineEntrypointOffset); DEF_OP(CycleCounter); DEF_OP(Add); + DEF_OP(AddNZCV); + DEF_OP(TestNZ); DEF_OP(Sub); + DEF_OP(SubNZCV); DEF_OP(Neg); DEF_OP(Abs); DEF_OP(Mul); @@ -256,6 +259,8 @@ private: DEF_OP(MulH); DEF_OP(UMulH); DEF_OP(Or); + DEF_OP(Orlshl); + DEF_OP(Orlshr); DEF_OP(And); DEF_OP(Andn); DEF_OP(Xor); diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h index 9cfe7a116..4db854ecd 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h @@ -1157,7 +1157,7 @@ private: void SetNZ_ZeroCV(unsigned SrcSize, OrderedNode *Res) { // The TestNZ opcode does this operation natively for 32-bit or 64-bit. // Otherwise we can implement the functionality ourselves with some bit math. - if (CTX->BackendFeatures.SupportsFlags && SrcSize >= 4) { + if (SrcSize >= 4) { CachedNZCV = _TestNZ(SrcSize, Res); PossiblySetNZCVBits = (1u << 31) | (1u << 30); } else { @@ -1180,7 +1180,7 @@ private: if (SetBits == 0) return _Lshl(OpSize::i64Bit, Value, _Constant(Bit)); - else if (CTX->BackendFeatures.SupportsShiftedBitwise && (SetBits & (1u << Bit)) == 0) + else if ((SetBits & (1u << Bit)) == 0) return _Orlshl(OpSize::i32Bit, NZCV, Value, Bit); else return _Bfi(OpSize::i32Bit, 1, Bit, NZCV, Value); diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp index 1d5733f83..518906cfe 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp @@ -143,8 +143,7 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) { // SF/ZF and N/Z are together on both arm64 and x86_64, so we special case that. bool GetNZ = (FlagsMask & (1 << FEXCore::X86State::RFLAG_SF_LOC)) && - (FlagsMask & (1 << FEXCore::X86State::RFLAG_ZF_LOC)) && - CTX->BackendFeatures.SupportsShiftedBitwise; + (FlagsMask & (1 << FEXCore::X86State::RFLAG_ZF_LOC)); // Handle CF first, since it's at bit 0 and hence doesn't need shift or OR. if (FlagsMask & (1 << FEXCore::X86State::RFLAG_CF_LOC)) { @@ -175,11 +174,7 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) { else Flag = GetRFLAG(FlagOffset); - if (CTX->BackendFeatures.SupportsShiftedBitwise) { - Original = _Orlshl(OpSize::i64Bit, Original, Flag, FlagOffset); - } else { - Original = _Bfi(OpSize::i32Bit, 1, FlagOffset, Original, Flag); - } + Original = _Orlshl(OpSize::i64Bit, Original, Flag, FlagOffset); } // OR in the SF/ZF flags at the end, allowing the lshr to fold with the OR @@ -537,7 +532,7 @@ void OpDispatchBuilder::CalculateFlags_SUB(uint8_t SrcSize, OrderedNode *Res, Or auto OldCF = UpdateCF ? nullptr : GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC); // TODO: Could do this path for small sources if we have FEAT_FlagM - if (CTX->BackendFeatures.SupportsFlags && SrcSize >= 4) { + if (SrcSize >= 4) { SetNZCV(_SubNZCV(OpSize, Src1, Src2)); } else { // SF/ZF @@ -580,7 +575,7 @@ void OpDispatchBuilder::CalculateFlags_ADD(uint8_t SrcSize, OrderedNode *Res, Or auto OldCF = UpdateCF ? nullptr : GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC); // TODO: Could do this path for small sources if we have FEAT_FlagM - if (CTX->BackendFeatures.SupportsFlags && SrcSize >= 4) { + if (SrcSize >= 4) { SetNZCV(_AddNZCV(OpSize, Src1, Src2)); } else { // SF/ZF diff --git a/FEXCore/include/FEXCore/Core/CPUBackend.h b/FEXCore/include/FEXCore/Core/CPUBackend.h index f5585985c..e8088600c 100644 --- a/FEXCore/include/FEXCore/Core/CPUBackend.h +++ b/FEXCore/include/FEXCore/Core/CPUBackend.h @@ -35,7 +35,6 @@ namespace CodeSerialize { namespace CPU { struct CPUBackendFeatures { bool SupportsStaticRegisterAllocation = false; - bool SupportsShiftedBitwise = false; bool SupportsFlags = false; bool SupportsSaturatingRoundingShifts = false; bool SupportsVTBL2 = false;