diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp index 57a915eb3..c9b73f40f 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp @@ -963,22 +963,25 @@ DEF_OP(VBSL) { } DEF_OP(VCMPEQ) { - auto Op = IROp->C(); + const auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; void *Src1 = GetSrc(Data->SSAData, Op->Vector1); void *Src2 = GetSrc(Data->SSAData, Op->Vector2); - uint8_t Tmp[16]; + uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE]; + + const uint8_t ElementSize = Op->Header.ElementSize; + const uint8_t Elements = OpSize / ElementSize; - const uint8_t Elements = OpSize / Op->Header.ElementSize; const auto Func = [](auto a, auto b) { return a == b ? ~0ULL : 0; }; - - switch (Op->Header.ElementSize) { + switch (ElementSize) { DO_VECTOR_OP(1, uint8_t, Func) DO_VECTOR_OP(2, uint16_t, Func) DO_VECTOR_OP(4, uint32_t, Func) DO_VECTOR_OP(8, uint64_t, Func) - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; } memcpy(GDP, Tmp, OpSize); diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp index 0a0866a93..c8cf92752 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp @@ -2277,42 +2277,97 @@ DEF_OP(VBSL) { } DEF_OP(VCMPEQ) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - if (Op->Header.ElementSize == OpSize) { - // Scalar - switch (Op->Header.ElementSize) { - case 4: { - cmeq(GetDst(Node).S(), GetSrc(Op->Vector1.ID()).S(), GetSrc(Op->Vector2.ID()).S()); - break; - } - case 8: { - cmeq(GetDst(Node).D(), GetSrc(Op->Vector1.ID()).D(), GetSrc(Op->Vector2.ID()).D()); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; - } - } - else { - // Vector - switch (Op->Header.ElementSize) { + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto ElementSize = Op->Header.ElementSize; + const auto IsScalar = ElementSize == OpSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto Vector1 = GetSrc(Op->Vector1.ID()); + const auto Vector2 = GetSrc(Op->Vector2.ID()); + + if (HostSupportsSVE && Is256Bit && !IsScalar) { + const auto Mask = PRED_TMP_32B.Zeroing(); + const auto ComparePred = p0; + + // Ensure no junk is in the temp (important for ensuring + // non-equal entries remain as zero during the final bitwise OR). + eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); + + // General idea is to compare for equality, not the equal vals + // from one of the registers, then or both together to make the + // relevant equal entries all 1s. + + switch (ElementSize) { case 1: { - cmeq(GetDst(Node).V16B(), GetSrc(Op->Vector1.ID()).V16B(), GetSrc(Op->Vector2.ID()).V16B()); - break; + cmpeq(ComparePred.VnB(), Mask, Vector1.Z().VnB(), Vector2.Z().VnB()); + not_(VTMP1.Z().VnB(), ComparePred.Merging(), Vector1.Z().VnB()); + orr(VTMP1.Z().VnB(), ComparePred.Merging(), VTMP1.Z().VnB(), Vector1.Z().VnB()); + break; } case 2: { - cmeq(GetDst(Node).V8H(), GetSrc(Op->Vector1.ID()).V8H(), GetSrc(Op->Vector2.ID()).V8H()); - break; + cmpeq(ComparePred.VnH(), Mask, Vector1.Z().VnH(), Vector2.Z().VnH()); + not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector1.Z().VnH()); + orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector1.Z().VnH()); + break; } case 4: { - cmeq(GetDst(Node).V4S(), GetSrc(Op->Vector1.ID()).V4S(), GetSrc(Op->Vector2.ID()).V4S()); - break; + cmpeq(ComparePred.VnS(), Mask, Vector1.Z().VnS(), Vector2.Z().VnS()); + not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector1.Z().VnS()); + orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector1.Z().VnS()); + break; } case 8: { - cmeq(GetDst(Node).V2D(), GetSrc(Op->Vector1.ID()).V2D(), GetSrc(Op->Vector2.ID()).V2D()); - break; + cmpeq(ComparePred.VnD(), Mask, Vector1.Z().VnD(), Vector2.Z().VnD()); + not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector1.Z().VnD()); + orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector1.Z().VnD()); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + return; } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; + + mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + } else { + if (IsScalar) { + switch (ElementSize) { + case 4: { + cmeq(Dst.S(), Vector1.S(), Vector2.S()); + break; + } + case 8: { + cmeq(Dst.D(), Vector1.D(), Vector2.D()); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } + } else { + switch (ElementSize) { + case 1: { + cmeq(Dst.V16B(), Vector1.V16B(), Vector2.V16B()); + break; + } + case 2: { + cmeq(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); + break; + } + case 4: { + cmeq(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); + break; + } + case 8: { + cmeq(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } } } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp index 58ce303ed..a14b14bc3 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp @@ -1417,22 +1417,30 @@ DEF_OP(VBSL) { } DEF_OP(VCMPEQ) { - auto Op = IROp->C(); + const auto Op = IROp->C(); - switch (Op->Header.ElementSize) { + const auto ElementSize = Op->Header.ElementSize; + + const auto Dst = ToYMM(GetDst(Node)); + const auto Vector1 = ToYMM(GetSrc(Op->Vector1.ID())); + const auto Vector2 = ToYMM(GetSrc(Op->Vector2.ID())); + + switch (ElementSize) { case 1: - vpcmpeqb(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID())); + vpcmpeqb(Dst, Vector1, Vector2); break; case 2: - vpcmpeqw(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID())); + vpcmpeqw(Dst, Vector1, Vector2); break; case 4: - vpcmpeqd(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID())); + vpcmpeqd(Dst, Vector1, Vector2); break; case 8: - vpcmpeqq(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID())); + vpcmpeqq(Dst, Vector1, Vector2); + break; + default: + LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); break; - default: LOGMAN_MSG_A_FMT("Unsupported element size: {}", Op->Header.ElementSize); } }