Merge pull request #2053 from lioncash/vfcmpord

IR: Handle 256-bit VFCMPORD/VFCMPUNO
This commit is contained in:
Ryan Houdek authored and GitHub committed 2022-10-09 19:43:36 -07:00
commit 7b4b9a80fa
3 files changed
+275 -123

No files matched your search

@@ -1274,7 +1274,7 @@ DEF_OP(VFCMPLE) {
}
DEF_OP(VFCMPORD) {
auto Op = IROp->C<IR::IROp_VFCMPORD>();
const auto Op = IROp->C<IR::IROp_VFCMPORD>();
const uint8_t OpSize = IROp->Size;
void *Src1 = GetSrc<void*>(Data->SSAData, Op->Vector1);
@@ -1282,21 +1282,28 @@ DEF_OP(VFCMPORD) {
const auto Func = [](auto a, auto b) { return (!std::isnan(a) && !std::isnan(b)) ? ~0ULL : 0; };
uint8_t Tmp[16];
const uint8_t Elements = OpSize / Op->Header.ElementSize;
uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE];
if (Op->Header.ElementSize == OpSize) {
switch (Op->Header.ElementSize) {
const uint8_t ElementSize = Op->Header.ElementSize;
const uint8_t Elements = OpSize / ElementSize;
const auto IsScalar = ElementSize == OpSize;
if (IsScalar) {
switch (ElementSize) {
DO_SCALAR_COMPARE_OP(4, float, uint32_t, Func);
DO_SCALAR_COMPARE_OP(8, double, uint64_t, Func);
default: LOGMAN_MSG_A_FMT("Unsupported elementSize: {}", Op->Header.ElementSize);
default:
LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize);
break;
}
}
else {
switch (Op->Header.ElementSize) {
switch (ElementSize) {
DO_VECTOR_COMPARE_OP(4, float, uint32_t, Func);
DO_VECTOR_COMPARE_OP(8, double, uint64_t, Func);
default: LOGMAN_MSG_A_FMT("Unsupported elementSize: {}", Op->Header.ElementSize);
default:
LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize);
break;
}
}
@@ -1304,7 +1311,7 @@ DEF_OP(VFCMPORD) {
}
DEF_OP(VFCMPUNO) {
auto Op = IROp->C<IR::IROp_VFCMPUNO>();
const auto Op = IROp->C<IR::IROp_VFCMPUNO>();
const uint8_t OpSize = IROp->Size;
void *Src1 = GetSrc<void*>(Data->SSAData, Op->Vector1);
@@ -1312,21 +1319,28 @@ DEF_OP(VFCMPUNO) {
const auto Func = [](auto a, auto b) { return (std::isnan(a) || std::isnan(b)) ? ~0ULL : 0; };
uint8_t Tmp[16];
const uint8_t Elements = OpSize / Op->Header.ElementSize;
uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE];
if (Op->Header.ElementSize == OpSize) {
switch (Op->Header.ElementSize) {
const uint8_t ElementSize = Op->Header.ElementSize;
const uint8_t Elements = OpSize / ElementSize;
const auto IsScalar = ElementSize == OpSize;
if (IsScalar) {
switch (ElementSize) {
DO_SCALAR_COMPARE_OP(4, float, uint32_t, Func);
DO_SCALAR_COMPARE_OP(8, double, uint64_t, Func);
default: LOGMAN_MSG_A_FMT("Unsupported elementSize: {}", Op->Header.ElementSize);
default:
LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize);
break;
}
}
else {
switch (Op->Header.ElementSize) {
switch (ElementSize) {
DO_VECTOR_COMPARE_OP(4, float, uint32_t, Func);
DO_VECTOR_COMPARE_OP(8, double, uint64_t, Func);
default: LOGMAN_MSG_A_FMT("Unsupported elementSize: {}", Op->Header.ElementSize);
default:
LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize);
break;
}
}
+185 -77
View File
@@ -3183,100 +3183,208 @@ DEF_OP(VFCMPLE) {
}
DEF_OP(VFCMPORD) {
auto Op = IROp->C<IR::IROp_VFCMPORD>();
const uint8_t OpSize = IROp->Size;
if (Op->Header.ElementSize == OpSize) {
// Scalar
switch (Op->Header.ElementSize) {
case 4: {
fcmge(VTMP1.S(), GetSrc(Op->Vector1.ID()).S(), GetSrc(Op->Vector2.ID()).S());
fcmgt(VTMP2.S(), GetSrc(Op->Vector2.ID()).S(), GetSrc(Op->Vector1.ID()).S());
orr(GetDst(Node).V8B(), VTMP1.V8B(), VTMP2.V8B());
break;
}
case 8: {
fcmge(VTMP1.D(), GetSrc(Op->Vector1.ID()).D(), GetSrc(Op->Vector2.ID()).D());
fcmgt(VTMP2.D(), GetSrc(Op->Vector2.ID()).D(), GetSrc(Op->Vector1.ID()).D());
orr(GetDst(Node).V8B(), VTMP1.V8B(), VTMP2.V8B());
break;
}
default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break;
}
}
else {
// Vector
switch (Op->Header.ElementSize) {
const auto Op = IROp->C<IR::IROp_VFCMPORD>();
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto IsScalar = ElementSize == OpSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Vector1 = GetSrc(Op->Vector1.ID());
const auto Vector2 = GetSrc(Op->Vector2.ID());
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = p0;
// Ensure there's no junk in the temporary.
eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD());
// The idea is like comparing for unordered, but we just
// invert the predicate from the comparison to instead
// select all ordered elements in the vector.
switch (ElementSize) {
case 2: {
fcmge(VTMP1.V8H(), GetSrc(Op->Vector1.ID()).V8H(), GetSrc(Op->Vector2.ID()).V8H());
fcmgt(VTMP2.V8H(), GetSrc(Op->Vector2.ID()).V8H(), GetSrc(Op->Vector1.ID()).V8H());
orr(GetDst(Node).V16B(), VTMP1.V16B(), VTMP2.V16B());
break;
fcmuo(ComparePred.VnH(), Mask, Vector1.Z().VnH(), Vector2.Z().VnH());
not_(ComparePred.VnB(), Mask, ComparePred.VnB());
not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector1.Z().VnH());
orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector1.Z().VnH());
break;
}
case 4: {
fcmge(VTMP1.V4S(), GetSrc(Op->Vector1.ID()).V4S(), GetSrc(Op->Vector2.ID()).V4S());
fcmgt(VTMP2.V4S(), GetSrc(Op->Vector2.ID()).V4S(), GetSrc(Op->Vector1.ID()).V4S());
orr(GetDst(Node).V16B(), VTMP1.V16B(), VTMP2.V16B());
break;
fcmuo(ComparePred.VnS(), Mask, Vector1.Z().VnS(), Vector2.Z().VnS());
not_(ComparePred.VnB(), Mask, ComparePred.VnB());
not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector1.Z().VnS());
orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector1.Z().VnS());
break;
}
case 8: {
fcmge(VTMP1.V2D(), GetSrc(Op->Vector1.ID()).V2D(), GetSrc(Op->Vector2.ID()).V2D());
fcmgt(VTMP2.V2D(), GetSrc(Op->Vector2.ID()).V2D(), GetSrc(Op->Vector1.ID()).V2D());
orr(GetDst(Node).V16B(), VTMP1.V16B(), VTMP2.V16B());
break;
fcmuo(ComparePred.VnD(), Mask, Vector1.Z().VnD(), Vector2.Z().VnD());
not_(ComparePred.VnB(), Mask, ComparePred.VnB());
not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector1.Z().VnD());
orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector1.Z().VnD());
break;
}
default:
LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
return;
}
mov(Dst.Z().VnD(), VTMP1.Z().VnD());
} else {
if (IsScalar) {
switch (ElementSize) {
case 2: {
fcmge(VTMP1.H(), Vector1.H(), Vector2.H());
fcmgt(VTMP2.H(), Vector2.H(), Vector1.H());
orr(Dst.V8B(), VTMP1.V8B(), VTMP2.V8B());
break;
}
case 4: {
fcmge(VTMP1.S(), Vector1.S(), Vector2.S());
fcmgt(VTMP2.S(), Vector2.S(), Vector1.S());
orr(Dst.V8B(), VTMP1.V8B(), VTMP2.V8B());
break;
}
case 8: {
fcmge(VTMP1.D(), Vector1.D(), Vector2.D());
fcmgt(VTMP2.D(), Vector2.D(), Vector1.D());
orr(Dst.V8B(), VTMP1.V8B(), VTMP2.V8B());
break;
}
default:
LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
break;
}
} else {
switch (ElementSize) {
case 2: {
fcmge(VTMP1.V8H(), Vector1.V8H(), Vector2.V8H());
fcmgt(VTMP2.V8H(), Vector2.V8H(), Vector1.V8H());
orr(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B());
break;
}
case 4: {
fcmge(VTMP1.V4S(), Vector1.V4S(), Vector2.V4S());
fcmgt(VTMP2.V4S(), Vector2.V4S(), Vector1.V4S());
orr(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B());
break;
}
case 8: {
fcmge(VTMP1.V2D(), Vector1.V2D(), Vector2.V2D());
fcmgt(VTMP2.V2D(), Vector2.V2D(), Vector1.V2D());
orr(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B());
break;
}
default:
LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
break;
}
default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break;
}
}
}
DEF_OP(VFCMPUNO) {
auto Op = IROp->C<IR::IROp_VFCMPUNO>();
const uint8_t OpSize = IROp->Size;
if (Op->Header.ElementSize == OpSize) {
// Scalar
switch (Op->Header.ElementSize) {
case 4: {
fcmge(VTMP1.S(), GetSrc(Op->Vector1.ID()).S(), GetSrc(Op->Vector2.ID()).S());
fcmgt(VTMP2.S(), GetSrc(Op->Vector2.ID()).S(), GetSrc(Op->Vector1.ID()).S());
orr(GetDst(Node).V8B(), VTMP1.V8B(), VTMP2.V8B());
mvn(GetDst(Node).V8B(), GetDst(Node).V8B());
break;
}
case 8: {
fcmge(VTMP1.D(), GetSrc(Op->Vector1.ID()).D(), GetSrc(Op->Vector2.ID()).D());
fcmgt(VTMP2.D(), GetSrc(Op->Vector2.ID()).D(), GetSrc(Op->Vector1.ID()).D());
orr(GetDst(Node).V8B(), VTMP1.V8B(), VTMP2.V8B());
mvn(GetDst(Node).V8B(), GetDst(Node).V8B());
break;
}
default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break;
}
}
else {
// Vector
switch (Op->Header.ElementSize) {
const auto Op = IROp->C<IR::IROp_VFCMPUNO>();
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto IsScalar = ElementSize == OpSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Vector1 = GetSrc(Op->Vector1.ID());
const auto Vector2 = GetSrc(Op->Vector2.ID());
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = p0;
// Ensure there's no junk in the temporary.
eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD());
switch (ElementSize) {
case 2: {
fcmge(VTMP1.V8H(), GetSrc(Op->Vector1.ID()).V8H(), GetSrc(Op->Vector2.ID()).V8H());
fcmgt(VTMP2.V8H(), GetSrc(Op->Vector2.ID()).V8H(), GetSrc(Op->Vector1.ID()).V8H());
orr(GetDst(Node).V16B(), VTMP1.V16B(), VTMP2.V16B());
mvn(GetDst(Node).V16B(), GetDst(Node).V16B());
break;
fcmuo(ComparePred.VnH(), Mask, Vector1.Z().VnH(), Vector2.Z().VnH());
not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector1.Z().VnH());
orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector1.Z().VnH());
break;
}
case 4: {
fcmge(VTMP1.V4S(), GetSrc(Op->Vector1.ID()).V4S(), GetSrc(Op->Vector2.ID()).V4S());
fcmgt(VTMP2.V4S(), GetSrc(Op->Vector2.ID()).V4S(), GetSrc(Op->Vector1.ID()).V4S());
orr(GetDst(Node).V16B(), VTMP1.V16B(), VTMP2.V16B());
mvn(GetDst(Node).V16B(), GetDst(Node).V16B());
break;
fcmuo(ComparePred.VnS(), Mask, Vector1.Z().VnS(), Vector2.Z().VnS());
not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector1.Z().VnS());
orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector1.Z().VnS());
break;
}
case 8: {
fcmge(VTMP1.V2D(), GetSrc(Op->Vector1.ID()).V2D(), GetSrc(Op->Vector2.ID()).V2D());
fcmgt(VTMP2.V2D(), GetSrc(Op->Vector2.ID()).V2D(), GetSrc(Op->Vector1.ID()).V2D());
orr(GetDst(Node).V16B(), VTMP1.V16B(), VTMP2.V16B());
mvn(GetDst(Node).V16B(), GetDst(Node).V16B());
break;
fcmuo(ComparePred.VnD(), Mask, Vector1.Z().VnD(), Vector2.Z().VnD());
not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector1.Z().VnD());
orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector1.Z().VnD());
break;
}
default:
LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
return;
}
mov(Dst.Z().VnD(), VTMP1.Z().VnD());
} else {
if (IsScalar) {
switch (ElementSize) {
case 2: {
fcmge(VTMP1.H(), Vector1.H(), Vector2.H());
fcmgt(VTMP2.H(), Vector2.H(), Vector1.H());
orr(Dst.V8B(), VTMP1.V8B(), VTMP2.V8B());
mvn(Dst.V8B(), Dst.V8B());
break;
}
case 4: {
fcmge(VTMP1.S(), Vector1.S(), Vector2.S());
fcmgt(VTMP2.S(), Vector2.S(), Vector1.S());
orr(Dst.V8B(), VTMP1.V8B(), VTMP2.V8B());
mvn(Dst.V8B(), Dst.V8B());
break;
}
case 8: {
fcmge(VTMP1.D(), Vector1.D(), Vector2.D());
fcmgt(VTMP2.D(), Vector2.D(), Vector1.D());
orr(Dst.V8B(), VTMP1.V8B(), VTMP2.V8B());
mvn(Dst.V8B(), Dst.V8B());
break;
}
default:
LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
break;
}
} else {
switch (ElementSize) {
case 2: {
fcmge(VTMP1.V8H(), Vector1.V8H(), Vector2.V8H());
fcmgt(VTMP2.V8H(), Vector2.V8H(), Vector1.V8H());
orr(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B());
mvn(Dst.V16B(), Dst.V16B());
break;
}
case 4: {
fcmge(VTMP1.V4S(), Vector1.V4S(), Vector2.V4S());
fcmgt(VTMP2.V4S(), Vector2.V4S(), Vector1.V4S());
orr(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B());
mvn(Dst.V16B(), Dst.V16B());
break;
}
case 8: {
fcmge(VTMP1.V2D(), Vector1.V2D(), Vector2.V2D());
fcmgt(VTMP2.V2D(), Vector2.V2D(), Vector1.V2D());
orr(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B());
mvn(Dst.V16B(), Dst.V16B());
break;
}
default:
LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
break;
}
default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break;
}
}
}
@@ -1781,57 +1781,87 @@ DEF_OP(VFCMPLE) {
}
DEF_OP(VFCMPORD) {
auto Op = IROp->C<IR::IROp_VFCMPORD>();
const uint8_t OpSize = IROp->Size;
const auto Op = IROp->C<IR::IROp_VFCMPORD>();
const auto OpSize = IROp->Size;
if (Op->Header.ElementSize == OpSize) {
switch (Op->Header.ElementSize) {
const auto ElementSize = Op->Header.ElementSize;
const auto IsScalar = ElementSize == OpSize;
const auto Dst = GetDst(Node);
const auto Vector1 = GetSrc(Op->Vector1.ID());
const auto Vector2 = GetSrc(Op->Vector2.ID());
if (IsScalar) {
switch (ElementSize) {
case 4:
vcmpss(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID()), 7);
break;
vcmpss(Dst, Vector1, Vector2, 7);
break;
case 8:
vcmpsd(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID()), 7);
break;
default: LOGMAN_MSG_A_FMT("Unsupported element size: {}", Op->Header.ElementSize);
vcmpsd(Dst, Vector1, Vector2, 7);
break;
default:
LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize);
break;
}
}
else {
switch (Op->Header.ElementSize) {
const auto DstYMM = ToYMM(Dst);
const auto Vector1YMM = ToYMM(Vector1);
const auto Vector2YMM = ToYMM(Vector2);
switch (ElementSize) {
case 4:
vcmpps(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID()), 7);
break;
vcmpps(DstYMM, Vector1YMM, Vector2YMM, 7);
break;
case 8:
vcmppd(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID()), 7);
break;
default: LOGMAN_MSG_A_FMT("Unsupported element size: {}", Op->Header.ElementSize);
vcmppd(DstYMM, Vector1YMM, Vector2YMM, 7);
break;
default:
LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize);
break;
}
}
}
DEF_OP(VFCMPUNO) {
auto Op = IROp->C<IR::IROp_VFCMPUNO>();
const uint8_t OpSize = IROp->Size;
const auto Op = IROp->C<IR::IROp_VFCMPUNO>();
const auto OpSize = IROp->Size;
if (Op->Header.ElementSize == OpSize) {
switch (Op->Header.ElementSize) {
const auto ElementSize = Op->Header.ElementSize;
const auto IsScalar = ElementSize == OpSize;
const auto Dst = GetDst(Node);
const auto Vector1 = GetSrc(Op->Vector1.ID());
const auto Vector2 = GetSrc(Op->Vector2.ID());
if (IsScalar) {
switch (ElementSize) {
case 4:
vcmpss(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID()), 3);
break;
vcmpss(Dst, Vector1, Vector2, 3);
break;
case 8:
vcmpsd(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID()), 3);
break;
default: LOGMAN_MSG_A_FMT("Unsupported element size: {}", Op->Header.ElementSize);
vcmpsd(Dst, Vector1, Vector2, 3);
break;
default:
LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize);
break;
}
}
else {
switch (Op->Header.ElementSize) {
const auto DstYMM = ToYMM(Dst);
const auto Vector1YMM = ToYMM(Vector1);
const auto Vector2YMM = ToYMM(Vector2);
switch (ElementSize) {
case 4:
vcmpps(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID()), 3);
break;
vcmpps(DstYMM, Vector1YMM, Vector2YMM, 3);
break;
case 8:
vcmppd(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID()), 3);
break;
default: LOGMAN_MSG_A_FMT("Unsupported element size: {}", Op->Header.ElementSize);
vcmppd(DstYMM, Vector1YMM, Vector2YMM, 3);
break;
default:
LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize);
break;
}
}
}