From 1d7b688c5593ffff925c9b99a7242239d2b8848f Mon Sep 17 00:00:00 2001 From: lioncash Date: Thu, 13 Oct 2022 18:49:26 +0000 Subject: [PATCH 1/2] IR: Handle 256-bit VUXTL Extends VUXTL to handle 256-bit values. --- .../Interface/Core/Interpreter/VectorOps.cpp | 13 ++-- .../Interface/Core/JIT/Arm64/VectorOps.cpp | 62 +++++++++++++++---- .../Interface/Core/JIT/x86_64/VectorOps.cpp | 40 +++++++++--- 3 files changed, 89 insertions(+), 26 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp index 3f6ae2280..10d1748a9 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp @@ -1784,20 +1784,23 @@ DEF_OP(VSXTL2) { } DEF_OP(VUXTL) { - auto Op = IROp->C(); + const auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; void *Src = GetSrc(Data->SSAData, Op->Vector); - uint8_t Tmp[16]{}; + uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE]{}; - const uint8_t Elements = OpSize / Op->Header.ElementSize; + const uint8_t ElementSize = Op->Header.ElementSize; + const uint8_t Elements = OpSize / ElementSize; const auto Func = [](auto a, auto min, auto max) { return a; }; - switch (Op->Header.ElementSize) { + switch (ElementSize) { DO_VECTOR_1SRC_2TYPE_OP(2, uint16_t, uint8_t, Func, 0, 0) DO_VECTOR_1SRC_2TYPE_OP(4, uint32_t, uint16_t, Func, 0, 0) DO_VECTOR_1SRC_2TYPE_OP(8, uint64_t, uint32_t, Func, 0, 0) - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; } memcpy(GDP, Tmp, OpSize); } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp index d2b80d371..3dbfd0e2a 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp @@ -4155,18 +4155,56 @@ DEF_OP(VSXTL2) { } DEF_OP(VUXTL) { - auto Op = IROp->C(); - switch (Op->Header.ElementSize) { - case 2: - uxtl(GetDst(Node).V8H(), GetSrc(Op->Vector.ID()).V8B()); - break; - case 4: - uxtl(GetDst(Node).V4S(), GetSrc(Op->Vector.ID()).V4H()); - break; - case 8: - uxtl(GetDst(Node).V2D(), GetSrc(Op->Vector.ID()).V2S()); - break; - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto Vector = GetSrc(Op->Vector.ID()); + + if (HostSupportsSVE && Is256Bit) { + // NOTE: See VSXTL implementation for an explanation on why + // UXTB/UXTH/UXTW aren't used, since the same behavior + // concerns applies here, but with zero-extension + // instead of sign-extension. + + switch (ElementSize) { + case 2: + ushllb(VTMP1.Z().VnH(), Vector.Z().VnB(), 0); + ushllt(VTMP2.Z().VnH(), Vector.Z().VnB(), 0); + zip1(Dst.Z().VnH(), VTMP1.Z().VnH(), VTMP2.Z().VnH()); + break; + case 4: + ushllb(VTMP1.Z().VnS(), Vector.Z().VnH(), 0); + ushllt(VTMP2.Z().VnS(), Vector.Z().VnH(), 0); + zip1(Dst.Z().VnS(), VTMP1.Z().VnS(), VTMP2.Z().VnS()); + break; + case 8: + ushllb(VTMP1.Z().VnD(), Vector.Z().VnS(), 0); + ushllt(VTMP2.Z().VnD(), Vector.Z().VnS(), 0); + zip1(Dst.Z().VnD(), VTMP1.Z().VnD(), VTMP2.Z().VnD()); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } + } else { + switch (ElementSize) { + case 2: + uxtl(Dst.V8H(), Vector.V8B()); + break; + case 4: + uxtl(Dst.V4S(), Vector.V4H()); + break; + case 8: + uxtl(Dst.V2D(), Vector.V2S()); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp index 6a1baf977..8794fd4be 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp @@ -2353,18 +2353,40 @@ DEF_OP(VSXTL2) { } DEF_OP(VUXTL) { - auto Op = IROp->C(); - switch (Op->Header.ElementSize) { + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto Vector = GetSrc(Op->Vector.ID()); + + switch (ElementSize) { case 2: - pmovzxbw(GetDst(Node), GetSrc(Op->Vector.ID())); - break; + if (Is256Bit) { + vpmovzxbw(ToYMM(Dst), Vector); + } else { + vpmovzxbw(Dst, Vector); + } + break; case 4: - pmovzxwd(GetDst(Node), GetSrc(Op->Vector.ID())); - break; + if (Is256Bit) { + vpmovzxwd(ToYMM(Dst), Vector); + } else { + vpmovzxwd(Dst, Vector); + } + break; case 8: - pmovzxdq(GetDst(Node), GetSrc(Op->Vector.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); + if (Is256Bit) { + vpmovzxdq(ToYMM(Dst), Vector); + } else { + vpmovzxdq(Dst, Vector); + } + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; } } From fc4a5ebfd38f52345a97300598e58098da6598f5 Mon Sep 17 00:00:00 2001 From: lioncash Date: Thu, 13 Oct 2022 19:10:22 +0000 Subject: [PATCH 2/2] IR: Handle 256-bit VUXTL2 Extends VUXTL2 to handle 256-bit vectors. --- .../Interface/Core/Interpreter/VectorOps.cpp | 13 ++-- .../Interface/Core/JIT/Arm64/VectorOps.cpp | 63 +++++++++++++++---- .../Interface/Core/JIT/x86_64/VectorOps.cpp | 56 ++++++++++++----- External/FEXCore/Source/Interface/IR/IR.json | 2 +- 4 files changed, 102 insertions(+), 32 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp index 10d1748a9..37aa5e9cb 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp @@ -1806,21 +1806,24 @@ DEF_OP(VUXTL) { } DEF_OP(VUXTL2) { - auto Op = IROp->C(); + const auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; void *Src = GetSrc(Data->SSAData, Op->Vector); - uint8_t Tmp[16]; + uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE]{}; - const uint8_t Elements = OpSize / Op->Header.ElementSize; + const uint8_t ElementSize = Op->Header.ElementSize; + const uint8_t Elements = OpSize / ElementSize; const auto Func = [](auto a, auto min, auto max) { return a; }; - switch (Op->Header.ElementSize) { + switch (ElementSize) { DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(2, uint16_t, uint8_t, Func, 0, 0) DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(4, uint32_t, uint16_t, Func, 0, 0) DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(8, uint64_t, uint32_t, Func, 0, 0) - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; } memcpy(GDP, Tmp, OpSize); } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp index 3dbfd0e2a..df609665f 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp @@ -4209,18 +4209,57 @@ DEF_OP(VUXTL) { } DEF_OP(VUXTL2) { - auto Op = IROp->C(); - switch (Op->Header.ElementSize) { - case 2: - uxtl2(GetDst(Node).V8H(), GetSrc(Op->Vector.ID()).V16B()); - break; - case 4: - uxtl2(GetDst(Node).V4S(), GetSrc(Op->Vector.ID()).V8H()); - break; - case 8: - uxtl2(GetDst(Node).V2D(), GetSrc(Op->Vector.ID()).V4S()); - break; - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); + const auto Op = IROp->C(); + + const auto OpSize = IROp->Size; + + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto Vector = GetSrc(Op->Vector.ID()); + + if (HostSupportsSVE && Is256Bit) { + // NOTE: See VSXTL implementation for an explanation on why + // UXTB/UXTH/UXTW aren't used, since the same behavior + // concerns applies here, but with zero-extension + // instead of sign-extension. + + switch (ElementSize) { + case 2: + ushllb(VTMP1.Z().VnH(), Vector.Z().VnB(), 0); + ushllt(VTMP2.Z().VnH(), Vector.Z().VnB(), 0); + zip2(Dst.Z().VnH(), VTMP1.Z().VnH(), VTMP2.Z().VnH()); + break; + case 4: + ushllb(VTMP1.Z().VnS(), Vector.Z().VnH(), 0); + ushllt(VTMP2.Z().VnS(), Vector.Z().VnH(), 0); + zip2(Dst.Z().VnS(), VTMP1.Z().VnS(), VTMP2.Z().VnS()); + break; + case 8: + ushllb(VTMP1.Z().VnD(), Vector.Z().VnS(), 0); + ushllt(VTMP2.Z().VnD(), Vector.Z().VnS(), 0); + zip2(Dst.Z().VnD(), VTMP1.Z().VnD(), VTMP2.Z().VnD()); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } + } else { + switch (ElementSize) { + case 2: + uxtl2(Dst.V8H(), Vector.V16B()); + break; + case 4: + uxtl2(Dst.V4S(), Vector.V8H()); + break; + case 8: + uxtl2(Dst.V2D(), Vector.V4S()); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp index 8794fd4be..30870371b 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp @@ -2391,21 +2391,49 @@ DEF_OP(VUXTL) { } DEF_OP(VUXTL2) { - auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; - vpsrldq(GetDst(Node), GetSrc(Op->Vector.ID()), OpSize / 2); - switch (Op->Header.ElementSize) { - case 2: - pmovzxbw(GetDst(Node), GetDst(Node)); - break; - case 4: - pmovzxwd(GetDst(Node), GetDst(Node)); - break; - case 8: - pmovzxdq(GetDst(Node), GetDst(Node)); - break; - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto Vector = GetSrc(Op->Vector.ID()); + + if (Is256Bit) { + const auto DstYMM = ToYMM(Dst); + + vextracti128(Dst, ToYMM(Vector), 1); + switch (ElementSize) { + case 2: + vpmovzxbw(DstYMM, Dst); + break; + case 4: + vpmovzxwd(DstYMM, Dst); + break; + case 8: + vpmovzxdq(DstYMM, Dst); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } + } else { + vpsrldq(Dst, Vector, OpSize / 2); + switch (ElementSize) { + case 2: + vpmovzxbw(Dst, Dst); + break; + case 4: + vpmovzxwd(Dst, Dst); + break; + case 8: + vpmovzxdq(Dst, Dst); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } } } diff --git a/External/FEXCore/Source/Interface/IR/IR.json b/External/FEXCore/Source/Interface/IR/IR.json index e3a114195..252c34215 100644 --- a/External/FEXCore/Source/Interface/IR/IR.json +++ b/External/FEXCore/Source/Interface/IR/IR.json @@ -1077,7 +1077,7 @@ }, "FPR = VUXTL2 u8:#RegisterSize, u8:#ElementSize, FPR:$Vector": { "Desc": ["Zero extends elements from the source element size to the next size up", - "Source elements come from the upper 64bits of the register" + "Source elements come from the upper half of the register" ], "DestSize": "RegisterSize", "NumElements": "RegisterSize / (ElementSize << 1)"