diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp index d71154e5f..3f6ae2280 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp @@ -1740,39 +1740,45 @@ DEF_OP(VUShrNI2) { } DEF_OP(VSXTL) { - auto Op = IROp->C(); + const auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; void *Src = GetSrc(Data->SSAData, Op->Vector); - uint8_t Tmp[16]{}; + uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE]{}; - const uint8_t Elements = OpSize / Op->Header.ElementSize; + const uint8_t ElementSize = Op->Header.ElementSize; + const uint8_t Elements = OpSize / ElementSize; const auto Func = [](auto a, auto min, auto max) { return a; }; - switch (Op->Header.ElementSize) { + switch (ElementSize) { DO_VECTOR_1SRC_2TYPE_OP(2, int16_t, int8_t, Func, 0, 0) DO_VECTOR_1SRC_2TYPE_OP(4, int32_t, int16_t, Func, 0, 0) DO_VECTOR_1SRC_2TYPE_OP(8, int64_t, int32_t, Func, 0, 0) - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; } memcpy(GDP, Tmp, OpSize); } DEF_OP(VSXTL2) { - auto Op = IROp->C(); + const auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; void *Src = GetSrc(Data->SSAData, Op->Vector); - uint8_t Tmp[16]; + uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE]; - const uint8_t Elements = OpSize / Op->Header.ElementSize; + const uint8_t ElementSize = Op->Header.ElementSize; + const uint8_t Elements = OpSize / ElementSize; const auto Func = [](auto a, auto min, auto max) { return a; }; - switch (Op->Header.ElementSize) { + switch (ElementSize) { DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(2, int16_t, int8_t, Func, 0, 0) DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(4, int32_t, int16_t, Func, 0, 0) DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(8, int64_t, int32_t, Func, 0, 0) - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; } memcpy(GDP, Tmp, OpSize); } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp index 59059dee3..d2b80d371 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp @@ -4012,34 +4012,145 @@ DEF_OP(VUShrNI2) { } DEF_OP(VSXTL) { - auto Op = IROp->C(); - switch (Op->Header.ElementSize) { - case 2: - sxtl(GetDst(Node).V8H(), GetSrc(Op->Vector.ID()).V8B()); - break; - case 4: - sxtl(GetDst(Node).V4S(), GetSrc(Op->Vector.ID()).V4H()); - break; - case 8: - sxtl(GetDst(Node).V2D(), GetSrc(Op->Vector.ID()).V2S()); - break; - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto Vector = GetSrc(Op->Vector.ID()); + + if (HostSupportsSVE && Is256Bit) { + // A little gross, but SVE SXTB/SXTH/SXTW would be a little + // more cumbersome to use here, since those instructions + // use the supplied element size to determine indexing across + // the vector. + // + // So for example if we were sign-extending a byte to a halfword + // with SXTB, assume the vector is like so: + // + // ╔═════════╗╔═════════╗╔═════════╗╔═════════╗ + // ║ Value 3 ║║ Value 2 ║║ Value 1 ║║ Value 0 ║ + // ╚═════════╝╚═════════╝╚═════════╝╚═════════╝ + // + // (Each element is 8 bits in size, and for brevity assume a vector + // that's only 32 bits wide). + // + // The operation + // + // SXTB Dst.VnH, Src.VnB + // + // Will sign-extend bytes based off the element size and also index + // the source vector on a by-element-size basis. + // + // The problem is, since we've specified halfwords as the element size + // (via Dst.VnH), the instruction will skip over Value 1 and sign-extend + // Value 2, place it into the Dst vector, and so on. So we'd be ignoring + // values and end up with something like: + // + // ╔════════════════════╗╔════════════════════╗ + // ║ Value 2 ║║ Value 0 ║ + // ╚════════════════════╝╚════════════════════╝ + // + // Uh oh! + // + // What we want is: + // + // ╔════════════════════╗╔════════════════════╗ + // ║ Value 1 ║║ Value 0 ║ + // ╚════════════════════╝╚════════════════════╝ + // + // We want the extending operation to handle each individual value from + // the source vector and not overlap or ignore them. + + switch (ElementSize) { + case 2: + sshllb(VTMP1.Z().VnH(), Vector.Z().VnB(), 0); + sshllt(VTMP2.Z().VnH(), Vector.Z().VnB(), 0); + zip1(Dst.Z().VnH(), VTMP1.Z().VnH(), VTMP2.Z().VnH()); + break; + case 4: + sshllb(VTMP1.Z().VnS(), Vector.Z().VnH(), 0); + sshllt(VTMP2.Z().VnS(), Vector.Z().VnH(), 0); + zip1(Dst.Z().VnS(), VTMP1.Z().VnS(), VTMP2.Z().VnS()); + break; + case 8: + sshllb(VTMP1.Z().VnD(), Vector.Z().VnS(), 0); + sshllt(VTMP2.Z().VnD(), Vector.Z().VnS(), 0); + zip1(Dst.Z().VnD(), VTMP1.Z().VnD(), VTMP2.Z().VnD()); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } + } else { + switch (ElementSize) { + case 2: + sxtl(Dst.V8H(), Vector.V8B()); + break; + case 4: + sxtl(Dst.V4S(), Vector.V4H()); + break; + case 8: + sxtl(Dst.V2D(), Vector.V2S()); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } } } DEF_OP(VSXTL2) { - auto Op = IROp->C(); - switch (Op->Header.ElementSize) { - case 2: - sxtl2(GetDst(Node).V8H(), GetSrc(Op->Vector.ID()).V16B()); - break; - case 4: - sxtl2(GetDst(Node).V4S(), GetSrc(Op->Vector.ID()).V8H()); - break; - case 8: - sxtl2(GetDst(Node).V2D(), GetSrc(Op->Vector.ID()).V4S()); - break; - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto Vector = GetSrc(Op->Vector.ID()); + + if (HostSupportsSVE && Is256Bit) { + // See VSXTL implementation for in depth explanation + // of all the instructions below. + + switch (ElementSize) { + case 2: + sshllb(VTMP1.Z().VnH(), Vector.Z().VnB(), 0); + sshllt(VTMP2.Z().VnH(), Vector.Z().VnB(), 0); + zip2(Dst.Z().VnH(), VTMP1.Z().VnH(), VTMP2.Z().VnH()); + break; + case 4: + sshllb(VTMP1.Z().VnS(), Vector.Z().VnH(), 0); + sshllt(VTMP2.Z().VnS(), Vector.Z().VnH(), 0); + zip2(Dst.Z().VnS(), VTMP1.Z().VnS(), VTMP2.Z().VnS()); + break; + case 8: + sshllb(VTMP1.Z().VnD(), Vector.Z().VnS(), 0); + sshllt(VTMP2.Z().VnD(), Vector.Z().VnS(), 0); + zip2(Dst.Z().VnD(), VTMP1.Z().VnD(), VTMP2.Z().VnD()); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } + } else { + switch (ElementSize) { + case 2: + sxtl2(Dst.V8H(), Vector.V16B()); + break; + case 4: + sxtl2(Dst.V4S(), Vector.V8H()); + break; + case 8: + sxtl2(Dst.V2D(), Vector.V4S()); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp index bfd0ce0d1..6a1baf977 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp @@ -2268,37 +2268,87 @@ DEF_OP(VUShrNI2) { } DEF_OP(VSXTL) { - auto Op = IROp->C(); - switch (Op->Header.ElementSize) { + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto Vector = GetSrc(Op->Vector.ID()); + + switch (ElementSize) { case 2: - pmovsxbw(GetDst(Node), GetSrc(Op->Vector.ID())); - break; + if (Is256Bit) { + vpmovsxbw(ToYMM(Dst), Vector); + } else { + vpmovsxbw(Dst, Vector); + } + break; case 4: - pmovsxwd(GetDst(Node), GetSrc(Op->Vector.ID())); - break; + if (Is256Bit) { + vpmovsxwd(ToYMM(Dst), Vector); + } else { + vpmovsxwd(Dst, Vector); + } + break; case 8: - pmovsxdq(GetDst(Node), GetSrc(Op->Vector.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); + if (Is256Bit) { + vpmovsxdq(ToYMM(Dst), Vector); + } else { + vpmovsxdq(Dst, Vector); + } + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; } } DEF_OP(VSXTL2) { - auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; - vpsrldq(GetDst(Node), GetSrc(Op->Vector.ID()), OpSize / 2); - switch (Op->Header.ElementSize) { - case 2: - pmovsxbw(GetDst(Node), GetDst(Node)); - break; - case 4: - pmovsxwd(GetDst(Node), GetDst(Node)); - break; - case 8: - pmovsxdq(GetDst(Node), GetDst(Node)); - break; - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto Vector = GetSrc(Op->Vector.ID()); + + if (Is256Bit) { + const auto DstYMM = ToYMM(Dst); + + vextracti128(Dst, ToYMM(Vector), 1); + switch (ElementSize) { + case 2: + vpmovsxbw(DstYMM, Dst); + break; + case 4: + vpmovsxwd(DstYMM, Dst); + break; + case 8: + vpmovsxdq(DstYMM, Dst); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } + } else { + vpsrldq(Dst, Vector, OpSize / 2); + switch (ElementSize) { + case 2: + vpmovsxbw(Dst, Dst); + break; + case 4: + vpmovsxwd(Dst, Dst); + break; + case 8: + vpmovsxdq(Dst, Dst); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } } } diff --git a/External/FEXCore/Source/Interface/IR/IR.json b/External/FEXCore/Source/Interface/IR/IR.json index d0740bda7..e3a114195 100644 --- a/External/FEXCore/Source/Interface/IR/IR.json +++ b/External/FEXCore/Source/Interface/IR/IR.json @@ -1065,7 +1065,7 @@ }, "FPR = VSXTL2 u8:#RegisterSize, u8:#ElementSize, FPR:$Vector": { "Desc": ["Sign extends elements from the source element size to the next size up", - "Source elements come from the upper 64bits of the register" + "Source elements come from the upper half of the register" ], "DestSize": "RegisterSize", "NumElements": "RegisterSize / (ElementSize << 1)"