diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp index d3aaba506..b9ecd1c56 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp @@ -358,23 +358,26 @@ DEF_OP(VAddP) { } DEF_OP(VAddV) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; void *Src = GetSrc(Data->SSAData, Op->Vector); - uint8_t Tmp[16]; + uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE]; - const uint8_t Elements = OpSize / Op->Header.ElementSize; + const uint8_t ElementSize = Op->Header.ElementSize; + const uint8_t Elements = OpSize / ElementSize; const auto Func = [](auto current, auto a) { return current + a; }; - switch (Op->Header.ElementSize) { + switch (ElementSize) { DO_VECTOR_REDUCE_1SRC_OP(1, int8_t, Func, 0) DO_VECTOR_REDUCE_1SRC_OP(2, int16_t, Func, 0) DO_VECTOR_REDUCE_1SRC_OP(4, int32_t, Func, 0) DO_VECTOR_REDUCE_1SRC_OP(8, int64_t, Func, 0) - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + return; } - memcpy(GDP, Tmp, Op->Header.ElementSize); + memcpy(GDP, Tmp, ElementSize); } DEF_OP(VUMinV) { diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp index 681570afa..0244270ac 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp @@ -621,20 +621,70 @@ DEF_OP(VAddP) { } DEF_OP(VAddV) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - const uint8_t Elements = OpSize / Op->Header.ElementSize; - // Vector - switch (Op->Header.ElementSize) { - case 1: - case 2: - case 4: - addv(GetDst(Node).VCast(Op->Header.ElementSize * 8, 1), GetSrc(Op->Vector.ID()).VCast(OpSize * 8, Elements)); - break; - case 8: - addp(GetDst(Node).VCast(OpSize * 8, 1), GetSrc(Op->Vector.ID()).VCast(OpSize * 8, Elements)); - break; - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto ElementSize = Op->Header.ElementSize; + const auto Elements = OpSize / ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto Vector = GetSrc(Op->Vector.ID()); + + if (HostSupportsSVE && Is256Bit) { + // SVE doesn't have an equivalent ADDV instruction, so we make do + // by performing two Adv. SIMD ADDV operations on the high and low + // 128-bit lanes and then sum them up. + + const auto Mask = PRED_TMP_32B.Zeroing(); + const auto CompactPred = p0; + + // Select all our upper elements to run ADDV over them. + not_(CompactPred.VnB(), Mask, PRED_TMP_16B.VnB()); + compact(VTMP1.Z().VnD(), CompactPred, Vector.Z().VnD()); + + switch (ElementSize) { + case 1: + addv(VTMP2.B(), Vector.V16B()); + addv(VTMP1.B(), VTMP1.V16B()); + add(Dst.V16B(), VTMP1.V16B(), VTMP2.V16B()); + break; + case 2: + addv(VTMP2.H(), Vector.V8H()); + addv(VTMP1.H(), VTMP1.V8H()); + add(Dst.V8H(), VTMP1.V8H(), VTMP2.V8H()); + break; + case 4: + addv(VTMP2.S(), Vector.V4S()); + addv(VTMP1.S(), VTMP1.V4S()); + add(Dst.V4S(), VTMP1.V4S(), VTMP2.V4S()); + break; + case 8: + addp(VTMP2.D(), Vector.V2D()); + addp(VTMP1.D(), VTMP1.V2D()); + add(Dst.V2D(), VTMP1.V2D(), VTMP2.V2D()); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } + } else { + const auto OpSizeBits = OpSize * 8; + const auto ElementSizeBits = ElementSize * 8; + + switch (ElementSize) { + case 1: + case 2: + case 4: + addv(Dst.VCast(ElementSizeBits, 1), Vector.VCast(OpSizeBits, Elements)); + break; + case 8: + addp(Dst.VCast(OpSizeBits, 1), Vector.VCast(OpSizeBits, Elements)); + break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp index d0540ba94..bc42905ff 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp @@ -404,37 +404,71 @@ DEF_OP(VAddP) { } DEF_OP(VAddV) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto Src = GetSrc(Op->Vector.ID()); + const auto Dest = GetDst(Node); + + const auto ElementSize = Op->Header.ElementSize; + const auto Elements = OpSize / ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - auto Src = GetSrc(Op->Vector.ID()); - auto Dest = GetDst(Node); vpxor(xmm15, xmm15, xmm15); - const uint8_t Elements = OpSize / Op->Header.ElementSize; - switch (Op->Header.ElementSize) { + switch (ElementSize) { case 2: { - for (int i = Elements; i > 1; i >>= 1) { - vphaddw(Dest, Src, Dest); - Src = Dest; + const auto HorizontalAdd = [this, Elements](const Xbyak::Xmm& dst, Xbyak::Xmm src, const Xbyak::Xmm& tmp) { + for (int i = Elements; i > 1; i >>= 1) { + vphaddw(dst, src, dst); + src = dst; + } + pextrw(eax, dst, 0); + pinsrw(tmp, eax, 0); + }; + + if (Is256Bit) { + vpxor(xmm13, xmm13, xmm13); + vextracti128(xmm14, ToYMM(Src), 1); + + HorizontalAdd(Dest, Src, xmm15); + HorizontalAdd(Dest, xmm14, xmm13); + + vpaddw(Dest, xmm13, xmm15); + } else { + HorizontalAdd(Dest, Src, xmm15); + vmovaps(Dest, xmm15); } - pextrw(eax, Dest, 0); - pinsrw(xmm15, eax, 0); - break; + break; } case 4: { - for (int i = Elements; i > 1; i >>= 1) { - vphaddd(Dest, Src, Dest); - Src = Dest; - } - pextrd(eax, Dest, 0); - pinsrd(xmm15, eax, 0); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; - } + const auto HorizontalAdd = [this, Elements](const Xbyak::Xmm& dst, Xbyak::Xmm src, const Xbyak::Xmm& tmp) { + for (int i = Elements; i > 1; i >>= 1) { + vphaddd(dst, src, dst); + src = dst; + } + pextrd(eax, dst, 0); + pinsrd(tmp, eax, 0); + }; - movaps(Dest, xmm15); + if (Is256Bit) { + vpxor(xmm13, xmm13, xmm13); + vextracti128(xmm14, ToYMM(Src), 1); + + HorizontalAdd(Dest, Src, xmm15); + HorizontalAdd(Dest, xmm14, xmm13); + + vpaddd(Dest, xmm13, xmm15); + } else { + HorizontalAdd(Dest, Src, xmm15); + vmovaps(Dest, xmm15); + } + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } } DEF_OP(VUMinV) {