diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp index 436c18669..a79504be2 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp @@ -448,6 +448,9 @@ DEF_OP(VBroadcastFromMem) { case 8: BroadcastElement(GDP, reinterpret_cast(MemData)); break; + case 16: + BroadcastElement(GDP, reinterpret_cast(MemData)); + break; default: LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem element size: {}", ElementSize); break; diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp index 2b2cd0ac2..60aba0a95 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp @@ -1439,8 +1439,9 @@ DEF_OP(VBroadcastFromMem) { const auto Dst = GetVReg(Node); const auto MemReg = GetReg(Op->Address.ID()); - LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, - "Invalid element size"); + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || + ElementSize == 4 || ElementSize == 8 || + ElementSize == 16, "Invalid element size"); if (HostSupportsSVE128 || HostSupportsSVE256) { if (Is256Bit) { @@ -1466,6 +1467,9 @@ DEF_OP(VBroadcastFromMem) { case 8: ld1rd(Dst.Z(), GoverningPredicate, MemReg); break; + case 16: + ld1rqb(Dst.Z(), GoverningPredicate, MemReg); + break; default: LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem size: {}", ElementSize); return; @@ -1484,6 +1488,10 @@ DEF_OP(VBroadcastFromMem) { case 8: ld1r(Dst.Q(), MemReg); break; + case 16: + // Normal load, like ld1rqb with 128-bit regs. + ldr(Dst.Q(), MemReg); + break; default: LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem size: {}", ElementSize); return; diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp index 19ec79182..cf6254047 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp @@ -871,6 +871,9 @@ DEF_OP(VBroadcastFromMem) { case 8: vpbroadcastq(DstYMM, qword [MemReg]); break; + case 16: + vbroadcasti128(DstYMM, xword [MemReg]); + break; default: LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem element size: {}", ElementSize); return; @@ -889,6 +892,9 @@ DEF_OP(VBroadcastFromMem) { case 8: vpbroadcastq(Dst, qword [MemReg]); break; + case 16: + vmovaps(Dst, xword [MemReg]); + break; default: LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem element size: {}", ElementSize); return; diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp index edcc3f0e3..a46948727 100644 --- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp +++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp @@ -1263,17 +1263,11 @@ void OpDispatchBuilder::VBROADCASTOp(OpcodeArgs) { OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1); Result = _VDupElement(DstSize, ElementSize, Src, 0); } else { - if constexpr (ElementSize == 16) { - // TODO: Make 128-bit loads go through the broadcast path. - OrderedNode *Src = LoadSource_WithOpSize(FPRClass, Op, Op->Src[0], ElementSize, Op->Flags, -1); - Result = _VDupElement(DstSize, ElementSize, Src, 0); - } else { - // Get the address to broadcast from into a GPR. - OrderedNode *Address = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], CTX->GetGPRSize(), Op->Flags, -1, false); - Address = AppendSegmentOffset(Address, Op->Flags); + // Get the address to broadcast from into a GPR. + OrderedNode *Address = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], CTX->GetGPRSize(), Op->Flags, -1, false); + Address = AppendSegmentOffset(Address, Op->Flags); - Result = _VBroadcastFromMem(DstSize, ElementSize, Address); - } + Result = _VBroadcastFromMem(DstSize, ElementSize, Address); } // No need to zero-extend result, since implementations diff --git a/unittests/InstructionCountCI/VEX_map2.json b/unittests/InstructionCountCI/VEX_map2.json index 149cf59c2..b02d3c709 100644 --- a/unittests/InstructionCountCI/VEX_map2.json +++ b/unittests/InstructionCountCI/VEX_map2.json @@ -1097,14 +1097,13 @@ ] }, "vbroadcastf128 ymm0, [rax]": { - "ExpectedInstructionCount": 3, + "ExpectedInstructionCount": 2, "Optimal": "No", "Comment": [ "Map 2 0b01 0x1a 256-bit" ], "ExpectedArm64ASM": [ - "ldr q4, [x4]", - "mov z4.q, q4", + "ld1rqb {z4.b}, p7/z, [x4]", "mov z16.d, p7/m, z4.d" ] }, @@ -2436,14 +2435,13 @@ ] }, "vbroadcasti128 ymm0, [rax]": { - "ExpectedInstructionCount": 3, + "ExpectedInstructionCount": 2, "Optimal": "No", "Comment": [ "Map 2 0b01 0x5a 256-bit" ], "ExpectedArm64ASM": [ - "ldr q4, [x4]", - "mov z4.q, q4", + "ld1rqb {z4.b}, p7/z, [x4]", "mov z16.d, p7/m, z4.d" ] },