IR: Allow 128-bit broadcasts in VBroadcastFromMem

Now all vbroadcast implementations go down the more optimal path.

For non-SVE 128-bit cases where we only have 128-bit wide registers,
we behave like ld1rqb and just act as a normal 128-bit load for
interface convenience.

External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp

@@ -448,6 +448,9 @@ DEF_OP(VBroadcastFromMem) {
   case 8:
     BroadcastElement(GDP, reinterpret_cast<const uint64_t*>(MemData));
     break;
+  case 16:
+    BroadcastElement(GDP, reinterpret_cast<const __uint128_t*>(MemData));
+    break;
   default:
     LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem element size: {}", ElementSize);
     break;

External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp

@@ -1439,8 +1439,9 @@ DEF_OP(VBroadcastFromMem) {
   const auto Dst = GetVReg(Node);
   const auto MemReg = GetReg(Op->Address.ID());
 
-  LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8,
-                      "Invalid element size");
+  LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 ||
+                      ElementSize == 4 || ElementSize == 8 ||
+                      ElementSize == 16, "Invalid element size");
 
   if (HostSupportsSVE128 || HostSupportsSVE256) {
     if (Is256Bit) {
@@ -1466,6 +1467,9 @@ DEF_OP(VBroadcastFromMem) {
     case 8:
       ld1rd(Dst.Z(), GoverningPredicate, MemReg);
       break;
+    case 16:
+      ld1rqb(Dst.Z(), GoverningPredicate, MemReg);
+      break;
     default:
       LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem size: {}", ElementSize);
       return;
@@ -1484,6 +1488,10 @@ DEF_OP(VBroadcastFromMem) {
     case 8:
       ld1r<ARMEmitter::SubRegSize::i64Bit>(Dst.Q(), MemReg);
       break;
+    case 16:
+      // Normal load, like ld1rqb with 128-bit regs.
+      ldr(Dst.Q(), MemReg);
+      break;
     default:
       LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem size: {}", ElementSize);
       return;

External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp

@@ -871,6 +871,9 @@ DEF_OP(VBroadcastFromMem) {
     case 8:
       vpbroadcastq(DstYMM, qword [MemReg]);
       break;
+    case 16:
+      vbroadcasti128(DstYMM, xword [MemReg]);
+      break;
     default:
       LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem element size: {}", ElementSize);
       return;
@@ -889,6 +892,9 @@ DEF_OP(VBroadcastFromMem) {
     case 8:
       vpbroadcastq(Dst, qword [MemReg]);
       break;
+    case 16:
+      vmovaps(Dst, xword [MemReg]);
+      break;
     default:
       LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem element size: {}", ElementSize);
       return;

External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp

@@ -1263,17 +1263,11 @@ void OpDispatchBuilder::VBROADCASTOp(OpcodeArgs) {
     OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
     Result = _VDupElement(DstSize, ElementSize, Src, 0);
   } else {
-    if constexpr (ElementSize == 16) {
-      // TODO: Make 128-bit loads go through the broadcast path.
-      OrderedNode *Src = LoadSource_WithOpSize(FPRClass, Op, Op->Src[0], ElementSize, Op->Flags, -1);
-      Result = _VDupElement(DstSize, ElementSize, Src, 0);
-    } else {
-      // Get the address to broadcast from into a GPR.
-      OrderedNode *Address = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], CTX->GetGPRSize(), Op->Flags, -1, false);
-      Address = AppendSegmentOffset(Address, Op->Flags);
+    // Get the address to broadcast from into a GPR.
+    OrderedNode *Address = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], CTX->GetGPRSize(), Op->Flags, -1, false);
+    Address = AppendSegmentOffset(Address, Op->Flags);
 
-      Result = _VBroadcastFromMem(DstSize, ElementSize, Address);
-    }
+    Result = _VBroadcastFromMem(DstSize, ElementSize, Address);
   }
 
   // No need to zero-extend result, since implementations

unittests/InstructionCountCI/VEX_map2.json

@@ -1097,14 +1097,13 @@
       ]
     },
     "vbroadcastf128 ymm0, [rax]": {
-      "ExpectedInstructionCount": 3,
+      "ExpectedInstructionCount": 2,
       "Optimal": "No",
       "Comment": [
         "Map 2 0b01 0x1a 256-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q4, [x4]",
-        "mov z4.q, q4",
+        "ld1rqb {z4.b}, p7/z, [x4]",
         "mov z16.d, p7/m, z4.d"
       ]
     },
@@ -2436,14 +2435,13 @@
       ]
     },
     "vbroadcasti128 ymm0, [rax]": {
-      "ExpectedInstructionCount": 3,
+      "ExpectedInstructionCount": 2,
       "Optimal": "No",
       "Comment": [
         "Map 2 0b01 0x5a 256-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q4, [x4]",
-        "mov z4.q, q4",
+        "ld1rqb {z4.b}, p7/z, [x4]",
         "mov z16.d, p7/m, z4.d"
       ]
     },