Merge pull request #6011 from javelina-pkwy/perf/gather-all-ones

Peephole optimization for VPCMPEQD + VGATHERDPS

FEXCore/Source/Interface/Core/DiskCache.h

@@ -249,7 +249,7 @@ namespace DiskCache {
   // The current version of the diskcache.
   // This must be changed any time codegen changes occur!
   // Be aware of the impact of changing this frequently!
-  static constexpr uint16_t FormatVersion = 29;
+  static constexpr uint16_t FormatVersion = 30;
 
   static constexpr uint32_t LOOKUP_KEY_MAX_BUCKET_DEPTH = 500;
 

FEXCore/Source/Interface/Core/JIT/JITClass.h

@@ -629,7 +629,8 @@ private:
   void Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize, ARMEmitter::VRegister Dst, ARMEmitter::VRegister IncomingDst,
                            std::optional<ARMEmitter::Register> BaseAddr, ARMEmitter::VRegister VectorIndexLow,
                            std::optional<ARMEmitter::VRegister> VectorIndexHigh, ARMEmitter::VRegister MaskReg, IR::OpSize VectorIndexSize,
-                           size_t DataElementOffsetStart, size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize);
+                           size_t DataElementOffsetStart, size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize,
+                           bool MaskAllOnes);
 
   void EmitTFCheck();
 

FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp

@@ -1058,7 +1058,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
                                        ARMEmitter::VRegister IncomingDst, std::optional<ARMEmitter::Register> BaseAddr,
                                        ARMEmitter::VRegister VectorIndexLow, std::optional<ARMEmitter::VRegister> VectorIndexHigh,
                                        ARMEmitter::VRegister MaskReg, IR::OpSize VectorIndexSize, size_t DataElementOffsetStart,
-                                       size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize) {
+                                       size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize, bool MaskAllOnes) {
   LOGMAN_THROW_A_FMT(ElementSize >= IR::OpSize::i8Bit && ElementSize <= IR::OpSize::i64Bit, "Invalid element size");
 
   const auto PerformSMove = [this](IR::OpSize ElementSize, const ARMEmitter::Register Dst, const ARMEmitter::VRegister Vector, int index) {
@@ -1116,11 +1116,13 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
 
   for (size_t i = DataElementOffsetStart, IndexElement = IndexElementOffsetStart; i < NumDataElements; ++i, ++IndexElement) {
     ARMEmitter::ForwardLabel Skip {};
-    // Extract mask element
-    PerformMove(ElementSize, WorkingReg, MaskReg, i);
+    if (!MaskAllOnes) {
+      // Extract mask element
+      PerformMove(ElementSize, WorkingReg, MaskReg, i);
 
-    // Skip if the mask's sign bit isn't set
-    (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
+      // Skip if the mask's sign bit isn't set
+      (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
+    }
 
     // Extract Index Element
     if ((IndexElement * IR::OpSizeToSize(VectorIndexSize)) >= 16) {
@@ -1216,12 +1218,15 @@ DEF_OP(VLoadVectorGatherMasked) {
 
     const auto SubRegSize = ConvertSubRegSize8(IROp);
 
-    const auto CMPPredicate = ARMEmitter::PReg::p0;
     const auto GoverningPredicate = Is256Bit ? PRED_TMP_32B : PRED_TMP_16B;
 
-    // Check if the sign bit is set for the given element size.
-    cmplt(SubRegSize, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);
-    auto TempDst = VTMP1;
+    // A mask known to be all-ones loads every element straight into the destination under the governing predicate.
+    const ARMEmitter::PRegister CMPPredicate = Op->MaskAllOnes ? GoverningPredicate : ARMEmitter::PReg::p0;
+    auto TempDst = Op->MaskAllOnes ? Dst : VTMP1;
+    if (!Op->MaskAllOnes) {
+      // Check if the sign bit is set for the given element size.
+      cmplt(SubRegSize, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);
+    }
 
     // No need to load a temporary register in the case that we weren't provided a base address and there is no scaling.
     ARMEmitter::SVEMemOperand MemDst {ARMEmitter::SVEMemOperand(VectorIndexLow.Z(), 0)};
@@ -1256,12 +1261,14 @@ DEF_OP(VLoadVectorGatherMasked) {
     default: break;
     }
 
-    ///< Merge elements based on predicate.
-    sel(SubRegSize, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());
+    if (!Op->MaskAllOnes) {
+      ///< Merge elements based on predicate.
+      sel(SubRegSize, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());
+    }
   } else {
     LOGMAN_THROW_A_FMT(!Is256Bit, "Can't emulate this gather load in the backend! Programming error!");
     Emulate128BitGather(IROp->Size, IROp->ElementSize, Dst, IncomingDst, BaseAddr, VectorIndexLow, VectorIndexHigh, MaskReg,
-                        VectorIndexSize, DataElementOffsetStart, IndexElementOffsetStart, OffsetScale, Op->AddrSize);
+                        VectorIndexSize, DataElementOffsetStart, IndexElementOffsetStart, OffsetScale, Op->AddrSize, Op->MaskAllOnes);
   }
 }
 
@@ -1295,16 +1302,20 @@ DEF_OP(VLoadVectorGatherMaskedQPS) {
     }
 
     const auto CMPPredicate = ARMEmitter::PReg::p0;
-    const auto CMPPredicate2 = ARMEmitter::PReg::p1;
+    // A mask known to be all-ones loads every element under the governing predicate and narrows straight into the destination.
+    const ARMEmitter::PRegister CMPPredicate2 = Op->MaskAllOnes ? PRED_TMP_16B : ARMEmitter::PReg::p1;
 
     const auto GoverningPredicate = PRED_TMP_16B;
 
-    // Check if the sign bit is set for the given element size.
-    // This will set the predicate bits for elements [0, 1, 2, 3]
-    // We then use punpklo to extend the low results to be for 64-bit elements.
-    cmplt(ARMEmitter::SubRegSize::i32Bit, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);
-    punpklo(CMPPredicate2, CMPPredicate);
+    if (!Op->MaskAllOnes) {
+      // Check if the sign bit is set for the given element size.
+      // This will set the predicate bits for elements [0, 1, 2, 3]
+      // We then use punpklo to extend the low results to be for 64-bit elements.
+      cmplt(ARMEmitter::SubRegSize::i32Bit, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);
+      punpklo(CMPPredicate2, CMPPredicate);
+    }
     auto TempDst = VTMP1;
+    const auto NarrowDst = Op->MaskAllOnes ? Dst : TempDst;
 
     auto GatherExtend = [this](ARMEmitter::VRegister Dst, std::optional<ARMEmitter::Register> BaseAddr, ARMEmitter::VRegister VectorIndex,
                                ARMEmitter::PRegister CMPPredicate, ARMEmitter::SVEModType ModType, uint8_t OffsetScale) {
@@ -1328,21 +1339,23 @@ DEF_OP(VLoadVectorGatherMaskedQPS) {
     GatherExtend(TempDst, BaseAddr, VectorIndexLow, CMPPredicate2, ModType, OffsetScale);
 
     if (VectorIndexHigh.has_value()) {
-      punpkhi(CMPPredicate2, CMPPredicate);
+      if (!Op->MaskAllOnes) {
+        punpkhi(CMPPredicate2, CMPPredicate);
+      }
       GatherExtend(VTMP2, BaseAddr, *VectorIndexHigh, CMPPredicate2, ModType, OffsetScale);
       // Move elements to the lower half.
-      uzp1(ARMEmitter::SubRegSize::i32Bit, TempDst.Q(), TempDst.Q(), VTMP2.Q());
-      ///< Merge elements based on predicate.
-      sel(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());
+      uzp1(ARMEmitter::SubRegSize::i32Bit, NarrowDst.Q(), TempDst.Q(), VTMP2.Q());
     } else {
       // Move elements to the lower half.
-      xtn(ARMEmitter::SubRegSize::i32Bit, TempDst.Q(), TempDst.Q());
+      xtn(ARMEmitter::SubRegSize::i32Bit, NarrowDst.Q(), TempDst.Q());
+    }
+    if (!Op->MaskAllOnes) {
       ///< Merge elements based on predicate.
       sel(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());
     }
   } else {
     Emulate128BitGather(IR::OpSize::i128Bit, IR::OpSize::i32Bit, Dst, IncomingDst, BaseAddr, VectorIndexLow, VectorIndexHigh, MaskReg,
-                        IR::OpSize::i64Bit, 0, 0, OffsetScale, Op->AddrSize);
+                        IR::OpSize::i64Bit, 0, 0, OffsetScale, Op->AddrSize, Op->MaskAllOnes);
   }
 }
 

FEXCore/Source/Interface/Core/OpcodeDispatcher.h

@@ -1084,8 +1084,9 @@ public:
   void AVX128_VFMAScalarImpl(OpcodeArgs, IROps IROp, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);
   void AVX128_VFMAddSubImpl(OpcodeArgs, bool AddSub, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);
 
-  RefPair AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB);
-  RefPair AVX128_VPGatherImpl(OpcodeArgs, OpSize Size, OpSize ElementLoadSize, OpSize AddrElementSize, RefPair Dest, RefPair Mask, RefVSIB VSIB);
+  RefPair AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB, bool MaskAllOnes);
+  RefPair AVX128_VPGatherImpl(OpcodeArgs, OpSize Size, OpSize ElementLoadSize, OpSize AddrElementSize, RefPair Dest, RefPair Mask,
+                              RefVSIB VSIB, bool MaskAllOnesLow = false, bool MaskAllOnesHigh = false);
 
   void AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize);
 
@@ -1178,6 +1179,11 @@ public:
     // At block boundaries, fix up the carry flag.
     if (!SRAOnly) {
       RectifyCarryInvert(CFInvertedABI);
+
+      // Also we can't trust the LastDef accross block boundaries, so clear that as well.
+      for (auto& Def : RegCache.LastDef) {
+        Def = nullptr;
+      }
     }
 
     if (!MMXOnly) {
@@ -1963,12 +1969,17 @@ private:
     uint64_t Partial;
 
     Ref Value[64];
+
+    // Keep track of where each register was last written to within the current block,
+    // for use in peephole optimizations.
+    Ref LastDef[64];
   } RegCache {};
 
   void InvalidateReg(uint8_t Index) {
     uint64_t Bit = (1ull << (uint64_t)Index);
     RegCache.Cached &= ~Bit;
     RegCache.Written &= ~Bit;
+    RegCache.LastDef[Index] = nullptr;
   }
 
   Ref LoadRegCache(uint64_t Offset, uint8_t Index, RegClass Class, IR::OpSize Size) {
@@ -2084,6 +2095,7 @@ private:
     RegCache.Value[Index] = Value;
     RegCache.Cached |= Bit;
     RegCache.Written |= Bit;
+    RegCache.LastDef[Index] = Value;
   }
 
   void InvalidateHighAVXRegisters() {

FEXCore/Source/Interface/Core/OpcodeDispatcher/AVX_128.cpp

@@ -2052,7 +2052,8 @@ void OpDispatchBuilder::AVX128_VFMAddSubImpl(OpcodeArgs, bool AddSub, uint8_t Sr
 }
 
 OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, OpSize Size, OpSize ElementLoadSize, OpSize AddrElementSize,
-                                                                  RefPair Dest, RefPair Mask, RefVSIB VSIB) {
+                                                                  RefPair Dest, RefPair Mask, RefVSIB VSIB, bool MaskAllOnesLow,
+                                                                  bool MaskAllOnesHigh) {
   LOGMAN_THROW_A_FMT(AddrElementSize == OpSize::i32Bit || AddrElementSize == OpSize::i64Bit, "Unknown address element size");
   const auto Is128Bit = Size == OpSize::i128Bit;
 
@@ -2102,7 +2103,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
   RefPair Result {};
   ///< Calculate the low-half.
   Result.Low = _VLoadVectorGatherMasked(OpSize::i128Bit, ElementLoadSize, Dest.Low, Mask.Low, BaseAddr, VSIB.Low, VSIB.High,
-                                        AddrElementSize, VSIB.Scale, 0, 0, AddrSize);
+                                        AddrElementSize, VSIB.Scale, 0, 0, AddrSize, MaskAllOnesLow);
 
   if (Is128Bit) {
     Result.High = LoadZeroVector(OpSize::i128Bit);
@@ -2117,6 +2118,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
 
     Ref DestReg = Dest.High;
     Ref MaskReg = Mask.High;
+    bool MaskAllOnes = MaskAllOnesHigh;
     uint8_t IndexElementOffset {};
     uint8_t DataElementOffset {};
     if (AddrElementSize == ElementLoadSize) {
@@ -2133,13 +2135,15 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
       AddrAddressing.High = Invalid();
       DestReg = Result.Low; ///< Start mixing with the low register.
       MaskReg = Mask.Low;   ///< Mask starts with the low mask here.
+      MaskAllOnes = MaskAllOnesLow;
       IndexElementOffset = 0;
       DataElementOffset = IR::NumElements(OpSize::i128Bit, ElementLoadSize) / 2;
     }
 
     ///< Calculate the high-half.
-    auto ResultHigh = _VLoadVectorGatherMasked(OpSize::i128Bit, ElementLoadSize, DestReg, MaskReg, BaseAddr, AddrAddressing.Low,
-                                               AddrAddressing.High, AddrElementSize, VSIB.Scale, DataElementOffset, IndexElementOffset, AddrSize);
+    auto ResultHigh =
+      _VLoadVectorGatherMasked(OpSize::i128Bit, ElementLoadSize, DestReg, MaskReg, BaseAddr, AddrAddressing.Low, AddrAddressing.High,
+                               AddrElementSize, VSIB.Scale, DataElementOffset, IndexElementOffset, AddrSize, MaskAllOnes);
 
     if (AddrElementSize == OpSize::i64Bit && ElementLoadSize == OpSize::i32Bit) {
       // If we only fetched 128-bits worth of data then the upper-result is all zero.
@@ -2152,7 +2156,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
   return Result;
 }
 
-OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB) {
+OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB, bool MaskAllOnes) {
 
   ///< BaseAddr doesn't need to exist, calculate that here.
   Ref BaseAddr = VSIB.BaseAddr;
@@ -2184,7 +2188,8 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(OpcodeArgs,
   auto AddrSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) != 0 ? (GPRSize >> 1) : GPRSize;
 
   ///< Calculate the low-half.
-  Result.Low = _VLoadVectorGatherMaskedQPS(OpSize::i128Bit, OpSize::i32Bit, Dest, Mask, BaseAddr, VSIB.Low, VSIB.High, VSIB.Scale, AddrSize);
+  Result.Low =
+    _VLoadVectorGatherMaskedQPS(OpSize::i128Bit, OpSize::i32Bit, Dest, Mask, BaseAddr, VSIB.Low, VSIB.High, VSIB.Scale, AddrSize, MaskAllOnes);
   Result.High = LoadZeroVector(OpSize::i128Bit);
   if (VSIB.High == Invalid()) {
     // Special case for only loading two floats.
@@ -2213,6 +2218,21 @@ void OpDispatchBuilder::AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize) {
   auto VSIB = AVX128_LoadVSIB(Op, Op->Src[0], Op->Flags, NeedsHighAddrBytes);
   auto Mask = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);
 
+  // Check if the mask was computed from a self comparison, which indicates an all-ones mask.
+  // This is a compiler idiom for an unmasked gather. In this case, we can apply a peephole and skip
+  // all mask checks in the codegen, reducing codesize substantially.
+  bool MaskAllOnesLow = false;
+  bool MaskAllOnesHigh = false;
+  if (Op->Src[1].IsGPR()) {
+    const auto MaskXMM = Op->Src[1].Data.GPR.GPR - X86State::REG_XMM_0;
+
+    const Ref LowDef = RegCache.LastDef[FPR0Index + MaskXMM];
+    const Ref HighDef = RegCache.LastDef[AVXHigh0Index + MaskXMM];
+
+    MaskAllOnesLow = LowDef && IsValueVectorAllOnes(WrapNode(LowDef));
+    MaskAllOnesHigh = !Is128Bit && HighDef && IsValueVectorAllOnes(WrapNode(HighDef));
+  }
+
   bool NeedsSVEScale = (VSIB.Scale == 2 || VSIB.Scale == 8) || (VSIB.BaseAddr == Invalid() && VSIB.Scale != 1);
 
   const bool NeedsExplicitSVEPath =
@@ -2243,15 +2263,15 @@ void OpDispatchBuilder::AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize) {
     }
 
     ///< AddressElementSize is now OpSize::i64Bit
-    Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIBLow);
+    Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIBLow, MaskAllOnesLow);
     if (NeedsHighAddrBytes) {
-      auto Res = AVX128_VPGatherQPSImpl(Op, Dest.High, Mask.High, VSIBHigh);
+      auto Res = AVX128_VPGatherQPSImpl(Op, Dest.High, Mask.High, VSIBHigh, MaskAllOnesHigh);
       Result.High = Res.Low;
     }
   } else if (AddrElementSize == OpSize::i64Bit && ElementLoadSize == OpSize::i32Bit) {
-    Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIB);
+    Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIB, MaskAllOnesLow);
   } else {
-    Result = AVX128_VPGatherImpl(Op, Size, ElementLoadSize, AddrElementSize, Dest, Mask, VSIB);
+    Result = AVX128_VPGatherImpl(Op, Size, ElementLoadSize, AddrElementSize, Dest, Mask, VSIB, MaskAllOnesLow, MaskAllOnesHigh);
   }
   AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
 

FEXCore/Source/Interface/IR/IR.json

@@ -604,11 +604,12 @@
         "DestSize": "RegisterSize",
         "ElementSize": "ElementSize"
       },
-      "FPR = VLoadVectorGatherMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$Mask, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, OpSize:$VectorIndexElementSize, u8:$OffsetScale, u8:$DataElementOffsetStart, u8:$IndexElementOffsetStart, OpSize:$AddrSize": {
+      "FPR = VLoadVectorGatherMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$Mask, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, OpSize:$VectorIndexElementSize, u8:$OffsetScale, u8:$DataElementOffsetStart, u8:$IndexElementOffsetStart, OpSize:$AddrSize, i1:$MaskAllOnes{false}": {
         "Desc": [
           "Does a masked load similar to VPGATHERD* where the upper bit of each element",
           "determines whether or not that element will be loaded from memory.",
-          "Most of VSIB encoding is passed directly through to the IR operation."
+          "Most of VSIB encoding is passed directly through to the IR operation.",
+          "MaskAllOnes is set when every mask element is provably all ones."
         ],
         "TiedSource": 0,
         "ImplicitFlagClobber": true,
@@ -618,12 +619,13 @@
           "$VectorIndexElementSize == OpSize::i32Bit || $VectorIndexElementSize == OpSize::i64Bit"
         ]
       },
-      "FPR = VLoadVectorGatherMaskedQPS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$MaskReg, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, u8:$OffsetScale, OpSize:$AddrSize": {
+      "FPR = VLoadVectorGatherMaskedQPS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$MaskReg, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, u8:$OffsetScale, OpSize:$AddrSize, i1:$MaskAllOnes{false}": {
         "Desc": [
           "Does a masked load similar to VPGATHERQPS where the upper bit of each element",
           "determines whether or not that element will be loaded from memory.",
           "Most of VSIB encoding is passed directly through to the IR operation.",
-          "Only supports the case of 32-bit data element sizes from 64-bit addresses"
+          "Only supports the case of 32-bit data element sizes from 64-bit addresses",
+          "MaskAllOnes is set when every mask element is provably all ones."
         ],
         "TiedSource": 0,
         "ImplicitFlagClobber": true,

FEXCore/Source/Interface/IR/IREmitter.h

@@ -341,6 +341,15 @@ public:
     return false;
   }
 
+  // Returns true if every bit of the vector value is set.
+  // Currently this only recognizes the self compare equal idiom.
+  // TODO: Should this recognize other patterns?
+  bool IsValueVectorAllOnes(OrderedNodeWrapper ssa) const {
+    auto RealNode = ssa.GetNode(DualListData.ListBegin());
+    const auto* IROp = RealNode->Op(DualListData.DataBegin());
+    return IROp->Op == OP_VCMPEQ && IROp->Args[0].ID() == IROp->Args[1].ID();
+  }
+
   FEXCore::IR::IROp_Header* GetOpHeader(OrderedNodeWrapper ssa) {
     Ref RealNode = ssa.GetNode(DualListData.ListBegin());
     return RealNode->Op(DualListData.DataBegin());

unittests/InstructionCountCI/AFP/H0F3A.json

@@ -8,7 +8,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "roundss xmm0, xmm1, 00000000b": {

unittests/InstructionCountCI/AFP/SVE256/Secondary.json

@@ -7,7 +7,7 @@
       "AFP"
     ],
     "DisabledHostFeatures": [],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "cvtpi2ps xmm0, [rax]": {

unittests/InstructionCountCI/AFP/SVE256/Secondary_REP.json

@@ -9,7 +9,7 @@
     "DisabledHostFeatures": [
       "RPRES"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "cvtsi2ss xmm0, eax": {

unittests/InstructionCountCI/AFP/SVE256/Secondary_REPNE.json

@@ -7,7 +7,7 @@
       "AFP"
     ],
     "DisabledHostFeatures": [],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "cvtsi2sd xmm0, eax": {

unittests/InstructionCountCI/AFP/Secondary.json

@@ -8,7 +8,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "cvtpi2ps xmm0, [rax]": {

unittests/InstructionCountCI/AFP/Secondary_REP.json

@@ -9,7 +9,7 @@
       "SVE256",
       "RPRES"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "cvtsi2ss xmm0, eax": {

unittests/InstructionCountCI/AFP/Secondary_REPNE.json

@@ -8,7 +8,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "cvtsi2sd xmm0, eax": {

unittests/InstructionCountCI/AFP/VEX_map1.json

@@ -9,7 +9,7 @@
     "DisabledHostFeatures": [
       "RPRES"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vsqrtss xmm0, xmm1, xmm2": {

unittests/InstructionCountCI/AFP/VEX_map3.json

@@ -7,7 +7,7 @@
       "AFP"
     ],
     "DisabledHostFeatures": [],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vroundss xmm0, xmm1, 00000000b": {

unittests/InstructionCountCI/AVX128/FMA4.json

@@ -7,7 +7,7 @@
       "SVE256",
       "SVE128"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vfmaddsubps xmm0, xmm1, xmm2, xmm3": {

unittests/InstructionCountCI/AVX128/VEX_map1.json

@@ -13,7 +13,7 @@
       "SVE256",
       "SVE128"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vmovups xmm0, xmm0": {

unittests/InstructionCountCI/AVX128/VEX_map1_FCMA.json

@@ -9,7 +9,7 @@
       "SVE256",
       "SVE128"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vaddsubpd xmm0, xmm1, xmm2": {

unittests/InstructionCountCI/AVX128/VEX_map1_SVE128.json

@@ -10,7 +10,7 @@
       "FLAGM2",
       "SVE256"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vmovntps [rax], xmm0": {

unittests/InstructionCountCI/AVX128/VEX_map1_flagm.json

@@ -10,7 +10,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vucomiss xmm0, xmm1": {

unittests/InstructionCountCI/AVX128/VEX_map2.json

@@ -11,7 +11,7 @@
       "I8MM",
       "DOTPROD"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vpshufb xmm0, xmm1, xmm2": {

unittests/InstructionCountCI/AVX128/VEX_map2_AFP.json

@@ -8,7 +8,7 @@
       "SVE256",
       "SVE128"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vfmadd132ss xmm0, xmm1, xmm2": {

unittests/InstructionCountCI/AVX128/VEX_map2_DOTPROD.json

@@ -12,7 +12,7 @@
       "SVE256",
       "I8MM"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vpdpbusd xmm0, xmm1, xmm2": {

unittests/InstructionCountCI/AVX128/VEX_map2_I8MM.json

@@ -11,7 +11,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vpdpbusd xmm0, xmm1, xmm2": {

unittests/InstructionCountCI/AVX128/VEX_map2_SVE128.json

@@ -10,7 +10,7 @@
       "FLAGM2",
       "SVE256"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vmovntdqa xmm0, [rax]": {

unittests/InstructionCountCI/AVX128/VEX_map2_flagm.json

@@ -11,7 +11,7 @@
       "SVE256",
       "SVEBITPERM"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vtestps xmm0, xmm1": {

unittests/InstructionCountCI/AVX128/VEX_map3.json

@@ -7,7 +7,7 @@
       "SVE256",
       "SVE128"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vpermq ymm0, ymm1, 1": {

unittests/InstructionCountCI/AVX128/VEX_map3_SVE128.json

@@ -8,7 +8,7 @@
       "AFP",
       "SVE256"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vblendvps xmm0, xmm1, xmm2, xmm3": {

unittests/InstructionCountCI/AVX128/VEX_map_group.json

@@ -9,7 +9,7 @@
       "SVE256",
       "SVE128"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vpsrlw xmm0, xmm1, 0": {

unittests/InstructionCountCI/Atomics.json

@@ -9,7 +9,7 @@
       "FLAGM",
       "FLAGM2"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "lock add byte [rax], cl": {

unittests/InstructionCountCI/Crypto/H0F38.json

@@ -9,7 +9,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "sha1nexte xmm0, xmm1": {

unittests/InstructionCountCI/Crypto/H0F38_SVE128.json

@@ -9,7 +9,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "sha1nexte xmm0, xmm1": {

unittests/InstructionCountCI/Crypto/H0F3A.json

@@ -9,7 +9,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "pclmulqdq xmm0, xmm1, 00000b": {

unittests/InstructionCountCI/DDD.json

@@ -10,7 +10,7 @@
       "AFP",
       "RPRES"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Comment": [
     "These 3DNow! instructions are optimal assuming that FEX doesn't SRA MMX registers",

unittests/InstructionCountCI/FEXOpt/AddressingLimitations.json

@@ -8,7 +8,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Comment": [
     "Instructions that explicitly push against the limits of ARM's loadstore instructions"

unittests/InstructionCountCI/FEXOpt/AddressingLimitations_32Bit.json

@@ -8,7 +8,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Comment": [
     "Instructions that explicitly push against the limits of ARM's loadstore instructions"

unittests/InstructionCountCI/FEXOpt/MultiInst.json

@@ -13,7 +13,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Comment": [
     "These are instruction combinations that could be more optimal if FEX optimized for them"

unittests/InstructionCountCI/FEXOpt/MultiInst_32bit.json

@@ -11,7 +11,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Comment": [
     "These are instruction combinations that could be more optimal if FEX optimized for them"

unittests/InstructionCountCI/FEXOpt/MultiInst_AFP.json

@@ -9,7 +9,7 @@
       "SVE256",
       "RPRES"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Comment": [
     "These are instruction combinations that could be more optimal if FEX optimized for them"

unittests/InstructionCountCI/FEXOpt/MultiInst_TSO.json

@@ -14,7 +14,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Comment": [
     "These are instruction combinations that could be more optimal if FEX optimized for them"

unittests/InstructionCountCI/FEXOpt/MultiInst_TSO_32bit.json

@@ -14,7 +14,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Comment": [
     "These are instruction combinations that could be more optimal if FEX optimized for them"

unittests/InstructionCountCI/FEXOpt/Peephole.json

@@ -0,0 +1,70 @@
+{
+  "Features": {
+    "Bitness": 64,
+    "EnabledHostFeatures": [],
+    "DisabledHostFeatures": [
+      "AFP",
+      "FLAGM",
+      "FLAGM2",
+      "SVE128",
+      "SVE256",
+      "I8MM",
+      "DOTPROD"
+    ],
+    "BinaryCacheVersion": 30
+  },
+  "Comment": [
+    "This file tracks the translations for the 'peephole' optimizations in FEX.",
+    "Peephole optimizations are small targetted optimizations that recognize",
+    "specific patterns in x86 code and specialize the translation accordingly."
+  ],
+  "Instructions": {
+    "vpcmpeqd + vpgatherdd": {
+      "x86InstructionCount": 2,
+      "ExpectedInstructionCount": 32,
+      "Comment": [
+        "Setting the mask to all-ones using vpcmpeqd, and then performing the gather operation.",
+        "This is an idiom emitted by Clang for an unmasked gather.",
+        "https://godbolt.org/z/4devEvfKf"
+      ],
+      "x86Insts": [
+        "vpcmpeqd ymm2, ymm2, ymm2",
+        "vpgatherdd ymm0, [ymm1*4 + rax], ymm2"
+      ],
+      "ExpectedArm64ASM": [
+        "ldr q2, [x28, #224]",
+        "cmeq v18.4s, v18.4s, v18.4s",
+        "cmeq v2.4s, v2.4s, v2.4s",
+        "ldr q3, [x28, #192]",
+        "ldr q4, [x28, #208]",
+        "smov x0, v17.s[0]",
+        "add x1, x4, w0, sxtw #2",
+        "ld1 {v16.s}[0], [x1]",
+        "smov x0, v17.s[1]",
+        "add x1, x4, w0, sxtw #2",
+        "ld1 {v16.s}[1], [x1]",
+        "smov x0, v17.s[2]",
+        "add x1, x4, w0, sxtw #2",
+        "ld1 {v16.s}[2], [x1]",
+        "smov x0, v17.s[3]",
+        "add x1, x4, w0, sxtw #2",
+        "ld1 {v16.s}[3], [x1]",
+        "smov x0, v4.s[0]",
+        "add x1, x4, w0, sxtw #2",
+        "ld1 {v3.s}[0], [x1]",
+        "smov x0, v4.s[1]",
+        "add x1, x4, w0, sxtw #2",
+        "ld1 {v3.s}[1], [x1]",
+        "smov x0, v4.s[2]",
+        "add x1, x4, w0, sxtw #2",
+        "ld1 {v3.s}[2], [x1]",
+        "smov x0, v4.s[3]",
+        "add x1, x4, w0, sxtw #2",
+        "ld1 {v3.s}[3], [x1]",
+        "movi v18.2d, #0x0",
+        "stp xzr, xzr, [x28, #224]",
+        "str q3, [x28, #192]"
+      ]
+    }
+  }
+}

unittests/InstructionCountCI/FEXOpt/Peephole_SVE128.json

@@ -0,0 +1,49 @@
+{
+  "Features": {
+    "Bitness": 64,
+    "EnabledHostFeatures": [
+      "SVE128"
+    ],
+    "DisabledHostFeatures": [
+      "AFP",
+      "FLAGM",
+      "FLAGM2",
+      "SVE256"
+    ],
+    "BinaryCacheVersion": 30
+  },
+  "Comment": [
+    "This file tracks the translations for the 'peephole' optimizations in FEX.",
+    "Peephole optimizations are small targetted optimizations that recognize",
+    "specific patterns in x86 code and specialize the translation accordingly."
+  ],
+  "Instructions": {
+    "vpcmpeqd + vpgatherdd": {
+      "x86InstructionCount": 2,
+      "ExpectedInstructionCount": 12,
+      "Comment": [
+        "Setting the mask to all-ones using vpcmpeqd, and then performing the gather operation.",
+        "This is an idiom emitted by Clang for an unmasked gather.",
+        "https://godbolt.org/z/4devEvfKf"
+      ],
+      "x86Insts": [
+        "vpcmpeqd ymm2, ymm2, ymm2",
+        "vpgatherdd ymm0, [ymm1*4 + rax], ymm2"
+      ],
+      "ExpectedArm64ASM": [
+        "ldr q2, [x28, #224]",
+        "cmeq v18.4s, v18.4s, v18.4s",
+        "cmeq v2.4s, v2.4s, v2.4s",
+        "ldr q3, [x28, #192]",
+        "ldr q4, [x28, #208]",
+        "mrs x20, nzcv",
+        "ld1w {z16.s}, p6/z, [x4, z17.s, sxtw #2]",
+        "ld1w {z3.s}, p6/z, [x4, z4.s, sxtw #2]",
+        "movi v18.2d, #0x0",
+        "msr nzcv, x20",
+        "stp xzr, xzr, [x28, #224]",
+        "str q3, [x28, #192]"
+      ]
+    }
+  }
+}

unittests/InstructionCountCI/FEXOpt/libnss.json

@@ -12,7 +12,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Comment": [],
   "Instructions": {

unittests/InstructionCountCI/FlagM/Atomics.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "lock add byte [rax], cl": {

unittests/InstructionCountCI/FlagM/FlagOpts.json

@@ -9,7 +9,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "Chained add": {

unittests/InstructionCountCI/FlagM/H0F38.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "ptest xmm0, xmm1": {

unittests/InstructionCountCI/FlagM/HotBlocks.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "The Witcher 3": {

unittests/InstructionCountCI/FlagM/HotBlocks_32Bit.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "Sonic Mania movie player": {

unittests/InstructionCountCI/FlagM/HotBlocks_AFP.json

@@ -10,7 +10,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "FMOD scalar loop": {

unittests/InstructionCountCI/FlagM/HotBlocks_TSO_32Bit.json

@@ -13,7 +13,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "The Sims 1 hot block": {

unittests/InstructionCountCI/FlagM/Primary.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "add bl, cl": {

unittests/InstructionCountCI/FlagM/PrimaryGroup.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "add al, 1": {

unittests/InstructionCountCI/FlagM/Primary_32Bit.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "push es": {

unittests/InstructionCountCI/FlagM/Secondary.json

@@ -11,7 +11,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "ucomiss xmm0, xmm1": {

unittests/InstructionCountCI/FlagM/SecondaryGroup.json

@@ -11,7 +11,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "sgdt [rax]": {

unittests/InstructionCountCI/FlagM/SecondaryModRM.json

@@ -11,7 +11,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "xgetbv": {

unittests/InstructionCountCI/FlagM/Secondary_OpSize.json

@@ -11,7 +11,7 @@
       "AFP",
       "FCMA"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "ucomisd xmm0, xmm1": {

unittests/InstructionCountCI/FlagM/Secondary_REP.json

@@ -12,7 +12,7 @@
       "AFP",
       "CSSC"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "popcnt ax, bx": {

unittests/InstructionCountCI/FlagM/Secondary_REP_CSSC.json

@@ -12,7 +12,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "popcnt ax, bx": {

unittests/InstructionCountCI/FlagM/VEX_map1.json

@@ -12,7 +12,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vucomiss xmm0, xmm1": {

unittests/InstructionCountCI/FlagM/VEX_map2.json

@@ -10,7 +10,7 @@
       "AFP",
       "SVEBITPERM"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vtestps xmm0, xmm1": {

unittests/InstructionCountCI/FlagM/VEX_map_group.json

@@ -9,7 +9,7 @@
     "DisabledHostFeatures": [
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "blsr eax, ebx": {

unittests/InstructionCountCI/FlagM/x87-Crysis2Max-fmodel.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "Block1": {

unittests/InstructionCountCI/FlagM/x87-HalfLife.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "Block1": {

unittests/InstructionCountCI/FlagM/x87-Oblivion.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "Block1": {

unittests/InstructionCountCI/FlagM/x87-Psychonauts.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "Block1": {

unittests/InstructionCountCI/FlagM/x87.json

@@ -11,7 +11,7 @@
       "CSSC",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "fadd dword [rax]": {

unittests/InstructionCountCI/FlagM/x87_f64-Crysis2Max-fmodel.json

@@ -13,7 +13,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "Block1": {

unittests/InstructionCountCI/FlagM/x87_f64-HalfLife.json

@@ -13,7 +13,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "Block1": {

unittests/InstructionCountCI/FlagM/x87_f64-Oblivion.json

@@ -13,7 +13,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "Block1": {

unittests/InstructionCountCI/FlagM/x87_f64-Psychonauts.json

@@ -13,7 +13,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "Block1": {

unittests/InstructionCountCI/FlagM/x87_f64.json

@@ -13,7 +13,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "fadd dword [rax]": {

unittests/InstructionCountCI/H0F38.json

@@ -10,7 +10,7 @@
       "FLAGM2",
       "CRYPTO"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "pshufb mm0, mm1": {

unittests/InstructionCountCI/H0F3A.json

@@ -8,7 +8,7 @@
       "AFP",
       "CRYPTO"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Comment": [
     "SSE4.2 string instructions are skipped here.",

unittests/InstructionCountCI/H0F3A_SVE128.json

@@ -8,7 +8,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "dpps xmm0, xmm1, 00000000b": {

unittests/InstructionCountCI/MOPS/Primary.json

@@ -11,7 +11,7 @@
       "FLAGM",
       "FLAGM2"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "rep movsb": {

unittests/InstructionCountCI/Primary.json

@@ -10,7 +10,7 @@
       "FLAGM2",
       "MOPS"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "add bl, cl": {

unittests/InstructionCountCI/PrimaryGroup.json

@@ -9,7 +9,7 @@
       "FLAGM",
       "FLAGM2"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Comment": [
     "Instructions in this table that are marked optimal don't have their flag calculation part of this assumption",

unittests/InstructionCountCI/Primary_32Bit.json

@@ -9,7 +9,7 @@
       "FlagM",
       "FlagM2"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "push es": {

unittests/InstructionCountCI/RPRES/DDD.json

@@ -9,7 +9,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "pfrcpv mm0, mm1": {

unittests/InstructionCountCI/RPRES/Secondary.json

@@ -9,7 +9,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "rsqrtps xmm0, xmm1": {

unittests/InstructionCountCI/RPRES/Secondary_REP_AFP.json

@@ -9,7 +9,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "rsqrtss xmm0, xmm1": {

unittests/InstructionCountCI/RPRES/VEX_map1_AFP.json

@@ -8,7 +8,7 @@
       "AFP"
     ],
     "DisabledHostFeatures": [],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "vrsqrtps xmm0, xmm1": {

unittests/InstructionCountCI/Repeat.json

@@ -6,7 +6,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {}
 }

unittests/InstructionCountCI/SSE42_Strings.json

@@ -33,7 +33,7 @@
       "      - 1b: ECX = MSB",
       "[7]   - Reserved"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "pcmpestrm xmm0, xmm1, 0_0_00_00_00b": {

unittests/InstructionCountCI/Secondary.json

@@ -12,7 +12,7 @@
       "FLAGM",
       "FLAGM2"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Comment": [
     "MMX instructions are defined as optimal without SRA being used for these instructions.",

unittests/InstructionCountCI/SecondaryGroup.json

@@ -11,7 +11,7 @@
       "FLAGM",
       "FLAGM2"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "sgdt [rax]": {

unittests/InstructionCountCI/SecondaryModRM.json

@@ -11,7 +11,7 @@
       "FLAGM",
       "FLAGM2"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "xgetbv": {

unittests/InstructionCountCI/Secondary_32Bit.json

@@ -7,7 +7,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "push fs": {

unittests/InstructionCountCI/Secondary_OpSize.json

@@ -12,7 +12,7 @@
       "FLAGM",
       "FLAGM2"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "movupd xmm0, xmm0": {

unittests/InstructionCountCI/Secondary_OpSize_FCMA.json

@@ -9,7 +9,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "addsubpd xmm0, xmm1": {

unittests/InstructionCountCI/Secondary_OpSize_SVE128.json

@@ -8,7 +8,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "psrlw xmm0, xmm1": {

unittests/InstructionCountCI/Secondary_OpSize_SVE256.json

@@ -7,7 +7,7 @@
       "AFP"
     ],
     "DisabledHostFeatures": [],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "pmulhuw xmm0, xmm1": {

unittests/InstructionCountCI/Secondary_REP.json

@@ -12,7 +12,7 @@
       "FRINTTS",
       "CSSC"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "movss xmm0, xmm1": {

unittests/InstructionCountCI/Secondary_REPNE.json

@@ -10,7 +10,7 @@
       "FCMA",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "movsd xmm0, xmm1": {

unittests/InstructionCountCI/Secondary_REPNE_FCMA.json

@@ -9,7 +9,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "addsubps xmm0, xmm1": {

unittests/InstructionCountCI/Secondary_REPNE_SVE128.json

@@ -10,7 +10,7 @@
       "FCMA",
       "AFP"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "cvtpd2dq xmm0, xmm1": {

unittests/InstructionCountCI/Secondary_REP_FRINTTS.json

@@ -12,7 +12,7 @@
       "FLAGM",
       "FLAGM2"
     ],
-    "BinaryCacheVersion": 29
+    "BinaryCacheVersion": 30
   },
   "Instructions": {
     "cvttss2si eax, xmm0": {