Merge pull request #4049 from alyssarosenzweig/opt/constprop-merges

ConstProp: speed it up

CodeEmitter/CodeEmitter/ALUOps.inl

@@ -301,8 +301,9 @@ public:
     xbfiz_helper(true, s, rd, rn, lsb, width);
   }
   void asr(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t shift) {
-    LOGMAN_THROW_A_FMT(shift <= RegSizeInBits(s), "Tried to asr a region larger than the register");
-    sbfm(s, rd, rn, shift, RegSizeInBits(s) - 1);
+    const auto RegSize_m1 = RegSizeInBits(s) - 1;
+    shift &= RegSize_m1;
+    sbfm(s, rd, rn, shift, RegSize_m1);
   }
 
   void uxtb(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn) {
@@ -325,14 +326,14 @@ public:
   }
 
   void lsl(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t shift) {
-    const auto RegSize = RegSizeInBits(s);
-    LOGMAN_THROW_A_FMT(shift < RegSize, "Tried to lsl a region larger than the register");
-    ubfm(s, rd, rn, (RegSize - shift) % RegSize, RegSize - shift - 1);
+    const auto RegSize_m1 = RegSizeInBits(s) - 1;
+    shift &= RegSize_m1;
+    ubfm(s, rd, rn, (RegSizeInBits(s) - shift) & RegSize_m1, RegSize_m1 - shift);
   }
   void lsr(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t shift) {
-    const auto RegSize = RegSizeInBits(s);
-    LOGMAN_THROW_A_FMT(shift < RegSize, "Tried to lsr a region larger than the register");
-    ubfm(s, rd, rn, shift, RegSize - 1);
+    const auto RegSize_m1 = RegSizeInBits(s) - 1;
+    shift &= RegSize_m1;
+    ubfm(s, rd, rn, shift, RegSize_m1);
   }
   void ubfx(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t lsb, uint32_t width) {
     LOGMAN_THROW_A_FMT(width > 0, "ubfx needs width > 0");
@@ -368,6 +369,7 @@ public:
   }
 
   void ror(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t Imm) {
+    Imm &= RegSizeInBits(s) - 1;
     extr(s, rd, rn, rn, Imm);
   }
 

FEXCore/Source/Interface/IR/Passes/ConstProp.cpp

@@ -74,13 +74,42 @@ public:
 private:
   void HandleConstantPools(IREmitter* IREmit, const IRListView& CurrentIR);
   void ConstantPropagation(IREmitter* IREmit, const IRListView& CurrentIR, Ref CodeNode, IROp_Header* IROp);
-  void ConstantInlining(IREmitter* IREmit, const IRListView& CurrentIR);
 
   fextl::unordered_map<uint64_t, Ref> ConstPool;
 
   bool SupportsTSOImm9 {};
   const FEXCore::CPUIDEmu* CPUID;
 
+  template<class F>
+  bool InlineIf(IREmitter* IREmit, const IRListView& CurrentIR, Ref CodeNode, IROp_Header* IROp, unsigned Index, F Filter) {
+    uint64_t Constant;
+    if (!IREmit->IsValueConstant(IROp->Args[Index], &Constant) || !Filter(Constant)) {
+      return false;
+    }
+
+    IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[Index]));
+    IREmit->ReplaceNodeArgument(CodeNode, Index, IREmit->_InlineConstant(Constant));
+    return true;
+  }
+
+  bool Inline(IREmitter* IREmit, const IRListView& CurrentIR, Ref CodeNode, IROp_Header* IROp, unsigned Index) {
+    return InlineIf(IREmit, CurrentIR, CodeNode, IROp, Index, [](uint64_t _) { return true; });
+  }
+
+  bool InlineIfZero(IREmitter* IREmit, const IRListView& CurrentIR, Ref CodeNode, IROp_Header* IROp, unsigned Index) {
+    return InlineIf(IREmit, CurrentIR, CodeNode, IROp, Index, [](uint64_t X) { return X == 0; });
+  }
+
+  bool InlineIfLargeAddSub(IREmitter* IREmit, const IRListView& CurrentIR, Ref CodeNode, IROp_Header* IROp, unsigned Index) {
+    // We don't allow 8/16-bit operations to have constants, since no
+    // constant would be in bounds after the JIT's 24/16 shift.
+    auto Filter = [&IROp](uint64_t X) {
+      return ARMEmitter::IsImmAddSub(X) && IROp->Size >= 4;
+    };
+
+    return InlineIf(IREmit, CurrentIR, CodeNode, IROp, Index, Filter);
+  }
+
   void InlineMemImmediate(IREmitter* IREmit, const IRListView& IR, Ref CodeNode, IROp_Header* IROp, OrderedNodeWrapper Offset,
                           MemOffsetType OffsetType, const size_t Offset_Index, uint8_t& OffsetScale, bool TSO) {
     uint64_t Imm {};
@@ -109,6 +138,9 @@ private:
 
 // Constants are pooled per block.
 void ConstProp::HandleConstantPools(IREmitter* IREmit, const IRListView& CurrentIR) {
+  const uint32_t SSACount = CurrentIR.GetSSACount();
+  fextl::vector<Ref> Remap(SSACount, NULL);
+
   for (auto [BlockNode, BlockIROp] : CurrentIR.GetBlocks()) {
     for (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {
       if (IROp->Op == OP_CONSTANT) {
@@ -116,11 +148,28 @@ void ConstProp::HandleConstantPools(IREmitter* IREmit, const IRListView& Current
         auto it = ConstPool.find(Op->Constant);
 
         if (it != ConstPool.end()) {
-          auto CodeIter = CurrentIR.at(CodeNode);
-          IREmit->ReplaceUsesWithAfter(CodeNode, it->second, CodeIter);
+          uint32_t Value = CurrentIR.GetID(CodeNode).Value;
+          LOGMAN_THROW_A_FMT(Value < SSACount, "def not yet remapped");
+
+          Remap[Value] = it->second;
         } else {
           ConstPool[Op->Constant] = CodeNode;
         }
+      } else {
+        const uint8_t NumArgs = IR::GetArgs(IROp->Op);
+        for (uint8_t i = 0; i < NumArgs; ++i) {
+          if (IROp->Args[i].IsInvalid()) {
+            continue;
+          }
+
+          uint32_t Value = IROp->Args[i].ID().Value;
+          LOGMAN_THROW_A_FMT(Value < SSACount, "src not yet remapped");
+
+          Ref New = Value < SSACount ? Remap[Value] : NULL;
+          if (New) {
+            IREmit->ReplaceNodeArgument(CodeNode, i, New);
+          }
+        }
       }
     }
     ConstPool.clear();
@@ -151,10 +200,14 @@ void ConstProp::ConstantPropagation(IREmitter* IREmit, const IRListView& Current
     if (IsConstant1 && IsConstant2 && IROp->Op == OP_ADD) {
       uint64_t NewConstant = (Constant1 + Constant2) & getMask(IROp);
       IREmit->ReplaceWithConstant(CodeNode, NewConstant);
+      break;
     } else if (IsConstant1 && IsConstant2 && IROp->Op == OP_SUB) {
       uint64_t NewConstant = (Constant1 - Constant2) & getMask(IROp);
       IREmit->ReplaceWithConstant(CodeNode, NewConstant);
-    } else if (IsConstant2 && !ARMEmitter::IsImmAddSub(Constant2) && ARMEmitter::IsImmAddSub(-Constant2)) {
+      break;
+    }
+
+    if (IsConstant2 && !ARMEmitter::IsImmAddSub(Constant2) && ARMEmitter::IsImmAddSub(-Constant2)) {
       // If the second argument is constant, the immediate is not ImmAddSub, but when negated is.
       // So, negate the operation to negate (and inline) the constant.
       if (IROp->Op == OP_ADD) {
@@ -175,6 +228,23 @@ void ConstProp::ConstantPropagation(IREmitter* IREmit, const IRListView& Current
       // Replace the second source with the negated constant.
       IREmit->ReplaceNodeArgument(CodeNode, Op->Src2_Index, NegConstant);
     }
+
+    if (!InlineIfLargeAddSub(IREmit, CurrentIR, CodeNode, IROp, 1) && (IROp->Op == OP_SUB || IROp->Op == OP_SUBWITHFLAGS)) {
+      // TODO: Generalize this
+      InlineIfZero(IREmit, CurrentIR, CodeNode, IROp, 0);
+    }
+
+    break;
+  }
+  case OP_ADDNZCV: {
+    InlineIfLargeAddSub(IREmit, CurrentIR, CodeNode, IROp, 1);
+    break;
+  }
+  case OP_SUBNZCV: {
+    if (!InlineIfLargeAddSub(IREmit, CurrentIR, CodeNode, IROp, 1)) {
+      // TODO: Generalize this
+      InlineIfZero(IREmit, CurrentIR, CodeNode, IROp, 0);
+    }
     break;
   }
   case OP_SUBSHIFT: {
@@ -194,10 +264,13 @@ void ConstProp::ConstantPropagation(IREmitter* IREmit, const IRListView& Current
     uint64_t Constant1 {};
     uint64_t Constant2 {};
 
+    bool Replaced = false;
+
     // Order matter for short circuit evaluation, subsequent ifs read constant2.
     if (IREmit->IsValueConstant(IROp->Args[1], &Constant2) && IREmit->IsValueConstant(IROp->Args[0], &Constant1)) {
       uint64_t NewConstant = (Constant1 & Constant2) & getMask(IROp);
       IREmit->ReplaceWithConstant(CodeNode, NewConstant);
+      Replaced = true;
     } else if (Constant2 == 1) {
       // happens from flag calcs
       auto val = IREmit->GetOpHeader(IROp->Args[0]);
@@ -206,10 +279,16 @@ void ConstProp::ConstantPropagation(IREmitter* IREmit, const IRListView& Current
       if (val->Op == OP_SELECT && IREmit->IsValueConstant(val->Args[2], &Constant2) && IREmit->IsValueConstant(val->Args[3], &Constant3) &&
           Constant2 == 1 && Constant3 == 0) {
         IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(IROp->Args[0]));
+        Replaced = true;
       }
     } else if (IROp->Args[0].ID() == IROp->Args[1].ID() || (Constant2 & getMask(IROp)) == getMask(IROp)) {
       // AND with same value results in original value
       IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(IROp->Args[0]));
+      Replaced = true;
+    }
+
+    if (!Replaced) {
+      InlineIf(IREmit, CurrentIR, CodeNode, IROp, 1, [&IROp](uint64_t X) { return IsImmLogical(X, IROp->Size * 8); });
     }
     break;
   }
@@ -223,6 +302,8 @@ void ConstProp::ConstantPropagation(IREmitter* IREmit, const IRListView& Current
     } else if (IROp->Args[0].ID() == IROp->Args[1].ID()) {
       // OR with same value results in original value
       IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(IROp->Args[0]));
+    } else {
+      InlineIf(IREmit, CurrentIR, CodeNode, IROp, 1, [&IROp](uint64_t X) { return IsImmLogical(X, IROp->Size * 8); });
     }
     break;
   }
@@ -239,6 +320,7 @@ void ConstProp::ConstantPropagation(IREmitter* IREmit, const IRListView& Current
       IREmit->ReplaceAllUsesWith(CodeNode, IREmit->_Constant(0));
     } else {
       // XOR with zero results in the nonzero source
+      bool Replaced = false;
       for (unsigned i = 0; i < 2; ++i) {
         if (!IREmit->IsValueConstant(IROp->Args[i], &Constant1)) {
           continue;
@@ -251,11 +333,22 @@ void ConstProp::ConstantPropagation(IREmitter* IREmit, const IRListView& Current
         IREmit->SetWriteCursor(CodeNode);
         Ref Arg = CurrentIR.GetNode(IROp->Args[1 - i]);
         IREmit->ReplaceAllUsesWith(CodeNode, Arg);
+        Replaced = true;
         break;
       }
+
+      if (!Replaced) {
+        InlineIf(IREmit, CurrentIR, CodeNode, IROp, 1, [&IROp](uint64_t X) { return IsImmLogical(X, IROp->Size * 8); });
+      }
     }
     break;
   }
+  case OP_ANDWITHFLAGS:
+  case OP_ANDN:
+  case OP_TESTNZ: {
+    InlineIf(IREmit, CurrentIR, CodeNode, IROp, 1, [&IROp](uint64_t X) { return IsImmLogical(X, IROp->Size * 8); });
+    break;
+  }
   case OP_NEG: {
     uint64_t Constant {};
 
@@ -265,6 +358,11 @@ void ConstProp::ConstantPropagation(IREmitter* IREmit, const IRListView& Current
     }
     break;
   }
+  case OP_ASHR:
+  case OP_ROR: {
+    Inline(IREmit, CurrentIR, CodeNode, IROp, 1);
+    break;
+  }
   case OP_LSHL: {
     uint64_t Constant1 {};
     uint64_t Constant2 {};
@@ -278,6 +376,8 @@ void ConstProp::ConstantPropagation(IREmitter* IREmit, const IRListView& Current
       IREmit->SetWriteCursor(CodeNode);
       Ref Arg = CurrentIR.GetNode(IROp->Args[0]);
       IREmit->ReplaceAllUsesWith(CodeNode, Arg);
+    } else {
+      Inline(IREmit, CurrentIR, CodeNode, IROp, 1);
     }
     break;
   }
@@ -298,6 +398,8 @@ void ConstProp::ConstantPropagation(IREmitter* IREmit, const IRListView& Current
       IREmit->SetWriteCursor(CodeNode);
       Ref Arg = CurrentIR.GetNode(IROp->Args[0]);
       IREmit->ReplaceAllUsesWith(CodeNode, Arg);
+    } else {
+      Inline(IREmit, CurrentIR, CodeNode, IROp, 1);
     }
     break;
   }
@@ -561,244 +663,101 @@ void ConstProp::ConstantPropagation(IREmitter* IREmit, const IRListView& Current
     break;
   }
 
-  default: break;
+  case OP_ADC:
+  case OP_ADCWITHFLAGS:
+  case OP_STORECONTEXT:
+  case OP_RMIFNZCV: {
+    InlineIfZero(IREmit, CurrentIR, CodeNode, IROp, 0);
+    break;
   }
-}
+  case OP_CONDADDNZCV:
+  case OP_CONDSUBNZCV: {
+    InlineIfZero(IREmit, CurrentIR, CodeNode, IROp, 0);
+    InlineIf(IREmit, CurrentIR, CodeNode, IROp, 1, ARMEmitter::IsImmAddSub);
+    break;
+  }
+  case OP_SELECT: {
+    InlineIf(IREmit, CurrentIR, CodeNode, IROp, 1, ARMEmitter::IsImmAddSub);
 
-void ConstProp::ConstantInlining(IREmitter* IREmit, const IRListView& CurrentIR) {
-  for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
-    switch (IROp->Op) {
-    case OP_LSHR:
-    case OP_ASHR:
-    case OP_ROR:
-    case OP_LSHL: {
-      uint64_t Constant2 {};
-      if (IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
-        IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
+    uint64_t AllOnes = IROp->Size == 8 ? 0xffff'ffff'ffff'ffffull : 0xffff'ffffull;
 
-        // this shouldn't be here, but rather on the emitter themselves or the constprop transformation?
-        if (IROp->Size <= 4) {
-          Constant2 &= 31;
-        } else {
-          Constant2 &= 63;
-        }
+    uint64_t Constant2 {};
+    uint64_t Constant3 {};
+    if (IREmit->IsValueConstant(IROp->Args[2], &Constant2) && IREmit->IsValueConstant(IROp->Args[3], &Constant3) &&
+        (Constant2 == 1 || Constant2 == AllOnes) && Constant3 == 0) {
+      IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[2]));
 
-        IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
-      }
-      break;
+      IREmit->ReplaceNodeArgument(CodeNode, 2, IREmit->_InlineConstant(Constant2));
+      IREmit->ReplaceNodeArgument(CodeNode, 3, IREmit->_InlineConstant(Constant3));
     }
-    case OP_ADD:
-    case OP_SUB:
-    case OP_ADDNZCV:
-    case OP_SUBNZCV:
-    case OP_ADDWITHFLAGS:
-    case OP_SUBWITHFLAGS: {
-      uint64_t Constant2 {};
-      if (IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
-        // We don't allow 8/16-bit operations to have constants, since no
-        // constant would be in bounds after the JIT's 24/16 shift.
-        if (ARMEmitter::IsImmAddSub(Constant2) && IROp->Size >= 4) {
-          IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
-          IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
-        }
-      } else if (IROp->Op == OP_SUBNZCV || IROp->Op == OP_SUBWITHFLAGS || IROp->Op == OP_SUB) {
-        // TODO: Generalize this
-        uint64_t Constant1 {};
-        if (IREmit->IsValueConstant(IROp->Args[0], &Constant1)) {
-          if (Constant1 == 0) {
-            IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[0]));
-            IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineConstant(0));
-          }
-        }
-      }
-
-      break;
-    }
-    case OP_ADC:
-    case OP_ADCWITHFLAGS:
-    case OP_STORECONTEXT: {
-      uint64_t Constant1 {};
-      if (IREmit->IsValueConstant(IROp->Args[0], &Constant1)) {
-        if (Constant1 == 0) {
-          IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[0]));
-          IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineConstant(0));
-        }
-      }
-
-      break;
-    }
-    case OP_RMIFNZCV: {
-      uint64_t Constant1 {};
-      if (IREmit->IsValueConstant(IROp->Args[0], &Constant1)) {
-        if (Constant1 == 0) {
-          IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[0]));
-          IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineConstant(0));
-        }
-      }
-
-      break;
-    }
-    case OP_CONDADDNZCV:
-    case OP_CONDSUBNZCV: {
-      uint64_t Constant2 {};
-      if (IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
-        if (ARMEmitter::IsImmAddSub(Constant2)) {
-          IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
-          IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
-        }
-      }
-
-      uint64_t Constant1 {};
-      if (IREmit->IsValueConstant(IROp->Args[0], &Constant1)) {
-        if (Constant1 == 0) {
-          IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[0]));
-          IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineConstant(0));
-        }
-      }
-      break;
-    }
-    case OP_TESTNZ: {
-      uint64_t Constant1 {};
-      if (IREmit->IsValueConstant(IROp->Args[1], &Constant1)) {
-        if (IsImmLogical(Constant1, IROp->Size * 8)) {
-          IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
-          IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant1));
-        }
-      }
-      break;
-    }
-    case OP_SELECT: {
-      uint64_t Constant1 {};
-      if (IREmit->IsValueConstant(IROp->Args[1], &Constant1)) {
-        if (ARMEmitter::IsImmAddSub(Constant1)) {
-          IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
-          IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant1));
-        }
-      }
 
+    break;
+  }
+  case OP_NZCVSELECT: {
+    // We always allow source 1 to be zero, but source 0 can only be a
+    // special 1/~0 constant if source 1 is 0.
+    if (InlineIfZero(IREmit, CurrentIR, CodeNode, IROp, 1)) {
       uint64_t AllOnes = IROp->Size == 8 ? 0xffff'ffff'ffff'ffffull : 0xffff'ffffull;
-
-      uint64_t Constant2 {};
-      uint64_t Constant3 {};
-      if (IREmit->IsValueConstant(IROp->Args[2], &Constant2) && IREmit->IsValueConstant(IROp->Args[3], &Constant3) &&
-          (Constant2 == 1 || Constant2 == AllOnes) && Constant3 == 0) {
-        IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[2]));
-
-        IREmit->ReplaceNodeArgument(CodeNode, 2, IREmit->_InlineConstant(Constant2));
-        IREmit->ReplaceNodeArgument(CodeNode, 3, IREmit->_InlineConstant(Constant3));
-      }
-
-      break;
+      InlineIf(IREmit, CurrentIR, CodeNode, IROp, 0, [&AllOnes](uint64_t X) { return X == 1 || X == AllOnes; });
     }
-    case OP_NZCVSELECT: {
-      uint64_t AllOnes = IROp->Size == 8 ? 0xffff'ffff'ffff'ffffull : 0xffff'ffffull;
+    break;
+  }
+  case OP_CONDJUMP: {
+    InlineIf(IREmit, CurrentIR, CodeNode, IROp, 1, ARMEmitter::IsImmAddSub);
+    break;
+  }
+  case OP_EXITFUNCTION: {
+    auto Op = IROp->C<IR::IROp_ExitFunction>();
 
-      // We always allow source 1 to be zero, but source 0 can only be a
-      // special 1/~0 constant if source 1 is 0.
-      uint64_t Constant0 {};
-      uint64_t Constant1 {};
-      if (IREmit->IsValueConstant(IROp->Args[1], &Constant1) && Constant1 == 0) {
-        IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
-        IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant1));
-
-        if (IREmit->IsValueConstant(IROp->Args[0], &Constant0) && (Constant0 == 1 || Constant0 == AllOnes)) {
-          IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[0]));
-          IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineConstant(Constant0));
-        }
-      }
-
-      break;
-    }
-    case OP_CONDJUMP: {
-      uint64_t Constant2 {};
-      if (IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
-        if (ARMEmitter::IsImmAddSub(Constant2)) {
-          IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
-          IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
-        }
-      }
-      break;
-    }
-    case OP_EXITFUNCTION: {
-      auto Op = IROp->C<IR::IROp_ExitFunction>();
-
-      uint64_t Constant {};
-      if (IREmit->IsValueConstant(Op->NewRIP, &Constant)) {
+    if (!Inline(IREmit, CurrentIR, CodeNode, IROp, Op->NewRIP_Index)) {
+      auto NewRIP = IREmit->GetOpHeader(Op->NewRIP);
+      if (NewRIP->Op == OP_ENTRYPOINTOFFSET) {
+        auto EO = NewRIP->C<IR::IROp_EntrypointOffset>();
         IREmit->SetWriteCursor(CurrentIR.GetNode(Op->NewRIP));
-        IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineConstant(Constant));
-      } else {
-        auto NewRIP = IREmit->GetOpHeader(Op->NewRIP);
-        if (NewRIP->Op == OP_ENTRYPOINTOFFSET) {
-          auto EO = NewRIP->C<IR::IROp_EntrypointOffset>();
-          IREmit->SetWriteCursor(CurrentIR.GetNode(Op->NewRIP));
 
-          IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineEntrypointOffset(IR::SizeToOpSize(EO->Header.Size), EO->Offset));
-        }
+        IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineEntrypointOffset(IR::SizeToOpSize(EO->Header.Size), EO->Offset));
       }
-      break;
     }
-    case OP_OR:
-    case OP_XOR:
-    case OP_AND:
-    case OP_ANDWITHFLAGS:
-    case OP_ANDN: {
-      uint64_t Constant2 {};
-      if (IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
-        if (IsImmLogical(Constant2, IROp->Size * 8)) {
-          IREmit->SetWriteCursor(CurrentIR.GetNode(IROp->Args[1]));
-          IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
-        }
-      }
-      break;
-    }
-    case OP_LOADMEM: {
-      auto Op = IROp->CW<IR::IROp_LoadMem>();
-      InlineMemImmediate(IREmit, CurrentIR, CodeNode, IROp, Op->Offset, Op->OffsetType, Op->Offset_Index, Op->OffsetScale, false);
-      break;
-    }
-    case OP_STOREMEM: {
-      auto Op = IROp->CW<IR::IROp_StoreMem>();
-      InlineMemImmediate(IREmit, CurrentIR, CodeNode, IROp, Op->Offset, Op->OffsetType, Op->Offset_Index, Op->OffsetScale, false);
-      break;
-    }
-    case OP_PREFETCH: {
-      auto Op = IROp->CW<IR::IROp_Prefetch>();
-      InlineMemImmediate(IREmit, CurrentIR, CodeNode, IROp, Op->Offset, Op->OffsetType, Op->Offset_Index, Op->OffsetScale, false);
-      break;
-    }
-    case OP_LOADMEMTSO: {
-      auto Op = IROp->CW<IR::IROp_LoadMemTSO>();
-      InlineMemImmediate(IREmit, CurrentIR, CodeNode, IROp, Op->Offset, Op->OffsetType, Op->Offset_Index, Op->OffsetScale, true);
-      break;
-    }
-    case OP_STOREMEMTSO: {
-      auto Op = IROp->CW<IR::IROp_StoreMemTSO>();
-      InlineMemImmediate(IREmit, CurrentIR, CodeNode, IROp, Op->Offset, Op->OffsetType, Op->Offset_Index, Op->OffsetScale, true);
-      break;
-    }
-    case OP_MEMCPY: {
-      auto Op = IROp->CW<IR::IROp_MemCpy>();
+    break;
+  }
 
-      uint64_t Constant {};
-      if (IREmit->IsValueConstant(Op->Direction, &Constant)) {
-        IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Direction));
-        IREmit->ReplaceNodeArgument(CodeNode, Op->Direction_Index, IREmit->_InlineConstant(Constant));
-      }
-      break;
-    }
-    case OP_MEMSET: {
-      auto Op = IROp->CW<IR::IROp_MemSet>();
+  case OP_LOADMEM: {
+    auto Op = IROp->CW<IR::IROp_LoadMem>();
+    InlineMemImmediate(IREmit, CurrentIR, CodeNode, IROp, Op->Offset, Op->OffsetType, Op->Offset_Index, Op->OffsetScale, false);
+    break;
+  }
+  case OP_STOREMEM: {
+    auto Op = IROp->CW<IR::IROp_StoreMem>();
+    InlineMemImmediate(IREmit, CurrentIR, CodeNode, IROp, Op->Offset, Op->OffsetType, Op->Offset_Index, Op->OffsetScale, false);
+    break;
+  }
+  case OP_PREFETCH: {
+    auto Op = IROp->CW<IR::IROp_Prefetch>();
+    InlineMemImmediate(IREmit, CurrentIR, CodeNode, IROp, Op->Offset, Op->OffsetType, Op->Offset_Index, Op->OffsetScale, false);
+    break;
+  }
+  case OP_LOADMEMTSO: {
+    auto Op = IROp->CW<IR::IROp_LoadMemTSO>();
+    InlineMemImmediate(IREmit, CurrentIR, CodeNode, IROp, Op->Offset, Op->OffsetType, Op->Offset_Index, Op->OffsetScale, true);
+    break;
+  }
+  case OP_STOREMEMTSO: {
+    auto Op = IROp->CW<IR::IROp_StoreMemTSO>();
+    InlineMemImmediate(IREmit, CurrentIR, CodeNode, IROp, Op->Offset, Op->OffsetType, Op->Offset_Index, Op->OffsetScale, true);
+    break;
+  }
+  case OP_MEMCPY: {
+    auto Op = IROp->CW<IR::IROp_MemCpy>();
+    Inline(IREmit, CurrentIR, CodeNode, IROp, Op->Direction_Index);
+    break;
+  }
+  case OP_MEMSET: {
+    auto Op = IROp->CW<IR::IROp_MemSet>();
+    Inline(IREmit, CurrentIR, CodeNode, IROp, Op->Direction_Index);
+    break;
+  }
 
-      uint64_t Constant {};
-      if (IREmit->IsValueConstant(Op->Direction, &Constant)) {
-        IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Direction));
-        IREmit->ReplaceNodeArgument(CodeNode, Op->Direction_Index, IREmit->_InlineConstant(Constant));
-      }
-      break;
-    }
-
-    default: break;
-    }
+  default: break;
   }
 }
 
@@ -807,13 +766,11 @@ void ConstProp::Run(IREmitter* IREmit) {
 
   auto CurrentIR = IREmit->ViewIR();
 
-  HandleConstantPools(IREmit, CurrentIR);
-
   for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
     ConstantPropagation(IREmit, CurrentIR, CodeNode, IROp);
   }
 
-  ConstantInlining(IREmit, CurrentIR);
+  HandleConstantPools(IREmit, IREmit->ViewIR());
 }
 
 fextl::unique_ptr<FEXCore::IR::Pass> CreateConstProp(bool SupportsTSOImm9, const FEXCore::CPUIDEmu* CPUID) {

unittests/InstructionCountCI/Atomics.json

@@ -687,13 +687,12 @@
       ]
     },
     "lock adc byte [rax], 1": {
-      "ExpectedInstructionCount": 18,
+      "ExpectedInstructionCount": 17,
       "Comment": "GROUP1 0x80 /2",
       "ExpectedArm64ASM": [
         "mov w20, #0x1",
-        "cinc w20, w20, lo",
-        "ldaddalb w20, w27, [x4]",
-        "mov w20, #0x1",
+        "cinc w21, w20, lo",
+        "ldaddalb w21, w27, [x4]",
         "cinc w20, w20, lo",
         "add w21, w27, w20",
         "uxtb w26, w21",
@@ -711,38 +710,36 @@
       ]
     },
     "lock adc byte [rax], 0xFF": {
-      "ExpectedInstructionCount": 19,
+      "ExpectedInstructionCount": 18,
       "Comment": "GROUP1 0x80 /2",
       "ExpectedArm64ASM": [
         "mov w20, #0xff",
+        "cinc w21, w20, lo",
+        "ldaddalb w21, w21, [x4]",
+        "mvn w27, w21",
         "cinc w20, w20, lo",
-        "ldaddalb w20, w20, [x4]",
-        "mvn w27, w20",
-        "mov w21, #0xff",
-        "cinc w21, w21, lo",
-        "add w22, w20, w21",
+        "add w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w26, w21",
-        "cset x21, hs",
+        "cmp w26, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
         "mrs x22, nzcv",
-        "orr w21, w22, w21, lsl #29",
-        "eor w22, w20, #0xff",
-        "eor w20, w26, w20",
-        "bic w20, w20, w22",
-        "ubfx x20, x20, #7, #1",
-        "orr w20, w21, w20, lsl #28",
+        "orr w20, w22, w20, lsl #29",
+        "eor w22, w21, #0xff",
+        "eor w21, w26, w21",
+        "bic w21, w21, w22",
+        "ubfx x21, x21, #7, #1",
+        "orr w20, w20, w21, lsl #28",
         "msr nzcv, x20"
       ]
     },
     "lock adc word [rax], 0x100": {
-      "ExpectedInstructionCount": 18,
+      "ExpectedInstructionCount": 17,
       "Comment": "GROUP1 0x81 /2",
       "ExpectedArm64ASM": [
         "mov w20, #0x100",
-        "cinc w20, w20, lo",
-        "ldaddalh w20, w27, [x4]",
-        "mov w20, #0x100",
+        "cinc w21, w20, lo",
+        "ldaddalh w21, w27, [x4]",
         "cinc w20, w20, lo",
         "add w21, w27, w20",
         "uxth w26, w21",
@@ -760,27 +757,26 @@
       ]
     },
     "lock adc word [rax], 0xFFFF": {
-      "ExpectedInstructionCount": 19,
+      "ExpectedInstructionCount": 18,
       "Comment": "GROUP1 0x81 /2",
       "ExpectedArm64ASM": [
         "mov w20, #0xffff",
+        "cinc w21, w20, lo",
+        "ldaddalh w21, w21, [x4]",
+        "mvn w27, w21",
         "cinc w20, w20, lo",
-        "ldaddalh w20, w20, [x4]",
-        "mvn w27, w20",
-        "mov w21, #0xffff",
-        "cinc w21, w21, lo",
-        "add w22, w20, w21",
+        "add w22, w21, w20",
         "uxth w26, w22",
-        "cmp w26, w21",
-        "cset x21, hs",
+        "cmp w26, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #16",
         "mrs x22, nzcv",
-        "orr w21, w22, w21, lsl #29",
-        "eor w22, w20, #0xffff",
-        "eor w20, w26, w20",
-        "bic w20, w20, w22",
-        "ubfx x20, x20, #15, #1",
-        "orr w20, w21, w20, lsl #28",
+        "orr w20, w22, w20, lsl #29",
+        "eor w22, w21, #0xffff",
+        "eor w21, w26, w21",
+        "bic w21, w21, w22",
+        "ubfx x21, x21, #15, #1",
+        "orr w20, w20, w21, lsl #28",
         "msr nzcv, x20"
       ]
     },
@@ -850,13 +846,12 @@
       ]
     },
     "lock adc word [rax], 1": {
-      "ExpectedInstructionCount": 18,
+      "ExpectedInstructionCount": 17,
       "Comment": "GROUP1 0x83 /2",
       "ExpectedArm64ASM": [
         "mov w20, #0x1",
-        "cinc w20, w20, lo",
-        "ldaddalh w20, w27, [x4]",
-        "mov w20, #0x1",
+        "cinc w21, w20, lo",
+        "ldaddalh w21, w27, [x4]",
         "cinc w20, w20, lo",
         "add w21, w27, w20",
         "uxth w26, w21",
@@ -906,108 +901,104 @@
       ]
     },
     "lock sbb byte [rax], 1": {
-      "ExpectedInstructionCount": 20,
+      "ExpectedInstructionCount": 19,
       "Comment": "GROUP1 0x80 /3",
       "ExpectedArm64ASM": [
         "mov w20, #0x1",
-        "cinc w20, w20, lo",
-        "neg w1, w20",
+        "cinc w21, w20, lo",
+        "neg w1, w21",
         "ldaddalb w1, w27, [x4]",
-        "uxtb w20, w27",
-        "mov w21, #0x1",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "uxtb w21, w27",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
         "mrs x22, nzcv",
-        "orr w21, w22, w21, lsl #29",
-        "eor w22, w20, #0x1",
-        "eor w20, w26, w20",
-        "and w20, w20, w22",
-        "ubfx x20, x20, #7, #1",
-        "orr w20, w21, w20, lsl #28",
+        "orr w20, w22, w20, lsl #29",
+        "eor w22, w21, #0x1",
+        "eor w21, w26, w21",
+        "and w21, w21, w22",
+        "ubfx x21, x21, #7, #1",
+        "orr w20, w20, w21, lsl #28",
         "msr nzcv, x20"
       ]
     },
     "lock sbb byte [rax], 0xFF": {
-      "ExpectedInstructionCount": 21,
+      "ExpectedInstructionCount": 20,
       "Comment": "GROUP1 0x80 /3",
       "ExpectedArm64ASM": [
         "mov w20, #0xff",
+        "cinc w21, w20, lo",
+        "neg w1, w21",
+        "ldaddalb w1, w21, [x4]",
+        "mvn w27, w21",
+        "uxtb w21, w21",
         "cinc w20, w20, lo",
-        "neg w1, w20",
-        "ldaddalb w1, w20, [x4]",
-        "mvn w27, w20",
-        "uxtb w20, w20",
-        "mov w21, #0xff",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "sub w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
         "mrs x22, nzcv",
-        "orr w21, w22, w21, lsl #29",
-        "eor w22, w20, #0xff",
-        "eor w20, w26, w20",
-        "and w20, w20, w22",
-        "ubfx x20, x20, #7, #1",
-        "orr w20, w21, w20, lsl #28",
+        "orr w20, w22, w20, lsl #29",
+        "eor w22, w21, #0xff",
+        "eor w21, w26, w21",
+        "and w21, w21, w22",
+        "ubfx x21, x21, #7, #1",
+        "orr w20, w20, w21, lsl #28",
         "msr nzcv, x20"
       ]
     },
     "lock sbb word [rax], 0x100": {
-      "ExpectedInstructionCount": 20,
+      "ExpectedInstructionCount": 19,
       "Comment": "GROUP1 0x81 /3",
       "ExpectedArm64ASM": [
         "mov w20, #0x100",
-        "cinc w20, w20, lo",
-        "neg w1, w20",
+        "cinc w21, w20, lo",
+        "neg w1, w21",
         "ldaddalh w1, w27, [x4]",
-        "uxth w20, w27",
-        "mov w21, #0x100",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "uxth w21, w27",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxth w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #16",
         "mrs x22, nzcv",
-        "orr w21, w22, w21, lsl #29",
-        "eor w22, w20, #0x100",
-        "eor w20, w26, w20",
-        "and w20, w20, w22",
-        "ubfx x20, x20, #15, #1",
-        "orr w20, w21, w20, lsl #28",
+        "orr w20, w22, w20, lsl #29",
+        "eor w22, w21, #0x100",
+        "eor w21, w26, w21",
+        "and w21, w21, w22",
+        "ubfx x21, x21, #15, #1",
+        "orr w20, w20, w21, lsl #28",
         "msr nzcv, x20"
       ]
     },
     "lock sbb word [rax], 0xFFFF": {
-      "ExpectedInstructionCount": 21,
+      "ExpectedInstructionCount": 20,
       "Comment": "GROUP1 0x81 /3",
       "ExpectedArm64ASM": [
         "mov w20, #0xffff",
+        "cinc w21, w20, lo",
+        "neg w1, w21",
+        "ldaddalh w1, w21, [x4]",
+        "mvn w27, w21",
+        "uxth w21, w21",
         "cinc w20, w20, lo",
-        "neg w1, w20",
-        "ldaddalh w1, w20, [x4]",
-        "mvn w27, w20",
-        "uxth w20, w20",
-        "mov w21, #0xffff",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "sub w22, w21, w20",
         "uxth w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #16",
         "mrs x22, nzcv",
-        "orr w21, w22, w21, lsl #29",
-        "eor w22, w20, #0xffff",
-        "eor w20, w26, w20",
-        "and w20, w20, w22",
-        "ubfx x20, x20, #15, #1",
-        "orr w20, w21, w20, lsl #28",
+        "orr w20, w22, w20, lsl #29",
+        "eor w22, w21, #0xffff",
+        "eor w21, w26, w21",
+        "and w21, w21, w22",
+        "ubfx x21, x21, #15, #1",
+        "orr w20, w20, w21, lsl #28",
         "msr nzcv, x20"
       ]
     },
@@ -1057,28 +1048,27 @@
       ]
     },
     "lock sbb word [rax], 1": {
-      "ExpectedInstructionCount": 20,
+      "ExpectedInstructionCount": 19,
       "Comment": "GROUP1 0x83 /3",
       "ExpectedArm64ASM": [
         "mov w20, #0x1",
-        "cinc w20, w20, lo",
-        "neg w1, w20",
+        "cinc w21, w20, lo",
+        "neg w1, w21",
         "ldaddalh w1, w27, [x4]",
-        "uxth w20, w27",
-        "mov w21, #0x1",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "uxth w21, w27",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxth w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #16",
         "mrs x22, nzcv",
-        "orr w21, w22, w21, lsl #29",
-        "eor w22, w20, #0x1",
-        "eor w20, w26, w20",
-        "and w20, w20, w22",
-        "ubfx x20, x20, #15, #1",
-        "orr w20, w21, w20, lsl #28",
+        "orr w20, w22, w20, lsl #29",
+        "eor w22, w21, #0x1",
+        "eor w21, w26, w21",
+        "and w21, w21, w22",
+        "ubfx x21, x21, #15, #1",
+        "orr w20, w20, w21, lsl #28",
         "msr nzcv, x20"
       ]
     },

unittests/InstructionCountCI/FlagM/Atomics.json

@@ -580,13 +580,12 @@
       ]
     },
     "lock adc byte [rax], 1": {
-      "ExpectedInstructionCount": 15,
+      "ExpectedInstructionCount": 14,
       "Comment": "GROUP1 0x80 /2",
       "ExpectedArm64ASM": [
         "mov w20, #0x1",
-        "cinc w20, w20, lo",
-        "ldaddalb w20, w27, [x4]",
-        "mov w20, #0x1",
+        "cinc w21, w20, lo",
+        "ldaddalb w21, w27, [x4]",
         "cinc w20, w20, lo",
         "add w21, w27, w20",
         "uxtb w26, w21",
@@ -601,35 +600,33 @@
       ]
     },
     "lock adc byte [rax], 0xFF": {
-      "ExpectedInstructionCount": 16,
+      "ExpectedInstructionCount": 15,
       "Comment": "GROUP1 0x80 /2",
       "ExpectedArm64ASM": [
         "mov w20, #0xff",
+        "cinc w21, w20, lo",
+        "ldaddalb w21, w21, [x4]",
+        "mvn w27, w21",
         "cinc w20, w20, lo",
-        "ldaddalb w20, w20, [x4]",
-        "mvn w27, w20",
-        "mov w21, #0xff",
-        "cinc w21, w21, lo",
-        "add w22, w20, w21",
+        "add w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w26, w21",
-        "cset x21, hs",
+        "cmp w26, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
-        "rmif x21, #63, #nzCv",
-        "eor w21, w20, #0xff",
-        "eor w20, w26, w20",
-        "bic w20, w20, w21",
+        "rmif x20, #63, #nzCv",
+        "eor w20, w21, #0xff",
+        "eor w21, w26, w21",
+        "bic w20, w21, w20",
         "rmif x20, #7, #nzcV"
       ]
     },
     "lock adc word [rax], 0x100": {
-      "ExpectedInstructionCount": 15,
+      "ExpectedInstructionCount": 14,
       "Comment": "GROUP1 0x81 /2",
       "ExpectedArm64ASM": [
         "mov w20, #0x100",
-        "cinc w20, w20, lo",
-        "ldaddalh w20, w27, [x4]",
-        "mov w20, #0x100",
+        "cinc w21, w20, lo",
+        "ldaddalh w21, w27, [x4]",
         "cinc w20, w20, lo",
         "add w21, w27, w20",
         "uxth w26, w21",
@@ -644,24 +641,23 @@
       ]
     },
     "lock adc word [rax], 0xFFFF": {
-      "ExpectedInstructionCount": 16,
+      "ExpectedInstructionCount": 15,
       "Comment": "GROUP1 0x81 /2",
       "ExpectedArm64ASM": [
         "mov w20, #0xffff",
+        "cinc w21, w20, lo",
+        "ldaddalh w21, w21, [x4]",
+        "mvn w27, w21",
         "cinc w20, w20, lo",
-        "ldaddalh w20, w20, [x4]",
-        "mvn w27, w20",
-        "mov w21, #0xffff",
-        "cinc w21, w21, lo",
-        "add w22, w20, w21",
+        "add w22, w21, w20",
         "uxth w26, w22",
-        "cmp w26, w21",
-        "cset x21, hs",
+        "cmp w26, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #16",
-        "rmif x21, #63, #nzCv",
-        "eor w21, w20, #0xffff",
-        "eor w20, w26, w20",
-        "bic w20, w20, w21",
+        "rmif x20, #63, #nzCv",
+        "eor w20, w21, #0xffff",
+        "eor w21, w26, w21",
+        "bic w20, w21, w20",
         "rmif x20, #15, #nzcV"
       ]
     },
@@ -715,13 +711,12 @@
       ]
     },
     "lock adc word [rax], 1": {
-      "ExpectedInstructionCount": 15,
+      "ExpectedInstructionCount": 14,
       "Comment": "GROUP1 0x83 /2",
       "ExpectedArm64ASM": [
         "mov w20, #0x1",
-        "cinc w20, w20, lo",
-        "ldaddalh w20, w27, [x4]",
-        "mov w20, #0x1",
+        "cinc w21, w20, lo",
+        "ldaddalh w21, w27, [x4]",
         "cinc w20, w20, lo",
         "add w21, w27, w20",
         "uxth w26, w21",
@@ -760,96 +755,92 @@
       ]
     },
     "lock sbb byte [rax], 1": {
-      "ExpectedInstructionCount": 17,
+      "ExpectedInstructionCount": 16,
       "Comment": "GROUP1 0x80 /3",
       "ExpectedArm64ASM": [
         "mov w20, #0x1",
-        "cinc w20, w20, lo",
-        "neg w1, w20",
+        "cinc w21, w20, lo",
+        "neg w1, w21",
         "ldaddalb w1, w27, [x4]",
-        "uxtb w20, w27",
-        "mov w21, #0x1",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "uxtb w21, w27",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
-        "rmif x21, #63, #nzCv",
-        "eor w21, w20, #0x1",
-        "eor w20, w26, w20",
-        "and w20, w20, w21",
+        "rmif x20, #63, #nzCv",
+        "eor w20, w21, #0x1",
+        "eor w21, w26, w21",
+        "and w20, w21, w20",
         "rmif x20, #7, #nzcV"
       ]
     },
     "lock sbb byte [rax], 0xFF": {
-      "ExpectedInstructionCount": 18,
+      "ExpectedInstructionCount": 17,
       "Comment": "GROUP1 0x80 /3",
       "ExpectedArm64ASM": [
         "mov w20, #0xff",
+        "cinc w21, w20, lo",
+        "neg w1, w21",
+        "ldaddalb w1, w21, [x4]",
+        "mvn w27, w21",
+        "uxtb w21, w21",
         "cinc w20, w20, lo",
-        "neg w1, w20",
-        "ldaddalb w1, w20, [x4]",
-        "mvn w27, w20",
-        "uxtb w20, w20",
-        "mov w21, #0xff",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "sub w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
-        "rmif x21, #63, #nzCv",
-        "eor w21, w20, #0xff",
-        "eor w20, w26, w20",
-        "and w20, w20, w21",
+        "rmif x20, #63, #nzCv",
+        "eor w20, w21, #0xff",
+        "eor w21, w26, w21",
+        "and w20, w21, w20",
         "rmif x20, #7, #nzcV"
       ]
     },
     "lock sbb word [rax], 0x100": {
-      "ExpectedInstructionCount": 17,
+      "ExpectedInstructionCount": 16,
       "Comment": "GROUP1 0x81 /3",
       "ExpectedArm64ASM": [
         "mov w20, #0x100",
-        "cinc w20, w20, lo",
-        "neg w1, w20",
+        "cinc w21, w20, lo",
+        "neg w1, w21",
         "ldaddalh w1, w27, [x4]",
-        "uxth w20, w27",
-        "mov w21, #0x100",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "uxth w21, w27",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxth w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #16",
-        "rmif x21, #63, #nzCv",
-        "eor w21, w20, #0x100",
-        "eor w20, w26, w20",
-        "and w20, w20, w21",
+        "rmif x20, #63, #nzCv",
+        "eor w20, w21, #0x100",
+        "eor w21, w26, w21",
+        "and w20, w21, w20",
         "rmif x20, #15, #nzcV"
       ]
     },
     "lock sbb word [rax], 0xFFFF": {
-      "ExpectedInstructionCount": 18,
+      "ExpectedInstructionCount": 17,
       "Comment": "GROUP1 0x81 /3",
       "ExpectedArm64ASM": [
         "mov w20, #0xffff",
+        "cinc w21, w20, lo",
+        "neg w1, w21",
+        "ldaddalh w1, w21, [x4]",
+        "mvn w27, w21",
+        "uxth w21, w21",
         "cinc w20, w20, lo",
-        "neg w1, w20",
-        "ldaddalh w1, w20, [x4]",
-        "mvn w27, w20",
-        "uxth w20, w20",
-        "mov w21, #0xffff",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "sub w22, w21, w20",
         "uxth w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #16",
-        "rmif x21, #63, #nzCv",
-        "eor w21, w20, #0xffff",
-        "eor w20, w26, w20",
-        "and w20, w20, w21",
+        "rmif x20, #63, #nzCv",
+        "eor w20, w21, #0xffff",
+        "eor w21, w26, w21",
+        "and w20, w21, w20",
         "rmif x20, #15, #nzcV"
       ]
     },
@@ -899,25 +890,24 @@
       ]
     },
     "lock sbb word [rax], 1": {
-      "ExpectedInstructionCount": 17,
+      "ExpectedInstructionCount": 16,
       "Comment": "GROUP1 0x83 /3",
       "ExpectedArm64ASM": [
         "mov w20, #0x1",
-        "cinc w20, w20, lo",
-        "neg w1, w20",
+        "cinc w21, w20, lo",
+        "neg w1, w21",
         "ldaddalh w1, w27, [x4]",
-        "uxth w20, w27",
-        "mov w21, #0x1",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "uxth w21, w27",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxth w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #16",
-        "rmif x21, #63, #nzCv",
-        "eor w21, w20, #0x1",
-        "eor w20, w26, w20",
-        "and w20, w20, w21",
+        "rmif x20, #63, #nzCv",
+        "eor w20, w21, #0x1",
+        "eor w21, w26, w21",
+        "and w20, w21, w20",
         "rmif x20, #15, #nzcV"
       ]
     },

unittests/InstructionCountCI/FlagM/Primary.json

@@ -593,8 +593,8 @@
       "ExpectedInstructionCount": 14,
       "Comment": "0x14",
       "ExpectedArm64ASM": [
-        "mvn w27, w4",
         "mov w20, #0xff",
+        "mvn w27, w4",
         "cinc w20, w20, lo",
         "add w21, w4, w20",
         "uxtb w26, w21",
@@ -613,8 +613,8 @@
       "ExpectedInstructionCount": 14,
       "Comment": "0x15",
       "ExpectedArm64ASM": [
-        "mvn w27, w4",
         "mov w20, #0xffff",
+        "mvn w27, w4",
         "cinc w20, w20, lo",
         "add w21, w4, w20",
         "uxth w26, w21",
@@ -789,18 +789,18 @@
       "ExpectedInstructionCount": 17,
       "Comment": "0x1C",
       "ExpectedArm64ASM": [
-        "uxtb w20, w4",
-        "mov w21, #0x1",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "mov w20, #0x1",
+        "uxtb w21, w4",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
-        "rmif x21, #63, #nzCv",
-        "eor w21, w20, #0x1",
-        "eor w20, w26, w20",
-        "and w20, w20, w21",
+        "rmif x20, #63, #nzCv",
+        "eor w20, w21, #0x1",
+        "eor w21, w26, w21",
+        "and w20, w21, w20",
         "rmif x20, #7, #nzcV",
         "mov x20, x4",
         "bfxil x20, x26, #0, #8",
@@ -812,18 +812,18 @@
       "ExpectedInstructionCount": 17,
       "Comment": "0x1D",
       "ExpectedArm64ASM": [
-        "uxth w20, w4",
-        "mov w21, #0x1",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "mov w20, #0x1",
+        "uxth w21, w4",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxth w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #16",
-        "rmif x21, #63, #nzCv",
-        "eor w21, w20, #0x1",
-        "eor w20, w26, w20",
-        "and w20, w20, w21",
+        "rmif x20, #63, #nzCv",
+        "eor w20, w21, #0x1",
+        "eor w21, w26, w21",
+        "and w20, w21, w20",
         "rmif x20, #15, #nzcV",
         "mov x20, x4",
         "bfxil x20, x26, #0, #16",
@@ -855,19 +855,19 @@
       "ExpectedInstructionCount": 15,
       "Comment": "0x1C",
       "ExpectedArm64ASM": [
+        "mov w20, #0xff",
         "mvn w27, w4",
-        "uxtb w20, w4",
-        "mov w21, #0xff",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "uxtb w21, w4",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
-        "rmif x21, #63, #nzCv",
-        "eor w21, w20, #0xff",
-        "eor w20, w26, w20",
-        "and w20, w20, w21",
+        "rmif x20, #63, #nzCv",
+        "eor w20, w21, #0xff",
+        "eor w21, w26, w21",
+        "and w20, w21, w20",
         "rmif x20, #7, #nzcV",
         "bfxil x4, x26, #0, #8"
       ]
@@ -876,19 +876,19 @@
       "ExpectedInstructionCount": 15,
       "Comment": "0x1D",
       "ExpectedArm64ASM": [
+        "mov w20, #0xffff",
         "mvn w27, w4",
-        "uxth w20, w4",
-        "mov w21, #0xffff",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "uxth w21, w4",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxth w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #16",
-        "rmif x21, #63, #nzCv",
-        "eor w21, w20, #0xffff",
-        "eor w20, w26, w20",
-        "and w20, w20, w21",
+        "rmif x20, #63, #nzCv",
+        "eor w20, w21, #0xffff",
+        "eor w21, w26, w21",
+        "and w20, w21, w20",
         "rmif x20, #15, #nzcV",
         "bfxil x4, x26, #0, #16"
       ]
@@ -1587,9 +1587,9 @@
       "ExpectedInstructionCount": 8,
       "Comment": "0x69",
       "ExpectedArm64ASM": [
-        "sxth x20, w6",
-        "mov w21, #0x101",
-        "mul x20, x20, x21",
+        "mov w20, #0x101",
+        "sxth x21, w6",
+        "mul x20, x21, x20",
         "sbfx x21, x20, #16, #16",
         "bfxil x4, x20, #0, #16",
         "sbfx x20, x20, #15, #1",
@@ -1626,9 +1626,9 @@
       "ExpectedInstructionCount": 8,
       "Comment": "0x6b",
       "ExpectedArm64ASM": [
-        "sxth x20, w6",
-        "mov w21, #0x3",
-        "mul x20, x20, x21",
+        "mov w20, #0x3",
+        "sxth x21, w6",
+        "mul x20, x21, x20",
         "sbfx x21, x20, #16, #16",
         "bfxil x4, x20, #0, #16",
         "sbfx x20, x20, #15, #1",

unittests/InstructionCountCI/FlagM/PrimaryGroup.json

@@ -63,18 +63,18 @@
       "ExpectedInstructionCount": 17,
       "Comment": "GROUP1 0x80 /3",
       "ExpectedArm64ASM": [
-        "uxtb w20, w4",
-        "mov w21, #0x1",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "mov w20, #0x1",
+        "uxtb w21, w4",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
-        "rmif x21, #63, #nzCv",
-        "eor w21, w20, #0x1",
-        "eor w20, w26, w20",
-        "and w20, w20, w21",
+        "rmif x20, #63, #nzCv",
+        "eor w20, w21, #0x1",
+        "eor w21, w26, w21",
+        "and w20, w21, w20",
         "rmif x20, #7, #nzcV",
         "mov x20, x4",
         "bfxil x20, x26, #0, #8",
@@ -154,8 +154,8 @@
       "ExpectedInstructionCount": 14,
       "Comment": "GROUP1 0x80 /2",
       "ExpectedArm64ASM": [
-        "mvn w27, w4",
         "mov w20, #0xff",
+        "mvn w27, w4",
         "cinc w20, w20, lo",
         "add w21, w4, w20",
         "uxtb w26, w21",
@@ -174,19 +174,19 @@
       "ExpectedInstructionCount": 15,
       "Comment": "GROUP1 0x80 /3",
       "ExpectedArm64ASM": [
+        "mov w20, #0xff",
         "mvn w27, w4",
-        "uxtb w20, w4",
-        "mov w21, #0xff",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "uxtb w21, w4",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
-        "rmif x21, #63, #nzCv",
-        "eor w21, w20, #0xff",
-        "eor w20, w26, w20",
-        "and w20, w20, w21",
+        "rmif x20, #63, #nzCv",
+        "eor w20, w21, #0xff",
+        "eor w21, w26, w21",
+        "and w20, w21, w20",
         "rmif x20, #7, #nzcV",
         "bfxil x4, x26, #0, #8"
       ]

unittests/InstructionCountCI/Primary.json

@@ -684,8 +684,8 @@
       "ExpectedInstructionCount": 17,
       "Comment": "0x14",
       "ExpectedArm64ASM": [
-        "mvn w27, w4",
         "mov w20, #0xff",
+        "mvn w27, w4",
         "cinc w20, w20, lo",
         "add w21, w4, w20",
         "uxtb w26, w21",
@@ -707,8 +707,8 @@
       "ExpectedInstructionCount": 17,
       "Comment": "0x15",
       "ExpectedArm64ASM": [
-        "mvn w27, w4",
         "mov w20, #0xffff",
+        "mvn w27, w4",
         "cinc w20, w20, lo",
         "add w21, w4, w20",
         "uxth w26, w21",
@@ -906,21 +906,21 @@
       "ExpectedInstructionCount": 20,
       "Comment": "0x1C",
       "ExpectedArm64ASM": [
-        "uxtb w20, w4",
-        "mov w21, #0x1",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "mov w20, #0x1",
+        "uxtb w21, w4",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
         "mrs x22, nzcv",
-        "orr w21, w22, w21, lsl #29",
-        "eor w22, w20, #0x1",
-        "eor w20, w26, w20",
-        "and w20, w20, w22",
-        "ubfx x20, x20, #7, #1",
-        "orr w20, w21, w20, lsl #28",
+        "orr w20, w22, w20, lsl #29",
+        "eor w22, w21, #0x1",
+        "eor w21, w26, w21",
+        "and w21, w21, w22",
+        "ubfx x21, x21, #7, #1",
+        "orr w20, w20, w21, lsl #28",
         "mov x21, x4",
         "bfxil x21, x26, #0, #8",
         "msr nzcv, x20",
@@ -932,21 +932,21 @@
       "ExpectedInstructionCount": 20,
       "Comment": "0x1D",
       "ExpectedArm64ASM": [
-        "uxth w20, w4",
-        "mov w21, #0x1",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "mov w20, #0x1",
+        "uxth w21, w4",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxth w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #16",
         "mrs x22, nzcv",
-        "orr w21, w22, w21, lsl #29",
-        "eor w22, w20, #0x1",
-        "eor w20, w26, w20",
-        "and w20, w20, w22",
-        "ubfx x20, x20, #15, #1",
-        "orr w20, w21, w20, lsl #28",
+        "orr w20, w22, w20, lsl #29",
+        "eor w22, w21, #0x1",
+        "eor w21, w26, w21",
+        "and w21, w21, w22",
+        "ubfx x21, x21, #15, #1",
+        "orr w20, w20, w21, lsl #28",
         "mov x21, x4",
         "bfxil x21, x26, #0, #16",
         "msr nzcv, x20",
@@ -978,22 +978,22 @@
       "ExpectedInstructionCount": 18,
       "Comment": "0x1C",
       "ExpectedArm64ASM": [
+        "mov w20, #0xff",
         "mvn w27, w4",
-        "uxtb w20, w4",
-        "mov w21, #0xff",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "uxtb w21, w4",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
         "mrs x22, nzcv",
-        "orr w21, w22, w21, lsl #29",
-        "eor w22, w20, #0xff",
-        "eor w20, w26, w20",
-        "and w20, w20, w22",
-        "ubfx x20, x20, #7, #1",
-        "orr w20, w21, w20, lsl #28",
+        "orr w20, w22, w20, lsl #29",
+        "eor w22, w21, #0xff",
+        "eor w21, w26, w21",
+        "and w21, w21, w22",
+        "ubfx x21, x21, #7, #1",
+        "orr w20, w20, w21, lsl #28",
         "bfxil x4, x26, #0, #8",
         "msr nzcv, x20"
       ]
@@ -1002,22 +1002,22 @@
       "ExpectedInstructionCount": 18,
       "Comment": "0x1D",
       "ExpectedArm64ASM": [
+        "mov w20, #0xffff",
         "mvn w27, w4",
-        "uxth w20, w4",
-        "mov w21, #0xffff",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "uxth w21, w4",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxth w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #16",
         "mrs x22, nzcv",
-        "orr w21, w22, w21, lsl #29",
-        "eor w22, w20, #0xffff",
-        "eor w20, w26, w20",
-        "and w20, w20, w22",
-        "ubfx x20, x20, #15, #1",
-        "orr w20, w21, w20, lsl #28",
+        "orr w20, w22, w20, lsl #29",
+        "eor w22, w21, #0xffff",
+        "eor w21, w26, w21",
+        "and w21, w21, w22",
+        "ubfx x21, x21, #15, #1",
+        "orr w20, w20, w21, lsl #28",
         "bfxil x4, x26, #0, #16",
         "msr nzcv, x20"
       ]
@@ -1816,9 +1816,9 @@
       "ExpectedInstructionCount": 8,
       "Comment": "0x69",
       "ExpectedArm64ASM": [
-        "sxth x20, w6",
-        "mov w21, #0x101",
-        "mul x20, x20, x21",
+        "mov w20, #0x101",
+        "sxth x21, w6",
+        "mul x20, x21, x20",
         "sbfx x21, x20, #16, #16",
         "bfxil x4, x20, #0, #16",
         "sbfx x20, x20, #15, #1",
@@ -1879,9 +1879,9 @@
       "ExpectedInstructionCount": 8,
       "Comment": "0x6b",
       "ExpectedArm64ASM": [
-        "sxth x20, w6",
-        "mov w21, #0x3",
-        "mul x20, x20, x21",
+        "mov w20, #0x3",
+        "sxth x21, w6",
+        "mul x20, x21, x20",
         "sbfx x21, x20, #16, #16",
         "bfxil x4, x20, #0, #16",
         "sbfx x20, x20, #15, #1",

unittests/InstructionCountCI/PrimaryGroup.json

@@ -73,21 +73,21 @@
       "ExpectedInstructionCount": 20,
       "Comment": "GROUP1 0x80 /3",
       "ExpectedArm64ASM": [
-        "uxtb w20, w4",
-        "mov w21, #0x1",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "mov w20, #0x1",
+        "uxtb w21, w4",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
         "mrs x22, nzcv",
-        "orr w21, w22, w21, lsl #29",
-        "eor w22, w20, #0x1",
-        "eor w20, w26, w20",
-        "and w20, w20, w22",
-        "ubfx x20, x20, #7, #1",
-        "orr w20, w21, w20, lsl #28",
+        "orr w20, w22, w20, lsl #29",
+        "eor w22, w21, #0x1",
+        "eor w21, w26, w21",
+        "and w21, w21, w22",
+        "ubfx x21, x21, #7, #1",
+        "orr w20, w20, w21, lsl #28",
         "mov x21, x4",
         "bfxil x21, x26, #0, #8",
         "msr nzcv, x20",
@@ -175,8 +175,8 @@
       "ExpectedInstructionCount": 17,
       "Comment": "GROUP1 0x80 /2",
       "ExpectedArm64ASM": [
-        "mvn w27, w4",
         "mov w20, #0xff",
+        "mvn w27, w4",
         "cinc w20, w20, lo",
         "add w21, w4, w20",
         "uxtb w26, w21",
@@ -198,22 +198,22 @@
       "ExpectedInstructionCount": 18,
       "Comment": "GROUP1 0x80 /3",
       "ExpectedArm64ASM": [
+        "mov w20, #0xff",
         "mvn w27, w4",
-        "uxtb w20, w4",
-        "mov w21, #0xff",
-        "cinc w21, w21, lo",
-        "sub w22, w20, w21",
+        "uxtb w21, w4",
+        "cinc w20, w20, lo",
+        "sub w22, w21, w20",
         "uxtb w26, w22",
-        "cmp w20, w21",
-        "cset x21, hs",
+        "cmp w21, w20",
+        "cset x20, hs",
         "cmn wzr, w26, lsl #24",
         "mrs x22, nzcv",
-        "orr w21, w22, w21, lsl #29",
-        "eor w22, w20, #0xff",
-        "eor w20, w26, w20",
-        "and w20, w20, w22",
-        "ubfx x20, x20, #7, #1",
-        "orr w20, w21, w20, lsl #28",
+        "orr w20, w22, w20, lsl #29",
+        "eor w22, w21, #0xff",
+        "eor w21, w26, w21",
+        "and w21, w21, w22",
+        "ubfx x21, x21, #7, #1",
+        "orr w20, w20, w21, lsl #28",
         "bfxil x4, x26, #0, #8",
         "msr nzcv, x20"
       ]