Docs: Update for release FEX-2511

f80 stack xchg optimization for fast path

CodeEmitter/CodeEmitter/ALUOps.inl

@@ -36,24 +36,33 @@ public:
     DataProcessing_PCRel_Imm(Op, rd, Imm);
   }
 
-  void adr(ARMEmitter::Register rd, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
     LOGMAN_THROW_A_FMT(IsADRRange(Imm), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0001'0000 << 24;
-    DataProcessing_PCRel_Imm(Op, rd, Imm);
+    if (IsADRRange(Imm)) {
+      constexpr uint32_t Op = 0b0001'0000 << 24;
+      DataProcessing_PCRel_Imm(Op, rd, Imm);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void adr(ARMEmitter::Register rd, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::ADR});
     constexpr uint32_t Op = 0b0001'0000 << 24;
     DataProcessing_PCRel_Imm(Op, rd, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void adr(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      adr(rd, &Label->Backward);
+      return adr(rd, &Label->Backward);
     } else {
-      adr(rd, &Label->Forward);
+      return adr(rd, &Label->Forward);
     }
   }
 
@@ -62,32 +71,42 @@ public:
     DataProcessing_PCRel_Imm(Op, rd, Imm);
   }
 
-  void adrp(ARMEmitter::Register rd, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, const BackwardLabel* Label) {
     int64_t Imm = reinterpret_cast<int64_t>(Label->Location) - (GetCursorAddress<int64_t>() & ~0xFFFLL);
     LOGMAN_THROW_A_FMT(IsADRPRange(Imm) && IsADRPAligned(Imm), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b1001'0000 << 24;
-    DataProcessing_PCRel_Imm(Op, rd, Imm);
+    if (IsADRPRange(Imm) && IsADRPAligned(Imm)) {
+      constexpr uint32_t Op = 0b1001'0000 << 24;
+      DataProcessing_PCRel_Imm(Op, rd, Imm);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void adrp(ARMEmitter::Register rd, ForwardLabel* Label) {
+
+  [[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::ADRP});
     constexpr uint32_t Op = 0b1001'0000 << 24;
     DataProcessing_PCRel_Imm(Op, rd, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void adrp(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      adrp(rd, &Label->Backward);
+      return adrp(rd, &Label->Backward);
     } else {
-      adrp(rd, &Label->Forward);
+      return adrp(rd, &Label->Forward);
     }
   }
 
-  void LongAddressGen(ARMEmitter::Register rd, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, const BackwardLabel* Label) {
     int64_t Imm = reinterpret_cast<int64_t>(Label->Location) - (GetCursorAddress<int64_t>());
     if (IsADRRange(Imm)) {
       // If the range is in ADR range then we can just use ADR.
-      adr(rd, Label);
+      return adr(rd, Label);
     } else if (IsADRPRange(Imm)) {
       int64_t ADRPImm = (reinterpret_cast<int64_t>(Label->Location) & ~0xFFFLL) - (GetCursorAddress<int64_t>() & ~0xFFFLL);
 
@@ -102,23 +121,28 @@ public:
         // Now even an add
         add(ARMEmitter::Size::i64Bit, rd, rd, AlignedOffset);
       }
-    } else {
-      LOGMAN_MSG_A_FMT("Unscaled offset too large");
-      FEX_UNREACHABLE;
+
+      return BranchEncodeSucceeded::Success;
     }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void LongAddressGen(ARMEmitter::Register rd, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::LONG_ADDRESS_GEN});
     // Emit a register index and a nop. These will be backpatched.
     dc32(rd.Idx());
     nop();
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void LongAddressGen(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      LongAddressGen(rd, &Label->Backward);
+      return LongAddressGen(rd, &Label->Backward);
     } else {
-      LongAddressGen(rd, &Label->Forward);
+      return LongAddressGen(rd, &Label->Forward);
     }
   }
 
@@ -862,12 +886,6 @@ public:
   }
 
 private:
-  static constexpr Condition InvertCondition(Condition cond) {
-    // These behave as always, so it makes no sense to allow inverting these.
-    LOGMAN_THROW_A_FMT(cond != Condition::CC_AL && cond != Condition::CC_NV, "Cannot invert CC_AL or CC_NV");
-    return static_cast<Condition>(FEXCore::ToUnderlying(cond) ^ 1);
-  }
-
   void and_(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t n, uint32_t immr, uint32_t imms) {
     constexpr uint32_t Op = 0b001'0010'00 << 22;
     DataProcessing_Logical_Imm(Op, s, rd, rn, n, immr, imms);

CodeEmitter/CodeEmitter/BranchOps.inl

@@ -20,23 +20,31 @@ public:
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 0, Cond, Imm);
   }
-  void b(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
-    constexpr uint32_t Op = 0b0101'010 << 25;
-    Branch_Conditional(Op, 0, 0, Cond, Imm >> 2);
+    if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) {
+      constexpr uint32_t Op = 0b0101'010 << 25;
+      Branch_Conditional(Op, 0, 0, Cond, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void b(ARMEmitter::Condition Cond, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 0, Cond, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void b(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      b(Cond, &Label->Backward);
+      return b(Cond, &Label->Backward);
     } else {
-      b(Cond, &Label->Forward);
+      return b(Cond, &Label->Forward);
     }
   }
 
@@ -45,24 +53,32 @@ public:
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 1, Cond, Imm);
   }
-  void bc(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
-    constexpr uint32_t Op = 0b0101'010 << 25;
-    Branch_Conditional(Op, 0, 1, Cond, Imm >> 2);
+    if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) {
+      constexpr uint32_t Op = 0b0101'010 << 25;
+      Branch_Conditional(Op, 0, 1, Cond, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void bc(ARMEmitter::Condition Cond, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 1, Cond, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void bc(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      bc(Cond, &Label->Backward);
+      return bc(Cond, &Label->Backward);
     } else {
-      bc(Cond, &Label->Forward);
+      return bc(Cond, &Label->Forward);
     }
   }
 
@@ -98,25 +114,32 @@ public:
 
     UnconditionalBranch(Op, Imm);
   }
-  void b(const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large");
-    constexpr uint32_t Op = 0b0001'01 << 26;
+    if (Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0)) {
+      constexpr uint32_t Op = 0b0001'01 << 26;
+      UnconditionalBranch(Op, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    UnconditionalBranch(Op, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void b(ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::B});
     constexpr uint32_t Op = 0b0001'01 << 26;
 
     UnconditionalBranch(Op, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void b(BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      b(&Label->Backward);
+      return b(&Label->Backward);
     } else {
-      b(&Label->Forward);
+      return b(&Label->Forward);
     }
   }
 
@@ -126,25 +149,33 @@ public:
     UnconditionalBranch(Op, Imm);
   }
 
-  void bl(const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bl(const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large");
-    constexpr uint32_t Op = 0b1001'01 << 26;
+    if (Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0)) {
+      constexpr uint32_t Op = 0b1001'01 << 26;
+      UnconditionalBranch(Op, Imm >> 2);
 
-    UnconditionalBranch(Op, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void bl(ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bl(ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::B});
     constexpr uint32_t Op = 0b1001'01 << 26;
 
     UnconditionalBranch(Op, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void bl(BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bl(BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      bl(&Label->Backward);
+      return bl(&Label->Backward);
     } else {
-      bl(&Label->Forward);
+      return bl(&Label->Forward);
     }
   }
 
@@ -155,28 +186,35 @@ public:
     CompareAndBranch(Op, s, rt, Imm);
   }
 
-  void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0011'0100 << 24;
+    if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) {
+      constexpr uint32_t Op = 0b0011'0100 << 24;
+      CompareAndBranch(Op, s, rt, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    CompareAndBranch(Op, s, rt, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
 
     constexpr uint32_t Op = 0b0011'0100 << 24;
 
     CompareAndBranch(Op, s, rt, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      cbz(s, rt, &Label->Backward);
+      return cbz(s, rt, &Label->Backward);
     } else {
-      cbz(s, rt, &Label->Forward);
+      return cbz(s, rt, &Label->Forward);
     }
   }
 
@@ -186,28 +224,35 @@ public:
     CompareAndBranch(Op, s, rt, Imm);
   }
 
-  void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0011'0101 << 24;
+    if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) {
+      constexpr uint32_t Op = 0b0011'0101 << 24;
+      CompareAndBranch(Op, s, rt, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    CompareAndBranch(Op, s, rt, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
 
     constexpr uint32_t Op = 0b0011'0101 << 24;
 
     CompareAndBranch(Op, s, rt, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      cbnz(s, rt, &Label->Backward);
+      return cbnz(s, rt, &Label->Backward);
     } else {
-      cbnz(s, rt, &Label->Forward);
+      return cbnz(s, rt, &Label->Forward);
     }
   }
 
@@ -217,28 +262,35 @@ public:
 
     TestAndBranch(Op, rt, Bit, Imm);
   }
-  void tbz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0011'0110 << 24;
+    if (Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0)) {
+      constexpr uint32_t Op = 0b0011'0110 << 24;
+      TestAndBranch(Op, rt, Bit, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    TestAndBranch(Op, rt, Bit, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void tbz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::TEST_BRANCH});
 
     constexpr uint32_t Op = 0b0011'0110 << 24;
 
     TestAndBranch(Op, rt, Bit, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void tbz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      tbz(rt, Bit, &Label->Backward);
+      return tbz(rt, Bit, &Label->Backward);
     } else {
-      tbz(rt, Bit, &Label->Forward);
+      return tbz(rt, Bit, &Label->Forward);
     }
   }
 
@@ -247,27 +299,35 @@ public:
 
     TestAndBranch(Op, rt, Bit, Imm);
   }
-  void tbnz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
     LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0011'0111 << 24;
+    if (Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0)) {
+      constexpr uint32_t Op = 0b0011'0111 << 24;
+      TestAndBranch(Op, rt, Bit, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    TestAndBranch(Op, rt, Bit, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void tbnz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::TEST_BRANCH});
     constexpr uint32_t Op = 0b0011'0111 << 24;
 
     TestAndBranch(Op, rt, Bit, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void tbnz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      tbnz(rt, Bit, &Label->Backward);
+      return tbnz(rt, Bit, &Label->Backward);
     } else {
-      tbnz(rt, Bit, &Label->Forward);
+      return tbnz(rt, Bit, &Label->Forward);
     }
   }
 

CodeEmitter/CodeEmitter/Emitter.h

@@ -586,6 +586,15 @@ concept IsXOrWRegister = std::is_same_v<T, XRegister> || std::is_same_v<T, WRegi
 template<typename T>
 concept IsQOrDRegister = std::is_same_v<T, QRegister> || std::is_same_v<T, DRegister>;
 
+template<typename T>
+concept IsLabel = std::is_same_v<T, ARMEmitter::ForwardLabel> || std::is_same_v<T, ARMEmitter::BackwardLabel> ||
+                  std::is_same_v<T, ARMEmitter::BiDirectionalLabel> || std::is_same_v<T, ARMEmitter::ForwardLabel::Reference>;
+
+enum class BranchEncodeSucceeded {
+  Success,
+  Failure,
+};
+
 // Whether or not a given set of vector registers are sequential
 // in increasing order as far as the register file is concerned (modulo its size)
 //
@@ -638,19 +647,25 @@ public:
 
   // Bind a backward label to an address.
   // Address that is bound is the current emitter location.
-  void Bind(BackwardLabel* Label) {
+  [[nodiscard]] bool Bind(BackwardLabel* Label) {
     LOGMAN_THROW_A_FMT(Label->Location == nullptr, "Trying to bind a label twice");
     Label->Location = GetCursorAddress<uint8_t*>();
+
+    // Always binds because it is only storing a location.
+    return true;
   }
 
-  void Bind(const ForwardLabel::Reference* Label) {
+  [[nodiscard]] bool Bind(const ForwardLabel::Reference* Label) {
     uint8_t* CurrentAddress = GetCursorAddress<uint8_t*>();
     // Patch up the instructions
     switch (Label->Type) {
     case ForwardLabel::InstType::ADR: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(IsADRRange(Imm), "Unscaled offset too large");
+      if (!IsADRRange(Imm)) {
+        // Can't bind.
+        return false;
+      }
       uint32_t InstMask = 0b11 << 29 | 0b1111'1111'1111'1111'111 << 5;
       uint32_t Offset = static_cast<uint32_t>(Imm) & 0x3F'FFFF;
       uint32_t Inst = *Instruction & ~InstMask;
@@ -662,7 +677,12 @@ public:
     case ForwardLabel::InstType::ADRP: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(IsADRPRange(Imm) && IsADRPAligned(Imm), "Unscaled offset too large");
+
+      if (!(IsADRPRange(Imm) && IsADRPAligned(Imm))) {
+        // Can't bind.
+        return false;
+      }
+
       Imm >>= 12;
       uint32_t InstMask = 0b11 << 29 | 0b1111'1111'1111'1111'111 << 5;
       uint32_t Offset = static_cast<uint32_t>(Imm) & 0x3F'FFFF;
@@ -672,11 +692,13 @@ public:
       *Instruction = Inst;
       break;
     }
-
     case ForwardLabel::InstType::B: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large");
+      if (!(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0))) {
+        // Can't bind.
+        return false;
+      }
       Imm >>= 2;
       uint32_t InstMask = 0x3FF'FFFF;
       uint32_t Offset = static_cast<uint32_t>(Imm) & InstMask;
@@ -686,11 +708,13 @@ public:
 
       break;
     }
-
     case ForwardLabel::InstType::TEST_BRANCH: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large");
+      if (!(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0))) {
+        // Can't bind.
+        return false;
+      }
       Imm >>= 2;
       uint32_t InstMask = 0x3FFF;
       uint32_t Offset = static_cast<uint32_t>(Imm) & InstMask;
@@ -704,7 +728,10 @@ public:
     case ForwardLabel::InstType::RELATIVE_LOAD: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
+      if (!(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0))) {
+        // Can't bind.
+        return false;
+      }
       Imm >>= 2;
       uint32_t InstMask = 0x7'FFFF;
       uint32_t Offset = static_cast<uint32_t>(Imm) & InstMask;
@@ -753,27 +780,41 @@ public:
     }
     default: LOGMAN_MSG_A_FMT("Unexpected inst type in label fixup");
     }
+
+    return true;
   }
 
   // Bind a forward label to a location.
   // This walks all the instructions in the label's vector.
   // Then backpatching all instructions that have used the label.
-  void Bind(ForwardLabel* Label) {
+  [[nodiscard]] bool Bind(ForwardLabel* Label) {
+    bool Bound = true;
     if (Label->FirstInst.Location) {
-      Bind(&Label->FirstInst);
+      Bound &= Bind(&Label->FirstInst);
     }
     for (auto& Inst : Label->Insts) {
-      Bind(&Inst);
+      Bound &= Bind(&Inst);
     }
+
+    return Bound;
   }
 
   // Bind a bidirectional location to a location.
   // Binds both forwards and backwards depending on how the label was used.
-  void Bind(BiDirectionalLabel* Label) {
+  [[nodiscard]] bool Bind(BiDirectionalLabel* Label) {
+    bool Bound = true;
     if (!Label->Backward.Location) {
-      Bind(&Label->Backward);
+      Bound &= Bind(&Label->Backward);
     }
-    Bind(&Label->Forward);
+    Bound &= Bind(&Label->Forward);
+
+    return Bound;
+  }
+
+  static constexpr Condition InvertCondition(Condition cond) {
+    // These behave as always, so it makes no sense to allow inverting these.
+    LOGMAN_THROW_A_FMT(cond != Condition::CC_AL && cond != Condition::CC_NV, "Cannot invert CC_AL or CC_NV");
+    return static_cast<Condition>(FEXCore::ToUnderlying(cond) ^ 1);
   }
 
 #include <CodeEmitter/VixlUtils.inl>

FEXCore/Source/CMakeLists.txt

@@ -66,6 +66,7 @@ set (SRCS
   Interface/IR/Passes/RedundantFlagCalculationElimination.cpp
   Interface/IR/Passes/RegisterAllocationPass.cpp
   Interface/IR/Passes/x87StackOptimizationPass.cpp
+  Utils/LongJump.cpp
   Utils/Telemetry.cpp
   Utils/Threads.cpp
   Utils/Profiler.cpp

FEXCore/Source/Common/VectorRegType.h

@@ -4,6 +4,8 @@
 #ifdef _M_X86_64
 #include <xmmintrin.h>
 #include <immintrin.h>
+#else
+#include <cstdint>
 #endif
 
 namespace FEXCore {

FEXCore/Source/Interface/Core/CPUID.cpp

@@ -88,6 +88,7 @@ namespace ProductNames {
   static const char ARM_Blizzard_M2Pro[] = "Apple Blizzard (M2 Pro)";
   static const char ARM_Avalanche_M2Max[] = "Apple Avalanche (M2 Max)";
   static const char ARM_Blizzard_M2Max[] = "Apple Blizzard (M2 Max)";
+  static const char ARM_AppleSilicon[] = "Apple Silicon";
 
   static const char ARM_ORYON_1[] = "Oryon-1";
   static const char ARM_Ampere_1[] = "AmpereOne";
@@ -188,6 +189,7 @@ void CPUIDEmu::SetupHostHybridFlag() {
     {0x61, 0x029, 1, ProductNames::ARM_Firestorm_M1Max}, // Apple Firestorm (M1 Max)
     {0x61, 0x025, 1, ProductNames::ARM_Firestorm_M1Pro}, // Apple Firestorm (M1 Pro)
     {0x61, 0x023, 1, ProductNames::ARM_Firestorm_M1},    // Apple Firestorm (M1)
+    {0x61, 0, 1, ProductNames::ARM_AppleSilicon},        // QEmu Apple Silicon
 
     {0x41, 0xd8c, 1, ProductNames::ARM_C1Ultra},   // C1-Ultra
     {0x41, 0xd90, 1, ProductNames::ARM_C1Premium}, // C1-Premium

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: MIT
 
-#include "Common/SoftFloat.h"
+#include "Common/VectorRegType.h"
 #include "Interface/Context/Context.h"
 #include "Interface/Core/CPUBackend.h"
 #include "Interface/Core/Dispatcher/Dispatcher.h"
@@ -26,9 +26,7 @@
 #endif
 
 #include <array>
-#include <atomic>
 #include <bit>
-#include <condition_variable>
 #include <csignal>
 #include <cstring>
 
@@ -95,12 +93,12 @@ void Dispatcher::EmitDispatcher() {
 
   FillStaticRegs();
   ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));
-  cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);
+  (void)cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);
 
   ARMEmitter::BiDirectionalLabel LoopTop {};
 
 #ifdef _M_ARM_64EC
-  b(&LoopTop);
+  (void)b(&LoopTop);
 
   AbsoluteLoopTopAddressEnterECFillSRA = GetCursorAddress<uint64_t>();
   ldr(STATE, EC_ENTRY_CPUAREA_REG, CPU_AREA_EMULATOR_DATA_OFFSET);
@@ -108,10 +106,10 @@ void Dispatcher::EmitDispatcher() {
 
   ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));
   // Force a single instruction block if ENTRY_FILL_SRA_SINGLE_INST_REG is nonzero entering the JIT, used for inline SMC handling.
-  cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);
+  (void)cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);
 
   // Enter JIT
-  b(&LoopTop);
+  (void)b(&LoopTop);
 
   AbsoluteLoopTopAddressEnterEC = GetCursorAddress<uint64_t>();
   // Load ThreadState and write the target PC there
@@ -132,7 +130,7 @@ void Dispatcher::EmitDispatcher() {
   ldp<ARMEmitter::IndexType::OFFSET>(TMP1, TMP2, REG_CALLRET_SP);
   // EC_CALL_CHECKER_PC_REG is REG_PF which isn't touched by any of the above
   sub(ARMEmitter::Size::i64Bit, TMP1, EC_CALL_CHECKER_PC_REG, TMP1);
-  cbnz(ARMEmitter::Size::i64Bit, TMP1, &LoopTop);
+  (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &LoopTop);
 
   // If the entry at the TOS is for the target address, pop it and return to the JIT code
   add(ARMEmitter::Size::i64Bit, REG_CALLRET_SP, REG_CALLRET_SP, 0x10);
@@ -144,7 +142,7 @@ void Dispatcher::EmitDispatcher() {
   // We want to ensure that we are 16 byte aligned at the top of this loop
   Align16B();
 
-  Bind(&LoopTop);
+  (void)Bind(&LoopTop);
   AbsoluteLoopTopAddress = GetCursorAddress<uint64_t>();
 
   // Load in our RIP
@@ -171,16 +169,16 @@ void Dispatcher::EmitDispatcher() {
   ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionEC));
   br(TMP2);
 
-  Bind(&l_NotECCode);
+  (void)Bind(&l_NotECCode);
 #endif
 
   ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
-  cbnz(ARMEmitter::Size::i32Bit, TMP1, &CompileSingleStep);
+  (void)cbnz(ARMEmitter::Size::i32Bit, TMP1, &CompileSingleStep);
 
   ARMEmitter::ForwardLabel NoBlock;
 
   if (DisableL2Cache()) {
-    b(&NoBlock);
+    (void)b(&NoBlock);
   } else {
     // This is the block cache lookup routine
     // It matches what is going on it LookupCache.h::FindBlock
@@ -203,7 +201,7 @@ void Dispatcher::EmitDispatcher() {
       ldr(TMP1, TMP1, TMP2, ARMEmitter::ExtendedType::LSL_64, 3);
 
       // If page pointer is zero then we have no block
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &NoBlock);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &NoBlock);
 
       // Steal the page offset
       and_(ARMEmitter::Size::i64Bit, TMP2, TMP4, 0x0FFF);
@@ -218,10 +216,10 @@ void Dispatcher::EmitDispatcher() {
 
       // If the guest address doesn't match, Compile the block.
       sub(TMP2, TMP2, RipReg);
-      cbnz(ARMEmitter::Size::i64Bit, TMP2, &NoBlock);
+      (void)cbnz(ARMEmitter::Size::i64Bit, TMP2, &NoBlock);
 
       // Check the host address to see if it matches, else compile the block.
-      cbz(ARMEmitter::Size::i64Bit, TMP4, &NoBlock);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP4, &NoBlock);
 
       // If we've made it here then we have a real compiled block
       {
@@ -313,7 +311,7 @@ void Dispatcher::EmitDispatcher() {
 
   // Need to create the block
   {
-    Bind(&NoBlock);
+    (void)Bind(&NoBlock);
 
     EmitSignalGuardedRegion([&]() {
       SpillStaticRegs(TMP1);
@@ -347,7 +345,7 @@ void Dispatcher::EmitDispatcher() {
   }
 
   {
-    Bind(&CompileSingleStep);
+    (void)Bind(&CompileSingleStep);
 
     EmitSignalGuardedRegion([&]() {
       SpillStaticRegs(TMP1);
@@ -509,7 +507,7 @@ void Dispatcher::EmitDispatcher() {
     stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
 
     // Now go back to the regular dispatcher loop
-    b(&LoopTop);
+    (void)b(&LoopTop);
   }
 
   auto EmitLongALUOpHandler = [&](auto R, auto Offset) {
@@ -578,14 +576,15 @@ void Dispatcher::EmitDispatcher() {
     }
   }
 
-  Bind(&l_CTX);
+  (void)Bind(&l_CTX);
   dc64(reinterpret_cast<uintptr_t>(CTX));
-  Bind(&l_Sleep);
+  (void)Bind(&l_Sleep);
   dc64(reinterpret_cast<uint64_t>(SleepThread));
-  Bind(&l_CompileBlock);
+  (void)Bind(&l_CompileBlock);
   FEXCore::Utils::MemberFunctionToPointerCast PMFCompileBlock(&FEXCore::Context::ContextImpl::CompileBlock);
   dc64(PMFCompileBlock.GetConvertedPointer());
-  Bind(&l_CompileSingleStep);
+  (void)Bind(&l_CompileSingleStep);
+
   FEXCore::Utils::MemberFunctionToPointerCast PMFCompileSingleStep(&FEXCore::Context::ContextImpl::CompileSingleStep);
   dc64(PMFCompileSingleStep.GetConvertedPointer());
 

FEXCore/Source/Interface/Core/JIT/ALUOps.cpp

@@ -588,7 +588,7 @@ DEF_OP(ShiftFlags) {
   and_(ARMEmitter::Size::i32Bit, TMP1, Src2, OpSize == IR::OpSize::i64Bit ? 0x3f : 0x1f);
 
   ARMEmitter::ForwardLabel Done;
-  cbz(EmitSize, TMP1, &Done);
+  (void)cbz(EmitSize, TMP1, &Done);
   {
     // PF/SF/ZF/OF
     if (OpSize >= IR::OpSize::i32Bit) {
@@ -652,7 +652,7 @@ DEF_OP(ShiftFlags) {
       msr(ARMEmitter::SystemRegister::NZCV, TMP2);
     }
   }
-  Bind(&Done);
+  (void)Bind(&Done);
 
   // TODO: Make RA less dumb so this can't happen (e.g. with late-kill).
   if (PFOutput != PFTemp) {
@@ -669,7 +669,7 @@ DEF_OP(RotateFlags) {
 
   // If shift=0, flags are unaffected. Wrap the whole implementation in a cbz.
   ARMEmitter::ForwardLabel Done;
-  cbz(EmitSize, Shift, &Done);
+  (void)cbz(EmitSize, Shift, &Done);
   {
     // Extract the last bit shifted in to CF
     const auto BitSize = IR::OpSizeToSize(Op->Size) * 8;
@@ -701,7 +701,7 @@ DEF_OP(RotateFlags) {
       msr(ARMEmitter::SystemRegister::NZCV, TMP3);
     }
   }
-  Bind(&Done);
+  (void)Bind(&Done);
 }
 
 DEF_OP(Extr) {
@@ -767,14 +767,14 @@ DEF_OP(PDep) {
     // Now, they're copied, so we can start setting Dest (even if it overlaps with
     // one of them).  Handle early exit case
     mov(EmitSize, Dest, 0);
-    cbz(EmitSize, OrigMask, &Done);
+    (void)cbz(EmitSize, OrigMask, &Done);
 
     // Setup for first iteration
     neg(EmitSize, T0, Mask);
     and_(EmitSize, T0, T0, Mask);
 
     // Main loop
-    Bind(&NextBit);
+    (void)Bind(&NextBit);
     sbfx(EmitSize, T1, Input, 0, 1);
     eor(EmitSize, Mask, Mask, T0);
     and_(EmitSize, T0, T1, T0);
@@ -782,10 +782,10 @@ DEF_OP(PDep) {
     orr(EmitSize, Dest, Dest, T0);
     lsr(EmitSize, Input, Input, 1);
     and_(EmitSize, T0, Mask, T1);
-    cbnz(EmitSize, T0, &NextBit);
+    (void)cbnz(EmitSize, T0, &NextBit);
 
     // All done with nothing to do.
-    Bind(&Done);
+    (void)Bind(&Done);
   }
 }
 
@@ -821,27 +821,27 @@ DEF_OP(PExt) {
     ARMEmitter::BackwardLabel NextBit;
     ARMEmitter::ForwardLabel Done;
 
-    cbz(EmitSize, Mask, &EarlyExit);
+    (void)cbz(EmitSize, Mask, &EarlyExit);
     mov(EmitSize, MaskReg, Mask);
     mov(EmitSize, ValueReg, Input);
     mov(EmitSize, Dest, ARMEmitter::Reg::zr);
 
     // Main loop
-    Bind(&NextBit);
-    cbz(EmitSize, MaskReg, &Done);
+    (void)Bind(&NextBit);
+    (void)cbz(EmitSize, MaskReg, &Done);
     clz(EmitSize, BitReg, MaskReg);
     lslv(EmitSize, ValueReg, ValueReg, BitReg);
     lslv(EmitSize, MaskReg, MaskReg, BitReg);
     extr(EmitSize, Dest, Dest, ValueReg, OpSizeBitsM1);
     bfc(EmitSize, MaskReg, OpSizeBitsM1, 1);
-    b(&NextBit);
+    (void)b(&NextBit);
 
     // Early exit
-    Bind(&EarlyExit);
+    (void)Bind(&EarlyExit);
     mov(EmitSize, Dest, ARMEmitter::Reg::zr);
 
     // All done with nothing to do.
-    Bind(&Done);
+    (void)Bind(&Done);
   }
 }
 
@@ -909,7 +909,7 @@ DEF_OP(Div) {
     eor(EmitSize, TMP1, TMP1, Upper);
 
     // If the sign bit matches then the result is zero
-    cbz(EmitSize, TMP1, &Only64Bit);
+    (void)cbz(EmitSize, TMP1, &Only64Bit);
 
     // Long divide
     {
@@ -928,17 +928,17 @@ DEF_OP(Div) {
       mov(EmitSize, Remainder, TMP2);
 
       // Skip 64-bit path
-      b(&LongDIVRet);
+      (void)b(&LongDIVRet);
     }
 
-    Bind(&Only64Bit);
+    (void)Bind(&Only64Bit);
     // 64-Bit only
     {
       sdiv(EmitSize, Quotient, Lower, Divisor);
       msub(EmitSize, Remainder, Quotient, Divisor, Lower);
     }
 
-    Bind(&LongDIVRet);
+    (void)Bind(&LongDIVRet);
     break;
   }
   default: LOGMAN_MSG_A_FMT("Unknown DIV Size: {}", OpSize); break;
@@ -992,7 +992,7 @@ DEF_OP(UDiv) {
 
     // Check the upper bits for zero
     // If the upper bits are zero then we can do a 64-bit divide
-    cbz(EmitSize, Upper, &Only64Bit);
+    (void)cbz(EmitSize, Upper, &Only64Bit);
 
     // Long divide
     {
@@ -1011,17 +1011,17 @@ DEF_OP(UDiv) {
       mov(EmitSize, Remainder, TMP2);
 
       // Skip 64-bit path
-      b(&LongDIVRet);
+      (void)b(&LongDIVRet);
     }
 
-    Bind(&Only64Bit);
+    (void)Bind(&Only64Bit);
     // 64-Bit only
     {
       udiv(EmitSize, Quotient, Lower, Divisor);
       msub(EmitSize, Remainder, Quotient, Divisor, Lower);
     }
 
-    Bind(&LongDIVRet);
+    (void)Bind(&LongDIVRet);
     break;
   }
   default: LOGMAN_MSG_A_FMT("Unknown LUDIV Size: {}", OpSize); break;

FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp

@@ -63,7 +63,7 @@ void Arm64JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair& Lit) {
   auto CurrentCursor = GetCursorAddress<uint8_t*>();
   Lit.MoveABI.NamedSymbolLiteral.Offset = CurrentCursor - CodeData.BlockBegin;
 
-  Bind(&Lit.Loc);
+  BindOrRestart(&Lit.Loc);
   dc64(Lit.Lit);
   Relocations.emplace_back(Lit.MoveABI);
 }

FEXCore/Source/Interface/Core/JIT/AtomicOps.cpp

@@ -62,27 +62,27 @@ DEF_OP(CASPair) {
     ARMEmitter::BackwardLabel LoopTop;
     ARMEmitter::ForwardLabel LoopNotExpected;
     ARMEmitter::ForwardLabel LoopExpected;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
 
     // This instruction sequence must be synced with HandleCASPAL_Armv8.
     ldaxp(EmitSize, TMP2, TMP3, MemSrc);
     cmp(EmitSize, TMP2, Expected0);
     ccmp(EmitSize, TMP3, Expected1, ARMEmitter::StatusFlags::None, ARMEmitter::Condition::CC_EQ);
-    b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
+    (void)b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
     stlxp(EmitSize, TMP2, Desired0, Desired1, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
+    (void)cbnz(EmitSize, TMP2, &LoopTop);
     mov(EmitSize, Dst0, Expected0);
     mov(EmitSize, Dst1, Expected1);
 
-    b(&LoopExpected);
+    (void)b(&LoopExpected);
 
-    Bind(&LoopNotExpected);
+    (void)Bind(&LoopNotExpected);
     mov(EmitSize, Dst0, TMP2.R());
     mov(EmitSize, Dst1, TMP3.R());
     // exclusive monitor needs to be cleared here
     // Might have hit the case where ldaxr was hit but stlxr wasn't
     clrex();
-    Bind(&LoopExpected);
+    (void)Bind(&LoopExpected);
 
     // Restore
     msr(ARMEmitter::SystemRegister::NZCV, TMP1);
@@ -114,7 +114,7 @@ DEF_OP(CAS) {
     ARMEmitter::BackwardLabel LoopTop;
     ARMEmitter::ForwardLabel LoopNotExpected;
     ARMEmitter::ForwardLabel LoopExpected;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     if (IROp->Size == IR::OpSize::i8Bit) {
       cmp(EmitSize, TMP2, Expected, ARMEmitter::ExtendedType::UXTB, 0);
@@ -123,18 +123,18 @@ DEF_OP(CAS) {
     } else {
       cmp(EmitSize, TMP2, Expected);
     }
-    b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
+    (void)b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
     stlxr(SubEmitSize, TMP3, Desired, MemSrc);
-    cbnz(EmitSize, TMP3, &LoopTop);
+    (void)cbnz(EmitSize, TMP3, &LoopTop);
     mov(EmitSize, Dst, Expected);
-    b(&LoopExpected);
+    (void)b(&LoopExpected);
 
-    Bind(&LoopNotExpected);
+    (void)Bind(&LoopNotExpected);
     mov(EmitSize, Dst, TMP2.R());
     // exclusive monitor needs to be cleared here
     // Might have hit the case where ldaxr was hit but stlxr wasn't
     clrex();
-    Bind(&LoopExpected);
+    (void)Bind(&LoopExpected);
   }
 }
 
@@ -150,11 +150,11 @@ DEF_OP(AtomicXor) {
     steorl(SubEmitSize, Src, MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     eor(EmitSize, TMP2, TMP2, Src);
     stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
+    (void)cbnz(EmitSize, TMP2, &LoopTop);
   }
 }
 
@@ -179,10 +179,10 @@ DEF_OP(AtomicSwap) {
     ldswpal(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     stlxr(SubEmitSize, TMP4, Src, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     ubfm(EmitSize, GetReg(Node), TMP2, 0, IR::OpSizeAsBits(OpSize) - 1);
   }
 }
@@ -199,11 +199,11 @@ DEF_OP(AtomicFetchAdd) {
     ldaddal(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     add(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -221,11 +221,11 @@ DEF_OP(AtomicFetchSub) {
     ldaddal(SubEmitSize, TMP2, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     sub(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -243,11 +243,11 @@ DEF_OP(AtomicFetchAnd) {
     ldclral(SubEmitSize, TMP2, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     and_(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -264,11 +264,11 @@ DEF_OP(AtomicFetchCLR) {
     ldclral(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     bic(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -285,11 +285,11 @@ DEF_OP(AtomicFetchOr) {
     ldsetal(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     orr(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -306,11 +306,11 @@ DEF_OP(AtomicFetchXor) {
     ldeoral(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     eor(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -326,20 +326,20 @@ DEF_OP(AtomicFetchNeg) {
     // Use a CAS loop to avoid needing to emulate unaligned LLSC atomics
     ldr(SubEmitSize, TMP2, MemSrc);
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     mov(EmitSize, TMP4, TMP2);
     neg(EmitSize, TMP3, TMP2);
     casal(SubEmitSize, TMP2, TMP3, MemSrc);
     sub(EmitSize, TMP3, TMP2, TMP4);
-    cbnz(EmitSize, TMP3, &LoopTop);
+    (void)cbnz(EmitSize, TMP3, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     neg(EmitSize, TMP3, TMP2);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -359,11 +359,11 @@ DEF_OP(TelemetrySetValue) {
     stsetl(ARMEmitter::SubRegSize::i64Bit, TMP1, TMP2);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(ARMEmitter::SubRegSize::i64Bit, TMP3, TMP2);
     orr(ARMEmitter::Size::i32Bit, TMP3, TMP3, Src);
     stlxr(ARMEmitter::SubRegSize::i64Bit, TMP3, TMP3, TMP2);
-    cbnz(ARMEmitter::Size::i32Bit, TMP3, &LoopTop);
+    (void)cbnz(ARMEmitter::Size::i32Bit, TMP3, &LoopTop);
   }
 #endif
 }

FEXCore/Source/Interface/Core/JIT/BranchOps.cpp

@@ -141,7 +141,7 @@ DEF_OP(ExitFunction) {
         if (!Op->CallReturnBlock.IsInvalid()) {
           auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();
           PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;
-          adr(TMP1, &l_CallReturn);
+          (void)adr(TMP1, &l_CallReturn);
           stp<ARMEmitter::IndexType::PRE>(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);
         } else {
           stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
@@ -149,16 +149,16 @@ DEF_OP(ExitFunction) {
       } else if (Op->Hint == IR::BranchHint::CheckTF) {
         ARMEmitter::ForwardLabel TFUnset;
         ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
-        cbz(ARMEmitter::Size::i32Bit, TMP1, &TFUnset);
+        (void)cbz(ARMEmitter::Size::i32Bit, TMP1, &TFUnset);
         LoadConstant(ARMEmitter::Size::i64Bit, TMP1, NewRIP);
         str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
         ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop));
         blr(TMP2);
-        Bind(&TFUnset);
+        (void)Bind(&TFUnset);
       }
 
       EmitLinkedBranch(NewRIP, Op->Hint == IR::BranchHint::Call);
-      Bind(&l_CallReturn);
+      (void)Bind(&l_CallReturn);
 #ifdef _M_ARM_64EC
     }
 #endif
@@ -170,7 +170,7 @@ DEF_OP(ExitFunction) {
       // First try to pop from the call-ret stack, otherwise follow the normal path (but ending in a ret)
       ldp<ARMEmitter::IndexType::POST>(TMP1, TMP2, REG_CALLRET_SP, 0x10);
       sub(TMP1, TMP1, RipReg.X());
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
     }
 
     // L1 Cache
@@ -185,23 +185,23 @@ DEF_OP(ExitFunction) {
 
     // Note: sub+cbnz used over cmp+br to preserve flags.
     sub(TMP1, TMP1, RipReg.X());
-    cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
+    (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
     ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop));
     str(RipReg.X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
 
-    Bind(&SkipFullLookup);
+    (void)Bind(&SkipFullLookup);
     if (Op->Hint == IR::BranchHint::Call) {
       ARMEmitter::ForwardLabel l_CallReturn;
       if (!Op->CallReturnBlock.IsInvalid()) {
         auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();
         PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;
-        adr(TMP1, &l_CallReturn);
+        (void)adr(TMP1, &l_CallReturn);
         stp<ARMEmitter::IndexType::PRE>(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);
       } else {
         stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
       }
       blr(TMP2);
-      Bind(&l_CallReturn);
+      (void)Bind(&l_CallReturn);
     } else if (Op->Hint == IR::BranchHint::Return) {
       ret(TMP2);
     } else {
@@ -222,7 +222,7 @@ DEF_OP(CondJump) {
   auto TrueTargetLabel = JumpTarget(Op->TrueBlock);
 
   if (Op->FromNZCV) {
-    b(MapCC(Op->Cond), TrueTargetLabel);
+    b_OrRestart(MapCC(Op->Cond), TrueTargetLabel);
   } else {
     uint64_t Const;
     const bool isConst = IsInlineConstant(Op->Cmp2, &Const);
@@ -235,16 +235,16 @@ DEF_OP(CondJump) {
 
     if (Op->Cond == IR::CondClass::EQ) {
       LOGMAN_THROW_A_FMT(Const == 0, "CondJump: Expected 0 source");
-      cbz(Size, Reg, TrueTargetLabel);
+      cbz_OrRestart(Size, Reg, TrueTargetLabel);
     } else if (Op->Cond == IR::CondClass::NEQ) {
       LOGMAN_THROW_A_FMT(Const == 0, "CondJump: Expected 0 source");
-      cbnz(Size, Reg, TrueTargetLabel);
+      cbnz_OrRestart(Size, Reg, TrueTargetLabel);
     } else if (Op->Cond == IR::CondClass::TSTZ) {
       LOGMAN_THROW_A_FMT(Const < 64, "CondJump: Expected valid bit source");
-      tbz(Reg, Const, TrueTargetLabel);
+      tbz_OrRestart(Reg, Const, TrueTargetLabel);
     } else if (Op->Cond == IR::CondClass::TSTNZ) {
       LOGMAN_THROW_A_FMT(Const < 64, "CondJump: Expected valid bit source");
-      tbnz(Reg, Const, TrueTargetLabel);
+      tbnz_OrRestart(Reg, Const, TrueTargetLabel);
     } else {
       LOGMAN_THROW_A_FMT(false, "CondJump expected simple condition");
     }
@@ -355,7 +355,7 @@ DEF_OP(ValidateCode) {
     while (len >= Size) {
       LoadData();
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP2);
-      cbnz(ARMEmitter::Size::i64Bit, TMP1, &Fail);
+      cbnz_OrRestart(ARMEmitter::Size::i64Bit, TMP1, &Fail);
       len -= Size;
       Offset += Size;
     }
@@ -383,10 +383,10 @@ DEF_OP(ValidateCode) {
 
   ARMEmitter::ForwardLabel End;
   LoadConstant(ARMEmitter::Size::i32Bit, Dst, 0);
-  b(&End);
-  Bind(&Fail);
+  b_OrRestart(&End);
+  BindOrRestart(&Fail);
   LoadConstant(ARMEmitter::Size::i32Bit, Dst, 1);
-  Bind(&End);
+  BindOrRestart(&End);
 }
 
 DEF_OP(ThreadRemoveCodeEntry) {

FEXCore/Source/Interface/Core/JIT/JIT.cpp

@@ -11,8 +11,6 @@ desc: Main glue logic of the arm64 splatter backend
 $end_info$
 */
 
-#include "Common/SoftFloat.h"
-
 #include "Interface/Context/Context.h"
 #include "Interface/Core/LookupCache.h"
 #include "Interface/Core/Dispatcher/Dispatcher.h"
@@ -30,6 +28,7 @@ $end_info$
 #include <FEXCore/Utils/CompilerDefs.h>
 #include <FEXCore/Utils/EnumUtils.h>
 #include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/LongJump.h>
 #include <FEXCore/Utils/Profiler.h>
 #include <FEXCore/Utils/Telemetry.h>
 #include <FEXCore/Utils/TypeDefines.h>
@@ -37,7 +36,6 @@ $end_info$
 
 #include <cstdio>
 #include <cstring>
-#include <limits>
 #include <unistd.h>
 
 namespace {
@@ -742,11 +740,11 @@ void Arm64JITCore::EmitTFCheck() {
   // Note that this needs to be before the below suspend checks, as X86 checks this flag immediately after executing an instruction.
   ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
 
-  cbz(ARMEmitter::Size::i32Bit, TMP1, &l_TFUnset);
+  (void)cbz(ARMEmitter::Size::i32Bit, TMP1, &l_TFUnset);
 
   // X86 semantically checks TF after executing each instruction, so e.g. setting a context with TF set will execute a single instruction
   // and then raise an exception. However on the FEX side this is simpler to implement by checking at the start of each instruction, handle this by having bit 1 being unset in the flag state indicate that TF is blocked for a single instruction.
-  tbz(TMP1, 1, &l_TFBlocked);
+  (void)tbz(TMP1, 1, &l_TFBlocked);
 
   // Block TF for a single instruction when the frontend jumps to a new context by unsetting bit 1.
   ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
@@ -769,11 +767,11 @@ void Arm64JITCore::EmitTFCheck() {
   ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP));
   br(TMP1);
 
-  Bind(&l_TFBlocked);
+  (void)Bind(&l_TFBlocked);
   // If TF was blocked for this instruction, unblock it for the next.
   LoadConstant(ARMEmitter::Size::i32Bit, TMP1, 0b11);
   strb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
-  Bind(&l_TFUnset);
+  (void)Bind(&l_TFUnset);
 }
 
 void Arm64JITCore::EmitSuspendInterruptCheck() {
@@ -789,16 +787,16 @@ void Arm64JITCore::EmitSuspendInterruptCheck() {
 
   ldr(TMP2.W(), STATE_PTR(CpuStateFrame, SuspendDoorbell));
   ARMEmitter::ForwardLabel l_NoSuspend;
-  cbz(ARMEmitter::Size::i32Bit, TMP2, &l_NoSuspend);
+  (void)cbz(ARMEmitter::Size::i32Bit, TMP2, &l_NoSuspend);
   brk(SuspendMagic);
-  Bind(&l_NoSuspend);
+  (void)Bind(&l_NoSuspend);
 #endif
 }
 
 void Arm64JITCore::EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF) {
   // Get the address of the JITCodeHeader and store in to the core state.
   // Two instruction cost, each 1 cycle.
-  adr(TMP1, &HeaderLabel);
+  adr_OrRestart(TMP1, &HeaderLabel);
   str(TMP1, STATE, offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader));
 
   if (CheckTF) {
@@ -815,21 +813,34 @@ void Arm64JITCore::EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool C
       sub(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, TMP1, ARMEmitter::ExtendedType::LSL_64, 0);
     }
   }
+
+  EmitSuspendInterruptCheck();
 }
 
 CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
                                                    FEXCore::Core::DebugData* DebugData, bool CheckTF) {
   FEXCORE_PROFILE_SCOPED("Arm64::CompileCode");
-
-  JumpTargets.clear();
-  CallReturnTargets.clear();
-  PendingJumpThunks.clear();
-  uint32_t SSACount = IR->GetSSACount();
-  JumpTargets.resize(IR->GetHeader()->BlockCount, {});
-
   this->Entry = Entry;
   this->DebugData = DebugData;
   this->IR = IR;
+  RequiresFarARM64Jumps = false;
+
+  // Prepare restart via long jump in case branch encoding fails.
+  // This uses UncheckedLongJump since we don't implement std::longjmp in WoA setups
+  switch (static_cast<RestartOptions::Control>(FEXCore::UncheckedLongJump::SetJump(RestartControl.RestartJump))) {
+  case RestartOptions::Control::Incoming:
+    // Nothing
+    break;
+  case RestartOptions::Control::EnableFarARM64Jumps: RequiresFarARM64Jumps = true; break;
+  default: ERROR_AND_DIE_FMT("Unhandled Arm64 restart condition!");
+  }
+
+  uint32_t SSACount = IR->GetSSACount();
+  JumpTargets.clear();
+  CallReturnTargets.clear();
+  PendingJumpThunks.clear();
+  JumpTargets.resize(IR->GetHeader()->BlockCount, {});
+
   CodeData.EntryPoints.clear();
 
   // Fairly excessive buffer range to make sure we don't overflow
@@ -844,7 +855,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
 
   // Put the code header at the start of the data block.
   ARMEmitter::BackwardLabel JITCodeHeaderLabel {};
-  Bind(&JITCodeHeaderLabel);
+  (void)Bind(&JITCodeHeaderLabel);
   JITCodeHeader* CodeHeader = GetCursorAddress<JITCodeHeader*>();
   CursorIncrement(sizeof(JITCodeHeader));
 
@@ -892,7 +903,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
         if (PendingTargetLabel->Backward.Location) {
           EmitSuspendInterruptCheck();
         }
-        b(PendingTargetLabel);
+        b_OrRestart(PendingTargetLabel);
         PendingTargetLabel = nullptr;
       }
 
@@ -902,14 +913,14 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
         const auto IsReturnTarget = CallReturnTargets.try_emplace(Node).first;
         if (PendingTargetLabel) {
           // If there is a fallthrough branch to this block, skip over the entrypoint code.
-          b(Target);
+          b_OrRestart(Target);
         } else if (PendingCallReturnTargetLabel && PendingCallReturnTargetLabel != &IsReturnTarget->second) {
           // If we just emitted a call, but the block we're now emitting is not the return block so don't fallthrough.
-          b(PendingCallReturnTargetLabel);
+          b_OrRestart(PendingCallReturnTargetLabel);
         }
         PendingCallReturnTargetLabel = nullptr;
 
-        Bind(&IsReturnTarget->second);
+        BindOrRestart(&IsReturnTarget->second);
         CodeData.EntryPoints.emplace(BlockStartRIP, GetCursorAddress<uint8_t*>());
         DebugData->GuestOpcodes.push_back({BlockIROp->GuestEntryOffset, GetCursorAddress<uint8_t*>() - CodeData.BlockBegin});
 
@@ -918,12 +929,12 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
 
       if (PendingCallReturnTargetLabel) {
         // If there is still a pending call return target, then the block we're emitting is not the return block so don't fallthrough.
-        b(PendingCallReturnTargetLabel);
+        b_OrRestart(PendingCallReturnTargetLabel);
         PendingCallReturnTargetLabel = nullptr;
       }
       PendingTargetLabel = nullptr;
 
-      Bind(Target);
+      BindOrRestart(Target);
     }
 
     for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {
@@ -948,7 +959,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
     if (PendingTargetLabel->Backward.Location) {
       EmitSuspendInterruptCheck();
     }
-    b(PendingTargetLabel);
+    b_OrRestart(PendingTargetLabel);
   }
   PendingTargetLabel = nullptr;
 
@@ -959,21 +970,21 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
 
     ARMEmitter::ForwardLabel l_DoLink;
     uint64_t ThunkAddress = GetCursorAddress<uint64_t>();
-    Bind(&PendingJumpThunk.Label);
-    b(&l_DoLink);
+    BindOrRestart(&PendingJumpThunk.Label);
+    b_OrRestart(&l_DoLink);
     br(TMP1);
-    Bind(&l_DoLink);
+    BindOrRestart(&l_DoLink);
     ldr(TMP1, &l_ExitLink);
     blr(TMP1);
 
     // This is a ExitFunctionLinkData struct
-    Bind(&l_ExitLink);
+    BindOrRestart(&l_ExitLink);
     dc64(0);                                             // HostCode
     dc64(PendingJumpThunk.GuestRIP);                     // GuestRIP
     dc64(PendingJumpThunk.CallerAddress - ThunkAddress); // CallerOffset
   }
 
-  Bind(&l_ExitLink);
+  BindOrRestart(&l_ExitLink);
   dc64(ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker);
 
   // CodeSize not including the header or tail data.

FEXCore/Source/Interface/Core/JIT/JITClass.h

@@ -23,6 +23,7 @@ $end_info$
 #include <FEXCore/fextl/memory.h>
 #include <FEXCore/fextl/string.h>
 #include <FEXCore/fextl/vector.h>
+#include <FEXCore/Utils/LongJump.h>
 
 #include <CodeEmitter/Emitter.h>
 
@@ -66,6 +67,19 @@ private:
   const bool HostSupportsRPRES {};
   const bool HostSupportsAFP {};
 
+  struct RestartOptions {
+    FEXCore::UncheckedLongJump::JumpBuf RestartJump;
+    enum class Control : uint64_t {
+      Incoming = 0,
+      EnableFarARM64Jumps = 1,
+    };
+  };
+
+  // FEXCore makes assumptions in the JIT about certain conditions being true.
+  // In the rare case when those assumptions are broken, FEX needs to safely restart the JIT.
+  RestartOptions RestartControl {};
+  bool RequiresFarARM64Jumps {};
+
   ARMEmitter::BiDirectionalLabel* PendingTargetLabel {};
   ARMEmitter::BiDirectionalLabel* PendingCallReturnTargetLabel {};
   FEXCore::Context::ContextImpl* CTX {};
@@ -329,14 +343,179 @@ private:
   void EmitLinkedBranch(uint64_t GuestRIP, bool Call) {
     PendingJumpThunks.push_back({GetCursorAddress<uint64_t>(), GuestRIP, {}});
     auto& Thunk = PendingJumpThunks.back();
-    Bind(&Thunk.Label);
+    BindOrRestart(&Thunk.Label);
     if (Call) {
-      bl(&Thunk.Label);
+      bl_OrRestart(&Thunk.Label);
     } else {
-      b(&Thunk.Label);
+      b_OrRestart(&Thunk.Label);
     }
   }
 
+  // Restart helpers
+  template<ARMEmitter::IsLabel T>
+  void bl_OrRestart(T* Label) {
+    if (bl(Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    // We can support this but currently unnecessary.
+    ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+    FEXCore::UncheckedLongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void b_OrRestart(T* Label) {
+    if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    // We can support this but currently unnecessary.
+    ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+    FEXCore::UncheckedLongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void b_OrRestart(ARMEmitter::Condition Cond, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)b(InvertCondition(Cond), &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (b(Cond, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::UncheckedLongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void cbz_OrRestart(ARMEmitter::Size s, ARMEmitter::Register rt, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)cbnz(s, rt, &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (cbz(s, rt, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::UncheckedLongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void cbnz_OrRestart(ARMEmitter::Size s, ARMEmitter::Register rt, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)cbz(s, rt, &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (cbnz(s, rt, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::UncheckedLongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void tbz_OrRestart(ARMEmitter::Register rt, uint32_t Bit, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)tbnz(rt, Bit, &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (tbz(rt, Bit, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::UncheckedLongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void tbnz_OrRestart(ARMEmitter::Register rt, uint32_t Bit, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)tbz(rt, Bit, &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (tbnz(rt, Bit, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::UncheckedLongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void adr_OrRestart(ARMEmitter::Register rd, T* Label) {
+    if (adr(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    // We can support this but currently unnecessary.
+    ERROR_AND_DIE_FMT("Long ADR currently unsupported!");
+    FEXCore::UncheckedLongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void adrp_OrRestart(ARMEmitter::Register rd, T* Label) {
+    if (adrp(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    // We can support this but currently unnecessary.
+    ERROR_AND_DIE_FMT("Long ADRP currently unsupported!");
+    FEXCore::UncheckedLongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void BindOrRestart(T* Label) {
+    if (Bind(Label)) {
+      return;
+    }
+
+    if (RequiresFarARM64Jumps) {
+      // This should have been caught before this point.
+      ERROR_AND_DIE_FMT("Unhandled long bind");
+      return;
+    }
+
+    FEXCore::UncheckedLongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
   // This is purely a debugging aid for developers to see if they are in JIT code space when inspecting raw memory
   void EmitDetectionString();
   IR::RegisterAllocationPass* RAPass {};

FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp

@@ -912,7 +912,7 @@ DEF_OP(VLoadVectorMasked) {
 
       // If the sign bit is zero then skip the load
       ARMEmitter::ForwardLabel Skip {};
-      tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
+      (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
       // Do the gather load for this element into the destination
       switch (IROp->ElementSize) {
       case IR::OpSize::i8Bit: ld1<ARMEmitter::SubRegSize::i8Bit>(TempDst.Q(), i, TempMemReg); break;
@@ -923,7 +923,7 @@ DEF_OP(VLoadVectorMasked) {
       default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, IROp->ElementSize); return;
       }
 
-      Bind(&Skip);
+      (void)Bind(&Skip);
 
       if ((i + 1) != NumElements) {
         // Handle register rename to save a move.
@@ -1013,7 +1013,7 @@ DEF_OP(VStoreVectorMasked) {
 
       // If the sign bit is zero then skip the load
       ARMEmitter::ForwardLabel Skip {};
-      tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
+      (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
       // Do the gather load for this element into the destination
       switch (IROp->ElementSize) {
       case IR::OpSize::i8Bit: st1<ARMEmitter::SubRegSize::i8Bit>(RegData.Q(), i, TempMemReg); break;
@@ -1024,7 +1024,7 @@ DEF_OP(VStoreVectorMasked) {
       default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, IROp->ElementSize); return;
       }
 
-      Bind(&Skip);
+      (void)Bind(&Skip);
 
       if ((i + 1) != NumElements) {
         // Handle register rename to save a move.
@@ -1102,7 +1102,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
     PerformMove(ElementSize, WorkingReg, MaskReg, i);
 
     // Skip if the mask's sign bit isn't set
-    tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
+    (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
 
     // Extract Index Element
     if ((IndexElement * IR::OpSizeToSize(VectorIndexSize)) >= 16) {
@@ -1140,7 +1140,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
     default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, ElementSize); FEX_UNREACHABLE;
     }
 
-    Bind(&Skip);
+    (void)Bind(&Skip);
   }
 
   if (NeedsDestTmp) {
@@ -1874,7 +1874,7 @@ DEF_OP(MemSet) {
 
   if (!DirectionIsInline) {
     // Backward or forwards implementation depends on flag
-    tbnz(DirectionReg, 1, &BackwardImpl);
+    (void)tbnz(DirectionReg, 1, &BackwardImpl);
   }
 
   auto MemStore = [this](auto Value, uint32_t OpSize, int32_t Size) {
@@ -1922,7 +1922,7 @@ DEF_OP(MemSet) {
     ARMEmitter::ForwardLabel DoneInternal {};
 
     // Early exit if zero count.
-    cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+    (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
     if (!IsAtomic) {
       ARMEmitter::ForwardLabel AgainInternal256Exit {};
@@ -1939,50 +1939,50 @@ DEF_OP(MemSet) {
       // Do this in two parts, to fallback to the byte by byte loop if size < 32, and to the
       // single copy loop if size < 64.
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal128Exit);
+      (void)tbnz(TMP1, 63, &AgainInternal128Exit);
 
       // Fill VTMP2 with the set pattern
       dup(SubRegSize, VTMP2.Q(), Value);
 
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal256Exit);
+      (void)tbnz(TMP1, 63, &AgainInternal256Exit);
 
-      Bind(&AgainInternal256);
+      (void)Bind(&AgainInternal256);
       stp<ARMEmitter::IndexType::POST>(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);
       stp<ARMEmitter::IndexType::POST>(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
-      tbz(TMP1, 63, &AgainInternal256);
+      (void)tbz(TMP1, 63, &AgainInternal256);
 
-      Bind(&AgainInternal256Exit);
+      (void)Bind(&AgainInternal256Exit);
       add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal128Exit);
-      Bind(&AgainInternal128);
+      (void)tbnz(TMP1, 63, &AgainInternal128Exit);
+      (void)Bind(&AgainInternal128);
       stp<ARMEmitter::IndexType::POST>(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbz(TMP1, 63, &AgainInternal128);
+      (void)tbz(TMP1, 63, &AgainInternal128);
 
-      Bind(&AgainInternal128Exit);
+      (void)Bind(&AgainInternal128Exit);
       add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
       if (Direction == -1) {
         add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);
       }
     }
 
-    Bind(&AgainInternal);
+    (void)Bind(&AgainInternal);
     if (IsAtomic) {
       MemStoreTSO(Value, OpSize, SizeDirection);
     } else {
       MemStore(Value, OpSize, SizeDirection);
     }
     sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 1);
-    cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
+    (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
 
-    Bind(&DoneInternal);
+    (void)Bind(&DoneInternal);
 
     if (SizeDirection >= 0) {
       switch (OpSize) {
@@ -2012,12 +2012,12 @@ DEF_OP(MemSet) {
       EmitMemset(Direction);
 
       if (Direction == 1) {
-        b(&Done);
-        Bind(&BackwardImpl);
+        (void)b(&Done);
+        (void)Bind(&BackwardImpl);
       }
     }
 
-    Bind(&Done);
+    (void)Bind(&Done);
     // Destination already set to the final pointer.
   }
 }
@@ -2067,7 +2067,7 @@ DEF_OP(MemCpy) {
 
   if (!DirectionIsInline) {
     // Backward or forwards implementation depends on flag
-    tbnz(DirectionReg, 1, &BackwardImpl);
+    (void)tbnz(DirectionReg, 1, &BackwardImpl);
   }
 
   auto MemCpy = [this](uint32_t OpSize, int32_t Size) {
@@ -2164,7 +2164,7 @@ DEF_OP(MemCpy) {
     ARMEmitter::ForwardLabel DoneInternal {};
 
     // Early exit if zero count.
-    cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+    (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
     if (!IsAtomic) {
       ARMEmitter::ForwardLabel AbsPos {};
@@ -2174,11 +2174,11 @@ DEF_OP(MemCpy) {
       ARMEmitter::BackwardLabel AgainInternal256 {};
 
       sub(ARMEmitter::Size::i64Bit, TMP4, TMP2, TMP3);
-      tbz(TMP4, 63, &AbsPos);
+      (void)tbz(TMP4, 63, &AbsPos);
       neg(ARMEmitter::Size::i64Bit, TMP4, TMP4);
-      Bind(&AbsPos);
+      (void)Bind(&AbsPos);
       sub(ARMEmitter::Size::i64Bit, TMP4, TMP4, 32);
-      tbnz(TMP4, 63, &AgainInternal);
+      (void)tbnz(TMP4, 63, &AgainInternal);
 
       if (Direction == -1) {
         sub(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);
@@ -2190,30 +2190,30 @@ DEF_OP(MemCpy) {
       // Do this in two parts, to fallback to the byte by byte loop if size < 32, and to the
       // single copy loop if size < 64.
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal128Exit);
+      (void)tbnz(TMP1, 63, &AgainInternal128Exit);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal256Exit);
+      (void)tbnz(TMP1, 63, &AgainInternal256Exit);
 
-      Bind(&AgainInternal256);
+      (void)Bind(&AgainInternal256);
       MemCpy(32, 32 * Direction);
       MemCpy(32, 32 * Direction);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
-      tbz(TMP1, 63, &AgainInternal256);
+      (void)tbz(TMP1, 63, &AgainInternal256);
 
-      Bind(&AgainInternal256Exit);
+      (void)Bind(&AgainInternal256Exit);
       add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal128Exit);
-      Bind(&AgainInternal128);
+      (void)tbnz(TMP1, 63, &AgainInternal128Exit);
+      (void)Bind(&AgainInternal128);
       MemCpy(32, 32 * Direction);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbz(TMP1, 63, &AgainInternal128);
+      (void)tbz(TMP1, 63, &AgainInternal128);
 
-      Bind(&AgainInternal128Exit);
+      (void)Bind(&AgainInternal128Exit);
       add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
       if (Direction == -1) {
         add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);
@@ -2221,16 +2221,16 @@ DEF_OP(MemCpy) {
       }
     }
 
-    Bind(&AgainInternal);
+    (void)Bind(&AgainInternal);
     if (IsAtomic) {
       MemCpyTSO(OpSize, SizeDirection);
     } else {
       MemCpy(OpSize, SizeDirection);
     }
     sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 1);
-    cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
+    (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
 
-    Bind(&DoneInternal);
+    (void)Bind(&DoneInternal);
 
     // Needs to use temporaries just in case of overwrite
     mov(TMP1, MemRegDest.X());
@@ -2288,11 +2288,11 @@ DEF_OP(MemCpy) {
     for (int32_t Direction : {1, -1}) {
       EmitMemcpy(Direction);
       if (Direction == 1) {
-        b(&Done);
-        Bind(&BackwardImpl);
+        (void)b(&Done);
+        (void)Bind(&BackwardImpl);
       }
     }
-    Bind(&Done);
+    (void)Bind(&Done);
     // Destination already set to the final pointer.
   }
 }

FEXCore/Source/Interface/Core/LookupCache.cpp

@@ -39,6 +39,8 @@ LookupCache::LookupCache(FEXCore::Context::ContextImpl* CTX)
   // We need one pointer per page of virtual memory
   // At 64GB of virtual memory this will allocate 128MB of virtual memory space
   PagePointer = reinterpret_cast<uintptr_t>(FEXCore::Allocator::VirtualAlloc(TotalCacheSize, false, false));
+  LOGMAN_THROW_A_FMT(PagePointer != -1ULL, "Failed to allocate PagePointer");
+
   FEXCore::Allocator::VirtualName("FEXMem_Lookup", reinterpret_cast<void*>(PagePointer),
                                   ctx->Config.VirtualMemSize / FEXCore::Utils::FEX_PAGE_SIZE * 8 + CODE_SIZE);
   CTX->SyscallHandler->MarkOvercommitRange(PagePointer, TotalCacheSize);
@@ -49,14 +51,11 @@ LookupCache::LookupCache(FEXCore::Context::ContextImpl* CTX)
   // We currently limit to 128MB of real memory for caching for the total cache size.
   // Can end up being inefficient if we compile a small number of blocks per page
   PageMemory = PagePointer + ctx->Config.VirtualMemSize / FEXCore::Utils::FEX_PAGE_SIZE * 8;
-  LOGMAN_THROW_A_FMT(PageMemory != -1ULL, "Failed to allocate page memory");
 
   // L1 Cache
   L1Pointer = PageMemory + CODE_SIZE;
   FEXCore::Allocator::VirtualName("FEXMem_Lookup_L1", reinterpret_cast<void*>(L1Pointer), MAX_L1_SIZE);
 
-  LOGMAN_THROW_A_FMT(L1Pointer != -1ULL, "Failed to allocate L1Pointer");
-
   VirtualMemSize = ctx->Config.VirtualMemSize;
 
   if (DynamicL1Cache()) {
@@ -76,7 +75,7 @@ LookupCache::~LookupCache() {
   // These will get freed when their memory allocators are deallocated.
 }
 
-void LookupCache::ClearL2Cache(const FEXCore::LookupCacheWriteLockToken& lk) {
+void LookupCache::ClearL2Cache(const FEXCore::LookupCacheReadLockToken& lk) {
   // Clear out the page memory
   // PagePointer and PageMemory are sequential with each other. Clear both at once.
   FEXCore::Allocator::VirtualDontNeed(reinterpret_cast<void*>(PagePointer),

FEXCore/Source/Interface/Core/LookupCache.h

@@ -3,6 +3,8 @@
 #include "Interface/Context/Context.h"
 #include <FEXCore/Utils/LogManager.h>
 #include <FEXCore/Utils/SHMStats.h>
+#include "Utils/WritePriorityMutex.h"
+
 #include <FEXCore/fextl/map.h>
 #include <FEXCore/fextl/memory_resource.h>
 #include <FEXCore/fextl/robin_map.h>
@@ -15,22 +17,35 @@
 #include <mutex>
 
 namespace FEXCore {
-
 struct LookupCacheWriteLockToken {
 private:
   // Only constructible by GuestToHostMap
   friend struct GuestToHostMap;
-  LookupCacheWriteLockToken(std::mutex& Mutex)
+  LookupCacheWriteLockToken(FEXCore::Utils::WritePriorityMutex::Mutex& Mutex)
     : Lock {Mutex} {}
-  std::lock_guard<std::mutex> Lock;
+  std::lock_guard<FEXCore::Utils::WritePriorityMutex::Mutex> Lock;
+};
+
+struct LookupCacheReadLockToken {
+private:
+  // Only constructible by GuestToHostMap
+  friend struct GuestToHostMap;
+  LookupCacheReadLockToken(FEXCore::Utils::WritePriorityMutex::Mutex& Mutex)
+    : Lock {Mutex} {}
+  std::shared_lock<FEXCore::Utils::WritePriorityMutex::Mutex> Lock;
 };
 
 struct GuestToHostMap {
-  std::mutex WriteLock;
+  FEXCore::Utils::WritePriorityMutex::Mutex Lock {};
 
   [[nodiscard]]
   LookupCacheWriteLockToken AcquireWriteLock() {
-    return LookupCacheWriteLockToken {WriteLock};
+    return LookupCacheWriteLockToken {Lock};
+  }
+
+  [[nodiscard]]
+  LookupCacheReadLockToken AcquireReadLock() {
+    return LookupCacheReadLockToken {Lock};
   }
 
   struct BlockLinkTag {
@@ -75,7 +90,7 @@ struct GuestToHostMap {
     BlockList[Address] = (uintptr_t)HostCode;
   }
 
-  std::optional<uintptr_t> FindBlock(uint64_t Address, const LookupCacheWriteLockToken&) {
+  std::optional<uintptr_t> FindBlock(uint64_t Address, const LookupCacheReadLockToken&) {
     auto HostCode = BlockList.find(Address);
     if (HostCode == BlockList.end()) {
       return std::nullopt;
@@ -144,7 +159,7 @@ public:
     {
       std::optional<FEXCore::SHMStats::AccumulationBlock<uint64_t>> LockTime(
         Thread->ThreadStats ? &Thread->ThreadStats->AccumulatedCacheReadLockTime : nullptr);
-      auto lk = Shared->AcquireWriteLock();
+      auto lk = Shared->AcquireReadLock();
       LockTime.reset();
 
       if (!DisableL2Cache()) {
@@ -302,7 +317,7 @@ public:
   }
 
   void ClearCache(const LookupCacheWriteLockToken&);
-  void ClearL2Cache(const LookupCacheWriteLockToken&);
+  void ClearL2Cache(const LookupCacheReadLockToken&);
   void ClearThreadLocalCaches(const LookupCacheWriteLockToken&);
 
   uintptr_t GetL1Pointer() const {
@@ -330,7 +345,7 @@ public:
   }
 
 private:
-  void CacheBlockMapping(uint64_t Address, uintptr_t HostCode, const LookupCacheWriteLockToken& lk) {
+  void CacheBlockMapping(uint64_t Address, uintptr_t HostCode, const LookupCacheReadLockToken& lk) {
     // Do L1
     auto& L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1PointerMask];
     L1Entry.GuestCode = Address;

FEXCore/Source/Interface/Core/OpcodeDispatcher/X87.cpp

@@ -17,7 +17,6 @@ $end_info$
 #include <FEXCore/Utils/LogManager.h>
 #include <FEXCore/Utils/FPState.h>
 
-#include <cmath>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -69,7 +68,7 @@ void OpDispatchBuilder::FLD(OpcodeArgs, IR::OpSize Width) {
   if (Width == OpSize::i32Bit || Width == OpSize::i64Bit) {
     ConvertedData = _F80CVTTo(Data, ReadWidth);
   }
-  _PushStack(ConvertedData, Data, ReadWidth, true);
+  _PushStack(ConvertedData, Data, ReadWidth);
 }
 
 // Float LoaD operation with memory operand
@@ -81,7 +80,7 @@ void OpDispatchBuilder::FBLD(OpcodeArgs) {
   // Read from memory
   Ref Data = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::f80Bit, Op->Flags);
   Ref ConvertedData = _F80BCDLoad(Data);
-  _PushStack(ConvertedData, Data, OpSize::i128Bit, true);
+  _PushStack(ConvertedData, Data, OpSize::i128Bit);
 }
 
 void OpDispatchBuilder::FBSTP(OpcodeArgs) {
@@ -93,7 +92,7 @@ void OpDispatchBuilder::FBSTP(OpcodeArgs) {
 void OpDispatchBuilder::FLD_Const(OpcodeArgs, NamedVectorConstant K) {
   // Update TOP
   Ref Data = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, K);
-  _PushStack(Data, Data, OpSize::i128Bit, true);
+  _PushStack(Data, Data, OpSize::i128Bit);
 }
 
 void OpDispatchBuilder::FILD(OpcodeArgs) {
@@ -124,7 +123,7 @@ void OpDispatchBuilder::FILD(OpcodeArgs) {
   auto upper = _Or(OpSize::i64Bit, sign, zeroed_exponent);
 
   Ref ConvertedData = _VLoadTwoGPRs(shifted, upper);
-  _PushStack(ConvertedData, Data, ReadWidth, false);
+  _PushStack(ConvertedData, Invalid(), ReadWidth);
 }
 
 void OpDispatchBuilder::FST(OpcodeArgs, IR::OpSize Width) {
@@ -132,7 +131,7 @@ void OpDispatchBuilder::FST(OpcodeArgs, IR::OpSize Width) {
   AddressMode A = DecodeAddress(Op, Op->Dest, MemoryAccessType::DEFAULT, false);
 
   A = SelectAddressMode(this, A, GetGPROpSize(), CTX->HostFeatures.SupportsTSOImm9, false, false, Width);
-  _StoreStackMem(SourceSize, Width, A.Base, A.Index, OpSize::iInvalid, A.IndexType, A.IndexScale, /*Float=*/true);
+  _StoreStackMem(SourceSize, Width, A.Base, A.Index, OpSize::iInvalid, A.IndexType, A.IndexScale);
 
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
@@ -878,8 +877,8 @@ void OpDispatchBuilder::X87FXTRACT(OpcodeArgs) {
   _PopStackDestroy();
   auto Exp = _F80XTRACT_EXP(Top);
   auto Sig = _F80XTRACT_SIG(Top);
-  _PushStack(Exp, Exp, OpSize::f80Bit, true);
-  _PushStack(Sig, Sig, OpSize::f80Bit, true);
+  _PushStack(Exp, Invalid(), OpSize::f80Bit);
+  _PushStack(Sig, Invalid(), OpSize::f80Bit);
 }
 
 } // namespace FEXCore::IR

FEXCore/Source/Interface/Core/OpcodeDispatcher/X87F64.cpp

@@ -68,7 +68,7 @@ void OpDispatchBuilder::FLDF64(OpcodeArgs, IR::OpSize Width) {
   } else if (Width == OpSize::f80Bit) {
     ConvertedData = _F80CVT(OpSize::i64Bit, Data);
   }
-  _PushStack(ConvertedData, Data, ReadWidth, true);
+  _PushStack(ConvertedData, Data, ReadWidth);
 }
 
 void OpDispatchBuilder::FBLDF64(OpcodeArgs) {
@@ -76,7 +76,7 @@ void OpDispatchBuilder::FBLDF64(OpcodeArgs) {
   Ref Data = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::f80Bit, Op->Flags);
   Ref ConvertedData = _F80BCDLoad(Data);
   ConvertedData = _F80CVT(OpSize::i64Bit, ConvertedData);
-  _PushStack(ConvertedData, Data, OpSize::i64Bit, true);
+  _PushStack(ConvertedData, Data, OpSize::i64Bit);
 }
 
 void OpDispatchBuilder::FBSTPF64(OpcodeArgs) {
@@ -88,7 +88,7 @@ void OpDispatchBuilder::FBSTPF64(OpcodeArgs) {
 
 void OpDispatchBuilder::FLDF64_Const(OpcodeArgs, uint64_t Num) {
   auto Data = _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, Constant(Num));
-  _PushStack(Data, Data, OpSize::i64Bit, true);
+  _PushStack(Data, Data, OpSize::i64Bit);
 }
 
 void OpDispatchBuilder::FILDF64(OpcodeArgs) {
@@ -100,7 +100,7 @@ void OpDispatchBuilder::FILDF64(OpcodeArgs) {
     Data = _Sbfe(OpSize::i64Bit, IR::OpSizeAsBits(ReadWidth), 0, Data);
   }
   auto ConvertedData = _Float_FromGPR_S(OpSize::i64Bit, ReadWidth == OpSize::i32Bit ? OpSize::i32Bit : OpSize::i64Bit, Data);
-  _PushStack(ConvertedData, Data, ReadWidth, false);
+  _PushStack(ConvertedData, Invalid(), ReadWidth);
 }
 
 void OpDispatchBuilder::FISTF64(OpcodeArgs, bool Truncate) {
@@ -397,7 +397,7 @@ void OpDispatchBuilder::X87FXTRACTF64(OpcodeArgs) {
   Ref Exp = _NZCVSelectV(OpSize::i64Bit, CondClass::EQ, ExpZV, ExpNZV);
 
   _PopStackDestroy();
-  _PushStack(Exp, Exp, OpSize::i64Bit, true);
-  _PushStack(Sig, Sig, OpSize::i64Bit, true);
+  _PushStack(Exp, Invalid(), OpSize::i64Bit);
+  _PushStack(Sig, Invalid(), OpSize::i64Bit);
 }
 } // namespace FEXCore::IR

FEXCore/Source/Interface/IR/IR.json

@@ -2818,17 +2818,13 @@
         "X87": true,
         "HasSideEffects": true
       },
-      "PushStack FPR:$X80Src, SSA:$OriginalValue, OpSize:$LoadSize, i1:$Float": {
+      "PushStack FPR:$X80Src, FPR:$OriginalValue, OpSize:$LoadSize": {
         "Desc": [
           "Pushes the provided X80Src source on to the x87 stack.",
-          "Tracks OriginalValue as the original value of X80Src.",
+          "Tracks OriginalValue as the original value of X80Src. OriginalValue can be Invalid() in which case no tracking is done.",
           "Opsize is 128bit for F80 values, 64-bit for low precision.",
           "LoadSize the original load size, i.e. of size of OriginalValue.",
-          "Float: 80-bit, 64-bit, 32-bit",
-          "Int: 64-bit, 32-bit, 16-bit"
-        ],
-        "EmitValidation": [
-          "WalkFindRegClass($OriginalValue) == RegClass::FPR || WalkFindRegClass($OriginalValue) == RegClass::GPR"
+          "Float: 80-bit, 64-bit, 32-bit"
         ],
         "HasSideEffects": true,
         "X87": true
@@ -2840,13 +2836,12 @@
         "HasSideEffects": true,
         "X87": true
       },
-      "StoreStackMem OpSize:$SourceSize, OpSize:$StoreSize, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale, i1:$Float": {
+      "StoreStackMem OpSize:$SourceSize, OpSize:$StoreSize, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale": {
         "Desc": [
           "Takes the top value off the x87 stack and stores it to memory.",
           "SourceSize is 128bit for F80 values, 64-bit for low precision.",
           "StoreSize is the store size for conversion:",
-          "Float: 80-bit, 64-bit, or 32-bit",
-          "Int: 64-bit, 32-bit, 16-bit"
+          "Float: 80-bit, 64-bit, or 32-bit"
         ],
         "HasSideEffects": true,
         "X87": true

FEXCore/Source/Interface/IR/Passes/x87StackOptimizationPass.cpp

@@ -6,7 +6,6 @@
 #include "Interface/IR/PassManager.h"
 #include "FEXCore/IR/IR.h"
 #include "FEXCore/Utils/Profiler.h"
-#include "FEXCore/Utils/MathUtils.h"
 #include "FEXCore/Core/HostFeatures.h"
 #include "Interface/Core/Addressing.h"
 
@@ -293,10 +292,9 @@ private:
     StackMemberInfo() {}
     StackMemberInfo(Ref Data)
       : StackDataNode(Data) {}
-    StackMemberInfo(Ref Data, Ref Source, OpSize Size, bool Float)
+    StackMemberInfo(Ref Data, Ref Source, OpSize Size)
       : StackDataNode(Data)
-      , Source({Size, Source})
-      , InterpretAsFloat(Float) {}
+      , Source({Size, Source}) {}
     Ref StackDataNode {}; // Reference to the data in the Stack.
                           // This is the source data node in the stack format, possibly converted to 64/80 bits.
     struct StackMemberData final {
@@ -306,7 +304,6 @@ private:
     // Tuple is only valid if we have information about the Source of the Stack Data Node.
     // In it's valid then OpSize is the original source size and Ref is the original source node.
     std::optional<StackMemberData> Source {};
-    bool InterpretAsFloat {false}; // True if this is a floating point value, false if integer
   };
 
   // StackData, TopCache need to be always properly set to ensure
@@ -927,8 +924,13 @@ void X87StackOptimization::Run(IREmitter* Emit) {
           StoreStackValueAtOffset_Slow(SourceNode);
         } else {
           auto* SourceNode = CurrentIR.GetNode(Op->X80Src);
-          auto* OriginalNode = CurrentIR.GetNode(Op->OriginalValue);
-          StackData.push(StackMemberInfo {SourceNode, OriginalNode, Op->LoadSize, Op->Float});
+          if (Op->OriginalValue.IsInvalid()) {
+            // No original value to track - just push the converted data
+            StackData.push(StackMemberInfo {SourceNode});
+          } else {
+            auto* OriginalNode = CurrentIR.GetNode(Op->OriginalValue);
+            StackData.push(StackMemberInfo {SourceNode, OriginalNode, Op->LoadSize});
+          }
         }
         break;
       }
@@ -993,9 +995,8 @@ void X87StackOptimization::Run(IREmitter* Emit) {
         // str w2, [x1]
         // or similar. As long as the source size and dest size are one and the same.
         // This will avoid any conversions between source and stack element size and conversion back.
-        if (!SlowPath && Value->Source && Value->Source->Size == Op->StoreSize && Value->InterpretAsFloat) {
-          const auto ClassType = Value->InterpretAsFloat ? RegClass::FPR : RegClass::GPR;
-          IREmit->_StoreMem(ClassType, Op->StoreSize, Value->Source->Node, AddrNode, Offset, Align, OffsetType, OffsetScale);
+        if (!SlowPath && Value->Source && Value->Source->Size == Op->StoreSize) {
+          IREmit->_StoreMemFPR(Op->StoreSize, Value->Source->Node, AddrNode, Offset, Align, OffsetType, OffsetScale);
           break;
         }
 
@@ -1035,11 +1036,26 @@ void X87StackOptimization::Run(IREmitter* Emit) {
       case OP_F80STACKXCHANGE: {
         const auto* Op = IROp->C<IROp_F80StackXchange>();
         auto Offset = Op->SrcStack;
-        Ref ValueTop = LoadStackValue();
-        Ref ValueOffset = LoadStackValue(Offset);
 
-        StoreStackValue(ValueOffset);
-        StoreStackValue(ValueTop, Offset);
+        if (Offset == 0) {
+          // No-op
+          break;
+        }
+
+        const auto [ValidTop, StackMemberTop] = StackData.top(0);
+        const auto [ValidOffset, StackMemberOffset] = StackData.top(Offset);
+
+        if (ValidTop != StackSlot::VALID || ValidOffset != StackSlot::VALID) {
+          // Slow path: do actual memory operations
+          Ref ValueTop = LoadStackValue();
+          Ref ValueOffset = LoadStackValue(Offset);
+          StoreStackValue(ValueOffset);
+          StoreStackValue(ValueTop, Offset);
+        } else {
+          // Fast path: swap complete StackMemberInfo preserving Source metadata
+          StackData.setTop(StackMemberOffset, 0);
+          StackData.setTop(StackMemberTop, Offset);
+        }
         break;
       }
 

FEXCore/Source/Utils/Allocator.cpp

@@ -140,10 +140,7 @@ void ClearHooks() {
   FEXCore::Allocator::mmap = ::mmap;
   FEXCore::Allocator::munmap = ::munmap;
 
-  // XXX: This is currently a leak.
-  // We can't work around this yet until static initializers that allocate memory are completely removed from our codebase
-  // Luckily we only remove this on process shutdown, so the kernel will do the cleanup for us
-  Alloc64.release();
+  Alloc::OSAllocator::ReleaseAllocatorWorkaround(std::move(Alloc64));
 }
 #pragma GCC diagnostic pop
 

FEXCore/Source/Utils/Allocator/64BitAllocator.cpp

@@ -207,7 +207,7 @@ OSAllocator_64Bit::LiveVMARegion* OSAllocator_64Bit::FindLiveRegionForAddress(ui
     uintptr_t RegionBegin = (*it)->SlabInfo->Base;
     uintptr_t RegionEnd = RegionBegin + (*it)->SlabInfo->RegionSize;
 
-    if (Addr >= RegionBegin && Addr < RegionEnd) {
+    if (Addr >= RegionBegin && AddrEnd < RegionEnd) {
       LiveRegion = *it;
       // Leave our loop
       break;
@@ -405,14 +405,18 @@ again:
     // Mark the pages as used
     uintptr_t RegionBegin = LiveRegion->SlabInfo->Base;
     uintptr_t MappedBegin = (AllocatedOffset - RegionBegin) >> FEXCore::Utils::FEX_PAGE_SHIFT;
+    size_t PagesSet {};
 
     for (size_t i = 0; i < NumberOfPages; ++i) {
-      LiveRegion->UsedPages.Set(MappedBegin + i);
+      PagesSet += LiveRegion->UsedPages.TestAndSet(MappedBegin + i) == false;
     }
 
     // Change our last allocation region
     LiveRegion->LastPageAllocation = MappedBegin + NumberOfPages;
-    LiveRegion->FreeSpace -= length;
+    LiveRegion->FreeSpace -= PagesSet * FEXCore::Utils::FEX_PAGE_SIZE;
+    LOGMAN_THROW_A_FMT(LiveRegion->FreeSpace <= LiveRegion->SlabInfo->RegionSize,
+                       "Corrupt LiveRegion free space! 0x{:x} > 0x{:x}. After allocating 0x{:x} (0x{:x} overlapped)", LiveRegion->FreeSpace,
+                       LiveRegion->SlabInfo->RegionSize, length, PagesSet);
   }
 
   if (!AllocatedOffset) {

FEXCore/Source/Utils/Allocator/FlexBitSet.h

@@ -27,6 +27,11 @@ struct FlexBitSet final {
     Memory[Element / MinimumSizeBits] &= ~(1ULL << (Element % MinimumSizeBits));
     return Value;
   }
+  bool TestAndSet(size_t Element) {
+    bool Value = Get(Element);
+    Memory[Element / MinimumSizeBits] |= (1ULL << (Element % MinimumSizeBits));
+    return Value;
+  }
   void Set(size_t Element) {
     Memory[Element / MinimumSizeBits] |= (1ULL << (Element % MinimumSizeBits));
   }
@@ -70,12 +75,17 @@ struct FlexBitSet final {
   template<bool WantUnset>
   BitsetScanResults BackwardScanForRange(size_t BeginningElement, size_t ElementCount, size_t MinimumElement) {
     bool FoundHole {};
-    for (size_t CurrentPage = BeginningElement; CurrentPage >= (MinimumElement + ElementCount);) {
+
+    // Final element to iterate to.
+    const size_t FinalElement = MinimumElement + ElementCount - 1;
+
+    for (size_t CurrentPage = BeginningElement; CurrentPage >= FinalElement;) {
       size_t Remaining = ElementCount;
-      LOGMAN_THROW_A_FMT(Remaining <= CurrentPage, "Scanning less than available range");
+      LOGMAN_THROW_A_FMT(CurrentPage <= BeginningElement && CurrentPage >= FinalElement, "BackwardScanForRange: Scanning less than "
+                                                                                         "available range");
 
       while (Remaining) {
-        if (this->Get(CurrentPage - Remaining) == WantUnset) {
+        if (this->Get(CurrentPage - Remaining + 1) == WantUnset) {
           // Has an intersecting range
           break;
         }
@@ -92,7 +102,7 @@ struct FlexBitSet final {
         CurrentPage -= Remaining;
       } else {
         // We have a slab range
-        return BitsetScanResults {CurrentPage - ElementCount, FoundHole};
+        return BitsetScanResults {CurrentPage - ElementCount + 1, FoundHole};
       }
     }
 
@@ -108,11 +118,15 @@ struct FlexBitSet final {
   BitsetScanResults ForwardScanForRange(size_t BeginningElement, size_t ElementCount, size_t ElementsInSet) {
     bool FoundHole {};
 
-    for (size_t CurrentElement = BeginningElement; CurrentElement < (ElementsInSet - ElementCount);) {
+    // Final element to iterate to.
+    const size_t FinalElement = ElementsInSet - ElementCount + 1;
+
+    for (size_t CurrentElement = BeginningElement; CurrentElement <= FinalElement;) {
       // If we have enough free space, check if we have enough free pages that are contiguous
       size_t Remaining = ElementCount;
 
-      LOGMAN_THROW_A_FMT((CurrentElement + Remaining - 1) < ElementsInSet, "Scanning less than available range");
+      LOGMAN_THROW_A_FMT(CurrentElement >= BeginningElement && CurrentElement <= FinalElement, "ForwardScanForRange: Scanning less than "
+                                                                                               "available range");
 
       while (Remaining) {
         if (this->Get(CurrentElement + Remaining - 1) == WantUnset) {

FEXCore/Source/Utils/Allocator/HostAllocator.h

@@ -53,4 +53,12 @@ public:
 namespace Alloc::OSAllocator {
 fextl::unique_ptr<Alloc::HostAllocator> Create64BitAllocator();
 fextl::unique_ptr<Alloc::HostAllocator> Create64BitAllocatorWithRegions(fextl::vector<FEXCore::Allocator::MemoryRegion>& Regions);
+static inline void ReleaseAllocatorWorkaround(fextl::unique_ptr<Alloc::HostAllocator> Allocator) {
+  // XXX: This is currently a leak.
+  // We can't work around this yet until static initializers that allocate memory are completely removed from our codebase
+  // The allocator is also intrusively allocated, so the unique_ptr tries to double free the HostAllocator object.
+  // Luckily we only remove this on process shutdown, so the kernel will do the cleanup for us
+  Allocator.release();
+}
+
 } // namespace Alloc::OSAllocator

FEXCore/Source/Utils/LongJump.cpp

@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: MIT
+#include <FEXCore/Utils/LongJump.h>
+
+namespace FEXCore::UncheckedLongJump {
+#if defined(_M_ARM_64)
+[[nodiscard]]
+FEX_DEFAULT_VISIBILITY FEX_NAKED uint64_t SetJump(JumpBuf& Buffer) {
+  __asm volatile(R"(
+      // x0 contains the jumpbuffer
+      stp x19, x20, [x0, #( 0 * 8)];
+      stp x21, x22, [x0, #( 2 * 8)];
+      stp x23, x24, [x0, #( 4 * 8)];
+      stp x25, x26, [x0, #( 6 * 8)];
+      stp x27, x28, [x0, #( 8 * 8)];
+      stp x29, x30, [x0, #(10 * 8)];
+
+      // FPRs
+      stp d8,   d9, [x0, #(12 * 8)];
+      stp d10, d11, [x0, #(14 * 8)];
+      stp d12, d13, [x0, #(16 * 8)];
+      stp d14, d15, [x0, #(18 * 8)];
+
+      // Move SP in to a temporary to store.
+      mov x1, sp;
+      str x1,  [x0, #(20 * 8)];
+
+      // Return zero to signify this is the SetJump.
+      mov x0, #0;
+      ret;
+    )" ::
+                   : "memory");
+}
+
+[[noreturn]]
+FEX_DEFAULT_VISIBILITY FEX_NAKED void LongJump(JumpBuf& Buffer, uint64_t Value) {
+  __asm volatile(R"(
+      // x0 contains the jumpbuffer
+      ldp x19, x20, [x0, #( 0 * 8)];
+      ldp x21, x22, [x0, #( 2 * 8)];
+      ldp x23, x24, [x0, #( 4 * 8)];
+      ldp x25, x26, [x0, #( 6 * 8)];
+      ldp x27, x28, [x0, #( 8 * 8)];
+      ldp x29, x30, [x0, #(10 * 8)];
+
+      // FPRs
+      ldp d8,   d9, [x0, #(12 * 8)];
+      ldp d10, d11, [x0, #(14 * 8)];
+      ldp d12, d13, [x0, #(16 * 8)];
+      ldp d14, d15, [x0, #(18 * 8)];
+
+      // Load SP in to temporary then move
+      ldr x0,  [x0, #(20 * 8)];
+      mov sp, x0;
+
+      // Move value in to result register
+      mov x0, x1;
+      ret;
+    )" ::
+                   : "memory");
+}
+#else
+[[nodiscard]]
+FEX_DEFAULT_VISIBILITY FEX_NAKED uint64_t SetJump(JumpBuf& Buffer) {
+  __asm volatile(R"(
+    .intel_syntax noprefix;
+    // rdi contains the jumpbuffer
+    mov [rdi + (0 * 8)], rbx;
+    mov [rdi + (1 * 8)], rsp;
+    mov [rdi + (2 * 8)], rbp;
+    mov [rdi + (3 * 8)], r12;
+    mov [rdi + (4 * 8)], r13;
+    mov [rdi + (5 * 8)], r14;
+    mov [rdi + (6 * 8)], r15;
+
+    // Return address is on the stack, load it and store
+    mov rsi, [rsp];
+    mov [rdi + (7 * 8)], rsi;
+
+    // Return zero to signify this is the SetJump.
+    mov rax, 0;
+    ret;
+
+    .att_syntax prefix;
+    )" ::
+                   : "memory");
+}
+
+[[noreturn]]
+FEX_DEFAULT_VISIBILITY FEX_NAKED void LongJump(JumpBuf& Buffer, uint64_t Value) {
+  __asm volatile(R"(
+    .intel_syntax noprefix;
+    // rdi contains the jumpbuffer
+    mov rbx, [rdi + (0 * 8)];
+    mov rsp, [rdi + (1 * 8)];
+    mov rbp, [rdi + (2 * 8)];
+    mov r12, [rdi + (3 * 8)];
+    mov r13, [rdi + (4 * 8)];
+    mov r14, [rdi + (5 * 8)];
+    mov r15, [rdi + (6 * 8)];
+
+    // Move value in to result register
+    mov rax, rsi;
+
+    // Pop the dead return address off the stack
+    pop rsi;
+
+    // Load the original return address from the jumpbuffer
+    mov rsi, [rdi + (7 * 8)];
+
+    // Return using a jump
+    jmp rsi;
+
+    .att_syntax prefix;
+    )" ::
+                   : "memory");
+}
+
+#endif
+} // namespace FEXCore::UncheckedLongJump

FEXCore/Source/Utils/SpinWaitLock.h

@@ -1,4 +1,6 @@
 // SPDX-License-Identifier: MIT
+#pragma once
+
 #include <atomic>
 #include <chrono>
 #include <mutex>
@@ -184,6 +186,26 @@ template bool Wait<uint16_t>(uint16_t*, uint16_t, const std::chrono::nanoseconds
 template bool Wait<uint32_t>(uint32_t*, uint32_t, const std::chrono::nanoseconds&);
 template bool Wait<uint64_t>(uint64_t*, uint64_t, const std::chrono::nanoseconds&);
 
+template<typename T>
+static inline T OneShotWFEBitComparison(T* Futex, T Mask, T Comp) {
+  auto AtomicFutex = std::atomic_ref<T>(*Futex);
+  T Result = AtomicFutex.load();
+
+  // Early exit if possible.
+  if ((Result & Mask) == Comp) {
+    return Result;
+  }
+
+  Result = LoadExclusive(Futex);
+  if ((Result & Mask) == Comp) {
+    return Result;
+  }
+
+  // Waits for write and returns result.
+  Result = WFELoadAtomic(Futex);
+  return Result;
+}
+
 #else
 template<typename T, typename TT>
 static inline void Wait(T* Futex, TT ExpectedValue) {

FEXCore/Source/Utils/WritePriorityMutex.h

@@ -0,0 +1,384 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+#include <atomic>
+#include <cstdint>
+
+#if !defined(_WIN32)
+#include <linux/futex.h> /* Definition of FUTEX_* constants */
+#include <sys/syscall.h> /* Definition of SYS_* constants */
+#include <unistd.h>
+#else
+#include <synchapi.h>
+#endif
+
+#include <FEXCore/Utils/LogManager.h>
+
+#include "Utils/SpinWaitLock.h"
+
+namespace FEXCore::Utils::WritePriorityMutex {
+
+#if !defined(_WIN32)
+// A custom mutex that prioritizes exclusive locks.
+// In highly contested scenarios, this can help minimize overall contention time.
+//
+// Features:
+//  - Up to 32767 pending exclusive locks ("writers")
+//  - Up to 32767 pending shared_locks ("readers")
+//  - Low-overhead waiting via WFE with a fallback to futex on timeout
+//  - Direct writer->reader hand-off and vice-versa to further reduce overhead
+//
+// Trade-offs:
+//  - No guaranteed order of wake-ups besides prioritizing writers
+//  - No support for recursive locking
+//  - We can't use FUTEX_LOCK_PI to enable priority inheritance
+class Mutex final {
+public:
+  Mutex() = default;
+
+  // Move-only type
+  Mutex(const Mutex&) = delete;
+  Mutex& operator=(const Mutex&) = delete;
+  Mutex(Mutex&& rhs) = delete;
+  Mutex& operator=(Mutex&&) = delete;
+
+  void lock() {
+    // Try a non-blocking lock first.
+    if (try_lock()) {
+      return;
+    }
+
+    // Try a quick WFE write-lock.
+    if (Attempt_WFE_WriteLock()) {
+      return;
+    }
+
+    // Still couldn't get it. Start waiting.
+    auto AtomicFutex = std::atomic_ref<uint32_t>(Futex);
+
+    uint32_t Expected {};
+    uint32_t Desired {};
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
+    Expected = AtomicFutex.load(std::memory_order_relaxed);
+    do {
+      // Increment the number of write waiters.
+      Desired = Expected + WRITE_WAITER_INCREMENT;
+
+      LOGMAN_THROW_A_FMT((Desired & WRITE_WAITER_COUNT_MASK) != 0, "Overflow in write-waiters!");
+    } while (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire) == false);
+#else
+    Expected = AtomicFutex.fetch_add(WRITE_WAITER_INCREMENT);
+    Desired = Expected + WRITE_WAITER_INCREMENT;
+#endif
+
+    // Thread added to waiter list.
+    Expected = Desired;
+
+    while (true) {
+      bool Sleep = false;
+
+      do {
+        if ((Expected & WRITE_OWNED_BIT) == 0 && (Expected & READ_OWNER_COUNT_MASK) == 0) {
+          // If not write-owned, and no read-owners, try to acquire.
+          LOGMAN_THROW_A_FMT((Expected & WRITE_WAITER_COUNT_MASK) != 0, "Underflow in write-waiters!");
+
+          // Add write-owned bit.
+          Desired = Expected | WRITE_OWNED_BIT;
+
+          // Remove ourselves from the wait list.
+          Desired -= WRITE_WAITER_INCREMENT;
+
+          Sleep = false;
+        } else {
+          // Already write-owned or read-locked. Go to sleep.
+          Desired = Expected;
+          Sleep = true;
+          break;
+        }
+
+      } while (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire) == false);
+
+      if (!Sleep) {
+        // Acquired early.
+        LOGMAN_THROW_A_FMT((Desired & WRITE_OWNED_BIT) == WRITE_OWNED_BIT, "Somehow acquired a write-lock without it being set!");
+        return;
+      }
+      FutexWaitForWriteAvailable(Desired);
+
+      Expected = AtomicFutex.load(std::memory_order_relaxed);
+    }
+  }
+
+  void lock_shared() {
+    // Try an uncontended lock first.
+    if (try_lock_shared()) {
+      return;
+    }
+
+    // Try a quick WFE read-lock.
+    if (Attempt_WFE_ReadLock()) {
+      return;
+    }
+
+    auto AtomicFutex = std::atomic_ref<uint32_t>(Futex);
+
+    uint32_t Expected = AtomicFutex.load(std::memory_order_relaxed);
+    uint32_t Desired {};
+
+    while (true) {
+
+      bool Sleep = false;
+      do {
+        if ((Expected & WRITE_OWNED_BIT) == 0 && (Expected & WRITE_WAITER_COUNT_MASK) == 0) {
+          // If no write-owner and no write-waiting, try and acquire.
+
+          Desired = Expected + READ_OWNER_INCREMENT;
+          LOGMAN_THROW_A_FMT((Desired & READ_OWNER_COUNT_MASK) != 0, "Overflow in read-owners!");
+          Sleep = false;
+        } else {
+          // Waiting for lock to become available. Add to waiters.
+          Desired = Expected | READ_WAITER_BIT;
+          Sleep = true;
+        }
+      } while (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire) == false);
+
+      if (!Sleep) {
+        // Acquired early.
+        LOGMAN_THROW_A_FMT((Desired & WRITE_OWNED_BIT) != WRITE_OWNED_BIT, "Somehow read-locked and got a write lock!");
+        return;
+      }
+
+      FutexWaitForReadAvailable(Desired);
+
+      Expected = AtomicFutex.load(std::memory_order_relaxed);
+    }
+  }
+
+  void unlock() {
+    auto AtomicFutex = std::atomic_ref<uint32_t>(Futex);
+
+    uint32_t Expected = AtomicFutex.load(std::memory_order_relaxed);
+    uint32_t Desired {};
+    do {
+      LOGMAN_THROW_A_FMT((Expected & WRITE_OWNED_BIT) == WRITE_OWNED_BIT, "Trying to write-unlock something not write-locked!");
+      // Remove the exclusive lock bit.
+      Desired = Expected & ~WRITE_OWNED_BIT;
+
+      // If no more writers, then make sure to clear the read-waiters bit as well.
+      if ((Desired & WRITE_WAITER_COUNT_MASK) == 0) {
+        Desired &= ~READ_WAITER_BIT;
+      }
+    } while (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire) == false);
+
+    // If success, then `Expected` has old value. Containing `READ_WAITER_BIT` which was just masked off, and also `WRITE_WAITER_COUNT_MASK`.
+    if ((Expected & WRITE_WAITER_COUNT_MASK)) {
+      // Handle write-write handoff.
+      FutexWakeWriter();
+    } else if ((Expected & READ_WAITER_BIT)) {
+      // Handle write-reader handoff.
+      FutexWakeReaders();
+    }
+  }
+
+  void unlock_shared() {
+    auto AtomicFutex = std::atomic_ref<uint32_t>(Futex);
+
+    uint32_t Desired {};
+#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
+    uint32_t Expected = AtomicFutex.load(std::memory_order_relaxed);
+    do {
+      LOGMAN_THROW_A_FMT((Expected & WRITE_OWNED_BIT) != WRITE_OWNED_BIT, "Trying to read-unlock something write-locked!");
+      LOGMAN_THROW_A_FMT((Expected & READ_OWNER_COUNT_MASK) != 0, "Trying to read-unlock something not read-locked!");
+
+      // Decrement the shared counter.
+      Desired = Expected - READ_OWNER_INCREMENT;
+    } while (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire) == false);
+#else
+    Desired = AtomicFutex.fetch_sub(READ_OWNER_INCREMENT) - READ_OWNER_INCREMENT;
+#endif
+
+    // Handle read->write handoff if there are any waiting writers, and no readers left.
+    if ((Desired & WRITE_WAITER_COUNT_MASK) && (Desired & READ_OWNER_COUNT_MASK) == 0) {
+      FutexWakeWriter();
+    }
+  }
+
+  bool try_lock() {
+    auto AtomicFutex = std::atomic_ref<uint32_t>(Futex);
+
+    uint32_t Expected = 0;
+
+    // Try and grab the owned bit.
+    uint32_t Desired = WRITE_OWNED_BIT;
+
+    // try to CAS immediately.
+    return AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire);
+  }
+
+  // Can race with other threads trying to lock shared!
+  bool try_lock_shared() {
+    auto AtomicFutex = std::atomic_ref<uint32_t>(Futex);
+    uint32_t Expected = AtomicFutex.load(std::memory_order_relaxed);
+
+    // Exclusively owned or has a list of waiting owners. Can't pass.
+    if ((Expected & WRITE_OWNED_BIT) || (Expected & WRITE_WAITER_COUNT_MASK)) {
+      return false;
+    }
+
+    // Try to add reader.
+    uint32_t Desired = Expected + READ_OWNER_INCREMENT;
+    LOGMAN_THROW_A_FMT((Desired & READ_OWNER_COUNT_MASK) != 0, "Overflow in read-owners!");
+
+    // Uncontended mutex check
+    return AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire);
+  }
+
+private:
+
+  void FutexWaitForWriteAvailable(uint32_t Expected) {
+    ::syscall(SYS_futex, &Futex, FUTEX_PRIVATE_FLAG | FUTEX_WAIT_BITSET, Expected, nullptr, nullptr, FUTEX_BITSET_WAIT_WRITERS);
+  }
+
+  // Read-lock waiting for writers to drain out.
+  void FutexWaitForReadAvailable(uint32_t Expected) {
+    ::syscall(SYS_futex, &Futex, FUTEX_PRIVATE_FLAG | FUTEX_WAIT_BITSET, Expected, nullptr, nullptr, FUTEX_BITSET_WAIT_READERS);
+  }
+
+  // Read-Lock or Write-lock unlocked, wake one writer.
+  // - Read->Write handoff.
+  // - Write->Write handoff.
+  void FutexWakeWriter() {
+    ::syscall(SYS_futex, &Futex, FUTEX_PRIVATE_FLAG | FUTEX_WAKE_BITSET, 1, nullptr, nullptr, FUTEX_BITSET_WAIT_WRITERS);
+  }
+
+  // Write-lock unlocked, wake read-locks waiting.
+  void FutexWakeReaders() {
+    // Wake all readers.
+    ::syscall(SYS_futex, &Futex, FUTEX_PRIVATE_FLAG | FUTEX_WAKE_BITSET, INT_MAX, nullptr, nullptr, FUTEX_BITSET_WAIT_READERS);
+  }
+
+  // Reuse the SpinWaitLock WFE implementations for read/write lock acquiring with WFE.
+  // Can't reuse the spin-lock directly as some bit-representations are different.
+  // WFE-write-lock is less likely to occur the more read-lock threads are participating. Can still occur so good to try.
+  // WFE-read-lock is actually quite likely to succeed.
+  // Return: true if the lock was acquired.
+  bool Attempt_WFE_WriteLock() {
+#ifdef _M_ARM_64
+    const auto Begin = FEXCore::Utils::SpinWaitLock::GetCycleCounter();
+    auto Now = Begin;
+    const auto Duration = FEXCore::Utils::SpinWaitLock::CycleCounterFrequency / CYCLECOUNT_DIVISOR;
+
+    auto AtomicFutex = std::atomic_ref<uint32_t>(Futex);
+    uint32_t Expected = AtomicFutex.load(std::memory_order_relaxed);
+
+    while ((Now - Begin) < Duration) {
+      if (Expected == 0) {
+        // Try and grab the owned bit.
+        uint32_t Desired = WRITE_OWNED_BIT;
+
+        if (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire)) {
+          return true;
+        }
+      }
+
+      // One-shot attempt to wait for mask to be zero.
+      Expected = FEXCore::Utils::SpinWaitLock::OneShotWFEBitComparison(&Futex, ~0U, 0U);
+      Now = FEXCore::Utils::SpinWaitLock::GetCycleCounter();
+    }
+#endif
+
+    return false;
+  }
+
+  // Return: true if the lock was acquired.
+  bool Attempt_WFE_ReadLock() {
+#ifdef _M_ARM_64
+    // Spin on a WFE for a short-amount of time, waiting for write-owned and writer-count to be zero.
+    //  - Attempt to acquire read-lock at that point.
+    //  - Don't add read-waiters bit on failure, return false.
+    const auto Begin = FEXCore::Utils::SpinWaitLock::GetCycleCounter();
+    auto Now = Begin;
+    const auto Duration = FEXCore::Utils::SpinWaitLock::CycleCounterFrequency / CYCLECOUNT_DIVISOR;
+
+    auto AtomicFutex = std::atomic_ref<uint32_t>(Futex);
+    uint32_t Expected = AtomicFutex.load(std::memory_order_relaxed);
+    uint32_t Desired {};
+
+    while ((Now - Begin) < Duration) {
+      if ((Expected & WRITE_OWNED_BIT) == 0 && (Expected & WRITE_WAITER_COUNT_MASK) == 0) {
+        // If no write-owner and no write-waiting, try and acquire.
+
+        Desired = Expected + READ_OWNER_INCREMENT;
+        LOGMAN_THROW_A_FMT((Desired & READ_OWNER_COUNT_MASK) != 0, "Overflow in read-owners!");
+        if (AtomicFutex.compare_exchange_strong(Expected, Desired, std::memory_order_acq_rel, std::memory_order_acquire)) {
+          return true;
+        }
+      }
+
+      // One-shot attempt to wait for mask to be zero.
+      Expected = FEXCore::Utils::SpinWaitLock::OneShotWFEBitComparison(&Futex, WRITE_OWNED_BIT | WRITE_WAITER_COUNT_MASK, 0U);
+      Now = FEXCore::Utils::SpinWaitLock::GetCycleCounter();
+    }
+#endif
+
+    return false;
+  }
+
+  constexpr static uint32_t WRITE_OWNED_BIT = 1U << 31;
+  constexpr static uint32_t READ_WAITER_BIT = 1U << 15;
+  constexpr static uint32_t WRITE_WAITER_OFFSET = 16;
+  constexpr static uint32_t WRITE_WAITER_INCREMENT = 1U << WRITE_WAITER_OFFSET;
+  constexpr static uint32_t READ_OWNER_INCREMENT = 1;
+
+  // Count masks
+  constexpr static uint32_t WRITE_WAITER_COUNT_MASK = 0x7FFFU << WRITE_WAITER_OFFSET;
+  constexpr static uint32_t READ_OWNER_COUNT_MASK = 0x7FFFU;
+
+  // Independent futex bit-set masks.
+  // Wait for readers to drain.
+  constexpr static uint32_t FUTEX_BITSET_WAIT_READERS = 1U << 0;
+  // Wait for writers to drain.
+  constexpr static uint32_t FUTEX_BITSET_WAIT_WRITERS = 1U << 1;
+
+  // Only spin on WFE for 0.01ms (10k ns).
+  constexpr static uint64_t CYCLECOUNT_DIVISOR = 1'000'000'000ULL / 10'000U;
+
+  // Layout:
+  //    Bits[31]: Write-lock bit.
+  // Bits[30:16]: Write-waiter count.
+  //    Bits[15]: Read-waiter bit.
+  //  Bits[14:0]: Read-owner count.
+  uint32_t Futex {};
+};
+#else
+// SRWLocks are already write-priority locks in WINE and Windows. Use them to avoid lock stampeding.
+class Mutex final {
+public:
+  void lock() {
+    AcquireSRWLockExclusive(&Futex);
+  }
+
+  void lock_shared() {
+    AcquireSRWLockShared(&Futex);
+  }
+
+  void unlock() {
+    ReleaseSRWLockExclusive(&Futex);
+  }
+
+  void unlock_shared() {
+    ReleaseSRWLockShared(&Futex);
+  }
+
+  bool try_lock() {
+    return TryAcquireSRWLockExclusive(&Futex);
+  }
+
+  bool try_lock_shared() {
+    return TryAcquireSRWLockShared(&Futex);
+  }
+
+private:
+  SRWLOCK Futex = SRWLOCK_INIT;
+};
+#endif
+} // namespace FEXCore::Utils::WritePriorityMutex

FEXCore/include/FEXCore/Core/CoreState.h

@@ -353,7 +353,6 @@ struct JITPointers {
     uint64_t ExitFunctionLinker {};
     uint64_t ThreadStopHandlerSpillSRA {};
     uint64_t ThreadPauseHandlerSpillSRA {};
-    uint64_t UnimplementedInstructionHandler {};
     uint64_t GuestSignal_SIGILL {};
     uint64_t GuestSignal_SIGTRAP {};
     uint64_t GuestSignal_SIGSEGV {};
@@ -371,8 +370,6 @@ struct JITPointers {
       // Process specific
       uint64_t LUDIV {};
       uint64_t LDIV {};
-      uint64_t LUREM {};
-      uint64_t LREM {};
 
       // Thread Specific
 
@@ -381,8 +378,6 @@ struct JITPointers {
        * @{ */
       uint64_t LUDIVHandler {};
       uint64_t LDIVHandler {};
-      uint64_t LUREMHandler {};
-      uint64_t LREMHandler {};
       /**  @} */
     } AArch64;
 

FEXCore/include/FEXCore/Utils/LongJump.h

@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+#include <FEXCore/Utils/CompilerDefs.h>
+
+#include <cstdint>
+
+// Reimplementation of longjmp without glibc fortification checks.
+// This is useful when false positives need to be avoided or when using
+// a libc implementation that does not implement std::longjmp.
+namespace FEXCore::UncheckedLongJump {
+// JumpBuf definition needs to be public because the frontend needs to understand it.
+#if defined(_M_ARM_64)
+struct JumpBuf {
+  // All the registers that are required by AAPCS64 to save.
+  // GPRs
+  // X19, X20, X21, X22,
+  // X23, X24, X25, X26,
+  // X27, X28, X29, X30,
+  //
+  // Lower 64-bits:
+  //  V8,  V9, V10, V11,
+  // V12, V13, V14, V15,
+  //
+  // SP,
+  uint64_t Registers[21];
+};
+#else
+struct JumpBuf {
+  // Registers to preserve
+  // RBX, RSP, RBP, R12, R13, R14, R15,
+  // <return address>
+  uint64_t Registers[8];
+};
+#endif
+
+[[nodiscard]] FEX_DEFAULT_VISIBILITY uint64_t SetJump(JumpBuf& Buffer);
+[[noreturn]] FEX_DEFAULT_VISIBILITY void LongJump(JumpBuf& Buffer, uint64_t Value);
+} // namespace FEXCore::UncheckedLongJump

FEXCore/unittests/APITests/Allocator.cpp

@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: MIT
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/generators/catch_generators_range.hpp>
+
+#include "Utils/Allocator/HostAllocator.h"
+#include <FEXCore/Utils/Allocator.h>
+#include <sys/mman.h>
+
+template<typename T>
+bool HasSyscallError(T Result) {
+  constexpr uint64_t MAX_ERRNO = 0xFFFF'FFFF'FFFF'0001ULL;
+  return reinterpret_cast<uint64_t>(Result) >= MAX_ERRNO;
+}
+
+TEST_CASE("Allocator - Fixed replacement") {
+  const auto RegionSize = 128 * 1024 * 1024;
+  fextl::vector<FEXCore::Allocator::MemoryRegion> MemoryRegions {};
+  for (size_t i = 0; i < 2; ++i) {
+    auto Ptr = mmap(nullptr, RegionSize, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    MemoryRegions.emplace_back(FEXCore::Allocator::MemoryRegion {
+      .Ptr = Ptr,
+      .Size = RegionSize,
+    });
+  }
+
+  auto Allocator = Alloc::OSAllocator::Create64BitAllocatorWithRegions(MemoryRegions);
+  auto Base = Allocator->Mmap(nullptr, 4096, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  REQUIRE(!HasSyscallError(Base));
+
+  // Allocate perfectly overlapping pages. Allocate as many pages as the region.
+  // FEX had a bug where the allocator could run out of memory with MAP_FIXED.
+  for (size_t i = 0; i < (RegionSize / 4096); ++i) {
+    auto NewBase = Allocator->Mmap(Base, 4096, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    REQUIRE(Base == NewBase);
+  }
+
+  Alloc::OSAllocator::ReleaseAllocatorWorkaround(std::move(Allocator));
+}
+
+TEST_CASE("Allocator - Non-Fit") {
+  const auto RegionSize = 128 * 1024 * 1024;
+  fextl::vector<FEXCore::Allocator::MemoryRegion> MemoryRegions {};
+  for (size_t i = 0; i < 2; ++i) {
+    auto Ptr = mmap(nullptr, RegionSize, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    MemoryRegions.emplace_back(FEXCore::Allocator::MemoryRegion {
+      .Ptr = Ptr,
+      .Size = RegionSize,
+    });
+  }
+
+  auto Allocator = Alloc::OSAllocator::Create64BitAllocatorWithRegions(MemoryRegions);
+  auto Base = Allocator->Mmap(nullptr, RegionSize / 4, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  REQUIRE(!HasSyscallError(Base));
+
+  // Try to allocate within the whole VMA size minus a small amount.
+  // FEX had a bug where if the allocation fit within a VMA region, it would try and allocate past the end without checking.
+  // Only occurred when `MAP_FIXED` was used.
+  auto NewBase = Allocator->Mmap(Base, RegionSize - (4096 * 64), PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  // Must either fit in the VMA region, or fail.
+  // - If it matches previous allocation, then it fit in the VMA region.
+  //   - This can happen if FEX's allocator gains support for VMA merging.
+  // - If it errors, then it doesn't fit in the VMA region.
+  REQUIRE((NewBase == Base || HasSyscallError(NewBase)));
+
+  Alloc::OSAllocator::ReleaseAllocatorWorkaround(std::move(Allocator));
+}

FEXCore/unittests/APITests/FlexBitSet.cpp

@@ -3,6 +3,7 @@
 #include <catch2/generators/catch_generators_range.hpp>
 
 #include "Utils/Allocator/FlexBitSet.h"
+#include <sys/mman.h>
 
 TEST_CASE("FlexBitSet - Sizing") {
   // Ensure that FlexBitSet sizing is correct.
@@ -40,3 +41,25 @@ TEST_CASE("FlexBitSet - Sizing") {
   CHECK(FEXCore::FlexBitSet<uint32_t>::SizeInBits(sizeof(uint32_t) * 8) == sizeof(uint32_t) * 8);
   CHECK(FEXCore::FlexBitSet<uint64_t>::SizeInBits(sizeof(uint64_t) * 8) == sizeof(uint64_t) * 8);
 }
+
+TEST_CASE("FlexBitSet - Limit") {
+  // Ensure that the FlexBitSet doesn't read past the limits, and returns correct indexes.
+  const auto Size = 4096 * 3;
+  auto Ptr = mmap(nullptr, Size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  auto PtrMiddle = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(Ptr) + 4096);
+  REQUIRE(mprotect(PtrMiddle, 4096, PROT_READ | PROT_WRITE) != -1);
+
+  using ElementType = uint8_t;
+  const size_t NumElements = 4096 * 8;
+  auto FlexBit = reinterpret_cast<FEXCore::FlexBitSet<ElementType>*>(PtrMiddle);
+
+  for (size_t i = 0; i < NumElements; ++i) {
+    auto Result = FlexBit->ForwardScanForRange<true>(i, 1, NumElements);
+    CHECK(Result.FoundElement == i);
+  }
+
+  for (size_t i = 0; i < NumElements; ++i) {
+    auto Result = FlexBit->BackwardScanForRange<true>(i, 1, 0);
+    CHECK(Result.FoundElement == i);
+  }
+}

FEXCore/unittests/Emitter/ALU_Tests.cpp

@@ -9,17 +9,17 @@ using namespace ARMEmitter;
 TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    adr(Reg::r30, &Label);
+    (void)adr(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x10fffffe);
   }
 
   {
     ForwardLabel Label;
-    adr(Reg::r30, &Label);
-    Bind(&Label);
+    (void)adr(Reg::r30, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x1000003e);
@@ -27,17 +27,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    adr(Reg::r30, &Label);
+    (void)adr(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x10fffffe);
   }
 
   {
     BiDirectionalLabel Label;
-    adr(Reg::r30, &Label);
-    Bind(&Label);
+    (void)adr(Reg::r30, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x1000003e);
@@ -45,42 +45,42 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    adrp(Reg::r30, &Label);
+    (void)adrp(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x9000001e);
   }
 
   {
     ForwardLabel Label;
-    adrp(Reg::r30, &Label);
+    (void)adrp(Reg::r30, &Label);
     // Move label a page away
     for (size_t i = 0; i < 1023; ++i) {
       nop();
     }
-    Bind(&Label);
+    (void)Bind(&Label);
 
     CHECK(DisassembleEncoding(0) == 0xb000001e);
   }
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    adrp(Reg::r30, &Label);
+    (void)adrp(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x9000001e);
   }
 
   {
     BiDirectionalLabel Label;
-    adrp(Reg::r30, &Label);
+    (void)adrp(Reg::r30, &Label);
     // Move label a page away
     for (size_t i = 0; i < 1023; ++i) {
       nop();
     }
-    Bind(&Label);
+    (void)Bind(&Label);
 
     CHECK(DisassembleEncoding(0) == 0xb000001e);
   }
@@ -88,17 +88,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate adr.
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
     CHECK(DisassembleEncoding(1) == 0x10fffffe);
   }
   {
     // Will generate nop + adr.
     ForwardLabel Label;
-    LongAddressGen(Reg::r30, &Label);
-    Bind(&Label);
+    (void)LongAddressGen(Reg::r30, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd503201f);
@@ -107,9 +107,9 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate adr.
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x10fffffe);
   }
@@ -117,8 +117,8 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate nop + adr.
     BiDirectionalLabel Label;
-    LongAddressGen(Reg::r30, &Label);
-    Bind(&Label);
+    (void)LongAddressGen(Reg::r30, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd503201f);
@@ -128,7 +128,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate adrp.
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     // Move adrp 1MB away.
@@ -136,7 +136,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
       nop();
     }
 
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
     nop();
     CHECK(DisassembleEncoding(262145) == 0x90fff81e);
     CHECK(DisassembleEncoding(262146) == 0xd503201f);
@@ -145,14 +145,14 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate nop + adrp.
     ForwardLabel Label;
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     // Move label 1MB away, plus a page, and then aligned to a page.
     for (size_t i = 0; i < ((1 * 1024 * 1024 + 4096) / 4 - 2); ++i) {
       nop();
     }
 
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd503201f);
@@ -162,14 +162,14 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate adrp + add.
     ForwardLabel Label;
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     // Move label 1MB away, plus a page, plus one instruction.
     for (size_t i = 0; i < ((1 * 1024 * 1024 + 4096) / 4 - 1); ++i) {
       nop();
     }
 
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb000081e);
@@ -180,7 +180,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate adrp.
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     // Move adrp 1MB away.
@@ -188,7 +188,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
       nop();
     }
 
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
     nop();
     CHECK(DisassembleEncoding(262145) == 0x90fff81e);
     CHECK(DisassembleEncoding(262146) == 0xd503201f);
@@ -197,14 +197,14 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate nop + adrp.
     BiDirectionalLabel Label;
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     // Move label 1MB away, plus a page, and then aligned to a page.
     for (size_t i = 0; i < ((1 * 1024 * 1024 + 4096) / 4 - 2); ++i) {
       nop();
     }
 
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd503201f);
@@ -214,14 +214,14 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate adrp + add.
     BiDirectionalLabel Label;
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     // Move label 1MB away, plus a page, plus one instruction.
     for (size_t i = 0; i < ((1 * 1024 * 1024 + 4096) / 4 - 1); ++i) {
       nop();
     }
 
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb000081e);

FEXCore/unittests/Emitter/Branch_Tests.cpp

@@ -9,17 +9,17 @@ using namespace ARMEmitter;
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Conditional branch immediate") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    b(Condition::CC_PL, &Label);
+    (void)b(Condition::CC_PL, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x54ffffe5);
   }
 
   {
     ForwardLabel Label;
-    b(Condition::CC_PL, &Label);
-    Bind(&Label);
+    (void)b(Condition::CC_PL, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x54000025);
@@ -27,17 +27,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Conditional branch immediat
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    b(Condition::CC_PL, &Label);
+    (void)b(Condition::CC_PL, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x54ffffe5);
   }
 
   {
     BiDirectionalLabel Label;
-    b(Condition::CC_PL, &Label);
-    Bind(&Label);
+    (void)b(Condition::CC_PL, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x54000025);
@@ -46,17 +46,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Conditional branch immediat
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Branch consistent conditional") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    bc(Condition::CC_PL, &Label);
+    (void)bc(Condition::CC_PL, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x54fffff5);
   }
 
   {
     ForwardLabel Label;
-    bc(Condition::CC_PL, &Label);
-    Bind(&Label);
+    (void)bc(Condition::CC_PL, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x54000035);
@@ -64,17 +64,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Branch consistent condition
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    bc(Condition::CC_PL, &Label);
+    (void)bc(Condition::CC_PL, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x54fffff5);
   }
 
   {
     BiDirectionalLabel Label;
-    bc(Condition::CC_PL, &Label);
-    Bind(&Label);
+    (void)bc(Condition::CC_PL, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x54000035);
@@ -89,17 +89,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch regist
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immediate") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    b(&Label);
+    (void)b(&Label);
 
     CHECK(DisassembleEncoding(1) == 0x17ffffff);
   }
 
   {
     ForwardLabel Label;
-    b(&Label);
-    Bind(&Label);
+    (void)b(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x14000001);
@@ -107,17 +107,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immedi
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    b(&Label);
+    (void)b(&Label);
 
     CHECK(DisassembleEncoding(1) == 0x17ffffff);
   }
 
   {
     BiDirectionalLabel Label;
-    b(&Label);
-    Bind(&Label);
+    (void)b(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x14000001);
@@ -125,17 +125,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immedi
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    bl(&Label);
+    (void)bl(&Label);
 
     CHECK(DisassembleEncoding(1) == 0x97ffffff);
   }
 
   {
     ForwardLabel Label;
-    bl(&Label);
-    Bind(&Label);
+    (void)bl(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x94000001);
@@ -143,17 +143,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immedi
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    bl(&Label);
+    (void)bl(&Label);
 
     CHECK(DisassembleEncoding(1) == 0x97ffffff);
   }
 
   {
     BiDirectionalLabel Label;
-    bl(&Label);
-    Bind(&Label);
+    (void)bl(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x94000001);
@@ -162,17 +162,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immedi
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbz(Size::i32Bit, Reg::r29, &Label);
+    (void)cbz(Size::i32Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x34fffffd);
   }
 
   {
     ForwardLabel Label;
-    cbz(Size::i32Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbz(Size::i32Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3400003d);
@@ -180,17 +180,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbz(Size::i32Bit, Reg::r29, &Label);
+    (void)cbz(Size::i32Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x34fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    cbz(Size::i32Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbz(Size::i32Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3400003d);
@@ -198,17 +198,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbz(Size::i64Bit, Reg::r29, &Label);
+    (void)cbz(Size::i64Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb4fffffd);
   }
 
   {
     ForwardLabel Label;
-    cbz(Size::i64Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbz(Size::i64Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb400003d);
@@ -216,17 +216,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbz(Size::i64Bit, Reg::r29, &Label);
+    (void)cbz(Size::i64Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb4fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    cbz(Size::i64Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbz(Size::i64Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb400003d);
@@ -234,17 +234,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbnz(Size::i32Bit, Reg::r29, &Label);
+    (void)cbnz(Size::i32Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x35fffffd);
   }
 
   {
     ForwardLabel Label;
-    cbnz(Size::i32Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbnz(Size::i32Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3500003d);
@@ -252,17 +252,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbnz(Size::i32Bit, Reg::r29, &Label);
+    (void)cbnz(Size::i32Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x35fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    cbnz(Size::i32Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbnz(Size::i32Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3500003d);
@@ -270,17 +270,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbnz(Size::i64Bit, Reg::r29, &Label);
+    (void)cbnz(Size::i64Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb5fffffd);
   }
 
   {
     ForwardLabel Label;
-    cbnz(Size::i64Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbnz(Size::i64Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb500003d);
@@ -288,17 +288,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbnz(Size::i64Bit, Reg::r29, &Label);
+    (void)cbnz(Size::i64Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb5fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    cbnz(Size::i64Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbnz(Size::i64Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb500003d);
@@ -307,17 +307,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbz(Reg::r29, 0, &Label);
+    (void)tbz(Reg::r29, 0, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x3607fffd);
   }
 
   {
     ForwardLabel Label;
-    tbz(Reg::r29, 0, &Label);
-    Bind(&Label);
+    (void)tbz(Reg::r29, 0, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3600003d);
@@ -325,17 +325,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbz(Reg::r29, 0, &Label);
+    (void)tbz(Reg::r29, 0, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x3607fffd);
   }
 
   {
     BiDirectionalLabel Label;
-    tbz(Reg::r29, 0, &Label);
-    Bind(&Label);
+    (void)tbz(Reg::r29, 0, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3600003d);
@@ -343,17 +343,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbz(Reg::r29, 63, &Label);
+    (void)tbz(Reg::r29, 63, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb6fffffd);
   }
 
   {
     ForwardLabel Label;
-    tbz(Reg::r29, 63, &Label);
-    Bind(&Label);
+    (void)tbz(Reg::r29, 63, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb6f8003d);
@@ -361,17 +361,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbz(Reg::r29, 63, &Label);
+    (void)tbz(Reg::r29, 63, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb6fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    tbz(Reg::r29, 63, &Label);
-    Bind(&Label);
+    (void)tbz(Reg::r29, 63, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb6f8003d);
@@ -379,17 +379,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbnz(Reg::r29, 0, &Label);
+    (void)tbnz(Reg::r29, 0, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x3707fffd);
   }
 
   {
     ForwardLabel Label;
-    tbnz(Reg::r29, 0, &Label);
-    Bind(&Label);
+    (void)tbnz(Reg::r29, 0, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3700003d);
@@ -397,17 +397,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbnz(Reg::r29, 0, &Label);
+    (void)tbnz(Reg::r29, 0, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x3707fffd);
   }
 
   {
     BiDirectionalLabel Label;
-    tbnz(Reg::r29, 0, &Label);
-    Bind(&Label);
+    (void)tbnz(Reg::r29, 0, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3700003d);
@@ -415,17 +415,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbnz(Reg::r29, 63, &Label);
+    (void)tbnz(Reg::r29, 63, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb7fffffd);
   }
 
   {
     ForwardLabel Label;
-    tbnz(Reg::r29, 63, &Label);
-    Bind(&Label);
+    (void)tbnz(Reg::r29, 63, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb7f8003d);
@@ -433,17 +433,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbnz(Reg::r29, 63, &Label);
+    (void)tbnz(Reg::r29, 63, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb7fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    tbnz(Reg::r29, 63, &Label);
-    Bind(&Label);
+    (void)tbnz(Reg::r29, 63, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb7f8003d);

FEXCore/unittests/Emitter/Loadstore_Tests.cpp

@@ -1323,7 +1323,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: LDAPR/STLR unscaled imme
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(WReg::w30, &Label);
 
@@ -1332,7 +1332,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(SReg::s30, &Label);
 
@@ -1341,7 +1341,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(XReg::x30, &Label);
 
@@ -1350,7 +1350,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(DReg::d30, &Label);
 
@@ -1359,7 +1359,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldrsw(XReg::x30, &Label);
 
@@ -1368,7 +1368,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(QReg::q30, &Label);
 
@@ -1377,7 +1377,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     prfm(Prefetch::PLDL1KEEP, &Label);
 
@@ -1387,7 +1387,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(WReg::w30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x1800003e);
@@ -1396,7 +1396,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(SReg::s30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x1c00003e);
@@ -1405,7 +1405,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(XReg::x30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x5800003e);
@@ -1414,7 +1414,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(DReg::d30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x5c00003e);
@@ -1423,7 +1423,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldrsw(XReg::x30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x9800003e);
@@ -1432,7 +1432,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(QReg::q30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x9c00003e);
@@ -1441,7 +1441,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     prfm(Prefetch::PLDL1KEEP, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd8000020);

Scripts/aarch64_fit_native.py

@@ -81,6 +81,8 @@ BigCoreIDs = {
             [ ["apple-a13", "0.0"], # If we aren't on 12.0+
               ["apple-a14", "12.0"], # Only exists in 12.0+
             ],
+        # QEmu HVF 10.2+
+        tuple([0x61, 0]): "apple-a13", # Can't determine variant, choose lowest.
 }
 
 LittleCoreIDs = {

Source/Tools/FEXConfig/main.qml

@@ -710,11 +710,6 @@ ApplicationWindow {
                 config: "X87ReducedPrecision"
             }
 
-            ConfigCheckBox {
-                text: qsTr("Unsafe local flags optimization")
-                config: "ABILocalFlags"
-            }
-
             ConfigCheckBox {
                 text: qsTr("Disable JIT optimization passes")
                 config: "O0"

Source/Tools/LinuxEmulation/ArchHelpers/MContext.cpp

@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: MIT
+#include "ArchHelpers/MContext.h"
+
+namespace FEX::ArchHelpers::Context {
+#ifdef _M_ARM_64
+std::string_view GetESRName(uint64_t ESR) {
+  switch ((ESR & ESR1_EC) >> 26) {
+  case 0b000'000: return "Unknown";
+  case 0b000'001: return "Trapped WF*";
+  case 0b000'011: return "Trapped MCR/MRC";
+  case 0b000'100: return "Trapped MCRR/MRRC";
+  case 0b000'101: return "Trapped MCR/MRC (coproc==0b1110)";
+  case 0b000'110: return "Trapped LDC/STC";
+  case 0b000'111: return "Trapped SME;SVE,ASIMD,FP";
+  case 0b001'010: return "Trapped non-covered instruction";
+  case 0b001'100: return "Trapped MRRC (coproc==0b1110)";
+  case 0b001'101: return "Branch target exception";
+  case 0b001'110: return "Illegal Execution State";
+  case 0b010'001: return "AArch32 SVC";
+  case 0b010'100: return "Trapped MSRR/MRRS/System instruction";
+  case 0b010'101: return "AArch64 SVC";
+  case 0b011'000: return "Trapped MSR/MRS/System instruction";
+  case 0b011'001: return "Trapped SVE from ZEN";
+  case 0b011'011: return "TSTART Exception";
+  case 0b011'100: return "PAC Exception";
+  case 0b011'101: return "Trapped SME from SMEN";
+  case 0b100'000: return "Instruction abort";
+  case 0b100'001: return "Instruction abort w/o change to exception level";
+  case 0b100'010: return "PC Alignment fault";
+  case 0b100'100: return "Data abort";
+  case 0b100'101: return "Data abort w/o change to exception level";
+  case 0b100'110: return "SP Alignment fault";
+  case 0b100'111: return "Memory operation exception";
+  case 0b101'000: return "AArch32 Trapped FP Exception";
+  case 0b101'100: return "AArch64 Trapped FP Exception";
+  case 0b101'101: return "GCS exception";
+  case 0b101'111: return "SError exception";
+  case 0b110'000: return "BP Exception";
+  case 0b110'001: return "BP Exception w/o change to exception level";
+  case 0b110'010: return "Software step Exception";
+  case 0b110'011: return "Software step Exception w/o change to exception level";
+  case 0b110'100: return "Watchpoint Exception";
+  case 0b110'101: return "Watchpoit Exception w/o change to exception level";
+  case 0b111'000: return "AArch32 BKPT";
+  case 0b111'100: return "AArch64 BRK";
+  case 0b111'101: return "Profiling Exception";
+  default: return "Reserved";
+  }
+}
+#endif
+} // namespace FEX::ArchHelpers::Context

Source/Tools/LinuxEmulation/ArchHelpers/MContext.h

@@ -203,9 +203,12 @@ constexpr static uint64_t ESR1_DataAbort_Level_EL2 = 0b01;
 constexpr static uint64_t ESR1_DataAbort_Level_EL1 = 0b10;
 constexpr static uint64_t ESR1_DataAbort_Level_EL0 = 0b11;
 
+std::string_view GetESRName(uint64_t ESR);
+
 static inline uint32_t GetProtectFlags(void* ucontext) {
   uint64_t ESR = GetArmESR(ucontext);
-  LOGMAN_THROW_A_FMT((ESR & ESR1_EC) == ESR1_EC_DataAbort, "Unknown ESR1 EC type: 0x{:x} != 0x{:x}", ESR & ESR1_EC, ESR1_EC_DataAbort);
+  LOGMAN_THROW_A_FMT((ESR & ESR1_EC) == ESR1_EC_DataAbort, "Unknown ESR1 EC type: 0x{:x} != 0x{:x}. Received '{}'", ESR & ESR1_EC,
+                     ESR1_EC_DataAbort, GetESRName(ESR));
 
   uint32_t ProtectFlags {};
   if ((ESR & ESR1_DataAbort_Level) == ESR1_DataAbort_Level_EL0) {

Source/Tools/LinuxEmulation/CMakeLists.txt

@@ -3,6 +3,7 @@ add_compile_options(-fno-operator-names)
 set (SRCS
   VDSO_Emulation.cpp
   Thunks.cpp
+  ArchHelpers/MContext.cpp
   GdbServer/Info.cpp
   LinuxSyscalls/GdbServer.cpp
   LinuxSyscalls/EmulatedFiles/EmulatedFiles.cpp

Source/Tools/LinuxEmulation/LinuxSyscalls/Seccomp/BPFEmitter.cpp

@@ -204,7 +204,7 @@ uint64_t BPFEmitter::HandleJmp(uint32_t BPFIP, uint32_t NumInst, const sock_filt
       TargetLabel = JumpLabels.try_emplace(Target, ARMEmitter::ForwardLabel {}).first;
     }
 
-    EMIT_INST(b(&TargetLabel->second));
+    EMIT_INST((void)b(&TargetLabel->second));
     break;
   }
   case BPF_JEQ:
@@ -248,8 +248,8 @@ uint64_t BPFEmitter::HandleJmp(uint32_t BPFIP, uint32_t NumInst, const sock_filt
       TargetFalseLabel = JumpLabels.try_emplace(TargetFalse, ARMEmitter::ForwardLabel {}).first;
     }
 
-    EMIT_INST(b(CompareResultOp, &TargetTrueLabel->second));
-    EMIT_INST(b(&TargetFalseLabel->second));
+    EMIT_INST((void)b(CompareResultOp, &TargetTrueLabel->second));
+    EMIT_INST((void)b(&TargetFalseLabel->second));
     break;
   }
   default: RETURN_ERROR(-EINVAL); // Unknown jump type
@@ -303,7 +303,7 @@ uint64_t BPFEmitter::HandleEmission(uint32_t flags, const sock_fprog* prog) {
     if constexpr (!CalculateSize) {
       auto jump_label = JumpLabels.find(i);
       if (jump_label != JumpLabels.end()) {
-        Bind(&jump_label->second);
+        (void)Bind(&jump_label->second);
       }
     }
 
@@ -401,7 +401,7 @@ uint64_t BPFEmitter::JITFilter(uint32_t flags, const sock_fprog* prog) {
   // Emit the constant pool.
   Align();
   for (auto& Const : ConstPool) {
-    Bind(&Const.second);
+    (void)Bind(&Const.second);
     dc32(Const.first);
   }
 

Source/Tools/LinuxEmulation/LinuxSyscalls/Utils/Threads.cpp

@@ -4,6 +4,7 @@
 
 #include <FEXCore/Core/Context.h>
 #include <FEXCore/Utils/Allocator.h>
+#include <FEXCore/Utils/LongJump.h>
 #include <FEXCore/Utils/Threads.h>
 
 namespace FEX::LinuxEmulation::Threads {
@@ -190,143 +191,6 @@ __attribute__((naked)) void StackPivotAndCall(void* Arg, FEXCore::Threads::Threa
 }
 #endif
 namespace PThreads {
-  namespace LongJump {
-    // This is a custom long jump implementation that avoids the glibc implementation.
-    // This is required behaviour because glibc's fortification checks don't understand stack pivots.
-    // FEX requires a stack pivot to work through a long jump, so these two features are at odds with each other.
-#ifdef _M_ARM_64
-    struct JumpBuf {
-      // All the registers that are required by AAPCS64 to save.
-      // GPRs
-      // X19, X20, X21, X22,
-      // X23, X24, X25, X26,
-      // X27, X28, X29, X30,
-      //
-      // Lower 64-bits:
-      //  V8,  V9, V10, V11,
-      // V12, V13, V14, V15,
-      //
-      // SP,
-      uint64_t Registers[21];
-    };
-    FEX_NAKED uint64_t SetJump(JumpBuf& Buffer) {
-      __asm volatile(R"(
-        // x0 contains the jumpbuffer
-        stp x19, x20, [x0, #( 0 * 8)];
-        stp x21, x22, [x0, #( 2 * 8)];
-        stp x23, x24, [x0, #( 4 * 8)];
-        stp x25, x26, [x0, #( 6 * 8)];
-        stp x27, x28, [x0, #( 8 * 8)];
-        stp x29, x30, [x0, #(10 * 8)];
-
-        // FPRs
-        stp d8,   d9, [x0, #(12 * 8)];
-        stp d10, d11, [x0, #(14 * 8)];
-        stp d12, d13, [x0, #(16 * 8)];
-        stp d14, d15, [x0, #(18 * 8)];
-
-        // Move SP in to a temporary to store.
-        mov x1, sp;
-        str x1,  [x0, #(20 * 8)];
-
-        // Return zero to signify this is the SetJump.
-        mov x0, #0;
-        ret;
-      )" ::
-                       : "memory");
-    }
-
-    [[noreturn]]
-    FEX_NAKED void LongJump(JumpBuf& Buffer, uint64_t Value) {
-      __asm volatile(R"(
-        // x0 contains the jumpbuffer
-        ldp x19, x20, [x0, #( 0 * 8)];
-        ldp x21, x22, [x0, #( 2 * 8)];
-        ldp x23, x24, [x0, #( 4 * 8)];
-        ldp x25, x26, [x0, #( 6 * 8)];
-        ldp x27, x28, [x0, #( 8 * 8)];
-        ldp x29, x30, [x0, #(10 * 8)];
-
-        // FPRs
-        ldp d8,   d9, [x0, #(12 * 8)];
-        ldp d10, d11, [x0, #(14 * 8)];
-        ldp d12, d13, [x0, #(16 * 8)];
-        ldp d14, d15, [x0, #(18 * 8)];
-
-        // Load SP in to temporary then move
-        ldr x0,  [x0, #(20 * 8)];
-        mov sp, x0;
-
-        // Move value in to result register
-        mov x0, x1;
-        ret;
-      )" ::
-                       : "memory");
-    }
-#else
-    struct JumpBuf {
-      // Registers to preserve
-      // RBX, RSP, RBP, R12, R13, R14, R15,
-      // <return address>
-      uint64_t Registers[8];
-    };
-
-    __attribute__((naked)) uint64_t SetJump(JumpBuf& Buffer) {
-      __asm volatile(R"(
-      .intel_syntax noprefix;
-      // rdi contains the jumpbuffer
-      mov [rdi + (0 * 8)], rbx;
-      mov [rdi + (1 * 8)], rsp;
-      mov [rdi + (2 * 8)], rbp;
-      mov [rdi + (3 * 8)], r12;
-      mov [rdi + (4 * 8)], r13;
-      mov [rdi + (5 * 8)], r14;
-      mov [rdi + (6 * 8)], r15;
-
-      // Return address is on the stack, load it and store
-      mov rsi, [rsp];
-      mov [rdi + (7 * 8)], rsi;
-
-      // Return zero to signify this is the SetJump.
-      mov rax, 0;
-      ret;
-
-      .att_syntax prefix;
-      )" ::
-                       : "memory");
-    }
-
-    [[noreturn]]
-    __attribute__((naked)) void LongJump(JumpBuf& Buffer, uint64_t Value) {
-      __asm volatile(R"(
-      .intel_syntax noprefix;
-      // rdi contains the jumpbuffer
-      mov rbx, [rdi + (0 * 8)];
-      mov rsp, [rdi + (1 * 8)];
-      mov rbp, [rdi + (2 * 8)];
-      mov r12, [rdi + (3 * 8)];
-      mov r13, [rdi + (4 * 8)];
-      mov r14, [rdi + (5 * 8)];
-      mov r15, [rdi + (6 * 8)];
-
-      // Move value in to result register
-      mov rax, rsi;
-
-      // Pop the dead return address off the stack
-      pop rsi;
-
-      // Load the original return address from the jumpbuffer
-      mov rsi, [rdi + (7 * 8)];
-
-      // Return using a jump
-      jmp rsi;
-
-      .att_syntax prefix;
-      )" ::
-                       : "memory");
-    }
-#endif
-  }; // namespace LongJump
   void* InitializeThread(void* Ptr);
 
   class PThread final : public FEXCore::Threads::Thread {
@@ -397,7 +261,7 @@ namespace PThreads {
       return STracker;
     }
 
-    void SetupLongJump(LongJump::JumpBuf* exit_resolver) {
+    void SetupLongJump(FEXCore::UncheckedLongJump::JumpBuf* exit_resolver) {
       _exit_resolver = exit_resolver;
     }
 
@@ -405,7 +269,7 @@ namespace PThreads {
     void LongJumpExit(FEX::HLE::ThreadStateObject* ThreadObject, uint32_t Status) {
       this->Status = Status;
       this->ThreadObject = ThreadObject;
-      LongJump::LongJump(*_exit_resolver, 1);
+      FEXCore::UncheckedLongJump::LongJump(*_exit_resolver, 1);
       FEX_UNREACHABLE;
     }
 
@@ -424,7 +288,9 @@ namespace PThreads {
     void* UserArg;
     void* Stack {};
 
-    LongJump::JumpBuf* _exit_resolver {};
+    // Use FEXCore's UncheckedLongJump to avoid fortification checks.
+    // This avoids a false positive since glibc does not understand stack pivots.
+    FEXCore::UncheckedLongJump::JumpBuf* _exit_resolver {};
     FEX::HLE::ThreadStateObject* ThreadObject {};
     uint32_t Status {};
   };
@@ -435,11 +301,11 @@ namespace PThreads {
     PThread* Thread {reinterpret_cast<PThread*>(Ptr)};
     StackBase = Thread->GetPivotStack();
     STracker = Thread->GetStackTracker();
-    LongJump::JumpBuf exit_resolver {};
+    FEXCore::UncheckedLongJump::JumpBuf exit_resolver {};
 
     bool LongJumpExit {};
 
-    if (LongJump::SetJump(exit_resolver) == 0) {
+    if (FEXCore::UncheckedLongJump::SetJump(exit_resolver) == 0) {
       Thread->SetupLongJump(&exit_resolver);
       // Run the user function.
       // `Thread` object is dead after this function returns.

Source/Tools/LinuxEmulation/LinuxSyscalls/x32/Ioctl/amdgpu_drm.inl

@@ -14,3 +14,6 @@ _BASIC_META(DRM_IOCTL_AMDGPU_WAIT_FENCES)
 _BASIC_META(DRM_IOCTL_AMDGPU_VM)
 _BASIC_META(DRM_IOCTL_AMDGPU_FENCE_TO_HANDLE)
 _BASIC_META(DRM_IOCTL_AMDGPU_SCHED)
+_BASIC_META(DRM_IOCTL_AMDGPU_USERQ)
+_BASIC_META(DRM_IOCTL_AMDGPU_USERQ_SIGNAL)
+_BASIC_META(DRM_IOCTL_AMDGPU_USERQ_WAIT)

Source/Tools/LinuxEmulation/LinuxSyscalls/x32/Ioctl/asahi_drm.inl

@@ -0,0 +1,11 @@
+_BASIC_META(DRM_IOCTL_ASAHI_GET_PARAMS)
+_BASIC_META(DRM_IOCTL_ASAHI_GET_TIME)
+_BASIC_META(DRM_IOCTL_ASAHI_VM_CREATE)
+_BASIC_META(DRM_IOCTL_ASAHI_VM_DESTROY)
+_BASIC_META(DRM_IOCTL_ASAHI_VM_BIND)
+_BASIC_META(DRM_IOCTL_ASAHI_GEM_CREATE)
+_BASIC_META(DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET)
+_BASIC_META(DRM_IOCTL_ASAHI_GEM_BIND_OBJECT)
+_BASIC_META(DRM_IOCTL_ASAHI_QUEUE_CREATE)
+_BASIC_META(DRM_IOCTL_ASAHI_QUEUE_DESTROY)
+_BASIC_META(DRM_IOCTL_ASAHI_SUBMIT)

Source/Tools/LinuxEmulation/LinuxSyscalls/x32/Ioctl/drm.h

@@ -15,10 +15,12 @@ extern "C" {
 #include "fex-drm/drm_mode.h"
 #include "fex-drm/i915_drm.h"
 #include "fex-drm/amdgpu_drm.h"
+#include "fex-drm/asahi_drm.h"
 #include "fex-drm/lima_drm.h"
 #include "fex-drm/panfrost_drm.h"
 #include "fex-drm/msm_drm.h"
 #include "fex-drm/nouveau_drm.h"
+#include "fex-drm/nova_drm.h"
 #include "fex-drm/radeon_drm.h"
 #include "fex-drm/vc4_drm.h"
 #include "fex-drm/v3d_drm.h"
@@ -1272,11 +1274,13 @@ namespace V3D {
 
 #include "LinuxSyscalls/x32/Ioctl/drm.inl"
 #include "LinuxSyscalls/x32/Ioctl/amdgpu_drm.inl"
+#include "LinuxSyscalls/x32/Ioctl/asahi_drm.inl"
 #include "LinuxSyscalls/x32/Ioctl/msm_drm.inl"
 #include "LinuxSyscalls/x32/Ioctl/i915_drm.inl"
 #include "LinuxSyscalls/x32/Ioctl/lima_drm.inl"
 #include "LinuxSyscalls/x32/Ioctl/panfrost_drm.inl"
 #include "LinuxSyscalls/x32/Ioctl/nouveau_drm.inl"
+#include "LinuxSyscalls/x32/Ioctl/nova_drm.inl"
 #include "LinuxSyscalls/x32/Ioctl/radeon_drm.inl"
 #include "LinuxSyscalls/x32/Ioctl/vc4_drm.inl"
 #include "LinuxSyscalls/x32/Ioctl/v3d_drm.inl"

Source/Tools/LinuxEmulation/LinuxSyscalls/x32/Ioctl/msm_drm.inl

@@ -10,4 +10,4 @@ _BASIC_META(DRM_IOCTL_MSM_GEM_MADVISE)
 _BASIC_META(DRM_IOCTL_MSM_SUBMITQUEUE_NEW)
 _BASIC_META(DRM_IOCTL_MSM_SUBMITQUEUE_CLOSE)
 _BASIC_META(DRM_IOCTL_MSM_SUBMITQUEUE_QUERY)
-
+_BASIC_META(DRM_IOCTL_MSM_VM_BIND)

Source/Tools/LinuxEmulation/LinuxSyscalls/x32/Ioctl/nova_drm.inl

@@ -0,0 +1,3 @@
+_BASIC_META(DRM_IOCTL_NOVA_GETPARAM)
+_BASIC_META(DRM_IOCTL_NOVA_GEM_CREATE)
+_BASIC_META(DRM_IOCTL_NOVA_GEM_INFO)

Source/Tools/LinuxEmulation/LinuxSyscalls/x32/Ioctl/panfrost_drm.inl

@@ -7,3 +7,4 @@ _BASIC_META(DRM_IOCTL_PANFROST_GET_BO_OFFSET)
 _BASIC_META(DRM_IOCTL_PANFROST_MADVISE)
 _BASIC_META(DRM_IOCTL_PANFROST_PERFCNT_ENABLE)
 _BASIC_META(DRM_IOCTL_PANFROST_PERFCNT_DUMP)
+_BASIC_META(DRM_IOCTL_PANFROST_SET_LABEL_BO)

Source/Tools/LinuxEmulation/LinuxSyscalls/x32/Ioctl/panthor_drm.inl

@@ -11,3 +11,5 @@ _BASIC_META(DRM_IOCTL_PANTHOR_GROUP_SUBMIT)
 _BASIC_META(DRM_IOCTL_PANTHOR_GROUP_GET_STATE)
 _BASIC_META(DRM_IOCTL_PANTHOR_TILER_HEAP_CREATE)
 _BASIC_META(DRM_IOCTL_PANTHOR_TILER_HEAP_DESTROY)
+_BASIC_META(DRM_IOCTL_PANTHOR_BO_SET_LABEL)
+_BASIC_META(DRM_IOCTL_PANTHOR_SET_USER_MMIO_OFFSET)

Source/Tools/LinuxEmulation/LinuxSyscalls/x32/Ioctl/v3d_drm.inl

@@ -11,3 +11,4 @@ _BASIC_META(DRM_IOCTL_V3D_PERFMON_DESTROY)
 _BASIC_META(DRM_IOCTL_V3D_PERFMON_GET_VALUES)
 _BASIC_META(DRM_IOCTL_V3D_SUBMIT_CPU)
 _BASIC_META(DRM_IOCTL_V3D_PERFMON_GET_COUNTER)
+_BASIC_META(DRM_IOCTL_V3D_PERFMON_SET_GLOBAL)

Source/Tools/pidof/pidof.cpp

@@ -87,13 +87,20 @@ bool FindWineFEXApplication(int64_t PID, std::string_view exe, const std::vector
   }
 
   // Wine was found, scan the mapped files to see if anything mapped "libarm64ecfex.dll" or "libwow64fex.dll"
-  for (const auto& Entry : std::filesystem::directory_iterator(fmt::format("/proc/{}/map_files", PID))) {
+
+  std::error_code ec {};
+  auto dir_iter = std::filesystem::directory_iterator(fmt::format("/proc/{}/map_files", PID), ec);
+  // If error reading symlink then skip.
+  if (ec) {
+    return false;
+  }
+
+  for (const auto& Entry : dir_iter) {
     // If not a symlink then skip.
     if (!Entry.is_symlink()) {
       continue;
     }
 
-    std::error_code ec {};
     const auto symlink_path = std::filesystem::read_symlink(Entry.path(), ec);
     // If error reading symlink then skip.
     if (ec) {

Source/Windows/Common/WinAPI/Sync.cpp

@@ -28,6 +28,18 @@ void ReleaseSRWLockExclusive(PSRWLOCK SRWLock) {
   RtlReleaseSRWLockExclusive(SRWLock);
 }
 
+void AcquireSRWLockShared(PSRWLOCK SRWLock) {
+  RtlAcquireSRWLockShared(SRWLock);
+}
+
+void ReleaseSRWLockShared(PSRWLOCK SRWLock) {
+  RtlReleaseSRWLockShared(SRWLock);
+}
+
+DLLEXPORT_FUNC(BOOLEAN, TryAcquireSRWLockShared, (PSRWLOCK SRWLock)) {
+  return RtlTryAcquireSRWLockShared(SRWLock);
+}
+
 DLLEXPORT_FUNC(BOOLEAN, TryAcquireSRWLockExclusive, (PSRWLOCK SRWLock)) {
   return RtlTryAcquireSRWLockExclusive(SRWLock);
 }

Source/Windows/WOW64/Module.cpp

@@ -529,6 +529,8 @@ void BTCpuProcessInit() {
 
   {
     auto HostFeatures = FEX::Windows::CPUFeatures::FetchHostFeatures(IsWine);
+    // AVX is unsupported for WOW64
+    HostFeatures.SupportsAVX = false;
     CTX = FEXCore::Context::Context::CreateNewContext(HostFeatures);
   }
 

Source/Windows/include/winternl.h

@@ -540,6 +540,9 @@ NTSTATUS WINAPI RtlWow64SetThreadContext(HANDLE, const WOW64_CONTEXT*);
 void WINAPI Wow64ProcessPendingCrossProcessItems(void);
 NTSTATUS WINAPI Wow64SystemServiceEx(UINT, UINT*);
 NTSTATUS WINAPI RtlWow64SuspendThread(HANDLE, ULONG*);
+void WINAPI RtlAcquireSRWLockShared(RTL_SRWLOCK*);
+void WINAPI RtlReleaseSRWLockShared(RTL_SRWLOCK*);
+BOOLEAN WINAPI RtlTryAcquireSRWLockShared(RTL_SRWLOCK*);
 
 #ifdef __cplusplus
 }

ThunkLibs/libGL/libGL_Guest.cpp

@@ -80,10 +80,10 @@ static void* malloc_wrapper(size_t size) {
 }
 
 static void OnInit() {
-  fexfn_pack_SetGuestMalloc((uintptr_t)malloc_wrapper, (uintptr_t)CallbackUnpack<decltype(malloc_wrapper)>::Unpack);
-  fexfn_pack_SetGuestXSync((uintptr_t)XSync, (uintptr_t)CallbackUnpack<decltype(XSync)>::Unpack);
-  fexfn_pack_SetGuestXGetVisualInfo((uintptr_t)XGetVisualInfo, (uintptr_t)CallbackUnpack<decltype(XGetVisualInfo)>::Unpack);
-  fexfn_pack_SetGuestXDisplayString((uintptr_t)XDisplayString, (uintptr_t)CallbackUnpack<decltype(XDisplayString)>::Unpack);
+  fexfn_pack_GL_SetGuestMalloc((uintptr_t)malloc_wrapper, (uintptr_t)CallbackUnpack<decltype(malloc_wrapper)>::Unpack);
+  fexfn_pack_GL_SetGuestXSync((uintptr_t)XSync, (uintptr_t)CallbackUnpack<decltype(XSync)>::Unpack);
+  fexfn_pack_GL_SetGuestXGetVisualInfo((uintptr_t)XGetVisualInfo, (uintptr_t)CallbackUnpack<decltype(XGetVisualInfo)>::Unpack);
+  fexfn_pack_GL_SetGuestXDisplayString((uintptr_t)XDisplayString, (uintptr_t)CallbackUnpack<decltype(XDisplayString)>::Unpack);
 }
 
 // libGL.so must pull in libX11.so as a dependency. Referencing some libX11

ThunkLibs/libGL/libGL_Host.cpp

@@ -50,19 +50,19 @@ host_layout<_XDisplay*>::~host_layout() {
 // Functions returning _XDisplay* should be handled explicitly via ptr_passthrough
 guest_layout<_XDisplay*> to_guest(host_layout<_XDisplay*>) = delete;
 
-static void fexfn_impl_libGL_SetGuestMalloc(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
+static void fexfn_impl_libGL_GL_SetGuestMalloc(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
   MakeHostTrampolineForGuestFunctionAt(GuestTarget, GuestUnpacker, &GuestMalloc);
 }
 
-static void fexfn_impl_libGL_SetGuestXGetVisualInfo(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
+static void fexfn_impl_libGL_GL_SetGuestXGetVisualInfo(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
   MakeHostTrampolineForGuestFunctionAt(GuestTarget, GuestUnpacker, &x11_manager.GuestXGetVisualInfo);
 }
 
-static void fexfn_impl_libGL_SetGuestXSync(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
+static void fexfn_impl_libGL_GL_SetGuestXSync(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
   MakeHostTrampolineForGuestFunctionAt(GuestTarget, GuestUnpacker, &x11_manager.GuestXSync);
 }
 
-static void fexfn_impl_libGL_SetGuestXDisplayString(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
+static void fexfn_impl_libGL_GL_SetGuestXDisplayString(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
   MakeHostTrampolineForGuestFunctionAt(GuestTarget, GuestUnpacker, &x11_manager.GuestXDisplayString);
 }
 

ThunkLibs/libGL/libGL_interface.cpp

@@ -22,18 +22,18 @@ template<>
 struct fex_gen_config<glXGetProcAddress> : fexgen::custom_host_impl, fexgen::custom_guest_entrypoint, fexgen::returns_guest_pointer {};
 
 // internal use
-void SetGuestMalloc(uintptr_t, uintptr_t);
-void SetGuestXSync(uintptr_t, uintptr_t);
-void SetGuestXGetVisualInfo(uintptr_t, uintptr_t);
-void SetGuestXDisplayString(uintptr_t, uintptr_t);
+void GL_SetGuestMalloc(uintptr_t, uintptr_t);
+void GL_SetGuestXSync(uintptr_t, uintptr_t);
+void GL_SetGuestXGetVisualInfo(uintptr_t, uintptr_t);
+void GL_SetGuestXDisplayString(uintptr_t, uintptr_t);
 template<>
-struct fex_gen_config<SetGuestMalloc> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
+struct fex_gen_config<GL_SetGuestMalloc> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
 template<>
-struct fex_gen_config<SetGuestXSync> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
+struct fex_gen_config<GL_SetGuestXSync> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
 template<>
-struct fex_gen_config<SetGuestXGetVisualInfo> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
+struct fex_gen_config<GL_SetGuestXGetVisualInfo> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
 template<>
-struct fex_gen_config<SetGuestXDisplayString> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
+struct fex_gen_config<GL_SetGuestXDisplayString> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
 
 template<typename>
 struct fex_gen_type {};

ThunkLibs/libvulkan/Guest.cpp

@@ -87,9 +87,9 @@ PFN_vkVoidFunction vkGetInstanceProcAddr(VkInstance a_0, const char* a_1) {
 void OnInit() {
   // TODO: Load libX11 on-demand instead
   void* libx11 = dlopen("libX11.so.6", RTLD_LAZY);
-  fexfn_pack_SetGuestXSync((uintptr_t)dlsym(libx11, "XSync"), (uintptr_t)CallbackUnpack<decltype(XSync)>::Unpack);
-  fexfn_pack_SetGuestXGetVisualInfo((uintptr_t)dlsym(libx11, "XGetVisualInfo"), (uintptr_t)CallbackUnpack<decltype(XGetVisualInfo)>::Unpack);
-  fexfn_pack_SetGuestXDisplayString((uintptr_t)dlsym(libx11, "XDisplayString"), (uintptr_t)CallbackUnpack<decltype(XDisplayString)>::Unpack);
+  fexfn_pack_Vulkan_SetGuestXSync((uintptr_t)dlsym(libx11, "XSync"), (uintptr_t)CallbackUnpack<decltype(XSync)>::Unpack);
+  fexfn_pack_Vulkan_SetGuestXGetVisualInfo((uintptr_t)dlsym(libx11, "XGetVisualInfo"), (uintptr_t)CallbackUnpack<decltype(XGetVisualInfo)>::Unpack);
+  fexfn_pack_Vulkan_SetGuestXDisplayString((uintptr_t)dlsym(libx11, "XDisplayString"), (uintptr_t)CallbackUnpack<decltype(XDisplayString)>::Unpack);
 }
 
 LOAD_LIB_INIT(libvulkan, OnInit)

ThunkLibs/libvulkan/Host.cpp

@@ -63,15 +63,15 @@ static void DoSetupWithInstance(VkInstance instance) {
 
 static X11Manager x11_manager;
 
-static void fexfn_impl_libvulkan_SetGuestXGetVisualInfo(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
+static void fexfn_impl_libvulkan_Vulkan_SetGuestXGetVisualInfo(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
   MakeHostTrampolineForGuestFunctionAt(GuestTarget, GuestUnpacker, &x11_manager.GuestXGetVisualInfo);
 }
 
-static void fexfn_impl_libvulkan_SetGuestXSync(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
+static void fexfn_impl_libvulkan_Vulkan_SetGuestXSync(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
   MakeHostTrampolineForGuestFunctionAt(GuestTarget, GuestUnpacker, &x11_manager.GuestXSync);
 }
 
-static void fexfn_impl_libvulkan_SetGuestXDisplayString(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
+static void fexfn_impl_libvulkan_Vulkan_SetGuestXDisplayString(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
   MakeHostTrampolineForGuestFunctionAt(GuestTarget, GuestUnpacker, &x11_manager.GuestXDisplayString);
 }
 

ThunkLibs/libvulkan/libvulkan_interface.cpp

@@ -29,15 +29,15 @@ template<typename>
 struct fex_gen_type {};
 
 // internal use
-void SetGuestXSync(uintptr_t, uintptr_t);
-void SetGuestXGetVisualInfo(uintptr_t, uintptr_t);
-void SetGuestXDisplayString(uintptr_t, uintptr_t);
+void Vulkan_SetGuestXSync(uintptr_t, uintptr_t);
+void Vulkan_SetGuestXGetVisualInfo(uintptr_t, uintptr_t);
+void Vulkan_SetGuestXDisplayString(uintptr_t, uintptr_t);
 template<>
-struct fex_gen_config<SetGuestXSync> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
+struct fex_gen_config<Vulkan_SetGuestXSync> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
 template<>
-struct fex_gen_config<SetGuestXGetVisualInfo> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
+struct fex_gen_config<Vulkan_SetGuestXGetVisualInfo> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
 template<>
-struct fex_gen_config<SetGuestXDisplayString> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
+struct fex_gen_config<Vulkan_SetGuestXDisplayString> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
 
 // So-called "dispatchable" handles are represented as opaque pointers.
 // In addition to marking them as such, API functions that create these objects

docs/SourceOutline.md

@@ -1,4 +1,4 @@
-# FEX-2510
+# FEX-2511
 
 ## FEXCore
 See [FEXCore/Readme.md](../FEXCore/Readme.md) for more details

unittests/ASM/FEX_bugs/BitConditionCheck.asm

@@ -0,0 +1,39 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "1"
+  },
+  "Env": { "FEX_MAXINST" : "41010", "FEX_MULTIBLOCK": "1", "FEX_TSOENABLED": "0" }
+}
+%endif
+
+; FEX-Emu had a bug where it tried to encode too large of a tbz/tbnz offset.
+
+%macro TooConditionalBitBranch 0
+  ; Stresses ARM64's tbz/tbnz branch target of +-32KB.
+  jmp %%top
+  %%top:
+
+  lea rax, [rel data]
+  mov rbx, 1
+  %rep 2000
+    add qword [rel data], rbx
+  %endrep
+  lea rax, [%%top]
+  jpe %%top
+%endmacro
+
+mov rax, 0
+cmp rax, 0
+mov rcx, 0
+
+jz long_jump
+
+TooConditionalBitBranch
+
+long_jump:
+mov rax, 1
+hlt
+
+data:
+dq 0, 0, 0, 0

unittests/ASM/FEX_bugs/BranchConditionCheck.asm

@@ -0,0 +1,40 @@
+
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "1"
+  },
+  "Env": { "FEX_MAXINST" : "41010", "FEX_MULTIBLOCK": "1", "FEX_TSOENABLED": "0" }
+}
+%endif
+
+; FEX-Emu had a bug where it tried encoding too large of a conditional branch offset.
+
+%macro TooLargeConditionalBranch 0
+  ; Stresses ARM64's b.cc/cbz/cbnz branch target of +-1MB.
+  jmp %%top
+  %%top:
+
+  lea rax, [rel data]
+  mov rbx, 1
+  %rep 37500
+    add qword [rel data], rbx
+  %endrep
+  lea rax, [%%top]
+  jnz %%top
+%endmacro
+
+mov rax, 0
+cmp rax, 0
+mov rcx, 0
+
+jz long_jump
+
+TooLargeConditionalBranch
+
+long_jump:
+mov rax, 1
+hlt
+
+data:
+dq 0, 0, 0, 0

unittests/InstructionCountCI/AVX128/FMA4.json

@@ -19,7 +19,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -33,7 +33,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -47,7 +47,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -61,7 +61,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -75,7 +75,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -89,7 +89,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -103,7 +103,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -117,7 +117,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -131,7 +131,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -145,7 +145,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -159,7 +159,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -173,7 +173,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -187,7 +187,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -201,7 +201,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -215,7 +215,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -229,7 +229,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -243,7 +243,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -257,7 +257,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -271,7 +271,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -285,7 +285,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -299,7 +299,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -313,7 +313,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -327,7 +327,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -341,7 +341,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -355,7 +355,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -369,7 +369,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -383,7 +383,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -397,7 +397,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -411,7 +411,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     },
@@ -425,7 +425,7 @@
         "str x20, [x28, #24]",
         "mov w1, #0x401",
         "str x1, [x28, #1488]",
-        "ldr x0, [x28, #2880]",
+        "ldr x0, [x28, #2872]",
         "br x0"
       ]
     }

unittests/InstructionCountCI/FlagM/x87_f64-Crysis2Max-fmodel.json

@@ -2085,220 +2085,220 @@
         "fcvt s2, d2",
         "stur s2, [x9, #-68]",
         "ldur s2, [x9, #-68]",
-        "fcvt d2, s2",
-        "ldur s4, [x9, #-72]",
-        "fcvt d4, s4",
-        "fadd d4, d2, d4",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-72]",
-        "ldur s4, [x9, #-72]",
-        "fcvt d4, s4",
-        "ldur s5, [x9, #-80]",
+        "fcvt d4, s2",
+        "ldur s5, [x9, #-72]",
         "fcvt d5, s5",
-        "fadd d4, d4, d5",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-80]",
-        "ldur s4, [x9, #-72]",
-        "fcvt d4, s4",
+        "fadd d5, d4, d5",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-72]",
+        "ldur s5, [x9, #-72]",
+        "fcvt d5, s5",
+        "ldur s6, [x9, #-80]",
+        "fcvt d6, s6",
+        "fadd d5, d5, d6",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-80]",
+        "ldur s5, [x9, #-72]",
+        "fcvt d5, s5",
+        "ldur s6, [x9, #-76]",
+        "fcvt d6, s6",
+        "fadd d5, d5, d6",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-72]",
         "ldur s5, [x9, #-76]",
         "fcvt d5, s5",
-        "fadd d4, d4, d5",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-72]",
-        "ldur s4, [x9, #-76]",
-        "fcvt d4, s4",
-        "fadd d4, d2, d4",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-76]",
-        "ldur s4, [x9, #-188]",
-        "fcvt d4, s4",
-        "ldur s5, [x9, #-192]",
-        "fcvt d5, s5",
-        "fadd d4, d4, d5",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-64]",
-        "ldur s4, [x9, #-192]",
-        "fcvt d4, s4",
+        "fadd d5, d4, d5",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-76]",
         "ldur s5, [x9, #-188]",
         "fcvt d5, s5",
-        "fsub d4, d4, d5",
-        "fmul d4, d4, d3",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-60]",
-        "ldur s4, [x9, #-180]",
-        "fcvt d4, s4",
-        "ldur s5, [x9, #-184]",
+        "ldur s6, [x9, #-192]",
+        "fcvt d6, s6",
+        "fadd d5, d5, d6",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-64]",
+        "ldur s5, [x9, #-192]",
         "fcvt d5, s5",
-        "fadd d4, d4, d5",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-56]",
-        "ldur s4, [x9, #-180]",
-        "fcvt d4, s4",
-        "ldur s5, [x9, #-184]",
+        "ldur s6, [x9, #-188]",
+        "fcvt d6, s6",
+        "fsub d5, d5, d6",
+        "fmul d5, d5, d3",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-60]",
+        "ldur s5, [x9, #-180]",
         "fcvt d5, s5",
-        "fsub d4, d4, d5",
-        "fmul d4, d4, d3",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-52]",
-        "ldur s4, [x9, #-56]",
-        "fcvt d4, s4",
-        "ldur s5, [x9, #-52]",
+        "ldur s6, [x9, #-184]",
+        "fcvt d6, s6",
+        "fadd d5, d5, d6",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-56]",
+        "ldur s5, [x9, #-180]",
         "fcvt d5, s5",
-        "fadd d4, d4, d5",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-56]",
-        "ldur s4, [x9, #-172]",
-        "fcvt d4, s4",
-        "ldur s5, [x9, #-176]",
+        "ldur s6, [x9, #-184]",
+        "fcvt d6, s6",
+        "fsub d5, d5, d6",
+        "fmul d5, d5, d3",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-52]",
+        "ldur s5, [x9, #-56]",
         "fcvt d5, s5",
-        "fadd d4, d4, d5",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-48]",
-        "ldur s4, [x9, #-176]",
-        "fcvt d4, s4",
+        "ldur s6, [x9, #-52]",
+        "fcvt d6, s6",
+        "fadd d5, d5, d6",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-56]",
         "ldur s5, [x9, #-172]",
         "fcvt d5, s5",
-        "fsub d4, d4, d5",
-        "fmul d4, d4, d3",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-44]",
-        "ldur s4, [x9, #-164]",
-        "fcvt d4, s4",
-        "ldur s5, [x9, #-168]",
+        "ldur s6, [x9, #-176]",
+        "fcvt d6, s6",
+        "fadd d5, d5, d6",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-48]",
+        "ldur s5, [x9, #-176]",
         "fcvt d5, s5",
-        "fadd d4, d4, d5",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-40]",
-        "ldur s4, [x9, #-164]",
-        "fcvt d4, s4",
-        "ldur s5, [x9, #-168]",
+        "ldur s6, [x9, #-172]",
+        "fcvt d6, s6",
+        "fsub d5, d5, d6",
+        "fmul d5, d5, d3",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-44]",
+        "ldur s5, [x9, #-164]",
         "fcvt d5, s5",
-        "fsub d4, d4, d5",
-        "fmul d4, d4, d3",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-36]",
-        "ldur s4, [x9, #-40]",
-        "fcvt d4, s4",
-        "ldur s5, [x9, #-36]",
+        "ldur s6, [x9, #-168]",
+        "fcvt d6, s6",
+        "fadd d5, d5, d6",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-40]",
+        "ldur s5, [x9, #-164]",
         "fcvt d5, s5",
-        "fadd d4, d4, d5",
+        "ldur s6, [x9, #-168]",
+        "fcvt d6, s6",
+        "fsub d5, d5, d6",
+        "fmul d5, d5, d3",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-36]",
+        "ldur s5, [x9, #-40]",
+        "fcvt d5, s5",
+        "ldur s6, [x9, #-36]",
+        "fcvt d6, s6",
+        "fadd d5, d5, d6",
         "strb wzr, [x28, #1049]",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-40]",
-        "ldur s4, [x9, #-48]",
-        "fcvt d4, s4",
-        "ldur s6, [x9, #-40]",
-        "fcvt d6, s6",
-        "fadd d4, d4, d6",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-48]",
-        "ldur s4, [x9, #-44]",
-        "fcvt d4, s4",
-        "ldur s6, [x9, #-40]",
-        "fcvt d6, s6",
-        "fadd d4, d4, d6",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-40]",
-        "ldur s4, [x9, #-44]",
-        "fcvt d4, s4",
-        "fadd d4, d4, d5",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-44]",
-        "ldur s4, [x9, #-160]",
-        "fcvt d4, s4",
-        "ldur s6, [x9, #-156]",
-        "fcvt d6, s6",
-        "fadd d4, d4, d6",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-32]",
-        "ldur s4, [x9, #-160]",
-        "fcvt d4, s4",
-        "ldur s6, [x9, #-156]",
-        "fcvt d6, s6",
-        "fsub d4, d4, d6",
-        "fmul d4, d4, d3",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-28]",
-        "ldur s4, [x9, #-152]",
-        "fcvt d4, s4",
-        "ldur s6, [x9, #-148]",
-        "fcvt d6, s6",
-        "fadd d4, d4, d6",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-24]",
-        "ldur s4, [x9, #-148]",
-        "fcvt d4, s4",
-        "ldur s6, [x9, #-152]",
-        "fcvt d6, s6",
-        "fsub d4, d4, d6",
-        "fmul d4, d4, d3",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-20]",
-        "ldur s4, [x9, #-24]",
-        "fcvt d4, s4",
-        "ldur s6, [x9, #-20]",
-        "fcvt d6, s6",
-        "fadd d4, d4, d6",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-24]",
-        "ldur s4, [x9, #-144]",
-        "fcvt d4, s4",
-        "ldur s6, [x9, #-140]",
-        "fcvt d6, s6",
-        "fadd d4, d4, d6",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-16]",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-40]",
+        "ldur s5, [x9, #-48]",
+        "fcvt d5, s5",
+        "ldur s7, [x9, #-40]",
+        "fcvt d7, s7",
+        "fadd d5, d5, d7",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-48]",
+        "ldur s5, [x9, #-44]",
+        "fcvt d5, s5",
+        "ldur s7, [x9, #-40]",
+        "fcvt d7, s7",
+        "fadd d5, d5, d7",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-40]",
+        "ldur s5, [x9, #-44]",
+        "fcvt d5, s5",
+        "fadd d5, d5, d6",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-44]",
+        "ldur s5, [x9, #-160]",
+        "fcvt d5, s5",
+        "ldur s7, [x9, #-156]",
+        "fcvt d7, s7",
+        "fadd d5, d5, d7",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-32]",
+        "ldur s5, [x9, #-160]",
+        "fcvt d5, s5",
+        "ldur s7, [x9, #-156]",
+        "fcvt d7, s7",
+        "fsub d5, d5, d7",
+        "fmul d5, d5, d3",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-28]",
+        "ldur s5, [x9, #-152]",
+        "fcvt d5, s5",
+        "ldur s7, [x9, #-148]",
+        "fcvt d7, s7",
+        "fadd d5, d5, d7",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-24]",
+        "ldur s5, [x9, #-148]",
+        "fcvt d5, s5",
+        "ldur s7, [x9, #-152]",
+        "fcvt d7, s7",
+        "fsub d5, d5, d7",
+        "fmul d5, d5, d3",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-20]",
+        "ldur s5, [x9, #-24]",
+        "fcvt d5, s5",
+        "ldur s7, [x9, #-20]",
+        "fcvt d7, s7",
+        "fadd d5, d5, d7",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-24]",
+        "ldur s5, [x9, #-144]",
+        "fcvt d5, s5",
+        "ldur s7, [x9, #-140]",
+        "fcvt d7, s7",
+        "fadd d5, d5, d7",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-16]",
         "ldr w4, [x9, #8]",
-        "ldur s4, [x9, #-144]",
-        "fcvt d4, s4",
+        "ldur s5, [x9, #-144]",
+        "fcvt d5, s5",
         "ldr w7, [x9, #12]",
-        "ldur s6, [x9, #-140]",
-        "fcvt d6, s6",
-        "fsub d4, d4, d6",
-        "fmul d4, d4, d3",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-12]",
-        "ldur s4, [x9, #-136]",
-        "fcvt d4, s4",
-        "ldur s6, [x9, #-132]",
-        "fcvt d6, s6",
-        "fadd d4, d4, d6",
-        "fcvt s4, d4",
-        "stur s4, [x9, #-8]",
-        "ldur s4, [x9, #-132]",
-        "fcvt d4, s4",
-        "ldur s6, [x9, #-136]",
-        "fcvt d6, s6",
-        "fsub d4, d4, d6",
-        "fmul d3, d3, d4",
+        "ldur s7, [x9, #-140]",
+        "fcvt d7, s7",
+        "fsub d5, d5, d7",
+        "fmul d5, d5, d3",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-12]",
+        "ldur s5, [x9, #-136]",
+        "fcvt d5, s5",
+        "ldur s7, [x9, #-132]",
+        "fcvt d7, s7",
+        "fadd d5, d5, d7",
+        "fcvt s5, d5",
+        "stur s5, [x9, #-8]",
+        "ldur s5, [x9, #-132]",
+        "fcvt d5, s5",
+        "ldur s7, [x9, #-136]",
+        "fcvt d7, s7",
+        "fsub d5, d5, d7",
+        "fmul d3, d3, d5",
         "strb wzr, [x28, #1049]",
         "fcvt s3, d3",
         "stur s3, [x9, #-4]",
         "ldur s3, [x9, #-8]",
         "fcvt d3, s3",
-        "ldur s4, [x9, #-4]",
-        "fcvt d4, s4",
-        "fadd d3, d3, d4",
+        "ldur s5, [x9, #-4]",
+        "fcvt d7, s5",
+        "fadd d3, d3, d7",
         "strb wzr, [x28, #1049]",
         "fcvt s3, d3",
         "stur s3, [x9, #-8]",
         "ldur s3, [x9, #-16]",
         "fcvt d3, s3",
-        "ldur s6, [x9, #-8]",
-        "fcvt d6, s6",
-        "fadd d3, d3, d6",
+        "ldur s8, [x9, #-8]",
+        "fcvt d8, s8",
+        "fadd d3, d3, d8",
         "fcvt s3, d3",
         "stur s3, [x9, #-16]",
         "ldur s3, [x9, #-12]",
         "fcvt d3, s3",
-        "ldur s6, [x9, #-8]",
-        "fcvt d6, s6",
-        "fadd d3, d3, d6",
+        "ldur s8, [x9, #-8]",
+        "fcvt d8, s8",
+        "fadd d3, d3, d8",
         "fcvt s3, d3",
         "stur s3, [x9, #-8]",
         "ldur s3, [x9, #-12]",
         "fcvt d3, s3",
-        "fadd d3, d3, d4",
+        "fadd d3, d3, d7",
         "fcvt s3, d3",
         "stur s3, [x9, #-12]",
         "ldur s3, [x9, #-128]",
@@ -2321,67 +2321,66 @@
         "str s3, [x7, #768]",
         "ldur s3, [x9, #-80]",
         "fcvt d3, s3",
-        "ldur s6, [x9, #-96]",
-        "fcvt d6, s6",
-        "fadd d3, d3, d6",
+        "ldur s8, [x9, #-96]",
+        "fcvt d8, s8",
+        "fadd d3, d3, d8",
         "fcvt s3, d3",
         "stur s3, [x9, #-96]",
         "ldur s3, [x9, #-96]",
         "str s3, [x4, #896]",
         "ldur s3, [x9, #-80]",
         "fcvt d3, s3",
-        "ldur s6, [x9, #-88]",
-        "fcvt d6, s6",
-        "fadd d3, d3, d6",
+        "ldur s8, [x9, #-88]",
+        "fcvt d8, s8",
+        "fadd d3, d3, d8",
         "fcvt s3, d3",
         "stur s3, [x9, #-80]",
         "ldur s3, [x9, #-80]",
         "str s3, [x4, #640]",
         "ldur s3, [x9, #-72]",
         "fcvt d3, s3",
-        "ldur s6, [x9, #-88]",
-        "fcvt d6, s6",
-        "fadd d3, d3, d6",
+        "ldur s8, [x9, #-88]",
+        "fcvt d8, s8",
+        "fadd d3, d3, d8",
         "fcvt s3, d3",
         "stur s3, [x9, #-88]",
         "ldur s3, [x9, #-88]",
         "str s3, [x4, #384]",
         "ldur s3, [x9, #-72]",
         "fcvt d3, s3",
-        "ldur s6, [x9, #-92]",
-        "fcvt d6, s6",
-        "fadd d3, d3, d6",
+        "ldur s8, [x9, #-92]",
+        "fcvt d8, s8",
+        "fadd d3, d3, d8",
         "fcvt s3, d3",
         "stur s3, [x9, #-72]",
         "ldur s3, [x9, #-72]",
         "str s3, [x4, #128]",
         "ldur s3, [x9, #-76]",
         "fcvt d3, s3",
-        "ldur s6, [x9, #-92]",
-        "fcvt d6, s6",
-        "fadd d3, d3, d6",
+        "ldur s8, [x9, #-92]",
+        "fcvt d8, s8",
+        "fadd d3, d3, d8",
         "fcvt s3, d3",
         "stur s3, [x9, #-92]",
         "ldur s3, [x9, #-92]",
         "str s3, [x7, #128]",
         "ldur s3, [x9, #-76]",
         "fcvt d3, s3",
-        "ldur s6, [x9, #-84]",
-        "fcvt d6, s6",
-        "fadd d3, d3, d6",
+        "ldur s8, [x9, #-84]",
+        "fcvt d8, s8",
+        "fadd d3, d3, d8",
         "fcvt s3, d3",
         "stur s3, [x9, #-76]",
         "ldur s3, [x9, #-76]",
         "str s3, [x7, #384]",
         "ldur s3, [x9, #-84]",
         "fcvt d3, s3",
-        "fadd d3, d2, d3",
+        "fadd d3, d4, d3",
         "fcvt s3, d3",
         "stur s3, [x9, #-84]",
         "ldur s3, [x9, #-84]",
         "str s3, [x7, #640]",
         "strb wzr, [x28, #1049]",
-        "fcvt s2, d2",
         "str s2, [x7, #896]",
         "ldur s2, [x9, #-32]",
         "fcvt d2, s2",
@@ -2511,7 +2510,7 @@
         "str s2, [x7, #448]",
         "ldur s2, [x9, #-20]",
         "fcvt d2, s2",
-        "fadd d2, d2, d4",
+        "fadd d2, d2, d7",
         "fcvt s2, d2",
         "stur s2, [x9, #-20]",
         "ldur s2, [x9, #-52]",
@@ -2523,15 +2522,14 @@
         "str s2, [x7, #576]",
         "ldur s2, [x9, #-20]",
         "fcvt d2, s2",
-        "fadd d2, d5, d2",
+        "fadd d2, d6, d2",
         "fcvt s2, d2",
         "str s2, [x7, #704]",
-        "fadd d2, d5, d4",
+        "fadd d2, d6, d7",
         "strb wzr, [x28, #1049]",
         "fcvt s2, d2",
         "str s2, [x7, #832]",
-        "fcvt s2, d4",
-        "str s2, [x7, #960]",
+        "str s5, [x7, #960]",
         "mov x8, x9",
         "ldp w9, w20, [x8], #8",
         "ldrb w21, [x28, #1051]",
@@ -2544,7 +2542,7 @@
         "strb w21, [x28, #1202]"
       ],
       "x86InstructionCount": 809,
-      "ExpectedInstructionCount": 1714
+      "ExpectedInstructionCount": 1712
     }
   }
 }

unittests/InstructionCountCI/FlagM/x87_f64-Oblivion.json

@@ -17,7 +17,7 @@
   "Instructions": {
     "Block1": {
       "x86InstructionCount": 911,
-      "ExpectedInstructionCount": 1697,
+      "ExpectedInstructionCount": 1695,
       "x86Insts": [
         "sub esp,0x118",
         "fld dword [ecx + 0x1084]",
@@ -2364,213 +2364,211 @@
         "fcvt s2, d2",
         "str s2, [x8, #60]",
         "ldr s2, [x8, #60]",
-        "fcvt d2, s2",
-        "ldr s3, [x8, #28]",
-        "fcvt d3, s3",
-        "fadd d4, d2, d3",
-        "strb wzr, [x28, #1049]",
-        "fcvt s4, d4",
-        "str s4, [x8, #196]",
-        "ldr s4, [x8, #196]",
+        "fcvt d3, s2",
+        "ldr s4, [x8, #28]",
         "fcvt d4, s4",
-        "ldr s5, [x8, #44]",
-        "fcvt d5, s5",
-        "fadd d6, d4, d5",
-        "strb wzr, [x28, #1049]",
-        "fcvt s6, d6",
-        "str s6, [x8, #188]",
-        "ldr s6, [x8, #188]",
-        "fcvt d6, s6",
-        "ldr s7, [x8, #20]",
-        "fcvt d7, s7",
-        "fadd d6, d6, d7",
-        "ldr s7, [x8, #52]",
-        "fcvt d7, s7",
-        "fadd d6, d6, d7",
-        "strb wzr, [x28, #1049]",
-        "fcvt s6, d6",
-        "str s6, [x8, #164]",
-        "fadd d6, d2, d5",
-        "ldr s8, [x8, #12]",
-        "fcvt d8, s8",
-        "fadd d6, d6, d8",
-        "fcvt s6, d6",
-        "str s6, [x8, #180]",
-        "ldr s6, [x8, #180]",
-        "fcvt d6, s6",
-        "fadd d6, d6, d7",
-        "fcvt s6, d6",
-        "str s6, [x8, #172]",
-        "fadd d6, d2, d7",
-        "ldr s8, [x8, #36]",
-        "fcvt d8, s8",
-        "fadd d6, d6, d8",
-        "fcvt s6, d6",
-        "str s6, [x8, #64]",
-        "ldr s6, [x8, #64]",
-        "fcvt d6, s6",
-        "ldr s8, [x8, #4]",
-        "fcvt d8, s8",
-        "fadd d6, d6, d8",
-        "fcvt s6, d6",
-        "str s6, [x8, #148]",
-        "ldr s6, [x8, #148]",
-        "fcvt d6, s6",
-        "fneg v6.2d, v6.2d",
-        "fcvt s6, d6",
-        "str s6, [x8, #272]",
-        "ldr s6, [x8, #272]",
-        "fcvt d6, s6",
-        "ldr s8, [x8, #56]",
-        "fcvt d8, s8",
-        "fsub d6, d6, d8",
-        "strb wzr, [x28, #1049]",
-        "fcvt s6, d6",
-        "str s6, [x8, #208]",
-        "ldr s6, [x8, #64]",
-        "fcvt d6, s6",
-        "ldr s9, [x8, #20]",
-        "fcvt d9, s9",
-        "fadd d6, d6, d9",
-        "fadd d6, d6, d3",
-        "fcvt s6, d6",
-        "str s6, [x8, #156]",
-        "ldr s6, [x8, #156]",
-        "fcvt d6, s6",
-        "fneg v6.2d, v6.2d",
-        "fcvt s6, d6",
-        "str s6, [x8, #136]",
-        "ldr s6, [x8, #136]",
-        "fcvt d6, s6",
-        "ldr s9, [x8, #24]",
-        "fcvt d9, s9",
-        "fsub d6, d6, d9",
-        "fsub d6, d6, d8",
-        "fcvt s6, d6",
-        "str s6, [x8, #216]",
-        "ldr s6, [x8, #40]",
-        "fcvt d6, s6",
-        "fneg v6.2d, v6.2d",
-        "fsub d5, d6, d5",
-        "fsub d5, d5, d8",
-        "fsub d5, d5, d2",
+        "fadd d5, d3, d4",
         "strb wzr, [x28, #1049]",
         "fcvt s5, d5",
-        "str s5, [x8, #64]",
-        "ldr s5, [x8, #64]",
-        "fcvt d5, s5",
-        "fsub d6, d5, d7",
-        "ldr s7, [x8, #8]",
+        "str s5, [x8, #196]",
+        "ldr s5, [x8, #196]",
+        "fcvt d6, s5",
+        "ldr s7, [x8, #44]",
         "fcvt d7, s7",
-        "fsub d7, d6, d7",
-        "ldr s9, [x8, #12]",
+        "fadd d8, d6, d7",
+        "strb wzr, [x28, #1049]",
+        "fcvt s8, d8",
+        "str s8, [x8, #188]",
+        "ldr s8, [x8, #188]",
+        "fcvt d8, s8",
+        "ldr s9, [x8, #20]",
         "fcvt d9, s9",
-        "fsub d7, d7, d9",
+        "fadd d8, d8, d9",
+        "ldr s9, [x8, #52]",
+        "fcvt d9, s9",
+        "fadd d8, d8, d9",
+        "strb wzr, [x28, #1049]",
+        "fcvt s8, d8",
+        "str s8, [x8, #164]",
+        "fadd d8, d3, d7",
+        "ldr s10, [x8, #12]",
+        "fcvt d10, s10",
+        "fadd d8, d8, d10",
+        "fcvt s8, d8",
+        "str s8, [x8, #180]",
+        "ldr s8, [x8, #180]",
+        "fcvt d8, s8",
+        "fadd d8, d8, d9",
+        "fcvt s8, d8",
+        "str s8, [x8, #172]",
+        "fadd d8, d3, d9",
+        "ldr s10, [x8, #36]",
+        "fcvt d10, s10",
+        "fadd d8, d8, d10",
+        "fcvt s8, d8",
+        "str s8, [x8, #64]",
+        "ldr s8, [x8, #64]",
+        "fcvt d8, s8",
+        "ldr s10, [x8, #4]",
+        "fcvt d10, s10",
+        "fadd d8, d8, d10",
+        "fcvt s8, d8",
+        "str s8, [x8, #148]",
+        "ldr s8, [x8, #148]",
+        "fcvt d8, s8",
+        "fneg v8.2d, v8.2d",
+        "fcvt s8, d8",
+        "str s8, [x8, #272]",
+        "ldr s8, [x8, #272]",
+        "fcvt d8, s8",
+        "ldr s10, [x8, #56]",
+        "fcvt d10, s10",
+        "fsub d8, d8, d10",
+        "strb wzr, [x28, #1049]",
+        "fcvt s8, d8",
+        "str s8, [x8, #208]",
+        "ldr s8, [x8, #64]",
+        "fcvt d8, s8",
+        "ldr s11, [x8, #20]",
+        "fcvt d11, s11",
+        "fadd d8, d8, d11",
+        "fadd d8, d8, d4",
+        "fcvt s8, d8",
+        "str s8, [x8, #156]",
+        "ldr s8, [x8, #156]",
+        "fcvt d8, s8",
+        "fneg v8.2d, v8.2d",
+        "fcvt s8, d8",
+        "str s8, [x8, #136]",
+        "ldr s8, [x8, #136]",
+        "fcvt d8, s8",
+        "ldr s11, [x8, #24]",
+        "fcvt d11, s11",
+        "fsub d8, d8, d11",
+        "fsub d8, d8, d10",
+        "fcvt s8, d8",
+        "str s8, [x8, #216]",
+        "ldr s8, [x8, #40]",
+        "fcvt d8, s8",
+        "fneg v8.2d, v8.2d",
+        "fsub d7, d8, d7",
+        "fsub d7, d7, d10",
+        "fsub d7, d7, d3",
+        "strb wzr, [x28, #1049]",
         "fcvt s7, d7",
-        "str s7, [x8, #232]",
-        "ldr s7, [x8, #20]",
+        "str s7, [x8, #64]",
+        "ldr s7, [x8, #64]",
         "fcvt d7, s7",
-        "fsub d6, d6, d7",
-        "ldr s7, [x8, #24]",
-        "fcvt d7, s7",
-        "fsub d6, d6, d7",
-        "fsub d6, d6, d3",
+        "fsub d8, d7, d9",
+        "ldr s9, [x8, #8]",
+        "fcvt d9, s9",
+        "fsub d9, d8, d9",
+        "ldr s11, [x8, #12]",
+        "fcvt d11, s11",
+        "fsub d9, d9, d11",
+        "fcvt s9, d9",
+        "str s9, [x8, #232]",
+        "ldr s9, [x8, #20]",
+        "fcvt d9, s9",
+        "fsub d8, d8, d9",
+        "ldr s9, [x8, #24]",
+        "fcvt d9, s9",
+        "fsub d8, d8, d9",
+        "fsub d8, d8, d4",
         "mov w20, #0x0",
         "strb wzr, [x28, #1049]",
-        "fcvt s6, d6",
-        "str s6, [x8, #224]",
-        "ldr s6, [x8, #48]",
-        "fcvt d6, s6",
-        "fsub d5, d5, d6",
-        "ldr s7, [x8, #8]",
-        "fcvt d7, s7",
-        "fsub d7, d5, d7",
-        "ldr s9, [x8, #12]",
+        "fcvt s8, d8",
+        "str s8, [x8, #224]",
+        "ldr s8, [x8, #48]",
+        "fcvt d8, s8",
+        "fsub d7, d7, d8",
+        "ldr s9, [x8, #8]",
         "fcvt d9, s9",
-        "fsub d7, d7, d9",
-        "fcvt s7, d7",
-        "str s7, [x8, #240]",
-        "ldr s7, [x8, #24]",
-        "fcvt d7, s7",
-        "ldr s9, [x8, #16]",
+        "fsub d9, d7, d9",
+        "ldr s11, [x8, #12]",
+        "fcvt d11, s11",
+        "fsub d9, d9, d11",
+        "fcvt s9, d9",
+        "str s9, [x8, #240]",
+        "ldr s9, [x8, #24]",
         "fcvt d9, s9",
-        "fadd d7, d7, d9",
-        "fadd d3, d3, d7",
+        "ldr s11, [x8, #16]",
+        "fcvt d11, s11",
+        "fadd d9, d9, d11",
+        "fadd d4, d4, d9",
         "ldr w4, [x7, #4100]",
         "ldr w5, [x7, #4096]",
         "strb wzr, [x28, #1049]",
         "add w4, w5, w4, lsl #2",
-        "fcvt s3, d3",
-        "str s3, [x8, #64]",
-        "ldr s3, [x8, #64]",
-        "fcvt d3, s3",
-        "fsub d5, d5, d3",
+        "fcvt s4, d4",
+        "str s4, [x8, #64]",
+        "ldr s4, [x8, #64]",
+        "fcvt d4, s4",
+        "fsub d7, d7, d4",
         "strb wzr, [x28, #1049]",
-        "fcvt s5, d5",
-        "str s5, [x8, #248]",
-        "ldr s5, [x8, #32]",
-        "fcvt d5, s5",
-        "fneg v5.2d, v5.2d",
-        "fsub d5, d5, d6",
+        "fcvt s7, d7",
+        "str s7, [x8, #248]",
+        "ldr s7, [x8, #32]",
+        "fcvt d7, s7",
+        "fneg v7.2d, v7.2d",
+        "fsub d7, d7, d8",
         "strb wzr, [x28, #1049]",
-        "fsub d5, d5, d8",
-        "fsub d5, d5, d2",
-        "fcvt s5, d5",
-        "str s5, [x8, #64]",
-        "ldr s5, [x8, #64]",
-        "fcvt d5, s5",
-        "ldr s6, [x8]",
-        "fcvt d6, s6",
-        "fsub d6, d5, d6",
-        "fcvt s6, d6",
-        "str s6, [x8, #264]",
-        "fsub d3, d5, d3",
+        "fsub d7, d7, d10",
+        "fsub d7, d7, d3",
+        "fcvt s7, d7",
+        "str s7, [x8, #64]",
+        "ldr s7, [x8, #64]",
+        "fcvt d7, s7",
+        "ldr s8, [x8]",
+        "fcvt d8, s8",
+        "fsub d8, d7, d8",
+        "fcvt s8, d8",
+        "str s8, [x8, #264]",
+        "fsub d4, d7, d4",
         "strb wzr, [x28, #1049]",
-        "fcvt s3, d3",
-        "str s3, [x8, #256]",
-        "ldr s3, [x8, #144]",
-        "str s3, [x4]",
-        "ldr s3, [x8, #148]",
-        "str s3, [x4, #64]",
-        "ldr s3, [x8, #152]",
-        "str s3, [x4, #128]",
-        "ldr s3, [x8, #156]",
-        "str s3, [x4, #192]",
-        "ldr s3, [x8, #160]",
-        "str s3, [x4, #256]",
-        "ldr s3, [x8, #164]",
-        "fcvt d5, s3",
-        "str s3, [x4, #320]",
-        "ldr s3, [x8, #168]",
-        "fcvt d6, s3",
-        "str s3, [x4, #384]",
-        "ldr s3, [x8, #172]",
-        "fcvt d7, s3",
-        "str s3, [x4, #448]",
-        "ldr s3, [x8, #176]",
-        "fcvt d8, s3",
-        "str s3, [x4, #512]",
-        "ldr s3, [x8, #180]",
-        "str s3, [x4, #576]",
-        "ldr s3, [x8, #184]",
-        "fcvt d9, s3",
-        "str s3, [x4, #640]",
-        "ldr s3, [x8, #188]",
-        "str s3, [x4, #704]",
-        "ldr s3, [x8, #192]",
-        "str s3, [x4, #768]",
+        "fcvt s4, d4",
+        "str s4, [x8, #256]",
+        "ldr s4, [x8, #144]",
+        "str s4, [x4]",
+        "ldr s4, [x8, #148]",
+        "str s4, [x4, #64]",
+        "ldr s4, [x8, #152]",
+        "str s4, [x4, #128]",
+        "ldr s4, [x8, #156]",
+        "str s4, [x4, #192]",
+        "ldr s4, [x8, #160]",
+        "str s4, [x4, #256]",
+        "ldr s4, [x8, #164]",
+        "fcvt d7, s4",
+        "str s4, [x4, #320]",
+        "ldr s4, [x8, #168]",
+        "fcvt d8, s4",
+        "str s4, [x4, #384]",
+        "ldr s4, [x8, #172]",
+        "fcvt d9, s4",
+        "str s4, [x4, #448]",
+        "ldr s4, [x8, #176]",
+        "fcvt d10, s4",
+        "str s4, [x4, #512]",
+        "ldr s4, [x8, #180]",
+        "str s4, [x4, #576]",
+        "ldr s4, [x8, #184]",
+        "fcvt d11, s4",
+        "str s4, [x4, #640]",
+        "ldr s4, [x8, #188]",
+        "str s4, [x4, #704]",
+        "ldr s4, [x8, #192]",
+        "str s4, [x4, #768]",
         "strb wzr, [x28, #1049]",
-        "fcvt s3, d4",
-        "str s3, [x4, #832]",
-        "ldr s3, [x8, #200]",
-        "str s3, [x4, #896]",
+        "str s5, [x4, #832]",
+        "ldr s4, [x8, #200]",
+        "str s4, [x4, #896]",
         "strb wzr, [x28, #1049]",
-        "fcvt s3, d2",
-        "str s3, [x4, #960]",
-        "fmov d3, x20",
-        "fcvt s3, d3",
-        "str s3, [x4, #1024]",
-        "fneg v2.2d, v2.2d",
+        "str s2, [x4, #960]",
+        "fmov d2, x20",
+        "fcvt s2, d2",
+        "str s2, [x4, #1024]",
+        "fneg v2.2d, v3.2d",
         "fcvt s2, d2",
         "str s2, [x4, #1088]",
         "ldr s2, [x8, #200]",
@@ -2579,7 +2577,7 @@
         "fcvt s2, d2",
         "str s2, [x4, #1152]",
         "strb wzr, [x28, #1049]",
-        "fneg v2.2d, v4.2d",
+        "fneg v2.2d, v6.2d",
         "fcvt s2, d2",
         "str s2, [x4, #1216]",
         "ldr s2, [x8, #192]",
@@ -2593,7 +2591,7 @@
         "fcvt s2, d2",
         "str s2, [x4, #1344]",
         "strb wzr, [x28, #1049]",
-        "fneg v2.2d, v9.2d",
+        "fneg v2.2d, v11.2d",
         "fcvt s2, d2",
         "str s2, [x4, #1408]",
         "ldr s2, [x8, #180]",
@@ -2602,18 +2600,18 @@
         "fcvt s2, d2",
         "str s2, [x4, #1472]",
         "strb wzr, [x28, #1049]",
-        "fneg v2.2d, v8.2d",
+        "fneg v2.2d, v10.2d",
         "fcvt s2, d2",
         "str s2, [x4, #1536]",
         "strb wzr, [x28, #1049]",
-        "fneg v2.2d, v7.2d",
+        "fneg v2.2d, v9.2d",
         "fcvt s2, d2",
         "str s2, [x4, #1600]",
         "strb wzr, [x28, #1049]",
-        "fneg v2.2d, v6.2d",
+        "fneg v2.2d, v8.2d",
         "fcvt s2, d2",
         "str s2, [x4, #1664]",
-        "fneg v2.2d, v5.2d",
+        "fneg v2.2d, v7.2d",
         "fcvt s2, d2",
         "str s2, [x4, #1728]",
         "ldr s2, [x8, #140]",
@@ -4209,7 +4207,7 @@
     },
     "Block3": {
       "x86InstructionCount": 649,
-      "ExpectedInstructionCount": 982,
+      "ExpectedInstructionCount": 958,
       "x86Insts": [
         "fld dword [esi + 0x64]",
         "mov eax,dword [esi + 0x88]",
@@ -4939,62 +4937,59 @@
         "fcvt s3, d3",
         "str s3, [x8, #40]",
         "ldr s3, [x8, #40]",
-        "fcvt d4, s3",
         "str s3, [x8, #16]",
-        "ldr s3, [x8, #80]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #40]",
-        "ldr s3, [x8, #40]",
-        "fcvt d5, s3",
-        "str s3, [x8, #44]",
-        "ldr s3, [x8, #88]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #40]",
-        "ldr s3, [x8, #40]",
-        "fcvt d6, s3",
-        "str s3, [x8, #20]",
-        "ldr s3, [x8, #140]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #76]",
-        "ldr s3, [x8, #76]",
-        "str s3, [x8, #68]",
-        "ldr s3, [x8, #124]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #140]",
-        "ldr s3, [x8, #140]",
-        "str s3, [x8, #72]",
-        "ldr s3, [x8, #132]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #88]",
-        "ldr s3, [x8, #88]",
-        "str s3, [x8, #64]",
-        "ldr s3, [x8, #92]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #80]",
-        "ldr s3, [x8, #80]",
-        "str s3, [x8, #132]",
-        "ldr s3, [x8, #96]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #84]",
-        "ldr s3, [x8, #84]",
-        "str s3, [x8, #124]",
-        "ldr s3, [x8, #100]",
-        "fcvt d3, s3",
-        "fmul d2, d2, d3",
+        "ldr s4, [x8, #80]",
+        "fcvt d4, s4",
+        "fmul d4, d4, d2",
+        "fcvt s4, d4",
+        "str s4, [x8, #40]",
+        "ldr s4, [x8, #40]",
+        "str s4, [x8, #44]",
+        "ldr s5, [x8, #88]",
+        "fcvt d5, s5",
+        "fmul d5, d5, d2",
+        "fcvt s5, d5",
+        "str s5, [x8, #40]",
+        "ldr s5, [x8, #40]",
+        "str s5, [x8, #20]",
+        "ldr s6, [x8, #140]",
+        "fcvt d6, s6",
+        "fmul d6, d6, d2",
+        "fcvt s6, d6",
+        "str s6, [x8, #76]",
+        "ldr s6, [x8, #76]",
+        "str s6, [x8, #68]",
+        "ldr s6, [x8, #124]",
+        "fcvt d6, s6",
+        "fmul d6, d6, d2",
+        "fcvt s6, d6",
+        "str s6, [x8, #140]",
+        "ldr s6, [x8, #140]",
+        "str s6, [x8, #72]",
+        "ldr s6, [x8, #132]",
+        "fcvt d6, s6",
+        "fmul d6, d6, d2",
+        "fcvt s6, d6",
+        "str s6, [x8, #88]",
+        "ldr s6, [x8, #88]",
+        "str s6, [x8, #64]",
+        "ldr s6, [x8, #92]",
+        "fcvt d6, s6",
+        "fmul d6, d6, d2",
+        "fcvt s6, d6",
+        "str s6, [x8, #80]",
+        "ldr s6, [x8, #80]",
+        "str s6, [x8, #132]",
+        "ldr s6, [x8, #96]",
+        "fcvt d6, s6",
+        "fmul d6, d6, d2",
+        "fcvt s6, d6",
+        "str s6, [x8, #84]",
+        "ldr s6, [x8, #84]",
+        "str s6, [x8, #124]",
+        "ldr s6, [x8, #100]",
+        "fcvt d6, s6",
+        "fmul d2, d2, d6",
         "strb wzr, [x28, #1049]",
         "fcvt s2, d2",
         "str s2, [x8, #40]",
@@ -5002,16 +4997,16 @@
         "str s2, [x8, #92]",
         "ldr s2, [x8, #148]",
         "fcvt d2, s2",
-        "ldr s3, [x8, #132]",
-        "fcvt d3, s3",
-        "fadd d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #132]",
-        "ldr s3, [x8, #152]",
-        "fcvt d3, s3",
+        "ldr s6, [x8, #132]",
+        "fcvt d6, s6",
+        "fadd d6, d6, d2",
+        "fcvt s6, d6",
+        "str s6, [x8, #132]",
+        "ldr s6, [x8, #152]",
+        "fcvt d6, s6",
         "ldr s7, [x8, #124]",
         "fcvt d7, s7",
-        "fadd d7, d7, d3",
+        "fadd d7, d7, d6",
         "fcvt s7, d7",
         "str s7, [x8, #124]",
         "ldr s7, [x8, #156]",
@@ -5076,14 +5071,11 @@
         "ldr w5, [x8, #100]",
         "strb wzr, [x28, #1049]",
         "str w5, [x8, #252]",
-        "fcvt s8, d4",
-        "str s8, [x8, #92]",
+        "str s3, [x8, #92]",
         "strb wzr, [x28, #1049]",
-        "fcvt s8, d5",
-        "str s8, [x8, #132]",
+        "str s4, [x8, #132]",
         "strb wzr, [x28, #1049]",
-        "fcvt s8, d6",
-        "str s8, [x8, #124]",
+        "str s5, [x8, #124]",
         "ldr s8, [x8, #76]",
         "str s8, [x8, #64]",
         "ldr s8, [x8, #140]",
@@ -5103,7 +5095,7 @@
         "str s8, [x8, #20]",
         "ldr s8, [x8, #44]",
         "fcvt d8, s8",
-        "fadd d8, d8, d3",
+        "fadd d8, d8, d6",
         "fcvt s8, d8",
         "str s8, [x8, #44]",
         "ldr s8, [x8, #16]",
@@ -5166,14 +5158,11 @@
         "ldr w5, [x8, #28]",
         "strb wzr, [x28, #1049]",
         "str w5, [x8, #264]",
-        "fcvt s8, d4",
-        "str s8, [x8, #92]",
+        "str s3, [x8, #92]",
         "strb wzr, [x28, #1049]",
-        "fcvt s8, d5",
-        "str s8, [x8, #132]",
+        "str s4, [x8, #132]",
         "strb wzr, [x28, #1049]",
-        "fcvt s8, d6",
-        "str s8, [x8, #124]",
+        "str s5, [x8, #124]",
         "ldr s8, [x8, #76]",
         "str s8, [x8, #64]",
         "ldr s8, [x8, #140]",
@@ -5193,7 +5182,7 @@
         "str s8, [x8, #20]",
         "ldr s8, [x8, #44]",
         "fcvt d8, s8",
-        "fadd d8, d8, d3",
+        "fadd d8, d8, d6",
         "fcvt s8, d8",
         "str s8, [x8, #44]",
         "ldr s8, [x8, #16]",
@@ -5256,35 +5245,32 @@
         "ldr w5, [x8, #28]",
         "strb wzr, [x28, #1049]",
         "str w5, [x8, #276]",
-        "fcvt s4, d4",
-        "str s4, [x8, #92]",
+        "str s3, [x8, #92]",
         "strb wzr, [x28, #1049]",
-        "fcvt s4, d5",
         "str s4, [x8, #132]",
         "strb wzr, [x28, #1049]",
-        "fcvt s4, d6",
-        "str s4, [x8, #124]",
-        "ldr s4, [x8, #76]",
-        "str s4, [x8, #64]",
-        "ldr s4, [x8, #140]",
-        "str s4, [x8, #72]",
-        "ldr s4, [x8, #88]",
-        "str s4, [x8, #68]",
-        "ldr s4, [x8, #80]",
-        "str s4, [x8, #20]",
-        "ldr s4, [x8, #84]",
-        "str s4, [x8, #44]",
-        "ldr s4, [x8, #40]",
-        "str s4, [x8, #16]",
-        "ldr s4, [x8, #20]",
-        "fcvt d4, s4",
-        "fadd d2, d2, d4",
+        "str s5, [x8, #124]",
+        "ldr s3, [x8, #76]",
+        "str s3, [x8, #64]",
+        "ldr s3, [x8, #140]",
+        "str s3, [x8, #72]",
+        "ldr s3, [x8, #88]",
+        "str s3, [x8, #68]",
+        "ldr s3, [x8, #80]",
+        "str s3, [x8, #20]",
+        "ldr s3, [x8, #84]",
+        "str s3, [x8, #44]",
+        "ldr s3, [x8, #40]",
+        "str s3, [x8, #16]",
+        "ldr s3, [x8, #20]",
+        "fcvt d3, s3",
+        "fadd d2, d2, d3",
         "strb wzr, [x28, #1049]",
         "fcvt s2, d2",
         "str s2, [x8, #20]",
         "ldr s2, [x8, #44]",
         "fcvt d2, s2",
-        "fadd d2, d3, d2",
+        "fadd d2, d6, d2",
         "strb wzr, [x28, #1049]",
         "fcvt s2, d2",
         "str s2, [x8, #44]",
@@ -5422,62 +5408,59 @@
         "fcvt s3, d3",
         "str s3, [x8, #20]",
         "ldr s3, [x8, #20]",
-        "fcvt d4, s3",
         "str s3, [x8, #92]",
-        "ldr s3, [x8, #44]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #20]",
-        "ldr s3, [x8, #20]",
-        "fcvt d5, s3",
-        "str s3, [x8, #132]",
-        "ldr s3, [x8, #16]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #20]",
-        "ldr s3, [x8, #20]",
-        "fcvt d6, s3",
-        "str s3, [x8, #124]",
-        "ldr s3, [x8, #64]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #40]",
-        "ldr s3, [x8, #40]",
-        "str s3, [x8, #64]",
-        "ldr s3, [x8, #72]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #84]",
-        "ldr s3, [x8, #84]",
-        "str s3, [x8, #72]",
-        "ldr s3, [x8, #68]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #80]",
-        "ldr s3, [x8, #80]",
-        "str s3, [x8, #68]",
-        "ldr s3, [x8, #112]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #88]",
-        "ldr s3, [x8, #88]",
-        "str s3, [x8, #20]",
-        "ldr s3, [x8, #116]",
-        "fcvt d3, s3",
-        "fmul d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #140]",
-        "ldr s3, [x8, #140]",
-        "str s3, [x8, #44]",
-        "ldr s3, [x8, #120]",
-        "fcvt d3, s3",
-        "fmul d2, d2, d3",
+        "ldr s4, [x8, #44]",
+        "fcvt d4, s4",
+        "fmul d4, d4, d2",
+        "fcvt s4, d4",
+        "str s4, [x8, #20]",
+        "ldr s4, [x8, #20]",
+        "str s4, [x8, #132]",
+        "ldr s5, [x8, #16]",
+        "fcvt d5, s5",
+        "fmul d5, d5, d2",
+        "fcvt s5, d5",
+        "str s5, [x8, #20]",
+        "ldr s5, [x8, #20]",
+        "str s5, [x8, #124]",
+        "ldr s6, [x8, #64]",
+        "fcvt d6, s6",
+        "fmul d6, d6, d2",
+        "fcvt s6, d6",
+        "str s6, [x8, #40]",
+        "ldr s6, [x8, #40]",
+        "str s6, [x8, #64]",
+        "ldr s6, [x8, #72]",
+        "fcvt d6, s6",
+        "fmul d6, d6, d2",
+        "fcvt s6, d6",
+        "str s6, [x8, #84]",
+        "ldr s6, [x8, #84]",
+        "str s6, [x8, #72]",
+        "ldr s6, [x8, #68]",
+        "fcvt d6, s6",
+        "fmul d6, d6, d2",
+        "fcvt s6, d6",
+        "str s6, [x8, #80]",
+        "ldr s6, [x8, #80]",
+        "str s6, [x8, #68]",
+        "ldr s6, [x8, #112]",
+        "fcvt d6, s6",
+        "fmul d6, d6, d2",
+        "fcvt s6, d6",
+        "str s6, [x8, #88]",
+        "ldr s6, [x8, #88]",
+        "str s6, [x8, #20]",
+        "ldr s6, [x8, #116]",
+        "fcvt d6, s6",
+        "fmul d6, d6, d2",
+        "fcvt s6, d6",
+        "str s6, [x8, #140]",
+        "ldr s6, [x8, #140]",
+        "str s6, [x8, #44]",
+        "ldr s6, [x8, #120]",
+        "fcvt d6, s6",
+        "fmul d2, d2, d6",
         "mov w20, #0x0",
         "strb wzr, [x28, #1049]",
         "fcvt s2, d2",
@@ -5486,16 +5469,16 @@
         "str s2, [x8, #16]",
         "ldr s2, [x8, #148]",
         "fcvt d2, s2",
-        "ldr s3, [x8, #20]",
-        "fcvt d3, s3",
-        "fadd d3, d3, d2",
-        "fcvt s3, d3",
-        "str s3, [x8, #20]",
-        "ldr s3, [x8, #152]",
-        "fcvt d3, s3",
+        "ldr s6, [x8, #20]",
+        "fcvt d6, s6",
+        "fadd d6, d6, d2",
+        "fcvt s6, d6",
+        "str s6, [x8, #20]",
+        "ldr s6, [x8, #152]",
+        "fcvt d6, s6",
         "ldr s7, [x8, #44]",
         "fcvt d7, s7",
-        "fadd d7, d7, d3",
+        "fadd d7, d7, d6",
         "fcvt s7, d7",
         "str s7, [x8, #44]",
         "ldr s7, [x8, #156]",
@@ -5560,14 +5543,11 @@
         "ldr w5, [x8, #120]",
         "strb wzr, [x28, #1049]",
         "str w5, [x8, #192]",
-        "fcvt s8, d4",
-        "str s8, [x8, #92]",
+        "str s3, [x8, #92]",
         "strb wzr, [x28, #1049]",
-        "fcvt s8, d5",
-        "str s8, [x8, #132]",
+        "str s4, [x8, #132]",
         "strb wzr, [x28, #1049]",
-        "fcvt s8, d6",
-        "str s8, [x8, #124]",
+        "str s5, [x8, #124]",
         "ldr s8, [x8, #40]",
         "str s8, [x8, #64]",
         "ldr s8, [x8, #84]",
@@ -5587,7 +5567,7 @@
         "str s8, [x8, #20]",
         "ldr s8, [x8, #44]",
         "fcvt d8, s8",
-        "fadd d8, d8, d3",
+        "fadd d8, d8, d6",
         "fcvt s8, d8",
         "str s8, [x8, #44]",
         "ldr s8, [x8, #16]",
@@ -5650,14 +5630,11 @@
         "ldr w5, [x8, #120]",
         "strb wzr, [x28, #1049]",
         "str w5, [x8, #204]",
-        "fcvt s8, d4",
-        "str s8, [x8, #92]",
+        "str s3, [x8, #92]",
         "strb wzr, [x28, #1049]",
-        "fcvt s8, d5",
-        "str s8, [x8, #132]",
+        "str s4, [x8, #132]",
         "strb wzr, [x28, #1049]",
-        "fcvt s8, d6",
-        "str s8, [x8, #124]",
+        "str s5, [x8, #124]",
         "ldr s8, [x8, #40]",
         "str s8, [x8, #64]",
         "ldr s8, [x8, #84]",
@@ -5677,7 +5654,7 @@
         "str s8, [x8, #20]",
         "ldr s8, [x8, #44]",
         "fcvt d8, s8",
-        "fadd d8, d8, d3",
+        "fadd d8, d8, d6",
         "fcvt s8, d8",
         "str s8, [x8, #44]",
         "ldr s8, [x8, #16]",
@@ -5741,35 +5718,32 @@
         "str w5, [x8, #216]",
         "strb wzr, [x28, #1049]",
         "str w20, [x8, #-4]!",
-        "fcvt s4, d4",
-        "str s4, [x8, #96]",
+        "str s3, [x8, #96]",
         "strb wzr, [x28, #1049]",
-        "fcvt s4, d5",
         "str s4, [x8, #136]",
         "strb wzr, [x28, #1049]",
-        "fcvt s4, d6",
-        "str s4, [x8, #128]",
-        "ldr s4, [x8, #44]",
-        "str s4, [x8, #68]",
-        "ldr s4, [x8, #88]",
-        "str s4, [x8, #76]",
-        "ldr s4, [x8, #84]",
-        "str s4, [x8, #72]",
-        "ldr s4, [x8, #92]",
-        "str s4, [x8, #24]",
-        "ldr s4, [x8, #144]",
-        "str s4, [x8, #48]",
-        "ldr s4, [x8, #80]",
-        "str s4, [x8, #20]",
-        "ldr s4, [x8, #24]",
-        "fcvt d4, s4",
-        "fadd d2, d2, d4",
+        "str s5, [x8, #128]",
+        "ldr s3, [x8, #44]",
+        "str s3, [x8, #68]",
+        "ldr s3, [x8, #88]",
+        "str s3, [x8, #76]",
+        "ldr s3, [x8, #84]",
+        "str s3, [x8, #72]",
+        "ldr s3, [x8, #92]",
+        "str s3, [x8, #24]",
+        "ldr s3, [x8, #144]",
+        "str s3, [x8, #48]",
+        "ldr s3, [x8, #80]",
+        "str s3, [x8, #20]",
+        "ldr s3, [x8, #24]",
+        "fcvt d3, s3",
+        "fadd d2, d2, d3",
         "strb wzr, [x28, #1049]",
         "fcvt s2, d2",
         "str s2, [x8, #24]",
         "ldr s2, [x8, #48]",
         "fcvt d2, s2",
-        "fadd d2, d3, d2",
+        "fadd d2, d6, d2",
         "strb wzr, [x28, #1049]",
         "fcvt s2, d2",
         "str s2, [x8, #48]",

unittests/InstructionCountCI/PrimaryGroup.json

@@ -2786,7 +2786,7 @@
         "mov x0, x5",
         "mov x1, x4",
         "mov x2, x6",
-        "ldr x3, [x28, #3584]",
+        "ldr x3, [x28, #3568]",
         "str x30, [sp, #-16]!",
         "blr x3",
         "ldr x30, [sp], #16",
@@ -2837,7 +2837,7 @@
         "mov x0, x5",
         "mov x1, x4",
         "mov x2, x6",
-        "ldr x3, [x28, #3592]",
+        "ldr x3, [x28, #3576]",
         "str x30, [sp, #-16]!",
         "blr x3",
         "ldr x30, [sp], #16",