JIT: Restore behaviour of emitting interrupt checks at every block entry

This is needed to handle suspend in infinite loops that occur as a
result of block-size constraints or indirect jumps. Fixes grow home.

CodeEmitter/CodeEmitter/ALUOps.inl

@@ -36,24 +36,33 @@ public:
     DataProcessing_PCRel_Imm(Op, rd, Imm);
   }
 
-  void adr(ARMEmitter::Register rd, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
     LOGMAN_THROW_A_FMT(IsADRRange(Imm), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0001'0000 << 24;
-    DataProcessing_PCRel_Imm(Op, rd, Imm);
+    if (IsADRRange(Imm)) [[likely]] {
+      constexpr uint32_t Op = 0b0001'0000 << 24;
+      DataProcessing_PCRel_Imm(Op, rd, Imm);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void adr(ARMEmitter::Register rd, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::ADR});
     constexpr uint32_t Op = 0b0001'0000 << 24;
     DataProcessing_PCRel_Imm(Op, rd, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void adr(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      adr(rd, &Label->Backward);
+      return adr(rd, &Label->Backward);
     } else {
-      adr(rd, &Label->Forward);
+      return adr(rd, &Label->Forward);
     }
   }
 
@@ -62,32 +71,42 @@ public:
     DataProcessing_PCRel_Imm(Op, rd, Imm);
   }
 
-  void adrp(ARMEmitter::Register rd, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, const BackwardLabel* Label) {
     int64_t Imm = reinterpret_cast<int64_t>(Label->Location) - (GetCursorAddress<int64_t>() & ~0xFFFLL);
     LOGMAN_THROW_A_FMT(IsADRPRange(Imm) && IsADRPAligned(Imm), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b1001'0000 << 24;
-    DataProcessing_PCRel_Imm(Op, rd, Imm);
+    if (IsADRPRange(Imm) && IsADRPAligned(Imm)) [[likely]] {
+      constexpr uint32_t Op = 0b1001'0000 << 24;
+      DataProcessing_PCRel_Imm(Op, rd, Imm);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void adrp(ARMEmitter::Register rd, ForwardLabel* Label) {
+
+  [[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::ADRP});
     constexpr uint32_t Op = 0b1001'0000 << 24;
     DataProcessing_PCRel_Imm(Op, rd, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void adrp(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      adrp(rd, &Label->Backward);
+      return adrp(rd, &Label->Backward);
     } else {
-      adrp(rd, &Label->Forward);
+      return adrp(rd, &Label->Forward);
     }
   }
 
-  void LongAddressGen(ARMEmitter::Register rd, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, const BackwardLabel* Label) {
     int64_t Imm = reinterpret_cast<int64_t>(Label->Location) - (GetCursorAddress<int64_t>());
     if (IsADRRange(Imm)) {
       // If the range is in ADR range then we can just use ADR.
-      adr(rd, Label);
+      return adr(rd, Label);
     } else if (IsADRPRange(Imm)) {
       int64_t ADRPImm = (reinterpret_cast<int64_t>(Label->Location) & ~0xFFFLL) - (GetCursorAddress<int64_t>() & ~0xFFFLL);
 
@@ -102,23 +121,28 @@ public:
         // Now even an add
         add(ARMEmitter::Size::i64Bit, rd, rd, AlignedOffset);
       }
-    } else {
-      LOGMAN_MSG_A_FMT("Unscaled offset too large");
-      FEX_UNREACHABLE;
+
+      return BranchEncodeSucceeded::Success;
     }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void LongAddressGen(ARMEmitter::Register rd, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::LONG_ADDRESS_GEN});
     // Emit a register index and a nop. These will be backpatched.
     dc32(rd.Idx());
     nop();
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void LongAddressGen(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      LongAddressGen(rd, &Label->Backward);
+      return LongAddressGen(rd, &Label->Backward);
     } else {
-      LongAddressGen(rd, &Label->Forward);
+      return LongAddressGen(rd, &Label->Forward);
     }
   }
 
@@ -862,12 +886,6 @@ public:
   }
 
 private:
-  static constexpr Condition InvertCondition(Condition cond) {
-    // These behave as always, so it makes no sense to allow inverting these.
-    LOGMAN_THROW_A_FMT(cond != Condition::CC_AL && cond != Condition::CC_NV, "Cannot invert CC_AL or CC_NV");
-    return static_cast<Condition>(FEXCore::ToUnderlying(cond) ^ 1);
-  }
-
   void and_(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t n, uint32_t immr, uint32_t imms) {
     constexpr uint32_t Op = 0b001'0010'00 << 22;
     DataProcessing_Logical_Imm(Op, s, rd, rn, n, immr, imms);

CodeEmitter/CodeEmitter/BranchOps.inl

@@ -20,23 +20,31 @@ public:
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 0, Cond, Imm);
   }
-  void b(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
-    constexpr uint32_t Op = 0b0101'010 << 25;
-    Branch_Conditional(Op, 0, 0, Cond, Imm >> 2);
+    if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0101'010 << 25;
+      Branch_Conditional(Op, 0, 0, Cond, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void b(ARMEmitter::Condition Cond, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 0, Cond, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void b(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      b(Cond, &Label->Backward);
+      return b(Cond, &Label->Backward);
     } else {
-      b(Cond, &Label->Forward);
+      return b(Cond, &Label->Forward);
     }
   }
 
@@ -45,24 +53,32 @@ public:
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 1, Cond, Imm);
   }
-  void bc(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
-    constexpr uint32_t Op = 0b0101'010 << 25;
-    Branch_Conditional(Op, 0, 1, Cond, Imm >> 2);
+    if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0101'010 << 25;
+      Branch_Conditional(Op, 0, 1, Cond, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void bc(ARMEmitter::Condition Cond, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 1, Cond, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void bc(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      bc(Cond, &Label->Backward);
+      return bc(Cond, &Label->Backward);
     } else {
-      bc(Cond, &Label->Forward);
+      return bc(Cond, &Label->Forward);
     }
   }
 
@@ -98,25 +114,32 @@ public:
 
     UnconditionalBranch(Op, Imm);
   }
-  void b(const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large");
-    constexpr uint32_t Op = 0b0001'01 << 26;
+    if (Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0001'01 << 26;
+      UnconditionalBranch(Op, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    UnconditionalBranch(Op, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void b(ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::B});
     constexpr uint32_t Op = 0b0001'01 << 26;
 
     UnconditionalBranch(Op, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void b(BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      b(&Label->Backward);
+      return b(&Label->Backward);
     } else {
-      b(&Label->Forward);
+      return b(&Label->Forward);
     }
   }
 
@@ -126,25 +149,33 @@ public:
     UnconditionalBranch(Op, Imm);
   }
 
-  void bl(const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bl(const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large");
-    constexpr uint32_t Op = 0b1001'01 << 26;
+    if (Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b1001'01 << 26;
+      UnconditionalBranch(Op, Imm >> 2);
 
-    UnconditionalBranch(Op, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void bl(ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bl(ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::B});
     constexpr uint32_t Op = 0b1001'01 << 26;
 
     UnconditionalBranch(Op, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void bl(BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bl(BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      bl(&Label->Backward);
+      return bl(&Label->Backward);
     } else {
-      bl(&Label->Forward);
+      return bl(&Label->Forward);
     }
   }
 
@@ -155,28 +186,35 @@ public:
     CompareAndBranch(Op, s, rt, Imm);
   }
 
-  void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0011'0100 << 24;
+    if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0011'0100 << 24;
+      CompareAndBranch(Op, s, rt, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    CompareAndBranch(Op, s, rt, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
 
     constexpr uint32_t Op = 0b0011'0100 << 24;
 
     CompareAndBranch(Op, s, rt, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      cbz(s, rt, &Label->Backward);
+      return cbz(s, rt, &Label->Backward);
     } else {
-      cbz(s, rt, &Label->Forward);
+      return cbz(s, rt, &Label->Forward);
     }
   }
 
@@ -186,28 +224,35 @@ public:
     CompareAndBranch(Op, s, rt, Imm);
   }
 
-  void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0011'0101 << 24;
+    if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0011'0101 << 24;
+      CompareAndBranch(Op, s, rt, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    CompareAndBranch(Op, s, rt, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
 
     constexpr uint32_t Op = 0b0011'0101 << 24;
 
     CompareAndBranch(Op, s, rt, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      cbnz(s, rt, &Label->Backward);
+      return cbnz(s, rt, &Label->Backward);
     } else {
-      cbnz(s, rt, &Label->Forward);
+      return cbnz(s, rt, &Label->Forward);
     }
   }
 
@@ -217,28 +262,35 @@ public:
 
     TestAndBranch(Op, rt, Bit, Imm);
   }
-  void tbz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0011'0110 << 24;
+    if (Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0011'0110 << 24;
+      TestAndBranch(Op, rt, Bit, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    TestAndBranch(Op, rt, Bit, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void tbz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::TEST_BRANCH});
 
     constexpr uint32_t Op = 0b0011'0110 << 24;
 
     TestAndBranch(Op, rt, Bit, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void tbz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      tbz(rt, Bit, &Label->Backward);
+      return tbz(rt, Bit, &Label->Backward);
     } else {
-      tbz(rt, Bit, &Label->Forward);
+      return tbz(rt, Bit, &Label->Forward);
     }
   }
 
@@ -247,27 +299,35 @@ public:
 
     TestAndBranch(Op, rt, Bit, Imm);
   }
-  void tbnz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
     LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0011'0111 << 24;
+    if (Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0011'0111 << 24;
+      TestAndBranch(Op, rt, Bit, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    TestAndBranch(Op, rt, Bit, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void tbnz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::TEST_BRANCH});
     constexpr uint32_t Op = 0b0011'0111 << 24;
 
     TestAndBranch(Op, rt, Bit, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void tbnz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      tbnz(rt, Bit, &Label->Backward);
+      return tbnz(rt, Bit, &Label->Backward);
     } else {
-      tbnz(rt, Bit, &Label->Forward);
+      return tbnz(rt, Bit, &Label->Forward);
     }
   }
 

CodeEmitter/CodeEmitter/Emitter.h

@@ -586,6 +586,15 @@ concept IsXOrWRegister = std::is_same_v<T, XRegister> || std::is_same_v<T, WRegi
 template<typename T>
 concept IsQOrDRegister = std::is_same_v<T, QRegister> || std::is_same_v<T, DRegister>;
 
+template<typename T>
+concept IsLabel = std::is_same_v<T, ARMEmitter::ForwardLabel> || std::is_same_v<T, ARMEmitter::BackwardLabel> ||
+                  std::is_same_v<T, ARMEmitter::BiDirectionalLabel> || std::is_same_v<T, ARMEmitter::ForwardLabel::Reference>;
+
+enum class BranchEncodeSucceeded {
+  Success,
+  Failure,
+};
+
 // Whether or not a given set of vector registers are sequential
 // in increasing order as far as the register file is concerned (modulo its size)
 //
@@ -638,19 +647,25 @@ public:
 
   // Bind a backward label to an address.
   // Address that is bound is the current emitter location.
-  void Bind(BackwardLabel* Label) {
+  [[nodiscard]] bool Bind(BackwardLabel* Label) {
     LOGMAN_THROW_A_FMT(Label->Location == nullptr, "Trying to bind a label twice");
     Label->Location = GetCursorAddress<uint8_t*>();
+
+    // Always binds because it is only storing a location.
+    return true;
   }
 
-  void Bind(const ForwardLabel::Reference* Label) {
+  [[nodiscard]] bool Bind(const ForwardLabel::Reference* Label) {
     uint8_t* CurrentAddress = GetCursorAddress<uint8_t*>();
     // Patch up the instructions
     switch (Label->Type) {
     case ForwardLabel::InstType::ADR: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(IsADRRange(Imm), "Unscaled offset too large");
+      if (!IsADRRange(Imm)) [[unlikely]] {
+        // Can't bind.
+        return false;
+      }
       uint32_t InstMask = 0b11 << 29 | 0b1111'1111'1111'1111'111 << 5;
       uint32_t Offset = static_cast<uint32_t>(Imm) & 0x3F'FFFF;
       uint32_t Inst = *Instruction & ~InstMask;
@@ -662,7 +677,12 @@ public:
     case ForwardLabel::InstType::ADRP: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(IsADRPRange(Imm) && IsADRPAligned(Imm), "Unscaled offset too large");
+
+      if (!(IsADRPRange(Imm) && IsADRPAligned(Imm))) [[unlikely]] {
+        // Can't bind.
+        return false;
+      }
+
       Imm >>= 12;
       uint32_t InstMask = 0b11 << 29 | 0b1111'1111'1111'1111'111 << 5;
       uint32_t Offset = static_cast<uint32_t>(Imm) & 0x3F'FFFF;
@@ -672,11 +692,13 @@ public:
       *Instruction = Inst;
       break;
     }
-
     case ForwardLabel::InstType::B: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large");
+      if (!(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0))) [[unlikely]] {
+        // Can't bind.
+        return false;
+      }
       Imm >>= 2;
       uint32_t InstMask = 0x3FF'FFFF;
       uint32_t Offset = static_cast<uint32_t>(Imm) & InstMask;
@@ -686,11 +708,13 @@ public:
 
       break;
     }
-
     case ForwardLabel::InstType::TEST_BRANCH: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large");
+      if (!(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0))) [[unlikely]] {
+        // Can't bind.
+        return false;
+      }
       Imm >>= 2;
       uint32_t InstMask = 0x3FFF;
       uint32_t Offset = static_cast<uint32_t>(Imm) & InstMask;
@@ -704,7 +728,10 @@ public:
     case ForwardLabel::InstType::RELATIVE_LOAD: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
+      if (!(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0))) [[unlikely]] {
+        // Can't bind.
+        return false;
+      }
       Imm >>= 2;
       uint32_t InstMask = 0x7'FFFF;
       uint32_t Offset = static_cast<uint32_t>(Imm) & InstMask;
@@ -753,27 +780,41 @@ public:
     }
     default: LOGMAN_MSG_A_FMT("Unexpected inst type in label fixup");
     }
+
+    return true;
   }
 
   // Bind a forward label to a location.
   // This walks all the instructions in the label's vector.
   // Then backpatching all instructions that have used the label.
-  void Bind(ForwardLabel* Label) {
+  [[nodiscard]] bool Bind(ForwardLabel* Label) {
+    bool Bound = true;
     if (Label->FirstInst.Location) {
-      Bind(&Label->FirstInst);
+      Bound &= Bind(&Label->FirstInst);
     }
     for (auto& Inst : Label->Insts) {
-      Bind(&Inst);
+      Bound &= Bind(&Inst);
     }
+
+    return Bound;
   }
 
   // Bind a bidirectional location to a location.
   // Binds both forwards and backwards depending on how the label was used.
-  void Bind(BiDirectionalLabel* Label) {
+  [[nodiscard]] bool Bind(BiDirectionalLabel* Label) {
+    bool Bound = true;
     if (!Label->Backward.Location) {
-      Bind(&Label->Backward);
+      Bound &= Bind(&Label->Backward);
     }
-    Bind(&Label->Forward);
+    Bound &= Bind(&Label->Forward);
+
+    return Bound;
+  }
+
+  static constexpr Condition InvertCondition(Condition cond) {
+    // These behave as always, so it makes no sense to allow inverting these.
+    LOGMAN_THROW_A_FMT(cond != Condition::CC_AL && cond != Condition::CC_NV, "Cannot invert CC_AL or CC_NV");
+    return static_cast<Condition>(FEXCore::ToUnderlying(cond) ^ 1);
   }
 
 #include <CodeEmitter/VixlUtils.inl>

FEXCore/Source/CMakeLists.txt

@@ -66,6 +66,7 @@ set (SRCS
   Interface/IR/Passes/RedundantFlagCalculationElimination.cpp
   Interface/IR/Passes/RegisterAllocationPass.cpp
   Interface/IR/Passes/x87StackOptimizationPass.cpp
+  Utils/LongJump.cpp
   Utils/Telemetry.cpp
   Utils/Threads.cpp
   Utils/Profiler.cpp

FEXCore/Source/Common/VectorRegType.h

@@ -4,6 +4,8 @@
 #ifdef _M_X86_64
 #include <xmmintrin.h>
 #include <immintrin.h>
+#else
+#include <cstdint>
 #endif
 
 namespace FEXCore {

FEXCore/Source/Interface/Context/Context.h

@@ -62,7 +62,7 @@ struct CustomIRResult {
     , Data(Data) {}
 };
 
-using BlockDelinkerFunc = void (*)(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record);
+using BlockDelinkerFunc = void (*)(FEXCore::Context::ExitFunctionLinkData* Record);
 constexpr uint32_t TSC_SCALE_MAXIMUM = 1'000'000'000; ///< 1Ghz
 
 class CodeCache : public AbstractCodeCache {
@@ -155,10 +155,10 @@ public:
     return CodeCache;
   }
 
-  void OnCodeBufferAllocated(CPU::CodeBuffer&) override;
+  void OnCodeBufferAllocated(const std::shared_ptr<CPU::CodeBuffer> &) override;
   void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) override;
-  void InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, uint64_t Start,
-                                uint64_t Length) override;
+  void InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) override;
+  void InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) override;
   FEXCore::ForkableSharedMutex& GetCodeInvalidationMutex() override {
     return CodeInvalidationMutex;
   }
@@ -231,8 +231,6 @@ public:
 
   ContextImpl(const FEXCore::HostFeatures& Features);
 
-  static bool ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, const FEXCore::LookupCacheWriteLockToken& lk);
-
   static void ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP);
 
   // This is used as a replacement for the SMC writes in the mono callsite backpatcher that avoids atomic operations
@@ -348,5 +346,8 @@ private:
 
   bool MonoDetected = false;
   std::atomic<uint64_t> MonoBackpatcherBlock;
+
+  std::mutex CodeBufferListLock;
+  fextl::vector<std::weak_ptr<CPU::CodeBuffer>> CodeBufferList;
 };
 } // namespace FEXCore::Context

FEXCore/Source/Interface/Core/CPUBackend.cpp

@@ -400,7 +400,7 @@ namespace CPU {
     Latest = Buffer;
     LatestOffset = 0;
 
-    OnCodeBufferAllocated(*Buffer);
+    OnCodeBufferAllocated(Buffer);
 
     return Buffer;
   }

FEXCore/Source/Interface/Core/CPUBackend.h

@@ -81,7 +81,7 @@ namespace CPU {
     // Protects writes to the latest CodeBuffer and changes to LatestOffset
     FEXCore::ForkableUniqueMutex CodeBufferWriteMutex;
 
-    virtual void OnCodeBufferAllocated(CodeBuffer&) {};
+    virtual void OnCodeBufferAllocated(const std::shared_ptr<CodeBuffer>&) {};
 
   private:
     fextl::shared_ptr<CodeBuffer> Latest;

FEXCore/Source/Interface/Core/Core.cpp

@@ -459,9 +459,14 @@ void ContextImpl::LockBeforeFork(FEXCore::Core::InternalThreadState* Thread) {
 }
 #endif
 
-void ContextImpl::OnCodeBufferAllocated(CPU::CodeBuffer& Buffer) {
+void ContextImpl::OnCodeBufferAllocated(const fextl::shared_ptr<CPU::CodeBuffer>& Buffer) {
   if (Config.GlobalJITNaming()) {
-    Symbols.RegisterJITSpace(Buffer.Ptr, Buffer.Size);
+    Symbols.RegisterJITSpace(Buffer->Ptr, Buffer->Size);
+  }
+
+  {
+    std::scoped_lock lk{CodeBufferListLock};
+    CodeBufferList.emplace_back(Buffer);
   }
 }
 
@@ -833,10 +838,14 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
     Thread->CPUBackend->ClearRelocations();
   }
 
+  fextl::vector<uint64_t> CodePages;
+
   if (NeedsAddGuestCodeRanges) {
     // Track in the guest to host map all entrypoints for all pages the compiled block touches, if any page didn't previously
     // contain code, inform the frontend so it can setup SMC detection.
     auto BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo();
+    CodePages.reserve(BlockInfo->CodePages.size());
+    CodePages.insert(CodePages.end(), BlockInfo->CodePages.begin(), BlockInfo->CodePages.end());
     for (auto CodePage : BlockInfo->CodePages) {
       if (Thread->LookupCache->AddBlockExecutableRange(Thread, BlockInfo->EntryPoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE)) {
         SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE);
@@ -845,8 +854,9 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
   }
 
   // Insert to lookup cache
+
   for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) {
-    Thread->LookupCache->AddBlockMapping(Thread, GuestAddr, HostAddr);
+    Thread->LookupCache->AddBlockMapping(Thread, GuestAddr, CodePages, HostAddr);
   }
 
   return (uintptr_t)CodePtr;
@@ -873,50 +883,37 @@ uintptr_t ContextImpl::CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, ui
   return (uintptr_t)CodePtr;
 }
 
-static void InvalidateGuestThreadCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator,
-                                           uint64_t Start, uint64_t Length) {
-  // Ensures now-modified mappings aren't cached as being in their previous non-executable state.
+void ContextImpl::InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) {
+  FEXCORE_PROFILE_SCOPED("InvalidateCodeBuffersCodeRange");
+
+  LogMan::Throw::AFmt(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here");
+  std::scoped_lock lk {CodeBufferListLock};
+  auto it = CodeBufferList.begin();
+  while (it != CodeBufferList.end()) {
+    if (auto Strong = it->lock(); Strong) {
+      Strong->LookupCache->InvalidateRange(Start, Length);
+      it++;
+    } else {
+      it = CodeBufferList.erase(it);
+    }
+  }
+}
+
+void ContextImpl::InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) {
+  LogMan::Throw::AFmt(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here");
+
+ // Ensures now-modified mappings aren't cached as being in their previous non-executable state.
   // Accessing FrontendDecoder is safe as the thread's code invalidation mutex must be locked here.
   Thread->FrontendDecoder->ResetExecutableRangeCache();
 
-  auto lk = Thread->LookupCache->AcquireWriteLock();
-  auto& CodePages = Thread->LookupCache->Shared->CodePages;
+  if (Thread->LookupCache->InvalidateCacheRange(Start, Length)) {
+    FEXCORE_PROFILE_SCOPED("InvalidateCallRet");
 
-  auto lower = CodePages.lower_bound(Start >> 12);
-  auto upper = CodePages.upper_bound((Start + Length - 1) >> 12);
-
-  for (auto it = lower; it != upper; it++) {
-    Accumulator.emplace_back(std::move(it->second));
-  }
-
-  bool InvalidatedAnyEntries = false;
-  for (const auto& PageEntries : Accumulator) {
-    for (const auto& Entry : PageEntries) {
-      if (ContextImpl::ThreadRemoveCodeEntry(Thread, Entry, lk)) {
-        InvalidatedAnyEntries = true;
-      }
-    }
-  }
-
-  if (InvalidatedAnyEntries) {
     // This may cause access violations in the thread on Windows as zeroing is not atomic, this is handled by the frontend
     Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);
   }
 }
 
-void ContextImpl::InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator,
-                                           uint64_t Start, uint64_t Length) {
-  InvalidateGuestThreadCodeRange(Thread, Accumulator, Start, Length);
-}
-
-bool ContextImpl::ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP,
-                                        const FEXCore::LookupCacheWriteLockToken& lk) {
-  LogMan::Throw::AFmt(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to "
-                                                                                                         "be unique_locked here");
-
-  return Thread->LookupCache->Erase(Thread->CurrentFrame, GuestRIP, lk);
-}
-
 void ContextImpl::ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP) {
   static_cast<ContextImpl*>(Frame->Thread->CTX)->SyscallHandler->InvalidateGuestCodeRange(Frame->Thread, GuestRIP, 1);
 }

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: MIT
 
-#include "Common/SoftFloat.h"
+#include "Common/VectorRegType.h"
 #include "Interface/Context/Context.h"
 #include "Interface/Core/CPUBackend.h"
 #include "Interface/Core/Dispatcher/Dispatcher.h"
@@ -26,9 +26,7 @@
 #endif
 
 #include <array>
-#include <atomic>
 #include <bit>
-#include <condition_variable>
 #include <csignal>
 #include <cstring>
 
@@ -95,7 +93,7 @@ void Dispatcher::EmitDispatcher() {
 
   FillStaticRegs();
   ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));
-  cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);
+  (void)cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);
 
   ARMEmitter::BiDirectionalLabel LoopTop {};
 
@@ -144,7 +142,7 @@ void Dispatcher::EmitDispatcher() {
   // We want to ensure that we are 16 byte aligned at the top of this loop
   Align16B();
 
-  Bind(&LoopTop);
+  (void)Bind(&LoopTop);
   AbsoluteLoopTopAddress = GetCursorAddress<uint64_t>();
 
   // Load in our RIP
@@ -171,16 +169,16 @@ void Dispatcher::EmitDispatcher() {
   ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionEC));
   br(TMP2);
 
-  Bind(&l_NotECCode);
+  (void)Bind(&l_NotECCode);
 #endif
 
   ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
-  cbnz(ARMEmitter::Size::i32Bit, TMP1, &CompileSingleStep);
+  (void)cbnz(ARMEmitter::Size::i32Bit, TMP1, &CompileSingleStep);
 
   ARMEmitter::ForwardLabel NoBlock;
 
   if (DisableL2Cache()) {
-    b(&NoBlock);
+    (void)b(&NoBlock);
   } else {
     // This is the block cache lookup routine
     // It matches what is going on it LookupCache.h::FindBlock
@@ -203,7 +201,7 @@ void Dispatcher::EmitDispatcher() {
       ldr(TMP1, TMP1, TMP2, ARMEmitter::ExtendedType::LSL_64, 3);
 
       // If page pointer is zero then we have no block
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &NoBlock);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &NoBlock);
 
       // Steal the page offset
       and_(ARMEmitter::Size::i64Bit, TMP2, TMP4, 0x0FFF);
@@ -218,10 +216,10 @@ void Dispatcher::EmitDispatcher() {
 
       // If the guest address doesn't match, Compile the block.
       sub(TMP2, TMP2, RipReg);
-      cbnz(ARMEmitter::Size::i64Bit, TMP2, &NoBlock);
+      (void)cbnz(ARMEmitter::Size::i64Bit, TMP2, &NoBlock);
 
       // Check the host address to see if it matches, else compile the block.
-      cbz(ARMEmitter::Size::i64Bit, TMP4, &NoBlock);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP4, &NoBlock);
 
       // If we've made it here then we have a real compiled block
       {
@@ -313,7 +311,7 @@ void Dispatcher::EmitDispatcher() {
 
   // Need to create the block
   {
-    Bind(&NoBlock);
+    (void)Bind(&NoBlock);
 
     EmitSignalGuardedRegion([&]() {
       SpillStaticRegs(TMP1);
@@ -347,7 +345,7 @@ void Dispatcher::EmitDispatcher() {
   }
 
   {
-    Bind(&CompileSingleStep);
+    (void)Bind(&CompileSingleStep);
 
     EmitSignalGuardedRegion([&]() {
       SpillStaticRegs(TMP1);
@@ -509,7 +507,7 @@ void Dispatcher::EmitDispatcher() {
     stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
 
     // Now go back to the regular dispatcher loop
-    b(&LoopTop);
+    (void)b(&LoopTop);
   }
 
   auto EmitLongALUOpHandler = [&](auto R, auto Offset) {
@@ -578,14 +576,15 @@ void Dispatcher::EmitDispatcher() {
     }
   }
 
-  Bind(&l_CTX);
+  (void)Bind(&l_CTX);
   dc64(reinterpret_cast<uintptr_t>(CTX));
-  Bind(&l_Sleep);
+  (void)Bind(&l_Sleep);
   dc64(reinterpret_cast<uint64_t>(SleepThread));
-  Bind(&l_CompileBlock);
+  (void)Bind(&l_CompileBlock);
   FEXCore::Utils::MemberFunctionToPointerCast PMFCompileBlock(&FEXCore::Context::ContextImpl::CompileBlock);
   dc64(PMFCompileBlock.GetConvertedPointer());
-  Bind(&l_CompileSingleStep);
+  (void)Bind(&l_CompileSingleStep);
+
   FEXCore::Utils::MemberFunctionToPointerCast PMFCompileSingleStep(&FEXCore::Context::ContextImpl::CompileSingleStep);
   dc64(PMFCompileSingleStep.GetConvertedPointer());
 

FEXCore/Source/Interface/Core/JIT/ALUOps.cpp

@@ -588,7 +588,7 @@ DEF_OP(ShiftFlags) {
   and_(ARMEmitter::Size::i32Bit, TMP1, Src2, OpSize == IR::OpSize::i64Bit ? 0x3f : 0x1f);
 
   ARMEmitter::ForwardLabel Done;
-  cbz(EmitSize, TMP1, &Done);
+  (void)cbz(EmitSize, TMP1, &Done);
   {
     // PF/SF/ZF/OF
     if (OpSize >= IR::OpSize::i32Bit) {
@@ -652,7 +652,7 @@ DEF_OP(ShiftFlags) {
       msr(ARMEmitter::SystemRegister::NZCV, TMP2);
     }
   }
-  Bind(&Done);
+  (void)Bind(&Done);
 
   // TODO: Make RA less dumb so this can't happen (e.g. with late-kill).
   if (PFOutput != PFTemp) {
@@ -669,7 +669,7 @@ DEF_OP(RotateFlags) {
 
   // If shift=0, flags are unaffected. Wrap the whole implementation in a cbz.
   ARMEmitter::ForwardLabel Done;
-  cbz(EmitSize, Shift, &Done);
+  (void)cbz(EmitSize, Shift, &Done);
   {
     // Extract the last bit shifted in to CF
     const auto BitSize = IR::OpSizeToSize(Op->Size) * 8;
@@ -701,7 +701,7 @@ DEF_OP(RotateFlags) {
       msr(ARMEmitter::SystemRegister::NZCV, TMP3);
     }
   }
-  Bind(&Done);
+  (void)Bind(&Done);
 }
 
 DEF_OP(Extr) {
@@ -767,14 +767,14 @@ DEF_OP(PDep) {
     // Now, they're copied, so we can start setting Dest (even if it overlaps with
     // one of them).  Handle early exit case
     mov(EmitSize, Dest, 0);
-    cbz(EmitSize, OrigMask, &Done);
+    (void)cbz(EmitSize, OrigMask, &Done);
 
     // Setup for first iteration
     neg(EmitSize, T0, Mask);
     and_(EmitSize, T0, T0, Mask);
 
     // Main loop
-    Bind(&NextBit);
+    (void)Bind(&NextBit);
     sbfx(EmitSize, T1, Input, 0, 1);
     eor(EmitSize, Mask, Mask, T0);
     and_(EmitSize, T0, T1, T0);
@@ -782,10 +782,10 @@ DEF_OP(PDep) {
     orr(EmitSize, Dest, Dest, T0);
     lsr(EmitSize, Input, Input, 1);
     and_(EmitSize, T0, Mask, T1);
-    cbnz(EmitSize, T0, &NextBit);
+    (void)cbnz(EmitSize, T0, &NextBit);
 
     // All done with nothing to do.
-    Bind(&Done);
+    (void)Bind(&Done);
   }
 }
 
@@ -821,27 +821,27 @@ DEF_OP(PExt) {
     ARMEmitter::BackwardLabel NextBit;
     ARMEmitter::ForwardLabel Done;
 
-    cbz(EmitSize, Mask, &EarlyExit);
+    (void)cbz(EmitSize, Mask, &EarlyExit);
     mov(EmitSize, MaskReg, Mask);
     mov(EmitSize, ValueReg, Input);
     mov(EmitSize, Dest, ARMEmitter::Reg::zr);
 
     // Main loop
-    Bind(&NextBit);
-    cbz(EmitSize, MaskReg, &Done);
+    (void)Bind(&NextBit);
+    (void)cbz(EmitSize, MaskReg, &Done);
     clz(EmitSize, BitReg, MaskReg);
     lslv(EmitSize, ValueReg, ValueReg, BitReg);
     lslv(EmitSize, MaskReg, MaskReg, BitReg);
     extr(EmitSize, Dest, Dest, ValueReg, OpSizeBitsM1);
     bfc(EmitSize, MaskReg, OpSizeBitsM1, 1);
-    b(&NextBit);
+    (void)b(&NextBit);
 
     // Early exit
-    Bind(&EarlyExit);
+    (void)Bind(&EarlyExit);
     mov(EmitSize, Dest, ARMEmitter::Reg::zr);
 
     // All done with nothing to do.
-    Bind(&Done);
+    (void)Bind(&Done);
   }
 }
 
@@ -909,7 +909,7 @@ DEF_OP(Div) {
     eor(EmitSize, TMP1, TMP1, Upper);
 
     // If the sign bit matches then the result is zero
-    cbz(EmitSize, TMP1, &Only64Bit);
+    (void)cbz(EmitSize, TMP1, &Only64Bit);
 
     // Long divide
     {
@@ -928,17 +928,17 @@ DEF_OP(Div) {
       mov(EmitSize, Remainder, TMP2);
 
       // Skip 64-bit path
-      b(&LongDIVRet);
+      (void)b(&LongDIVRet);
     }
 
-    Bind(&Only64Bit);
+    (void)Bind(&Only64Bit);
     // 64-Bit only
     {
       sdiv(EmitSize, Quotient, Lower, Divisor);
       msub(EmitSize, Remainder, Quotient, Divisor, Lower);
     }
 
-    Bind(&LongDIVRet);
+    (void)Bind(&LongDIVRet);
     break;
   }
   default: LOGMAN_MSG_A_FMT("Unknown DIV Size: {}", OpSize); break;
@@ -992,7 +992,7 @@ DEF_OP(UDiv) {
 
     // Check the upper bits for zero
     // If the upper bits are zero then we can do a 64-bit divide
-    cbz(EmitSize, Upper, &Only64Bit);
+    (void)cbz(EmitSize, Upper, &Only64Bit);
 
     // Long divide
     {
@@ -1011,17 +1011,17 @@ DEF_OP(UDiv) {
       mov(EmitSize, Remainder, TMP2);
 
       // Skip 64-bit path
-      b(&LongDIVRet);
+      (void)b(&LongDIVRet);
     }
 
-    Bind(&Only64Bit);
+    (void)Bind(&Only64Bit);
     // 64-Bit only
     {
       udiv(EmitSize, Quotient, Lower, Divisor);
       msub(EmitSize, Remainder, Quotient, Divisor, Lower);
     }
 
-    Bind(&LongDIVRet);
+    (void)Bind(&LongDIVRet);
     break;
   }
   default: LOGMAN_MSG_A_FMT("Unknown LUDIV Size: {}", OpSize); break;

FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp

@@ -63,7 +63,7 @@ void Arm64JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair& Lit) {
   auto CurrentCursor = GetCursorAddress<uint8_t*>();
   Lit.MoveABI.NamedSymbolLiteral.Offset = CurrentCursor - CodeData.BlockBegin;
 
-  Bind(&Lit.Loc);
+  BindOrRestart(&Lit.Loc);
   dc64(Lit.Lit);
   Relocations.emplace_back(Lit.MoveABI);
 }

FEXCore/Source/Interface/Core/JIT/AtomicOps.cpp

@@ -62,27 +62,27 @@ DEF_OP(CASPair) {
     ARMEmitter::BackwardLabel LoopTop;
     ARMEmitter::ForwardLabel LoopNotExpected;
     ARMEmitter::ForwardLabel LoopExpected;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
 
     // This instruction sequence must be synced with HandleCASPAL_Armv8.
     ldaxp(EmitSize, TMP2, TMP3, MemSrc);
     cmp(EmitSize, TMP2, Expected0);
     ccmp(EmitSize, TMP3, Expected1, ARMEmitter::StatusFlags::None, ARMEmitter::Condition::CC_EQ);
-    b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
+    (void)b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
     stlxp(EmitSize, TMP2, Desired0, Desired1, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
+    (void)cbnz(EmitSize, TMP2, &LoopTop);
     mov(EmitSize, Dst0, Expected0);
     mov(EmitSize, Dst1, Expected1);
 
-    b(&LoopExpected);
+    (void)b(&LoopExpected);
 
-    Bind(&LoopNotExpected);
+    (void)Bind(&LoopNotExpected);
     mov(EmitSize, Dst0, TMP2.R());
     mov(EmitSize, Dst1, TMP3.R());
     // exclusive monitor needs to be cleared here
     // Might have hit the case where ldaxr was hit but stlxr wasn't
     clrex();
-    Bind(&LoopExpected);
+    (void)Bind(&LoopExpected);
 
     // Restore
     msr(ARMEmitter::SystemRegister::NZCV, TMP1);
@@ -114,7 +114,7 @@ DEF_OP(CAS) {
     ARMEmitter::BackwardLabel LoopTop;
     ARMEmitter::ForwardLabel LoopNotExpected;
     ARMEmitter::ForwardLabel LoopExpected;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     if (IROp->Size == IR::OpSize::i8Bit) {
       cmp(EmitSize, TMP2, Expected, ARMEmitter::ExtendedType::UXTB, 0);
@@ -123,18 +123,18 @@ DEF_OP(CAS) {
     } else {
       cmp(EmitSize, TMP2, Expected);
     }
-    b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
+    (void)b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
     stlxr(SubEmitSize, TMP3, Desired, MemSrc);
-    cbnz(EmitSize, TMP3, &LoopTop);
+    (void)cbnz(EmitSize, TMP3, &LoopTop);
     mov(EmitSize, Dst, Expected);
-    b(&LoopExpected);
+    (void)b(&LoopExpected);
 
-    Bind(&LoopNotExpected);
+    (void)Bind(&LoopNotExpected);
     mov(EmitSize, Dst, TMP2.R());
     // exclusive monitor needs to be cleared here
     // Might have hit the case where ldaxr was hit but stlxr wasn't
     clrex();
-    Bind(&LoopExpected);
+    (void)Bind(&LoopExpected);
   }
 }
 
@@ -150,11 +150,11 @@ DEF_OP(AtomicXor) {
     steorl(SubEmitSize, Src, MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     eor(EmitSize, TMP2, TMP2, Src);
     stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
+    (void)cbnz(EmitSize, TMP2, &LoopTop);
   }
 }
 
@@ -179,10 +179,10 @@ DEF_OP(AtomicSwap) {
     ldswpal(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     stlxr(SubEmitSize, TMP4, Src, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     ubfm(EmitSize, GetReg(Node), TMP2, 0, IR::OpSizeAsBits(OpSize) - 1);
   }
 }
@@ -199,11 +199,11 @@ DEF_OP(AtomicFetchAdd) {
     ldaddal(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     add(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -221,11 +221,11 @@ DEF_OP(AtomicFetchSub) {
     ldaddal(SubEmitSize, TMP2, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     sub(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -243,11 +243,11 @@ DEF_OP(AtomicFetchAnd) {
     ldclral(SubEmitSize, TMP2, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     and_(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -264,11 +264,11 @@ DEF_OP(AtomicFetchCLR) {
     ldclral(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     bic(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -285,11 +285,11 @@ DEF_OP(AtomicFetchOr) {
     ldsetal(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     orr(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -306,11 +306,11 @@ DEF_OP(AtomicFetchXor) {
     ldeoral(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     eor(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -326,20 +326,20 @@ DEF_OP(AtomicFetchNeg) {
     // Use a CAS loop to avoid needing to emulate unaligned LLSC atomics
     ldr(SubEmitSize, TMP2, MemSrc);
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     mov(EmitSize, TMP4, TMP2);
     neg(EmitSize, TMP3, TMP2);
     casal(SubEmitSize, TMP2, TMP3, MemSrc);
     sub(EmitSize, TMP3, TMP2, TMP4);
-    cbnz(EmitSize, TMP3, &LoopTop);
+    (void)cbnz(EmitSize, TMP3, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     neg(EmitSize, TMP3, TMP2);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -359,11 +359,11 @@ DEF_OP(TelemetrySetValue) {
     stsetl(ARMEmitter::SubRegSize::i64Bit, TMP1, TMP2);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(ARMEmitter::SubRegSize::i64Bit, TMP3, TMP2);
     orr(ARMEmitter::Size::i32Bit, TMP3, TMP3, Src);
     stlxr(ARMEmitter::SubRegSize::i64Bit, TMP3, TMP3, TMP2);
-    cbnz(ARMEmitter::Size::i32Bit, TMP3, &LoopTop);
+    (void)cbnz(ARMEmitter::Size::i32Bit, TMP3, &LoopTop);
   }
 #endif
 }

FEXCore/Source/Interface/Core/JIT/BranchOps.cpp

@@ -141,7 +141,7 @@ DEF_OP(ExitFunction) {
         if (!Op->CallReturnBlock.IsInvalid()) {
           auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();
           PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;
-          adr(TMP1, &l_CallReturn);
+          (void)adr(TMP1, &l_CallReturn);
           stp<ARMEmitter::IndexType::PRE>(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);
         } else {
           stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
@@ -149,16 +149,16 @@ DEF_OP(ExitFunction) {
       } else if (Op->Hint == IR::BranchHint::CheckTF) {
         ARMEmitter::ForwardLabel TFUnset;
         ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
-        cbz(ARMEmitter::Size::i32Bit, TMP1, &TFUnset);
+        (void)cbz(ARMEmitter::Size::i32Bit, TMP1, &TFUnset);
         LoadConstant(ARMEmitter::Size::i64Bit, TMP1, NewRIP);
         str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
         ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop));
         blr(TMP2);
-        Bind(&TFUnset);
+        (void)Bind(&TFUnset);
       }
 
       EmitLinkedBranch(NewRIP, Op->Hint == IR::BranchHint::Call);
-      Bind(&l_CallReturn);
+      (void)Bind(&l_CallReturn);
 #ifdef _M_ARM_64EC
     }
 #endif
@@ -170,7 +170,7 @@ DEF_OP(ExitFunction) {
       // First try to pop from the call-ret stack, otherwise follow the normal path (but ending in a ret)
       ldp<ARMEmitter::IndexType::POST>(TMP1, TMP2, REG_CALLRET_SP, 0x10);
       sub(TMP1, TMP1, RipReg.X());
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
     }
 
     // L1 Cache
@@ -185,23 +185,23 @@ DEF_OP(ExitFunction) {
 
     // Note: sub+cbnz used over cmp+br to preserve flags.
     sub(TMP1, TMP1, RipReg.X());
-    cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
+    (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
     ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop));
     str(RipReg.X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
 
-    Bind(&SkipFullLookup);
+    (void)Bind(&SkipFullLookup);
     if (Op->Hint == IR::BranchHint::Call) {
       ARMEmitter::ForwardLabel l_CallReturn;
       if (!Op->CallReturnBlock.IsInvalid()) {
         auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();
         PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;
-        adr(TMP1, &l_CallReturn);
+        (void)adr(TMP1, &l_CallReturn);
         stp<ARMEmitter::IndexType::PRE>(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);
       } else {
         stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
       }
       blr(TMP2);
-      Bind(&l_CallReturn);
+      (void)Bind(&l_CallReturn);
     } else if (Op->Hint == IR::BranchHint::Return) {
       ret(TMP2);
     } else {
@@ -222,7 +222,7 @@ DEF_OP(CondJump) {
   auto TrueTargetLabel = JumpTarget(Op->TrueBlock);
 
   if (Op->FromNZCV) {
-    b(MapCC(Op->Cond), TrueTargetLabel);
+    b_OrRestart(MapCC(Op->Cond), TrueTargetLabel);
   } else {
     uint64_t Const;
     const bool isConst = IsInlineConstant(Op->Cmp2, &Const);
@@ -235,16 +235,16 @@ DEF_OP(CondJump) {
 
     if (Op->Cond == IR::CondClass::EQ) {
       LOGMAN_THROW_A_FMT(Const == 0, "CondJump: Expected 0 source");
-      cbz(Size, Reg, TrueTargetLabel);
+      cbz_OrRestart(Size, Reg, TrueTargetLabel);
     } else if (Op->Cond == IR::CondClass::NEQ) {
       LOGMAN_THROW_A_FMT(Const == 0, "CondJump: Expected 0 source");
-      cbnz(Size, Reg, TrueTargetLabel);
+      cbnz_OrRestart(Size, Reg, TrueTargetLabel);
     } else if (Op->Cond == IR::CondClass::TSTZ) {
       LOGMAN_THROW_A_FMT(Const < 64, "CondJump: Expected valid bit source");
-      tbz(Reg, Const, TrueTargetLabel);
+      tbz_OrRestart(Reg, Const, TrueTargetLabel);
     } else if (Op->Cond == IR::CondClass::TSTNZ) {
       LOGMAN_THROW_A_FMT(Const < 64, "CondJump: Expected valid bit source");
-      tbnz(Reg, Const, TrueTargetLabel);
+      tbnz_OrRestart(Reg, Const, TrueTargetLabel);
     } else {
       LOGMAN_THROW_A_FMT(false, "CondJump expected simple condition");
     }
@@ -355,7 +355,7 @@ DEF_OP(ValidateCode) {
     while (len >= Size) {
       LoadData();
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP2);
-      cbnz(ARMEmitter::Size::i64Bit, TMP1, &Fail);
+      cbnz_OrRestart(ARMEmitter::Size::i64Bit, TMP1, &Fail);
       len -= Size;
       Offset += Size;
     }
@@ -383,10 +383,10 @@ DEF_OP(ValidateCode) {
 
   ARMEmitter::ForwardLabel End;
   LoadConstant(ARMEmitter::Size::i32Bit, Dst, 0);
-  b(&End);
-  Bind(&Fail);
+  b_OrRestart(&End);
+  BindOrRestart(&Fail);
   LoadConstant(ARMEmitter::Size::i32Bit, Dst, 1);
-  Bind(&End);
+  BindOrRestart(&End);
 }
 
 DEF_OP(ThreadRemoveCodeEntry) {

FEXCore/Source/Interface/Core/JIT/JIT.cpp

@@ -11,8 +11,6 @@ desc: Main glue logic of the arm64 splatter backend
 $end_info$
 */
 
-#include "Common/SoftFloat.h"
-
 #include "Interface/Context/Context.h"
 #include "Interface/Core/LookupCache.h"
 #include "Interface/Core/Dispatcher/Dispatcher.h"
@@ -30,6 +28,7 @@ $end_info$
 #include <FEXCore/Utils/CompilerDefs.h>
 #include <FEXCore/Utils/EnumUtils.h>
 #include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/LongJump.h>
 #include <FEXCore/Utils/Profiler.h>
 #include <FEXCore/Utils/Telemetry.h>
 #include <FEXCore/Utils/TypeDefines.h>
@@ -37,7 +36,6 @@ $end_info$
 
 #include <cstdio>
 #include <cstring>
-#include <limits>
 #include <unistd.h>
 
 namespace {
@@ -495,7 +493,7 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
   }
 }
 
-static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record, bool Call) {
+static void DirectBlockDelinker(FEXCore::Context::ExitFunctionLinkData* Record, bool Call) {
   uintptr_t JumpThunkStartAddress = reinterpret_cast<uintptr_t>(Record) - 0x10;
   uintptr_t CallerAddress = JumpThunkStartAddress + Record->CallerOffset;
   auto BranchOffset = JumpThunkStartAddress / 4 - CallerAddress / 4;
@@ -513,7 +511,7 @@ static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Co
   ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(CallerAddress), 4);
 }
 
-static void IndirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
+static void IndirectBlockDelinker(FEXCore::Context::ExitFunctionLinkData* Record) {
   uintptr_t JumpThunkStartAddress = reinterpret_cast<uintptr_t>(Record) - 0x10;
   uint32_t BranchInst = 0;
   ARMEmitter::Emitter BranchEmit(reinterpret_cast<uint8_t*>(&BranchInst), 4);
@@ -580,13 +578,13 @@ uint64_t Arm64JITCore::ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEX
       BranchEmit.bl(BranchOffset);
       Thread->LookupCache->AddBlockLink(
         GuestRip, Record,
-        [](FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Frame, Record, true); }, lk);
+        [](FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Record, true); }, lk);
     } else {
       BranchEmit.b(BranchOffset);
       Thread->LookupCache->AddBlockLink(
         GuestRip, Record,
-        [](FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
-          DirectBlockDelinker(Frame, Record, false);
+        [](FEXCore::Context::ExitFunctionLinkData* Record) {
+          DirectBlockDelinker(Record, false);
         },
         lk);
     }
@@ -742,11 +740,11 @@ void Arm64JITCore::EmitTFCheck() {
   // Note that this needs to be before the below suspend checks, as X86 checks this flag immediately after executing an instruction.
   ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
 
-  cbz(ARMEmitter::Size::i32Bit, TMP1, &l_TFUnset);
+  (void)cbz(ARMEmitter::Size::i32Bit, TMP1, &l_TFUnset);
 
   // X86 semantically checks TF after executing each instruction, so e.g. setting a context with TF set will execute a single instruction
   // and then raise an exception. However on the FEX side this is simpler to implement by checking at the start of each instruction, handle this by having bit 1 being unset in the flag state indicate that TF is blocked for a single instruction.
-  tbz(TMP1, 1, &l_TFBlocked);
+  (void)tbz(TMP1, 1, &l_TFBlocked);
 
   // Block TF for a single instruction when the frontend jumps to a new context by unsetting bit 1.
   ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
@@ -769,11 +767,11 @@ void Arm64JITCore::EmitTFCheck() {
   ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP));
   br(TMP1);
 
-  Bind(&l_TFBlocked);
+  (void)Bind(&l_TFBlocked);
   // If TF was blocked for this instruction, unblock it for the next.
   LoadConstant(ARMEmitter::Size::i32Bit, TMP1, 0b11);
   strb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
-  Bind(&l_TFUnset);
+  (void)Bind(&l_TFUnset);
 }
 
 void Arm64JITCore::EmitSuspendInterruptCheck() {
@@ -791,14 +789,14 @@ void Arm64JITCore::EmitSuspendInterruptCheck() {
   ARMEmitter::ForwardLabel l_NoSuspend;
   cbz(ARMEmitter::Size::i32Bit, TMP2, &l_NoSuspend);
   brk(SuspendMagic);
-  Bind(&l_NoSuspend);
+  (void)Bind(&l_NoSuspend);
 #endif
 }
 
 void Arm64JITCore::EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF) {
   // Get the address of the JITCodeHeader and store in to the core state.
   // Two instruction cost, each 1 cycle.
-  adr(TMP1, &HeaderLabel);
+  adr_OrRestart(TMP1, &HeaderLabel);
   str(TMP1, STATE, offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader));
 
   if (CheckTF) {
@@ -815,21 +813,32 @@ void Arm64JITCore::EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool C
       sub(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, TMP1, ARMEmitter::ExtendedType::LSL_64, 0);
     }
   }
+
+  EmitSuspendInterruptCheck();
 }
 
 CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
                                                    FEXCore::Core::DebugData* DebugData, bool CheckTF) {
   FEXCORE_PROFILE_SCOPED("Arm64::CompileCode");
-
-  JumpTargets.clear();
-  CallReturnTargets.clear();
-  PendingJumpThunks.clear();
-  uint32_t SSACount = IR->GetSSACount();
-  JumpTargets.resize(IR->GetHeader()->BlockCount, {});
-
   this->Entry = Entry;
   this->DebugData = DebugData;
   this->IR = IR;
+  RequiresFarARM64Jumps = false;
+
+  switch (static_cast<RestartOptions::Control>(FEXCore::LongJump::SetJump(RestartControl.RestartJump))) {
+  case RestartOptions::Control::Incoming:
+    // Nothing
+    break;
+  case RestartOptions::Control::EnableFarARM64Jumps: RequiresFarARM64Jumps = true; break;
+  default: ERROR_AND_DIE_FMT("Unhandled Arm64 restart condition!");
+  }
+
+  uint32_t SSACount = IR->GetSSACount();
+  JumpTargets.clear();
+  CallReturnTargets.clear();
+  PendingJumpThunks.clear();
+  JumpTargets.resize(IR->GetHeader()->BlockCount, {});
+
   CodeData.EntryPoints.clear();
 
   // Fairly excessive buffer range to make sure we don't overflow
@@ -844,7 +853,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
 
   // Put the code header at the start of the data block.
   ARMEmitter::BackwardLabel JITCodeHeaderLabel {};
-  Bind(&JITCodeHeaderLabel);
+  (void)Bind(&JITCodeHeaderLabel);
   JITCodeHeader* CodeHeader = GetCursorAddress<JITCodeHeader*>();
   CursorIncrement(sizeof(JITCodeHeader));
 
@@ -892,7 +901,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
         if (PendingTargetLabel->Backward.Location) {
           EmitSuspendInterruptCheck();
         }
-        b(PendingTargetLabel);
+        b_OrRestart(PendingTargetLabel);
         PendingTargetLabel = nullptr;
       }
 
@@ -902,14 +911,14 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
         const auto IsReturnTarget = CallReturnTargets.try_emplace(Node).first;
         if (PendingTargetLabel) {
           // If there is a fallthrough branch to this block, skip over the entrypoint code.
-          b(Target);
+          b_OrRestart(Target);
         } else if (PendingCallReturnTargetLabel && PendingCallReturnTargetLabel != &IsReturnTarget->second) {
           // If we just emitted a call, but the block we're now emitting is not the return block so don't fallthrough.
-          b(PendingCallReturnTargetLabel);
+          b_OrRestart(PendingCallReturnTargetLabel);
         }
         PendingCallReturnTargetLabel = nullptr;
 
-        Bind(&IsReturnTarget->second);
+        BindOrRestart(&IsReturnTarget->second);
         CodeData.EntryPoints.emplace(BlockStartRIP, GetCursorAddress<uint8_t*>());
         DebugData->GuestOpcodes.push_back({BlockIROp->GuestEntryOffset, GetCursorAddress<uint8_t*>() - CodeData.BlockBegin});
 
@@ -918,12 +927,12 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
 
       if (PendingCallReturnTargetLabel) {
         // If there is still a pending call return target, then the block we're emitting is not the return block so don't fallthrough.
-        b(PendingCallReturnTargetLabel);
+        b_OrRestart(PendingCallReturnTargetLabel);
         PendingCallReturnTargetLabel = nullptr;
       }
       PendingTargetLabel = nullptr;
 
-      Bind(Target);
+      BindOrRestart(Target);
     }
 
     for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {
@@ -948,7 +957,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
     if (PendingTargetLabel->Backward.Location) {
       EmitSuspendInterruptCheck();
     }
-    b(PendingTargetLabel);
+    b_OrRestart(PendingTargetLabel);
   }
   PendingTargetLabel = nullptr;
 
@@ -959,21 +968,21 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
 
     ARMEmitter::ForwardLabel l_DoLink;
     uint64_t ThunkAddress = GetCursorAddress<uint64_t>();
-    Bind(&PendingJumpThunk.Label);
-    b(&l_DoLink);
+    BindOrRestart(&PendingJumpThunk.Label);
+    b_OrRestart(&l_DoLink);
     br(TMP1);
-    Bind(&l_DoLink);
+    BindOrRestart(&l_DoLink);
     ldr(TMP1, &l_ExitLink);
     blr(TMP1);
 
     // This is a ExitFunctionLinkData struct
-    Bind(&l_ExitLink);
+    BindOrRestart(&l_ExitLink);
     dc64(0);                                             // HostCode
     dc64(PendingJumpThunk.GuestRIP);                     // GuestRIP
     dc64(PendingJumpThunk.CallerAddress - ThunkAddress); // CallerOffset
   }
 
-  Bind(&l_ExitLink);
+  BindOrRestart(&l_ExitLink);
   dc64(ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker);
 
   // CodeSize not including the header or tail data.

FEXCore/Source/Interface/Core/JIT/JITClass.h

@@ -23,6 +23,7 @@ $end_info$
 #include <FEXCore/fextl/memory.h>
 #include <FEXCore/fextl/string.h>
 #include <FEXCore/fextl/vector.h>
+#include <FEXCore/Utils/LongJump.h>
 
 #include <CodeEmitter/Emitter.h>
 
@@ -66,6 +67,19 @@ private:
   const bool HostSupportsRPRES {};
   const bool HostSupportsAFP {};
 
+  struct RestartOptions {
+    FEXCore::LongJump::JumpBuf RestartJump;
+    enum class Control : uint64_t {
+      Incoming = 0,
+      EnableFarARM64Jumps = 1,
+    };
+  };
+
+  // FEXCore makes assumptions in the JIT about certain conditions being true.
+  // In the rare case when those assumptions are broken, FEX needs to safely restart the JIT.
+  RestartOptions RestartControl {};
+  bool RequiresFarARM64Jumps {};
+
   ARMEmitter::BiDirectionalLabel* PendingTargetLabel {};
   ARMEmitter::BiDirectionalLabel* PendingCallReturnTargetLabel {};
   FEXCore::Context::ContextImpl* CTX {};
@@ -329,14 +343,179 @@ private:
   void EmitLinkedBranch(uint64_t GuestRIP, bool Call) {
     PendingJumpThunks.push_back({GetCursorAddress<uint64_t>(), GuestRIP, {}});
     auto& Thunk = PendingJumpThunks.back();
-    Bind(&Thunk.Label);
+    BindOrRestart(&Thunk.Label);
     if (Call) {
-      bl(&Thunk.Label);
+      bl_OrRestart(&Thunk.Label);
     } else {
-      b(&Thunk.Label);
+      b_OrRestart(&Thunk.Label);
     }
   }
 
+  // Restart helpers
+  template<ARMEmitter::IsLabel T>
+  void bl_OrRestart(T* Label) {
+    if (bl(Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    // We can support this but currently unnecessary.
+    ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void b_OrRestart(T* Label) {
+    if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    // We can support this but currently unnecessary.
+    ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void b_OrRestart(ARMEmitter::Condition Cond, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)b(InvertCondition(Cond), &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (b(Cond, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void cbz_OrRestart(ARMEmitter::Size s, ARMEmitter::Register rt, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)cbnz(s, rt, &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (cbz(s, rt, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void cbnz_OrRestart(ARMEmitter::Size s, ARMEmitter::Register rt, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)cbz(s, rt, &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (cbnz(s, rt, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void tbz_OrRestart(ARMEmitter::Register rt, uint32_t Bit, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)tbnz(rt, Bit, &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (tbz(rt, Bit, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void tbnz_OrRestart(ARMEmitter::Register rt, uint32_t Bit, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)tbz(rt, Bit, &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (tbnz(rt, Bit, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void adr_OrRestart(ARMEmitter::Register rd, T* Label) {
+    if (adr(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    // We can support this but currently unnecessary.
+    ERROR_AND_DIE_FMT("Long ADR currently unsupported!");
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void adrp_OrRestart(ARMEmitter::Register rd, T* Label) {
+    if (adrp(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    // We can support this but currently unnecessary.
+    ERROR_AND_DIE_FMT("Long ADRP currently unsupported!");
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<ARMEmitter::IsLabel T>
+  void BindOrRestart(T* Label) {
+    if (Bind(Label)) {
+      return;
+    }
+
+    if (RequiresFarARM64Jumps) {
+      // This should have been caught before this point.
+      ERROR_AND_DIE_FMT("Unhandled long bind");
+      return;
+    }
+
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
   // This is purely a debugging aid for developers to see if they are in JIT code space when inspecting raw memory
   void EmitDetectionString();
   IR::RegisterAllocationPass* RAPass {};

FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp

@@ -912,7 +912,7 @@ DEF_OP(VLoadVectorMasked) {
 
       // If the sign bit is zero then skip the load
       ARMEmitter::ForwardLabel Skip {};
-      tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
+      (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
       // Do the gather load for this element into the destination
       switch (IROp->ElementSize) {
       case IR::OpSize::i8Bit: ld1<ARMEmitter::SubRegSize::i8Bit>(TempDst.Q(), i, TempMemReg); break;
@@ -923,7 +923,7 @@ DEF_OP(VLoadVectorMasked) {
       default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, IROp->ElementSize); return;
       }
 
-      Bind(&Skip);
+      (void)Bind(&Skip);
 
       if ((i + 1) != NumElements) {
         // Handle register rename to save a move.
@@ -1013,7 +1013,7 @@ DEF_OP(VStoreVectorMasked) {
 
       // If the sign bit is zero then skip the load
       ARMEmitter::ForwardLabel Skip {};
-      tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
+      (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
       // Do the gather load for this element into the destination
       switch (IROp->ElementSize) {
       case IR::OpSize::i8Bit: st1<ARMEmitter::SubRegSize::i8Bit>(RegData.Q(), i, TempMemReg); break;
@@ -1024,7 +1024,7 @@ DEF_OP(VStoreVectorMasked) {
       default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, IROp->ElementSize); return;
       }
 
-      Bind(&Skip);
+      (void)Bind(&Skip);
 
       if ((i + 1) != NumElements) {
         // Handle register rename to save a move.
@@ -1102,7 +1102,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
     PerformMove(ElementSize, WorkingReg, MaskReg, i);
 
     // Skip if the mask's sign bit isn't set
-    tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
+    (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
 
     // Extract Index Element
     if ((IndexElement * IR::OpSizeToSize(VectorIndexSize)) >= 16) {
@@ -1140,7 +1140,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
     default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, ElementSize); FEX_UNREACHABLE;
     }
 
-    Bind(&Skip);
+    (void)Bind(&Skip);
   }
 
   if (NeedsDestTmp) {
@@ -1874,7 +1874,7 @@ DEF_OP(MemSet) {
 
   if (!DirectionIsInline) {
     // Backward or forwards implementation depends on flag
-    tbnz(DirectionReg, 1, &BackwardImpl);
+    (void)tbnz(DirectionReg, 1, &BackwardImpl);
   }
 
   auto MemStore = [this](auto Value, uint32_t OpSize, int32_t Size) {
@@ -1922,7 +1922,7 @@ DEF_OP(MemSet) {
     ARMEmitter::ForwardLabel DoneInternal {};
 
     // Early exit if zero count.
-    cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+    (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
     if (!IsAtomic) {
       ARMEmitter::ForwardLabel AgainInternal256Exit {};
@@ -1939,50 +1939,50 @@ DEF_OP(MemSet) {
       // Do this in two parts, to fallback to the byte by byte loop if size < 32, and to the
       // single copy loop if size < 64.
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal128Exit);
+      (void)tbnz(TMP1, 63, &AgainInternal128Exit);
 
       // Fill VTMP2 with the set pattern
       dup(SubRegSize, VTMP2.Q(), Value);
 
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal256Exit);
+      (void)tbnz(TMP1, 63, &AgainInternal256Exit);
 
-      Bind(&AgainInternal256);
+      (void)Bind(&AgainInternal256);
       stp<ARMEmitter::IndexType::POST>(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);
       stp<ARMEmitter::IndexType::POST>(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
-      tbz(TMP1, 63, &AgainInternal256);
+      (void)tbz(TMP1, 63, &AgainInternal256);
 
-      Bind(&AgainInternal256Exit);
+      (void)Bind(&AgainInternal256Exit);
       add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal128Exit);
-      Bind(&AgainInternal128);
+      (void)tbnz(TMP1, 63, &AgainInternal128Exit);
+      (void)Bind(&AgainInternal128);
       stp<ARMEmitter::IndexType::POST>(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbz(TMP1, 63, &AgainInternal128);
+      (void)tbz(TMP1, 63, &AgainInternal128);
 
-      Bind(&AgainInternal128Exit);
+      (void)Bind(&AgainInternal128Exit);
       add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
       if (Direction == -1) {
         add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);
       }
     }
 
-    Bind(&AgainInternal);
+    (void)Bind(&AgainInternal);
     if (IsAtomic) {
       MemStoreTSO(Value, OpSize, SizeDirection);
     } else {
       MemStore(Value, OpSize, SizeDirection);
     }
     sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 1);
-    cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
+    (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
 
-    Bind(&DoneInternal);
+    (void)Bind(&DoneInternal);
 
     if (SizeDirection >= 0) {
       switch (OpSize) {
@@ -2012,12 +2012,12 @@ DEF_OP(MemSet) {
       EmitMemset(Direction);
 
       if (Direction == 1) {
-        b(&Done);
-        Bind(&BackwardImpl);
+        (void)b(&Done);
+        (void)Bind(&BackwardImpl);
       }
     }
 
-    Bind(&Done);
+    (void)Bind(&Done);
     // Destination already set to the final pointer.
   }
 }
@@ -2067,7 +2067,7 @@ DEF_OP(MemCpy) {
 
   if (!DirectionIsInline) {
     // Backward or forwards implementation depends on flag
-    tbnz(DirectionReg, 1, &BackwardImpl);
+    (void)tbnz(DirectionReg, 1, &BackwardImpl);
   }
 
   auto MemCpy = [this](uint32_t OpSize, int32_t Size) {
@@ -2164,7 +2164,7 @@ DEF_OP(MemCpy) {
     ARMEmitter::ForwardLabel DoneInternal {};
 
     // Early exit if zero count.
-    cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+    (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
     if (!IsAtomic) {
       ARMEmitter::ForwardLabel AbsPos {};
@@ -2174,11 +2174,11 @@ DEF_OP(MemCpy) {
       ARMEmitter::BackwardLabel AgainInternal256 {};
 
       sub(ARMEmitter::Size::i64Bit, TMP4, TMP2, TMP3);
-      tbz(TMP4, 63, &AbsPos);
+      (void)tbz(TMP4, 63, &AbsPos);
       neg(ARMEmitter::Size::i64Bit, TMP4, TMP4);
-      Bind(&AbsPos);
+      (void)Bind(&AbsPos);
       sub(ARMEmitter::Size::i64Bit, TMP4, TMP4, 32);
-      tbnz(TMP4, 63, &AgainInternal);
+      (void)tbnz(TMP4, 63, &AgainInternal);
 
       if (Direction == -1) {
         sub(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);
@@ -2190,30 +2190,30 @@ DEF_OP(MemCpy) {
       // Do this in two parts, to fallback to the byte by byte loop if size < 32, and to the
       // single copy loop if size < 64.
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal128Exit);
+      (void)tbnz(TMP1, 63, &AgainInternal128Exit);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal256Exit);
+      (void)tbnz(TMP1, 63, &AgainInternal256Exit);
 
-      Bind(&AgainInternal256);
+      (void)Bind(&AgainInternal256);
       MemCpy(32, 32 * Direction);
       MemCpy(32, 32 * Direction);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
-      tbz(TMP1, 63, &AgainInternal256);
+      (void)tbz(TMP1, 63, &AgainInternal256);
 
-      Bind(&AgainInternal256Exit);
+      (void)Bind(&AgainInternal256Exit);
       add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal128Exit);
-      Bind(&AgainInternal128);
+      (void)tbnz(TMP1, 63, &AgainInternal128Exit);
+      (void)Bind(&AgainInternal128);
       MemCpy(32, 32 * Direction);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbz(TMP1, 63, &AgainInternal128);
+      (void)tbz(TMP1, 63, &AgainInternal128);
 
-      Bind(&AgainInternal128Exit);
+      (void)Bind(&AgainInternal128Exit);
       add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
       if (Direction == -1) {
         add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);
@@ -2221,16 +2221,16 @@ DEF_OP(MemCpy) {
       }
     }
 
-    Bind(&AgainInternal);
+    (void)Bind(&AgainInternal);
     if (IsAtomic) {
       MemCpyTSO(OpSize, SizeDirection);
     } else {
       MemCpy(OpSize, SizeDirection);
     }
     sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 1);
-    cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
+    (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
 
-    Bind(&DoneInternal);
+    (void)Bind(&DoneInternal);
 
     // Needs to use temporaries just in case of overwrite
     mov(TMP1, MemRegDest.X());
@@ -2288,11 +2288,11 @@ DEF_OP(MemCpy) {
     for (int32_t Direction : {1, -1}) {
       EmitMemcpy(Direction);
       if (Direction == 1) {
-        b(&Done);
-        Bind(&BackwardImpl);
+        (void)b(&Done);
+        (void)Bind(&BackwardImpl);
       }
     }
-    Bind(&Done);
+    (void)Bind(&Done);
     // Destination already set to the final pointer.
   }
 }

FEXCore/Source/Interface/Core/LookupCache.cpp

@@ -87,12 +87,12 @@ void LookupCache::ClearL2Cache(const FEXCore::LookupCacheWriteLockToken& lk) {
 void LookupCache::ClearThreadLocalCaches(const LookupCacheWriteLockToken&) {
   // Clear L1 and L2 by clearing the full cache.
   FEXCore::Allocator::VirtualDontNeed(reinterpret_cast<void*>(PagePointer), TotalCacheSize, false);
+  CachedCodePages.clear();
 }
 
 void LookupCache::ClearCache(const LookupCacheWriteLockToken& lk) {
   // Clear L1 and L2 by clearing the full cache.
-  FEXCore::Allocator::VirtualDontNeed(reinterpret_cast<void*>(PagePointer), TotalCacheSize, false);
-
+  ClearThreadLocalCaches(lk);
   Shared->ClearCache(lk);
 }
 

FEXCore/Source/Interface/Core/LookupCache.h

@@ -7,6 +7,7 @@
 #include <FEXCore/fextl/memory_resource.h>
 #include <FEXCore/fextl/robin_map.h>
 #include <FEXCore/fextl/vector.h>
+#include <FEXCore/fextl/unordered_set.h>
 #include <FEXCore/fextl/memory_resource.h>
 
 #include <cstdint>
@@ -59,42 +60,61 @@ struct GuestToHostMap {
   fextl::unique_ptr<std::pmr::polymorphic_allocator<std::byte>> BlockLinks_pma;
   BlockLinksMapType* BlockLinks;
 
-  fextl::robin_map<uint64_t, uint64_t> BlockList;
+  struct BlockEntry {
+    uint64_t HostCode;
+    fextl::vector<uint64_t> CodePages;
+  };
+
+  fextl::robin_map<uint64_t, BlockEntry> BlockList;
 
   fextl::map<uint64_t, fextl::vector<uint64_t>> CodePages;
 
   GuestToHostMap();
 
   // Adds to Guest -> Host code mapping
-  void AddBlockMapping(uint64_t Address, void* HostCode, const LookupCacheWriteLockToken&) {
+  const BlockEntry& AddBlockMapping(uint64_t Address, const fextl::vector<uint64_t>& CodePages, void* HostCode, const LookupCacheWriteLockToken&) {
     // This may replace an existing mapping
     // NOTE: Generally no previous entry should exist, however there is one exception:
     //       If the backend updates the active thread's CodeBuffer, the new associated LookupCache
     //       may already contain the block address. Since is comparatively rare, we'll just leak
     //       one of the two blocks in this case.
-    BlockList[Address] = (uintptr_t)HostCode;
+    return BlockList.insert_or_assign(Address, BlockEntry {(uintptr_t)HostCode, CodePages}).first->second;
   }
 
-  std::optional<uintptr_t> FindBlock(uint64_t Address, const LookupCacheWriteLockToken&) {
+  const BlockEntry* FindBlock(uint64_t Address, const LookupCacheWriteLockToken&) {
     auto HostCode = BlockList.find(Address);
     if (HostCode == BlockList.end()) {
-      return std::nullopt;
+      return nullptr;
     }
-    return HostCode->second;
+    return &HostCode->second;
   }
 
-  bool Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address, const LookupCacheWriteLockToken&) {
+  bool Erase(uint64_t Address, const LookupCacheWriteLockToken&) {
     // Sever any links to this block
     auto lower = BlockLinks->lower_bound({Address, nullptr});
     auto upper = BlockLinks->upper_bound({Address, reinterpret_cast<FEXCore::Context::ExitFunctionLinkData*>(UINTPTR_MAX)});
     for (auto it = lower; it != upper; it = BlockLinks->erase(it)) {
-      it->second(Frame, it->first.HostLink);
+      it->second(it->first.HostLink);
     }
 
     // Remove from BlockList
     return BlockList.erase(Address) != 0;
   }
 
+  void InvalidateRange(uint64_t Start, uint64_t Length) {
+    auto lk = AcquireWriteLock();
+
+    auto lower = CodePages.lower_bound(Start >> 12);
+    auto upper = CodePages.upper_bound((Start + Length - 1) >> 12);
+
+    for (auto it = lower; it != upper; it++) {
+      for (const auto& Entry : it->second) {
+        Erase(Entry, lk);
+      }
+    }
+    CodePages.erase(lower, upper);
+  }
+
   void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink,
                     const FEXCore::Context::BlockDelinkerFunc& delinker, const LookupCacheWriteLockToken&) {
     BlockLinks->insert({{GuestDestination, HostLink}, delinker});
@@ -170,10 +190,10 @@ public:
 
       if (!HostPtr) {
         // Try L3
-        auto HostCode = Shared->FindBlock(Address, lk);
-        if (HostCode) {
-          CacheBlockMapping(Address, HostCode.value(), lk);
-          HostPtr = HostCode.value();
+        auto Entry = Shared->FindBlock(Address, lk);
+        if (Entry) {
+          CacheBlockMapping(Address, *Entry, false, lk);
+          HostPtr = Entry->HostCode;
         }
       }
     }
@@ -244,32 +264,25 @@ public:
   }
 
   // Adds to Guest -> Host code mapping
-  void AddBlockMapping(FEXCore::Core::InternalThreadState* Thread, uint64_t Address, void* HostCode) {
+  void AddBlockMapping(FEXCore::Core::InternalThreadState* Thread, uint64_t Address, const fextl::vector<uint64_t>& CodePages, void* HostCode) {
     std::optional<FEXCore::SHMStats::AccumulationBlock<uint64_t>> LockTime(
       Thread->ThreadStats ? &Thread->ThreadStats->AccumulatedCacheWriteLockTime : nullptr);
     auto lk = Shared->AcquireWriteLock();
     LockTime.reset();
 
-    Shared->AddBlockMapping(Address, HostCode, lk);
+    const auto& Entry = Shared->AddBlockMapping(Address, CodePages, HostCode, lk);
 
     // There is no need to update L1 or L2, they will get updated on first lookup
     // However, adding to L1 here increases performance
-    auto& L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1PointerMask];
-    L1Entry.GuestCode = Address;
-    L1Entry.HostCode = (uintptr_t)HostCode;
+    CacheBlockMapping(Address, Entry, true, lk);
   }
 
-  // NOTE: It's the caller's responsibility to call Erase() for all other
-  //       GuestToHostMaps that share the same LookupCache. Otherwise, the
-  //       L1/L2 caches will contain stale references to deallocated memory.
-  bool Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address, const LookupCacheWriteLockToken& lk) {
-    bool ErasedAny = Shared->Erase(Frame, Address, lk);
-
+  // Invalidates L1/L2 for a given guest block
+  void InvalidateCache(uint64_t Address, const LookupCacheWriteLockToken& lk) {
     // Do L1
     auto& L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1PointerMask];
     if (L1Entry.GuestCode == Address) {
       L1Entry.GuestCode = 0;
-      ErasedAny = true;
       // Leave L1Entry.HostCode as is, so that concurrent lookups won't read a null pointer
       // This is a soft guarantee for cross thread invalidation, as atomics are not used
       // and it hasn't been thoroughly tested
@@ -285,7 +298,7 @@ public:
       uint64_t LocalPagePointer = Pointers[Address];
       if (!LocalPagePointer) {
         // Page for this code didn't even exist, nothing to do
-        return ErasedAny;
+        return;
       }
 
       // Page exists, just set the offset to zero
@@ -293,7 +306,22 @@ public:
       BlockPointers[PageOffset].GuestCode = 0;
       BlockPointers[PageOffset].HostCode = 0;
     }
-    return true;
+  }
+
+  // Invalidates all L1/L2 entries for all guest block that intersect the given range
+  bool InvalidateCacheRange(uint64_t Start, uint64_t Length) {
+    auto lk = Shared->AcquireWriteLock();
+
+    auto lower = CachedCodePages.lower_bound(Start >> 12);
+    auto upper = CachedCodePages.upper_bound((Start + Length - 1) >> 12);
+
+    for (auto it = lower; it != upper; it++) {
+      for (const auto& Entry : it->second) {
+        InvalidateCache(Entry, lk);
+      }
+    }
+    CachedCodePages.erase(lower, upper);
+    return upper != lower;
   }
 
   void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink,
@@ -330,13 +358,17 @@ public:
   }
 
 private:
-  void CacheBlockMapping(uint64_t Address, uintptr_t HostCode, const LookupCacheWriteLockToken& lk) {
+  void CacheBlockMapping(uint64_t Address, const GuestToHostMap::BlockEntry& Entry, bool L1Only, const LookupCacheWriteLockToken& lk) {
+    for (const auto& CodePage : Entry.CodePages) {
+      CachedCodePages[CodePage >> 12].insert(Address);
+    }
+
     // Do L1
     auto& L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1PointerMask];
     L1Entry.GuestCode = Address;
-    L1Entry.HostCode = HostCode;
+    L1Entry.HostCode = Entry.HostCode;
 
-    if (!DisableL2Cache()) {
+    if (!DisableL2Cache() && !L1Only) {
       // Do ful map
       auto FullAddress = Address;
       Address = Address & (VirtualMemSize - 1);
@@ -353,7 +385,7 @@ private:
         if (!NewPageBacking) {
           // Couldn't allocate, clear L2 and retry
           ClearL2Cache(lk);
-          CacheBlockMapping(Address, HostCode, lk);
+          CacheBlockMapping(Address, Entry, false, lk);
           return;
         }
         Pointers[Address] = NewPageBacking;
@@ -365,7 +397,7 @@ private:
 
       // This silently replaces existing mappings
       BlockPointers[PageOffset].GuestCode = FullAddress;
-      BlockPointers[PageOffset].HostCode = HostCode;
+      BlockPointers[PageOffset].HostCode = Entry.HostCode;
     }
   }
 
@@ -383,6 +415,9 @@ private:
     return PageMemory + NewBase;
   }
 
+  // Maps from a page index to all blocks in the page that have at some point been fetched into L1/L2
+  fextl::map<uint64_t, fextl::unordered_set<uint64_t>> CachedCodePages;
+
   uintptr_t PagePointer;
   uintptr_t PageMemory;
   uintptr_t L1Pointer;

FEXCore/Source/Utils/LongJump.cpp

@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: MIT
+#include <FEXCore/Utils/LongJump.h>
+
+namespace FEXCore::LongJump {
+#if defined(_M_ARM_64)
+[[nodiscard]]
+FEX_DEFAULT_VISIBILITY FEX_NAKED uint64_t SetJump(JumpBuf& Buffer) {
+  __asm volatile(R"(
+      // x0 contains the jumpbuffer
+      stp x19, x20, [x0, #( 0 * 8)];
+      stp x21, x22, [x0, #( 2 * 8)];
+      stp x23, x24, [x0, #( 4 * 8)];
+      stp x25, x26, [x0, #( 6 * 8)];
+      stp x27, x28, [x0, #( 8 * 8)];
+      stp x29, x30, [x0, #(10 * 8)];
+
+      // FPRs
+      stp d8,   d9, [x0, #(12 * 8)];
+      stp d10, d11, [x0, #(14 * 8)];
+      stp d12, d13, [x0, #(16 * 8)];
+      stp d14, d15, [x0, #(18 * 8)];
+
+      // Move SP in to a temporary to store.
+      mov x1, sp;
+      str x1,  [x0, #(20 * 8)];
+
+      // Return zero to signify this is the SetJump.
+      mov x0, #0;
+      ret;
+    )" ::
+                   : "memory");
+}
+
+[[noreturn]]
+FEX_DEFAULT_VISIBILITY FEX_NAKED void LongJump(JumpBuf& Buffer, uint64_t Value) {
+  __asm volatile(R"(
+      // x0 contains the jumpbuffer
+      ldp x19, x20, [x0, #( 0 * 8)];
+      ldp x21, x22, [x0, #( 2 * 8)];
+      ldp x23, x24, [x0, #( 4 * 8)];
+      ldp x25, x26, [x0, #( 6 * 8)];
+      ldp x27, x28, [x0, #( 8 * 8)];
+      ldp x29, x30, [x0, #(10 * 8)];
+
+      // FPRs
+      ldp d8,   d9, [x0, #(12 * 8)];
+      ldp d10, d11, [x0, #(14 * 8)];
+      ldp d12, d13, [x0, #(16 * 8)];
+      ldp d14, d15, [x0, #(18 * 8)];
+
+      // Load SP in to temporary then move
+      ldr x0,  [x0, #(20 * 8)];
+      mov sp, x0;
+
+      // Move value in to result register
+      mov x0, x1;
+      ret;
+    )" ::
+                   : "memory");
+}
+#else
+[[nodiscard]]
+FEX_DEFAULT_VISIBILITY FEX_NAKED uint64_t SetJump(JumpBuf& Buffer) {
+  __asm volatile(R"(
+    .intel_syntax noprefix;
+    // rdi contains the jumpbuffer
+    mov [rdi + (0 * 8)], rbx;
+    mov [rdi + (1 * 8)], rsp;
+    mov [rdi + (2 * 8)], rbp;
+    mov [rdi + (3 * 8)], r12;
+    mov [rdi + (4 * 8)], r13;
+    mov [rdi + (5 * 8)], r14;
+    mov [rdi + (6 * 8)], r15;
+
+    // Return address is on the stack, load it and store
+    mov rsi, [rsp];
+    mov [rdi + (7 * 8)], rsi;
+
+    // Return zero to signify this is the SetJump.
+    mov rax, 0;
+    ret;
+
+    .att_syntax prefix;
+    )" ::
+                   : "memory");
+}
+
+[[noreturn]]
+FEX_DEFAULT_VISIBILITY FEX_NAKED void LongJump(JumpBuf& Buffer, uint64_t Value) {
+  __asm volatile(R"(
+    .intel_syntax noprefix;
+    // rdi contains the jumpbuffer
+    mov rbx, [rdi + (0 * 8)];
+    mov rsp, [rdi + (1 * 8)];
+    mov rbp, [rdi + (2 * 8)];
+    mov r12, [rdi + (3 * 8)];
+    mov r13, [rdi + (4 * 8)];
+    mov r14, [rdi + (5 * 8)];
+    mov r15, [rdi + (6 * 8)];
+
+    // Move value in to result register
+    mov rax, rsi;
+
+    // Pop the dead return address off the stack
+    pop rsi;
+
+    // Load the original return address from the jumpbuffer
+    mov rsi, [rdi + (7 * 8)];
+
+    // Return using a jump
+    jmp rsi;
+
+    .att_syntax prefix;
+    )" ::
+                   : "memory");
+}
+
+#endif
+} // namespace FEXCore::LongJump

FEXCore/include/FEXCore/Core/Context.h

@@ -42,9 +42,6 @@ enum OperatingMode {
 
 using CodeRangeInvalidationFn = std::function<void(uint64_t start, uint64_t Length)>;
 
-// Nested vector of guest block entrypoints
-using InvalidatedEntryAccumulator = fextl::vector<fextl::vector<uint64_t>>;
-
 using CustomIREntrypointHandler = std::function<void(uintptr_t Entrypoint, IR::IREmitter*)>;
 
 using ExitHandler = std::function<void(Core::InternalThreadState* Thread)>;
@@ -141,8 +138,9 @@ public:
   virtual AbstractCodeCache& GetCodeCache() = 0;
 
   FEX_DEFAULT_VISIBILITY virtual void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) = 0;
-  FEX_DEFAULT_VISIBILITY virtual void InvalidateGuestCodeRange(
-    FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, uint64_t Start, uint64_t Length) = 0;
+  FEX_DEFAULT_VISIBILITY virtual void InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) = 0;
+  FEX_DEFAULT_VISIBILITY virtual void
+  InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) = 0;
   FEX_DEFAULT_VISIBILITY virtual FEXCore::ForkableSharedMutex& GetCodeInvalidationMutex() = 0;
 
   FEX_DEFAULT_VISIBILITY virtual void

FEXCore/include/FEXCore/Utils/LongJump.h

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+#include <FEXCore/Utils/CompilerDefs.h>
+
+#include <cstdint>
+
+// Reimplementation of longjmp without glibc fortification checks.
+// This is useful to avoid false positives reported by glibc.
+namespace FEXCore::LongJump {
+// JumpBuf definition needs to be public because the frontend needs to understand it.
+#if defined(_M_ARM_64)
+struct JumpBuf {
+  // All the registers that are required by AAPCS64 to save.
+  // GPRs
+  // X19, X20, X21, X22,
+  // X23, X24, X25, X26,
+  // X27, X28, X29, X30,
+  //
+  // Lower 64-bits:
+  //  V8,  V9, V10, V11,
+  // V12, V13, V14, V15,
+  //
+  // SP,
+  uint64_t Registers[21];
+};
+#else
+struct JumpBuf {
+  // Registers to preserve
+  // RBX, RSP, RBP, R12, R13, R14, R15,
+  // <return address>
+  uint64_t Registers[8];
+};
+#endif
+
+[[nodiscard]] FEX_DEFAULT_VISIBILITY uint64_t SetJump(JumpBuf& Buffer);
+[[noreturn]] FEX_DEFAULT_VISIBILITY void LongJump(JumpBuf& Buffer, uint64_t Value);
+} // namespace FEXCore::LongJump

FEXCore/unittests/Emitter/ALU_Tests.cpp

@@ -9,17 +9,17 @@ using namespace ARMEmitter;
 TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    adr(Reg::r30, &Label);
+    (void)adr(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x10fffffe);
   }
 
   {
     ForwardLabel Label;
-    adr(Reg::r30, &Label);
-    Bind(&Label);
+    (void)adr(Reg::r30, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x1000003e);
@@ -27,17 +27,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    adr(Reg::r30, &Label);
+    (void)adr(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x10fffffe);
   }
 
   {
     BiDirectionalLabel Label;
-    adr(Reg::r30, &Label);
-    Bind(&Label);
+    (void)adr(Reg::r30, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x1000003e);
@@ -45,42 +45,42 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    adrp(Reg::r30, &Label);
+    (void)adrp(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x9000001e);
   }
 
   {
     ForwardLabel Label;
-    adrp(Reg::r30, &Label);
+    (void)adrp(Reg::r30, &Label);
     // Move label a page away
     for (size_t i = 0; i < 1023; ++i) {
       nop();
     }
-    Bind(&Label);
+    (void)Bind(&Label);
 
     CHECK(DisassembleEncoding(0) == 0xb000001e);
   }
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    adrp(Reg::r30, &Label);
+    (void)adrp(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x9000001e);
   }
 
   {
     BiDirectionalLabel Label;
-    adrp(Reg::r30, &Label);
+    (void)adrp(Reg::r30, &Label);
     // Move label a page away
     for (size_t i = 0; i < 1023; ++i) {
       nop();
     }
-    Bind(&Label);
+    (void)Bind(&Label);
 
     CHECK(DisassembleEncoding(0) == 0xb000001e);
   }
@@ -88,17 +88,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate adr.
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
     CHECK(DisassembleEncoding(1) == 0x10fffffe);
   }
   {
     // Will generate nop + adr.
     ForwardLabel Label;
-    LongAddressGen(Reg::r30, &Label);
-    Bind(&Label);
+    (void)LongAddressGen(Reg::r30, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd503201f);
@@ -107,9 +107,9 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate adr.
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x10fffffe);
   }
@@ -117,8 +117,8 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate nop + adr.
     BiDirectionalLabel Label;
-    LongAddressGen(Reg::r30, &Label);
-    Bind(&Label);
+    (void)LongAddressGen(Reg::r30, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd503201f);
@@ -128,7 +128,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate adrp.
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     // Move adrp 1MB away.
@@ -136,7 +136,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
       nop();
     }
 
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
     nop();
     CHECK(DisassembleEncoding(262145) == 0x90fff81e);
     CHECK(DisassembleEncoding(262146) == 0xd503201f);
@@ -145,14 +145,14 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate nop + adrp.
     ForwardLabel Label;
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     // Move label 1MB away, plus a page, and then aligned to a page.
     for (size_t i = 0; i < ((1 * 1024 * 1024 + 4096) / 4 - 2); ++i) {
       nop();
     }
 
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd503201f);
@@ -162,14 +162,14 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate adrp + add.
     ForwardLabel Label;
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     // Move label 1MB away, plus a page, plus one instruction.
     for (size_t i = 0; i < ((1 * 1024 * 1024 + 4096) / 4 - 1); ++i) {
       nop();
     }
 
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb000081e);
@@ -180,7 +180,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate adrp.
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     // Move adrp 1MB away.
@@ -188,7 +188,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
       nop();
     }
 
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
     nop();
     CHECK(DisassembleEncoding(262145) == 0x90fff81e);
     CHECK(DisassembleEncoding(262146) == 0xd503201f);
@@ -197,14 +197,14 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate nop + adrp.
     BiDirectionalLabel Label;
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     // Move label 1MB away, plus a page, and then aligned to a page.
     for (size_t i = 0; i < ((1 * 1024 * 1024 + 4096) / 4 - 2); ++i) {
       nop();
     }
 
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd503201f);
@@ -214,14 +214,14 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     // Will generate adrp + add.
     BiDirectionalLabel Label;
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     // Move label 1MB away, plus a page, plus one instruction.
     for (size_t i = 0; i < ((1 * 1024 * 1024 + 4096) / 4 - 1); ++i) {
       nop();
     }
 
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb000081e);

FEXCore/unittests/Emitter/Branch_Tests.cpp

@@ -9,17 +9,17 @@ using namespace ARMEmitter;
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Conditional branch immediate") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    b(Condition::CC_PL, &Label);
+    (void)b(Condition::CC_PL, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x54ffffe5);
   }
 
   {
     ForwardLabel Label;
-    b(Condition::CC_PL, &Label);
-    Bind(&Label);
+    (void)b(Condition::CC_PL, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x54000025);
@@ -27,17 +27,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Conditional branch immediat
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    b(Condition::CC_PL, &Label);
+    (void)b(Condition::CC_PL, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x54ffffe5);
   }
 
   {
     BiDirectionalLabel Label;
-    b(Condition::CC_PL, &Label);
-    Bind(&Label);
+    (void)b(Condition::CC_PL, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x54000025);
@@ -46,17 +46,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Conditional branch immediat
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Branch consistent conditional") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    bc(Condition::CC_PL, &Label);
+    (void)bc(Condition::CC_PL, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x54fffff5);
   }
 
   {
     ForwardLabel Label;
-    bc(Condition::CC_PL, &Label);
-    Bind(&Label);
+    (void)bc(Condition::CC_PL, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x54000035);
@@ -64,17 +64,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Branch consistent condition
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    bc(Condition::CC_PL, &Label);
+    (void)bc(Condition::CC_PL, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x54fffff5);
   }
 
   {
     BiDirectionalLabel Label;
-    bc(Condition::CC_PL, &Label);
-    Bind(&Label);
+    (void)bc(Condition::CC_PL, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x54000035);
@@ -89,17 +89,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch regist
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immediate") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    b(&Label);
+    (void)b(&Label);
 
     CHECK(DisassembleEncoding(1) == 0x17ffffff);
   }
 
   {
     ForwardLabel Label;
-    b(&Label);
-    Bind(&Label);
+    (void)b(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x14000001);
@@ -107,17 +107,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immedi
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    b(&Label);
+    (void)b(&Label);
 
     CHECK(DisassembleEncoding(1) == 0x17ffffff);
   }
 
   {
     BiDirectionalLabel Label;
-    b(&Label);
-    Bind(&Label);
+    (void)b(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x14000001);
@@ -125,17 +125,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immedi
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    bl(&Label);
+    (void)bl(&Label);
 
     CHECK(DisassembleEncoding(1) == 0x97ffffff);
   }
 
   {
     ForwardLabel Label;
-    bl(&Label);
-    Bind(&Label);
+    (void)bl(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x94000001);
@@ -143,17 +143,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immedi
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    bl(&Label);
+    (void)bl(&Label);
 
     CHECK(DisassembleEncoding(1) == 0x97ffffff);
   }
 
   {
     BiDirectionalLabel Label;
-    bl(&Label);
-    Bind(&Label);
+    (void)bl(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x94000001);
@@ -162,17 +162,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immedi
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbz(Size::i32Bit, Reg::r29, &Label);
+    (void)cbz(Size::i32Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x34fffffd);
   }
 
   {
     ForwardLabel Label;
-    cbz(Size::i32Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbz(Size::i32Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3400003d);
@@ -180,17 +180,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbz(Size::i32Bit, Reg::r29, &Label);
+    (void)cbz(Size::i32Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x34fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    cbz(Size::i32Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbz(Size::i32Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3400003d);
@@ -198,17 +198,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbz(Size::i64Bit, Reg::r29, &Label);
+    (void)cbz(Size::i64Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb4fffffd);
   }
 
   {
     ForwardLabel Label;
-    cbz(Size::i64Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbz(Size::i64Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb400003d);
@@ -216,17 +216,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbz(Size::i64Bit, Reg::r29, &Label);
+    (void)cbz(Size::i64Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb4fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    cbz(Size::i64Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbz(Size::i64Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb400003d);
@@ -234,17 +234,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbnz(Size::i32Bit, Reg::r29, &Label);
+    (void)cbnz(Size::i32Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x35fffffd);
   }
 
   {
     ForwardLabel Label;
-    cbnz(Size::i32Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbnz(Size::i32Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3500003d);
@@ -252,17 +252,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbnz(Size::i32Bit, Reg::r29, &Label);
+    (void)cbnz(Size::i32Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x35fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    cbnz(Size::i32Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbnz(Size::i32Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3500003d);
@@ -270,17 +270,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbnz(Size::i64Bit, Reg::r29, &Label);
+    (void)cbnz(Size::i64Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb5fffffd);
   }
 
   {
     ForwardLabel Label;
-    cbnz(Size::i64Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbnz(Size::i64Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb500003d);
@@ -288,17 +288,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbnz(Size::i64Bit, Reg::r29, &Label);
+    (void)cbnz(Size::i64Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb5fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    cbnz(Size::i64Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbnz(Size::i64Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb500003d);
@@ -307,17 +307,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbz(Reg::r29, 0, &Label);
+    (void)tbz(Reg::r29, 0, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x3607fffd);
   }
 
   {
     ForwardLabel Label;
-    tbz(Reg::r29, 0, &Label);
-    Bind(&Label);
+    (void)tbz(Reg::r29, 0, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3600003d);
@@ -325,17 +325,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbz(Reg::r29, 0, &Label);
+    (void)tbz(Reg::r29, 0, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x3607fffd);
   }
 
   {
     BiDirectionalLabel Label;
-    tbz(Reg::r29, 0, &Label);
-    Bind(&Label);
+    (void)tbz(Reg::r29, 0, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3600003d);
@@ -343,17 +343,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbz(Reg::r29, 63, &Label);
+    (void)tbz(Reg::r29, 63, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb6fffffd);
   }
 
   {
     ForwardLabel Label;
-    tbz(Reg::r29, 63, &Label);
-    Bind(&Label);
+    (void)tbz(Reg::r29, 63, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb6f8003d);
@@ -361,17 +361,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbz(Reg::r29, 63, &Label);
+    (void)tbz(Reg::r29, 63, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb6fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    tbz(Reg::r29, 63, &Label);
-    Bind(&Label);
+    (void)tbz(Reg::r29, 63, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb6f8003d);
@@ -379,17 +379,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbnz(Reg::r29, 0, &Label);
+    (void)tbnz(Reg::r29, 0, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x3707fffd);
   }
 
   {
     ForwardLabel Label;
-    tbnz(Reg::r29, 0, &Label);
-    Bind(&Label);
+    (void)tbnz(Reg::r29, 0, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3700003d);
@@ -397,17 +397,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbnz(Reg::r29, 0, &Label);
+    (void)tbnz(Reg::r29, 0, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x3707fffd);
   }
 
   {
     BiDirectionalLabel Label;
-    tbnz(Reg::r29, 0, &Label);
-    Bind(&Label);
+    (void)tbnz(Reg::r29, 0, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3700003d);
@@ -415,17 +415,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbnz(Reg::r29, 63, &Label);
+    (void)tbnz(Reg::r29, 63, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb7fffffd);
   }
 
   {
     ForwardLabel Label;
-    tbnz(Reg::r29, 63, &Label);
-    Bind(&Label);
+    (void)tbnz(Reg::r29, 63, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb7f8003d);
@@ -433,17 +433,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbnz(Reg::r29, 63, &Label);
+    (void)tbnz(Reg::r29, 63, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb7fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    tbnz(Reg::r29, 63, &Label);
-    Bind(&Label);
+    (void)tbnz(Reg::r29, 63, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb7f8003d);

FEXCore/unittests/Emitter/Loadstore_Tests.cpp

@@ -1323,7 +1323,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: LDAPR/STLR unscaled imme
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(WReg::w30, &Label);
 
@@ -1332,7 +1332,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(SReg::s30, &Label);
 
@@ -1341,7 +1341,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(XReg::x30, &Label);
 
@@ -1350,7 +1350,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(DReg::d30, &Label);
 
@@ -1359,7 +1359,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldrsw(XReg::x30, &Label);
 
@@ -1368,7 +1368,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(QReg::q30, &Label);
 
@@ -1377,7 +1377,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     prfm(Prefetch::PLDL1KEEP, &Label);
 
@@ -1387,7 +1387,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(WReg::w30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x1800003e);
@@ -1396,7 +1396,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(SReg::s30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x1c00003e);
@@ -1405,7 +1405,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(XReg::x30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x5800003e);
@@ -1414,7 +1414,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(DReg::d30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x5c00003e);
@@ -1423,7 +1423,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldrsw(XReg::x30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x9800003e);
@@ -1432,7 +1432,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(QReg::q30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x9c00003e);
@@ -1441,7 +1441,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     prfm(Prefetch::PLDL1KEEP, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd8000020);

Source/Tools/CodeSizeValidation/Main.cpp

@@ -52,8 +52,8 @@ public:
 
     {
       auto CodeInvalidationlk = FEXCore::GuardSignalDeferringSection(CTX->GetCodeInvalidationMutex(), Thread);
-      FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-      CTX->InvalidateGuestCodeRange(Thread, Accumulator, reinterpret_cast<uint64_t>(CodeStart), MAX_CODE_SIZE);
+      CTX->InvalidateCodeBuffersCodeRange(reinterpret_cast<uint64_t>(CodeStart), MAX_CODE_SIZE);
+      CTX->InvalidateThreadCachedCodeRange(Thread, reinterpret_cast<uint64_t>(CodeStart), MAX_CODE_SIZE);
     }
 
     ClearStats();

Source/Tools/FEXInterpreter/FEXInterpreter.cpp

@@ -222,6 +222,28 @@ void CheckForGCS() {
 }
 } // namespace FEX::GCS
 
+namespace FEX::UnalignedAtomic {
+void SetupKernelUnalignedAtomics() {
+#ifndef PR_ARM64_SET_UNALIGN_ATOMIC
+#define PR_ARM64_SET_UNALIGN_ATOMIC 0x46455849
+#define PR_ARM64_UNALIGN_ATOMIC_EMULATE (1UL << 0)
+#define PR_ARM64_UNALIGN_ATOMIC_BACKPATCH (1UL << 1)
+#define PR_ARM64_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS (1UL << 2)
+#endif
+
+  // Interfaces with downstream FEX kernel patches to control unaligned atomic handling
+  FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
+  FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS);
+
+  uint64_t Flags = (StrictInProcessSplitLocks() ? PR_ARM64_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS : 0) |
+                   (ParanoidTSO() ? 0 : PR_ARM64_UNALIGN_ATOMIC_BACKPATCH) | PR_ARM64_UNALIGN_ATOMIC_EMULATE;
+
+  if (prctl(PR_ARM64_SET_UNALIGN_ATOMIC, Flags, 0, 0, 0) != -1) {
+    LogMan::Msg::IFmt("FEX: Kernel unaligned atomics enabled!");
+  }
+}
+} // namespace FEX::UnalignedAtomic
+
 /**
  * @brief Get an FD from an environment variable and then unset the environment variable.
  *
@@ -458,6 +480,7 @@ int main(int argc, char** argv, char** const envp) {
 
   // Setup TSO hardware emulation immediately after initializing the context.
   FEX::TSO::SetupTSOEmulation(CTX.get());
+  FEX::UnalignedAtomic::SetupKernelUnalignedAtomics();
 
   if (!Loader.Is64BitMode()) {
     // Tell the kernel we want to use the compat input syscalls even though we're

Source/Tools/LinuxEmulation/LinuxSyscalls/Seccomp/BPFEmitter.cpp

@@ -204,7 +204,7 @@ uint64_t BPFEmitter::HandleJmp(uint32_t BPFIP, uint32_t NumInst, const sock_filt
       TargetLabel = JumpLabels.try_emplace(Target, ARMEmitter::ForwardLabel {}).first;
     }
 
-    EMIT_INST(b(&TargetLabel->second));
+    EMIT_INST((void)b(&TargetLabel->second));
     break;
   }
   case BPF_JEQ:
@@ -248,8 +248,8 @@ uint64_t BPFEmitter::HandleJmp(uint32_t BPFIP, uint32_t NumInst, const sock_filt
       TargetFalseLabel = JumpLabels.try_emplace(TargetFalse, ARMEmitter::ForwardLabel {}).first;
     }
 
-    EMIT_INST(b(CompareResultOp, &TargetTrueLabel->second));
-    EMIT_INST(b(&TargetFalseLabel->second));
+    EMIT_INST((void)b(CompareResultOp, &TargetTrueLabel->second));
+    EMIT_INST((void)b(&TargetFalseLabel->second));
     break;
   }
   default: RETURN_ERROR(-EINVAL); // Unknown jump type
@@ -303,7 +303,7 @@ uint64_t BPFEmitter::HandleEmission(uint32_t flags, const sock_fprog* prog) {
     if constexpr (!CalculateSize) {
       auto jump_label = JumpLabels.find(i);
       if (jump_label != JumpLabels.end()) {
-        Bind(&jump_label->second);
+        (void)Bind(&jump_label->second);
       }
     }
 
@@ -401,7 +401,7 @@ uint64_t BPFEmitter::JITFilter(uint32_t flags, const sock_fprog* prog) {
   // Emit the constant pool.
   Align();
   for (auto& Const : ConstPool) {
-    Bind(&Const.second);
+    (void)Bind(&Const.second);
     dc32(Const.first);
   }
 

Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h

@@ -208,10 +208,9 @@ public:
     // Thread object isn't valid very early in frontend's initialization.
     // To be more optimal the frontend should provide this code with a valid Thread object earlier.
     auto CodeInvalidationlk = GuardSignalDeferringSectionWithFallback(CTX->GetCodeInvalidationMutex(), CallingThread);
-    FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-
+    CTX->InvalidateCodeBuffersCodeRange(Start, Length);
     for (auto& Thread : Threads) {
-      CTX->InvalidateGuestCodeRange(Thread->Thread, Accumulator, Start, Length);
+      CTX->InvalidateThreadCachedCodeRange(Thread->Thread, Start, Length);
     }
   }
 
@@ -223,10 +222,9 @@ public:
     // Thread object isn't valid very early in frontend's initialization.
     // To be more optimal the frontend should provide this code with a valid Thread object earlier.
     auto CodeInvalidationlk = GuardSignalDeferringSectionWithFallback(CTX->GetCodeInvalidationMutex(), CallingThread);
-    FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-
+    CTX->InvalidateCodeBuffersCodeRange(Start, Length);
     for (auto& Thread : Threads) {
-      CTX->InvalidateGuestCodeRange(Thread->Thread, Accumulator, Start, Length);
+      CTX->InvalidateThreadCachedCodeRange(Thread->Thread, Start, Length);
     }
 
     // Callback while holding the locks.

Source/Tools/LinuxEmulation/LinuxSyscalls/Utils/Threads.cpp

@@ -4,6 +4,7 @@
 
 #include <FEXCore/Core/Context.h>
 #include <FEXCore/Utils/Allocator.h>
+#include <FEXCore/Utils/LongJump.h>
 #include <FEXCore/Utils/Threads.h>
 
 namespace FEX::LinuxEmulation::Threads {
@@ -190,143 +191,6 @@ __attribute__((naked)) void StackPivotAndCall(void* Arg, FEXCore::Threads::Threa
 }
 #endif
 namespace PThreads {
-  namespace LongJump {
-    // This is a custom long jump implementation that avoids the glibc implementation.
-    // This is required behaviour because glibc's fortification checks don't understand stack pivots.
-    // FEX requires a stack pivot to work through a long jump, so these two features are at odds with each other.
-#ifdef _M_ARM_64
-    struct JumpBuf {
-      // All the registers that are required by AAPCS64 to save.
-      // GPRs
-      // X19, X20, X21, X22,
-      // X23, X24, X25, X26,
-      // X27, X28, X29, X30,
-      //
-      // Lower 64-bits:
-      //  V8,  V9, V10, V11,
-      // V12, V13, V14, V15,
-      //
-      // SP,
-      uint64_t Registers[21];
-    };
-    FEX_NAKED uint64_t SetJump(JumpBuf& Buffer) {
-      __asm volatile(R"(
-        // x0 contains the jumpbuffer
-        stp x19, x20, [x0, #( 0 * 8)];
-        stp x21, x22, [x0, #( 2 * 8)];
-        stp x23, x24, [x0, #( 4 * 8)];
-        stp x25, x26, [x0, #( 6 * 8)];
-        stp x27, x28, [x0, #( 8 * 8)];
-        stp x29, x30, [x0, #(10 * 8)];
-
-        // FPRs
-        stp d8,   d9, [x0, #(12 * 8)];
-        stp d10, d11, [x0, #(14 * 8)];
-        stp d12, d13, [x0, #(16 * 8)];
-        stp d14, d15, [x0, #(18 * 8)];
-
-        // Move SP in to a temporary to store.
-        mov x1, sp;
-        str x1,  [x0, #(20 * 8)];
-
-        // Return zero to signify this is the SetJump.
-        mov x0, #0;
-        ret;
-      )" ::
-                       : "memory");
-    }
-
-    [[noreturn]]
-    FEX_NAKED void LongJump(JumpBuf& Buffer, uint64_t Value) {
-      __asm volatile(R"(
-        // x0 contains the jumpbuffer
-        ldp x19, x20, [x0, #( 0 * 8)];
-        ldp x21, x22, [x0, #( 2 * 8)];
-        ldp x23, x24, [x0, #( 4 * 8)];
-        ldp x25, x26, [x0, #( 6 * 8)];
-        ldp x27, x28, [x0, #( 8 * 8)];
-        ldp x29, x30, [x0, #(10 * 8)];
-
-        // FPRs
-        ldp d8,   d9, [x0, #(12 * 8)];
-        ldp d10, d11, [x0, #(14 * 8)];
-        ldp d12, d13, [x0, #(16 * 8)];
-        ldp d14, d15, [x0, #(18 * 8)];
-
-        // Load SP in to temporary then move
-        ldr x0,  [x0, #(20 * 8)];
-        mov sp, x0;
-
-        // Move value in to result register
-        mov x0, x1;
-        ret;
-      )" ::
-                       : "memory");
-    }
-#else
-    struct JumpBuf {
-      // Registers to preserve
-      // RBX, RSP, RBP, R12, R13, R14, R15,
-      // <return address>
-      uint64_t Registers[8];
-    };
-
-    __attribute__((naked)) uint64_t SetJump(JumpBuf& Buffer) {
-      __asm volatile(R"(
-      .intel_syntax noprefix;
-      // rdi contains the jumpbuffer
-      mov [rdi + (0 * 8)], rbx;
-      mov [rdi + (1 * 8)], rsp;
-      mov [rdi + (2 * 8)], rbp;
-      mov [rdi + (3 * 8)], r12;
-      mov [rdi + (4 * 8)], r13;
-      mov [rdi + (5 * 8)], r14;
-      mov [rdi + (6 * 8)], r15;
-
-      // Return address is on the stack, load it and store
-      mov rsi, [rsp];
-      mov [rdi + (7 * 8)], rsi;
-
-      // Return zero to signify this is the SetJump.
-      mov rax, 0;
-      ret;
-
-      .att_syntax prefix;
-      )" ::
-                       : "memory");
-    }
-
-    [[noreturn]]
-    __attribute__((naked)) void LongJump(JumpBuf& Buffer, uint64_t Value) {
-      __asm volatile(R"(
-      .intel_syntax noprefix;
-      // rdi contains the jumpbuffer
-      mov rbx, [rdi + (0 * 8)];
-      mov rsp, [rdi + (1 * 8)];
-      mov rbp, [rdi + (2 * 8)];
-      mov r12, [rdi + (3 * 8)];
-      mov r13, [rdi + (4 * 8)];
-      mov r14, [rdi + (5 * 8)];
-      mov r15, [rdi + (6 * 8)];
-
-      // Move value in to result register
-      mov rax, rsi;
-
-      // Pop the dead return address off the stack
-      pop rsi;
-
-      // Load the original return address from the jumpbuffer
-      mov rsi, [rdi + (7 * 8)];
-
-      // Return using a jump
-      jmp rsi;
-
-      .att_syntax prefix;
-      )" ::
-                       : "memory");
-    }
-#endif
-  }; // namespace LongJump
   void* InitializeThread(void* Ptr);
 
   class PThread final : public FEXCore::Threads::Thread {
@@ -397,7 +261,7 @@ namespace PThreads {
       return STracker;
     }
 
-    void SetupLongJump(LongJump::JumpBuf* exit_resolver) {
+    void SetupLongJump(FEXCore::LongJump::JumpBuf* exit_resolver) {
       _exit_resolver = exit_resolver;
     }
 
@@ -405,7 +269,7 @@ namespace PThreads {
     void LongJumpExit(FEX::HLE::ThreadStateObject* ThreadObject, uint32_t Status) {
       this->Status = Status;
       this->ThreadObject = ThreadObject;
-      LongJump::LongJump(*_exit_resolver, 1);
+      FEXCore::LongJump::LongJump(*_exit_resolver, 1);
       FEX_UNREACHABLE;
     }
 
@@ -424,7 +288,9 @@ namespace PThreads {
     void* UserArg;
     void* Stack {};
 
-    LongJump::JumpBuf* _exit_resolver {};
+    // Use FEXCore's LongJump to avoid fortification checks.
+    // This avoids a false positive since glibc does not understand stack pivots.
+    FEXCore::LongJump::JumpBuf* _exit_resolver {};
     FEX::HLE::ThreadStateObject* ThreadObject {};
     uint32_t Status {};
   };
@@ -435,11 +301,11 @@ namespace PThreads {
     PThread* Thread {reinterpret_cast<PThread*>(Ptr)};
     StackBase = Thread->GetPivotStack();
     STracker = Thread->GetStackTracker();
-    LongJump::JumpBuf exit_resolver {};
+    FEXCore::LongJump::JumpBuf exit_resolver {};
 
     bool LongJumpExit {};
 
-    if (LongJump::SetJump(exit_resolver) == 0) {
+    if (FEXCore::LongJump::SetJump(exit_resolver) == 0) {
       Thread->SetupLongJump(&exit_resolver);
       // Run the user function.
       // `Thread` object is dead after this function returns.

Source/Windows/ARM64EC/Module.cpp

@@ -639,7 +639,6 @@ NTSTATUS ProcessInit() {
 
   SignalDelegator = fextl::make_unique<FEX::DummyHandlers::DummySignalDelegator>();
   SyscallHandler = fextl::make_unique<Exception::ECSyscallHandler>();
-  Exception::HandlerConfig.emplace();
 
   const auto NtDll = GetModuleHandle("ntdll.dll");
   const bool IsWine = !!GetProcAddress(NtDll, "wine_get_version");
@@ -653,7 +652,7 @@ NTSTATUS ProcessInit() {
   CTX->SetSignalDelegator(SignalDelegator.get());
   CTX->SetSyscallHandler(SyscallHandler.get());
   CTX->InitCore();
-
+  Exception::HandlerConfig.emplace(*CTX);
   InvalidationTracker.emplace(*CTX, Threads);
 
   HandleImageMap(NtDllBase);

Source/Windows/Common/InvalidationTracker.cpp

@@ -56,11 +56,7 @@ void InvalidationTracker::HandleMemoryProtectionNotification(uint64_t Address, u
 
   if (NeedsInvalidate) {
     // IntervalsLock cannot be held during invalidation
-    std::scoped_lock Lock(CTX.GetCodeInvalidationMutex());
-    FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-    for (auto Thread : Threads) {
-      CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, AlignedBase, AlignedSize);
-    }
+    InvalidateIntervalInternal(AlignedBase, AlignedSize);
   }
 }
 
@@ -115,13 +111,8 @@ InvalidationTracker::InvalidateContainingSectionResult InvalidationTracker::Inva
          reinterpret_cast<uint64_t>(Info.AllocationBase) == SectionBase) {
     SectionSize += Info.RegionSize;
   }
-  {
-    std::scoped_lock Lock(CTX.GetCodeInvalidationMutex());
-    FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-    for (auto Thread : Threads) {
-      CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, SectionBase, SectionSize);
-    }
-  }
+
+  InvalidateIntervalInternal(SectionBase, SectionSize);
 
   if (Free) {
     std::unique_lock Lock(IntervalsLock);
@@ -141,13 +132,7 @@ void InvalidationTracker::InvalidateAlignedInterval(uint64_t Address, uint64_t S
   const auto AlignedBase = Address & FEXCore::Utils::FEX_PAGE_MASK;
   const auto AlignedSize = std::max(Size, (Address - AlignedBase + Size + FEXCore::Utils::FEX_PAGE_SIZE - 1) & FEXCore::Utils::FEX_PAGE_MASK);
 
-  {
-    std::scoped_lock Lock(CTX.GetCodeInvalidationMutex());
-    FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-    for (auto Thread : Threads) {
-      CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, AlignedBase, AlignedSize);
-    }
-  }
+  InvalidateIntervalInternal(AlignedBase, AlignedSize);
 
   if (Free) {
     std::unique_lock Lock(IntervalsLock);
@@ -197,14 +182,8 @@ bool InvalidationTracker::HandleRWXAccessViolation(FEXCore::Core::InternalThread
   }(FaultAddress);
 
   if (NeedsInvalidate) {
-    {
-      // IntervalsLock cannot be held during invalidation
-      std::scoped_lock Lock(CTX.GetCodeInvalidationMutex());
-      FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-      for (auto Thread : Threads) {
-        CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, FaultAddress & FEXCore::Utils::FEX_PAGE_MASK, FEXCore::Utils::FEX_PAGE_SIZE);
-      }
-    }
+    // IntervalsLock cannot be held during invalidation
+    InvalidateIntervalInternal(FaultAddress & FEXCore::Utils::FEX_PAGE_MASK, FEXCore::Utils::FEX_PAGE_SIZE);
     DetectMonoBackpatcherBlock(Thread, HostPc);
     return true;
   }
@@ -274,4 +253,12 @@ void InvalidationTracker::DisableSMCDetection() {
   } while (Query.Size);
 }
 
+void InvalidationTracker::InvalidateIntervalInternal(uint64_t Address, uint64_t Size) {
+  std::scoped_lock Lock(CTX.GetCodeInvalidationMutex());
+  CTX.InvalidateCodeBuffersCodeRange(Address, Size);
+  for (auto Thread : Threads) {
+    CTX.InvalidateThreadCachedCodeRange(Thread.second, Address, Size);
+  }
+}
+
 } // namespace FEX::Windows

Source/Windows/Common/InvalidationTracker.h

@@ -4,6 +4,7 @@
 #include <FEXCore/Utils/IntervalList.h>
 #include <FEXCore/HLE/SyscallHandler.h>
 #include <mutex>
+#include <shared_mutex>
 #include <unordered_map>
 #include <string_view>
 
@@ -37,6 +38,8 @@ public:
 private:
   void DetectMonoBackpatcherBlock(FEXCore::Core::InternalThreadState* Thread, uint64_t HostPC);
   void DisableSMCDetection();
+  void InvalidateIntervalInternal(uint64_t Address, uint64_t Size);
+
 
   FEXCore::IntervalList<uint64_t> XIntervals;
   FEXCore::IntervalList<uint64_t> RWXIntervals;

Source/Windows/Common/TSOHandlerConfig.h

@@ -1,18 +1,34 @@
 // SPDX-License-Identifier: MIT
 #pragma once
 
+#include <FEXCore/Core/Context.h>
 #include <FEXCore/Config/Config.h>
 #include <FEXCore/Utils/ArchHelpers/Arm64.h>
 
 namespace FEX::Windows {
 class TSOHandlerConfig final {
 public:
-  TSOHandlerConfig() {
+  TSOHandlerConfig(FEXCore::Context::Context& CTX) {
     if (HalfBarrierTSOEnabled()) {
       UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::HalfBarrier;
     } else {
       UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::NonAtomic;
     }
+
+    if (TSOEnabled()) {
+      BOOL Enable = TRUE;
+      NTSTATUS Status = NtSetInformationProcess(NtCurrentProcess(), ProcessFexHardwareTso, &Enable, sizeof(Enable));
+      if (Status == STATUS_SUCCESS) {
+        CTX.SetHardwareTSOSupport(true);
+      }
+    }
+
+    uint64_t Flags = (StrictInProcessSplitLocks() ? FEX_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS : 0) |
+                     FEX_UNALIGN_ATOMIC_BACKPATCH | FEX_UNALIGN_ATOMIC_EMULATE;
+
+    if (NtSetInformationProcess(NtCurrentProcess(), ProcessFexUnalignAtomic, &Flags, sizeof(Flags)) == STATUS_SUCCESS) {
+      LogMan::Msg::IFmt("FEX: Kernel unaligned atomics enabled!");
+    }
   }
 
   FEXCore::ArchHelpers::Arm64::UnalignedHandlerType GetUnalignedHandlerType() const {
@@ -20,7 +36,9 @@ public:
   }
 
 private:
+  FEX_CONFIG_OPT(TSOEnabled, TSOENABLED);
   FEX_CONFIG_OPT(HalfBarrierTSOEnabled, HALFBARRIERTSOENABLED);
+  FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS);
 
   FEXCore::ArchHelpers::Arm64::UnalignedHandlerType UnalignedHandlerType {FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::HalfBarrier};
 };

Source/Windows/WOW64/Module.cpp

@@ -522,7 +522,6 @@ void BTCpuProcessInit() {
 
   SignalDelegator = fextl::make_unique<FEX::DummyHandlers::DummySignalDelegator>();
   SyscallHandler = fextl::make_unique<WowSyscallHandler>();
-  Context::HandlerConfig.emplace();
   const auto NtDll = GetModuleHandle("ntdll.dll");
   const bool IsWine = !!GetProcAddress(NtDll, "wine_get_version");
   OvercommitTracker.emplace(IsWine);
@@ -535,7 +534,7 @@ void BTCpuProcessInit() {
   CTX->SetSignalDelegator(SignalDelegator.get());
   CTX->SetSyscallHandler(SyscallHandler.get());
   CTX->InitCore();
-
+  Context::HandlerConfig.emplace(*CTX);
   InvalidationTracker.emplace(*CTX, Threads);
 
   auto NtDllX86 = reinterpret_cast<SYSTEM_DLL_INIT_BLOCK*>(GetProcAddress(NtDll, "LdrSystemDllInitBlock"))->ntdll_handle;

Source/Windows/include/winternl.h

@@ -455,6 +455,12 @@ typedef enum _MEMORY_INFORMATION_CLASS {
 #define SystemEmulationBasicInformation (SYSTEM_INFORMATION_CLASS)62
 
 #define ProcessFexHardwareTso (PROCESSINFOCLASS)2000
+#define ProcessFexUnalignAtomic (PROCESSINFOCLASS)2001
+
+// These match the prctl flag values
+#define FEX_UNALIGN_ATOMIC_EMULATE (1ULL << 0)
+#define FEX_UNALIGN_ATOMIC_BACKPATCH (1ULL << 1)
+#define FEX_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS (1ULL << 2)
 
 typedef enum _KEY_VALUE_INFORMATION_CLASS {
   KeyValueBasicInformation,

unittests/ASM/FEX_bugs/BitConditionCheck.asm

@@ -0,0 +1,39 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "1"
+  },
+  "Env": { "FEX_MAXINST" : "41010", "FEX_MULTIBLOCK": "1", "FEX_TSOENABLED": "0" }
+}
+%endif
+
+; FEX-Emu had a bug where it tried to encode too large of a tbz/tbnz offset.
+
+%macro TooConditionalBitBranch 0
+  ; Stresses ARM64's tbz/tbnz branch target of +-32KB.
+  jmp %%top
+  %%top:
+
+  lea rax, [rel data]
+  mov rbx, 1
+  %rep 2000
+    add qword [rel data], rbx
+  %endrep
+  lea rax, [%%top]
+  jpe %%top
+%endmacro
+
+mov rax, 0
+cmp rax, 0
+mov rcx, 0
+
+jz long_jump
+
+TooConditionalBitBranch
+
+long_jump:
+mov rax, 1
+hlt
+
+data:
+dq 0, 0, 0, 0

unittests/ASM/FEX_bugs/BranchConditionCheck.asm

@@ -0,0 +1,40 @@
+
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "1"
+  },
+  "Env": { "FEX_MAXINST" : "41010", "FEX_MULTIBLOCK": "1", "FEX_TSOENABLED": "0" }
+}
+%endif
+
+; FEX-Emu had a bug where it tried encoding too large of a conditional branch offset.
+
+%macro TooLargeConditionalBranch 0
+  ; Stresses ARM64's b.cc/cbz/cbnz branch target of +-1MB.
+  jmp %%top
+  %%top:
+
+  lea rax, [rel data]
+  mov rbx, 1
+  %rep 37500
+    add qword [rel data], rbx
+  %endrep
+  lea rax, [%%top]
+  jnz %%top
+%endmacro
+
+mov rax, 0
+cmp rax, 0
+mov rcx, 0
+
+jz long_jump
+
+TooLargeConditionalBranch
+
+long_jump:
+mov rax, 1
+hlt
+
+data:
+dq 0, 0, 0, 0