FEX: Print a log when kernel unaligned atomics are used

The only functional difference is that the new handler always uses
half-barriers for vector atomics. There's no technical reason for
paranoid TSO not to use these and it was just missed initially.

CodeEmitter/CodeEmitter/ALUOps.inl

@@ -36,24 +36,33 @@ public:
     DataProcessing_PCRel_Imm(Op, rd, Imm);
   }
 
-  void adr(ARMEmitter::Register rd, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
     LOGMAN_THROW_A_FMT(IsADRRange(Imm), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0001'0000 << 24;
-    DataProcessing_PCRel_Imm(Op, rd, Imm);
+    if (IsADRRange(Imm)) [[likely]] {
+      constexpr uint32_t Op = 0b0001'0000 << 24;
+      DataProcessing_PCRel_Imm(Op, rd, Imm);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void adr(ARMEmitter::Register rd, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::ADR});
     constexpr uint32_t Op = 0b0001'0000 << 24;
     DataProcessing_PCRel_Imm(Op, rd, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void adr(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      adr(rd, &Label->Backward);
+      return adr(rd, &Label->Backward);
     } else {
-      adr(rd, &Label->Forward);
+      return adr(rd, &Label->Forward);
     }
   }
 
@@ -62,32 +71,42 @@ public:
     DataProcessing_PCRel_Imm(Op, rd, Imm);
   }
 
-  void adrp(ARMEmitter::Register rd, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, const BackwardLabel* Label) {
     int64_t Imm = reinterpret_cast<int64_t>(Label->Location) - (GetCursorAddress<int64_t>() & ~0xFFFLL);
     LOGMAN_THROW_A_FMT(IsADRPRange(Imm) && IsADRPAligned(Imm), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b1001'0000 << 24;
-    DataProcessing_PCRel_Imm(Op, rd, Imm);
+    if (IsADRPRange(Imm) && IsADRPAligned(Imm)) [[likely]] {
+      constexpr uint32_t Op = 0b1001'0000 << 24;
+      DataProcessing_PCRel_Imm(Op, rd, Imm);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void adrp(ARMEmitter::Register rd, ForwardLabel* Label) {
+
+  [[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::ADRP});
     constexpr uint32_t Op = 0b1001'0000 << 24;
     DataProcessing_PCRel_Imm(Op, rd, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void adrp(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      adrp(rd, &Label->Backward);
+      return adrp(rd, &Label->Backward);
     } else {
-      adrp(rd, &Label->Forward);
+      return adrp(rd, &Label->Forward);
     }
   }
 
-  void LongAddressGen(ARMEmitter::Register rd, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, const BackwardLabel* Label) {
     int64_t Imm = reinterpret_cast<int64_t>(Label->Location) - (GetCursorAddress<int64_t>());
     if (IsADRRange(Imm)) {
       // If the range is in ADR range then we can just use ADR.
-      adr(rd, Label);
+      return adr(rd, Label);
     } else if (IsADRPRange(Imm)) {
       int64_t ADRPImm = (reinterpret_cast<int64_t>(Label->Location) & ~0xFFFLL) - (GetCursorAddress<int64_t>() & ~0xFFFLL);
 
@@ -102,23 +121,28 @@ public:
         // Now even an add
         add(ARMEmitter::Size::i64Bit, rd, rd, AlignedOffset);
       }
-    } else {
-      LOGMAN_MSG_A_FMT("Unscaled offset too large");
-      FEX_UNREACHABLE;
+
+      return BranchEncodeSucceeded::Success;
     }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void LongAddressGen(ARMEmitter::Register rd, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::LONG_ADDRESS_GEN});
     // Emit a register index and a nop. These will be backpatched.
     dc32(rd.Idx());
     nop();
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void LongAddressGen(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      LongAddressGen(rd, &Label->Backward);
+      return LongAddressGen(rd, &Label->Backward);
     } else {
-      LongAddressGen(rd, &Label->Forward);
+      return LongAddressGen(rd, &Label->Forward);
     }
   }
 
@@ -862,12 +886,6 @@ public:
   }
 
 private:
-  static constexpr Condition InvertCondition(Condition cond) {
-    // These behave as always, so it makes no sense to allow inverting these.
-    LOGMAN_THROW_A_FMT(cond != Condition::CC_AL && cond != Condition::CC_NV, "Cannot invert CC_AL or CC_NV");
-    return static_cast<Condition>(FEXCore::ToUnderlying(cond) ^ 1);
-  }
-
   void and_(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t n, uint32_t immr, uint32_t imms) {
     constexpr uint32_t Op = 0b001'0010'00 << 22;
     DataProcessing_Logical_Imm(Op, s, rd, rn, n, immr, imms);

CodeEmitter/CodeEmitter/BranchOps.inl

@@ -20,23 +20,31 @@ public:
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 0, Cond, Imm);
   }
-  void b(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
-    constexpr uint32_t Op = 0b0101'010 << 25;
-    Branch_Conditional(Op, 0, 0, Cond, Imm >> 2);
+    if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0101'010 << 25;
+      Branch_Conditional(Op, 0, 0, Cond, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void b(ARMEmitter::Condition Cond, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 0, Cond, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void b(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      b(Cond, &Label->Backward);
+      return b(Cond, &Label->Backward);
     } else {
-      b(Cond, &Label->Forward);
+      return b(Cond, &Label->Forward);
     }
   }
 
@@ -45,24 +53,32 @@ public:
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 1, Cond, Imm);
   }
-  void bc(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
-    constexpr uint32_t Op = 0b0101'010 << 25;
-    Branch_Conditional(Op, 0, 1, Cond, Imm >> 2);
+    if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0101'010 << 25;
+      Branch_Conditional(Op, 0, 1, Cond, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void bc(ARMEmitter::Condition Cond, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 1, Cond, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void bc(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      bc(Cond, &Label->Backward);
+      return bc(Cond, &Label->Backward);
     } else {
-      bc(Cond, &Label->Forward);
+      return bc(Cond, &Label->Forward);
     }
   }
 
@@ -98,25 +114,32 @@ public:
 
     UnconditionalBranch(Op, Imm);
   }
-  void b(const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large");
-    constexpr uint32_t Op = 0b0001'01 << 26;
+    if (Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0001'01 << 26;
+      UnconditionalBranch(Op, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    UnconditionalBranch(Op, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void b(ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::B});
     constexpr uint32_t Op = 0b0001'01 << 26;
 
     UnconditionalBranch(Op, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void b(BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded b(BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      b(&Label->Backward);
+      return b(&Label->Backward);
     } else {
-      b(&Label->Forward);
+      return b(&Label->Forward);
     }
   }
 
@@ -126,25 +149,33 @@ public:
     UnconditionalBranch(Op, Imm);
   }
 
-  void bl(const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bl(const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large");
-    constexpr uint32_t Op = 0b1001'01 << 26;
+    if (Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b1001'01 << 26;
+      UnconditionalBranch(Op, Imm >> 2);
 
-    UnconditionalBranch(Op, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
+
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
-  void bl(ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bl(ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::B});
     constexpr uint32_t Op = 0b1001'01 << 26;
 
     UnconditionalBranch(Op, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void bl(BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded bl(BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      bl(&Label->Backward);
+      return bl(&Label->Backward);
     } else {
-      bl(&Label->Forward);
+      return bl(&Label->Forward);
     }
   }
 
@@ -155,28 +186,35 @@ public:
     CompareAndBranch(Op, s, rt, Imm);
   }
 
-  void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0011'0100 << 24;
+    if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0011'0100 << 24;
+      CompareAndBranch(Op, s, rt, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    CompareAndBranch(Op, s, rt, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
 
     constexpr uint32_t Op = 0b0011'0100 << 24;
 
     CompareAndBranch(Op, s, rt, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      cbz(s, rt, &Label->Backward);
+      return cbz(s, rt, &Label->Backward);
     } else {
-      cbz(s, rt, &Label->Forward);
+      return cbz(s, rt, &Label->Forward);
     }
   }
 
@@ -186,28 +224,35 @@ public:
     CompareAndBranch(Op, s, rt, Imm);
   }
 
-  void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0011'0101 << 24;
+    if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0011'0101 << 24;
+      CompareAndBranch(Op, s, rt, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    CompareAndBranch(Op, s, rt, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
 
     constexpr uint32_t Op = 0b0011'0101 << 24;
 
     CompareAndBranch(Op, s, rt, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      cbnz(s, rt, &Label->Backward);
+      return cbnz(s, rt, &Label->Backward);
     } else {
-      cbnz(s, rt, &Label->Forward);
+      return cbnz(s, rt, &Label->Forward);
     }
   }
 
@@ -217,28 +262,35 @@ public:
 
     TestAndBranch(Op, rt, Bit, Imm);
   }
-  void tbz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
-    LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0011'0110 << 24;
+    if (Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0011'0110 << 24;
+      TestAndBranch(Op, rt, Bit, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    TestAndBranch(Op, rt, Bit, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void tbz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::TEST_BRANCH});
 
     constexpr uint32_t Op = 0b0011'0110 << 24;
 
     TestAndBranch(Op, rt, Bit, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void tbz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      tbz(rt, Bit, &Label->Backward);
+      return tbz(rt, Bit, &Label->Backward);
     } else {
-      tbz(rt, Bit, &Label->Forward);
+      return tbz(rt, Bit, &Label->Forward);
     }
   }
 
@@ -247,27 +299,35 @@ public:
 
     TestAndBranch(Op, rt, Bit, Imm);
   }
-  void tbnz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
     int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
     LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large");
 
-    constexpr uint32_t Op = 0b0011'0111 << 24;
+    if (Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0)) [[likely]] {
+      constexpr uint32_t Op = 0b0011'0111 << 24;
+      TestAndBranch(Op, rt, Bit, Imm >> 2);
+      return BranchEncodeSucceeded::Success;
+    }
 
-    TestAndBranch(Op, rt, Bit, Imm >> 2);
+    // Can't encode.
+    return BranchEncodeSucceeded::Failure;
   }
 
-  void tbnz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
     AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::TEST_BRANCH});
     constexpr uint32_t Op = 0b0011'0111 << 24;
 
     TestAndBranch(Op, rt, Bit, 0);
+
+    // Forward label doesn't know if it can encode until Bind.
+    return BranchEncodeSucceeded::Success;
   }
 
-  void tbnz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
+  [[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
     if (Label->Backward.Location) {
-      tbnz(rt, Bit, &Label->Backward);
+      return tbnz(rt, Bit, &Label->Backward);
     } else {
-      tbnz(rt, Bit, &Label->Forward);
+      return tbnz(rt, Bit, &Label->Forward);
     }
   }
 

CodeEmitter/CodeEmitter/Emitter.h

@@ -586,6 +586,11 @@ concept IsXOrWRegister = std::is_same_v<T, XRegister> || std::is_same_v<T, WRegi
 template<typename T>
 concept IsQOrDRegister = std::is_same_v<T, QRegister> || std::is_same_v<T, DRegister>;
 
+enum class BranchEncodeSucceeded {
+  Success,
+  Failure,
+};
+
 // Whether or not a given set of vector registers are sequential
 // in increasing order as far as the register file is concerned (modulo its size)
 //
@@ -638,19 +643,25 @@ public:
 
   // Bind a backward label to an address.
   // Address that is bound is the current emitter location.
-  void Bind(BackwardLabel* Label) {
+  [[nodiscard]] bool Bind(BackwardLabel* Label) {
     LOGMAN_THROW_A_FMT(Label->Location == nullptr, "Trying to bind a label twice");
     Label->Location = GetCursorAddress<uint8_t*>();
+
+    // Always binds because it is only storing a location.
+    return true;
   }
 
-  void Bind(const ForwardLabel::Reference* Label) {
+  [[nodiscard]] bool Bind(const ForwardLabel::Reference* Label) {
     uint8_t* CurrentAddress = GetCursorAddress<uint8_t*>();
     // Patch up the instructions
     switch (Label->Type) {
     case ForwardLabel::InstType::ADR: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(IsADRRange(Imm), "Unscaled offset too large");
+      if (!IsADRRange(Imm)) [[unlikely]] {
+        // Can't bind.
+        return false;
+      }
       uint32_t InstMask = 0b11 << 29 | 0b1111'1111'1111'1111'111 << 5;
       uint32_t Offset = static_cast<uint32_t>(Imm) & 0x3F'FFFF;
       uint32_t Inst = *Instruction & ~InstMask;
@@ -662,7 +673,12 @@ public:
     case ForwardLabel::InstType::ADRP: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(IsADRPRange(Imm) && IsADRPAligned(Imm), "Unscaled offset too large");
+
+      if (!(IsADRPRange(Imm) && IsADRPAligned(Imm))) [[unlikely]] {
+        // Can't bind.
+        return false;
+      }
+
       Imm >>= 12;
       uint32_t InstMask = 0b11 << 29 | 0b1111'1111'1111'1111'111 << 5;
       uint32_t Offset = static_cast<uint32_t>(Imm) & 0x3F'FFFF;
@@ -672,11 +688,13 @@ public:
       *Instruction = Inst;
       break;
     }
-
     case ForwardLabel::InstType::B: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large");
+      if (!(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0))) [[unlikely]] {
+        // Can't bind.
+        return false;
+      }
       Imm >>= 2;
       uint32_t InstMask = 0x3FF'FFFF;
       uint32_t Offset = static_cast<uint32_t>(Imm) & InstMask;
@@ -686,11 +704,13 @@ public:
 
       break;
     }
-
     case ForwardLabel::InstType::TEST_BRANCH: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large");
+      if (!(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0))) [[unlikely]] {
+        // Can't bind.
+        return false;
+      }
       Imm >>= 2;
       uint32_t InstMask = 0x3FFF;
       uint32_t Offset = static_cast<uint32_t>(Imm) & InstMask;
@@ -704,7 +724,10 @@ public:
     case ForwardLabel::InstType::RELATIVE_LOAD: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
-      LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
+      if (!(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0))) [[unlikely]] {
+        // Can't bind.
+        return false;
+      }
       Imm >>= 2;
       uint32_t InstMask = 0x7'FFFF;
       uint32_t Offset = static_cast<uint32_t>(Imm) & InstMask;
@@ -753,27 +776,41 @@ public:
     }
     default: LOGMAN_MSG_A_FMT("Unexpected inst type in label fixup");
     }
+
+    return true;
   }
 
   // Bind a forward label to a location.
   // This walks all the instructions in the label's vector.
   // Then backpatching all instructions that have used the label.
-  void Bind(ForwardLabel* Label) {
+  [[nodiscard]] bool Bind(ForwardLabel* Label) {
+    bool Bound = true;
     if (Label->FirstInst.Location) {
-      Bind(&Label->FirstInst);
+      Bound &= Bind(&Label->FirstInst);
     }
     for (auto& Inst : Label->Insts) {
-      Bind(&Inst);
+      Bound &= Bind(&Inst);
     }
+
+    return Bound;
   }
 
   // Bind a bidirectional location to a location.
   // Binds both forwards and backwards depending on how the label was used.
-  void Bind(BiDirectionalLabel* Label) {
+  [[nodiscard]] bool Bind(BiDirectionalLabel* Label) {
+    bool Bound = true;
     if (!Label->Backward.Location) {
-      Bind(&Label->Backward);
+      Bound &= Bind(&Label->Backward);
     }
-    Bind(&Label->Forward);
+    Bound &= Bind(&Label->Forward);
+
+    return Bound;
+  }
+
+  static constexpr Condition InvertCondition(Condition cond) {
+    // These behave as always, so it makes no sense to allow inverting these.
+    LOGMAN_THROW_A_FMT(cond != Condition::CC_AL && cond != Condition::CC_NV, "Cannot invert CC_AL or CC_NV");
+    return static_cast<Condition>(FEXCore::ToUnderlying(cond) ^ 1);
   }
 
 #include <CodeEmitter/VixlUtils.inl>

FEXCore/Source/CMakeLists.txt

@@ -66,6 +66,7 @@ set (SRCS
   Interface/IR/Passes/RedundantFlagCalculationElimination.cpp
   Interface/IR/Passes/RegisterAllocationPass.cpp
   Interface/IR/Passes/x87StackOptimizationPass.cpp
+  Utils/LongJump.cpp
   Utils/Telemetry.cpp
   Utils/Threads.cpp
   Utils/Profiler.cpp

FEXCore/Source/Common/VectorRegType.h

@@ -4,6 +4,8 @@
 #ifdef _M_X86_64
 #include <xmmintrin.h>
 #include <immintrin.h>
+#else
+#include <cstdint>
 #endif
 
 namespace FEXCore {

FEXCore/Source/Interface/Config/Config.json.in

@@ -327,6 +327,13 @@
           "Enables FEX's low-overhead sampling profile statistics.",
           "Requires a supported version of Mangohud to see the results"
         ]
+      },
+      "TraceProfiler": {
+        "Type": "bool",
+        "Default": "false",
+        "Desc": [
+          "Enables FEX's trace profiler. Using gpuvis or tracy"
+        ]
       }
     },
     "Hacks": {

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: MIT
 
-#include "Common/SoftFloat.h"
+#include "Common/VectorRegType.h"
 #include "Interface/Context/Context.h"
 #include "Interface/Core/CPUBackend.h"
 #include "Interface/Core/Dispatcher/Dispatcher.h"
@@ -26,9 +26,7 @@
 #endif
 
 #include <array>
-#include <atomic>
 #include <bit>
-#include <condition_variable>
 #include <csignal>
 #include <cstring>
 
@@ -93,7 +91,7 @@ void Dispatcher::EmitDispatcher() {
 
   FillStaticRegs();
   ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));
-  cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);
+  (void)cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);
 
   ARMEmitter::BiDirectionalLabel LoopTop {};
 
@@ -142,7 +140,7 @@ void Dispatcher::EmitDispatcher() {
   // We want to ensure that we are 16 byte aligned at the top of this loop
   Align16B();
 
-  Bind(&LoopTop);
+  (void)Bind(&LoopTop);
   AbsoluteLoopTopAddress = GetCursorAddress<uint64_t>();
 
   // Load in our RIP
@@ -169,11 +167,11 @@ void Dispatcher::EmitDispatcher() {
   ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionEC));
   br(TMP2);
 
-  Bind(&l_NotECCode);
+  (void)!Bind(&l_NotECCode);
 #endif
 
   ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
-  cbnz(ARMEmitter::Size::i32Bit, TMP1, &CompileSingleStep);
+  (void)cbnz(ARMEmitter::Size::i32Bit, TMP1, &CompileSingleStep);
 
   // This is the block cache lookup routine
   // It matches what is going on it LookupCache.h::FindBlock
@@ -198,7 +196,7 @@ void Dispatcher::EmitDispatcher() {
     ldr(TMP1, TMP1, TMP2, ARMEmitter::ExtendedType::LSL_64, 3);
 
     // If page pointer is zero then we have no block
-    cbz(ARMEmitter::Size::i64Bit, TMP1, &NoBlock);
+    (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &NoBlock);
 
     // Steal the page offset
     and_(ARMEmitter::Size::i64Bit, TMP2, TMP4, 0x0FFF);
@@ -213,10 +211,11 @@ void Dispatcher::EmitDispatcher() {
 
     // If the guest address doesn't match, Compile the block.
     sub(TMP2, TMP2, RipReg);
-    cbnz(ARMEmitter::Size::i64Bit, TMP2, &NoBlock);
+    (void)cbnz(ARMEmitter::Size::i64Bit, TMP2, &NoBlock);
+
 
     // Check the host address to see if it matches, else compile the block.
-    cbz(ARMEmitter::Size::i64Bit, TMP4, &NoBlock);
+    (void)cbz(ARMEmitter::Size::i64Bit, TMP4, &NoBlock);
 
     // If we've made it here then we have a real compiled block
     {
@@ -304,7 +303,7 @@ void Dispatcher::EmitDispatcher() {
 
   // Need to create the block
   {
-    Bind(&NoBlock);
+    (void)Bind(&NoBlock);
 
     EmitSignalGuardedRegion([&]() {
       SpillStaticRegs(TMP1);
@@ -338,7 +337,7 @@ void Dispatcher::EmitDispatcher() {
   }
 
   {
-    Bind(&CompileSingleStep);
+    (void)Bind(&CompileSingleStep);
 
     EmitSignalGuardedRegion([&]() {
       SpillStaticRegs(TMP1);
@@ -500,7 +499,7 @@ void Dispatcher::EmitDispatcher() {
     stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
 
     // Now go back to the regular dispatcher loop
-    b(&LoopTop);
+    (void)b(&LoopTop);
   }
 
   auto EmitLongALUOpHandler = [&](auto R, auto Offset) {
@@ -569,14 +568,15 @@ void Dispatcher::EmitDispatcher() {
     }
   }
 
-  Bind(&l_CTX);
+  (void)Bind(&l_CTX);
   dc64(reinterpret_cast<uintptr_t>(CTX));
-  Bind(&l_Sleep);
+  (void)Bind(&l_Sleep);
   dc64(reinterpret_cast<uint64_t>(SleepThread));
-  Bind(&l_CompileBlock);
+  (void)Bind(&l_CompileBlock);
   FEXCore::Utils::MemberFunctionToPointerCast PMFCompileBlock(&FEXCore::Context::ContextImpl::CompileBlock);
   dc64(PMFCompileBlock.GetConvertedPointer());
-  Bind(&l_CompileSingleStep);
+  (void)Bind(&l_CompileSingleStep);
+
   FEXCore::Utils::MemberFunctionToPointerCast PMFCompileSingleStep(&FEXCore::Context::ContextImpl::CompileSingleStep);
   dc64(PMFCompileSingleStep.GetConvertedPointer());
 

FEXCore/Source/Interface/Core/JIT/ALUOps.cpp

@@ -588,7 +588,7 @@ DEF_OP(ShiftFlags) {
   and_(ARMEmitter::Size::i32Bit, TMP1, Src2, OpSize == IR::OpSize::i64Bit ? 0x3f : 0x1f);
 
   ARMEmitter::ForwardLabel Done;
-  cbz(EmitSize, TMP1, &Done);
+  (void)cbz(EmitSize, TMP1, &Done);
   {
     // PF/SF/ZF/OF
     if (OpSize >= IR::OpSize::i32Bit) {
@@ -652,7 +652,7 @@ DEF_OP(ShiftFlags) {
       msr(ARMEmitter::SystemRegister::NZCV, TMP2);
     }
   }
-  Bind(&Done);
+  (void)Bind(&Done);
 
   // TODO: Make RA less dumb so this can't happen (e.g. with late-kill).
   if (PFOutput != PFTemp) {
@@ -669,7 +669,7 @@ DEF_OP(RotateFlags) {
 
   // If shift=0, flags are unaffected. Wrap the whole implementation in a cbz.
   ARMEmitter::ForwardLabel Done;
-  cbz(EmitSize, Shift, &Done);
+  (void)cbz(EmitSize, Shift, &Done);
   {
     // Extract the last bit shifted in to CF
     const auto BitSize = IR::OpSizeToSize(Op->Size) * 8;
@@ -701,7 +701,7 @@ DEF_OP(RotateFlags) {
       msr(ARMEmitter::SystemRegister::NZCV, TMP3);
     }
   }
-  Bind(&Done);
+  (void)Bind(&Done);
 }
 
 DEF_OP(Extr) {
@@ -767,14 +767,14 @@ DEF_OP(PDep) {
     // Now, they're copied, so we can start setting Dest (even if it overlaps with
     // one of them).  Handle early exit case
     mov(EmitSize, Dest, 0);
-    cbz(EmitSize, OrigMask, &Done);
+    (void)cbz(EmitSize, OrigMask, &Done);
 
     // Setup for first iteration
     neg(EmitSize, T0, Mask);
     and_(EmitSize, T0, T0, Mask);
 
     // Main loop
-    Bind(&NextBit);
+    (void)Bind(&NextBit);
     sbfx(EmitSize, T1, Input, 0, 1);
     eor(EmitSize, Mask, Mask, T0);
     and_(EmitSize, T0, T1, T0);
@@ -782,10 +782,10 @@ DEF_OP(PDep) {
     orr(EmitSize, Dest, Dest, T0);
     lsr(EmitSize, Input, Input, 1);
     and_(EmitSize, T0, Mask, T1);
-    cbnz(EmitSize, T0, &NextBit);
+    (void)cbnz(EmitSize, T0, &NextBit);
 
     // All done with nothing to do.
-    Bind(&Done);
+    (void)Bind(&Done);
   }
 }
 
@@ -821,27 +821,27 @@ DEF_OP(PExt) {
     ARMEmitter::BackwardLabel NextBit;
     ARMEmitter::ForwardLabel Done;
 
-    cbz(EmitSize, Mask, &EarlyExit);
+    (void)cbz(EmitSize, Mask, &EarlyExit);
     mov(EmitSize, MaskReg, Mask);
     mov(EmitSize, ValueReg, Input);
     mov(EmitSize, Dest, ARMEmitter::Reg::zr);
 
     // Main loop
-    Bind(&NextBit);
-    cbz(EmitSize, MaskReg, &Done);
+    (void)Bind(&NextBit);
+    (void)cbz(EmitSize, MaskReg, &Done);
     clz(EmitSize, BitReg, MaskReg);
     lslv(EmitSize, ValueReg, ValueReg, BitReg);
     lslv(EmitSize, MaskReg, MaskReg, BitReg);
     extr(EmitSize, Dest, Dest, ValueReg, OpSizeBitsM1);
     bfc(EmitSize, MaskReg, OpSizeBitsM1, 1);
-    b(&NextBit);
+    (void)b(&NextBit);
 
     // Early exit
-    Bind(&EarlyExit);
+    (void)Bind(&EarlyExit);
     mov(EmitSize, Dest, ARMEmitter::Reg::zr);
 
     // All done with nothing to do.
-    Bind(&Done);
+    (void)Bind(&Done);
   }
 }
 
@@ -909,7 +909,7 @@ DEF_OP(Div) {
     eor(EmitSize, TMP1, TMP1, Upper);
 
     // If the sign bit matches then the result is zero
-    cbz(EmitSize, TMP1, &Only64Bit);
+    (void)cbz(EmitSize, TMP1, &Only64Bit);
 
     // Long divide
     {
@@ -928,17 +928,17 @@ DEF_OP(Div) {
       mov(EmitSize, Remainder, TMP2);
 
       // Skip 64-bit path
-      b(&LongDIVRet);
+      (void)b(&LongDIVRet);
     }
 
-    Bind(&Only64Bit);
+    (void)Bind(&Only64Bit);
     // 64-Bit only
     {
       sdiv(EmitSize, Quotient, Lower, Divisor);
       msub(EmitSize, Remainder, Quotient, Divisor, Lower);
     }
 
-    Bind(&LongDIVRet);
+    (void)Bind(&LongDIVRet);
     break;
   }
   default: LOGMAN_MSG_A_FMT("Unknown DIV Size: {}", OpSize); break;
@@ -992,7 +992,7 @@ DEF_OP(UDiv) {
 
     // Check the upper bits for zero
     // If the upper bits are zero then we can do a 64-bit divide
-    cbz(EmitSize, Upper, &Only64Bit);
+    (void)cbz(EmitSize, Upper, &Only64Bit);
 
     // Long divide
     {
@@ -1011,17 +1011,17 @@ DEF_OP(UDiv) {
       mov(EmitSize, Remainder, TMP2);
 
       // Skip 64-bit path
-      b(&LongDIVRet);
+      (void)b(&LongDIVRet);
     }
 
-    Bind(&Only64Bit);
+    (void)Bind(&Only64Bit);
     // 64-Bit only
     {
       udiv(EmitSize, Quotient, Lower, Divisor);
       msub(EmitSize, Remainder, Quotient, Divisor, Lower);
     }
 
-    Bind(&LongDIVRet);
+    (void)Bind(&LongDIVRet);
     break;
   }
   default: LOGMAN_MSG_A_FMT("Unknown LUDIV Size: {}", OpSize); break;

FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp

@@ -63,7 +63,7 @@ void Arm64JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair& Lit) {
   auto CurrentCursor = GetCursorAddress<uint8_t*>();
   Lit.MoveABI.NamedSymbolLiteral.Offset = CurrentCursor - CodeData.BlockBegin;
 
-  Bind(&Lit.Loc);
+  BindOrRestart(&Lit.Loc);
   dc64(Lit.Lit);
   Relocations.emplace_back(Lit.MoveABI);
 }

FEXCore/Source/Interface/Core/JIT/AtomicOps.cpp

@@ -62,27 +62,27 @@ DEF_OP(CASPair) {
     ARMEmitter::BackwardLabel LoopTop;
     ARMEmitter::ForwardLabel LoopNotExpected;
     ARMEmitter::ForwardLabel LoopExpected;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
 
     // This instruction sequence must be synced with HandleCASPAL_Armv8.
     ldaxp(EmitSize, TMP2, TMP3, MemSrc);
     cmp(EmitSize, TMP2, Expected0);
     ccmp(EmitSize, TMP3, Expected1, ARMEmitter::StatusFlags::None, ARMEmitter::Condition::CC_EQ);
-    b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
+    (void)b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
     stlxp(EmitSize, TMP2, Desired0, Desired1, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
+    (void)cbnz(EmitSize, TMP2, &LoopTop);
     mov(EmitSize, Dst0, Expected0);
     mov(EmitSize, Dst1, Expected1);
 
-    b(&LoopExpected);
+    (void)b(&LoopExpected);
 
-    Bind(&LoopNotExpected);
+    (void)Bind(&LoopNotExpected);
     mov(EmitSize, Dst0, TMP2.R());
     mov(EmitSize, Dst1, TMP3.R());
     // exclusive monitor needs to be cleared here
     // Might have hit the case where ldaxr was hit but stlxr wasn't
     clrex();
-    Bind(&LoopExpected);
+    (void)Bind(&LoopExpected);
 
     // Restore
     msr(ARMEmitter::SystemRegister::NZCV, TMP1);
@@ -114,7 +114,7 @@ DEF_OP(CAS) {
     ARMEmitter::BackwardLabel LoopTop;
     ARMEmitter::ForwardLabel LoopNotExpected;
     ARMEmitter::ForwardLabel LoopExpected;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     if (IROp->Size == IR::OpSize::i8Bit) {
       cmp(EmitSize, TMP2, Expected, ARMEmitter::ExtendedType::UXTB, 0);
@@ -123,18 +123,18 @@ DEF_OP(CAS) {
     } else {
       cmp(EmitSize, TMP2, Expected);
     }
-    b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
+    (void)b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
     stlxr(SubEmitSize, TMP3, Desired, MemSrc);
-    cbnz(EmitSize, TMP3, &LoopTop);
+    (void)cbnz(EmitSize, TMP3, &LoopTop);
     mov(EmitSize, Dst, Expected);
-    b(&LoopExpected);
+    (void)b(&LoopExpected);
 
-    Bind(&LoopNotExpected);
+    (void)Bind(&LoopNotExpected);
     mov(EmitSize, Dst, TMP2.R());
     // exclusive monitor needs to be cleared here
     // Might have hit the case where ldaxr was hit but stlxr wasn't
     clrex();
-    Bind(&LoopExpected);
+    (void)Bind(&LoopExpected);
   }
 }
 
@@ -150,11 +150,11 @@ DEF_OP(AtomicXor) {
     steorl(SubEmitSize, Src, MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     eor(EmitSize, TMP2, TMP2, Src);
     stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
-    cbnz(EmitSize, TMP2, &LoopTop);
+    (void)cbnz(EmitSize, TMP2, &LoopTop);
   }
 }
 
@@ -179,10 +179,10 @@ DEF_OP(AtomicSwap) {
     ldswpal(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     stlxr(SubEmitSize, TMP4, Src, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     ubfm(EmitSize, GetReg(Node), TMP2, 0, IR::OpSizeAsBits(OpSize) - 1);
   }
 }
@@ -199,11 +199,11 @@ DEF_OP(AtomicFetchAdd) {
     ldaddal(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     add(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -221,11 +221,11 @@ DEF_OP(AtomicFetchSub) {
     ldaddal(SubEmitSize, TMP2, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     sub(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -243,11 +243,11 @@ DEF_OP(AtomicFetchAnd) {
     ldclral(SubEmitSize, TMP2, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     and_(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -264,11 +264,11 @@ DEF_OP(AtomicFetchCLR) {
     ldclral(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     bic(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -285,11 +285,11 @@ DEF_OP(AtomicFetchOr) {
     ldsetal(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     orr(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -306,11 +306,11 @@ DEF_OP(AtomicFetchXor) {
     ldeoral(SubEmitSize, Src, GetReg(Node), MemSrc);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     eor(EmitSize, TMP3, TMP2, Src);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -326,20 +326,20 @@ DEF_OP(AtomicFetchNeg) {
     // Use a CAS loop to avoid needing to emulate unaligned LLSC atomics
     ldr(SubEmitSize, TMP2, MemSrc);
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     mov(EmitSize, TMP4, TMP2);
     neg(EmitSize, TMP3, TMP2);
     casal(SubEmitSize, TMP2, TMP3, MemSrc);
     sub(EmitSize, TMP3, TMP2, TMP4);
-    cbnz(EmitSize, TMP3, &LoopTop);
+    (void)cbnz(EmitSize, TMP3, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     neg(EmitSize, TMP3, TMP2);
     stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
-    cbnz(EmitSize, TMP4, &LoopTop);
+    (void)cbnz(EmitSize, TMP4, &LoopTop);
     mov(EmitSize, GetReg(Node), TMP2.R());
   }
 }
@@ -359,11 +359,11 @@ DEF_OP(TelemetrySetValue) {
     stsetl(ARMEmitter::SubRegSize::i64Bit, TMP1, TMP2);
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    Bind(&LoopTop);
+    (void)Bind(&LoopTop);
     ldaxr(ARMEmitter::SubRegSize::i64Bit, TMP3, TMP2);
     orr(ARMEmitter::Size::i32Bit, TMP3, TMP3, Src);
     stlxr(ARMEmitter::SubRegSize::i64Bit, TMP3, TMP3, TMP2);
-    cbnz(ARMEmitter::Size::i32Bit, TMP3, &LoopTop);
+    (void)cbnz(ARMEmitter::Size::i32Bit, TMP3, &LoopTop);
   }
 #endif
 }

FEXCore/Source/Interface/Core/JIT/BranchOps.cpp

@@ -141,7 +141,7 @@ DEF_OP(ExitFunction) {
         if (!Op->CallReturnBlock.IsInvalid()) {
           auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();
           PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;
-          adr(TMP1, &l_CallReturn);
+          (void)adr(TMP1, &l_CallReturn);
           stp<ARMEmitter::IndexType::PRE>(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);
         } else {
           stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
@@ -149,16 +149,16 @@ DEF_OP(ExitFunction) {
       } else if (Op->Hint == IR::BranchHint::CheckTF) {
         ARMEmitter::ForwardLabel TFUnset;
         ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
-        cbz(ARMEmitter::Size::i32Bit, TMP1, &TFUnset);
+        (void)cbz(ARMEmitter::Size::i32Bit, TMP1, &TFUnset);
         LoadConstant(ARMEmitter::Size::i64Bit, TMP1, NewRIP);
         str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
         ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop));
         blr(TMP2);
-        Bind(&TFUnset);
+        (void)Bind(&TFUnset);
       }
 
       EmitLinkedBranch(NewRIP, Op->Hint == IR::BranchHint::Call);
-      Bind(&l_CallReturn);
+      (void)Bind(&l_CallReturn);
 #ifdef _M_ARM_64EC
     }
 #endif
@@ -170,7 +170,7 @@ DEF_OP(ExitFunction) {
       // First try to pop from the call-ret stack, otherwise follow the normal path (but ending in a ret)
       ldp<ARMEmitter::IndexType::POST>(TMP1, TMP2, REG_CALLRET_SP, 0x10);
       sub(TMP1, TMP1, RipReg.X());
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
     }
 
     // L1 Cache
@@ -187,23 +187,23 @@ DEF_OP(ExitFunction) {
 
     // Note: sub+cbnz used over cmp+br to preserve flags.
     sub(TMP1, TMP1, RipReg.X());
-    cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
+    (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
     ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop));
     str(RipReg.X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
 
-    Bind(&SkipFullLookup);
+    (void)Bind(&SkipFullLookup);
     if (Op->Hint == IR::BranchHint::Call) {
       ARMEmitter::ForwardLabel l_CallReturn;
       if (!Op->CallReturnBlock.IsInvalid()) {
         auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();
         PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;
-        adr(TMP1, &l_CallReturn);
+        (void)adr(TMP1, &l_CallReturn);
         stp<ARMEmitter::IndexType::PRE>(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);
       } else {
         stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
       }
       blr(TMP2);
-      Bind(&l_CallReturn);
+      (void)Bind(&l_CallReturn);
     } else if (Op->Hint == IR::BranchHint::Return) {
       ret(TMP2);
     } else {
@@ -224,7 +224,7 @@ DEF_OP(CondJump) {
   auto TrueTargetLabel = JumpTarget(Op->TrueBlock);
 
   if (Op->FromNZCV) {
-    b(MapCC(Op->Cond), TrueTargetLabel);
+    b_OrRestart(MapCC(Op->Cond), TrueTargetLabel);
   } else {
     uint64_t Const;
     const bool isConst = IsInlineConstant(Op->Cmp2, &Const);
@@ -237,16 +237,16 @@ DEF_OP(CondJump) {
 
     if (Op->Cond.Val == FEXCore::IR::COND_EQ) {
       LOGMAN_THROW_A_FMT(Const == 0, "CondJump: Expected 0 source");
-      cbz(Size, Reg, TrueTargetLabel);
+      cbz_OrRestart(Size, Reg, TrueTargetLabel);
     } else if (Op->Cond.Val == FEXCore::IR::COND_NEQ) {
       LOGMAN_THROW_A_FMT(Const == 0, "CondJump: Expected 0 source");
-      cbnz(Size, Reg, TrueTargetLabel);
+      cbnz_OrRestart(Size, Reg, TrueTargetLabel);
     } else if (Op->Cond.Val == FEXCore::IR::COND_TSTZ) {
       LOGMAN_THROW_A_FMT(Const < 64, "CondJump: Expected valid bit source");
-      tbz(Reg, Const, TrueTargetLabel);
+      tbz_OrRestart(Reg, Const, TrueTargetLabel);
     } else if (Op->Cond.Val == FEXCore::IR::COND_TSTNZ) {
       LOGMAN_THROW_A_FMT(Const < 64, "CondJump: Expected valid bit source");
-      tbnz(Reg, Const, TrueTargetLabel);
+      tbnz_OrRestart(Reg, Const, TrueTargetLabel);
     } else {
       LOGMAN_THROW_A_FMT(false, "CondJump expected simple condition");
     }
@@ -458,7 +458,7 @@ DEF_OP(ValidateCode) {
     while (len >= Size) {
       LoadData();
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP2);
-      cbnz(ARMEmitter::Size::i64Bit, TMP1, &Fail);
+      cbnz_OrRestart(ARMEmitter::Size::i64Bit, TMP1, &Fail);
       len -= Size;
       Offset += Size;
     }
@@ -486,10 +486,10 @@ DEF_OP(ValidateCode) {
 
   ARMEmitter::ForwardLabel End;
   LoadConstant(ARMEmitter::Size::i32Bit, Dst, 0);
-  b(&End);
-  Bind(&Fail);
+  b_OrRestart(&End);
+  BindOrRestart(&Fail);
   LoadConstant(ARMEmitter::Size::i32Bit, Dst, 1);
-  Bind(&End);
+  BindOrRestart(&End);
 }
 
 DEF_OP(ThreadRemoveCodeEntry) {

FEXCore/Source/Interface/Core/JIT/JIT.cpp

@@ -11,8 +11,6 @@ desc: Main glue logic of the arm64 splatter backend
 $end_info$
 */
 
-#include "Common/SoftFloat.h"
-
 #include "Interface/Context/Context.h"
 #include "Interface/Core/LookupCache.h"
 #include "Interface/Core/Dispatcher/Dispatcher.h"
@@ -30,6 +28,7 @@ $end_info$
 #include <FEXCore/Utils/CompilerDefs.h>
 #include <FEXCore/Utils/EnumUtils.h>
 #include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/LongJump.h>
 #include <FEXCore/Utils/Profiler.h>
 #include <FEXCore/Utils/Telemetry.h>
 #include <FEXCore/Utils/TypeDefines.h>
@@ -37,7 +36,6 @@ $end_info$
 
 #include <cstdio>
 #include <cstring>
-#include <limits>
 #include <unistd.h>
 
 namespace {
@@ -669,15 +667,6 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::In
 
   CurrentCodeBuffer = CodeBuffers.GetLatest();
   ThreadState->LookupCache->Shared = CurrentCodeBuffer->LookupCache.get();
-
-  // Setup dynamic dispatch.
-  if (ParanoidTSO()) {
-    RT_LoadMemTSO = &Arm64JITCore::Op_ParanoidLoadMemTSO;
-    RT_StoreMemTSO = &Arm64JITCore::Op_ParanoidStoreMemTSO;
-  } else {
-    RT_LoadMemTSO = &Arm64JITCore::Op_LoadMemTSO;
-    RT_StoreMemTSO = &Arm64JITCore::Op_StoreMemTSO;
-  }
 }
 
 void Arm64JITCore::EmitDetectionString() {
@@ -748,11 +737,11 @@ void Arm64JITCore::EmitTFCheck() {
   // Note that this needs to be before the below suspend checks, as X86 checks this flag immediately after executing an instruction.
   ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
 
-  cbz(ARMEmitter::Size::i32Bit, TMP1, &l_TFUnset);
+  (void)cbz(ARMEmitter::Size::i32Bit, TMP1, &l_TFUnset);
 
   // X86 semantically checks TF after executing each instruction, so e.g. setting a context with TF set will execute a single instruction
   // and then raise an exception. However on the FEX side this is simpler to implement by checking at the start of each instruction, handle this by having bit 1 being unset in the flag state indicate that TF is blocked for a single instruction.
-  tbz(TMP1, 1, &l_TFBlocked);
+  (void)tbz(TMP1, 1, &l_TFBlocked);
 
   // Block TF for a single instruction when the frontend jumps to a new context by unsetting bit 1.
   ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
@@ -775,11 +764,11 @@ void Arm64JITCore::EmitTFCheck() {
   ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP));
   br(TMP1);
 
-  Bind(&l_TFBlocked);
+  (void)Bind(&l_TFBlocked);
   // If TF was blocked for this instruction, unblock it for the next.
   LoadConstant(ARMEmitter::Size::i32Bit, TMP1, 0b11);
   strb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
-  Bind(&l_TFUnset);
+  (void)Bind(&l_TFUnset);
 }
 
 void Arm64JITCore::EmitSuspendInterruptCheck() {
@@ -797,14 +786,14 @@ void Arm64JITCore::EmitSuspendInterruptCheck() {
   ARMEmitter::ForwardLabel l_NoSuspend;
   cbz(ARMEmitter::Size::i32Bit, TMP2, &l_NoSuspend);
   brk(SuspendMagic);
-  Bind(&l_NoSuspend);
+  (void)Bind(&l_NoSuspend);
 #endif
 }
 
 void Arm64JITCore::EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF) {
   // Get the address of the JITCodeHeader and store in to the core state.
   // Two instruction cost, each 1 cycle.
-  adr(TMP1, &HeaderLabel);
+  adr_OrRestart(TMP1, &HeaderLabel);
   str(TMP1, STATE, offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader));
 
   if (CheckTF) {
@@ -826,16 +815,25 @@ void Arm64JITCore::EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool C
 CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
                                                    FEXCore::Core::DebugData* DebugData, bool CheckTF) {
   FEXCORE_PROFILE_SCOPED("Arm64::CompileCode");
-
-  JumpTargets.clear();
-  CallReturnTargets.clear();
-  PendingJumpThunks.clear();
-  uint32_t SSACount = IR->GetSSACount();
-  JumpTargets.resize(IR->GetHeader()->BlockCount, {});
-
   this->Entry = Entry;
   this->DebugData = DebugData;
   this->IR = IR;
+  RequiresFarARM64Jumps = false;
+
+  switch (static_cast<RestartOptions::Control>(FEXCore::LongJump::SetJump(RestartControl.RestartJump))) {
+  case RestartOptions::Control::Incoming:
+    // Nothing
+    break;
+  case RestartOptions::Control::EnableFarARM64Jumps: RequiresFarARM64Jumps = true; break;
+  default: ERROR_AND_DIE_FMT("Unhandled Arm64 restart condition!");
+  }
+
+  uint32_t SSACount = IR->GetSSACount();
+  JumpTargets.clear();
+  CallReturnTargets.clear();
+  PendingJumpThunks.clear();
+  JumpTargets.resize(IR->GetHeader()->BlockCount, {});
+
   CodeData.EntryPoints.clear();
 
   // Fairly excessive buffer range to make sure we don't overflow
@@ -850,7 +848,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
 
   // Put the code header at the start of the data block.
   ARMEmitter::BackwardLabel JITCodeHeaderLabel {};
-  Bind(&JITCodeHeaderLabel);
+  (void)Bind(&JITCodeHeaderLabel);
   JITCodeHeader* CodeHeader = GetCursorAddress<JITCodeHeader*>();
   CursorIncrement(sizeof(JITCodeHeader));
 
@@ -898,7 +896,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
         if (PendingTargetLabel->Backward.Location) {
           EmitSuspendInterruptCheck();
         }
-        b(PendingTargetLabel);
+        b_OrRestart(PendingTargetLabel);
         PendingTargetLabel = nullptr;
       }
 
@@ -908,14 +906,14 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
         const auto IsReturnTarget = CallReturnTargets.try_emplace(Node).first;
         if (PendingTargetLabel) {
           // If there is a fallthrough branch to this block, skip over the entrypoint code.
-          b(Target);
+          b_OrRestart(Target);
         } else if (PendingCallReturnTargetLabel && PendingCallReturnTargetLabel != &IsReturnTarget->second) {
           // If we just emitted a call, but the block we're now emitting is not the return block so don't fallthrough.
-          b(PendingCallReturnTargetLabel);
+          b_OrRestart(PendingCallReturnTargetLabel);
         }
         PendingCallReturnTargetLabel = nullptr;
 
-        Bind(&IsReturnTarget->second);
+        BindOrRestart(&IsReturnTarget->second);
         CodeData.EntryPoints.emplace(BlockStartRIP, GetCursorAddress<uint8_t*>());
         DebugData->GuestOpcodes.push_back({BlockIROp->GuestEntryOffset, GetCursorAddress<uint8_t*>() - CodeData.BlockBegin});
 
@@ -924,12 +922,12 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
 
       if (PendingCallReturnTargetLabel) {
         // If there is still a pending call return target, then the block we're emitting is not the return block so don't fallthrough.
-        b(PendingCallReturnTargetLabel);
+        b_OrRestart(PendingCallReturnTargetLabel);
         PendingCallReturnTargetLabel = nullptr;
       }
       PendingTargetLabel = nullptr;
 
-      Bind(Target);
+      BindOrRestart(Target);
     }
 
     for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {
@@ -956,7 +954,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
     if (PendingTargetLabel->Backward.Location) {
       EmitSuspendInterruptCheck();
     }
-    b(PendingTargetLabel);
+    b_OrRestart(PendingTargetLabel);
   }
   PendingTargetLabel = nullptr;
 
@@ -967,21 +965,21 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
 
     ARMEmitter::ForwardLabel l_DoLink;
     uint64_t ThunkAddress = GetCursorAddress<uint64_t>();
-    Bind(&PendingJumpThunk.Label);
-    b(&l_DoLink);
+    BindOrRestart(&PendingJumpThunk.Label);
+    b_OrRestart(&l_DoLink);
     br(TMP1);
-    Bind(&l_DoLink);
+    BindOrRestart(&l_DoLink);
     ldr(TMP1, &l_ExitLink);
     blr(TMP1);
 
     // This is a ExitFunctionLinkData struct
-    Bind(&l_ExitLink);
+    BindOrRestart(&l_ExitLink);
     dc64(0);                                             // HostCode
     dc64(PendingJumpThunk.GuestRIP);                     // GuestRIP
     dc64(PendingJumpThunk.CallerAddress - ThunkAddress); // CallerOffset
   }
 
-  Bind(&l_ExitLink);
+  BindOrRestart(&l_ExitLink);
   dc64(ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker);
 
   // CodeSize not including the header or tail data.

FEXCore/Source/Interface/Core/JIT/JITClass.h

@@ -19,6 +19,7 @@ $end_info$
 #include <FEXCore/fextl/map.h>
 #include <FEXCore/fextl/string.h>
 #include <FEXCore/fextl/vector.h>
+#include <FEXCore/Utils/LongJump.h>
 
 #include <CodeEmitter/Emitter.h>
 
@@ -54,6 +55,7 @@ public:
 
 private:
   FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
+  FEX_CONFIG_OPT(HalfBarrierTSOEnabled, HALFBARRIERTSOENABLED);
 
   const bool HostSupportsSVE128 {};
   const bool HostSupportsSVE256 {};
@@ -61,6 +63,19 @@ private:
   const bool HostSupportsRPRES {};
   const bool HostSupportsAFP {};
 
+  struct RestartOptions {
+    FEXCore::LongJump::JumpBuf RestartJump;
+    enum class Control : uint64_t {
+      Incoming = 0,
+      EnableFarARM64Jumps = 1,
+    };
+  };
+
+  // FEXCore makes assumptions in the JIT about certain conditions being true.
+  // In the rare case when those assumptions are broken, FEX needs to safely restart the JIT.
+  RestartOptions RestartControl {};
+  bool RequiresFarARM64Jumps {};
+
   ARMEmitter::BiDirectionalLabel* PendingTargetLabel {};
   ARMEmitter::BiDirectionalLabel* PendingCallReturnTargetLabel {};
   FEXCore::Context::ContextImpl* CTX {};
@@ -315,14 +330,199 @@ private:
   void EmitLinkedBranch(uint64_t GuestRIP, bool Call) {
     PendingJumpThunks.push_back({GetCursorAddress<uint64_t>(), GuestRIP, {}});
     auto& Thunk = PendingJumpThunks.back();
-    Bind(&Thunk.Label);
+    BindOrRestart(&Thunk.Label);
     if (Call) {
-      bl(&Thunk.Label);
+      bl_OrRestart(&Thunk.Label);
     } else {
-      b(&Thunk.Label);
+      b_OrRestart(&Thunk.Label);
     }
   }
 
+  // Restart helpers
+  template<typename T>
+  requires (std::is_same_v<T, ARMEmitter::ForwardLabel> || std::is_same_v<T, ARMEmitter::BackwardLabel> ||
+            std::is_same_v<T, ARMEmitter::BiDirectionalLabel> || std::is_same_v<T, ARMEmitter::ForwardLabel::Reference>)
+  void bl_OrRestart(T* Label) {
+    if (bl(Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    // We can support this but currently unnecessary.
+    ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<typename T>
+  requires (std::is_same_v<T, ARMEmitter::ForwardLabel> || std::is_same_v<T, ARMEmitter::BackwardLabel> ||
+            std::is_same_v<T, ARMEmitter::BiDirectionalLabel> || std::is_same_v<T, ARMEmitter::ForwardLabel::Reference>)
+  void b_OrRestart(T* Label) {
+    if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    // We can support this but currently unnecessary.
+    ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<typename T>
+  requires (std::is_same_v<T, ARMEmitter::ForwardLabel> || std::is_same_v<T, ARMEmitter::BackwardLabel> ||
+            std::is_same_v<T, ARMEmitter::BiDirectionalLabel> || std::is_same_v<T, ARMEmitter::ForwardLabel::Reference>)
+  void b_OrRestart(ARMEmitter::Condition Cond, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)b(InvertCondition(Cond), &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (b(Cond, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<typename T>
+  requires (std::is_same_v<T, ARMEmitter::ForwardLabel> || std::is_same_v<T, ARMEmitter::BackwardLabel> ||
+            std::is_same_v<T, ARMEmitter::BiDirectionalLabel> || std::is_same_v<T, ARMEmitter::ForwardLabel::Reference>)
+  void cbz_OrRestart(ARMEmitter::Size s, ARMEmitter::Register rt, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)cbnz(s, rt, &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (cbz(s, rt, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<typename T>
+  requires (std::is_same_v<T, ARMEmitter::ForwardLabel> || std::is_same_v<T, ARMEmitter::BackwardLabel> ||
+            std::is_same_v<T, ARMEmitter::BiDirectionalLabel> || std::is_same_v<T, ARMEmitter::ForwardLabel::Reference>)
+  void cbnz_OrRestart(ARMEmitter::Size s, ARMEmitter::Register rt, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)cbz(s, rt, &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (cbnz(s, rt, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<typename T>
+  requires (std::is_same_v<T, ARMEmitter::ForwardLabel> || std::is_same_v<T, ARMEmitter::BackwardLabel> ||
+            std::is_same_v<T, ARMEmitter::BiDirectionalLabel> || std::is_same_v<T, ARMEmitter::ForwardLabel::Reference>)
+  void tbz_OrRestart(ARMEmitter::Register rt, uint32_t Bit, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)tbnz(rt, Bit, &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (tbz(rt, Bit, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<typename T>
+  requires (std::is_same_v<T, ARMEmitter::ForwardLabel> || std::is_same_v<T, ARMEmitter::BackwardLabel> ||
+            std::is_same_v<T, ARMEmitter::BiDirectionalLabel> || std::is_same_v<T, ARMEmitter::ForwardLabel::Reference>)
+  void tbnz_OrRestart(ARMEmitter::Register rt, uint32_t Bit, T* Label) {
+    if (RequiresFarARM64Jumps) {
+      ARMEmitter::ForwardLabel Skip {};
+      // Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
+      (void)tbz(rt, Bit, &Skip);
+      if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
+        ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
+      }
+
+      (void)Bind(&Skip);
+      return;
+    }
+
+    if (tbnz(rt, Bit, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<typename T>
+  requires (std::is_same_v<T, ARMEmitter::ForwardLabel> || std::is_same_v<T, ARMEmitter::BackwardLabel> ||
+            std::is_same_v<T, ARMEmitter::BiDirectionalLabel> || std::is_same_v<T, ARMEmitter::ForwardLabel::Reference>)
+  void adr_OrRestart(ARMEmitter::Register rd, T* Label) {
+    if (adr(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    // We can support this but currently unnecessary.
+    ERROR_AND_DIE_FMT("Long ADR currently unsupported!");
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<typename T>
+  requires (std::is_same_v<T, ARMEmitter::ForwardLabel> || std::is_same_v<T, ARMEmitter::BackwardLabel> ||
+            std::is_same_v<T, ARMEmitter::BiDirectionalLabel> || std::is_same_v<T, ARMEmitter::ForwardLabel::Reference>)
+  void adrp_OrRestart(ARMEmitter::Register rd, T* Label) {
+    if (adrp(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
+      return;
+    }
+
+    // We can support this but currently unnecessary.
+    ERROR_AND_DIE_FMT("Long ADRP currently unsupported!");
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
+  template<typename T>
+  requires (std::is_same_v<T, ARMEmitter::ForwardLabel> || std::is_same_v<T, ARMEmitter::BackwardLabel> ||
+            std::is_same_v<T, ARMEmitter::BiDirectionalLabel> || std::is_same_v<T, ARMEmitter::ForwardLabel::Reference>)
+  void BindOrRestart(T* Label) {
+    if (Bind(Label)) {
+      return;
+    }
+
+    if (RequiresFarARM64Jumps) {
+      // This should have been caught before this point.
+      ERROR_AND_DIE_FMT("Oops. Unhandled long bind.");
+      return;
+    }
+
+    FEXCore::LongJump::LongJump(RestartControl.RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
+  }
+
   // This is purely a debugging aid for developers to see if they are in JIT code space when inspecting raw memory
   void EmitDetectionString();
   IR::RegisterAllocationPass* RAPass {};
@@ -414,17 +614,8 @@ private:
 
   void EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF);
 
-  // Runtime selection;
-  // Load and store TSO memory style
-  OpType RT_LoadMemTSO;
-  OpType RT_StoreMemTSO;
-
 #define DEF_OP(x) void Op_##x(IR::IROp_Header const* IROp, IR::Ref Node)
 
-  // Dynamic Dispatcher supporting operations
-  DEF_OP(ParanoidLoadMemTSO);
-  DEF_OP(ParanoidStoreMemTSO);
-
   ///< Unhandled handler
   DEF_OP(Unhandled);
 

FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp

@@ -776,8 +776,10 @@ DEF_OP(LoadMemTSO) {
       case IR::OpSize::i64Bit: ldapur(Dst.X(), MemReg, Offset); break;
       default: LOGMAN_MSG_A_FMT("Unhandled LoadMemTSO size: {}", OpSize); break;
       }
-      // Half-barrier once back-patched.
-      nop();
+      if (HalfBarrierTSOEnabled() && !ParanoidTSO()) {
+        // Half-barrier once back-patched.
+        nop();
+      }
     }
   } else if (CTX->HostFeatures.SupportsRCPC && Op->Class == FEXCore::IR::GPRClass) {
     const auto Dst = GetReg(Node);
@@ -791,8 +793,10 @@ DEF_OP(LoadMemTSO) {
       case IR::OpSize::i64Bit: ldapr(Dst.X(), MemReg); break;
       default: LOGMAN_MSG_A_FMT("Unhandled LoadMemTSO size: {}", OpSize); break;
       }
-      // Half-barrier once back-patched.
-      nop();
+      if (HalfBarrierTSOEnabled() && !ParanoidTSO()) {
+        // Half-barrier once back-patched.
+        nop();
+      }
     }
   } else if (Op->Class == FEXCore::IR::GPRClass) {
     const auto Dst = GetReg(Node);
@@ -806,8 +810,10 @@ DEF_OP(LoadMemTSO) {
       case IR::OpSize::i64Bit: ldar(Dst.X(), MemReg); break;
       default: LOGMAN_MSG_A_FMT("Unhandled LoadMemTSO size: {}", OpSize); break;
       }
-      // Half-barrier once back-patched.
-      nop();
+      if (HalfBarrierTSOEnabled() && !ParanoidTSO()) {
+        // Half-barrier once back-patched.
+        nop();
+      }
     }
   } else {
     const auto Dst = GetVReg(Node);
@@ -911,7 +917,7 @@ DEF_OP(VLoadVectorMasked) {
 
       // If the sign bit is zero then skip the load
       ARMEmitter::ForwardLabel Skip {};
-      tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
+      (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
       // Do the gather load for this element into the destination
       switch (IROp->ElementSize) {
       case IR::OpSize::i8Bit: ld1<ARMEmitter::SubRegSize::i8Bit>(TempDst.Q(), i, TempMemReg); break;
@@ -922,7 +928,7 @@ DEF_OP(VLoadVectorMasked) {
       default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, IROp->ElementSize); return;
       }
 
-      Bind(&Skip);
+      (void)Bind(&Skip);
 
       if ((i + 1) != NumElements) {
         // Handle register rename to save a move.
@@ -1012,7 +1018,7 @@ DEF_OP(VStoreVectorMasked) {
 
       // If the sign bit is zero then skip the load
       ARMEmitter::ForwardLabel Skip {};
-      tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
+      (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
       // Do the gather load for this element into the destination
       switch (IROp->ElementSize) {
       case IR::OpSize::i8Bit: st1<ARMEmitter::SubRegSize::i8Bit>(RegData.Q(), i, TempMemReg); break;
@@ -1023,7 +1029,7 @@ DEF_OP(VStoreVectorMasked) {
       default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, IROp->ElementSize); return;
       }
 
-      Bind(&Skip);
+      (void)Bind(&Skip);
 
       if ((i + 1) != NumElements) {
         // Handle register rename to save a move.
@@ -1101,7 +1107,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
     PerformMove(ElementSize, WorkingReg, MaskReg, i);
 
     // Skip if the mask's sign bit isn't set
-    tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
+    (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
 
     // Extract Index Element
     if ((IndexElement * IR::OpSizeToSize(VectorIndexSize)) >= 16) {
@@ -1139,7 +1145,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
     default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, ElementSize); FEX_UNREACHABLE;
     }
 
-    Bind(&Skip);
+    (void)Bind(&Skip);
   }
 
   if (NeedsDestTmp) {
@@ -1777,8 +1783,10 @@ DEF_OP(StoreMemTSO) {
       // 8bit load is always aligned to natural alignment
       stlurb(Src, MemReg, Offset);
     } else {
-      // Half-barrier once back-patched.
-      nop();
+      if (HalfBarrierTSOEnabled() && !ParanoidTSO()) {
+        // Half-barrier once back-patched.
+        nop();
+      }
       switch (OpSize) {
       case IR::OpSize::i16Bit: stlurh(Src, MemReg, Offset); break;
       case IR::OpSize::i32Bit: stlur(Src.W(), MemReg, Offset); break;
@@ -1793,8 +1801,10 @@ DEF_OP(StoreMemTSO) {
       // 8bit load is always aligned to natural alignment
       stlrb(Src, MemReg);
     } else {
-      // Half-barrier once back-patched.
-      nop();
+      if (HalfBarrierTSOEnabled() && !ParanoidTSO()) {
+        // Half-barrier once back-patched.
+        nop();
+      }
       switch (OpSize) {
       case IR::OpSize::i16Bit: stlrh(Src, MemReg); break;
       case IR::OpSize::i32Bit: stlr(Src.W(), MemReg); break;
@@ -1870,7 +1880,7 @@ DEF_OP(MemSet) {
 
   if (!DirectionIsInline) {
     // Backward or forwards implementation depends on flag
-    tbnz(DirectionReg, 1, &BackwardImpl);
+    (void)tbnz(DirectionReg, 1, &BackwardImpl);
   }
 
   auto MemStore = [this](auto Value, uint32_t OpSize, int32_t Size) {
@@ -1888,7 +1898,9 @@ DEF_OP(MemSet) {
       // 8bit load is always aligned to natural alignment
       stlrb(Value.W(), TMP2);
     } else {
-      nop();
+      if (HalfBarrierTSOEnabled() && !ParanoidTSO()) {
+        nop();
+      }
       switch (OpSize) {
       case 2: stlrh(Value.W(), TMP2); break;
       case 4: stlr(Value.W(), TMP2); break;
@@ -1918,7 +1930,7 @@ DEF_OP(MemSet) {
     ARMEmitter::ForwardLabel DoneInternal {};
 
     // Early exit if zero count.
-    cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+    (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
     if (!IsAtomic) {
       ARMEmitter::ForwardLabel AgainInternal256Exit {};
@@ -1935,50 +1947,50 @@ DEF_OP(MemSet) {
       // Do this in two parts, to fallback to the byte by byte loop if size < 32, and to the
       // single copy loop if size < 64.
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal128Exit);
+      (void)tbnz(TMP1, 63, &AgainInternal128Exit);
 
       // Fill VTMP2 with the set pattern
       dup(SubRegSize, VTMP2.Q(), Value);
 
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal256Exit);
+      (void)tbnz(TMP1, 63, &AgainInternal256Exit);
 
-      Bind(&AgainInternal256);
+      (void)Bind(&AgainInternal256);
       stp<ARMEmitter::IndexType::POST>(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);
       stp<ARMEmitter::IndexType::POST>(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
-      tbz(TMP1, 63, &AgainInternal256);
+      (void)tbz(TMP1, 63, &AgainInternal256);
 
-      Bind(&AgainInternal256Exit);
+      (void)Bind(&AgainInternal256Exit);
       add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal128Exit);
-      Bind(&AgainInternal128);
+      (void)tbnz(TMP1, 63, &AgainInternal128Exit);
+      (void)Bind(&AgainInternal128);
       stp<ARMEmitter::IndexType::POST>(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbz(TMP1, 63, &AgainInternal128);
+      (void)tbz(TMP1, 63, &AgainInternal128);
 
-      Bind(&AgainInternal128Exit);
+      (void)Bind(&AgainInternal128Exit);
       add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
       if (Direction == -1) {
         add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);
       }
     }
 
-    Bind(&AgainInternal);
+    (void)Bind(&AgainInternal);
     if (IsAtomic) {
       MemStoreTSO(Value, OpSize, SizeDirection);
     } else {
       MemStore(Value, OpSize, SizeDirection);
     }
     sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 1);
-    cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
+    (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
 
-    Bind(&DoneInternal);
+    (void)Bind(&DoneInternal);
 
     if (SizeDirection >= 0) {
       switch (OpSize) {
@@ -2008,12 +2020,12 @@ DEF_OP(MemSet) {
       EmitMemset(Direction);
 
       if (Direction == 1) {
-        b(&Done);
-        Bind(&BackwardImpl);
+        (void)b(&Done);
+        (void)Bind(&BackwardImpl);
       }
     }
 
-    Bind(&Done);
+    (void)Bind(&Done);
     // Destination already set to the final pointer.
   }
 }
@@ -2063,7 +2075,7 @@ DEF_OP(MemCpy) {
 
   if (!DirectionIsInline) {
     // Backward or forwards implementation depends on flag
-    tbnz(DirectionReg, 1, &BackwardImpl);
+    (void)tbnz(DirectionReg, 1, &BackwardImpl);
   }
 
   auto MemCpy = [this](uint32_t OpSize, int32_t Size) {
@@ -2106,9 +2118,11 @@ DEF_OP(MemCpy) {
         default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size); break;
         }
 
-        // Placeholders for backpatching barriers (one per load/store)
-        nop();
-        nop();
+        if (HalfBarrierTSOEnabled() && !ParanoidTSO()) {
+          // Placeholders for backpatching barriers (one per load/store)
+          nop();
+          nop();
+        }
 
         switch (OpSize) {
         case 2: stlrh(TMP4.W(), TMP2); break;
@@ -2130,9 +2144,11 @@ DEF_OP(MemCpy) {
         default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size); break;
         }
 
-        // Placeholders for backpatching barriers (one per load/store)
-        nop();
-        nop();
+        if (HalfBarrierTSOEnabled() && !ParanoidTSO()) {
+          // Placeholders for backpatching barriers (one per load/store)
+          nop();
+          nop();
+        }
 
         switch (OpSize) {
         case 2: stlrh(TMP4.W(), TMP2); break;
@@ -2160,7 +2176,7 @@ DEF_OP(MemCpy) {
     ARMEmitter::ForwardLabel DoneInternal {};
 
     // Early exit if zero count.
-    cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+    (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
     if (!IsAtomic) {
       ARMEmitter::ForwardLabel AbsPos {};
@@ -2170,11 +2186,11 @@ DEF_OP(MemCpy) {
       ARMEmitter::BackwardLabel AgainInternal256 {};
 
       sub(ARMEmitter::Size::i64Bit, TMP4, TMP2, TMP3);
-      tbz(TMP4, 63, &AbsPos);
+      (void)tbz(TMP4, 63, &AbsPos);
       neg(ARMEmitter::Size::i64Bit, TMP4, TMP4);
-      Bind(&AbsPos);
+      (void)Bind(&AbsPos);
       sub(ARMEmitter::Size::i64Bit, TMP4, TMP4, 32);
-      tbnz(TMP4, 63, &AgainInternal);
+      (void)tbnz(TMP4, 63, &AgainInternal);
 
       if (Direction == -1) {
         sub(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);
@@ -2186,30 +2202,30 @@ DEF_OP(MemCpy) {
       // Do this in two parts, to fallback to the byte by byte loop if size < 32, and to the
       // single copy loop if size < 64.
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal128Exit);
+      (void)tbnz(TMP1, 63, &AgainInternal128Exit);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal256Exit);
+      (void)tbnz(TMP1, 63, &AgainInternal256Exit);
 
-      Bind(&AgainInternal256);
+      (void)Bind(&AgainInternal256);
       MemCpy(32, 32 * Direction);
       MemCpy(32, 32 * Direction);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
-      tbz(TMP1, 63, &AgainInternal256);
+      (void)tbz(TMP1, 63, &AgainInternal256);
 
-      Bind(&AgainInternal256Exit);
+      (void)Bind(&AgainInternal256Exit);
       add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbnz(TMP1, 63, &AgainInternal128Exit);
-      Bind(&AgainInternal128);
+      (void)tbnz(TMP1, 63, &AgainInternal128Exit);
+      (void)Bind(&AgainInternal128);
       MemCpy(32, 32 * Direction);
       sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      tbz(TMP1, 63, &AgainInternal128);
+      (void)tbz(TMP1, 63, &AgainInternal128);
 
-      Bind(&AgainInternal128Exit);
+      (void)Bind(&AgainInternal128Exit);
       add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
-      cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
+      (void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
 
       if (Direction == -1) {
         add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);
@@ -2217,16 +2233,16 @@ DEF_OP(MemCpy) {
       }
     }
 
-    Bind(&AgainInternal);
+    (void)Bind(&AgainInternal);
     if (IsAtomic) {
       MemCpyTSO(OpSize, SizeDirection);
     } else {
       MemCpy(OpSize, SizeDirection);
     }
     sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 1);
-    cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
+    (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
 
-    Bind(&DoneInternal);
+    (void)Bind(&DoneInternal);
 
     // Needs to use temporaries just in case of overwrite
     mov(TMP1, MemRegDest.X());
@@ -2284,186 +2300,15 @@ DEF_OP(MemCpy) {
     for (int32_t Direction : {1, -1}) {
       EmitMemcpy(Direction);
       if (Direction == 1) {
-        b(&Done);
-        Bind(&BackwardImpl);
+        (void)b(&Done);
+        (void)Bind(&BackwardImpl);
       }
     }
-    Bind(&Done);
+    (void)Bind(&Done);
     // Destination already set to the final pointer.
   }
 }
 
-DEF_OP(ParanoidLoadMemTSO) {
-  const auto Op = IROp->C<IR::IROp_LoadMemTSO>();
-  const auto OpSize = IROp->Size;
-
-  auto MemReg = GetReg(Op->Addr);
-
-  if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == FEXCore::IR::GPRClass) {
-    const auto Dst = GetReg(Node);
-    uint64_t Offset = 0;
-    if (!Op->Offset.IsInvalid()) {
-      if (!IsInlineConstant(Op->Offset, &Offset)) {
-        MemReg = ApplyMemOperand(OpSize, MemReg, TMP4, Op->Offset, Op->OffsetType, Op->OffsetScale);
-      }
-    }
-
-    if (OpSize == IR::OpSize::i8Bit) {
-      // 8bit load is always aligned to natural alignment
-      const auto Dst = GetReg(Node);
-      ldapurb(Dst, MemReg, Offset);
-    } else {
-      switch (OpSize) {
-      case IR::OpSize::i16Bit: ldapurh(Dst, MemReg, Offset); break;
-      case IR::OpSize::i32Bit: ldapur(Dst.W(), MemReg, Offset); break;
-      case IR::OpSize::i64Bit: ldapur(Dst.X(), MemReg, Offset); break;
-      default: LOGMAN_MSG_A_FMT("Unhandled ParanoidLoadMemTSO size: {}", OpSize); break;
-      }
-    }
-  } else if (CTX->HostFeatures.SupportsRCPC && Op->Class == FEXCore::IR::GPRClass) {
-    const auto Dst = GetReg(Node);
-    MemReg = ApplyMemOperand(OpSize, MemReg, TMP4, Op->Offset, Op->OffsetType, Op->OffsetScale);
-    if (OpSize == IR::OpSize::i8Bit) {
-      // 8bit load is always aligned to natural alignment
-      ldaprb(Dst.W(), MemReg);
-    } else {
-      switch (OpSize) {
-      case IR::OpSize::i16Bit: ldaprh(Dst.W(), MemReg); break;
-      case IR::OpSize::i32Bit: ldapr(Dst.W(), MemReg); break;
-      case IR::OpSize::i64Bit: ldapr(Dst.X(), MemReg); break;
-      default: LOGMAN_MSG_A_FMT("Unhandled ParanoidLoadMemTSO size: {}", OpSize); break;
-      }
-    }
-  } else if (Op->Class == FEXCore::IR::GPRClass) {
-    const auto Dst = GetReg(Node);
-    MemReg = ApplyMemOperand(OpSize, MemReg, TMP4, Op->Offset, Op->OffsetType, Op->OffsetScale);
-    switch (OpSize) {
-    case IR::OpSize::i8Bit: ldarb(Dst, MemReg); break;
-    case IR::OpSize::i16Bit: ldarh(Dst, MemReg); break;
-    case IR::OpSize::i32Bit: ldar(Dst.W(), MemReg); break;
-    case IR::OpSize::i64Bit: ldar(Dst.X(), MemReg); break;
-    default: LOGMAN_MSG_A_FMT("Unhandled ParanoidLoadMemTSO size: {}", OpSize); break;
-    }
-  } else {
-    const auto Dst = GetVReg(Node);
-    MemReg = ApplyMemOperand(OpSize, MemReg, TMP4, Op->Offset, Op->OffsetType, Op->OffsetScale);
-    switch (OpSize) {
-    case IR::OpSize::i8Bit:
-      ldarb(TMP1, MemReg);
-      fmov(ARMEmitter::Size::i32Bit, Dst.S(), TMP1.W());
-      break;
-    case IR::OpSize::i16Bit:
-      ldarh(TMP1, MemReg);
-      fmov(ARMEmitter::Size::i32Bit, Dst.S(), TMP1.W());
-      break;
-    case IR::OpSize::i32Bit:
-      ldar(TMP1.W(), MemReg);
-      fmov(ARMEmitter::Size::i32Bit, Dst.S(), TMP1.W());
-      break;
-    case IR::OpSize::i64Bit:
-      ldar(TMP1, MemReg);
-      fmov(ARMEmitter::Size::i64Bit, Dst.D(), TMP1);
-      break;
-    case IR::OpSize::i128Bit:
-      ldaxp(ARMEmitter::Size::i64Bit, TMP1, TMP2, MemReg);
-      clrex();
-      ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, TMP1);
-      ins(ARMEmitter::SubRegSize::i64Bit, Dst, 1, TMP2);
-      break;
-    case IR::OpSize::i256Bit:
-      LOGMAN_THROW_A_FMT(HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
-      dmb(ARMEmitter::BarrierScope::ISH);
-      ld1b<ARMEmitter::SubRegSize::i8Bit>(Dst.Z(), PRED_TMP_32B.Zeroing(), MemReg);
-      dmb(ARMEmitter::BarrierScope::ISH);
-      break;
-    default: LOGMAN_MSG_A_FMT("Unhandled ParanoidLoadMemTSO size: {}", OpSize); break;
-    }
-  }
-}
-
-DEF_OP(ParanoidStoreMemTSO) {
-  const auto Op = IROp->C<IR::IROp_StoreMemTSO>();
-  const auto OpSize = IROp->Size;
-
-  auto MemReg = GetReg(Op->Addr);
-
-  if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == FEXCore::IR::GPRClass) {
-    const auto Src = GetZeroableReg(Op->Value);
-    uint64_t Offset = 0;
-    if (!Op->Offset.IsInvalid()) {
-      if (!IsInlineConstant(Op->Offset, &Offset)) {
-        MemReg = ApplyMemOperand(OpSize, MemReg, TMP1, Op->Offset, Op->OffsetType, Op->OffsetScale);
-      }
-    }
-
-    if (OpSize == IR::OpSize::i8Bit) {
-      // 8bit load is always aligned to natural alignment
-      stlurb(Src, MemReg, Offset);
-    } else {
-      switch (OpSize) {
-      case IR::OpSize::i16Bit: stlurh(Src, MemReg, Offset); break;
-      case IR::OpSize::i32Bit: stlur(Src.W(), MemReg, Offset); break;
-      case IR::OpSize::i64Bit: stlur(Src.X(), MemReg, Offset); break;
-      default: LOGMAN_MSG_A_FMT("Unhandled ParanoidStoreMemTSO size: {}", OpSize); break;
-      }
-    }
-  } else if (Op->Class == FEXCore::IR::GPRClass) {
-    const auto Src = GetZeroableReg(Op->Value);
-    MemReg = ApplyMemOperand(OpSize, MemReg, TMP1, Op->Offset, Op->OffsetType, Op->OffsetScale);
-    switch (OpSize) {
-    case IR::OpSize::i8Bit: stlrb(Src, MemReg); break;
-    case IR::OpSize::i16Bit: stlrh(Src, MemReg); break;
-    case IR::OpSize::i32Bit: stlr(Src.W(), MemReg); break;
-    case IR::OpSize::i64Bit: stlr(Src.X(), MemReg); break;
-    default: LOGMAN_MSG_A_FMT("Unhandled ParanoidStoreMemTSO size: {}", OpSize); break;
-    }
-  } else {
-    const auto Src = GetVReg(Op->Value);
-
-    MemReg = ApplyMemOperand(OpSize, MemReg, TMP4, Op->Offset, Op->OffsetType, Op->OffsetScale);
-
-    switch (OpSize) {
-    case IR::OpSize::i8Bit:
-      umov<ARMEmitter::SubRegSize::i8Bit>(TMP1, Src, 0);
-      stlrb(TMP1, MemReg);
-      break;
-    case IR::OpSize::i16Bit:
-      umov<ARMEmitter::SubRegSize::i16Bit>(TMP1, Src, 0);
-      stlrh(TMP1, MemReg);
-      break;
-    case IR::OpSize::i32Bit:
-      umov<ARMEmitter::SubRegSize::i32Bit>(TMP1, Src, 0);
-      stlr(TMP1.W(), MemReg);
-      break;
-    case IR::OpSize::i64Bit:
-      umov<ARMEmitter::SubRegSize::i64Bit>(TMP1, Src, 0);
-      stlr(TMP1, MemReg);
-      break;
-    case IR::OpSize::i128Bit: {
-      // Move vector to GPRs
-      umov<ARMEmitter::SubRegSize::i64Bit>(TMP1, Src, 0);
-      umov<ARMEmitter::SubRegSize::i64Bit>(TMP2, Src, 1);
-      ARMEmitter::BackwardLabel B;
-      Bind(&B);
-
-      // ldaxp must not have both the destination registers be the same
-      ldaxp(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::zr, TMP3, MemReg); // <- Can hit SIGBUS. Overwritten with DMB
-      stlxp(ARMEmitter::Size::i64Bit, TMP3, TMP1, TMP2, MemReg);          // <- Can also hit SIGBUS
-      cbnz(ARMEmitter::Size::i64Bit, TMP3, &B);                           // < Overwritten with DMB
-      break;
-    }
-    case IR::OpSize::i256Bit: {
-      LOGMAN_THROW_A_FMT(HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
-      dmb(ARMEmitter::BarrierScope::ISH);
-      st1b<ARMEmitter::SubRegSize::i8Bit>(Src.Z(), PRED_TMP_32B, MemReg, 0);
-      dmb(ARMEmitter::BarrierScope::ISH);
-      break;
-    }
-    default: LOGMAN_MSG_A_FMT("Unhandled ParanoidStoreMemTSO size: {}", OpSize); break;
-    }
-  }
-}
-
 DEF_OP(CacheLineClear) {
   if (!CTX->HostFeatures.SupportsCacheMaintenanceOps) {
     dmb(ARMEmitter::BarrierScope::SY);

FEXCore/Source/Interface/IR/IR.json

@@ -604,8 +604,7 @@
         "Desc": ["Does a x86 TSO compatible load from memory. Offset must be Invalid()."
                 ],
         "Inline": ["", "Memtso"],
-        "DestSize": "Size",
-        "DynamicDispatch": true
+        "DestSize": "Size"
       },
 
       "StoreMemTSO RegisterClass:$Class, OpSize:#Size, SSA:$Value, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale": {
@@ -613,8 +612,7 @@
                 ],
         "Inline": ["Zero", "", "Memtso"],
         "HasSideEffects": true,
-        "DestSize": "Size",
-        "DynamicDispatch": true
+        "DestSize": "Size"
       },
 
       "FPR = VLoadVectorMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Mask, GPR:$Addr, GPR:$Offset, MemOffsetType:$OffsetType, u8:$OffsetScale": {

FEXCore/Source/Utils/LongJump.cpp

@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: MIT
+#include <FEXCore/Utils/LongJump.h>
+
+namespace FEXCore::LongJump {
+#if defined(_M_ARM_64)
+[[nodiscard]]
+FEX_DEFAULT_VISIBILITY FEX_NAKED uint64_t SetJump(JumpBuf& Buffer) {
+  __asm volatile(R"(
+      // x0 contains the jumpbuffer
+      stp x19, x20, [x0, #( 0 * 8)];
+      stp x21, x22, [x0, #( 2 * 8)];
+      stp x23, x24, [x0, #( 4 * 8)];
+      stp x25, x26, [x0, #( 6 * 8)];
+      stp x27, x28, [x0, #( 8 * 8)];
+      stp x29, x30, [x0, #(10 * 8)];
+
+      // FPRs
+      stp d8,   d9, [x0, #(12 * 8)];
+      stp d10, d11, [x0, #(14 * 8)];
+      stp d12, d13, [x0, #(16 * 8)];
+      stp d14, d15, [x0, #(18 * 8)];
+
+      // Move SP in to a temporary to store.
+      mov x1, sp;
+      str x1,  [x0, #(20 * 8)];
+
+      // Return zero to signify this is the SetJump.
+      mov x0, #0;
+      ret;
+    )" ::
+                   : "memory");
+}
+
+[[noreturn]]
+FEX_DEFAULT_VISIBILITY FEX_NAKED void LongJump(JumpBuf& Buffer, uint64_t Value) {
+  __asm volatile(R"(
+      // x0 contains the jumpbuffer
+      ldp x19, x20, [x0, #( 0 * 8)];
+      ldp x21, x22, [x0, #( 2 * 8)];
+      ldp x23, x24, [x0, #( 4 * 8)];
+      ldp x25, x26, [x0, #( 6 * 8)];
+      ldp x27, x28, [x0, #( 8 * 8)];
+      ldp x29, x30, [x0, #(10 * 8)];
+
+      // FPRs
+      ldp d8,   d9, [x0, #(12 * 8)];
+      ldp d10, d11, [x0, #(14 * 8)];
+      ldp d12, d13, [x0, #(16 * 8)];
+      ldp d14, d15, [x0, #(18 * 8)];
+
+      // Load SP in to temporary then move
+      ldr x0,  [x0, #(20 * 8)];
+      mov sp, x0;
+
+      // Move value in to result register
+      mov x0, x1;
+      ret;
+    )" ::
+                   : "memory");
+}
+#else
+[[nodiscard]]
+FEX_DEFAULT_VISIBILITY FEX_NAKED uint64_t SetJump(JumpBuf& Buffer) {
+  __asm volatile(R"(
+    .intel_syntax noprefix;
+    // rdi contains the jumpbuffer
+    mov [rdi + (0 * 8)], rbx;
+    mov [rdi + (1 * 8)], rsp;
+    mov [rdi + (2 * 8)], rbp;
+    mov [rdi + (3 * 8)], r12;
+    mov [rdi + (4 * 8)], r13;
+    mov [rdi + (5 * 8)], r14;
+    mov [rdi + (6 * 8)], r15;
+
+    // Return address is on the stack, load it and store
+    mov rsi, [rsp];
+    mov [rdi + (7 * 8)], rsi;
+
+    // Return zero to signify this is the SetJump.
+    mov rax, 0;
+    ret;
+
+    .att_syntax prefix;
+    )" ::
+                   : "memory");
+}
+
+[[noreturn]]
+FEX_DEFAULT_VISIBILITY FEX_NAKED void LongJump(JumpBuf& Buffer, uint64_t Value) {
+  __asm volatile(R"(
+    .intel_syntax noprefix;
+    // rdi contains the jumpbuffer
+    mov rbx, [rdi + (0 * 8)];
+    mov rsp, [rdi + (1 * 8)];
+    mov rbp, [rdi + (2 * 8)];
+    mov r12, [rdi + (3 * 8)];
+    mov r13, [rdi + (4 * 8)];
+    mov r14, [rdi + (5 * 8)];
+    mov r15, [rdi + (6 * 8)];
+
+    // Move value in to result register
+    mov rax, rsi;
+
+    // Pop the dead return address off the stack
+    pop rsi;
+
+    // Load the original return address from the jumpbuffer
+    mov rsi, [rdi + (7 * 8)];
+
+    // Return using a jump
+    jmp rsi;
+
+    .att_syntax prefix;
+    )" ::
+                   : "memory");
+}
+
+#endif
+} // namespace FEXCore::LongJump

FEXCore/include/FEXCore/Utils/LongJump.h

@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+#include <FEXCore/Utils/CompilerDefs.h>
+
+#include <cstdint>
+
+// It's longjump without glibc fortification checks.
+namespace FEXCore::LongJump {
+// JumpBuf definition needs to be public because the frontend needs to understand it.
+#if defined(_M_ARM_64)
+struct JumpBuf {
+  // All the registers that are required by AAPCS64 to save.
+  // GPRs
+  // X19, X20, X21, X22,
+  // X23, X24, X25, X26,
+  // X27, X28, X29, X30,
+  //
+  // Lower 64-bits:
+  //  V8,  V9, V10, V11,
+  // V12, V13, V14, V15,
+  //
+  // SP,
+  uint64_t Registers[21];
+};
+#else
+struct JumpBuf {
+  // Registers to preserve
+  // RBX, RSP, RBP, R12, R13, R14, R15,
+  // <return address>
+  uint64_t Registers[8];
+};
+#endif
+
+[[nodiscard]] FEX_DEFAULT_VISIBILITY uint64_t SetJump(JumpBuf& Buffer);
+[[noreturn]] FEX_DEFAULT_VISIBILITY void LongJump(JumpBuf& Buffer, uint64_t Value);
+} // namespace FEXCore::LongJump

FEXCore/unittests/Emitter/ALU_Tests.cpp

@@ -9,17 +9,17 @@ using namespace ARMEmitter;
 TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    adr(Reg::r30, &Label);
+    (void)adr(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x10fffffe);
   }
 
   {
     ForwardLabel Label;
-    adr(Reg::r30, &Label);
-    Bind(&Label);
+    (void)adr(Reg::r30, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x1000003e);
@@ -27,17 +27,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    adr(Reg::r30, &Label);
+    (void)adr(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x10fffffe);
   }
 
   {
     BiDirectionalLabel Label;
-    adr(Reg::r30, &Label);
-    Bind(&Label);
+    (void)adr(Reg::r30, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x1000003e);
@@ -45,80 +45,80 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    adrp(Reg::r30, &Label);
+    (void)adrp(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x9000001e);
   }
 
   {
     ForwardLabel Label;
-    adrp(Reg::r30, &Label);
+    (void)adrp(Reg::r30, &Label);
     // Move label a page away
     for (size_t i = 0; i < 1023; ++i) {
       nop();
     }
-    Bind(&Label);
+    (void)Bind(&Label);
 
     CHECK(DisassembleEncoding(0) == 0xb000001e);
   }
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    adrp(Reg::r30, &Label);
+    (void)adrp(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x9000001e);
   }
 
   {
     BiDirectionalLabel Label;
-    adrp(Reg::r30, &Label);
+    (void)adrp(Reg::r30, &Label);
     // Move label a page away
     for (size_t i = 0; i < 1023; ++i) {
       nop();
     }
-    Bind(&Label);
+    (void)Bind(&Label);
 
     CHECK(DisassembleEncoding(0) == 0xb000001e);
   }
 
   {
-    // Will generate adr.
+    // Will generate (void)adr.
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
     CHECK(DisassembleEncoding(1) == 0x10fffffe);
   }
   {
-    // Will generate nop + adr.
+    // Will generate nop + (void)adr.
     ForwardLabel Label;
-    LongAddressGen(Reg::r30, &Label);
-    Bind(&Label);
+    (void)LongAddressGen(Reg::r30, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd503201f);
     CHECK(DisassembleEncoding(1) == 0x1000003e);
   }
   {
-    // Will generate adr.
+    // Will generate (void)adr.
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x10fffffe);
   }
 
   {
-    // Will generate nop + adr.
+    // Will generate nop + (void)adr.
     BiDirectionalLabel Label;
-    LongAddressGen(Reg::r30, &Label);
-    Bind(&Label);
+    (void)LongAddressGen(Reg::r30, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd503201f);
@@ -126,33 +126,33 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   }
 
   {
-    // Will generate adrp.
+    // Will generate (void)adrp.
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
-    // Move adrp 1MB away.
+    // Move (void)adrp 1MB away.
     for (size_t i = 0; i < (1 * 1024 * 1024 / 4); ++i) {
       nop();
     }
 
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
     nop();
     CHECK(DisassembleEncoding(262145) == 0x90fff81e);
     CHECK(DisassembleEncoding(262146) == 0xd503201f);
   }
 
   {
-    // Will generate nop + adrp.
+    // Will generate nop + (void)adrp.
     ForwardLabel Label;
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     // Move label 1MB away, plus a page, and then aligned to a page.
     for (size_t i = 0; i < ((1 * 1024 * 1024 + 4096) / 4 - 2); ++i) {
       nop();
     }
 
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd503201f);
@@ -160,16 +160,16 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   }
 
   {
-    // Will generate adrp + add.
+    // Will generate (void)adrp + add.
     ForwardLabel Label;
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     // Move label 1MB away, plus a page, plus one instruction.
     for (size_t i = 0; i < ((1 * 1024 * 1024 + 4096) / 4 - 1); ++i) {
       nop();
     }
 
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb000081e);
@@ -178,33 +178,33 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
 
 
   {
-    // Will generate adrp.
+    // Will generate (void)adrp.
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
-    // Move adrp 1MB away.
+    // Move (void)adrp 1MB away.
     for (size_t i = 0; i < (1 * 1024 * 1024 / 4); ++i) {
       nop();
     }
 
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
     nop();
     CHECK(DisassembleEncoding(262145) == 0x90fff81e);
     CHECK(DisassembleEncoding(262146) == 0xd503201f);
   }
 
   {
-    // Will generate nop + adrp.
+    // Will generate nop + (void)adrp.
     BiDirectionalLabel Label;
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     // Move label 1MB away, plus a page, and then aligned to a page.
     for (size_t i = 0; i < ((1 * 1024 * 1024 + 4096) / 4 - 2); ++i) {
       nop();
     }
 
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd503201f);
@@ -212,16 +212,16 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ALU: PC relative") {
   }
 
   {
-    // Will generate adrp + add.
+    // Will generate (void)adrp + add.
     BiDirectionalLabel Label;
-    LongAddressGen(Reg::r30, &Label);
+    (void)LongAddressGen(Reg::r30, &Label);
 
     // Move label 1MB away, plus a page, plus one instruction.
     for (size_t i = 0; i < ((1 * 1024 * 1024 + 4096) / 4 - 1); ++i) {
       nop();
     }
 
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb000081e);

FEXCore/unittests/Emitter/Branch_Tests.cpp

@@ -9,17 +9,17 @@ using namespace ARMEmitter;
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Conditional branch immediate") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    b(Condition::CC_PL, &Label);
+    (void)b(Condition::CC_PL, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x54ffffe5);
   }
 
   {
     ForwardLabel Label;
-    b(Condition::CC_PL, &Label);
-    Bind(&Label);
+    (void)b(Condition::CC_PL, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x54000025);
@@ -27,17 +27,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Conditional branch immediat
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    b(Condition::CC_PL, &Label);
+    (void)b(Condition::CC_PL, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x54ffffe5);
   }
 
   {
     BiDirectionalLabel Label;
-    b(Condition::CC_PL, &Label);
-    Bind(&Label);
+    (void)b(Condition::CC_PL, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x54000025);
@@ -46,17 +46,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Conditional branch immediat
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Branch consistent conditional") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    bc(Condition::CC_PL, &Label);
+    (void)bc(Condition::CC_PL, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x54fffff5);
   }
 
   {
     ForwardLabel Label;
-    bc(Condition::CC_PL, &Label);
-    Bind(&Label);
+    (void)bc(Condition::CC_PL, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x54000035);
@@ -64,17 +64,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Branch consistent condition
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    bc(Condition::CC_PL, &Label);
+    (void)bc(Condition::CC_PL, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x54fffff5);
   }
 
   {
     BiDirectionalLabel Label;
-    bc(Condition::CC_PL, &Label);
-    Bind(&Label);
+    (void)bc(Condition::CC_PL, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x54000035);
@@ -89,17 +89,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch regist
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immediate") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    b(&Label);
+    (void)b(&Label);
 
     CHECK(DisassembleEncoding(1) == 0x17ffffff);
   }
 
   {
     ForwardLabel Label;
-    b(&Label);
-    Bind(&Label);
+    (void)b(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x14000001);
@@ -107,17 +107,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immedi
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    b(&Label);
+    (void)b(&Label);
 
     CHECK(DisassembleEncoding(1) == 0x17ffffff);
   }
 
   {
     BiDirectionalLabel Label;
-    b(&Label);
-    Bind(&Label);
+    (void)b(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x14000001);
@@ -125,17 +125,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immedi
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    bl(&Label);
+    (void)bl(&Label);
 
     CHECK(DisassembleEncoding(1) == 0x97ffffff);
   }
 
   {
     ForwardLabel Label;
-    bl(&Label);
-    Bind(&Label);
+    (void)bl(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x94000001);
@@ -143,17 +143,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immedi
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    bl(&Label);
+    (void)bl(&Label);
 
     CHECK(DisassembleEncoding(1) == 0x97ffffff);
   }
 
   {
     BiDirectionalLabel Label;
-    bl(&Label);
-    Bind(&Label);
+    (void)bl(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x94000001);
@@ -162,17 +162,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Unconditional branch immedi
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbz(Size::i32Bit, Reg::r29, &Label);
+    (void)cbz(Size::i32Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x34fffffd);
   }
 
   {
     ForwardLabel Label;
-    cbz(Size::i32Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbz(Size::i32Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3400003d);
@@ -180,17 +180,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbz(Size::i32Bit, Reg::r29, &Label);
+    (void)cbz(Size::i32Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x34fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    cbz(Size::i32Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbz(Size::i32Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3400003d);
@@ -198,17 +198,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbz(Size::i64Bit, Reg::r29, &Label);
+    (void)cbz(Size::i64Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb4fffffd);
   }
 
   {
     ForwardLabel Label;
-    cbz(Size::i64Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbz(Size::i64Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb400003d);
@@ -216,17 +216,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbz(Size::i64Bit, Reg::r29, &Label);
+    (void)cbz(Size::i64Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb4fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    cbz(Size::i64Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbz(Size::i64Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb400003d);
@@ -234,17 +234,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbnz(Size::i32Bit, Reg::r29, &Label);
+    (void)cbnz(Size::i32Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x35fffffd);
   }
 
   {
     ForwardLabel Label;
-    cbnz(Size::i32Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbnz(Size::i32Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3500003d);
@@ -252,17 +252,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbnz(Size::i32Bit, Reg::r29, &Label);
+    (void)cbnz(Size::i32Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x35fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    cbnz(Size::i32Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbnz(Size::i32Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3500003d);
@@ -270,17 +270,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbnz(Size::i64Bit, Reg::r29, &Label);
+    (void)cbnz(Size::i64Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb5fffffd);
   }
 
   {
     ForwardLabel Label;
-    cbnz(Size::i64Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbnz(Size::i64Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb500003d);
@@ -288,17 +288,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    cbnz(Size::i64Bit, Reg::r29, &Label);
+    (void)cbnz(Size::i64Bit, Reg::r29, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb5fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    cbnz(Size::i64Bit, Reg::r29, &Label);
-    Bind(&Label);
+    (void)cbnz(Size::i64Bit, Reg::r29, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb500003d);
@@ -307,17 +307,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Compare and branch") {
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbz(Reg::r29, 0, &Label);
+    (void)tbz(Reg::r29, 0, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x3607fffd);
   }
 
   {
     ForwardLabel Label;
-    tbz(Reg::r29, 0, &Label);
-    Bind(&Label);
+    (void)tbz(Reg::r29, 0, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3600003d);
@@ -325,17 +325,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbz(Reg::r29, 0, &Label);
+    (void)tbz(Reg::r29, 0, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x3607fffd);
   }
 
   {
     BiDirectionalLabel Label;
-    tbz(Reg::r29, 0, &Label);
-    Bind(&Label);
+    (void)tbz(Reg::r29, 0, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3600003d);
@@ -343,17 +343,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbz(Reg::r29, 63, &Label);
+    (void)tbz(Reg::r29, 63, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb6fffffd);
   }
 
   {
     ForwardLabel Label;
-    tbz(Reg::r29, 63, &Label);
-    Bind(&Label);
+    (void)tbz(Reg::r29, 63, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb6f8003d);
@@ -361,17 +361,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbz(Reg::r29, 63, &Label);
+    (void)tbz(Reg::r29, 63, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb6fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    tbz(Reg::r29, 63, &Label);
-    Bind(&Label);
+    (void)tbz(Reg::r29, 63, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb6f8003d);
@@ -379,17 +379,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbnz(Reg::r29, 0, &Label);
+    (void)tbnz(Reg::r29, 0, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x3707fffd);
   }
 
   {
     ForwardLabel Label;
-    tbnz(Reg::r29, 0, &Label);
-    Bind(&Label);
+    (void)tbnz(Reg::r29, 0, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3700003d);
@@ -397,17 +397,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbnz(Reg::r29, 0, &Label);
+    (void)tbnz(Reg::r29, 0, &Label);
 
     CHECK(DisassembleEncoding(1) == 0x3707fffd);
   }
 
   {
     BiDirectionalLabel Label;
-    tbnz(Reg::r29, 0, &Label);
-    Bind(&Label);
+    (void)tbnz(Reg::r29, 0, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x3700003d);
@@ -415,17 +415,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbnz(Reg::r29, 63, &Label);
+    (void)tbnz(Reg::r29, 63, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb7fffffd);
   }
 
   {
     ForwardLabel Label;
-    tbnz(Reg::r29, 63, &Label);
-    Bind(&Label);
+    (void)tbnz(Reg::r29, 63, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb7f8003d);
@@ -433,17 +433,17 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Branch: Test and branch immediate")
 
   {
     BiDirectionalLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
-    tbnz(Reg::r29, 63, &Label);
+    (void)tbnz(Reg::r29, 63, &Label);
 
     CHECK(DisassembleEncoding(1) == 0xb7fffffd);
   }
 
   {
     BiDirectionalLabel Label;
-    tbnz(Reg::r29, 63, &Label);
-    Bind(&Label);
+    (void)tbnz(Reg::r29, 63, &Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xb7f8003d);

FEXCore/unittests/Emitter/Loadstore_Tests.cpp

@@ -1323,7 +1323,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: LDAPR/STLR unscaled imme
 TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal") {
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(WReg::w30, &Label);
 
@@ -1332,7 +1332,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(SReg::s30, &Label);
 
@@ -1341,7 +1341,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(XReg::x30, &Label);
 
@@ -1350,7 +1350,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(DReg::d30, &Label);
 
@@ -1359,7 +1359,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldrsw(XReg::x30, &Label);
 
@@ -1368,7 +1368,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     ldr(QReg::q30, &Label);
 
@@ -1377,7 +1377,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
 
   {
     BackwardLabel Label;
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
     prfm(Prefetch::PLDL1KEEP, &Label);
 
@@ -1387,7 +1387,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(WReg::w30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x1800003e);
@@ -1396,7 +1396,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(SReg::s30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x1c00003e);
@@ -1405,7 +1405,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(XReg::x30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x5800003e);
@@ -1414,7 +1414,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(DReg::d30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x5c00003e);
@@ -1423,7 +1423,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldrsw(XReg::x30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x9800003e);
@@ -1432,7 +1432,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     ldr(QReg::q30, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0x9c00003e);
@@ -1441,7 +1441,7 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Loadstore: Load register literal")
   {
     ForwardLabel Label;
     prfm(Prefetch::PLDL1KEEP, &Label);
-    Bind(&Label);
+    (void)Bind(&Label);
     dc32(0);
 
     CHECK(DisassembleEncoding(0) == 0xd8000020);

Source/Tools/CMakeLists.txt

@@ -23,8 +23,7 @@ if (NOT MINGW_BUILD)
 
   add_subdirectory(FEXInterpreter/)
   add_subdirectory(pidof/)
-endif()
-
-if (BUILD_TESTING)
-  add_subdirectory(TestHarnessRunner/)
+  if (BUILD_TESTING)
+    add_subdirectory(TestHarnessRunner/)
+  endif()
 endif()

Source/Tools/FEXInterpreter/FEXInterpreter.cpp

@@ -256,6 +256,28 @@ void CheckForGCS() {
 }
 } // namespace FEX::GCS
 
+namespace FEX::UnalignedAtomic {
+void SetupKernelUnalignedAtomics() {
+#ifndef PR_ARM64_SET_UNALIGN_ATOMIC
+#define PR_ARM64_SET_UNALIGN_ATOMIC 0x46455849
+#define PR_ARM64_UNALIGN_ATOMIC_EMULATE (1UL << 0)
+#define PR_ARM64_UNALIGN_ATOMIC_BACKPATCH (1UL << 1)
+#define PR_ARM64_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS (1UL << 2)
+#endif
+
+  // Interfaces with downstream FEX kernel patches to control unaligned atomic handling
+  FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
+  FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS);
+
+  uint64_t Flags = (StrictInProcessSplitLocks() ? PR_ARM64_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS : 0) |
+                   (ParanoidTSO() ? 0 : PR_ARM64_UNALIGN_ATOMIC_BACKPATCH) | PR_ARM64_UNALIGN_ATOMIC_EMULATE;
+
+  if (prctl(PR_ARM64_SET_UNALIGN_ATOMIC, Flags, 0, 0, 0) != -1) {
+    LogMan::Msg::IFmt("FEX: Kernel unaligned atomics enabled!");
+  }
+}
+} // namespace FEX::UnalignedAtomic
+
 /**
  * @brief Get an FD from an environment variable and then unset the environment variable.
  *
@@ -480,7 +502,12 @@ int main(int argc, char** argv, char** const envp) {
     free(data);
   }
 
-  FEXCore::Profiler::Init(Program.ProgramName, Program.ProgramPath);
+  {
+    FEX_CONFIG_OPT(TraceProfiler, TRACEPROFILER);
+    if (TraceProfiler()) {
+      FEXCore::Profiler::Init(Program.ProgramName, Program.ProgramPath);
+    }
+  }
 
   bool SupportsAVX {};
   fextl::unique_ptr<FEXCore::Context::Context> CTX;
@@ -492,6 +519,7 @@ int main(int argc, char** argv, char** const envp) {
 
   // Setup TSO hardware emulation immediately after initializing the context.
   FEX::TSO::SetupTSOEmulation(CTX.get());
+  FEX::UnalignedAtomic::SetupKernelUnalignedAtomics();
 
   if (!Loader.Is64BitMode()) {
     // Tell the kernel we want to use the compat input syscalls even though we're

Source/Tools/LinuxEmulation/LinuxSyscalls/Seccomp/BPFEmitter.cpp

@@ -164,7 +164,7 @@ uint64_t BPFEmitter::HandleJmp(uint32_t BPFIP, uint32_t NumInst, const sock_filt
       TargetLabel = JumpLabels.try_emplace(Target, ARMEmitter::ForwardLabel {}).first;
     }
 
-    EMIT_INST(b(&TargetLabel->second));
+    EMIT_INST((void)b(&TargetLabel->second));
     break;
   }
   case BPF_JEQ:
@@ -208,8 +208,8 @@ uint64_t BPFEmitter::HandleJmp(uint32_t BPFIP, uint32_t NumInst, const sock_filt
       TargetFalseLabel = JumpLabels.try_emplace(TargetFalse, ARMEmitter::ForwardLabel {}).first;
     }
 
-    EMIT_INST(b(CompareResultOp, &TargetTrueLabel->second));
-    EMIT_INST(b(&TargetFalseLabel->second));
+    EMIT_INST((void)b(CompareResultOp, &TargetTrueLabel->second));
+    EMIT_INST((void)b(&TargetFalseLabel->second));
     break;
   }
   default: RETURN_ERROR(-EINVAL); // Unknown jump type
@@ -263,7 +263,7 @@ uint64_t BPFEmitter::HandleEmission(uint32_t flags, const sock_fprog* prog) {
     if constexpr (!CalculateSize) {
       auto jump_label = JumpLabels.find(i);
       if (jump_label != JumpLabels.end()) {
-        Bind(&jump_label->second);
+        (void)Bind(&jump_label->second);
       }
     }
 
@@ -359,7 +359,7 @@ uint64_t BPFEmitter::JITFilter(uint32_t flags, const sock_fprog* prog) {
   // Emit the constant pool.
   Align();
   for (auto& Const : ConstPool) {
-    Bind(&Const.second);
+    (void)Bind(&Const.second);
     dc32(Const.first);
   }
 

Source/Tools/LinuxEmulation/LinuxSyscalls/Utils/Threads.cpp

@@ -4,6 +4,7 @@
 
 #include <FEXCore/Core/Context.h>
 #include <FEXCore/Utils/Allocator.h>
+#include <FEXCore/Utils/LongJump.h>
 #include <FEXCore/Utils/Threads.h>
 
 namespace FEX::LinuxEmulation::Threads {
@@ -189,143 +190,6 @@ __attribute__((naked)) void StackPivotAndCall(void* Arg, FEXCore::Threads::Threa
 }
 #endif
 namespace PThreads {
-  namespace LongJump {
-    // This is a custom long jump implementation that avoids the glibc implementation.
-    // This is required behaviour because glibc's fortification checks don't understand stack pivots.
-    // FEX requires a stack pivot to work through a long jump, so these two features are at odds with each other.
-#ifdef _M_ARM_64
-    struct JumpBuf {
-      // All the registers that are required by AAPCS64 to save.
-      // GPRs
-      // X19, X20, X21, X22,
-      // X23, X24, X25, X26,
-      // X27, X28, X29, X30,
-      //
-      // Lower 64-bits:
-      //  V8,  V9, V10, V11,
-      // V12, V13, V14, V15,
-      //
-      // SP,
-      uint64_t Registers[21];
-    };
-    FEX_NAKED uint64_t SetJump(JumpBuf& Buffer) {
-      __asm volatile(R"(
-        // x0 contains the jumpbuffer
-        stp x19, x20, [x0, #( 0 * 8)];
-        stp x21, x22, [x0, #( 2 * 8)];
-        stp x23, x24, [x0, #( 4 * 8)];
-        stp x25, x26, [x0, #( 6 * 8)];
-        stp x27, x28, [x0, #( 8 * 8)];
-        stp x29, x30, [x0, #(10 * 8)];
-
-        // FPRs
-        stp d8,   d9, [x0, #(12 * 8)];
-        stp d10, d11, [x0, #(14 * 8)];
-        stp d12, d13, [x0, #(16 * 8)];
-        stp d14, d15, [x0, #(18 * 8)];
-
-        // Move SP in to a temporary to store.
-        mov x1, sp;
-        str x1,  [x0, #(19 * 8)];
-
-        // Return zero to signify this is the SetJump.
-        mov x0, #0;
-        ret;
-      )" ::
-                       : "memory");
-    }
-
-    [[noreturn]]
-    FEX_NAKED void LongJump(JumpBuf& Buffer, uint64_t Value) {
-      __asm volatile(R"(
-        // x0 contains the jumpbuffer
-        ldp x19, x20, [x0, #( 0 * 8)];
-        ldp x21, x22, [x0, #( 2 * 8)];
-        ldp x23, x24, [x0, #( 4 * 8)];
-        ldp x25, x26, [x0, #( 6 * 8)];
-        ldp x27, x28, [x0, #( 8 * 8)];
-        ldp x29, x30, [x0, #(10 * 8)];
-
-        // FPRs
-        ldp d8,   d9, [x0, #(12 * 8)];
-        ldp d10, d11, [x0, #(14 * 8)];
-        ldp d12, d13, [x0, #(16 * 8)];
-        ldp d14, d15, [x0, #(18 * 8)];
-
-        // Load SP in to temporary then move
-        ldr x0,  [x0, #(19 * 8)];
-        mov sp, x0;
-
-        // Move value in to result register
-        mov x0, x1;
-        ret;
-      )" ::
-                       : "memory");
-    }
-#else
-    struct JumpBuf {
-      // Registers to preserve
-      // RBX, RSP, RBP, R12, R13, R14, R15,
-      // <return address>
-      uint64_t Registers[8];
-    };
-
-    __attribute__((naked)) uint64_t SetJump(JumpBuf& Buffer) {
-      __asm volatile(R"(
-      .intel_syntax noprefix;
-      // rdi contains the jumpbuffer
-      mov [rdi + (0 * 8)], rbx;
-      mov [rdi + (1 * 8)], rsp;
-      mov [rdi + (2 * 8)], rbp;
-      mov [rdi + (3 * 8)], r12;
-      mov [rdi + (4 * 8)], r13;
-      mov [rdi + (5 * 8)], r14;
-      mov [rdi + (6 * 8)], r15;
-
-      // Return address is on the stack, load it and store
-      mov rsi, [rsp];
-      mov [rdi + (7 * 8)], rsi;
-
-      // Return zero to signify this is the SetJump.
-      mov rax, 0;
-      ret;
-
-      .att_syntax prefix;
-      )" ::
-                       : "memory");
-    }
-
-    [[noreturn]]
-    __attribute__((naked)) void LongJump(JumpBuf& Buffer, uint64_t Value) {
-      __asm volatile(R"(
-      .intel_syntax noprefix;
-      // rdi contains the jumpbuffer
-      mov rbx, [rdi + (0 * 8)];
-      mov rsp, [rdi + (1 * 8)];
-      mov rbp, [rdi + (2 * 8)];
-      mov r12, [rdi + (3 * 8)];
-      mov r13, [rdi + (4 * 8)];
-      mov r14, [rdi + (5 * 8)];
-      mov r15, [rdi + (6 * 8)];
-
-      // Move value in to result register
-      mov rax, rsi;
-
-      // Pop the dead return address off the stack
-      pop rsi;
-
-      // Load the original return address from the jumpbuffer
-      mov rsi, [rdi + (7 * 8)];
-
-      // Return using a jump
-      jmp rsi;
-
-      .att_syntax prefix;
-      )" ::
-                       : "memory");
-    }
-#endif
-  }; // namespace LongJump
   void* InitializeThread(void* Ptr);
 
   class PThread final : public FEXCore::Threads::Thread {
@@ -396,7 +260,7 @@ namespace PThreads {
       return STracker;
     }
 
-    void SetupLongJump(LongJump::JumpBuf* exit_resolver) {
+    void SetupLongJump(FEXCore::LongJump::JumpBuf* exit_resolver) {
       _exit_resolver = exit_resolver;
     }
 
@@ -404,7 +268,7 @@ namespace PThreads {
     void LongJumpExit(FEX::HLE::ThreadStateObject* ThreadObject, uint32_t Status) {
       this->Status = Status;
       this->ThreadObject = ThreadObject;
-      LongJump::LongJump(*_exit_resolver, 1);
+      FEXCore::LongJump::LongJump(*_exit_resolver, 1);
       FEX_UNREACHABLE;
     }
 
@@ -423,7 +287,7 @@ namespace PThreads {
     void* UserArg;
     void* Stack {};
 
-    LongJump::JumpBuf* _exit_resolver {};
+    FEXCore::LongJump::JumpBuf* _exit_resolver {};
     FEX::HLE::ThreadStateObject* ThreadObject {};
     uint32_t Status {};
   };
@@ -434,11 +298,11 @@ namespace PThreads {
     PThread* Thread {reinterpret_cast<PThread*>(Ptr)};
     StackBase = Thread->GetPivotStack();
     STracker = Thread->GetStackTracker();
-    LongJump::JumpBuf exit_resolver {};
+    FEXCore::LongJump::JumpBuf exit_resolver {};
 
     bool LongJumpExit {};
 
-    if (LongJump::SetJump(exit_resolver) == 0) {
+    if (FEXCore::LongJump::SetJump(exit_resolver) == 0) {
       Thread->SetupLongJump(&exit_resolver);
       // Run the user function.
       // `Thread` object is dead after this function returns.

Source/Windows/ARM64EC/Module.cpp

@@ -640,14 +640,18 @@ NTSTATUS ProcessInit() {
 
   FEXCore::Config::Set(FEXCore::Config::CONFIG_IS64BIT_MODE, "1");
 
-  FEXCore::Profiler::Init("", "");
+  {
+    FEX_CONFIG_OPT(TraceProfiler, TRACEPROFILER);
+    if (TraceProfiler()) {
+      FEXCore::Profiler::Init("", "");
+    }
+  }
 
   FEX_CONFIG_OPT(ExtendedVolatileMetadataConfig, EXTENDEDVOLATILEMETADATA);
   ExtendedMetaData = FEX::VolatileMetadata::ParseExtendedVolatileMetadata(ExtendedVolatileMetadataConfig());
 
   SignalDelegator = fextl::make_unique<FEX::DummyHandlers::DummySignalDelegator>();
   SyscallHandler = fextl::make_unique<Exception::ECSyscallHandler>();
-  Exception::HandlerConfig.emplace();
 
   const auto NtDll = GetModuleHandle("ntdll.dll");
   const bool IsWine = !!GetProcAddress(NtDll, "wine_get_version");
@@ -661,7 +665,7 @@ NTSTATUS ProcessInit() {
   CTX->SetSignalDelegator(SignalDelegator.get());
   CTX->SetSyscallHandler(SyscallHandler.get());
   CTX->InitCore();
-
+  Exception::HandlerConfig.emplace(*CTX);
   InvalidationTracker.emplace(*CTX, Threads);
 
   HandleImageMap(NtDllBase);

Source/Windows/Common/TSOHandlerConfig.h

@@ -1,13 +1,14 @@
 // SPDX-License-Identifier: MIT
 #pragma once
 
+#include <FEXCore/Core/Context.h>
 #include <FEXCore/Config/Config.h>
 #include <FEXCore/Utils/ArchHelpers/Arm64.h>
 
 namespace FEX::Windows {
 class TSOHandlerConfig final {
 public:
-  TSOHandlerConfig() {
+  TSOHandlerConfig(FEXCore::Context::Context& CTX) {
     if (ParanoidTSO()) {
       UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::Paranoid;
     } else if (HalfBarrierTSOEnabled()) {
@@ -15,6 +16,21 @@ public:
     } else {
       UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::NonAtomic;
     }
+
+    if (TSOEnabled()) {
+      BOOL Enable = TRUE;
+      NTSTATUS Status = NtSetInformationProcess(NtCurrentProcess(), ProcessFexHardwareTso, &Enable, sizeof(Enable));
+      if (Status == STATUS_SUCCESS) {
+        CTX.SetHardwareTSOSupport(true);
+      }
+    }
+
+    uint64_t Flags = (StrictInProcessSplitLocks() ? FEX_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS : 0) |
+                     (ParanoidTSO() ? 0 : FEX_UNALIGN_ATOMIC_BACKPATCH) | FEX_UNALIGN_ATOMIC_EMULATE;
+
+    if (NtSetInformationProcess(NtCurrentProcess(), ProcessFexUnalignAtomic, &Flags, sizeof(Flags)) == STATUS_SUCCESS) {
+      LogMan::Msg::IFmt("FEX: Kernel unaligned atomics enabled!");
+    }
   }
 
   FEXCore::ArchHelpers::Arm64::UnalignedHandlerType GetUnalignedHandlerType() const {
@@ -22,8 +38,10 @@ public:
   }
 
 private:
+  FEX_CONFIG_OPT(TSOEnabled, TSOENABLED);
   FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
   FEX_CONFIG_OPT(HalfBarrierTSOEnabled, HALFBARRIERTSOENABLED);
+  FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS);
 
   FEXCore::ArchHelpers::Arm64::UnalignedHandlerType UnalignedHandlerType {FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::HalfBarrier};
 };

Source/Windows/WOW64/Module.cpp

@@ -516,14 +516,18 @@ void BTCpuProcessInit() {
   FEXCore::Config::Set(FEXCore::Config::CONFIG_INTERPRETER_INSTALLED, "0");
   FEXCore::Config::Set(FEXCore::Config::CONFIG_IS64BIT_MODE, "0");
 
-  FEXCore::Profiler::Init("", "");
+  {
+    FEX_CONFIG_OPT(TraceProfiler, TRACEPROFILER);
+    if (TraceProfiler()) {
+      FEXCore::Profiler::Init("", "");
+    }
+  }
 
   FEX_CONFIG_OPT(ExtendedVolatileMetadataConfig, EXTENDEDVOLATILEMETADATA);
   ExtendedMetaData = FEX::VolatileMetadata::ParseExtendedVolatileMetadata(ExtendedVolatileMetadataConfig());
 
   SignalDelegator = fextl::make_unique<FEX::DummyHandlers::DummySignalDelegator>();
   SyscallHandler = fextl::make_unique<WowSyscallHandler>();
-  Context::HandlerConfig.emplace();
   const auto NtDll = GetModuleHandle("ntdll.dll");
   const bool IsWine = !!GetProcAddress(NtDll, "wine_get_version");
   OvercommitTracker.emplace(IsWine);
@@ -536,7 +540,7 @@ void BTCpuProcessInit() {
   CTX->SetSignalDelegator(SignalDelegator.get());
   CTX->SetSyscallHandler(SyscallHandler.get());
   CTX->InitCore();
-
+  Context::HandlerConfig.emplace(*CTX);
   InvalidationTracker.emplace(*CTX, Threads);
 
   auto NtDllX86 = reinterpret_cast<SYSTEM_DLL_INIT_BLOCK*>(GetProcAddress(NtDll, "LdrSystemDllInitBlock"))->ntdll_handle;

Source/Windows/include/winternl.h

@@ -455,6 +455,12 @@ typedef enum _MEMORY_INFORMATION_CLASS {
 #define SystemEmulationBasicInformation (SYSTEM_INFORMATION_CLASS)62
 
 #define ProcessFexHardwareTso (PROCESSINFOCLASS)2000
+#define ProcessFexUnalignAtomic (PROCESSINFOCLASS)2001
+
+// These match the prctl flag values
+#define FEX_UNALIGN_ATOMIC_EMULATE (1ULL << 0)
+#define FEX_UNALIGN_ATOMIC_BACKPATCH (1ULL << 1)
+#define FEX_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS (1ULL << 2)
 
 typedef enum _KEY_VALUE_INFORMATION_CLASS {
   KeyValueBasicInformation,

unittests/ASM/FEX_bugs/BitConditionCheck.asm

@@ -0,0 +1,39 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "1"
+  },
+  "Env": { "FEX_MAXINST" : "41010", "FEX_MULTIBLOCK": "1", "FEX_TSOENABLED": "0" }
+}
+%endif
+
+; FEX-Emu had a bug where it tried to encode too large of a tbz/tbnz offset.
+
+%macro TooConditionalBitBranch 0
+  ; Stresses ARM64's tbz/tbnz branch target of +-32KB.
+  jmp %%top
+  %%top:
+
+  lea rax, [rel data]
+  mov rbx, 1
+  %rep 2000
+    add qword [rel data], rbx
+  %endrep
+  lea rax, [%%top]
+  jpe %%top
+%endmacro
+
+mov rax, 0
+cmp rax, 0
+mov rcx, 0
+
+jz long_jump
+
+TooConditionalBitBranch
+
+long_jump:
+mov rax, 1
+hlt
+
+data:
+dq 0, 0, 0, 0

unittests/ASM/FEX_bugs/BranchConditionCheck.asm

@@ -0,0 +1,40 @@
+
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "1"
+  },
+  "Env": { "FEX_MAXINST" : "41010", "FEX_MULTIBLOCK": "1", "FEX_TSOENABLED": "0" }
+}
+%endif
+
+; FEX-Emu had a bug where it tried encoding too large of a conditional branch offset.
+
+%macro TooLargeConditionalBranch 0
+  ; Stresses ARM64's b.cc/cbz/cbnz branch target of +-1MB.
+  jmp %%top
+  %%top:
+
+  lea rax, [rel data]
+  mov rbx, 1
+  %rep 37500
+    add qword [rel data], rbx
+  %endrep
+  lea rax, [%%top]
+  jnz %%top
+%endmacro
+
+mov rax, 0
+cmp rax, 0
+mov rcx, 0
+
+jz long_jump
+
+TooLargeConditionalBranch
+
+long_jump:
+mov rax, 1
+hlt
+
+data:
+dq 0, 0, 0, 0