From 1ad928dc2cda89e2e0ae5cf60fc3899dd29564a4 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 28 Jul 2023 09:42:24 -0400
Subject: [PATCH 01/23] IR: Add special NZCV "flag"

Reserve 4 bytes of "flags" to model the 32-bit arm64 NZCV register, so
we can start porting FEX's flag handling code over to using NZCV without
needing the whole compiler to be aware of instructions that might
clobber host flags.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Interface/Core/Interpreter/MemoryOps.cpp  | 22 ++++++++++++++-----
 .../Interface/Core/JIT/Arm64/MemoryOps.cpp    | 12 ++++++++--
 .../Interface/Core/JIT/x86_64/MemoryOps.cpp   | 12 ++++++++--
 .../FEXCore/include/FEXCore/Core/X86Enums.h   |  7 ++++++
 4 files changed, 44 insertions(+), 9 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp
index 096da6f6a..d4c355b97 100644
--- a/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp
+++ b/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp
@@ -190,19 +190,31 @@ DEF_OP(LoadFlag) {
   uintptr_t ContextPtr = reinterpret_cast<uintptr_t>(Data->State->CurrentFrame);
   ContextPtr += offsetof(FEXCore::Core::CPUState, flags[0]);
   ContextPtr += Op->Flag;
-  uint8_t const *MemData = reinterpret_cast<uint8_t const*>(ContextPtr);
-  GD = *MemData;
+
+  if (Op->Flag == 24 /* NZCV */) {
+    uint32_t const *MemData = reinterpret_cast<uint32_t const*>(ContextPtr);
+    GD = *MemData;
+  } else {
+    uint8_t const *MemData = reinterpret_cast<uint8_t const*>(ContextPtr);
+    GD = *MemData;
+  }
 }
 
 DEF_OP(StoreFlag) {
   auto Op = IROp->C<IR::IROp_StoreFlag>();
-  uint8_t Arg = *GetSrc<uint8_t*>(Data->SSAData, Op->Value);
+  uint32_t Arg = *GetSrc<uint32_t*>(Data->SSAData, Op->Value);
 
   uintptr_t ContextPtr = reinterpret_cast<uintptr_t>(Data->State->CurrentFrame);
   ContextPtr += offsetof(FEXCore::Core::CPUState, flags[0]);
   ContextPtr += Op->Flag;
-  uint8_t *MemData = reinterpret_cast<uint8_t*>(ContextPtr);
-  *MemData = Arg;
+
+  if (Op->Flag == 24 /* NZCV */) {
+    uint32_t *MemData = reinterpret_cast<uint32_t*>(ContextPtr);
+    *MemData = Arg;
+  } else {
+    uint8_t *MemData = reinterpret_cast<uint8_t*>(ContextPtr);
+    *MemData = Arg;
+  }
 }
 
 DEF_OP(LoadMem) {
diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp
index 8805637f6..33aedd381 100644
--- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp
+++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp
@@ -1051,12 +1051,20 @@ DEF_OP(FillRegister) {
 DEF_OP(LoadFlag) {
   auto Op = IROp->C<IR::IROp_LoadFlag>();
   auto Dst = GetReg(Node);
-  ldrb(Dst, STATE, offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag);
+
+  if (Op->Flag == 24 /* NZCV */)
+    ldr(Dst.W(), STATE, offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag);
+  else
+    ldrb(Dst, STATE, offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag);
 }
 
 DEF_OP(StoreFlag) {
   auto Op = IROp->C<IR::IROp_StoreFlag>();
-  strb(GetReg(Op->Value.ID()), STATE, offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag);
+
+  if (Op->Flag == 24 /* NZCV */)
+    str(GetReg(Op->Value.ID()).W(), STATE, offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag);
+  else
+    strb(GetReg(Op->Value.ID()), STATE, offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag);
 }
 
 FEXCore::ARMEmitter::ExtendedMemOperand Arm64JITCore::GenerateMemOperand(uint8_t AccessSize,
diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp
index 98a7f3807..775d4d27c 100644
--- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp
+++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp
@@ -601,14 +601,22 @@ DEF_OP(LoadFlag) {
   auto Op = IROp->C<IR::IROp_LoadFlag>();
 
   auto Dst = GetDst<RA_64>(Node);
-  movzx(Dst, byte [STATE + (offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)]);
+
+  if (Op->Flag == 24 /* NZCV */)
+    mov(Dst.cvt32(), dword [STATE + (offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)]);
+  else
+    movzx(Dst, byte [STATE + (offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)]);
 }
 
 DEF_OP(StoreFlag) {
   auto Op = IROp->C<IR::IROp_StoreFlag>();
 
   mov (rax, GetSrc<RA_64>(Op->Value.ID()));
-  mov(byte [STATE + (offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)], al);
+
+  if (Op->Flag == 24 /* NZCV */)
+    mov(dword [STATE + (offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)], eax);
+  else
+    mov(byte [STATE + (offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)], al);
 }
 
 Xbyak::RegExp X86JITCore::GenerateModRM(Xbyak::Reg Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale) const {
diff --git a/External/FEXCore/include/FEXCore/Core/X86Enums.h b/External/FEXCore/include/FEXCore/Core/X86Enums.h
index a28c942cd..21a36f071 100644
--- a/External/FEXCore/include/FEXCore/Core/X86Enums.h
+++ b/External/FEXCore/include/FEXCore/Core/X86Enums.h
@@ -74,6 +74,13 @@ enum X86RegLocation : uint32_t {
   RFLAG_VIP_LOC   = 20,
   RFLAG_ID_LOC    = 21,
 
+  // So we can implement arm64-like flag manipulaton on the interpreter/x86 jit..
+  // SF/ZF/CF/OF packed into a 32-bit word, matching arm64's NZCV structure (not semantics).
+  RFLAG_NZCV_LOC   = 24,
+  RFLAG_NZCV_1_LOC = 25,
+  RFLAG_NZCV_2_LOC = 26,
+  RFLAG_NZCV_3_LOC = 27,
+
 // So we can share flag handling logic, we put x87 flags after RFLAGS
   X87FLAG_BASE    = 32,
   X87FLAG_IE_LOC  = 32,

From 4d13f5d97d3a7a37ffb18414f107dfef2cdd14fc Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Tue, 25 Jul 2023 13:52:11 -0400
Subject: [PATCH 02/23] IR: Add Orlshl op

On arm64, orr (with a shifted register) is maybe fewer cycles and
definitely easier on the RA than bfi (=> fewer moves generated). So,
it's preferred when we know the corresponding bit is 0 in the
destination.

It's not useful on other targets, so it's gated behind a backend feature
bit.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Source/Interface/Core/JIT/Arm64/ALUOps.cpp  | 17 +++++++++++++++++
 .../Source/Interface/Core/JIT/Arm64/JIT.cpp     |  4 +++-
 .../Source/Interface/Core/JIT/Arm64/JITClass.h  |  1 +
 External/FEXCore/Source/Interface/IR/IR.json    |  4 ++++
 .../Source/Interface/IR/Passes/ConstProp.cpp    | 13 +++++++++++++
 .../FEXCore/include/FEXCore/Core/CPUBackend.h   |  1 +
 6 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp
index 3cc7a4b61..c8a4d61b9 100644
--- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp
+++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp
@@ -331,6 +331,23 @@ DEF_OP(Or) {
   }
 }
 
+DEF_OP(Orlshl) {
+  auto Op = IROp->C<IR::IROp_Orlshl>();
+  const uint8_t OpSize = IROp->Size;
+  const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;
+
+  const auto Dst = GetReg(Node);
+  const auto Src1 = GetReg(Op->Src1.ID());
+
+  uint64_t Const;
+  if (IsInlineConstant(Op->Src2, &Const)) {
+    orr(EmitSize, Dst, Src1, Const << Op->BitShift);
+  } else {
+    const auto Src2 = GetReg(Op->Src2.ID());
+    orr(EmitSize, Dst, Src1, Src2, ARMEmitter::ShiftType::LSL, Op->BitShift);
+  }
+}
+
 DEF_OP(And) {
   auto Op = IROp->C<IR::IROp_And>();
   const uint8_t OpSize = IROp->Size;
diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp
index b1ae84127..cf37d1aa6 100644
--- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp
+++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp
@@ -857,6 +857,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry,
         REGISTER_OP(MULH,              MulH);
         REGISTER_OP(UMULH,             UMulH);
         REGISTER_OP(OR,                Or);
+        REGISTER_OP(ORLSHL,            Orlshl);
         REGISTER_OP(AND,               And);
         REGISTER_OP(ANDN,              Andn);
         REGISTER_OP(XOR,               Xor);
@@ -1181,7 +1182,8 @@ fextl::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::ContextImpl *
 
 CPUBackendFeatures GetArm64JITBackendFeatures() {
   return CPUBackendFeatures {
-    .SupportsStaticRegisterAllocation = true
+    .SupportsStaticRegisterAllocation = true,
+    .SupportsShiftedBitwise = true,
   };
 }
 
diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h
index 047883c02..91e881d2f 100644
--- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h
+++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h
@@ -248,6 +248,7 @@ private:
   DEF_OP(MulH);
   DEF_OP(UMulH);
   DEF_OP(Or);
+  DEF_OP(Orlshl);
   DEF_OP(And);
   DEF_OP(Andn);
   DEF_OP(Xor);
diff --git a/External/FEXCore/Source/Interface/IR/IR.json b/External/FEXCore/Source/Interface/IR/IR.json
index 9ad62c729..d0fe371dd 100644
--- a/External/FEXCore/Source/Interface/IR/IR.json
+++ b/External/FEXCore/Source/Interface/IR/IR.json
@@ -782,6 +782,10 @@
         "Desc": ["Integer binary or"
                 ]
       },
+      "GPR = Orlshl GPR:$Src1, GPR:$Src2, u8:$BitShift": {
+        "Desc": ["Integer binary or with logical shift left"
+                ]
+      },
       "GPR = Xor GPR:$Src1, GPR:$Src2": {
         "Desc": ["Integer binary exclusive or"
                 ]
diff --git a/External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp b/External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp
index 239dafa67..5d0dff524 100644
--- a/External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp
+++ b/External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp
@@ -707,6 +707,19 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
       }
     break;
     }
+    case OP_ORLSHL: {
+      auto Op = IROp->CW<IR::IROp_Orlshl>();
+      uint64_t Constant1{};
+      uint64_t Constant2{};
+
+      if (IREmit->IsValueConstant(Op->Header.Args[0], &Constant1) &&
+          IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
+        uint64_t NewConstant = Constant1 | (Constant2 << Op->BitShift);
+        IREmit->ReplaceWithConstant(CodeNode, NewConstant);
+        Changed = true;
+      }
+    break;
+    }
     case OP_XOR: {
       auto Op = IROp->C<IR::IROp_Xor>();
       uint64_t Constant1{};
diff --git a/External/FEXCore/include/FEXCore/Core/CPUBackend.h b/External/FEXCore/include/FEXCore/Core/CPUBackend.h
index c766b7b41..661eef908 100644
--- a/External/FEXCore/include/FEXCore/Core/CPUBackend.h
+++ b/External/FEXCore/include/FEXCore/Core/CPUBackend.h
@@ -35,6 +35,7 @@ namespace CodeSerialize {
 namespace CPU {
   struct CPUBackendFeatures {
     bool SupportsStaticRegisterAllocation = false;
+    bool SupportsShiftedBitwise = false;
   };
 
   class CPUBackend {

From 6e05494ad05e8e14f13380c62c73bfcd9089a1ad Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 28 Jul 2023 12:52:55 -0400
Subject: [PATCH 03/23] IR: Add Orlshr

Similar to Orlshl. This will let us save an instruction in
GetPackedRFLAG.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Source/Interface/Core/JIT/Arm64/ALUOps.cpp  | 17 +++++++++++++++++
 .../Source/Interface/Core/JIT/Arm64/JIT.cpp     |  1 +
 .../Source/Interface/Core/JIT/Arm64/JITClass.h  |  1 +
 External/FEXCore/Source/Interface/IR/IR.json    |  4 ++++
 .../Source/Interface/IR/Passes/ConstProp.cpp    | 13 +++++++++++++
 5 files changed, 36 insertions(+)

diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp
index c8a4d61b9..a69845b3c 100644
--- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp
+++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp
@@ -348,6 +348,23 @@ DEF_OP(Orlshl) {
   }
 }
 
+DEF_OP(Orlshr) {
+  auto Op = IROp->C<IR::IROp_Orlshr>();
+  const uint8_t OpSize = IROp->Size;
+  const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;
+
+  const auto Dst = GetReg(Node);
+  const auto Src1 = GetReg(Op->Src1.ID());
+
+  uint64_t Const;
+  if (IsInlineConstant(Op->Src2, &Const)) {
+    orr(EmitSize, Dst, Src1, Const >> Op->BitShift);
+  } else {
+    const auto Src2 = GetReg(Op->Src2.ID());
+    orr(EmitSize, Dst, Src1, Src2, ARMEmitter::ShiftType::LSR, Op->BitShift);
+  }
+}
+
 DEF_OP(And) {
   auto Op = IROp->C<IR::IROp_And>();
   const uint8_t OpSize = IROp->Size;
diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp
index cf37d1aa6..14600ff7b 100644
--- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp
+++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp
@@ -858,6 +858,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry,
         REGISTER_OP(UMULH,             UMulH);
         REGISTER_OP(OR,                Or);
         REGISTER_OP(ORLSHL,            Orlshl);
+        REGISTER_OP(ORLSHR,            Orlshr);
         REGISTER_OP(AND,               And);
         REGISTER_OP(ANDN,              Andn);
         REGISTER_OP(XOR,               Xor);
diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h
index 91e881d2f..8f92f9f6f 100644
--- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h
+++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h
@@ -249,6 +249,7 @@ private:
   DEF_OP(UMulH);
   DEF_OP(Or);
   DEF_OP(Orlshl);
+  DEF_OP(Orlshr);
   DEF_OP(And);
   DEF_OP(Andn);
   DEF_OP(Xor);
diff --git a/External/FEXCore/Source/Interface/IR/IR.json b/External/FEXCore/Source/Interface/IR/IR.json
index d0fe371dd..620466333 100644
--- a/External/FEXCore/Source/Interface/IR/IR.json
+++ b/External/FEXCore/Source/Interface/IR/IR.json
@@ -786,6 +786,10 @@
         "Desc": ["Integer binary or with logical shift left"
                 ]
       },
+      "GPR = Orlshr GPR:$Src1, GPR:$Src2, u8:$BitShift": {
+        "Desc": ["Integer binary or with logical shift right"
+                ]
+      },
       "GPR = Xor GPR:$Src1, GPR:$Src2": {
         "Desc": ["Integer binary exclusive or"
                 ]
diff --git a/External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp b/External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp
index 5d0dff524..7c5da65cc 100644
--- a/External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp
+++ b/External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp
@@ -720,6 +720,19 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
       }
     break;
     }
+    case OP_ORLSHR: {
+      auto Op = IROp->CW<IR::IROp_Orlshr>();
+      uint64_t Constant1{};
+      uint64_t Constant2{};
+
+      if (IREmit->IsValueConstant(Op->Header.Args[0], &Constant1) &&
+          IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
+        uint64_t NewConstant = Constant1 | (Constant2 >> Op->BitShift);
+        IREmit->ReplaceWithConstant(CodeNode, NewConstant);
+        Changed = true;
+      }
+    break;
+    }
     case OP_XOR: {
       auto Op = IROp->C<IR::IROp_Xor>();
       uint64_t Constant1{};

From 64bac687b55709b5c724b57238a97020454b2185 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Mon, 24 Jul 2023 17:15:04 -0400
Subject: [PATCH 04/23] IR: Add TEST opcode

Maps to arm64 tst, except properly SSA. This will need some RA support
to avoid redundant mrs/msr sequences.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Source/Interface/Core/JIT/Arm64/ALUOps.cpp    | 15 +++++++++++++++
 .../Source/Interface/Core/JIT/Arm64/JIT.cpp       |  2 ++
 .../Source/Interface/Core/JIT/Arm64/JITClass.h    |  1 +
 External/FEXCore/Source/Interface/IR/IR.json      |  4 ++++
 .../Source/Interface/IR/Passes/ConstProp.cpp      | 14 ++++++++++++++
 .../FEXCore/include/FEXCore/Core/CPUBackend.h     |  1 +
 6 files changed, 37 insertions(+)

diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp
index a69845b3c..c1ca65196 100644
--- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp
+++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp
@@ -84,6 +84,21 @@ DEF_OP(Add) {
   }
 }
 
+DEF_OP(TestNZ) {
+  auto Op = IROp->C<IR::IROp_TestNZ>();
+  const uint8_t OpSize = Op->Size;
+
+  LOGMAN_THROW_AA_FMT(OpSize == 4 || OpSize == 8, "Unsupported {} size: {}", __func__, OpSize);
+  const auto EmitSize = OpSize == 8 ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;
+
+  const auto Dst = GetReg(Node);
+  const auto ZeroReg = ARMEmitter::Reg::zr;
+  cmn(EmitSize, GetReg(Op->Src1.ID()), ZeroReg);
+
+  // TODO: Optimize this out
+  mrs(Dst, ARMEmitter::SystemRegister::NZCV);
+}
+
 DEF_OP(Sub) {
   auto Op = IROp->C<IR::IROp_Sub>();
   const uint8_t OpSize = IROp->Size;
diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp
index 14600ff7b..0062098b7 100644
--- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp
+++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp
@@ -845,6 +845,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry,
         REGISTER_OP(INLINEENTRYPOINTOFFSET,  InlineEntrypointOffset);
         REGISTER_OP(CYCLECOUNTER,      CycleCounter);
         REGISTER_OP(ADD,               Add);
+        REGISTER_OP(TESTNZ,            TestNZ);
         REGISTER_OP(SUB,               Sub);
         REGISTER_OP(NEG,               Neg);
         REGISTER_OP(ABS,               Abs);
@@ -1185,6 +1186,7 @@ CPUBackendFeatures GetArm64JITBackendFeatures() {
   return CPUBackendFeatures {
     .SupportsStaticRegisterAllocation = true,
     .SupportsShiftedBitwise = true,
+    .SupportsFlags = true,
   };
 }
 
diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h
index 8f92f9f6f..cf3b45c2b 100644
--- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h
+++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h
@@ -236,6 +236,7 @@ private:
   DEF_OP(InlineEntrypointOffset);
   DEF_OP(CycleCounter);
   DEF_OP(Add);
+  DEF_OP(TestNZ);
   DEF_OP(Sub);
   DEF_OP(Neg);
   DEF_OP(Abs);
diff --git a/External/FEXCore/Source/Interface/IR/IR.json b/External/FEXCore/Source/Interface/IR/IR.json
index 620466333..3583adf3c 100644
--- a/External/FEXCore/Source/Interface/IR/IR.json
+++ b/External/FEXCore/Source/Interface/IR/IR.json
@@ -802,6 +802,10 @@
         "Desc": ["Integer binary AND NOT. Performs the equivalent of Src1 & ~Src2"],
         "DestSize": "std::max<uint8_t>(4, GetOpSize(_Src1))"
       },
+      "GPR = TestNZ u8:$Size, GPR:$Src1": {
+        "Desc": ["Return NZCV for a GPR, setting N and Z accordingly and zeroing C and V"],
+        "DestSize": "4"
+      },
       "GPR = Lshl u8:#Size, GPR:$Src1, GPR:$Src2": {
         "Desc": ["Integer logical shift left"
                 ],
diff --git a/External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp b/External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp
index 7c5da65cc..a4c1cf60d 100644
--- a/External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp
+++ b/External/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp
@@ -690,6 +690,20 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
       }
     break;
     }
+    case OP_TESTNZ: {
+      auto Op = IROp->CW<IR::IROp_TestNZ>();
+      uint64_t Constant1{};
+
+      if (IREmit->IsValueConstant(Op->Header.Args[0], &Constant1)) {
+        bool N = Constant1 & (1ull << ((Op->Size * 8) - 1));
+        bool Z = Constant1 == 0;
+        uint32_t NZVC = (N ? (1u << 31) : 0) | (Z ? (1u << 30) : 0);
+
+        IREmit->ReplaceWithConstant(CodeNode, NZVC);
+        Changed = true;
+      }
+    break;
+    }
     case OP_OR: {
       auto Op = IROp->CW<IR::IROp_Or>();
       uint64_t Constant1{};
diff --git a/External/FEXCore/include/FEXCore/Core/CPUBackend.h b/External/FEXCore/include/FEXCore/Core/CPUBackend.h
index 661eef908..881aee72e 100644
--- a/External/FEXCore/include/FEXCore/Core/CPUBackend.h
+++ b/External/FEXCore/include/FEXCore/Core/CPUBackend.h
@@ -36,6 +36,7 @@ namespace CPU {
   struct CPUBackendFeatures {
     bool SupportsStaticRegisterAllocation = false;
     bool SupportsShiftedBitwise = false;
+    bool SupportsFlags = false;
   };
 
   class CPUBackend {

From 8a049aa0c32390a61cc7d7340d63ce2b281d0b23 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 28 Jul 2023 11:42:09 -0400
Subject: [PATCH 05/23] Context: Make BackendFeatures public

So that the OpcodeDispatcher can check the supported features and emit
code accordingly.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 External/FEXCore/Source/Interface/Context/Context.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/External/FEXCore/Source/Interface/Context/Context.h b/External/FEXCore/Source/Interface/Context/Context.h
index 62a621bb7..7bac3c17e 100644
--- a/External/FEXCore/Source/Interface/Context/Context.h
+++ b/External/FEXCore/Source/Interface/Context/Context.h
@@ -381,6 +381,8 @@ namespace FEXCore::Context {
 
     bool ExitOnHLTEnabled() const { return ExitOnHLT; }
 
+    FEXCore::CPU::CPUBackendFeatures BackendFeatures;
+
   protected:
     void ClearCodeCache(FEXCore::Core::InternalThreadState *Thread);
 
@@ -436,7 +438,6 @@ namespace FEXCore::Context {
 
     std::shared_mutex CustomIRMutex;
     fextl::unordered_map<uint64_t, std::tuple<CustomIREntrypointHandler, void *, void *>> CustomIRHandlers;
-    FEXCore::CPU::CPUBackendFeatures BackendFeatures;
     FEXCore::CPU::DispatcherConfig DispatcherConfig;
   };
 

From 4786aa479cb7c059f945fe961dbb19409b601107 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Tue, 25 Jul 2023 15:03:12 -0400
Subject: [PATCH 06/23] OpcodeDispatcher: Unify SetRFLAG impls

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
index fd5de7de1..2613dfaba 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
@@ -1043,8 +1043,7 @@ private:
 
   template<unsigned BitOffset>
   void SetRFLAG(OrderedNode *Value) {
-    flagsOp = SelectionFlag::Nothing;
-    _StoreFlag(Value, BitOffset);
+    SetRFLAG(Value, BitOffset);
   }
 
   void SetRFLAG(OrderedNode *Value, unsigned BitOffset) {

From 4af42477a741d0f34fdaf93555de2789f493eaf4 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 28 Jul 2023 09:41:39 -0400
Subject: [PATCH 07/23] OpcodeDispatcher: Use GetRFLAG more

We'll add an extra caching layer in a moment so can't call _LoadFlag
directly and expect correct results.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
index 1caf92fb9..e9458c772 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -74,7 +74,7 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) {
     // the byte is allowed to be garbage.
     OrderedNode *Flag = FlagOffset == FEXCore::X86State::RFLAG_PF_LOC ?
                         LoadPF() :
-                        _LoadFlag(FlagOffset);
+                        GetRFLAG(FlagOffset);
 
     Original = _Bfi(4, 1, FlagOffset, Original, Flag);
   }

From 2832afd0f7f4443e299889a3ae0be892b4a162b1 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 28 Jul 2023 10:11:32 -0400
Subject: [PATCH 08/23] OpcodeDispatcher: Calculate deferred flags more

If we read or write NZCV flags we need to call CalculateDeferredFlags on
block boundaries, if only to flush out the cached copy.

Also, when leaving a block we call it to flush out. This is annoyingly
invasive but I don't know of a better way to do this that doesn't
involve rearchitecting the dispatcher.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Interface/Core/OpcodeDispatcher.cpp       | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp
index 19cbf6894..8cdca5893 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp
@@ -145,6 +145,7 @@ void OpDispatchBuilder::SyscallOp(OpcodeArgs) {
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_BLOCK_END) {
     // RIP could have been updated after coming back from the Syscall.
     NewRIP = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, rip));
+    CalculateDeferredFlags();
     _ExitFunction(NewRIP);
   }
 }
@@ -178,6 +179,7 @@ void OpDispatchBuilder::ThunkOp(OpcodeArgs) {
 
   // Store the new stack pointer
   StoreGPRRegister(X86State::REG_RSP, NewSP);
+  CalculateDeferredFlags();
 
   // Store the new RIP
   _ExitFunction(NewRIP);
@@ -240,6 +242,7 @@ void OpDispatchBuilder::RETOp(OpcodeArgs) {
 
   // Store the new stack pointer
   StoreGPRRegister(X86State::REG_RSP, NewSP);
+  CalculateDeferredFlags();
 
   // Store the new RIP
   _ExitFunction(NewRIP);
@@ -304,6 +307,7 @@ void OpDispatchBuilder::IRETOp(OpcodeArgs) {
     StoreGPRRegister(X86State::REG_RSP, SP);
   }
 
+  CalculateDeferredFlags();
   _ExitFunction(NewRIP);
   BlockSetRIP = true;
 }
@@ -313,6 +317,7 @@ void OpDispatchBuilder::CallbackReturnOp(OpcodeArgs) {
   // Store the new RIP
   _CallbackReturn();
   auto NewRIP = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, rip));
+  CalculateDeferredFlags();
   // This ExitFunction won't actually get hit but needs to exist
   _ExitFunction(NewRIP);
   BlockSetRIP = true;
@@ -810,6 +815,7 @@ void OpDispatchBuilder::CALLOp(OpcodeArgs) {
   LOGMAN_THROW_A_FMT(Op->Src[0].IsLiteral(), "Had wrong operand type");
   const uint64_t TargetRIP = Op->PC + Op->InstSize + Op->Src[0].Data.Literal.Value;
 
+  CalculateDeferredFlags();
   if (NextRIP != TargetRIP) {
     // Store the RIP
     _ExitFunction(NewRIP); // If we get here then leave the function now
@@ -840,6 +846,7 @@ void OpDispatchBuilder::CALLAbsoluteOp(OpcodeArgs) {
   _StoreMem(GPRClass, Size, NewSP, ConstantPCReturn, Size);
 
   // Store the RIP
+  CalculateDeferredFlags();
   _ExitFunction(JMPPCOffset); // If we get here then leave the function now
 }
 
@@ -1124,6 +1131,7 @@ void OpDispatchBuilder::CondJUMPOp(OpcodeArgs) {
     Target &= 0xFFFFFFFFU;
   }
 
+  CalculateDeferredFlags();
   auto TrueBlock = JumpTargets.find(Target);
   auto FalseBlock = JumpTargets.find(Op->PC + Op->InstSize);
 
@@ -1269,6 +1277,7 @@ void OpDispatchBuilder::LoopOp(OpcodeArgs) {
     SrcCond = _And(SrcCond, ZF);
   }
 
+  CalculateDeferredFlags();
   auto TrueBlock = JumpTargets.find(Target);
   auto FalseBlock = JumpTargets.find(Op->PC + Op->InstSize);
 
@@ -1337,6 +1346,7 @@ void OpDispatchBuilder::JUMPOp(OpcodeArgs) {
     TargetRIP &= 0xFFFFFFFFU;
   }
 
+  CalculateDeferredFlags();
   // This is just an unconditional relative literal jump
   if (Multiblock) {
     auto JumpBlock = JumpTargets.find(TargetRIP);
@@ -1375,6 +1385,7 @@ void OpDispatchBuilder::JUMPAbsoluteOp(OpcodeArgs) {
   // This uses ModRM to determine its location
   // No way to use this effectively in multiblock
   auto RIPOffset = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1);
+  CalculateDeferredFlags();
 
   // Store the new RIP
   _ExitFunction(RIPOffset);
@@ -3407,12 +3418,14 @@ void OpDispatchBuilder::DAAOp(OpcodeArgs) {
   auto AL = LoadGPRRegister(X86State::REG_RAX, 1);
 
   SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Constant(0));
+  CalculateDeferredFlags();
 
   auto Cond = _Or(AF, _Select(FEXCore::IR::COND_UGT, _And(AL, _Constant(0xF)), _Constant(9), _Constant(1), _Constant(0)));
   auto FalseBlock = CreateNewCodeBlockAfter(GetCurrentBlock());
   auto TrueBlock = CreateNewCodeBlockAfter(FalseBlock);
   auto EndBlock = CreateNewCodeBlockAfter(TrueBlock);
 
+  CalculateDeferredFlags();
   _CondJump(Cond, TrueBlock, FalseBlock);
   SetCurrentCodeBlock(FalseBlock);
   {
@@ -3431,6 +3444,7 @@ void OpDispatchBuilder::DAAOp(OpcodeArgs) {
     // So Or(CF, _Constant(0)) ill mean CF gets updated to the old value in the true case?
     SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Or(CF, NewCF));
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(_Constant(1));
+    CalculateDeferredFlags();
     _Jump(EndBlock);
   }
   SetCurrentCodeBlock(EndBlock);
@@ -3443,6 +3457,7 @@ void OpDispatchBuilder::DAAOp(OpcodeArgs) {
   SetCurrentCodeBlock(FalseBlock);
   {
     SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Constant(0));
+    CalculateDeferredFlags();
     _Jump(EndBlock);
   }
   SetCurrentCodeBlock(TrueBlock);
@@ -3452,6 +3467,7 @@ void OpDispatchBuilder::DAAOp(OpcodeArgs) {
     auto NewAL = _Add(AL, _Constant(0x60));
     StoreGPRRegister(X86State::REG_RAX, NewAL, 1);
     SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Constant(1));
+    CalculateDeferredFlags();
     _Jump(EndBlock);
   }
   SetCurrentCodeBlock(EndBlock);
@@ -3470,12 +3486,14 @@ void OpDispatchBuilder::DASOp(OpcodeArgs) {
   auto AL = LoadGPRRegister(X86State::REG_RAX, 1);
 
   SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Constant(0));
+  CalculateDeferredFlags();
 
   auto Cond = _Or(AF, _Select(FEXCore::IR::COND_UGT, _And(AL, _Constant(0xf)), _Constant(9), _Constant(1), _Constant(0)));
   auto FalseBlock = CreateNewCodeBlockAfter(GetCurrentBlock());
   auto TrueBlock = CreateNewCodeBlockAfter(FalseBlock);
   auto EndBlock = CreateNewCodeBlockAfter(TrueBlock);
 
+  CalculateDeferredFlags();
   _CondJump(Cond, TrueBlock, FalseBlock);
   SetCurrentCodeBlock(FalseBlock);
   {
@@ -3494,6 +3512,7 @@ void OpDispatchBuilder::DASOp(OpcodeArgs) {
     // So Or(CF, _Constant(0)) ill mean CF gets updated to the old value in the true case?
     SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Or(CF, NewCF));
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(_Constant(1));
+    CalculateDeferredFlags();
     _Jump(EndBlock);
   }
   SetCurrentCodeBlock(EndBlock);
@@ -3506,6 +3525,7 @@ void OpDispatchBuilder::DASOp(OpcodeArgs) {
   SetCurrentCodeBlock(FalseBlock);
   {
     SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Constant(0));
+    CalculateDeferredFlags();
     _Jump(EndBlock);
   }
   SetCurrentCodeBlock(TrueBlock);
@@ -3514,6 +3534,7 @@ void OpDispatchBuilder::DASOp(OpcodeArgs) {
     auto NewAL = _Sub(AL, _Constant(0x60));
     StoreGPRRegister(X86State::REG_RAX, NewAL, 1);
     SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Constant(1));
+    CalculateDeferredFlags();
     _Jump(EndBlock);
   }
   SetCurrentCodeBlock(EndBlock);
@@ -3543,6 +3564,7 @@ void OpDispatchBuilder::AAAOp(OpcodeArgs) {
     StoreGPRRegister(X86State::REG_RAX, NewAX, 2);
     SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Constant(0));
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(_Constant(0));
+    CalculateDeferredFlags();
     _Jump(EndBlock);
   }
   SetCurrentCodeBlock(TrueBlock);
@@ -3552,6 +3574,7 @@ void OpDispatchBuilder::AAAOp(OpcodeArgs) {
     StoreGPRRegister(X86State::REG_RAX, Result, 2);
     SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Constant(1));
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(_Constant(1));
+    CalculateDeferredFlags();
     _Jump(EndBlock);
   }
   SetCurrentCodeBlock(EndBlock);
@@ -3576,6 +3599,7 @@ void OpDispatchBuilder::AASOp(OpcodeArgs) {
     StoreGPRRegister(X86State::REG_RAX, NewAX, 2);
     SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Constant(0));
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(_Constant(0));
+    CalculateDeferredFlags();
     _Jump(EndBlock);
   }
   SetCurrentCodeBlock(TrueBlock);
@@ -3586,6 +3610,7 @@ void OpDispatchBuilder::AASOp(OpcodeArgs) {
     StoreGPRRegister(X86State::REG_RAX, Result, 2);
     SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(_Constant(1));
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(_Constant(1));
+    CalculateDeferredFlags();
     _Jump(EndBlock);
   }
   SetCurrentCodeBlock(EndBlock);
@@ -4024,6 +4049,7 @@ void OpDispatchBuilder::CMPSOp(OpcodeArgs) {
       StoreGPRRegister(X86State::REG_RSI, Dest_RSI);
 
       OrderedNode *ZF = GetRFLAG(FEXCore::X86State::RFLAG_ZF_LOC);
+      CalculateDeferredFlags();
       InternalCondJump = _CondJump(ZF, {REPE ? COND_NEQ : COND_EQ});
 
       // Jump back to the start if we have more work to do
@@ -4242,6 +4268,7 @@ void OpDispatchBuilder::SCASOp(OpcodeArgs) {
       StoreGPRRegister(X86State::REG_RDI, TailDest_RDI);
 
       OrderedNode *ZF = GetRFLAG(FEXCore::X86State::RFLAG_ZF_LOC);
+      CalculateDeferredFlags();
       InternalCondJump = _CondJump(ZF, {REPE ? COND_NEQ : COND_EQ});
 
       // Jump back to the start if we have more work to do
@@ -4675,6 +4702,7 @@ void OpDispatchBuilder::CMPXCHGPairOp(OpcodeArgs) {
 
   // Set ZF
   SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(ZFResult);
+  CalculateDeferredFlags();
 
   auto CondJump = _CondJump(ZFResult);
 
@@ -4739,6 +4767,7 @@ void OpDispatchBuilder::Finalize() {
 
     // We haven't emitted. Dump out to the dispatcher
     SetCurrentCodeBlock(Handler.second.BlockEntry);
+    CalculateDeferredFlags();
     _ExitFunction(_EntrypointOffset(Handler.first - Entry, GPRSize));
   }
 }
@@ -5459,6 +5488,7 @@ void OpDispatchBuilder::INTOp(OpcodeArgs) {
 
   if (Op->OP == 0xCE) { // Conditional to only break if Overflow == 1
     auto Flag = GetRFLAG(FEXCore::X86State::RFLAG_OF_LOC);
+    CalculateDeferredFlags();
 
     // If condition doesn't hold then keep going
     auto CondJump = _CondJump(Flag, {COND_EQ});

From 0fb390388974aea4c7f5d8b42aa59317fa24b84f Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 28 Jul 2023 09:43:29 -0400
Subject: [PATCH 09/23] OpcodeDispatcher: Pack NZCV flags together

Later, this will let us take advantage of the arm64 flags. For
now, this just turns some strb's into bfi's for dubious benefit.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Source/Interface/Core/OpcodeDispatcher.h  | 55 ++++++++++++++++++-
 .../Interface/Core/OpcodeDispatcher/Flags.cpp |  9 +++
 2 files changed, 62 insertions(+), 2 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
index 2613dfaba..ce10544d9 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
@@ -83,6 +83,10 @@ public:
 
   void StartNewBlock() {
     flagsOp = SelectionFlag::Nothing;
+
+    // If we loaded flags but didn't change them, invalidate the cached copy and move on.
+    // Changes get stored out by CalculateDeferredFlags.
+    CachedNZCV = nullptr;
   }
 
   bool FinishOp(uint64_t NextRIP, bool LastOp) {
@@ -834,6 +838,21 @@ private:
   OrderedNode* flagsOpDestSigned{};
   OrderedNode* flagsOpSrcSigned{};
 
+  static bool IsNZCV(unsigned BitOffset) {
+    switch (BitOffset) {
+      case FEXCore::X86State::RFLAG_CF_LOC:
+      case FEXCore::X86State::RFLAG_ZF_LOC:
+      case FEXCore::X86State::RFLAG_SF_LOC:
+      case FEXCore::X86State::RFLAG_OF_LOC:
+        return true;
+
+      default:
+        return false;
+    }
+  }
+
+  OrderedNode* CachedNZCV = {};
+
   fextl::map<uint64_t, JumpTargetInfo> JumpTargets;
   bool HandledLock{false};
   bool DecodeFailure{false};
@@ -1041,6 +1060,31 @@ private:
   [[nodiscard]] uint32_t GetDstBitSize(X86Tables::DecodedOp Op) const;
   [[nodiscard]] uint32_t GetSrcBitSize(X86Tables::DecodedOp Op) const;
 
+  static inline constexpr unsigned IndexNZCV(unsigned BitOffset) {
+    switch (BitOffset) {
+      case FEXCore::X86State::RFLAG_OF_LOC: return 28;
+      case FEXCore::X86State::RFLAG_CF_LOC: return 29;
+      case FEXCore::X86State::RFLAG_ZF_LOC: return 30;
+      case FEXCore::X86State::RFLAG_SF_LOC: return 31;
+      default: FEX_UNREACHABLE;
+    }
+  }
+
+  OrderedNode *GetNZCV() {
+    if (!CachedNZCV)
+      CachedNZCV = _LoadFlag(FEXCore::X86State::RFLAG_NZCV_LOC);
+
+    return CachedNZCV;
+  }
+
+  void SetNZCV(OrderedNode *Value) {
+    CachedNZCV = Value;
+  }
+
+  OrderedNode *InsertNZCV(OrderedNode *NZCV, unsigned BitOffset, OrderedNode *Value) {
+    return _Bfi(4, 1, IndexNZCV(BitOffset), NZCV, Value);
+  }
+
   template<unsigned BitOffset>
   void SetRFLAG(OrderedNode *Value) {
     SetRFLAG(Value, BitOffset);
@@ -1048,11 +1092,18 @@ private:
 
   void SetRFLAG(OrderedNode *Value, unsigned BitOffset) {
     flagsOp = SelectionFlag::Nothing;
-    _StoreFlag(Value, BitOffset);
+
+    if (IsNZCV(BitOffset))
+      SetNZCV(InsertNZCV(GetNZCV(), BitOffset, Value));
+    else
+      _StoreFlag(Value, BitOffset);
   }
 
   OrderedNode *GetRFLAG(unsigned BitOffset) {
-    return _LoadFlag(BitOffset);
+    if (IsNZCV(BitOffset))
+      return _Bfe(1, 1, IndexNZCV(BitOffset), GetNZCV());
+    else
+      return _LoadFlag(BitOffset);
   }
 
   OrderedNode *SelectCC(uint8_t OP, OrderedNode *TrueValue, OrderedNode *FalseValue);
diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
index e9458c772..3d0d79545 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -138,6 +138,10 @@ void OpDispatchBuilder::CalculatePF(OrderedNode *Res, OrderedNode *condition) {
 void OpDispatchBuilder::CalculateDeferredFlags(uint32_t FlagsToCalculateMask) {
   if (CurrentDeferredFlags.Type == FlagsGenerationType::TYPE_NONE) {
     // Nothing to do
+    if (CachedNZCV)
+      _StoreFlag(CachedNZCV, FEXCore::X86State::RFLAG_NZCV_LOC);
+
+    CachedNZCV = nullptr;
     return;
   }
 
@@ -320,6 +324,11 @@ void OpDispatchBuilder::CalculateDeferredFlags(uint32_t FlagsToCalculateMask) {
 
   // Done calculating
   CurrentDeferredFlags.Type = FlagsGenerationType::TYPE_NONE;
+
+  if (CachedNZCV)
+    _StoreFlag(CachedNZCV, FEXCore::X86State::RFLAG_NZCV_LOC);
+
+  CachedNZCV = nullptr;
 }
 
 void OpDispatchBuilder::CalculateFlags_ADC(uint8_t SrcSize, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2, OrderedNode *CF) {

From 52ce027e9b8fe9437ce67d46f5c3f428fdd49e8b Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Mon, 31 Jul 2023 13:54:24 -0400
Subject: [PATCH 10/23] OpcodeDispatcher: Set N flag more efficiently

We can set N more efficiently with some bit math, and zero ZCV at the
same time. In the future we'll be able to use TST for this to make it
even faster.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Source/Interface/Core/OpcodeDispatcher.h  |  32 +++++
 .../Interface/Core/OpcodeDispatcher/Flags.cpp | 123 ++++--------------
 2 files changed, 58 insertions(+), 97 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
index ce10544d9..13138c4b4 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
@@ -1081,6 +1081,38 @@ private:
     CachedNZCV = Value;
   }
 
+  void SetN_ZeroZCV(unsigned SrcSize, OrderedNode *Res) {
+    static_assert(IndexNZCV(FEXCore::X86State::RFLAG_SF_LOC) == 31);
+
+    unsigned NBit = 31;
+    unsigned SignBit = (SrcSize * 8) - 1;
+
+    OrderedNode *Shifted;
+
+    // Shift the sign bit into the N bit
+    if (SignBit > NBit)
+      Shifted = _Ashr(Res, _Constant(SignBit - NBit));
+    else if (SignBit < NBit)
+      Shifted = _Lshl(Res, _Constant(NBit - SignBit));
+    else
+      Shifted = Res;
+
+    // Mask off just the N bit, which now equals the sign bit
+    CachedNZCV = _And(Shifted, _Constant(1u << NBit));
+  }
+
+  // TODO: Replace with a TST instruction
+  void SetNZ_ZeroCV(unsigned SrcSize, OrderedNode *Res) {
+    // N
+    SetN_ZeroZCV(SrcSize, Res);
+
+    // Z
+    auto Zero = _Constant(0);
+    auto One = _Constant(1);
+    auto SelectOp = _Select(FEXCore::IR::COND_EQ, Res, Zero, One, Zero);
+    SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
+  }
+
   OrderedNode *InsertNZCV(OrderedNode *NZCV, unsigned BitOffset, OrderedNode *Value) {
     return _Bfi(4, 1, IndexNZCV(BitOffset), NZCV, Value);
   }
diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
index 3d0d79545..8ecfa9e4a 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -341,20 +341,10 @@ void OpDispatchBuilder::CalculateFlags_ADC(uint8_t SrcSize, OrderedNode *Res, Or
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(AFRes);
   }
 
-  // SF
-  {
-    auto SignOp = _Bfe(1, SrcSize * 8 - 1, Res);
-    SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignOp);
-  }
-
   CalculatePF(Res);
 
-  // ZF
-  {
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ,
-        Res, Zero, One, Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
-  }
+  // SF/ZF
+  SetNZ_ZeroCV(SrcSize, Res);
 
   // CF
   // Unsigned
@@ -380,20 +370,10 @@ void OpDispatchBuilder::CalculateFlags_SBB(uint8_t SrcSize, OrderedNode *Res, Or
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(AFRes);
   }
 
-  // SF
-  {
-    auto SignOp = _Bfe(1, SrcSize * 8 - 1, Res);
-    SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignOp);
-  }
-
   CalculatePF(Res);
 
-  // ZF
-  {
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ,
-        Res, Zero, One, Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
-  }
+  // SF/ZF
+  SetNZ_ZeroCV(SrcSize, Res);
 
   // CF
   // Unsigned
@@ -426,20 +406,13 @@ void OpDispatchBuilder::CalculateFlags_SUB(uint8_t SrcSize, OrderedNode *Res, Or
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(AFRes);
   }
 
-  // SF
-  {
-    auto SignOp = _Bfe(1, SrcSize * 8 - 1, Res);
-    SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignOp);
-  }
-
   CalculatePF(Res);
 
-  // ZF
-  {
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ,
-        Res, Zero, One, Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
-  }
+  // Stash CF before zeroing it
+  auto OldCF = GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC);
+
+  // SF/ZF
+  SetNZ_ZeroCV(SrcSize, Res);
 
   // CF
   if (UpdateCF) {
@@ -447,7 +420,10 @@ void OpDispatchBuilder::CalculateFlags_SUB(uint8_t SrcSize, OrderedNode *Res, Or
         Src1, Src2, One, Zero);
 
     SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(SelectOp);
+  } else {
+    SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(OldCF);
   }
+
   // OF
   {
     auto XorOp1 = _Xor(Src1, Src2);
@@ -471,25 +447,21 @@ void OpDispatchBuilder::CalculateFlags_ADD(uint8_t SrcSize, OrderedNode *Res, Or
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(AFRes);
   }
 
-  // SF
-  {
-    auto SignOp = _Bfe(1, SrcSize * 8 - 1, Res);
-    SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignOp);
-  }
-
   CalculatePF(Res);
 
-  // ZF
-  {
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ,
-        Res, Zero, One, Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
-  }
+  // Stash CF before zeroing it
+  auto OldCF = GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC);
+
+  // SF/ZF
+  SetNZ_ZeroCV(SrcSize, Res);
+
   // CF
   if (UpdateCF) {
     auto SelectOp = _Select(FEXCore::IR::COND_ULT, Res, Src2, One, Zero);
 
     SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(SelectOp);
+  } else {
+    SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(OldCF);
   }
 
   CalculateOF_Add(SrcSize, Res, Src1, Src2);
@@ -549,7 +521,6 @@ void OpDispatchBuilder::CalculateFlags_UMUL(OrderedNode *High) {
 
 void OpDispatchBuilder::CalculateFlags_Logical(uint8_t SrcSize, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2) {
   auto Zero = _Constant(0);
-  auto One = _Constant(1);
   // AF
   {
     // Undefined
@@ -557,26 +528,10 @@ void OpDispatchBuilder::CalculateFlags_Logical(uint8_t SrcSize, OrderedNode *Res
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(Zero);
   }
 
-  // SF
-  {
-    auto SignOp = _Bfe(1, SrcSize * 8 - 1, Res);
-    SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignOp);
-  }
-
   CalculatePF(Res);
 
-  // ZF
-  {
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ,
-        Res, Zero, One, Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
-  }
-
-  // CF/OF
-  {
-    SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(Zero);
-  }
+  // SF/ZF/CF/OF
+  SetNZ_ZeroCV(SrcSize, Res);
 }
 
 #define COND_FLAG_SET(cond, flag, newflag) \
@@ -1047,7 +1002,8 @@ void OpDispatchBuilder::CalculateFlags_BLSI(uint8_t SrcSize, OrderedNode *Src) {
   auto Zero = _Constant(0);
   auto One = _Constant(1);
 
-  SetRFLAG<X86State::RFLAG_OF_LOC>(Zero);
+  SetNZ_ZeroCV(SrcSize, Src);
+
   SetRFLAG<X86State::RFLAG_AF_LOC>(Zero);
   if (CTX->Config.ABINoPF) {
     _InvalidateFlags(1UL << X86State::RFLAG_PF_LOC);
@@ -1055,14 +1011,6 @@ void OpDispatchBuilder::CalculateFlags_BLSI(uint8_t SrcSize, OrderedNode *Src) {
     SetRFLAG<X86State::RFLAG_PF_LOC>(Zero);
   }
 
-  // ZF
-  {
-    auto ZFOp = _Select(IR::COND_EQ,
-                        Src, Zero,
-                        One, Zero);
-    SetRFLAG<X86State::RFLAG_ZF_LOC>(ZFOp);
-  }
-
   // CF
   {
     auto CFOp = _Select(IR::COND_EQ,
@@ -1070,12 +1018,6 @@ void OpDispatchBuilder::CalculateFlags_BLSI(uint8_t SrcSize, OrderedNode *Src) {
                         Zero, One);
     SetRFLAG<X86State::RFLAG_CF_LOC>(CFOp);
   }
-
-  // SF
-  {
-    auto SignOp = _Bfe(1, SrcSize * 8 - 1, Src);
-    SetRFLAG<X86State::RFLAG_SF_LOC>(SignOp);
-  }
 }
 
 void OpDispatchBuilder::CalculateFlags_BLSMSK(OrderedNode *Src) {
@@ -1104,7 +1046,8 @@ void OpDispatchBuilder::CalculateFlags_BLSR(uint8_t SrcSize, OrderedNode *Result
   auto Zero = _Constant(0);
   auto One = _Constant(1);
 
-  SetRFLAG<X86State::RFLAG_OF_LOC>(Zero);
+  SetNZ_ZeroCV(SrcSize, Result);
+
   SetRFLAG<X86State::RFLAG_AF_LOC>(Zero);
   if (CTX->Config.ABINoPF) {
     _InvalidateFlags(1UL << X86State::RFLAG_PF_LOC);
@@ -1112,14 +1055,6 @@ void OpDispatchBuilder::CalculateFlags_BLSR(uint8_t SrcSize, OrderedNode *Result
     SetRFLAG<X86State::RFLAG_PF_LOC>(Zero);
   }
 
-  // ZF
-  {
-    auto ZFOp = _Select(IR::COND_EQ,
-                        Result, Zero,
-                        One, Zero);
-    SetRFLAG<X86State::RFLAG_ZF_LOC>(ZFOp);
-  }
-
   // CF
   {
     auto CFOp = _Select(IR::COND_EQ,
@@ -1127,12 +1062,6 @@ void OpDispatchBuilder::CalculateFlags_BLSR(uint8_t SrcSize, OrderedNode *Result
                         Zero, One);
     SetRFLAG<X86State::RFLAG_CF_LOC>(CFOp);
   }
-
-  // SF
-  {
-    auto SignOp = _Bfe(1, SrcSize * 8 - 1, Result);
-    SetRFLAG<X86State::RFLAG_SF_LOC>(SignOp);
-  }
 }
 
 void OpDispatchBuilder::CalculateFlags_POPCOUNT(OrderedNode *Src) {

From 747c5e17f88ed369ba84dfddb701e61290435306 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 28 Jul 2023 10:41:59 -0400
Subject: [PATCH 11/23] OpcodeDispatcher: Add ZeroNZCV helper

In some cases we just want to insert in one bit at a time, add a helper
to zero the 4 flags together so we can avoid the extra RMW cycle at the
beginning.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Source/Interface/Core/OpcodeDispatcher.h  |  4 +++
 .../Interface/Core/OpcodeDispatcher/Flags.cpp | 33 +++++++------------
 2 files changed, 15 insertions(+), 22 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
index 13138c4b4..b9f3d3494 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
@@ -1081,6 +1081,10 @@ private:
     CachedNZCV = Value;
   }
 
+  void ZeroNZCV() {
+    CachedNZCV = _Constant(0);
+  }
+
   void SetN_ZeroZCV(unsigned SrcSize, OrderedNode *Res) {
     static_assert(IndexNZCV(FEXCore::X86State::RFLAG_SF_LOC) == 31);
 
diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
index 8ecfa9e4a..bbd8e9af8 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -476,8 +476,7 @@ void OpDispatchBuilder::CalculateFlags_MUL(uint8_t SrcSize, OrderedNode *Res, Or
   {
     SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(Zero);
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(Zero);
+    ZeroNZCV();
   }
 
   // CF/OF
@@ -502,9 +501,8 @@ void OpDispatchBuilder::CalculateFlags_UMUL(OrderedNode *High) {
   // Undefined
   {
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(Zero);
     SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(Zero);
+    ZeroNZCV();
   }
 
   // CF/OF
@@ -964,14 +962,10 @@ void OpDispatchBuilder::CalculateFlags_BEXTR(OrderedNode *Src) {
   //
   // CF and OF are defined as being set to zero
   //
-  SetRFLAG<X86State::RFLAG_CF_LOC>(Zero);
-  SetRFLAG<X86State::RFLAG_OF_LOC>(Zero);
-
   // Every other flag is considered undefined after a
   // BEXTR instruction, but we opt to reliably clear them.
   //
-  SetRFLAG<X86State::RFLAG_AF_LOC>(Zero);
-  SetRFLAG<X86State::RFLAG_SF_LOC>(Zero);
+  ZeroNZCV();
 
   // PF
   if (CTX->Config.ABINoPF) {
@@ -1072,12 +1066,11 @@ void OpDispatchBuilder::CalculateFlags_POPCOUNT(OrderedNode *Src) {
       _Constant(1), Zero);
 
   // Set flags
-  SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(Zero);
+  ZeroNZCV();
+  SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(ZFResult);
+
   SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(Zero);
   SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(Zero);
-  SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(ZFResult);
-  SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(Zero);
-  SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(Zero);
 }
 
 void OpDispatchBuilder::CalculateFlags_BZHI(uint8_t SrcSize, OrderedNode *Result, OrderedNode *Src) {
@@ -1110,16 +1103,12 @@ void OpDispatchBuilder::CalculateFlags_BZHI(uint8_t SrcSize, OrderedNode *Result
                         One, Zero);
     SetRFLAG<X86State::RFLAG_CF_LOC>(CFOp);
   }
-
-  // SF
-  {
-    auto SignOp = _Bfe(1, SrcSize * 8 - 1, Result);
-    SetRFLAG<X86State::RFLAG_SF_LOC>(SignOp);
-  }
 }
 
 void OpDispatchBuilder::CalculateFlags_TZCNT(OrderedNode *Src) {
   // OF, SF, AF, PF all undefined
+  ZeroNZCV();
+
   auto Zero = _Constant(0);
   auto ZFResult = _Select(FEXCore::IR::COND_EQ,
       Src,  Zero,
@@ -1132,6 +1121,7 @@ void OpDispatchBuilder::CalculateFlags_TZCNT(OrderedNode *Src) {
 
 void OpDispatchBuilder::CalculateFlags_LZCNT(uint8_t SrcSize, OrderedNode *Src) {
   // OF, SF, AF, PF all undefined
+  ZeroNZCV();
 
   auto Zero = _Constant(0);
   auto ZFResult = _Select(FEXCore::IR::COND_EQ,
@@ -1145,6 +1135,7 @@ void OpDispatchBuilder::CalculateFlags_LZCNT(uint8_t SrcSize, OrderedNode *Src)
 
 void OpDispatchBuilder::CalculateFlags_BITSELECT(OrderedNode *Src) {
   // OF, SF, AF, PF, CF all undefined
+  ZeroNZCV();
 
   auto ZeroConst = _Constant(0);
   auto OneConst = _Constant(1);
@@ -1160,11 +1151,9 @@ void OpDispatchBuilder::CalculateFlags_BITSELECT(OrderedNode *Src) {
 void OpDispatchBuilder::CalculateFlags_RDRAND(OrderedNode *Src) {
   // OF, SF, ZF, AF, PF all zero
   // CF is set to the incoming source
+  ZeroNZCV();
 
   auto ZeroConst = _Constant(0);
-  SetRFLAG<X86State::RFLAG_OF_LOC>(ZeroConst);
-  SetRFLAG<X86State::RFLAG_SF_LOC>(ZeroConst);
-  SetRFLAG<X86State::RFLAG_ZF_LOC>(ZeroConst);
   SetRFLAG<X86State::RFLAG_AF_LOC>(ZeroConst);
   SetRFLAG<X86State::RFLAG_PF_LOC>(ZeroConst);
   SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(Src);

From 8244c7f2a6f02ce61dbbdf8f5cdb81c8fb9fe6db Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 28 Jul 2023 10:38:17 -0400
Subject: [PATCH 12/23] OpcodeDispatcher: Use orlshl when possible

If we can prove that a flag bit could not possibly be set, we can use
orlshl rather than bfi, which can be more efficient.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Source/Interface/Core/OpcodeDispatcher.h  | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
index b9f3d3494..fc17ab1b6 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
@@ -852,6 +852,7 @@ private:
   }
 
   OrderedNode* CachedNZCV = {};
+  uint32_t PossiblySetNZCVBits = 0;
 
   fextl::map<uint64_t, JumpTargetInfo> JumpTargets;
   bool HandledLock{false};
@@ -1071,9 +1072,13 @@ private:
   }
 
   OrderedNode *GetNZCV() {
-    if (!CachedNZCV)
+    if (!CachedNZCV) {
       CachedNZCV = _LoadFlag(FEXCore::X86State::RFLAG_NZCV_LOC);
 
+      // We don't know what's set
+      PossiblySetNZCVBits = ~0;
+    }
+
     return CachedNZCV;
   }
 
@@ -1083,6 +1088,7 @@ private:
 
   void ZeroNZCV() {
     CachedNZCV = _Constant(0);
+    PossiblySetNZCVBits = 0;
   }
 
   void SetN_ZeroZCV(unsigned SrcSize, OrderedNode *Res) {
@@ -1103,6 +1109,7 @@ private:
 
     // Mask off just the N bit, which now equals the sign bit
     CachedNZCV = _And(Shifted, _Constant(1u << NBit));
+    PossiblySetNZCVBits = (1u << NBit);
   }
 
   // TODO: Replace with a TST instruction
@@ -1118,7 +1125,17 @@ private:
   }
 
   OrderedNode *InsertNZCV(OrderedNode *NZCV, unsigned BitOffset, OrderedNode *Value) {
-    return _Bfi(4, 1, IndexNZCV(BitOffset), NZCV, Value);
+    unsigned Bit = IndexNZCV(BitOffset);
+
+    uint32_t SetBits = PossiblySetNZCVBits;
+    PossiblySetNZCVBits |= (1u << Bit);
+
+    if (SetBits == 0)
+      return _Lshl(Value, _Constant(Bit));
+    else if (CTX->BackendFeatures.SupportsShiftedBitwise && (SetBits & (1u << Bit)) == 0)
+      return _Orlshl(NZCV, Value, Bit);
+    else
+      return _Bfi(4, 1, Bit, NZCV, Value);
   }
 
   template<unsigned BitOffset>

From e285c7c9a019d736995a2dba4fc499bfc40e95de Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 28 Jul 2023 11:59:29 -0400
Subject: [PATCH 13/23] OpcodeDispatcher: Use TEST when possible

Faster sign/negate testing for 32-bit/64-bit inputs. This could maybe be
extended to 8/16-bit if we have FlagM but that's for later.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Source/Interface/Core/OpcodeDispatcher.h  | 22 ++++++++++++-------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
index fc17ab1b6..58a5d822d 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
@@ -1112,16 +1112,22 @@ private:
     PossiblySetNZCVBits = (1u << NBit);
   }
 
-  // TODO: Replace with a TST instruction
   void SetNZ_ZeroCV(unsigned SrcSize, OrderedNode *Res) {
-    // N
-    SetN_ZeroZCV(SrcSize, Res);
+    // The TestNZ opcode does this operation natively for 32-bit or 64-bit.
+    // Otherwise we can implement the functionality ourselves with some bit math.
+    if (CTX->BackendFeatures.SupportsFlags && SrcSize >= 4) {
+      CachedNZCV = _TestNZ(SrcSize, Res);
+      PossiblySetNZCVBits = (1u << 31) | (1u << 30);
+    } else {
+      // N
+      SetN_ZeroZCV(SrcSize, Res);
 
-    // Z
-    auto Zero = _Constant(0);
-    auto One = _Constant(1);
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ, Res, Zero, One, Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
+      // Z
+      auto Zero = _Constant(0);
+      auto One = _Constant(1);
+      auto SelectOp = _Select(FEXCore::IR::COND_EQ, Res, Zero, One, Zero);
+      SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
+    }
   }
 
   OrderedNode *InsertNZCV(OrderedNode *NZCV, unsigned BitOffset, OrderedNode *Value) {

From 1e4456ec409a736a85b92e65d4329a3d5b6c9a80 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 28 Jul 2023 12:41:49 -0400
Subject: [PATCH 14/23] OpcodeDispatcher: Use orlshl in GetPackedRFLAG

Saves some moves.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
index bbd8e9af8..6c57ea186 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -76,7 +76,10 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) {
                         LoadPF() :
                         GetRFLAG(FlagOffset);
 
-    Original = _Bfi(4, 1, FlagOffset, Original, Flag);
+    if (CTX->BackendFeatures.SupportsShiftedBitwise)
+      Original = _Orlshl(Original, Flag, FlagOffset);
+    else
+      Original = _Bfi(4, 1, FlagOffset, Original, Flag);
   }
   return Original;
 }

From d5026f5815319164b7f3b65d65fad81a7513d457 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 28 Jul 2023 12:50:56 -0400
Subject: [PATCH 15/23] OpcodeDispatcher: Handle SF/ZF together for
 GetPackedRFLAG

They're together on both x86 and arm64, so this is faster if we're
getting both.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Interface/Core/OpcodeDispatcher/Flags.cpp | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
index 6c57ea186..d8b325716 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -64,12 +64,25 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) {
   CalculateDeferredFlags();
 
   OrderedNode *Original = _Constant((1U << X86State::RFLAG_RESERVED_LOC) & FlagsMask ? 2 : 0);
+
+  // SF/ZF and N/Z are together on both arm64 and x86_64, so we special case that.
+  bool GetNZ = (FlagsMask & (1 << FEXCore::X86State::RFLAG_SF_LOC)) &&
+               (FlagsMask & (1 << FEXCore::X86State::RFLAG_ZF_LOC)) &&
+               CTX->BackendFeatures.SupportsShiftedBitwise;
+
+
   for (size_t i = 0; i < FlagOffsets.size(); ++i) {
     const auto FlagOffset = FlagOffsets[i];
     if (!((1U << FlagOffset) & FlagsMask)) {
       continue;
     }
 
+    if (GetNZ && (FlagOffset == FEXCore::X86State::RFLAG_SF_LOC ||
+                  FlagOffset == FEXCore::X86State::RFLAG_ZF_LOC)) {
+      // Already handled
+      continue;
+    }
+
     // Note that the Bfi only considers the bottom bit of the flag, the rest of
     // the byte is allowed to be garbage.
     OrderedNode *Flag = FlagOffset == FEXCore::X86State::RFLAG_PF_LOC ?
@@ -81,6 +94,15 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) {
     else
       Original = _Bfi(4, 1, FlagOffset, Original, Flag);
   }
+
+  // OR in the SF/ZF flags at the end, allowing the lshr to fold with the OR
+  if (GetNZ) {
+    static_assert(FEXCore::X86State::RFLAG_SF_LOC == (FEXCore::X86State::RFLAG_ZF_LOC + 1));
+    auto NZCV = GetNZCV();
+    auto NZ = _And(NZCV, _Constant(0b11u << 30));
+    Original = _Orlshr(Original, NZ, 31 - FEXCore::X86State::RFLAG_SF_LOC);
+  }
+
   return Original;
 }
 

From cec98637c97484db81cdfde8677ea82d3c507401 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 28 Jul 2023 13:03:42 -0400
Subject: [PATCH 16/23] OpcodeDispatcher: Avoid extra OR in GetPackedRFLAG

We know that bit 0 is CF, so we can do CF first and then avoid setting
Original to 2 (for reserved) with a silly `or xzr, #2` instruction.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Interface/Core/OpcodeDispatcher/Flags.cpp    | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
index d8b325716..20da8e9fa 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -63,13 +63,18 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) {
   // Calculate flags early.
   CalculateDeferredFlags();
 
-  OrderedNode *Original = _Constant((1U << X86State::RFLAG_RESERVED_LOC) & FlagsMask ? 2 : 0);
+  OrderedNode *Original = _Constant(0);
 
   // SF/ZF and N/Z are together on both arm64 and x86_64, so we special case that.
   bool GetNZ = (FlagsMask & (1 << FEXCore::X86State::RFLAG_SF_LOC)) &&
                (FlagsMask & (1 << FEXCore::X86State::RFLAG_ZF_LOC)) &&
                CTX->BackendFeatures.SupportsShiftedBitwise;
 
+  // Handle CF first, since it's at bit 0 and hence doesn't need shift or OR.
+  if (FlagsMask & (1 << FEXCore::X86State::RFLAG_CF_LOC)) {
+    static_assert(FEXCore::X86State::RFLAG_CF_LOC == 0);
+    Original = GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC);
+  }
 
   for (size_t i = 0; i < FlagOffsets.size(); ++i) {
     const auto FlagOffset = FlagOffsets[i];
@@ -77,8 +82,9 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) {
       continue;
     }
 
-    if (GetNZ && (FlagOffset == FEXCore::X86State::RFLAG_SF_LOC ||
-                  FlagOffset == FEXCore::X86State::RFLAG_ZF_LOC)) {
+    if ((GetNZ && (FlagOffset == FEXCore::X86State::RFLAG_SF_LOC ||
+                   FlagOffset == FEXCore::X86State::RFLAG_ZF_LOC)) ||
+        FlagOffset == FEXCore::X86State::RFLAG_CF_LOC) {
       // Already handled
       continue;
     }
@@ -103,6 +109,10 @@ OrderedNode *OpDispatchBuilder::GetPackedRFLAG(uint32_t FlagsMask) {
     Original = _Orlshr(Original, NZ, 31 - FEXCore::X86State::RFLAG_SF_LOC);
   }
 
+  // The constant is OR'ed in at the end, to avoid a pointless or xzr, #2.
+  if ((1U << X86State::RFLAG_RESERVED_LOC) & FlagsMask)
+    Original = _Or(Original, _Constant(2));
+
   return Original;
 }
 

From a4ea792d0327740f21c69a05550c1e7696b8f994 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Fri, 28 Jul 2023 13:07:56 -0400
Subject: [PATCH 17/23] OpcodeDispatcher: Optimize GetRFLAG of definitely-0
 flag

We can just return zero, no need to do a pointless Bfe. Saves yet
another instruction for GetPackedRLAG in a test I'm looking at.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../FEXCore/Source/Interface/Core/OpcodeDispatcher.h   | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
index 58a5d822d..8d03c9ab2 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
@@ -1159,10 +1159,14 @@ private:
   }
 
   OrderedNode *GetRFLAG(unsigned BitOffset) {
-    if (IsNZCV(BitOffset))
-      return _Bfe(1, 1, IndexNZCV(BitOffset), GetNZCV());
-    else
+    if (IsNZCV(BitOffset)) {
+      if (!CachedNZCV || (PossiblySetNZCVBits & (1u << IndexNZCV(BitOffset))))
+        return _Bfe(1, 1, IndexNZCV(BitOffset), GetNZCV());
+      else
+        return _Constant(0);
+    } else {
       return _LoadFlag(BitOffset);
+    }
   }
 
   OrderedNode *SelectCC(uint8_t OP, OrderedNode *TrueValue, OrderedNode *FalseValue);

From 7c523752671467633d05124d333a526739dd56f5 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Wed, 2 Aug 2023 13:27:02 -0400
Subject: [PATCH 18/23] OpcodeDispatcher: Optimize (U)MUL flags calculation

csel the value we want directly. Saves 2 instructions. Could do better
still but hey, progress is progress. Currently looks like:

2995: 0x0000ffff6c600040  320407f5		orr w21, wzr, #0x30000000
2995: 0x0000ffff6c600044  f10000df		cmp x6, #0x0 (0)
2995: 0x0000ffff6c600048  9a950294		csel x20, x20, x21, eq
2995: 0x0000ffff6c60004c  b902db94		str w20, [x28, #728]

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Interface/Core/OpcodeDispatcher/Flags.cpp   | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
index 20da8e9fa..9f1863a99 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -504,14 +504,12 @@ void OpDispatchBuilder::CalculateFlags_ADD(uint8_t SrcSize, OrderedNode *Res, Or
 
 void OpDispatchBuilder::CalculateFlags_MUL(uint8_t SrcSize, OrderedNode *Res, OrderedNode *High) {
   auto Zero = _Constant(0);
-  auto One = _Constant(1);
 
   // PF/AF/ZF/SF
   // Undefined
   {
     SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(Zero);
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(Zero);
-    ZeroNZCV();
   }
 
   // CF/OF
@@ -521,23 +519,22 @@ void OpDispatchBuilder::CalculateFlags_MUL(uint8_t SrcSize, OrderedNode *Res, Or
 
     auto SignBit = _Sbfe(1, SrcSize * 8 - 1, Res);
 
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ, High, SignBit, Zero, One);
+    auto CV = _Constant((1u << IndexNZCV(FEXCore::X86State::RFLAG_CF_LOC)) |
+                        (1u << IndexNZCV(FEXCore::X86State::RFLAG_OF_LOC)));
 
-    SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(SelectOp);
-    SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(SelectOp);
+    // Set CV accordingly and zero NZ regardless
+    SetNZCV(_Select(FEXCore::IR::COND_EQ, High, SignBit, Zero, CV));
   }
 }
 
 void OpDispatchBuilder::CalculateFlags_UMUL(OrderedNode *High) {
   auto Zero = _Constant(0);
-  auto One = _Constant(1);
 
   // AF/SF/PF/ZF
   // Undefined
   {
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(Zero);
     SetRFLAG<FEXCore::X86State::RFLAG_PF_LOC>(Zero);
-    ZeroNZCV();
   }
 
   // CF/OF
@@ -545,10 +542,10 @@ void OpDispatchBuilder::CalculateFlags_UMUL(OrderedNode *High) {
     // CF and OF are set if the result of the operation can't be fit in to the destination register
     // The result register will be all zero if it can't fit due to how multiplication behaves
 
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ, High, Zero, Zero, One);
+    auto CV = _Constant((1u << IndexNZCV(FEXCore::X86State::RFLAG_CF_LOC)) |
+                        (1u << IndexNZCV(FEXCore::X86State::RFLAG_OF_LOC)));
 
-    SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(SelectOp);
-    SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(SelectOp);
+    SetNZCV(_Select(FEXCore::IR::COND_EQ, High, Zero, Zero, CV));
   }
 }
 

From 7ceaf564078c46764829a9675998e064646957ae Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Wed, 2 Aug 2023 13:44:14 -0400
Subject: [PATCH 19/23] OpcodeDispatcher: Use SetNZ_ZeroCV for immediate shifts

We need to be careful to preserve V if needed. For `shl 1` and `shr 1`,
saves 2 instruction overall compared to before the PR. For `sar 1`,
saves 3 overall.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Interface/Core/OpcodeDispatcher/Flags.cpp | 82 +++++++------------
 1 file changed, 31 insertions(+), 51 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
index 9f1863a99..6a597ae98 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -701,7 +701,10 @@ void OpDispatchBuilder::CalculateFlags_ShiftLeftImmediate(uint8_t SrcSize, Order
   if (Shift == 0) return;
 
   auto Zero = _Constant(0);
-  auto One = _Constant(1);
+
+  // Stash OF before overwriting it
+  auto OldOF = Shift != 1 ? GetRFLAG(FEXCore::X86State::RFLAG_OF_LOC) : NULL;
+  SetNZ_ZeroCV(SrcSize, Res);
 
   // CF
   {
@@ -722,24 +725,14 @@ void OpDispatchBuilder::CalculateFlags_ShiftLeftImmediate(uint8_t SrcSize, Order
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(Zero);
   }
 
-  // ZF
-  {
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ,
-        Res, Zero, One, Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
-  }
-
-  // SF
-  {
-    auto SignOp = _Bfe(1, SrcSize * 8 - 1, Res);
-    SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignOp);
-
-    // OF
-    // In the case of left shift. OF is only set from the result of <Top Source Bit> XOR <Top Result Bit>
-    if (Shift == 1) {
-      auto SourceBit = _Bfe(1, SrcSize * 8 - 1, Src1);
-      SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(_Xor(SourceBit, SignOp));
-    }
+  // OF
+  // In the case of left shift. OF is only set from the result of <Top Source Bit> XOR <Top Result Bit>
+  if (Shift == 1) {
+    auto Xor = _Xor(Res, Src1);
+    auto OF = _Bfe(1, SrcSize * 8 - 1, Xor);
+    SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(OF);
+  } else {
+    SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(OldOF);
   }
 }
 
@@ -748,7 +741,10 @@ void OpDispatchBuilder::CalculateFlags_SignShiftRightImmediate(uint8_t SrcSize,
   if (Shift == 0) return;
 
   auto Zero = _Constant(0);
-  auto One = _Constant(1);
+
+  // Stash OF before overwriting it
+  auto OldOF = Shift != 1 ? GetRFLAG(FEXCore::X86State::RFLAG_OF_LOC) : NULL;
+  SetNZ_ZeroCV(SrcSize, Res);
 
   // CF
   {
@@ -765,31 +761,23 @@ void OpDispatchBuilder::CalculateFlags_SignShiftRightImmediate(uint8_t SrcSize,
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(Zero);
   }
 
-  // ZF
-  {
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ,
-        Res, Zero, One, Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
-  }
-
-  // SF
-  {
-    auto SignBitOp = _Bfe(1, SrcSize * 8 - 1, Res);
-    SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignBitOp);
-
-    // OF
-    // Only defined when Shift is 1 else undefined
-    // Only is set if the top bit was set to 1 when shifted
-    // So it is set to same value as SF
-    if (Shift == 1) {
-      SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(Zero);
-    }
+  // OF
+  // Only defined when Shift is 1 else undefined
+  // Only is set if the top bit was set to 1 when shifted
+  // So it is set to same value as SF
+  if (Shift == 1) {
+    /* Already zeroed */
+  } else {
+    SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(OldOF);
   }
 }
 
 void OpDispatchBuilder::CalculateFlags_ShiftRightImmediateCommon(uint8_t SrcSize, OrderedNode *Res, OrderedNode *Src1, uint64_t Shift) {
   auto Zero = _Constant(0);
-  auto One = _Constant(1);
+
+  // Stash OF before overwriting it
+  auto OldOF = Shift != 1 ? GetRFLAG(FEXCore::X86State::RFLAG_OF_LOC) : NULL;
+  SetNZ_ZeroCV(SrcSize, Res);
 
   // CF
   {
@@ -806,17 +794,9 @@ void OpDispatchBuilder::CalculateFlags_ShiftRightImmediateCommon(uint8_t SrcSize
     SetRFLAG<FEXCore::X86State::RFLAG_AF_LOC>(Zero);
   }
 
-  // ZF
-  {
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ,
-        Res, Zero, One, Zero);
-    SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(SelectOp);
-  }
-
-  // SF
-  {
-    auto SignBitOp = _Bfe(1, SrcSize * 8 - 1, Res);
-    SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignBitOp);
+  // Preserve OF if it won't be written
+  if (Shift != 1) {
+    SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(OldOF);
   }
 }
 

From 969d2e4b6a70601d9ceef903d0a8f8aea0f8f07a Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Wed, 2 Aug 2023 14:14:08 -0400
Subject: [PATCH 20/23] OpcodeDispatcher: Zero OF for shift > 1

It is undefined in this case. We prefer to zero (rather than preserve
the existing value) as it avoids a costly RMW of the NZCV register.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Interface/Core/OpcodeDispatcher/Flags.cpp   | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
index 6a597ae98..ab3b4cfec 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -702,8 +702,6 @@ void OpDispatchBuilder::CalculateFlags_ShiftLeftImmediate(uint8_t SrcSize, Order
 
   auto Zero = _Constant(0);
 
-  // Stash OF before overwriting it
-  auto OldOF = Shift != 1 ? GetRFLAG(FEXCore::X86State::RFLAG_OF_LOC) : NULL;
   SetNZ_ZeroCV(SrcSize, Res);
 
   // CF
@@ -732,7 +730,7 @@ void OpDispatchBuilder::CalculateFlags_ShiftLeftImmediate(uint8_t SrcSize, Order
     auto OF = _Bfe(1, SrcSize * 8 - 1, Xor);
     SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(OF);
   } else {
-    SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(OldOF);
+    // Undefined, we choose to zero as part of SetNZ_ZeroCV
   }
 }
 
@@ -742,8 +740,6 @@ void OpDispatchBuilder::CalculateFlags_SignShiftRightImmediate(uint8_t SrcSize,
 
   auto Zero = _Constant(0);
 
-  // Stash OF before overwriting it
-  auto OldOF = Shift != 1 ? GetRFLAG(FEXCore::X86State::RFLAG_OF_LOC) : NULL;
   SetNZ_ZeroCV(SrcSize, Res);
 
   // CF
@@ -762,14 +758,9 @@ void OpDispatchBuilder::CalculateFlags_SignShiftRightImmediate(uint8_t SrcSize,
   }
 
   // OF
-  // Only defined when Shift is 1 else undefined
-  // Only is set if the top bit was set to 1 when shifted
-  // So it is set to same value as SF
-  if (Shift == 1) {
-    /* Already zeroed */
-  } else {
-    SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(OldOF);
-  }
+  // Only defined when Shift is 1 else undefined. Only is set if the top bit was set to 1 when
+  // shifted So it is set to zero.  In the undefined case we choose to zero as well. Since it was
+  // already zeroed there's nothing to do here.
 }
 
 void OpDispatchBuilder::CalculateFlags_ShiftRightImmediateCommon(uint8_t SrcSize, OrderedNode *Res, OrderedNode *Src1, uint64_t Shift) {

From 18783948f7845000ec9ff8ccfe755435d438cd0e Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Wed, 2 Aug 2023 14:31:42 -0400
Subject: [PATCH 21/23] OpcodeDispatcher: Optimize non-immediate shift

Similar to the immediate case, but now we select between the entire old
and new NZCV registers. This is faster than selecting each bit
independently. Saves 11 instructions for calculating flags for "shl eax,
cl".
---
 .../Interface/Core/OpcodeDispatcher/Flags.cpp | 27 ++++++++-----------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
index ab3b4cfec..c60a7fdf0 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -571,7 +571,11 @@ SetRFLAG<FEXCore::X86State::flag>(newval);
 
 void OpDispatchBuilder::CalculateFlags_ShiftLeft(uint8_t SrcSize, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2) {
   auto Zero = _Constant(0);
-  auto One = _Constant(1);
+
+  auto OldNZCV = GetNZCV();
+  uint32_t OldSetNZCVBits = PossiblySetNZCVBits;
+
+  SetNZ_ZeroCV(SrcSize, Res);
 
   // CF
   {
@@ -579,7 +583,7 @@ void OpDispatchBuilder::CalculateFlags_ShiftLeft(uint8_t SrcSize, OrderedNode *R
     auto Size = _Constant(SrcSize * 8);
     auto ShiftAmt = _Sub(Size, Src2);
     auto LastBit = _Bfe(1, 0, _Lshr(Src1, ShiftAmt));
-    COND_FLAG_SET(Src2, RFLAG_CF_LOC, LastBit);
+    SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(LastBit);
   }
 
   CalculatePF(Res, Src2);
@@ -591,26 +595,17 @@ void OpDispatchBuilder::CalculateFlags_ShiftLeft(uint8_t SrcSize, OrderedNode *R
     COND_FLAG_SET(Src2, RFLAG_AF_LOC, Zero);
   }
 
-  // ZF
-  {
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ,
-        Res, Zero, One, Zero);
-    COND_FLAG_SET(Src2, RFLAG_ZF_LOC, SelectOp);
-  }
-
-  // SF
-  {
-    auto val = _Bfe(1, SrcSize * 8 - 1, Res);
-    COND_FLAG_SET(Src2, RFLAG_SF_LOC, val);
-  }
-
   // OF
   {
     // In the case of left shift. OF is only set from the result of <Top Source Bit> XOR <Top Result Bit>
     // When Shift > 1 then OF is undefined
     auto val = _Bfe(1, SrcSize * 8 - 1, _Xor(Src1, Res));
-    COND_FLAG_SET(Src2, RFLAG_OF_LOC, val);
+    SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(val);
   }
+
+  // Now select between the two
+  SetNZCV(_Select(FEXCore::IR::COND_EQ, Src2, Zero, OldNZCV, GetNZCV()));
+  PossiblySetNZCVBits |= OldSetNZCVBits;
 }
 
 void OpDispatchBuilder::CalculateFlags_ShiftRight(uint8_t SrcSize, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2) {

From fa42c1616e3faeb3a49a4a34237e9dc1458475b8 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Wed, 2 Aug 2023 14:36:19 -0400
Subject: [PATCH 22/23] OpcodeDispatcher: Preserve AF for non-immediate shift

The selection logic is expensive. Saves 5 instructions.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Source/Interface/Core/OpcodeDispatcher/Flags.cpp        | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
index c60a7fdf0..ce86c4d4e 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -589,11 +589,7 @@ void OpDispatchBuilder::CalculateFlags_ShiftLeft(uint8_t SrcSize, OrderedNode *R
   CalculatePF(Res, Src2);
 
   // AF
-  {
-    // Undefined
-    // Set to zero anyway
-    COND_FLAG_SET(Src2, RFLAG_AF_LOC, Zero);
-  }
+  // Undefined
 
   // OF
   {

From 7a0119b092632db17d025c40f55866bdf1618ce7 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Wed, 2 Aug 2023 14:43:21 -0400
Subject: [PATCH 23/23] OpcodeDispatcher: Optimize right shifts

Same technique as the left shifts. Gets rid of all our COND_FLAG_SET
use, which is good because it's a performance footgun.

Overall saves 17 instructions (!!!!) from the flag calculation code for
`sar eax, cl`.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
---
 .../Interface/Core/OpcodeDispatcher/Flags.cpp | 71 ++++++-------------
 1 file changed, 23 insertions(+), 48 deletions(-)

diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
index ce86c4d4e..97a0ba7a0 100644
--- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
+++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Flags.cpp
@@ -564,11 +564,6 @@ void OpDispatchBuilder::CalculateFlags_Logical(uint8_t SrcSize, OrderedNode *Res
   SetNZ_ZeroCV(SrcSize, Res);
 }
 
-#define COND_FLAG_SET(cond, flag, newflag) \
-auto oldflag = GetRFLAG(FEXCore::X86State::flag);\
-auto newval = _Select(FEXCore::IR::COND_EQ, cond, Zero, oldflag, newflag);\
-SetRFLAG<FEXCore::X86State::flag>(newval);
-
 void OpDispatchBuilder::CalculateFlags_ShiftLeft(uint8_t SrcSize, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2) {
   auto Zero = _Constant(0);
 
@@ -608,83 +603,63 @@ void OpDispatchBuilder::CalculateFlags_ShiftRight(uint8_t SrcSize, OrderedNode *
   auto Zero = _Constant(0);
   auto One = _Constant(1);
 
+  auto OldNZCV = GetNZCV();
+  uint32_t OldSetNZCVBits = PossiblySetNZCVBits;
+
+  SetNZ_ZeroCV(SrcSize, Res);
+
   // CF
   {
     // Extract the last bit shifted in to CF
     auto ShiftAmt = _Sub(Src2, One);
     auto LastBit = _Bfe(1, 0, _Lshr(Src1, ShiftAmt));
-    COND_FLAG_SET(Src2, RFLAG_CF_LOC, LastBit);
+    SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(LastBit);
   }
 
   CalculatePF(Res, Src2);
 
   // AF
-  {
-    // Undefined
-    // Set to zero anyway
-    COND_FLAG_SET(Src2, RFLAG_AF_LOC, Zero);
-  }
-
-  // ZF
-  {
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ,
-        Res, Zero, One, Zero);
-    COND_FLAG_SET(Src2, RFLAG_ZF_LOC, SelectOp);
-  }
-
-  // SF
-  {
-    auto val =_Bfe(1, SrcSize * 8 - 1, Res);
-    COND_FLAG_SET(Src2, RFLAG_SF_LOC, val);
-  }
+  // Undefined
 
   // OF
   {
     // Only defined when Shift is 1 else undefined
     // OF flag is set if a sign change occurred
     auto val = _Bfe(1, SrcSize * 8 - 1, _Xor(Src1, Res));
-    COND_FLAG_SET(Src2, RFLAG_OF_LOC, val);
+    SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(val);
   }
+
+  // Now select between the two
+  SetNZCV(_Select(FEXCore::IR::COND_EQ, Src2, Zero, OldNZCV, GetNZCV()));
+  PossiblySetNZCVBits |= OldSetNZCVBits;
 }
 
 void OpDispatchBuilder::CalculateFlags_SignShiftRight(uint8_t SrcSize, OrderedNode *Res, OrderedNode *Src1, OrderedNode *Src2) {
   auto Zero = _Constant(0);
   auto One = _Constant(1);
 
+  auto OldNZCV = GetNZCV();
+  uint32_t OldSetNZCVBits = PossiblySetNZCVBits;
+
+  // SF/ZF/OF
+  SetNZ_ZeroCV(SrcSize, Res);
+
   // CF
   {
     // Extract the last bit shifted in to CF
     auto ShiftAmt = _Sub(Src2, One);
     auto LastBit = _Bfe(1, 0, _Lshr(Src1, ShiftAmt));
-    COND_FLAG_SET(Src2, RFLAG_CF_LOC, LastBit);
+    SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(LastBit);
   }
 
   CalculatePF(Res, Src2);
 
   // AF
-  {
-    // Undefined
-    // Set to zero anyway
-    COND_FLAG_SET(Src2, RFLAG_AF_LOC, Zero);
-  }
+  // Undefined
 
-  // ZF
-  {
-    auto SelectOp = _Select(FEXCore::IR::COND_EQ,
-        Res, Zero, One, Zero);
-    COND_FLAG_SET(Src2, RFLAG_ZF_LOC, SelectOp);
-  }
-
-  // SF
-  {
-    auto SignBitOp = _Bfe(1, SrcSize * 8 - 1, Res);
-    COND_FLAG_SET(Src2, RFLAG_SF_LOC, SignBitOp);
-  }
-
-  // OF
-  {
-    COND_FLAG_SET(Src2, RFLAG_OF_LOC, Zero);
-  }
+  // Now select between the two
+  SetNZCV(_Select(FEXCore::IR::COND_EQ, Src2, Zero, OldNZCV, GetNZCV()));
+  PossiblySetNZCVBits |= OldSetNZCVBits;
 }
 
 void OpDispatchBuilder::CalculateFlags_ShiftLeftImmediate(uint8_t SrcSize, OrderedNode *Res, OrderedNode *Src1, uint64_t Shift) {