FEXCore: Implement a mostly NOP implementation of waitpkg

Part of waitpkg is the TPAUSE instruction. This instruction gives an
RDTSC deadline to go in to a low power sleep mode with the CPU.

Semantically we can't implement umonitor and umwait with ARM's exclusive
monitor implementation, but a nop implementation is sane. Just need to
make sure to clear the pre-req flags.

This lowers power consumption of UE5 games since their job handler now
goes to a tpause based implementation instead of a `pause` spinloop
implementation.

FEXCore/Source/Interface/Core/CPUID.cpp

@@ -626,6 +626,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
     // Only enable EnhancedREPMOVS if atomic memcpy tso emulation isn't enabled.
     const uint32_t SupportsEnhancedREPMOVS = CTX->IsMemcpyAtomicTSOEnabled() == false;
     const uint32_t SupportsVPCLMULQDQ = CTX->HostFeatures.SupportsPMULL_128Bit && SupportsAVX();
+    const uint32_t SupportsWFXT = CTX->HostFeatures.SupportsWFXT;
 
     // Number of subfunctions
     Res.eax = 0x0;
@@ -667,7 +668,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (0 << 2) |                                // Usermode instruction prevention
               (0 << 3) |                                // Protection keys for user mode pages
               (0 << 4) |                                // OS protection keys
-              (0 << 5) |                                // waitpkg
+              (SupportsWFXT << 5) |                     // waitpkg
               (0 << 6) |                                // AVX512_VBMI2
               (0 << 7) |                                // CET shadow stack
               (0 << 8) |                                // GFNI

FEXCore/Source/Interface/Core/JIT/MiscOps.cpp

@@ -17,6 +17,26 @@ $end_info$
 
 namespace FEXCore::CPU {
 
+DEF_OP(WFET) {
+  auto Op = IROp->C<IR::IROp_WFET>();
+  const auto Lower = GetReg(Op->Lower);
+  const auto Upper = GetReg(Op->Upper);
+
+  // Combine registers.
+  mov(ARMEmitter::Size::i64Bit, TMP1, Lower);
+  bfi(ARMEmitter::Size::i64Bit, TMP1, Upper, 32, 32);
+  if (CTX->Config.TSCScale) {
+    // Scale back to ARM64 TSC scale if necessary
+    lsr(ARMEmitter::Size::i64Bit, TMP1, TMP1, CTX->Config.TSCScale);
+  }
+
+  // Clear the exclusive monitor so it can't spuriously wake up with that event.
+  clrex();
+
+  // Execute wfet to wait until the TSC.
+  wfet(TMP1);
+}
+
 DEF_OP(GuestOpcode) {
   auto Op = IROp->C<IR::IROp_GuestOpcode>();
   // metadata

FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp

@@ -4658,9 +4658,27 @@ void OpDispatchBuilder::MOVBEOp(OpcodeArgs) {
   }
 }
 
-void OpDispatchBuilder::CLWB(OpcodeArgs) {
-  Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
-  _CacheLineClean(DestMem);
+void OpDispatchBuilder::CLWBOrTPause(OpcodeArgs) {
+  if (DestIsMem(Op)) {
+    Ref DestMem = MakeSegmentAddress(Op, Op->Dest);
+    _CacheLineClean(DestMem);
+  }
+  else {
+    if (!CTX->HostFeatures.SupportsWFXT) {
+      UnimplementedOp(Op);
+    } else {
+      auto RAX = LoadGPRRegister(X86State::REG_RAX);
+      auto RDX = LoadGPRRegister(X86State::REG_RDX);
+
+      // Incoming source register is unused.
+      _WFET(RDX, RAX);
+
+      // OF, SF, ZF, AF, PF, CF all zero.
+      // CF is used if the OS deadline is set, which we don't do anything with.
+      ZeroPF_AF();
+      ZeroNZCV();
+    }
+  }
 }
 
 void OpDispatchBuilder::CLFLUSHOPT(OpcodeArgs) {
@@ -4697,6 +4715,30 @@ void OpDispatchBuilder::StoreFenceOrCLFlush(OpcodeArgs) {
   }
 }
 
+void OpDispatchBuilder::UMonitorOrCLRSSBSY(OpcodeArgs) {
+  if (DestIsMem(Op) || !CTX->HostFeatures.SupportsWFXT) {
+    // CLRSSBSY
+    UnimplementedOp(Op);
+  }
+  else {
+    // Explicit NOP implementation of umonitor.
+  }
+}
+
+void OpDispatchBuilder::UMWaitOp(OpcodeArgs) {
+  if (DestIsMem(Op) || !CTX->HostFeatures.SupportsWFXT) {
+    UnimplementedOp(Op);
+  }
+  else {
+    // Explicit NOP implementation of umwait.
+    // Still zero flags.
+    //
+    // OF, SF, ZF, AF, PF, CF all zero.
+    ZeroPF_AF();
+    ZeroNZCV();
+  }
+}
+
 void OpDispatchBuilder::CLZeroOp(OpcodeArgs) {
   Ref DestMem = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, {.LoadData = false});
   _CacheLineZero(DestMem);

FEXCore/Source/Interface/Core/OpcodeDispatcher.h

@@ -824,11 +824,13 @@ public:
   void PHADDS(OpcodeArgs);
   void PHSUBS(OpcodeArgs);
 
-  void CLWB(OpcodeArgs);
+  void CLWBOrTPause(OpcodeArgs);
   void CLFLUSHOPT(OpcodeArgs);
   void LoadFenceOrXRSTOR(OpcodeArgs);
   void MemFenceOrXSAVEOPT(OpcodeArgs);
   void StoreFenceOrCLFlush(OpcodeArgs);
+  void UMonitorOrCLRSSBSY(OpcodeArgs);
+  void UMWaitOp(OpcodeArgs);
   void CLZeroOp(OpcodeArgs);
   void RDTSCPOp(OpcodeArgs);
   void RDPIDOp(OpcodeArgs);

FEXCore/Source/Interface/Core/OpcodeDispatcher/SecondaryGroupTables.h

@@ -113,11 +113,13 @@ constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> OpDis
   {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 7), 1, &OpDispatchBuilder::StoreFenceOrCLFlush}, // SFENCE (or CLFLUSH)
 
   {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F3, 5), 1, &OpDispatchBuilder::UnimplementedOp},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F3, 6), 1, &OpDispatchBuilder::UnimplementedOp},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F3, 6), 1, &OpDispatchBuilder::UMonitorOrCLRSSBSY},
 
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_66, 6), 1, &OpDispatchBuilder::CLWB},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_66, 6), 1, &OpDispatchBuilder::CLWBOrTPause},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_66, 7), 1, &OpDispatchBuilder::CLFLUSHOPT},
 
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F2, 6), 1, &OpDispatchBuilder::UMWaitOp},
+
   // GROUP 16
   {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_NONE, 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, true, 1>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_NONE, 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::Prefetch, false, false, 1>},

FEXCore/Source/Interface/Core/X86Tables/SecondaryGroupTables.cpp

@@ -347,7 +347,7 @@ std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGroupOps = [
     {OPD(TYPE_GROUP_15, PF_F3, 3), 1, X86InstInfo{"WRGSBASE", TYPE_INST, GenFlagsDstSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY,          0, nullptr}},
     {OPD(TYPE_GROUP_15, PF_F3, 4), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
     {OPD(TYPE_GROUP_15, PF_F3, 5), 1, X86InstInfo{"INCSSPQ", TYPE_INST, FLAGS_MODRM,                    0, nullptr}},
-    {OPD(TYPE_GROUP_15, PF_F3, 6), 1, X86InstInfo{"CLRSSBSY", TYPE_INST, FLAGS_NONE,                    0, nullptr}},
+    {OPD(TYPE_GROUP_15, PF_F3, 6), 1, X86InstInfo{"UMONITOR/CLRSSBSY", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,       0, nullptr}},
     {OPD(TYPE_GROUP_15, PF_F3, 7), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
 
     {OPD(TYPE_GROUP_15, PF_66, 0), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
@@ -365,7 +365,7 @@ std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGroupOps = [
     {OPD(TYPE_GROUP_15, PF_F2, 3), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
     {OPD(TYPE_GROUP_15, PF_F2, 4), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
     {OPD(TYPE_GROUP_15, PF_F2, 5), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
-    {OPD(TYPE_GROUP_15, PF_F2, 6), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
+    {OPD(TYPE_GROUP_15, PF_F2, 6), 1, X86InstInfo{"UMWAIT", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,       0, nullptr}},
     {OPD(TYPE_GROUP_15, PF_F2, 7), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE,                    0, nullptr}},
 
     // GROUP 16

FEXCore/Source/Interface/IR/IR.json

@@ -284,6 +284,13 @@
         "HasSideEffects": true,
         "Desc": ["This is a hint instruction that the CPU is likely to do a spin so it might want to pause to help out SMP",
                  "Can be implemented as a NOP if necessary"]
+      },
+      "WFET GPR:$Upper, GPR:$Lower": {
+        "HasSideEffects": true,
+        "Desc": [
+          "Implement a low power wait attempting to sleep until RDTSC >= Upper:Lower.",
+          "Will spuriously wake up."
+        ]
       }
     },
     "Branch": {

FEXCore/include/FEXCore/Core/HostFeatures.h

@@ -38,6 +38,7 @@ struct HostFeatures {
   bool SupportsCPUIndexInTPIDRRO {};
   bool SupportsFRINTTS {};
   bool SupportsECV {};
+  bool SupportsWFXT {};
 
   // Float exception behaviour
   bool SupportsAFP {};

Source/Common/HostFeatures.cpp

@@ -484,6 +484,7 @@ FEXCore::HostFeatures FetchHostFeatures(FEX::CPUFeatures& Features, bool Support
   HostFeatures.SupportsRPRES = Features.Supports(CPUFeatures::Feature::RPRES);
   HostFeatures.SupportsSVEBitPerm = Features.Supports(CPUFeatures::Feature::SVE_BitPerm);
   HostFeatures.SupportsECV = Features.Supports(CPUFeatures::Feature::ECV);
+  HostFeatures.SupportsWFXT = Features.Supports(CPUFeatures::Feature::WFxt);
 
 #ifdef VIXL_SIMULATOR
   // Hardcode enable SVE with 256-bit wide registers.