JIT: Let DiskCache patch ValidateCode guest CRC

So we can cache blocks with FullSMCTracking finally.

FEXCore/Source/Interface/Core/CodeCache.cpp

@@ -30,6 +30,9 @@
 #include <FEXCore/Utils/AllocatorHooks.h>
 
 #include <fstream>
+#if defined(ARCHITECTURE_arm64)
+#include <arm_acle.h>
+#endif
 
 namespace FEXCore {
 
@@ -555,6 +558,25 @@ static inline void ApplyPatchableDataRelocation(uint64_t SiteAddress, uint8_t Va
                        CPU::Arm64Emitter::PadType::DOPAD);
 }
 
+static inline uint32_t crc32(const uint8_t* Ptr, size_t Size) {
+#if defined(ARCHITECTURE_arm64)
+  uint32_t Result {};
+#define do_crc(type, suffix)                                               \
+  while (Size >= sizeof(type)) {                                           \
+    Result = __crc32##suffix(Result, *reinterpret_cast<const type*>(Ptr)); \
+    Ptr += sizeof(type);                                                   \
+    Size -= sizeof(type);                                                  \
+  }
+  do_crc(uint64_t, d);
+  do_crc(uint32_t, w);
+  do_crc(uint16_t, h);
+  do_crc(uint8_t, b);
+  return Result;
+#else
+  // Unsupported on non-arm.
+  return 0;
+#endif
+};
 
 static inline void ApplyPatchableRIPLiteralRelocation(uint64_t SiteAddress, uint8_t ValueSize, CPU::Arm64Emitter& Emitter) {
   Emitter.dc64(SiteAddress + ValueSize + ReadLiveGuestData(SiteAddress, ValueSize));
@@ -565,6 +587,11 @@ static inline void ApplyPatchableRIPMoveRelocation(uint64_t SiteAddress, uint8_t
   Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Target, CPU::Arm64Emitter::PadType::DOPAD);
 }
 
+static inline void ApplyPatchableCRCMoveRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) {
+  const uint64_t Target = crc32(reinterpret_cast<const uint8_t*>(SiteAddress), ValueSize);
+  Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Target, CPU::Arm64Emitter::PadType::DOPAD);
+}
+
 bool CodeCache::ApplyPackedCodeRelocations(uint64_t GuestEntry, std::span<std::byte> Code,
                                            std::span<const DiskCache::BlobSmallRelocation> SmallRelocs,
                                            std::span<const DiskCache::BlobThunkRelocation> ThunkRelocs) {
@@ -599,6 +626,11 @@ bool CodeCache::ApplyPackedCodeRelocations(uint64_t GuestEntry, std::span<std::b
                                       Reloc.PatchableData.RegisterIndex, Emitter);
       break;
     }
+    case FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE: {
+      ApplyPatchableCRCMoveRelocation(GuestEntry + Reloc.PatchableData.SiteOffset, Reloc.PatchableData.ValueSize,
+                                      Reloc.PatchableData.RegisterIndex, Emitter);
+      break;
+    }
     default: ERROR_AND_DIE_FMT("Unknown packed relocation type {}", ToUnderlying((CPU::RelocationTypes)Reloc.Type));
     }
   }

FEXCore/Source/Interface/Core/Core.cpp

@@ -523,6 +523,8 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
 
   bool HasCustomIR {};
 
+  bool WantsDiskCachePatching = DiskCache.IsReadingDiskCache() || DiskCache.IsWritingDiskCache();
+
   if (HasCustomIRHandlers.load(std::memory_order_relaxed)) {
     std::shared_lock lk(CustomIRMutex);
     auto Handler = CustomIRHandlers.find(GuestRIP);
@@ -656,8 +658,11 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
             return 0;
 #endif
           };
-          auto CodeChanged = Thread->OpDispatcher->_ValidateCode(
-            Thread->OpDispatcher->Constant(crc32(ExistingCodePtr, DecodedInfo->InstSize)), InstAddressReg, DecodedInfo->InstSize);
+          auto Value = crc32(ExistingCodePtr, DecodedInfo->InstSize);
+          auto CRC = WantsDiskCachePatching ?
+                       Thread->OpDispatcher->_PatchableGuestCRC(IR::OpSize::i64Bit, Value, (int64_t)ExistingCodePtr, DecodedInfo->InstSize) :
+                       Thread->OpDispatcher->Constant(Value);
+          auto CodeChanged = Thread->OpDispatcher->_ValidateCode(CRC, InstAddressReg, DecodedInfo->InstSize);
 
           auto InvalidateCodeCond = Thread->OpDispatcher->CondJump(CodeChanged);
 

FEXCore/Source/Interface/Core/DiskCache.cpp

@@ -986,7 +986,8 @@ namespace DiskCache {
       }
       case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE:
       case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE:
-      case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_LITERAL: {
+      case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_LITERAL:
+      case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE: {
         // same data for all, relative vs. not and register vs. literal will depend on type on apply
         BlobSmallRelocation SmallReloc = {};
         SmallReloc.Offset = Reloc.Header.Offset;

FEXCore/Source/Interface/Core/Frontend.cpp

@@ -1522,10 +1522,6 @@ void Decoder::PruneInlinedBranchDataMasks() {
 void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) {
   // counter-intuitively, the masks are also needed for lookup on anon prefix decodes, not just stores
   bool WantsDataMasks = CTX->DiskCache.IsReadingDiskCache() || CTX->DiskCache.IsWritingDiskCache();
-  // remove this if we ever fixup ValidateCode crc constant after relocations
-  if (CTX->Config.SMCChecks == FEXCore::Config::CONFIG_SMC_FULL) {
-    WantsDataMasks = false;
-  }
 
   while (!FinalInstruction && (Paused || !BlocksToDecode.empty())) {
     bool Pausing = false;
@@ -1679,9 +1675,6 @@ void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) {
           // NOTE: This will invalidate BlockIt, this is fine as we immediately break from the loop and EraseBlock cannot be true
           if (CTX->AreMonoHacksActive() && IsBranchMonoTailcall(BlockIt->NumInstructions)) {
             BlockIt->ForceFullSMCDetection = true;
-            // todo abandon patching this for now, as the crc will fail and it will lock up redoing it over and over
-            // we should fix the crc at relocation if this is important
-            BlockIt->DataMasks.clear();
           }
           BranchTargetInMultiblockRange();
         }

FEXCore/Source/Interface/Core/JIT/ALUOps.cpp

@@ -9,6 +9,7 @@ $end_info$
 #include "FEXCore/IR/IR.h"
 #include "Interface/Context/Context.h"
 #include "Interface/Core/JIT/JITClass.h"
+#include "Interface/Core/JIT/Relocations.h"
 #include "Interface/IR/Passes/RegisterAllocationPass.h"
 
 namespace FEXCore::CPU {
@@ -66,12 +67,17 @@ DEF_OP(EntrypointOffset) {
 
 DEF_OP(PatchableGuestData) {
   auto Op = IROp->C<IR::IROp_PatchableGuestData>();
-  InsertGuestPatchableDataMove(GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize);
+  InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE, GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize);
 }
 
 DEF_OP(PatchableGuestRIP) {
   auto Op = IROp->C<IR::IROp_PatchableGuestRIP>();
-  InsertGuestPatchableRIPMove(GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize);
+  InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE, GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize);
+}
+
+DEF_OP(PatchableGuestCRC) {
+  auto Op = IROp->C<IR::IROp_PatchableGuestRIP>();
+  InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE, GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize);
 }
 
 DEF_OP(InlineConstant) {

FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp

@@ -121,21 +121,10 @@ auto Arm64JITCore::InsertGuestPatchableRIPLiteral(uint64_t GuestRIP, uint64_t Si
   };
 }
 
-void Arm64JITCore::InsertGuestPatchableDataMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize) {
+void Arm64JITCore::InsertGuestPatchableMove(FEXCore::CPU::RelocationTypes Type, ARMEmitter::Register Reg, uint64_t Value,
+                                            uint64_t SiteAddress, uint8_t ValueSize) {
   Relocation MoveABI = Relocation::Default();
-  MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE};
-  MoveABI.GuestPatchableData.RegisterIndex = Reg.Idx();
-  MoveABI.GuestPatchableData.ValueSize = ValueSize;
-  MoveABI.GuestPatchableData.SiteAddress = SiteAddress;
-
-  // this might get patched on disk cache load
-  LoadConstant(ARMEmitter::Size::i64Bit, Reg, Value, FEXCore::CPU::Arm64Emitter::PadType::DOPAD);
-  Relocations.emplace_back(MoveABI);
-}
-
-void Arm64JITCore::InsertGuestPatchableRIPMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize) {
-  Relocation MoveABI = Relocation::Default();
-  MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE};
+  MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = Type};
   MoveABI.GuestPatchableData.RegisterIndex = Reg.Idx();
   MoveABI.GuestPatchableData.ValueSize = ValueSize;
   MoveABI.GuestPatchableData.SiteAddress = SiteAddress;

FEXCore/Source/Interface/Core/JIT/BranchOps.cpp

@@ -84,7 +84,8 @@ DEF_OP(ExitFunction) {
       str(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));
       add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0);
       if (Op->PatchSiteAddress) {
-        InsertGuestPatchableRIPMove(EC_CALL_CHECKER_PC_REG, NewRIP, Op->PatchSiteAddress, Op->PatchSiteSize);
+        InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE, EC_CALL_CHECKER_PC_REG, NewRIP, Op->PatchSiteAddress,
+                                 Op->PatchSiteSize);
       } else {
         InsertGuestRIPMove(EC_CALL_CHECKER_PC_REG, NewRIP);
       }

FEXCore/Source/Interface/Core/JIT/JITClass.h

@@ -564,8 +564,8 @@ private:
    */
   void InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constant);
 
-  void InsertGuestPatchableDataMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize);
-  void InsertGuestPatchableRIPMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize);
+  void InsertGuestPatchableMove(FEXCore::CPU::RelocationTypes Type, ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress,
+                                uint8_t ValueSize);
 
   /**
    * @brief Inserts a named symbol as a literal in memory

FEXCore/Source/Interface/Core/JIT/Relocations.h

@@ -37,6 +37,10 @@ enum class RelocationTypes : uint32_t {
   // Like PATCHABLE_RIP_LITERAL but puts it in a register
   // Aligned to struct RelocGuestPatchableData
   RELOC_GUEST_PATCHABLE_RIP_MOVE,
+
+  // Patchable guest CRC
+  // Aligned to struct RelocGuestPatchableData
+  RELOC_GUEST_PATCHABLE_CRC_MOVE,
 };
 
 struct FEX_PACKED RelocationHeader final {

FEXCore/Source/Interface/IR/IR.json

@@ -967,6 +967,13 @@
         "DestSize": "Size"
       },
 
+      "GPR = PatchableGuestCRC OpSize:#Size, i64:$Value, i64:$GuestAddress, i64:$GuestSize": {
+        "Desc": ["Loads Guest CRC in a patchable way",
+                 "On disk cache load the value is recomputed from live guest bytes and patched"
+                ],
+        "DestSize": "Size"
+      },
+
       "GPR = Constant i64:$Constant, ConstPad:$Pad{IR::ConstPad::NoPad}, i32:$MaxBytes{0}": {
         "Desc": ["Generates a 64bit constant inside of a GPR",
                  "Unsupported to create a constant in FPR"

FEXCore/include/FEXCore/Core/DiskCache.h

@@ -223,7 +223,7 @@ namespace DiskCache {
 
   // TODO: This header is in global installed header path, but uses internal headers.
   // Migrate this once that is fixed.
-  static constexpr uint16_t FormatVersion = 22;
+  static constexpr uint16_t FormatVersion = 23;
   FEX_DEFAULT_VISIBILITY uint16_t GetFormatVersion();
 
 } // namespace DiskCache