From 5efc3212d2ba69b941215e86bfed7dc80959eabe Mon Sep 17 00:00:00 2001 From: "Pierre-Loup A. Griffais" Date: Sun, 13 Sep 2026 20:48:45 -0700 Subject: [PATCH] JIT: Let DiskCache patch ValidateCode guest CRC So we can cache blocks with FullSMCTracking finally. --- FEXCore/Source/Interface/Core/CodeCache.cpp | 32 +++++++++++++++++++ FEXCore/Source/Interface/Core/Core.cpp | 9 ++++-- FEXCore/Source/Interface/Core/DiskCache.cpp | 3 +- FEXCore/Source/Interface/Core/Frontend.cpp | 7 ---- FEXCore/Source/Interface/Core/JIT/ALUOps.cpp | 10 ++++-- .../Interface/Core/JIT/Arm64Relocations.cpp | 17 ++-------- .../Source/Interface/Core/JIT/BranchOps.cpp | 3 +- FEXCore/Source/Interface/Core/JIT/JITClass.h | 4 +-- .../Source/Interface/Core/JIT/Relocations.h | 4 +++ FEXCore/Source/Interface/IR/IR.json | 7 ++++ FEXCore/include/FEXCore/Core/DiskCache.h | 2 +- 11 files changed, 68 insertions(+), 30 deletions(-) diff --git a/FEXCore/Source/Interface/Core/CodeCache.cpp b/FEXCore/Source/Interface/Core/CodeCache.cpp index d2f8fa20a..a35fc674d 100644 --- a/FEXCore/Source/Interface/Core/CodeCache.cpp +++ b/FEXCore/Source/Interface/Core/CodeCache.cpp @@ -30,6 +30,9 @@ #include #include +#if defined(ARCHITECTURE_arm64) +#include +#endif namespace FEXCore { @@ -555,6 +558,25 @@ static inline void ApplyPatchableDataRelocation(uint64_t SiteAddress, uint8_t Va CPU::Arm64Emitter::PadType::DOPAD); } +static inline uint32_t crc32(const uint8_t* Ptr, size_t Size) { +#if defined(ARCHITECTURE_arm64) + uint32_t Result {}; +#define do_crc(type, suffix) \ + while (Size >= sizeof(type)) { \ + Result = __crc32##suffix(Result, *reinterpret_cast(Ptr)); \ + Ptr += sizeof(type); \ + Size -= sizeof(type); \ + } + do_crc(uint64_t, d); + do_crc(uint32_t, w); + do_crc(uint16_t, h); + do_crc(uint8_t, b); + return Result; +#else + // Unsupported on non-arm. + return 0; +#endif +}; static inline void ApplyPatchableRIPLiteralRelocation(uint64_t SiteAddress, uint8_t ValueSize, CPU::Arm64Emitter& Emitter) { Emitter.dc64(SiteAddress + ValueSize + ReadLiveGuestData(SiteAddress, ValueSize)); @@ -565,6 +587,11 @@ static inline void ApplyPatchableRIPMoveRelocation(uint64_t SiteAddress, uint8_t Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Target, CPU::Arm64Emitter::PadType::DOPAD); } +static inline void ApplyPatchableCRCMoveRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) { + const uint64_t Target = crc32(reinterpret_cast(SiteAddress), ValueSize); + Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Target, CPU::Arm64Emitter::PadType::DOPAD); +} + bool CodeCache::ApplyPackedCodeRelocations(uint64_t GuestEntry, std::span Code, std::span SmallRelocs, std::span ThunkRelocs) { @@ -599,6 +626,11 @@ bool CodeCache::ApplyPackedCodeRelocations(uint64_t GuestEntry, std::spanOpDispatcher->_ValidateCode( - Thread->OpDispatcher->Constant(crc32(ExistingCodePtr, DecodedInfo->InstSize)), InstAddressReg, DecodedInfo->InstSize); + auto Value = crc32(ExistingCodePtr, DecodedInfo->InstSize); + auto CRC = WantsDiskCachePatching ? + Thread->OpDispatcher->_PatchableGuestCRC(IR::OpSize::i64Bit, Value, (int64_t)ExistingCodePtr, DecodedInfo->InstSize) : + Thread->OpDispatcher->Constant(Value); + auto CodeChanged = Thread->OpDispatcher->_ValidateCode(CRC, InstAddressReg, DecodedInfo->InstSize); auto InvalidateCodeCond = Thread->OpDispatcher->CondJump(CodeChanged); diff --git a/FEXCore/Source/Interface/Core/DiskCache.cpp b/FEXCore/Source/Interface/Core/DiskCache.cpp index b56b81173..83105fe09 100644 --- a/FEXCore/Source/Interface/Core/DiskCache.cpp +++ b/FEXCore/Source/Interface/Core/DiskCache.cpp @@ -986,7 +986,8 @@ namespace DiskCache { } case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE: case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE: - case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_LITERAL: { + case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_LITERAL: + case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE: { // same data for all, relative vs. not and register vs. literal will depend on type on apply BlobSmallRelocation SmallReloc = {}; SmallReloc.Offset = Reloc.Header.Offset; diff --git a/FEXCore/Source/Interface/Core/Frontend.cpp b/FEXCore/Source/Interface/Core/Frontend.cpp index fe983b19f..e59c1cd26 100644 --- a/FEXCore/Source/Interface/Core/Frontend.cpp +++ b/FEXCore/Source/Interface/Core/Frontend.cpp @@ -1522,10 +1522,6 @@ void Decoder::PruneInlinedBranchDataMasks() { void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) { // counter-intuitively, the masks are also needed for lookup on anon prefix decodes, not just stores bool WantsDataMasks = CTX->DiskCache.IsReadingDiskCache() || CTX->DiskCache.IsWritingDiskCache(); - // remove this if we ever fixup ValidateCode crc constant after relocations - if (CTX->Config.SMCChecks == FEXCore::Config::CONFIG_SMC_FULL) { - WantsDataMasks = false; - } while (!FinalInstruction && (Paused || !BlocksToDecode.empty())) { bool Pausing = false; @@ -1679,9 +1675,6 @@ void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) { // NOTE: This will invalidate BlockIt, this is fine as we immediately break from the loop and EraseBlock cannot be true if (CTX->AreMonoHacksActive() && IsBranchMonoTailcall(BlockIt->NumInstructions)) { BlockIt->ForceFullSMCDetection = true; - // todo abandon patching this for now, as the crc will fail and it will lock up redoing it over and over - // we should fix the crc at relocation if this is important - BlockIt->DataMasks.clear(); } BranchTargetInMultiblockRange(); } diff --git a/FEXCore/Source/Interface/Core/JIT/ALUOps.cpp b/FEXCore/Source/Interface/Core/JIT/ALUOps.cpp index d59ffca44..b65bf80fc 100644 --- a/FEXCore/Source/Interface/Core/JIT/ALUOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/ALUOps.cpp @@ -9,6 +9,7 @@ $end_info$ #include "FEXCore/IR/IR.h" #include "Interface/Context/Context.h" #include "Interface/Core/JIT/JITClass.h" +#include "Interface/Core/JIT/Relocations.h" #include "Interface/IR/Passes/RegisterAllocationPass.h" namespace FEXCore::CPU { @@ -66,12 +67,17 @@ DEF_OP(EntrypointOffset) { DEF_OP(PatchableGuestData) { auto Op = IROp->C(); - InsertGuestPatchableDataMove(GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize); + InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE, GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize); } DEF_OP(PatchableGuestRIP) { auto Op = IROp->C(); - InsertGuestPatchableRIPMove(GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize); + InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE, GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize); +} + +DEF_OP(PatchableGuestCRC) { + auto Op = IROp->C(); + InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE, GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize); } DEF_OP(InlineConstant) { diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp b/FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp index 587784468..bb00718c4 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp +++ b/FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp @@ -121,21 +121,10 @@ auto Arm64JITCore::InsertGuestPatchableRIPLiteral(uint64_t GuestRIP, uint64_t Si }; } -void Arm64JITCore::InsertGuestPatchableDataMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize) { +void Arm64JITCore::InsertGuestPatchableMove(FEXCore::CPU::RelocationTypes Type, ARMEmitter::Register Reg, uint64_t Value, + uint64_t SiteAddress, uint8_t ValueSize) { Relocation MoveABI = Relocation::Default(); - MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE}; - MoveABI.GuestPatchableData.RegisterIndex = Reg.Idx(); - MoveABI.GuestPatchableData.ValueSize = ValueSize; - MoveABI.GuestPatchableData.SiteAddress = SiteAddress; - - // this might get patched on disk cache load - LoadConstant(ARMEmitter::Size::i64Bit, Reg, Value, FEXCore::CPU::Arm64Emitter::PadType::DOPAD); - Relocations.emplace_back(MoveABI); -} - -void Arm64JITCore::InsertGuestPatchableRIPMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize) { - Relocation MoveABI = Relocation::Default(); - MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE}; + MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = Type}; MoveABI.GuestPatchableData.RegisterIndex = Reg.Idx(); MoveABI.GuestPatchableData.ValueSize = ValueSize; MoveABI.GuestPatchableData.SiteAddress = SiteAddress; diff --git a/FEXCore/Source/Interface/Core/JIT/BranchOps.cpp b/FEXCore/Source/Interface/Core/JIT/BranchOps.cpp index 4cbf45c39..5852d1e8f 100644 --- a/FEXCore/Source/Interface/Core/JIT/BranchOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/BranchOps.cpp @@ -84,7 +84,8 @@ DEF_OP(ExitFunction) { str(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp)); add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0); if (Op->PatchSiteAddress) { - InsertGuestPatchableRIPMove(EC_CALL_CHECKER_PC_REG, NewRIP, Op->PatchSiteAddress, Op->PatchSiteSize); + InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE, EC_CALL_CHECKER_PC_REG, NewRIP, Op->PatchSiteAddress, + Op->PatchSiteSize); } else { InsertGuestRIPMove(EC_CALL_CHECKER_PC_REG, NewRIP); } diff --git a/FEXCore/Source/Interface/Core/JIT/JITClass.h b/FEXCore/Source/Interface/Core/JIT/JITClass.h index 8758f9a64..7c60a2708 100644 --- a/FEXCore/Source/Interface/Core/JIT/JITClass.h +++ b/FEXCore/Source/Interface/Core/JIT/JITClass.h @@ -564,8 +564,8 @@ private: */ void InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constant); - void InsertGuestPatchableDataMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize); - void InsertGuestPatchableRIPMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize); + void InsertGuestPatchableMove(FEXCore::CPU::RelocationTypes Type, ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, + uint8_t ValueSize); /** * @brief Inserts a named symbol as a literal in memory diff --git a/FEXCore/Source/Interface/Core/JIT/Relocations.h b/FEXCore/Source/Interface/Core/JIT/Relocations.h index c2b7db98d..cbd5b2aab 100644 --- a/FEXCore/Source/Interface/Core/JIT/Relocations.h +++ b/FEXCore/Source/Interface/Core/JIT/Relocations.h @@ -37,6 +37,10 @@ enum class RelocationTypes : uint32_t { // Like PATCHABLE_RIP_LITERAL but puts it in a register // Aligned to struct RelocGuestPatchableData RELOC_GUEST_PATCHABLE_RIP_MOVE, + + // Patchable guest CRC + // Aligned to struct RelocGuestPatchableData + RELOC_GUEST_PATCHABLE_CRC_MOVE, }; struct FEX_PACKED RelocationHeader final { diff --git a/FEXCore/Source/Interface/IR/IR.json b/FEXCore/Source/Interface/IR/IR.json index 19eb43f16..d201e63fb 100644 --- a/FEXCore/Source/Interface/IR/IR.json +++ b/FEXCore/Source/Interface/IR/IR.json @@ -967,6 +967,13 @@ "DestSize": "Size" }, + "GPR = PatchableGuestCRC OpSize:#Size, i64:$Value, i64:$GuestAddress, i64:$GuestSize": { + "Desc": ["Loads Guest CRC in a patchable way", + "On disk cache load the value is recomputed from live guest bytes and patched" + ], + "DestSize": "Size" + }, + "GPR = Constant i64:$Constant, ConstPad:$Pad{IR::ConstPad::NoPad}, i32:$MaxBytes{0}": { "Desc": ["Generates a 64bit constant inside of a GPR", "Unsupported to create a constant in FPR" diff --git a/FEXCore/include/FEXCore/Core/DiskCache.h b/FEXCore/include/FEXCore/Core/DiskCache.h index 671776d08..88b84bb36 100644 --- a/FEXCore/include/FEXCore/Core/DiskCache.h +++ b/FEXCore/include/FEXCore/Core/DiskCache.h @@ -223,7 +223,7 @@ namespace DiskCache { // TODO: This header is in global installed header path, but uses internal headers. // Migrate this once that is fixed. - static constexpr uint16_t FormatVersion = 22; + static constexpr uint16_t FormatVersion = 23; FEX_DEFAULT_VISIBILITY uint16_t GetFormatVersion(); } // namespace DiskCache