From 8bf22b63829b4a93961fa0004ade5a23fd7172d2 Mon Sep 17 00:00:00 2001 From: "Pierre-Loup A. Griffais" Date: Thu, 3 Sep 2026 17:51:11 -0700 Subject: [PATCH] DiskCache: add plumbing for guest-patching for branches To catch many conditional jumps found in Mono JITted code. --- FEXCore/Source/Interface/Core/CodeCache.cpp | 30 ++++++++++++++ FEXCore/Source/Interface/Core/DiskCache.cpp | 5 ++- FEXCore/Source/Interface/Core/Frontend.cpp | 39 ++++++++++++++++++- FEXCore/Source/Interface/Core/Frontend.h | 4 ++ .../Interface/Core/JIT/Arm64Relocations.cpp | 33 +++++++++++++++- .../Source/Interface/Core/JIT/BranchOps.cpp | 9 ++++- FEXCore/Source/Interface/Core/JIT/JIT.cpp | 12 ++++-- FEXCore/Source/Interface/Core/JIT/JITClass.h | 12 +++++- .../Source/Interface/Core/JIT/Relocations.h | 8 ++++ .../Source/Interface/Core/OpcodeDispatcher.h | 19 ++++++--- .../Interface/Core/X86Tables/X86Tables.h | 2 +- FEXCore/Source/Interface/IR/IR.json | 4 +- FEXCore/include/FEXCore/Core/DiskCache.h | 2 +- 13 files changed, 159 insertions(+), 20 deletions(-) diff --git a/FEXCore/Source/Interface/Core/CodeCache.cpp b/FEXCore/Source/Interface/Core/CodeCache.cpp index 5246c0247..f6feedca4 100644 --- a/FEXCore/Source/Interface/Core/CodeCache.cpp +++ b/FEXCore/Source/Interface/Core/CodeCache.cpp @@ -544,6 +544,27 @@ static inline void ApplyPatchableDataRelocation(uint64_t SiteAddress, uint8_t Va Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Value, CPU::Arm64Emitter::PadType::DOPAD); } +static inline int64_t ReadLiveGuestDisplacement(uint64_t SiteAddress, uint8_t ValueSize) { + uint64_t Raw = 0; + memcpy(&Raw, reinterpret_cast(SiteAddress), ValueSize); + // manual sign-extension from guest live bytes + // 1/2 sizes not permitted in DetectDataMasks currently + if (ValueSize == 4) { + return (int32_t)Raw; + } else { + return (int64_t)Raw; + } +} + +static inline void ApplyPatchableRIPLiteralRelocation(uint64_t SiteAddress, uint8_t ValueSize, CPU::Arm64Emitter& Emitter) { + Emitter.dc64(SiteAddress + ValueSize + ReadLiveGuestDisplacement(SiteAddress, ValueSize)); +} + +static inline void ApplyPatchableRIPMoveRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) { + const uint64_t Target = SiteAddress + ValueSize + ReadLiveGuestDisplacement(SiteAddress, ValueSize); + Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Target, CPU::Arm64Emitter::PadType::DOPAD); +} + bool CodeCache::ApplyPackedCodeRelocations(uint64_t GuestEntry, std::span Code, std::span SmallRelocs, std::span ThunkRelocs) { @@ -569,6 +590,15 @@ bool CodeCache::ApplyPackedCodeRelocations(uint64_t GuestEntry, std::spanOP >= 0xB8 && DecodeInst->OP <= 0xBF) { @@ -1416,12 +1417,20 @@ void Decoder::DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block) { // if (LiteralToPatch && Value < 0x1000000ULL) { // LiteralToPatch = nullptr; // } + Type = DataMaskType::MOV; + } + + // jmp/call branches that use a literal rip-relative offset + // some of those may be inlined by multiblock and will be cleaned up at decode end + if (DecodeInst->TableInfo->Flags & X86Tables::InstFlags::FLAGS_SETS_RIP && DecodeInst->Src[0].IsLiteral()) { + LiteralToPatch = &DecodeInst->Src[0]; + Type = DataMaskType::BRANCH; } // todo add a bunch more if (LiteralToPatch) { - Block.DataMasks.push_back({OpAddress + LastFieldReadOffset, LastFieldReadSize}); + Block.DataMasks.push_back({OpAddress + LastFieldReadOffset, Type, LastFieldReadSize}); LiteralToPatch->Type = X86Tables::DecodedOperand::OpType::LiteralPatchable; LiteralToPatch->Data.LiteralPatchable.FieldOffset = LastFieldReadOffset; @@ -1429,6 +1438,29 @@ void Decoder::DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block) { } } +void Decoder::PruneInlinedBranchDataMasks() { + for (auto& Block : BlockInfo.Blocks) { + if (!Block.DataMasks.size()) { + continue; + } + const auto& LastInst = Block.DecodedInstructions[Block.NumInstructions - 1]; + const auto& LastMask = Block.DataMasks.back(); + + if (LastMask.Type != DataMaskType::BRANCH) { + continue; + } + + const uint64_t NextInst = LastInst.PC + LastInst.InstSize; + if (LastMask.FieldAddress < LastInst.PC || LastMask.FieldAddress + LastMask.ValueSize > NextInst) { + continue; + } + + if (std::ranges::binary_search(BlockInfo.Blocks, NextInst + LastInst.Src[0].Data.LiteralPatchable.Value, std::less {}, &DecodedBlocks::Entry)) { + Block.DataMasks.pop_back(); + } + } +} + void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) { // counter-intuitively, the masks are also needed for lookup on anon prefix decodes, not just stores bool WantsDataMasks = CTX->DiskCache.IsReadingDiskCache() || CTX->DiskCache.IsWritingDiskCache(); @@ -1640,6 +1672,11 @@ void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) { for (auto& Block : BlockInfo.Blocks) { Block.IsEntryPoint = BlockInfo.EntryPoints.contains(Block.Entry); } + + // now that multiblock has settled down, remove any branch masks we put down that didn't end the block + if (WantsDataMasks) { + PruneInlinedBranchDataMasks(); + } } void Decoder::SetupDecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t PC, uint64_t MaxInst) { diff --git a/FEXCore/Source/Interface/Core/Frontend.h b/FEXCore/Source/Interface/Core/Frontend.h index 5a3ec199f..d71a9c073 100644 --- a/FEXCore/Source/Interface/Core/Frontend.h +++ b/FEXCore/Source/Interface/Core/Frontend.h @@ -32,8 +32,11 @@ public: UNIMPLEMENTED_INST, }; + enum class DataMaskType : uint8_t { MOV, BRANCH }; + struct DataMask final { uint64_t FieldAddress; + DataMaskType Type; uint8_t ValueSize; }; @@ -110,6 +113,7 @@ private: void AddBranchTarget(uint64_t Target); void DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block); + void PruneInlinedBranchDataMasks(); bool CheckRangeExecutable(uint64_t Address, uint64_t Size); diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp b/FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp index 47c4f1159..587784468 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp +++ b/FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp @@ -58,7 +58,8 @@ Arm64JITCore::NamedSymbolLiteralPair Arm64JITCore::InsertNamedSymbolLiteral(FEXC void Arm64JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair Lit) { switch (Lit.MoveABI.Header.Type) { case RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: - case RelocationTypes::RELOC_GUEST_RIP_LITERAL: { + case RelocationTypes::RELOC_GUEST_RIP_LITERAL: + case RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_LITERAL: { Lit.MoveABI.Header.Offset = GetCursorOffset(); break; } @@ -102,6 +103,24 @@ void Arm64JITCore::InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constan Relocations.emplace_back(MoveABI); } +auto Arm64JITCore::InsertGuestPatchableRIPLiteral(uint64_t GuestRIP, uint64_t SiteAddress, uint8_t ValueSize) -> NamedSymbolLiteralPair { + return { + .Lit = GuestRIP, + .MoveABI = + { + .GuestPatchableData = {.Header = + { + .Offset = 0, // Set by PlaceNamedSymbolLiteral + .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_LITERAL, + }, + .RegisterIndex = 0, // unused + .ValueSize = ValueSize, + // NOTE: Cache serialization will subtract the unit entry address later + .SiteAddress = SiteAddress}, + }, + }; +} + void Arm64JITCore::InsertGuestPatchableDataMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize) { Relocation MoveABI = Relocation::Default(); MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE}; @@ -114,6 +133,18 @@ void Arm64JITCore::InsertGuestPatchableDataMove(ARMEmitter::Register Reg, uint64 Relocations.emplace_back(MoveABI); } +void Arm64JITCore::InsertGuestPatchableRIPMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize) { + Relocation MoveABI = Relocation::Default(); + MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE}; + MoveABI.GuestPatchableData.RegisterIndex = Reg.Idx(); + MoveABI.GuestPatchableData.ValueSize = ValueSize; + MoveABI.GuestPatchableData.SiteAddress = SiteAddress; + + // this might get patched on disk cache load + LoadConstant(ARMEmitter::Size::i64Bit, Reg, Value, FEXCore::CPU::Arm64Emitter::PadType::DOPAD); + Relocations.emplace_back(MoveABI); +} + fextl::vector Arm64JITCore::TakeRelocations(uint64_t GuestBaseAddress) { // Rebase relocations to library base address for (auto& Relocation : Relocations) { diff --git a/FEXCore/Source/Interface/Core/JIT/BranchOps.cpp b/FEXCore/Source/Interface/Core/JIT/BranchOps.cpp index ddf3e870a..4cbf45c39 100644 --- a/FEXCore/Source/Interface/Core/JIT/BranchOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/BranchOps.cpp @@ -83,7 +83,11 @@ DEF_OP(ExitFunction) { if (NewRIP < EC_CODE_BITMAP_MAX_ADDRESS && RtlIsEcCode(NewRIP)) { str(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp)); add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0); - InsertGuestRIPMove(EC_CALL_CHECKER_PC_REG, NewRIP); + if (Op->PatchSiteAddress) { + InsertGuestPatchableRIPMove(EC_CALL_CHECKER_PC_REG, NewRIP, Op->PatchSiteAddress, Op->PatchSiteSize); + } else { + InsertGuestRIPMove(EC_CALL_CHECKER_PC_REG, NewRIP); + } ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.ExitFunctionEC)); br(TMP2); } else { @@ -173,6 +177,7 @@ DEF_OP(ExitFunction) { ARMEmitter::ForwardLabel TFUnset; ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC])); (void)cbz(ARMEmitter::Size::i32Bit, TMP1, &TFUnset); + // todo do we need to account for cache patching here? InsertGuestRIPMove(TMP1, NewRIP); str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip)); ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.DispatcherLoopTop)); @@ -180,7 +185,7 @@ DEF_OP(ExitFunction) { (void)Bind(&TFUnset); } - EmitLinkedBranch(NewRIP, Op->Hint == IR::BranchHint::Call); + EmitLinkedBranch(NewRIP, Op->Hint == IR::BranchHint::Call, Op->PatchSiteAddress, Op->PatchSiteSize); (void)Bind(&l_CallReturn); #ifdef ARCHITECTURE_arm64ec } diff --git a/FEXCore/Source/Interface/Core/JIT/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/JIT.cpp index 7330e40a6..c23f01a3d 100644 --- a/FEXCore/Source/Interface/Core/JIT/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/JIT.cpp @@ -1019,9 +1019,15 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size // This is a ExitFunctionLinkData struct BindOrRestart(&l_ExitLink); - dc64(0); // HostCode - PlaceNamedSymbolLiteral(InsertGuestRIPLiteral(PendingJumpThunk.GuestRIP)); // GuestRIP - dc64(PendingJumpThunk.CallerAddress - ThunkAddress); // CallerOffset + dc64(0); // HostCode + if (PendingJumpThunk.PatchSiteAddress) { + // GuestRIP with an extra step + PlaceNamedSymbolLiteral( + InsertGuestPatchableRIPLiteral(PendingJumpThunk.GuestRIP, PendingJumpThunk.PatchSiteAddress, PendingJumpThunk.PatchSiteSize)); + } else { + PlaceNamedSymbolLiteral(InsertGuestRIPLiteral(PendingJumpThunk.GuestRIP)); // GuestRIP + } + dc64(PendingJumpThunk.CallerAddress - ThunkAddress); // CallerOffset } BindOrRestart(&l_ExitLink); diff --git a/FEXCore/Source/Interface/Core/JIT/JITClass.h b/FEXCore/Source/Interface/Core/JIT/JITClass.h index 2a56a7efe..8758f9a64 100644 --- a/FEXCore/Source/Interface/Core/JIT/JITClass.h +++ b/FEXCore/Source/Interface/Core/JIT/JITClass.h @@ -105,6 +105,8 @@ private: uint64_t CallerAddress; uint64_t GuestRIP; ARMEmitter::ForwardLabel Label; + uint64_t PatchSiteAddress = 0; + uint8_t PatchSiteSize = 0; }; fextl::vector PendingJumpThunks; @@ -345,8 +347,8 @@ private: uint32_t End; }; - void EmitLinkedBranch(uint64_t GuestRIP, bool Call) { - PendingJumpThunks.push_back({GetCursorAddress(), GuestRIP, {}}); + void EmitLinkedBranch(uint64_t GuestRIP, bool Call, uint64_t PatchSiteAddress = 0, uint8_t PatchSiteSize = 0) { + PendingJumpThunks.push_back({GetCursorAddress(), GuestRIP, {}, PatchSiteAddress, PatchSiteSize}); auto& Thunk = PendingJumpThunks.back(); BindOrRestart(&Thunk.Label); if (Call) { @@ -563,6 +565,7 @@ private: void InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constant); void InsertGuestPatchableDataMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize); + void InsertGuestPatchableRIPMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize); /** * @brief Inserts a named symbol as a literal in memory @@ -583,6 +586,11 @@ private: */ NamedSymbolLiteralPair InsertGuestRIPLiteral(uint64_t GuestRIP); + /** + * @brief Like InsertGuestRIPLiteral, but with patch information to recompute value from live guest bytes at cache load time + */ + NamedSymbolLiteralPair InsertGuestPatchableRIPLiteral(uint64_t GuestRIP, uint64_t SiteAddress, uint8_t ValueSize); + /** * @brief Place the named symbol literal relocation in memory * diff --git a/FEXCore/Source/Interface/Core/JIT/Relocations.h b/FEXCore/Source/Interface/Core/JIT/Relocations.h index ec268874d..c2b7db98d 100644 --- a/FEXCore/Source/Interface/Core/JIT/Relocations.h +++ b/FEXCore/Source/Interface/Core/JIT/Relocations.h @@ -29,6 +29,14 @@ enum class RelocationTypes : uint32_t { // The frontend flagged those regions as patchable by the disk cache // Aligned to struct RelocGuestPatchableData RELOC_GUEST_PATCHABLE_DATA_MOVE, + + // Same as GuestRipLiteral but patchable + // Aligned to struct RelocGuestPatchableData + RELOC_GUEST_PATCHABLE_RIP_LITERAL, + + // Like PATCHABLE_RIP_LITERAL but puts it in a register + // Aligned to struct RelocGuestPatchableData + RELOC_GUEST_PATCHABLE_RIP_MOVE, }; struct FEX_PACKED RelocationHeader final { diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h index 5a5737131..ff5c38951 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h @@ -202,9 +202,10 @@ public: FlushRegisterCache(); return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, InvalidNode, InvalidNode); } - IRPair ExitFunction(Ref NewRIP, BranchHint Hint, Ref CallReturnAddress, Ref CallReturnBlock) { + IRPair ExitFunction(Ref NewRIP, BranchHint Hint, Ref CallReturnAddress, Ref CallReturnBlock, + uint64_t PatchSiteAddress = 0, uint64_t PatchSiteSize = 0) { FlushRegisterCache(); - return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, CallReturnAddress, CallReturnBlock); + return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, CallReturnAddress, CallReturnBlock, PatchSiteAddress, PatchSiteSize); } IRPair Break(BreakDefinition Reason) { FlushRegisterCache(); @@ -1510,12 +1511,18 @@ private: return _GetRelocatedPC(Op, Offset, false); } - void ExitRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset = 0) { - ExitFunction(_GetRelocatedPC(Op, Offset, true /* Inline */)); + void ExitRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset, BranchHint Hint, Ref CallReturnAddress, Ref CallReturnBlock) { + uint64_t PatchOffset = 0; + uint64_t PatchSize = 0; + if (Op->Src[0].IsLiteralPatchable() && Offset && Offset == (int64_t)Op->Src[0].Literal()) { + PatchOffset = Op->PC + Op->Src[0].Data.LiteralPatchable.FieldOffset; + PatchSize = Op->Src[0].Data.LiteralPatchable.Width; + } + ExitFunction(_GetRelocatedPC(Op, Offset, true /* Inline */), Hint, CallReturnAddress, CallReturnBlock, PatchOffset, PatchSize); } - void ExitRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset, BranchHint Hint, Ref CallReturnAddress, Ref CallReturnBlock) { - ExitFunction(_GetRelocatedPC(Op, Offset, true /* Inline */), Hint, CallReturnAddress, CallReturnBlock); + void ExitRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset = 0) { + ExitRelocatedPC(Op, Offset, BranchHint::None, InvalidNode, InvalidNode); } [[nodiscard]] diff --git a/FEXCore/Source/Interface/Core/X86Tables/X86Tables.h b/FEXCore/Source/Interface/Core/X86Tables/X86Tables.h index 9ab0cfbaa..bae27564a 100644 --- a/FEXCore/Source/Interface/Core/X86Tables/X86Tables.h +++ b/FEXCore/Source/Interface/Core/X86Tables/X86Tables.h @@ -171,7 +171,7 @@ struct DecodedOperand { } uint64_t Literal() const { - LOGMAN_THROW_A_FMT(IsLiteral(), "Precondition: must be a literal"); + LOGMAN_THROW_A_FMT(IsLiteral() || IsLiteralPatchable(), "Precondition: must be a literal"); return Data.Literal.Value; } diff --git a/FEXCore/Source/Interface/IR/IR.json b/FEXCore/Source/Interface/IR/IR.json index 61d0e0fd0..c1a80970f 100644 --- a/FEXCore/Source/Interface/IR/IR.json +++ b/FEXCore/Source/Interface/IR/IR.json @@ -312,8 +312,8 @@ "HasSideEffects": true, "RAOverride": "2" }, - "ExitFunction OpSize:#Size, GPR:$NewRIP, BranchHint:$Hint, GPR:$CallReturnAddress, SSA:$CallReturnBlock": { - "Desc": ["Exits the current JIT function with a target RIP" + "ExitFunction OpSize:#Size, GPR:$NewRIP, BranchHint:$Hint, GPR:$CallReturnAddress, SSA:$CallReturnBlock, i64:$PatchSiteAddress{0}, i64:$PatchSiteSize{0}": { + "Desc": ["Exits the current JIT function with a target RIP - optionally patchable from guest bytes for caching" ], "Inline": ["Any"], "HasSideEffects": true, diff --git a/FEXCore/include/FEXCore/Core/DiskCache.h b/FEXCore/include/FEXCore/Core/DiskCache.h index 4071a8572..7ece69444 100644 --- a/FEXCore/include/FEXCore/Core/DiskCache.h +++ b/FEXCore/include/FEXCore/Core/DiskCache.h @@ -221,7 +221,7 @@ namespace DiskCache { // TODO: This header is in global installed header path, but uses internal headers. // Migrate this once that is fixed. - static constexpr uint16_t FormatVersion = 17; + static constexpr uint16_t FormatVersion = 18; FEX_DEFAULT_VISIBILITY uint16_t GetFormatVersion(); } // namespace DiskCache