diff --git a/FEXCore/Source/Interface/Core/CodeCache.cpp b/FEXCore/Source/Interface/Core/CodeCache.cpp index f6feedca4..d2f8fa20a 100644 --- a/FEXCore/Source/Interface/Core/CodeCache.cpp +++ b/FEXCore/Source/Interface/Core/CodeCache.cpp @@ -538,13 +538,7 @@ ApplyRIPMoveRelocation(ContextImpl& CTX, uint64_t GuestRIP, uint8_t RegisterInde Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Pointer, CPU::Arm64Emitter::PadType::DOPAD); } -static inline void ApplyPatchableDataRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) { - uint64_t Value = 0; - memcpy(&Value, reinterpret_cast(SiteAddress), ValueSize); - Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Value, CPU::Arm64Emitter::PadType::DOPAD); -} - -static inline int64_t ReadLiveGuestDisplacement(uint64_t SiteAddress, uint8_t ValueSize) { +static inline int64_t ReadLiveGuestData(uint64_t SiteAddress, uint8_t ValueSize) { uint64_t Raw = 0; memcpy(&Raw, reinterpret_cast(SiteAddress), ValueSize); // manual sign-extension from guest live bytes @@ -556,12 +550,18 @@ static inline int64_t ReadLiveGuestDisplacement(uint64_t SiteAddress, uint8_t Va } } +static inline void ApplyPatchableDataRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) { + Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), ReadLiveGuestData(SiteAddress, ValueSize), + CPU::Arm64Emitter::PadType::DOPAD); +} + + static inline void ApplyPatchableRIPLiteralRelocation(uint64_t SiteAddress, uint8_t ValueSize, CPU::Arm64Emitter& Emitter) { - Emitter.dc64(SiteAddress + ValueSize + ReadLiveGuestDisplacement(SiteAddress, ValueSize)); + Emitter.dc64(SiteAddress + ValueSize + ReadLiveGuestData(SiteAddress, ValueSize)); } static inline void ApplyPatchableRIPMoveRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) { - const uint64_t Target = SiteAddress + ValueSize + ReadLiveGuestDisplacement(SiteAddress, ValueSize); + const uint64_t Target = SiteAddress + ValueSize + ReadLiveGuestData(SiteAddress, ValueSize); Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Target, CPU::Arm64Emitter::PadType::DOPAD); } diff --git a/FEXCore/Source/Interface/Core/DiskCache.cpp b/FEXCore/Source/Interface/Core/DiskCache.cpp index 281426fe3..b56b81173 100644 --- a/FEXCore/Source/Interface/Core/DiskCache.cpp +++ b/FEXCore/Source/Interface/Core/DiskCache.cpp @@ -879,7 +879,9 @@ namespace DiskCache { } CurStartExtent = Mask.FieldAddress + Mask.ValueSize; - DataMaskAddresses.insert(Mask.FieldAddress); + if (Mask.Type != Frontend::Decoder::DataMaskType::NOP) { + DataMaskAddresses.insert(Mask.FieldAddress); + } } LastBlock = &SubBlock; } diff --git a/FEXCore/Source/Interface/Core/Frontend.cpp b/FEXCore/Source/Interface/Core/Frontend.cpp index 6ae165ca1..41f6d7a32 100644 --- a/FEXCore/Source/Interface/Core/Frontend.cpp +++ b/FEXCore/Source/Interface/Core/Frontend.cpp @@ -8,6 +8,7 @@ $end_info$ #include "Interface/Context/Context.h" #include "Interface/Core/Frontend.h" +#include "Interface/Core/OpcodeDispatcher.h" #include "Interface/Core/X86Tables/X86Tables.h" #include "Interface/Core/LookupCache.h" @@ -216,6 +217,7 @@ void Decoder::DecodeModRM_16(X86Tables::DecodedOperand* Operand, X86Tables::ModR Operand->Type = DecodedOperand::OpType::SIB; Operand->Data.SIB.Scale = 1; Operand->Data.SIB.Offset = Literal; + Operand->Data.SIB.PatchableDisp = false; // Only called when ModRM.mod != 0b11 struct Encodings { @@ -291,6 +293,7 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand* Operand, X86Tables::ModR // SIB Operand->Type = DecodedOperand::OpType::SIB; Operand->Data.SIB.Scale = 1 << SIB.scale; + Operand->Data.SIB.PatchableDisp = false; // The invalid encoding types are described at Table 1-12. "promoted nsigned is always non-zero" { @@ -329,6 +332,7 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand* Operand, X86Tables::ModR auto [Literal, IsRelocation] = ReadData(4); Operand->Type = IsRelocation ? DecodedOperand::OpType::RIPRelativeRelocation : DecodedOperand::OpType::RIPRelative; Operand->Data.RIPLiteral.Value = Literal; + Operand->Data.RIPLiteral.PatchableDisp = false; } else { // Register-direct addressing Operand->Type = DecodedOperand::OpType::GPRDirect; @@ -344,6 +348,7 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand* Operand, X86Tables::ModR Operand->Type = IsRelocation ? DecodedOperand::OpType::GPRIndirectRelocation : DecodedOperand::OpType::GPRIndirect; Operand->Data.GPRIndirect.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, false, false, false, false); Operand->Data.GPRIndirect.Displacement = Literal; + Operand->Data.GPRIndirect.PatchableDisp = false; } } @@ -1401,7 +1406,7 @@ void Decoder::DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block) { } FEXCore::X86Tables::DecodedOperand* LiteralToPatch = nullptr; - DataMaskType Type; + DataMaskType Type = DataMaskType::NONE; // mov reg,imm if (DecodeInst->OP >= 0xB8 && DecodeInst->OP <= 0xBF) { @@ -1417,9 +1422,63 @@ void Decoder::DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block) { // if (LiteralToPatch && Value < 0x1000000ULL) { // LiteralToPatch = nullptr; // } - Type = DataMaskType::MOV; + if (LiteralToPatch) { + Type = DataMaskType::MOV; + } } + // anything that has a patchable disp32 + bool TryDisp = DecodeInst->Flags & X86Tables::DecodeFlags::FLAG_DECODED_MODRM; + if (TryDisp) { + FEXCore::X86Tables::ModRMDecoded ModRM; + ModRM.Hex = DecodeInst->ModRM; + { + // todo we could handle both imm + disp with more load tracking + bool FoundLiteral = false; + FEXCore::X86Tables::DecodedOperand* OpToPatch = nullptr; + for (auto& Src : DecodeInst->Src) { + if (Src.IsLiteral()) { + FoundLiteral = true; + break; + } + if (Src.IsRIPRelative() || Src.IsGPRIndirect() || Src.IsSIB()) { + OpToPatch = &Src; + } + } + if (DecodeInst->Dest.IsLiteral()) { + FoundLiteral = true; + } + if (DecodeInst->Dest.IsRIPRelative() || DecodeInst->Dest.IsGPRIndirect() || DecodeInst->Dest.IsSIB()) { + OpToPatch = &DecodeInst->Dest; + } + if (!FoundLiteral && OpToPatch) { + if (DecodeInst->TableInfo->OpcodeDispatcher.OpDispatch == &IR::OpDispatchBuilder::NOPOp) { + // if it's a nop disp, only mask data out, no other action required + Type = DataMaskType::NOP; + } else if (OpToPatch->IsRIPRelative()) { + Type = DataMaskType::DISP; + OpToPatch->Data.RIPLiteral.PatchableDisp = true; + OpToPatch->Data.RIPLiteral.DispOffset = LastFieldReadOffset; + } else if (OpToPatch->IsGPRIndirect()) { + // filter out disp8 + if (!(ModRM.mod == 1)) { + Type = DataMaskType::DISP; + OpToPatch->Data.GPRIndirect.PatchableDisp = true; + OpToPatch->Data.GPRIndirect.DispOffset = LastFieldReadOffset; + } + } else if (OpToPatch->IsSIB()) { + FEXCore::X86Tables::SIBDecoded SIB; + SIB.Hex = DecodeInst->SIB; + // two disp32 cases here + if (ModRM.mod == 0b10 || (ModRM.mod == 0 && SIB.base == 0b101)) { + Type = DataMaskType::DISP; + OpToPatch->Data.SIB.PatchableDisp = true; + OpToPatch->Data.SIB.DispOffset = LastFieldReadOffset; + } + } + } + } + } // jmp/call branches that use a literal rip-relative offset // some of those may be inlined by multiblock and will be cleaned up at decode end if (DecodeInst->TableInfo->Flags & X86Tables::InstFlags::FLAGS_SETS_RIP && DecodeInst->Src[0].IsLiteral()) { @@ -1429,12 +1488,14 @@ void Decoder::DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block) { // todo add a bunch more - if (LiteralToPatch) { + if (Type != DataMaskType::NONE) { Block.DataMasks.push_back({OpAddress + LastFieldReadOffset, Type, LastFieldReadSize}); - LiteralToPatch->Type = X86Tables::DecodedOperand::OpType::LiteralPatchable; - LiteralToPatch->Data.LiteralPatchable.FieldOffset = LastFieldReadOffset; - LiteralToPatch->Data.LiteralPatchable.Width = LastFieldReadSize; + if (LiteralToPatch) { + LiteralToPatch->Type = X86Tables::DecodedOperand::OpType::LiteralPatchable; + LiteralToPatch->Data.LiteralPatchable.FieldOffset = LastFieldReadOffset; + LiteralToPatch->Data.LiteralPatchable.Width = LastFieldReadSize; + } } } diff --git a/FEXCore/Source/Interface/Core/Frontend.h b/FEXCore/Source/Interface/Core/Frontend.h index d71a9c073..1fc0e6ed3 100644 --- a/FEXCore/Source/Interface/Core/Frontend.h +++ b/FEXCore/Source/Interface/Core/Frontend.h @@ -32,7 +32,7 @@ public: UNIMPLEMENTED_INST, }; - enum class DataMaskType : uint8_t { MOV, BRANCH }; + enum class DataMaskType : uint8_t { NONE, MOV, BRANCH, DISP, NOP }; struct DataMask final { uint64_t FieldAddress; diff --git a/FEXCore/Source/Interface/Core/JIT/ALUOps.cpp b/FEXCore/Source/Interface/Core/JIT/ALUOps.cpp index 3691bcab8..d59ffca44 100644 --- a/FEXCore/Source/Interface/Core/JIT/ALUOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/ALUOps.cpp @@ -69,6 +69,11 @@ DEF_OP(PatchableGuestData) { InsertGuestPatchableDataMove(GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize); } +DEF_OP(PatchableGuestRIP) { + auto Op = IROp->C(); + InsertGuestPatchableRIPMove(GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize); +} + DEF_OP(InlineConstant) { // nop } diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp index 9c4957fc9..dbb02496c 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp @@ -4308,12 +4308,23 @@ AddressMode OpDispatchBuilder::DecodeAddress(const X86Tables::DecodedOp& Op, con if (Operand.IsGPRIndirectRelocation()) { A.Base = Add(GPRSize, _EntrypointOffset(GPRSize, Operand.Data.GPRIndirect.Displacement), A.Base); } else { - A.Offset = static_cast(Operand.Data.GPRIndirect.Displacement); + if (Operand.Data.GPRIndirect.PatchableDisp) { + A.Base = Add(GPRSize, A.Base, + _PatchableGuestData(OpSize::i64Bit, static_cast(Operand.Data.GPRIndirect.Displacement), + Op->PC + Operand.Data.GPRIndirect.DispOffset, 4)); + } else { + A.Offset = static_cast(Operand.Data.GPRIndirect.Displacement); + } } A.NonTSO |= IsNonTSOReg(AccessType, Operand.Data.GPRIndirect.GPR); } else if (Operand.IsRIPRelative() || Operand.IsRIPRelativeRelocation()) { if (Is64BitMode) { - A.Base = GetRelocatedPC(Op, static_cast(Operand.Data.RIPLiteral.Value)); + if (Operand.IsRIPRelative() && Operand.Data.RIPLiteral.PatchableDisp) { + A.Base = _PatchableGuestRIP(OpSize::i64Bit, Op->PC + Op->InstSize + static_cast(Operand.Data.RIPLiteral.Value), + Op->PC + Operand.Data.RIPLiteral.DispOffset, 4); + } else { + A.Base = GetRelocatedPC(Op, static_cast(Operand.Data.RIPLiteral.Value)); + } } else { // 32bit this isn't RIP relative but instead absolute if (Operand.IsRIPRelativeRelocation()) { @@ -4350,6 +4361,14 @@ AddressMode OpDispatchBuilder::DecodeAddress(const X86Tables::DecodedOp& Op, con } else { A.Base = EPOffset; } + } else if (Operand.Data.SIB.PatchableDisp) { + Ref PatchedDisp = + _PatchableGuestData(OpSize::i64Bit, static_cast(Operand.Data.SIB.Offset), Op->PC + Operand.Data.SIB.DispOffset, 4); + if (A.Base) { + A.Base = Add(OpSize::i64Bit, A.Base, PatchedDisp); + } else { + A.Base = PatchedDisp; + } } else { A.Offset = static_cast(Operand.Data.SIB.Offset); } diff --git a/FEXCore/Source/Interface/Core/X86Tables/X86Tables.h b/FEXCore/Source/Interface/Core/X86Tables/X86Tables.h index 281379799..5de48bf53 100644 --- a/FEXCore/Source/Interface/Core/X86Tables/X86Tables.h +++ b/FEXCore/Source/Interface/Core/X86Tables/X86Tables.h @@ -185,10 +185,14 @@ struct DecodedOperand { struct { int64_t Displacement; uint8_t GPR; + bool PatchableDisp; + uint8_t DispOffset; } GPRIndirect; // Shared with GPRIndirectRelocation struct { int64_t Value; + bool PatchableDisp; + uint8_t DispOffset; } RIPLiteral; // Shared with RIPLiteralRelocation struct LiteralType { @@ -211,7 +215,9 @@ struct DecodedOperand { uint8_t Scale; uint8_t Index; // ~0 invalid uint8_t Base; // ~0 invalid - } SIB; // Shared with SIBRelocation + bool PatchableDisp; + uint8_t DispOffset; + } SIB; // Shared with SIBRelocation }; TypeUnion Data; diff --git a/FEXCore/Source/Interface/IR/IR.json b/FEXCore/Source/Interface/IR/IR.json index 26aa45371..19eb43f16 100644 --- a/FEXCore/Source/Interface/IR/IR.json +++ b/FEXCore/Source/Interface/IR/IR.json @@ -960,6 +960,13 @@ "DestSize": "Size" }, + "GPR = PatchableGuestRIP OpSize:#Size, i64:$Value, i64:$SiteAddress, i64:$SiteSize": { + "Desc": ["Loads GuestRIP-relative Value in a patchable way", + "On disk cache load the value is patched from live guest PC and live displacement at SiteAddress" + ], + "DestSize": "Size" + }, + "GPR = Constant i64:$Constant, ConstPad:$Pad{IR::ConstPad::NoPad}, i32:$MaxBytes{0}": { "Desc": ["Generates a 64bit constant inside of a GPR", "Unsupported to create a constant in FPR" diff --git a/FEXCore/include/FEXCore/Core/DiskCache.h b/FEXCore/include/FEXCore/Core/DiskCache.h index 0be0a639f..671776d08 100644 --- a/FEXCore/include/FEXCore/Core/DiskCache.h +++ b/FEXCore/include/FEXCore/Core/DiskCache.h @@ -223,7 +223,7 @@ namespace DiskCache { // TODO: This header is in global installed header path, but uses internal headers. // Migrate this once that is fixed. - static constexpr uint16_t FormatVersion = 21; + static constexpr uint16_t FormatVersion = 22; FEX_DEFAULT_VISIBILITY uint16_t GetFormatVersion(); } // namespace DiskCache