diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index df8e8716a..9421cf103 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -128,6 +128,11 @@ public: */ [[nodiscard]] bool ApplyCodeRelocations(uint64_t GuestDelta, std::span Code, std::span Relocations, bool ForStorage); + + // Same but on disk cache packed relocations + [[nodiscard]] + bool ApplyPackedCodeRelocations(uint64_t GuestDelta, std::span Code, std::span SmallRelocs, + std::span ThunkRelocs, bool ForStorage); }; class ContextImpl final : public FEXCore::Context::Context, public CPU::SharedCodeBufferManager { diff --git a/FEXCore/Source/Interface/Core/CodeCache.cpp b/FEXCore/Source/Interface/Core/CodeCache.cpp index 8c32e5ae3..b3982e47d 100644 --- a/FEXCore/Source/Interface/Core/CodeCache.cpp +++ b/FEXCore/Source/Interface/Core/CodeCache.cpp @@ -507,6 +507,70 @@ void CodeCache::Validate(const ExecutableFileSectionInfo& Section, fextl::set(CTX.ThunkHandler->LookupThunk(*Symbol)); + if (Pointer == ~0ULL) { + return false; + } + // TODO: Pointers are required to fit within 48-bit VA space. + // But forcing 6-byte broke relocations. + Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Pointer, CPU::Arm64Emitter::PadType::DOPAD); + return true; +} + +static inline void ApplyRIPLiteralRelocation(ContextImpl& CTX, uint64_t GuestRIP, uint64_t GuestEntry, CPU::Arm64Emitter& Emitter) { + Emitter.dc64(GuestEntry + GuestRIP); +} + +static inline void +ApplyRIPMoveRelocation(ContextImpl& CTX, uint64_t GuestRIP, uint8_t RegisterIndex, uint64_t GuestEntry, CPU::Arm64Emitter& Emitter) { + uint64_t Pointer = GuestRIP + GuestEntry; + // TODO: Pointers are required to fit within 48-bit VA space. + // But forcing 6-byte broke relocations. + Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Pointer, CPU::Arm64Emitter::PadType::DOPAD); +} + +bool CodeCache::ApplyPackedCodeRelocations(uint64_t GuestEntry, std::span Code, + std::span SmallRelocs, + std::span ThunkRelocs, bool ForStorage) { + CPU::Arm64Emitter Emitter(&CTX, Code.data(), Code.size_bytes()); + for (auto& Reloc : SmallRelocs) { + LOGMAN_THROW_A_FMT(Reloc.Offset < Code.size_bytes(), "Invalid relocation offset"); + Emitter.SetCursorOffset(Reloc.Offset); + switch ((CPU::RelocationTypes)Reloc.Type) { + case FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: { + ApplySymbolLiteralRelocation(CTX, (CPU::RelocNamedSymbolLiteral::NamedSymbol)Reloc.Named.Symbol, GuestEntry, Emitter, ForStorage); + break; + } + case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL: { + ApplyRIPLiteralRelocation(CTX, Reloc.RIPLiteral.GuestRIP, GuestEntry, Emitter); + break; + } + case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE: { + ApplyRIPMoveRelocation(CTX, Reloc.RIPMove.GuestRIP, Reloc.RIPMove.RegisterIndex, GuestEntry, Emitter); + break; + } + default: ERROR_AND_DIE_FMT("Unknown packed relocation type {}", ToUnderlying((CPU::RelocationTypes)Reloc.Type)); + } + } + for (auto& Reloc : ThunkRelocs) { + LOGMAN_THROW_A_FMT(Reloc.Offset < Code.size_bytes(), "Invalid relocation offset"); + Emitter.SetCursorOffset(Reloc.Offset); + if (!ApplyThunkMoveRelocation(CTX, (const IR::SHA256Sum*)Reloc.SymbolHash, Reloc.RegisterIndex, Emitter, ForStorage)) { + return false; + } + } + return true; +} + bool CodeCache::ApplyCodeRelocations(uint64_t GuestEntry, std::span Code, std::span EntryRelocations, bool ForStorage) { CPU::Arm64Emitter Emitter(&CTX, Code.data(), Code.size_bytes()); @@ -517,31 +581,21 @@ bool CodeCache::ApplyCodeRelocations(uint64_t GuestEntry, std::span C switch (Reloc.Header.Type) { case FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: { - // Generate a literal so we can place it - uint64_t Pointer = ForStorage ? 0 : GetNamedSymbolLiteral(CTX, Reloc.NamedSymbolLiteral.Symbol); - Emitter.dc64(Pointer); + ApplySymbolLiteralRelocation(CTX, Reloc.NamedSymbolLiteral.Symbol, GuestEntry, Emitter, ForStorage); break; } case FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE: { - uint64_t Pointer = ForStorage ? 0 : reinterpret_cast(CTX.ThunkHandler->LookupThunk(Reloc.NamedThunkMove.Symbol)); - if (Pointer == ~0ULL) { + if (!ApplyThunkMoveRelocation(CTX, &Reloc.NamedThunkMove.Symbol, Reloc.NamedThunkMove.RegisterIndex, Emitter, ForStorage)) { return false; } - // TODO: Pointers are required to fit within 48-bit VA space. - // But forcing 6-byte broke relocations. - Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(Reloc.NamedThunkMove.RegisterIndex), Pointer, - CPU::Arm64Emitter::PadType::DOPAD); break; } case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL: { - Emitter.dc64(GuestEntry + Reloc.GuestRIP.GuestRIP); + ApplyRIPLiteralRelocation(CTX, Reloc.GuestRIP.GuestRIP, GuestEntry, Emitter); break; } case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE: { - uint64_t Pointer = Reloc.GuestRIP.GuestRIP + GuestEntry; - // TODO: Pointers are required to fit within 48-bit VA space. - // But forcing 6-byte broke relocations. - Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(Reloc.GuestRIP.RegisterIndex), Pointer, CPU::Arm64Emitter::PadType::DOPAD); + ApplyRIPMoveRelocation(CTX, Reloc.GuestRIP.GuestRIP, Reloc.GuestRIP.RegisterIndex, GuestEntry, Emitter); break; } diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index e05d84930..ec8b4b770 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -864,7 +864,8 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_ FEXCORE_PROFILE_ACCUMULATION(Thread, AccumulatedDiskCacheLookupTime); Hit = DiskCache.Lookup(Thread, *Region, GuestRIP); if (Hit) { - DiskCacheHitRelocationsApplied = CodeCache.ApplyCodeRelocations(GuestRIP, std::as_writable_bytes(Hit->HostCode), Hit->Relocations, false); + DiskCacheHitRelocationsApplied = + CodeCache.ApplyPackedCodeRelocations(GuestRIP, std::as_writable_bytes(Hit->HostCode), Hit->SmallRelocs, Hit->ThunkRelocs, false); if (DiskCacheHitRelocationsApplied && LoadDiskCacheCode) { auto LoadedCode = Thread->CPUBackend->LoadCachedCode(Hit->HostCode, Hit->EntryPoints); diff --git a/FEXCore/Source/Interface/Core/DiskCache.cpp b/FEXCore/Source/Interface/Core/DiskCache.cpp index 59ccbc58b..e2b13c291 100644 --- a/FEXCore/Source/Interface/Core/DiskCache.cpp +++ b/FEXCore/Source/Interface/Core/DiskCache.cpp @@ -451,45 +451,18 @@ namespace DiskCache { return std::nullopt; } - HitData.HostCode = {HitData.Blob.data() + sizeof(Header), Header.HostSize}; - HitData.EntryPoints = {reinterpret_cast(HitData.Blob.data() + sizeof(Header) + Header.HostSize), Header.EntryPointCount}; + uint32_t BlobOffset = sizeof(Header); - auto* SmallRelocs = reinterpret_cast( - HitData.Blob.data() + sizeof(Header) + Header.HostSize + Header.EntryPointCount * sizeof(BlobEntryPoint)); - auto* ThunkRelocs = reinterpret_cast( - reinterpret_cast(SmallRelocs) + Header.SmallRelocCount * sizeof(BlobSmallRelocation)); + HitData.HostCode = {HitData.Blob.data() + BlobOffset, Header.HostSize}; + BlobOffset += Header.HostSize; + HitData.EntryPoints = {reinterpret_cast(HitData.Blob.data() + BlobOffset), Header.EntryPointCount}; + BlobOffset += Header.EntryPointCount * sizeof(BlobEntryPoint); + HitData.SmallRelocs = {reinterpret_cast(HitData.Blob.data() + BlobOffset), Header.SmallRelocCount}; + BlobOffset += Header.SmallRelocCount * sizeof(BlobSmallRelocation); + HitData.ThunkRelocs = {reinterpret_cast(HitData.Blob.data() + BlobOffset), Header.ThunkRelocCount}; + BlobOffset += Header.ThunkRelocCount * sizeof(BlobThunkRelocation); - HitData.Relocations.reserve(Header.SmallRelocCount + Header.ThunkRelocCount); - for (uint32_t i = 0; i < Header.SmallRelocCount; ++i) { - const auto& SmallReloc = SmallRelocs[i]; - FEXCore::CPU::Relocation Reloc = FEXCore::CPU::Relocation::Default(); - Reloc.Header.Type = (CPU::RelocationTypes)SmallReloc.Type; - Reloc.Header.Offset = SmallReloc.Offset; - switch (SmallReloc.Type) { - case uint8_t(CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL): - Reloc.NamedSymbolLiteral.Symbol = CPU::RelocNamedSymbolLiteral::NamedSymbol(SmallReloc.Named.Symbol); - break; - case uint8_t(CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL): Reloc.GuestRIP.GuestRIP = SmallReloc.RIPLiteral.GuestRIP; break; - case uint8_t(CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE): - Reloc.GuestRIP.RegisterIndex = SmallReloc.RIPMove.RegisterIndex; - Reloc.GuestRIP.GuestRIP = SmallReloc.RIPMove.GuestRIP; - break; - default: return std::nullopt; - } - HitData.Relocations.push_back(Reloc); - } - for (uint32_t i = 0; i < Header.ThunkRelocCount; ++i) { - const auto& BigReloc = ThunkRelocs[i]; - FEXCore::CPU::Relocation Reloc = FEXCore::CPU::Relocation::Default(); - Reloc.NamedThunkMove.Header.Offset = BigReloc.Offset; - Reloc.NamedThunkMove.Header.Type = CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE; - Reloc.NamedThunkMove.RegisterIndex = BigReloc.RegisterIndex; - memcpy(&Reloc.NamedThunkMove.Symbol, BigReloc.SymbolHash, sizeof(BigReloc.SymbolHash)); - HitData.Relocations.push_back(Reloc); - } - - auto* PageOffsets = - reinterpret_cast(reinterpret_cast(ThunkRelocs) + Header.ThunkRelocCount * sizeof(BlobThunkRelocation)); + auto* PageOffsets = reinterpret_cast(HitData.Blob.data() + BlobOffset); HitData.GuestPages.reserve(Header.TouchedGuestPagesCount); for (uint32_t i = 0; i < Header.TouchedGuestPagesCount; ++i) { HitData.GuestPages.push_back(GuestRIP + PageOffsets[i]); diff --git a/FEXCore/include/FEXCore/Core/DiskCache.h b/FEXCore/include/FEXCore/Core/DiskCache.h index 4bd5ce763..7eebbf23a 100644 --- a/FEXCore/include/FEXCore/Core/DiskCache.h +++ b/FEXCore/include/FEXCore/Core/DiskCache.h @@ -96,7 +96,8 @@ namespace DiskCache { fextl::vector Blob; std::span HostCode; std::span EntryPoints; - fextl::vector Relocations; + std::span SmallRelocs; + std::span ThunkRelocs; fextl::vector GuestPages; // the spans above point to memory owned by the Blob vec, so it's important this can't be copied