From 3157685f0fbbcca06fd8b507f2ae6530cc868af6 Mon Sep 17 00:00:00 2001 From: "Pierre-Loup A. Griffais" Date: Tue, 1 Sep 2026 01:02:10 -0700 Subject: [PATCH] DiskCache: initial anon caching Decode a few bytes in advance to get a hashable prefix to use as key. Generate touched pages dynamically since they can be misaligned now, as the cached-hit guest code isn't necessarily in the same spot as the store was. --- .../Source/Interface/Config/Config.json.in | 8 + FEXCore/Source/Interface/Core/Core.cpp | 30 ++- FEXCore/Source/Interface/Core/DiskCache.cpp | 115 ++++++---- FEXCore/Source/Interface/Core/Frontend.cpp | 204 +++++++++++------- FEXCore/Source/Interface/Core/Frontend.h | 13 +- FEXCore/include/FEXCore/Core/DiskCache.h | 13 +- 6 files changed, 240 insertions(+), 143 deletions(-) diff --git a/FEXCore/Source/Interface/Config/Config.json.in b/FEXCore/Source/Interface/Config/Config.json.in index f1eaf5eeb..77414f300 100644 --- a/FEXCore/Source/Interface/Config/Config.json.in +++ b/FEXCore/Source/Interface/Config/Config.json.in @@ -168,6 +168,14 @@ "Don't cache blocks with relocations pointing outside of any known region" ] }, + "DiskCacheAnonCaching": { + "Type": "bool", + "Default": "true", + "AffectsCodeGen": "false", + "Desc": [ + "Attempt to cache anonymous code" + ] + }, "DiskCachePath": { "Type": "str", "Default": "", diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index 9e4f548a6..3cdbcb5f0 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -542,7 +542,7 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue if (!HasCustomIR) { const auto* GuestCode = reinterpret_cast(GuestRIP); - Thread->FrontendDecoder->DecodeInstructionsAtEntry(Thread, GuestCode, GuestRIP, MaxInst); + Thread->FrontendDecoder->DecodeLoop(GuestCode); const auto* BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo(); const auto& CodeBlocks = BlockInfo->Blocks; @@ -883,13 +883,16 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_ return HostCode; } + Thread->FrontendDecoder->SetupDecodeInstructionsAtEntry(Thread, GuestRIP, MaxInst); + std::optional Region = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP); std::optional Hit; + std::optional DiskCacheGuestCodeKey; bool DiskCacheHitRelocationsApplied = false; bool LoadDiskCacheCode = true; - if (Region && Region->FileStartVA != 0) { + { FEXCORE_PROFILE_ACCUMULATION(Thread, AccumulatedDiskCacheLookupTime); - Hit = DiskCache.Lookup(Thread, *Region, GuestRIP); + Hit = DiskCache.Lookup(Thread, Region, GuestRIP, DiskCacheGuestCodeKey); if (Hit) { DiskCacheHitRelocationsApplied = CodeCache.ApplyPackedCodeRelocations(GuestRIP, std::as_writable_bytes(Hit->HostCode), Hit->SmallRelocs, Hit->ThunkRelocs, false); @@ -917,6 +920,7 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_ LOGMAN_THROW_A_FMT(CachedHostCode != 0, "Couldn't find GuestRIP in Disk Cache entrypoints!"); FEXCORE_PROFILE_INSTANT_INCREMENT(Thread, AccumulatedDiskCacheHitCount, 1); + Thread->FrontendDecoder->DelayedDisownBuffer(); return CachedHostCode; } } @@ -994,16 +998,19 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_ } // Disk Cache - if (Region && Region->FileStartVA != 0 && !CodeCache.IsGeneratingCache) { - std::span Relocations; - if (DebugData && DebugData->Relocations) { - Relocations = *DebugData->Relocations; + if (!CodeCache.IsGeneratingCache) { + if (DiskCacheGuestCodeKey) { + std::span Relocations; + if (DebugData && DebugData->Relocations) { + Relocations = *DebugData->Relocations; + } + std::span GuestCode = {reinterpret_cast(StartAddr), Length}; + const Frontend::Decoder::DecodedBlockInformation* BlockInfo = + NeedsAddGuestCodeRanges ? Thread->FrontendDecoder->GetDecodedBlockInfo() : nullptr; + DiskCache.Store(Thread, Region, GuestRIP, *DiskCacheGuestCodeKey, GuestCode, CompiledCode, Relocations, BlockInfo); } - std::span GuestCode = {reinterpret_cast(StartAddr), Length}; - const Frontend::Decoder::DecodedBlockInformation* BlockInfo = NeedsAddGuestCodeRanges ? Thread->FrontendDecoder->GetDecodedBlockInfo() : nullptr; - DiskCache.Store(Thread, *Region, GuestRIP, GuestCode, CompiledCode, Relocations, BlockInfo); - if (CodeMapWriter) { + if (CodeMapWriter && Region && Region->FileStartVA != 0) { CodeMapWriter->AppendBlock(*Region, GuestRIP); } } @@ -1030,6 +1037,7 @@ uintptr_t ContextImpl::CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, ui // Invalidate might take a unique lock on this, to guarantee that during invalidation no code gets compiled auto lk = GuardSignalDeferringSection(CodeInvalidationMutex, Thread); + Thread->FrontendDecoder->SetupDecodeInstructionsAtEntry(Thread, GuestRIP, 1); auto [CompiledCode, DebugData, StartAddr, Length, _] = CompileCode(Thread, GuestRIP, 1); auto CodePtr = CompiledCode.EntryPoints[GuestRIP]; if (CodePtr == nullptr) { diff --git a/FEXCore/Source/Interface/Core/DiskCache.cpp b/FEXCore/Source/Interface/Core/DiskCache.cpp index cb4782afa..18c12279b 100644 --- a/FEXCore/Source/Interface/Core/DiskCache.cpp +++ b/FEXCore/Source/Interface/Core/DiskCache.cpp @@ -2,6 +2,8 @@ #define XXH_STATIC_LINKING_ONLY +#include "FEXCore/Utils/TypeDefines.h" +#include "Interface/Core/Frontend.h" #include "FEXCore/Config/Config.h" #include "FEXCore/fextl/string.h" #include "FEXHeaderUtils/Filesystem.h" @@ -237,6 +239,9 @@ namespace DiskCache { if (!ExtentsValid) { continue; } + } else { + NewEntry.GuestExtents.push_back(0); + NewEntry.GuestExtents.push_back(IndexBlobExtra->GuestSize); } CacheIndex.insert({IndexBlobCommon->hash, {std::move(NewEntry)}}); } else { @@ -325,6 +330,9 @@ namespace DiskCache { NewEntry.GuestExtents.resize(IndexBlobHeader->GuestExtentsCount); memcpy(NewEntry.GuestExtents.data(), reinterpret_cast(IndexBlob.data() + sizeof(IndexExtraBlobHeader)), IndexBlobHeader->GuestExtentsCount * sizeof(uint32_t)); + } else { + NewEntry.GuestExtents.push_back(0); + NewEntry.GuestExtents.push_back(IndexBlobHeader->GuestSize); } std::lock_guard Guard(IndexMutex); Index.insert_or_assign(Hash, std::move(NewEntry)); @@ -431,22 +439,44 @@ namespace DiskCache { } } - uint64_t DiskCache::MakeBlobKey(const uint64_t ModuleOffset) { + uint64_t DiskCache::MakeBlobKey(const uint64_t CodeKey) { struct { - uint64_t ModuleOffset; + uint64_t CodeKey; XXH128_hash_t BucketHash; - } BlobKeyBytes = {ModuleOffset, BucketHash}; + } BlobKeyBytes = {CodeKey, BucketHash}; return XXH3_64bits(&BlobKeyBytes, sizeof(BlobKeyBytes)); } - std::optional DiskCache::Lookup(Core::InternalThreadState* Thread, const ExecutableFileSectionInfo& Region, uint64_t GuestRIP) { + std::optional DiskCache::Lookup(Core::InternalThreadState* Thread, std::optional Region, + uint64_t GuestRIP, std::optional& GuestCodeKey) { if (!IsReadingDiskCache()) { return std::nullopt; } - uint64_t ModuleOffset = GuestRIP - Region.FileStartVA; + if (Region && Region->FileStartVA) { + GuestCodeKey = GuestRIP - Region->FileStartVA; + } else { + if (!AnonCaching) { + return std::nullopt; + } + Thread->FrontendDecoder->DecodeLoop(reinterpret_cast(GuestRIP), AnonPrefixGuestBytes); + XXH3_state_t HashState; + XXH3_64bits_reset(&HashState); + for (auto& SubBlock : Thread->FrontendDecoder->GetDecodedBlockInfo()->Blocks) { + XXH3_64bits_update(&HashState, reinterpret_cast(SubBlock.Entry), SubBlock.Size); + if (SubBlock.BlockStatus != Frontend::Decoder::DecodedBlockStatus::SUCCESS) { + return std::nullopt; + } + } + GuestCodeKey = XXH3_64bits_digest(&HashState); + // if (TotalSize < AnonPrefixGuestBytes) { + // GuestCodeKey = 0; + // return std::nullopt; + // } + // LogMan::Msg::IFmt("anon lookup! length {:d} {}", GuestCodeKey, TotalSize); + } - uint64_t Hash = MakeBlobKey(ModuleOffset); + uint64_t Hash = MakeBlobKey(*GuestCodeKey); IndexEntry Entry; { @@ -479,16 +509,22 @@ namespace DiskCache { // LogMan::Msg::IFmt("extent {} {}", Entry.GuestExtents[i], Entry.GuestExtents[i]+Entry.GuestExtents[i+1]); // } // } - if (Entry.GuestExtents.size() == 0) { - LiveGuestHash = XXH3_128bits(reinterpret_cast(GuestRIP), Entry.GuestSize); - } else { - XXH3_state_t HashState; - XXH3_128bits_reset(&HashState); - for (uint32_t i = 0; i < Entry.GuestExtents.size(); i += 2) { - XXH3_128bits_update(&HashState, reinterpret_cast(GuestRIP) + Entry.GuestExtents[i], Entry.GuestExtents[i + 1]); + fextl::vector GuestPages; + + XXH3_state_t HashState; + XXH3_128bits_reset(&HashState); + for (uint32_t i = 0; i < Entry.GuestExtents.size(); i += 2) { + XXH3_128bits_update(&HashState, reinterpret_cast(GuestRIP) + Entry.GuestExtents[i], Entry.GuestExtents[i + 1]); + uint64_t FirstPage = (Entry.GuestExtents[i] + GuestRIP) & Utils::FEX_PAGE_MASK; + uint64_t LastPage = (Entry.GuestExtents[i] + Entry.GuestExtents[i + 1] + GuestRIP) & Utils::FEX_PAGE_MASK; + for (uint64_t Page = FirstPage; Page <= LastPage; Page += Utils::FEX_PAGE_SIZE) { + if (GuestPages.size() == 0 || Page != GuestPages.back()) { + GuestPages.push_back(Page); + } } - LiveGuestHash = XXH3_128bits_digest(&HashState); } + LiveGuestHash = XXH3_128bits_digest(&HashState); + if (!XXH128_isEqual(LiveGuestHash, Entry.GuestHash)) { // LogMan::Msg::IFmt("hash mismatch! length {:d}", Header.GuestSize); return std::nullopt; @@ -498,8 +534,10 @@ namespace DiskCache { // this seems to be a full hit, pull from disk and check the entry is big enough to have everything (except GuestCode) CodeHitData HitData; uint32_t EntrySizeWithoutGuestCode = Entry.Size - Entry.GuestSize; - HitData.Blob.resize(EntrySizeWithoutGuestCode); - if (!Entry.DB->ReadCacheBlob(Entry.Offset, HitData.Blob)) { + HitData.Blob.resize(GuestPages.size() * sizeof(uint64_t) + EntrySizeWithoutGuestCode); + memcpy(HitData.Blob.data(), GuestPages.data(), GuestPages.size() * sizeof(uint64_t)); + uint32_t BlobOffset = GuestPages.size() * sizeof(uint64_t); + if (!Entry.DB->ReadCacheBlob(Entry.Offset, {HitData.Blob.data() + BlobOffset, EntrySizeWithoutGuestCode})) { return std::nullopt; } @@ -507,11 +545,11 @@ namespace DiskCache { return std::nullopt; } BlobFixedHeader Header; - memcpy(&Header, HitData.Blob.data(), sizeof(Header)); + memcpy(&Header, HitData.Blob.data() + BlobOffset, sizeof(Header)); + BlobOffset += sizeof(Header); uint32_t SizeNeeded = sizeof(Header) + Header.HostSize + Header.EntryPointCount * (sizeof(uint64_t) + sizeof(uint32_t)); - SizeNeeded += Header.SmallRelocCount * sizeof(BlobSmallRelocation) + Header.ThunkRelocCount * sizeof(BlobThunkRelocation) + - Header.TouchedGuestPagesCount * sizeof(uint64_t); + SizeNeeded += Header.SmallRelocCount * sizeof(BlobSmallRelocation) + Header.ThunkRelocCount * sizeof(BlobThunkRelocation); if (EntrySizeWithoutGuestCode != SizeNeeded) { return std::nullopt; } @@ -520,12 +558,8 @@ namespace DiskCache { return std::nullopt; } - uint32_t BlobOffset = sizeof(Header); - HitData.HostCode = {HitData.Blob.data() + BlobOffset, Header.HostSize}; BlobOffset += Header.HostSize; - HitData.GuestPages = {reinterpret_cast(HitData.Blob.data() + BlobOffset), Header.TouchedGuestPagesCount}; - BlobOffset += Header.TouchedGuestPagesCount * sizeof(uint64_t); HitData.EntryPointRIPs = {reinterpret_cast(HitData.Blob.data() + BlobOffset), Header.EntryPointCount}; BlobOffset += Header.EntryPointCount * sizeof(uint64_t); HitData.EntryPointHostOffsets = {reinterpret_cast(HitData.Blob.data() + BlobOffset), Header.EntryPointCount}; @@ -535,9 +569,8 @@ namespace DiskCache { HitData.ThunkRelocs = {reinterpret_cast(HitData.Blob.data() + BlobOffset), Header.ThunkRelocCount}; BlobOffset += Header.ThunkRelocCount * sizeof(BlobThunkRelocation); - for (auto& PageOffset : HitData.GuestPages) { - PageOffset += GuestRIP; - } + HitData.GuestPages = {reinterpret_cast(HitData.Blob.data()), GuestPages.size()}; + for (auto& EntryPointRip : HitData.EntryPointRIPs) { EntryPointRip += GuestRIP; } @@ -563,8 +596,8 @@ namespace DiskCache { } }; - bool DiskCache::Store(Core::InternalThreadState* Thread, const ExecutableFileSectionInfo& Region, uint64_t GuestRIP, - std::span GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode, + bool DiskCache::Store(Core::InternalThreadState* Thread, std::optional Region, uint64_t GuestRIP, + uint64_t GuestCodeKey, std::span GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode, std::span Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo) { if (!IsWritingDiskCache()) { return false; @@ -572,20 +605,19 @@ namespace DiskCache { if (!DecodedBlockInfo) { return false; } - // check for any reloc targets outside of our jurisdiction // todo what are they exactly? caching those blocks is great when it works, so need to figure this out and make finer-grained if we can - if (RelocationFilter) { + if (RelocationFilter && Region) { for (const auto& Reloc : Relocations) { if (Reloc.Header.Type != CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL && Reloc.Header.Type != CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE) { continue; } uint64_t Target = Reloc.GuestRIP.GuestRIP; - if (Target >= Region.BeginVA && Target < Region.EndVA) { + if (Target >= Region->BeginVA && Target < Region->EndVA) { continue; } auto TargetSection = CTX->SyscallHandler->LookupExecutableFileSection(Thread, Target); - if (!TargetSection || TargetSection->FileInfo.FileId != Region.FileInfo.FileId) { + if (!TargetSection || TargetSection->FileInfo.FileId != Region->FileInfo.FileId) { // we don't know where it's pointing, so we don't know how to encode the offset, so we can't cache atm return false; } @@ -606,6 +638,9 @@ namespace DiskCache { uint64_t CurStartExtent = 0, CurEndExtent = 0; const Frontend::Decoder::DecodedBlocks* LastBlock = nullptr; for (auto& SubBlock : DecodedBlockInfo->Blocks) { + if (SubBlock.BlockStatus != Frontend::Decoder::DecodedBlockStatus::SUCCESS) { + return false; + } if (!CurStartExtent) { CurStartExtent = SubBlock.Entry; CurEndExtent = SubBlock.Entry + SubBlock.Size; @@ -635,12 +670,10 @@ namespace DiskCache { // } const uint32_t EntryPointCount = (uint32_t)CompiledCode.EntryPoints.size(); - const uint32_t TouchedGuestPagesCount = DecodedBlockInfo ? (uint32_t)DecodedBlockInfo->CodePages.size() : 0; const size_t HeaderOffset = 0; const size_t HostCodeOffset = HeaderOffset + sizeof(BlobFixedHeader); - const size_t TouchedGuestPagesOffset = HostCodeOffset + CompiledCode.Size; - const size_t EntryPointRIPsOffset = TouchedGuestPagesOffset + TouchedGuestPagesCount * sizeof(uint64_t); + const size_t EntryPointRIPsOffset = HostCodeOffset + CompiledCode.Size; const size_t EntryPointHostOffsetsOffset = EntryPointRIPsOffset + EntryPointCount * sizeof(uint64_t); const size_t SmallRelocsOffset = EntryPointHostOffsetsOffset + EntryPointCount * sizeof(uint32_t); const size_t ThunkRelocsOffset = SmallRelocsOffset + SmallRelocCount * sizeof(BlobSmallRelocation); @@ -652,9 +685,7 @@ namespace DiskCache { Blob.resize(TotalSize); uint8_t* BlobData = Blob.data(); - uint64_t ModuleOffset = GuestRIP - Region.FileStartVA; - - uint64_t BlobKey = MakeBlobKey(ModuleOffset); + uint64_t BlobKey = MakeBlobKey(GuestCodeKey); MesaFOZ::foz_payload_key Key = {}; fextl::string BlobName = fextl::fmt::format("{:016x}", BlobKey); memcpy(Key.bytes, BlobName.data(), BlobName.size()); @@ -665,7 +696,6 @@ namespace DiskCache { .EntryPointCount = EntryPointCount, .SmallRelocCount = SmallRelocCount, .ThunkRelocCount = ThunkRelocCount, - .TouchedGuestPagesCount = TouchedGuestPagesCount, }; if (ExactGuestCodeExtents.size() == 0) { @@ -681,13 +711,6 @@ namespace DiskCache { memcpy(BlobData + HeaderOffset, &Header, sizeof(Header)); memcpy(BlobData + HostCodeOffset, CompiledCode.BlockBegin, CompiledCode.Size); - // relocate touched pages relative to GuestRIP - auto* PageOffsets = reinterpret_cast(BlobData + TouchedGuestPagesOffset); - uint32_t PageIdx = 0; - for (auto GuestPage : DecodedBlockInfo->CodePages) { - PageOffsets[PageIdx++] = GuestPage - GuestRIP; - } - // pack and relocate entrypoints auto* EntryRIPs = reinterpret_cast(BlobData + EntryPointRIPsOffset); auto* EntryHostOffsets = reinterpret_cast(BlobData + EntryPointHostOffsetsOffset); diff --git a/FEXCore/Source/Interface/Core/Frontend.cpp b/FEXCore/Source/Interface/Core/Frontend.cpp index c5efe3c8c..822fd9278 100644 --- a/FEXCore/Source/Interface/Core/Frontend.cpp +++ b/FEXCore/Source/Interface/Core/Frontend.cpp @@ -1380,107 +1380,67 @@ const Decoder::DecodeStream Decoder::AdjustAddrForSpecialRegion(const uint8_t* _ } bool Decoder::CheckIfCacheable(FEXCore::Core::InternalThreadState& Thread, const uint8_t* InstStream, uint64_t PC, uint64_t MaxInst) { - DecodeInstructionsAtEntry(&Thread, InstStream, PC, MaxInst); + SetupDecodeInstructionsAtEntry(&Thread, PC, MaxInst); + DecodeLoop(InstStream); bool Uncacheable = HitBadRelocation; DelayedDisownBuffer(); return !Uncacheable; } -void Decoder::DecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState* Thread, const uint8_t* _InstStream, uint64_t PC, uint64_t MaxInst) { - FEXCORE_PROFILE_SCOPED("DecodeInstructions"); - BlockInfo.TotalInstructionCount = 0; - BlockInfo.Blocks.clear(); - VisitedBlocks.clear(); - // Reset internal state management - DecodedSize = 0; - MaxCondBranchForward = 0; - MaxCondBranchBackwards = ~0ULL; - DecodedBuffer = PoolObject.ReownOrClaimBuffer(); +void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) { + while (!FinalInstruction && (Paused || !BlocksToDecode.empty())) { + bool Pausing = false; + fextl::vector::iterator BlockIt; + if (!Paused || BlockResume == -1) { + auto BlockDecodeIt = BlocksToDecode.begin(); + uint64_t RIPToDecode = *BlockDecodeIt; + BlocksToDecode.erase(BlockDecodeIt); + VisitedBlocks.emplace(RIPToDecode); - // Decode operating mode from thread's CS segment. - const auto CSSegment = Core::CPUState::GetSegmentFromIndex(Thread->CurrentFrame->State, Thread->CurrentFrame->State.cs_idx); - BlockInfo.Is64BitMode = CSSegment->L == 1; - LOGMAN_THROW_A_FMT(BlockInfo.Is64BitMode == CTX->Config.Is64BitMode, "Expected operating mode to not change at runtime!"); + auto BlockSuccIt = std::lower_bound(BlockInfo.Blocks.begin(), BlockInfo.Blocks.end(), RIPToDecode, + [](const auto& a, uint64_t Address) { return a.Entry < Address; }); - EntryPoint = PC; - BlockInfo.EntryPoints = {PC}; + LOGMAN_THROW_A_FMT(BlockSuccIt == BlockInfo.Blocks.end() || BlockSuccIt->Entry != RIPToDecode, "unexpected"); - uint64_t TotalInstructions {}; + NextBlockStartAddress = ~0ULL; + if (!BlocksToDecode.empty()) { + // We just erased the lowest, the front is then the second lowest + NextBlockStartAddress = *BlocksToDecode.begin(); + } + if (BlockSuccIt != BlockInfo.Blocks.end() && BlockSuccIt->Entry < NextBlockStartAddress) { + NextBlockStartAddress = BlockSuccIt->Entry; + } - SectionMinAddress = 0; - SectionMaxAddress = ~0ULL; - Relocations = nullptr; + LOGMAN_THROW_A_FMT(NextBlockStartAddress == ~0ULL || NextBlockStartAddress > RIPToDecode, "unexpected"); - if (CTX->GetCodeCache().IsGeneratingCache || EnableCodeCacheValidation) { - // If generating cache, attempt to load section bounds and relocations - if (auto SectionInfo = CTX->SyscallHandler->LookupExecutableFileSection(Thread, EntryPoint)) { - SectionMinAddress = SectionInfo->FileStartVA; - SectionMaxAddress = SectionInfo->EndVA; - Relocations = &SectionInfo->FileInfo.Relocations; - } - } + // Insert the block now so it can be looked up and split if necessary on a backward edge + BlockIt = BlockInfo.Blocks.emplace(BlockSuccIt); - DecodedMinAddress = EntryPoint; - DecodedMaxAddress = EntryPoint; + BlockIt->Entry = RIPToDecode; + BlockIt->Size = 0; + BlockIt->IsEntryPoint = EntryBlock; - // Entry is a jump target - BlocksToDecode = {PC}; + PCOffset = 0; + BlockStartOffset = DecodedSize; + EraseBlock = true; // Unset once the block contains an instruction - uint64_t CurrentCodePage = PC & FEXCore::Utils::FEX_PAGE_MASK; + BlockIt->DecodedInstructions = &DecodedBuffer[BlockStartOffset]; + BlockIt->NumInstructions = 0; - BlockInfo.CodePages = {CurrentCodePage}; - - if (MaxInst == 0) { - MaxInst = CTX->Config.MaxInstPerBlock; - } - - bool EntryBlock {true}; - bool FinalInstruction {false}; - - while (!FinalInstruction && !BlocksToDecode.empty()) { - auto BlockDecodeIt = BlocksToDecode.begin(); - uint64_t RIPToDecode = *BlockDecodeIt; - BlocksToDecode.erase(BlockDecodeIt); - VisitedBlocks.emplace(RIPToDecode); - - auto BlockSuccIt = std::lower_bound(BlockInfo.Blocks.begin(), BlockInfo.Blocks.end(), RIPToDecode, - [](const auto& a, uint64_t Address) { return a.Entry < Address; }); - - LOGMAN_THROW_A_FMT(BlockSuccIt == BlockInfo.Blocks.end() || BlockSuccIt->Entry != RIPToDecode, "unexpected"); - - NextBlockStartAddress = ~0ULL; - if (!BlocksToDecode.empty()) { - // We just erased the lowest, the front is then the second lowest - NextBlockStartAddress = *BlocksToDecode.begin(); - } - if (BlockSuccIt != BlockInfo.Blocks.end() && BlockSuccIt->Entry < NextBlockStartAddress) { - NextBlockStartAddress = BlockSuccIt->Entry; + // Do a bit of pointer math to figure out where we are in code + InstStream = AdjustAddrForSpecialRegion(_InstStream, EntryPoint, RIPToDecode); + } else if (BlockResume != -1) { + BlockIt = BlockInfo.Blocks.begin() + BlockResume; + BlockResume = -1; } - LOGMAN_THROW_A_FMT(NextBlockStartAddress == ~0ULL || NextBlockStartAddress > RIPToDecode, "unexpected"); - - // Insert the block now so it can be looked up and split if necessary on a backward edge - auto BlockIt = BlockInfo.Blocks.emplace(BlockSuccIt); - - BlockIt->Entry = RIPToDecode; - BlockIt->Size = 0; - BlockIt->IsEntryPoint = EntryBlock; - - uint64_t PCOffset = 0; - uint64_t BlockStartOffset = DecodedSize; - bool EraseBlock = true; // Unset once the block contains an instruction - - BlockIt->DecodedInstructions = &DecodedBuffer[BlockStartOffset]; - BlockIt->NumInstructions = 0; - - // Do a bit of pointer math to figure out where we are in code - InstStream = AdjustAddrForSpecialRegion(_InstStream, EntryPoint, RIPToDecode); + Paused = false; while (1) { InstructionSize = 0; // MAX_INST_SIZE assumes worst case - auto OpAddress = RIPToDecode + PCOffset; + auto OpAddress = BlockIt->Entry + PCOffset; auto OpMaxAddress = OpAddress + MAX_INST_SIZE; auto OpMinPage = OpAddress & FEXCore::Utils::FEX_PAGE_MASK; @@ -1548,6 +1508,15 @@ void Decoder::DecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState* Thre break; } + if (GuestSizePause) { + if (GuestSizePause > DecodeInst->InstSize) { + GuestSizePause -= DecodeInst->InstSize; + } else { + GuestSizePause = 0; + Pausing = true; + } + } + // Check if we need to end the entire multiblock FinalInstruction = DecodedSize >= MaxInst || DecodedSize >= DefaultDecodedBufferSize || TotalInstructions >= MaxInst; if (FinalInstruction) { @@ -1569,6 +1538,17 @@ void Decoder::DecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState* Thre PCOffset += DecodeInst->InstSize; InstStream += DecodeInst->InstSize; + + if (Pausing) { + Pausing = false; + Paused = true; + BlockResume = BlockIt - BlockInfo.Blocks.begin(); + break; + } + } + + if (Paused) { + break; } // NOTE: BlockIt is only valid here in the EraseBlock case @@ -1580,6 +1560,16 @@ void Decoder::DecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState* Thre CurrentBlockTargets.clear(); EntryBlock = false; + + if (Pausing && !BlocksToDecode.empty() && !FinalInstruction) { + Paused = true; + BlockResume = -1; + break; + } + } + + if (Paused) { + return; } BlockInfo.TotalInstructionCount = TotalInstructions; @@ -1589,4 +1579,58 @@ void Decoder::DecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState* Thre } } +void Decoder::SetupDecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t PC, uint64_t MaxInst) { + FEXCORE_PROFILE_SCOPED("DecodeInstructions"); + BlockInfo.TotalInstructionCount = 0; + BlockInfo.Blocks.clear(); + VisitedBlocks.clear(); + // Reset internal state management + Paused = false; + BlockResume = -1; + DecodedSize = 0; + if (MaxInst == 0) { + MaxInst = CTX->Config.MaxInstPerBlock; + } + this->MaxInst = MaxInst; + MaxCondBranchForward = 0; + MaxCondBranchBackwards = ~0ULL; + DecodedBuffer = PoolObject.ReownOrClaimBuffer(); + + // Decode operating mode from thread's CS segment. + const auto CSSegment = Core::CPUState::GetSegmentFromIndex(Thread->CurrentFrame->State, Thread->CurrentFrame->State.cs_idx); + BlockInfo.Is64BitMode = CSSegment->L == 1; + LOGMAN_THROW_A_FMT(BlockInfo.Is64BitMode == CTX->Config.Is64BitMode, "Expected operating mode to not change at runtime!"); + + EntryPoint = PC; + BlockInfo.EntryPoints = {PC}; + + TotalInstructions = 0; + + SectionMinAddress = 0; + SectionMaxAddress = ~0ULL; + Relocations = nullptr; + + if (CTX->GetCodeCache().IsGeneratingCache || EnableCodeCacheValidation) { + // If generating cache, attempt to load section bounds and relocations + if (auto SectionInfo = CTX->SyscallHandler->LookupExecutableFileSection(Thread, EntryPoint)) { + SectionMinAddress = SectionInfo->FileStartVA; + SectionMaxAddress = SectionInfo->EndVA; + Relocations = &SectionInfo->FileInfo.Relocations; + } + } + + DecodedMinAddress = EntryPoint; + DecodedMaxAddress = EntryPoint; + + // Entry is a jump target + BlocksToDecode = {PC}; + + CurrentCodePage = PC & FEXCore::Utils::FEX_PAGE_MASK; + + BlockInfo.CodePages = {CurrentCodePage}; + + EntryBlock = true; + FinalInstruction = false; +} + } // namespace FEXCore::Frontend diff --git a/FEXCore/Source/Interface/Core/Frontend.h b/FEXCore/Source/Interface/Core/Frontend.h index 7a1e00f26..fe846347d 100644 --- a/FEXCore/Source/Interface/Core/Frontend.h +++ b/FEXCore/Source/Interface/Core/Frontend.h @@ -54,7 +54,8 @@ public: Decoder(FEXCore::Core::InternalThreadState* Thread); bool CheckIfCacheable(FEXCore::Core::InternalThreadState&, const uint8_t* InstStream, uint64_t PC, uint64_t MaxInst); - void DecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState* Thread, const uint8_t* InstStream, uint64_t PC, uint64_t MaxInst); + void SetupDecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t PC, uint64_t MaxInst); + void DecodeLoop(const uint8_t* InstStream, uint64_t GuestPause = 0); const DecodedBlockInformation* GetDecodedBlockInfo() const { return &BlockInfo; @@ -117,6 +118,16 @@ private: FEXCore::X86Tables::DecodedInst* DecodedBuffer {}; Utils::PoolBufferWithTimedRetirement PoolObject; size_t DecodedSize {}; + uint64_t TotalInstructions {}; + uint64_t CurrentCodePage {}; + bool EntryBlock {}; + bool FinalInstruction {}; + uint64_t MaxInst {}; + bool Paused {}; + int64_t BlockResume = -1; + uint64_t PCOffset {}; + uint64_t BlockStartOffset {}; + bool EraseBlock {}; uint64_t ExecutableRangeBase {}; uint64_t ExecutableRangeEnd {}; diff --git a/FEXCore/include/FEXCore/Core/DiskCache.h b/FEXCore/include/FEXCore/Core/DiskCache.h index e5ba84253..4d5654030 100644 --- a/FEXCore/include/FEXCore/Core/DiskCache.h +++ b/FEXCore/include/FEXCore/Core/DiskCache.h @@ -67,7 +67,6 @@ namespace DiskCache { uint32_t EntryPointCount; uint32_t SmallRelocCount; uint32_t ThunkRelocCount; - uint32_t TouchedGuestPagesCount; XXH128_hash_t GuestHash; }; @@ -99,7 +98,7 @@ namespace DiskCache { struct CodeHitData { fextl::vector Blob; std::span HostCode; - std::span GuestPages; + std::span GuestPages; std::span EntryPointRIPs; std::span EntryPointHostOffsets; std::span SmallRelocs; @@ -170,8 +169,9 @@ namespace DiskCache { public: void Init(FEXCore::Context::ContextImpl* CTX); - std::optional Lookup(Core::InternalThreadState* Thread, const ExecutableFileSectionInfo& Region, uint64_t GuestRIP); - bool Store(Core::InternalThreadState* Thread, const ExecutableFileSectionInfo& Region, uint64_t GuestRIP, + std::optional Lookup(Core::InternalThreadState* Thread, std::optional Region, uint64_t GuestRIP, + std::optional& GuestCodeKey); + bool Store(Core::InternalThreadState* Thread, std::optional Region, uint64_t GuestRIP, uint64_t GuestCodeKey, std::span GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode, std::span Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo); @@ -201,13 +201,16 @@ namespace DiskCache { FEX_CONFIG_OPT(EnableDiskCache, DISKCACHE); FEX_CONFIG_OPT(MapDiskCacheFiles, DISKCACHEFILEMAPPING); FEX_CONFIG_OPT(RelocationFilter, DISKCACHERELOCATIONFILTER); + FEX_CONFIG_OPT(AnonCaching, DISKCACHEANONCACHING); FEX_CONFIG_OPT(BasePathOverride, DISKCACHEPATH); FEX_CONFIG_OPT(RODBNames, DISKCACHERODBNAMES); }; + static constexpr uint16_t AnonPrefixGuestBytes = 64; + // TODO: This header is in global installed header path, but uses internal headers. // Migrate this once that is fixed. - static constexpr uint16_t FormatVersion = 10; + static constexpr uint16_t FormatVersion = 11; FEX_DEFAULT_VISIBILITY uint16_t GetFormatVersion(); } // namespace DiskCache