steam-frame/FEX-Emu--FEX
1
0 Fork 0
mirror of https://github.com/FEX-Emu/FEX.git synced 2026-10-06 08:00:15 +02:00
Code Issues Packages Projects Releases 66 Wiki Activity

DiskCache: optional validation-only mode

Browse Source 
Diffs guest code in lookup miss hash mismatches and host code after lookup hit.
This commit is contained in:
Pierre-Loup A. Griffais committed 2026-09-03 23:31:05 -07:00
1 parent cfef3d2cbf
commit 40bfff985f
4 files changed
+110 -19
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

FEXCore/Source/Interface/Config/Config.json.in
+8
View File
@@ -160,6 +160,14 @@
"Maps cache files for faster reading"
]
},
"DiskCacheValidation": {
"Type": "bool",
"Default": "false",
"AffectsCodeGen": "false",
"Desc": [
"Debug mode that does nothing but validate code hits"
]
},
"DiskCacheRelocationFilter": {
"Type": "bool",
"Default": "true",
FEXCore/Source/Interface/Core/Core.cpp
+10 -13
View File
@@ -894,22 +894,15 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
std::optional<ExecutableFileSectionInfo> Region = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP);
std::optional<DiskCache::CodeHitData> Hit;
std::optional<uint64_t> DiskCacheGuestCodeKey;
bool DiskCacheHitRelocationsApplied = false;
bool LoadDiskCacheCode = true;
{
FEXCORE_PROFILE_ACCUMULATION(Thread, AccumulatedDiskCacheLookupTime);
Hit = DiskCache.Lookup(Thread, Region, GuestRIP, DiskCacheGuestCodeKey);
if (Hit) {
DiskCacheHitRelocationsApplied =
CodeCache.ApplyPackedCodeRelocations(GuestRIP, std::as_writable_bytes(Hit->HostCode), Hit->SmallRelocs, Hit->ThunkRelocs, false);
if (DiskCacheHitRelocationsApplied && LoadDiskCacheCode) {
auto LoadedCode = Thread->CPUBackend->LoadCachedCode(Hit->HostCode);
if (LoadedCode.BlockBegin) {
for (auto& CodePage : Hit->GuestPages) {
if (Thread->LookupCache->AddBlockExecutableRange(Thread, Hit->EntryPointRIPs, CodePage, FEXCore::Utils::FEX_PAGE_SIZE)) {
SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE);
}
if (Hit && !DiskCache.IsValidating()) {
auto LoadedCode = Thread->CPUBackend->LoadCachedCode(Hit->HostCode);
if (LoadedCode.BlockBegin) {
for (auto& CodePage : Hit->GuestPages) {
if (Thread->LookupCache->AddBlockExecutableRange(Thread, Hit->EntryPointRIPs, CodePage, FEXCore::Utils::FEX_PAGE_SIZE)) {
SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE);
}
LOGMAN_THROW_A_FMT(Hit->EntryPointRIPs.size() == Hit->EntryPointHostOffsets.size(), "Mismatched Disk Cache entrypoint pairs!");
@@ -949,6 +942,10 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
return reinterpret_cast<uintptr_t>(CodePtr);
}
if (DiskCacheGuestCodeKey && Hit && DiskCache.IsValidating()) {
DiskCache.Validate(*DiskCacheGuestCodeKey, *Hit, CompiledCode, Region);
}
// if this ever fires, we need to serialize the offset into disk cache
LOGMAN_THROW_A_FMT(StartAddr == GuestRIP, "StartAddr offset from GuestRIP");
FEXCore/Source/Interface/Core/DiskCache.cpp
+86 -4
View File
@@ -526,10 +526,47 @@ namespace DiskCache {
LiveGuestHash = XXH3_128bits_digest(&HashState);
if (!XXH128_isEqual(LiveGuestHash, Entry.GuestHash)) {
// LogMan::Msg::IFmt("hash mismatch! length {:d}", Header.GuestSize);
if (Validation) {
fextl::vector<uint8_t> GuestCode(Entry.GuestSize);
if (Entry.Size >= Entry.GuestSize && Entry.DB->ReadCacheBlob(Entry.Offset + Entry.Size - Entry.GuestSize, GuestCode)) {
const uint8_t* CachedGuest = GuestCode.data();
const uint8_t* LiveGuest = reinterpret_cast<const uint8_t*>(GuestRIP);
uint64_t DiffCount = 0;
uint64_t DiffOffset = 0;
for (uint32_t i = 0; i < Entry.GuestExtents.size(); i += 2) {
uint32_t Begin = Entry.GuestExtents[i];
uint32_t End = Begin + Entry.GuestExtents[i + 1];
bool PreviousByteDiff = false;
for (uint32_t Offset = Begin; Offset < End; Offset++) {
if (LiveGuest[Offset] != CachedGuest[Offset]) {
if (DiffCount == 0) {
DiffOffset = Offset;
}
if (!PreviousByteDiff) {
DiffCount++;
}
PreviousByteDiff = true;
} else {
PreviousByteDiff = false;
}
}
}
if (DiffCount) {
uint64_t DiffStart = DiffOffset >= 8 ? DiffOffset - 8 : 0;
uint64_t DiffContextBytes = std::min<uint64_t>(16, Entry.GuestSize - DiffStart);
auto LiveDump = fmt::join(std::span<const uint8_t> {LiveGuest + DiffStart, DiffContextBytes}, " ");
auto CacheDump = fmt::join(std::span<const uint8_t> {CachedGuest + DiffStart, DiffContextBytes}, " ");
auto KeyPrefix = (Region && Region->FileStartVA) ? "file" : "anon";
LogMan::Msg::IFmt("DiskCache: lookup guest hash mismatch key={}-{:x} gsize={}, ndiff={}, first diff: offset={} live=[{:02x}] "
"cached=[{:02x}]",
KeyPrefix, *GuestCodeKey, Entry.GuestSize, DiffCount, DiffOffset, LiveDump, CacheDump);
} else {
LogMan::Msg::IFmt("DiskCache: guest hash mismatch but no diff?");
}
}
}
return std::nullopt;
}
// LogMan::Msg::IFmt("hash ok! length {:d}", Header.GuestSize);
// this seems to be a full hit, pull from disk and check the entry is big enough to have everything (except GuestCode)
CodeHitData HitData;
@@ -564,9 +601,11 @@ namespace DiskCache {
BlobOffset += Header.EntryPointCount * sizeof(uint64_t);
HitData.EntryPointHostOffsets = {reinterpret_cast<const uint32_t*>(HitData.Blob.data() + BlobOffset), Header.EntryPointCount};
BlobOffset += Header.EntryPointCount * sizeof(uint32_t);
HitData.SmallRelocs = {reinterpret_cast<const BlobSmallRelocation*>(HitData.Blob.data() + BlobOffset), Header.SmallRelocCount};
std::span<const BlobSmallRelocation> SmallRelocs = {reinterpret_cast<const BlobSmallRelocation*>(HitData.Blob.data() + BlobOffset),
Header.SmallRelocCount};
BlobOffset += Header.SmallRelocCount * sizeof(BlobSmallRelocation);
HitData.ThunkRelocs = {reinterpret_cast<const BlobThunkRelocation*>(HitData.Blob.data() + BlobOffset), Header.ThunkRelocCount};
std::span<const BlobThunkRelocation> ThunkRelocs = {reinterpret_cast<const BlobThunkRelocation*>(HitData.Blob.data() + BlobOffset),
Header.ThunkRelocCount};
BlobOffset += Header.ThunkRelocCount * sizeof(BlobThunkRelocation);
HitData.GuestPages = {reinterpret_cast<uint64_t*>(HitData.Blob.data()), GuestPages.size()};
@@ -575,9 +614,52 @@ namespace DiskCache {
EntryPointRip += GuestRIP;
}
if (!CTX->CodeCache.ApplyPackedCodeRelocations(GuestRIP, std::as_writable_bytes(HitData.HostCode), SmallRelocs, ThunkRelocs, false)) {
return std::nullopt;
}
return HitData;
}
void DiskCache::Validate(uint64_t GuestCodeKey, const CodeHitData& Hit, const CPU::CPUBackend::CompiledCode& CompiledCode,
std::optional<ExecutableFileSectionInfo> Region) {
if (!Validation) {
return;
}
auto KeyPrefix = (Region && Region->FileStartVA) ? "file" : "anon";
if (Hit.HostCode.size() != CompiledCode.Size) {
LogMan::Msg::EFmt("DiskCache: validate host size mismatch key={}-{:x} cached={} live={}", KeyPrefix, GuestCodeKey,
Hit.HostCode.size(), CompiledCode.Size);
} else if (memcmp(Hit.HostCode.data(), CompiledCode.BlockBegin, CompiledCode.Size) != 0) {
bool PreviousByteDiff = false;
size_t FirstDiff = 0;
size_t DiffCount = 0;
for (size_t i = 0; i < CompiledCode.Size; i++) {
if (Hit.HostCode[i] != CompiledCode.BlockBegin[i]) {
if (DiffCount == 0) {
FirstDiff = i;
}
if (!PreviousByteDiff) {
DiffCount++;
}
PreviousByteDiff = true;
} else {
PreviousByteDiff = false;
}
}
// align to 4 bytes to read host arm easier
const size_t DiffStart = (FirstDiff & ~3ULL) >= 8 ? (FirstDiff & ~3ULL) - 8 : 0;
const size_t ContextBytes = std::min<size_t>(16, CompiledCode.Size - DiffStart);
LogMan::Msg::EFmt("DiskCache: validate host code mismatch key={}-{:x} size={} firstoffset={} ndiff={} cached=[{:02x}] live=[{:02x}]",
KeyPrefix, GuestCodeKey, CompiledCode.Size, FirstDiff, DiffCount,
fmt::join(std::span<const uint8_t> {Hit.HostCode.data() + DiffStart, ContextBytes}, " "),
fmt::join(std::span<const uint8_t> {CompiledCode.BlockBegin + DiffStart, ContextBytes}, " "));
}
}
struct DiskCache::CacheStoreWorkItem final : WorkQueueThread::WorkItem {
DiskCache* Self;
IndexedDB* DB;
FEXCore/include/FEXCore/Core/DiskCache.h
+6 -2
View File
@@ -101,8 +101,6 @@ namespace DiskCache {
std::span<const uint64_t> GuestPages;
std::span<uint64_t> EntryPointRIPs;
std::span<const uint32_t> EntryPointHostOffsets;
std::span<const BlobSmallRelocation> SmallRelocs;
std::span<const BlobThunkRelocation> ThunkRelocs;
// the spans above point to memory owned by the Blob vec, so it's important this can't be copied
CodeHitData() = default;
@@ -171,6 +169,8 @@ namespace DiskCache {
std::optional<CodeHitData> Lookup(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region, uint64_t GuestRIP,
std::optional<uint64_t>& GuestCodeKey);
void Validate(uint64_t GuestCodeKey, const CodeHitData& Hit, const CPU::CPUBackend::CompiledCode& CompiledCode,
std::optional<ExecutableFileSectionInfo> Region);
bool Store(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region, uint64_t GuestRIP, uint64_t GuestCodeKey,
std::span<const uint8_t> GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode,
std::span<const FEXCore::CPU::Relocation> Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo);
@@ -181,6 +181,9 @@ namespace DiskCache {
bool IsReadingDiskCache() const {
return !ROCacheDBs.empty() || RWCacheDB != nullptr;
}
bool IsValidating() const {
return Validation;
}
private:
bool OpenCacheDB(const fextl::string& CacheDBName, bool ReadOnly);
@@ -199,6 +202,7 @@ namespace DiskCache {
fextl::unique_ptr<WorkQueueThread> Writer;
FEX_CONFIG_OPT(EnableDiskCache, DISKCACHE);
FEX_CONFIG_OPT(Validation, DISKCACHEVALIDATION);
FEX_CONFIG_OPT(MapDiskCacheFiles, DISKCACHEFILEMAPPING);
FEX_CONFIG_OPT(RelocationFilter, DISKCACHERELOCATIONFILTER);
FEX_CONFIG_OPT(AnonCaching, DISKCACHEANONCACHING);