steam-frame/FEX-Emu--FEX
1
0 Fork 0
mirror of https://github.com/FEX-Emu/FEX.git synced 2026-10-06 10:00:16 +02:00
Code Issues Packages Projects Releases 66 Wiki Activity
Files
d3dcc740b80824f7c883bb61dc668e9370bb26ba
FEX-Emu--FEX/FEXCore/Source/Interface/Core/DiskCache.cpp
T
Ryan Houdek 47557a6184 FEXCore/DiskCache: Prune old entries automatically 
Adds a config option to disable pruning if desired.

Had to change FEXCore::HostFeatures slightly to remove the HostType from
the Hash for caching, as we don't want the bucket to inadvertently
delete wow64,arm64ec,linux buckets when executing in the same namespace.

Minor changes to the DiskCache::Init code to handle returning two
values, moving the HostType to be a part of the "ProcessBucketHash"
instead of the "MachineBucketHash".

Then uses the directory walking function added in the previous commit to
find mismatched/old "MachineBucketHash" directories and remove them.
2026-09-24 15:21:48 -07:00

1257 lines
49 KiB
C++
Raw Blame History

// SPDX-License-Identifier: MIT
#define XXH_STATIC_LINKING_ONLY
#include "Interface/Core/Frontend.h"
#include "Interface/Context/Context.h"
#include <FEXCore/Config/Config.h>
#include <FEXCore/Core/DiskCache.h>
#include <FEXCore/Core/DiskCacheFileMapper.h>
#include <FEXCore/fextl/memory.h>
#include <FEXCore/fextl/string.h>
#include <FEXCore/HLE/SyscallHandler.h>
#include <FEXCore/Utils/File.h>
#include <FEXCore/Utils/FileUtils.h>
#include <FEXCore/Utils/LogManager.h>
#include <FEXCore/Utils/Profiler.h>
#include <FEXCore/Utils/TypeDefines.h>
#include <FEXHeaderUtils/Filesystem.h>
#include <cstdint>
#include <cstring>
#include <atomic>
namespace FEXCore {
namespace DiskCache {
namespace MesaFOZ {
enum { FOSSILIZE_COMPRESSION_NONE = 1, FOSSILIZE_COMPRESSION_DEFLATE = 2 };
enum { FOSSILIZE_FORMAT_VERSION = 6, FOSSILIZE_FORMAT_MIN_COMPAT_VERSION = 5 };
#define FOZ_REF_MAGIC_SIZE 16
static const uint8_t stream_reference_magic_and_version[FOZ_REF_MAGIC_SIZE] = {
0x81, 'F', 'O', 'S', 'S', 'I', 'L', 'I', 'Z', 'E', 'D', 'B', 0, 0, 0, FOSSILIZE_FORMAT_VERSION, /* 4 bytes to use for versioning. */
};
struct __attribute__((packed)) mesa_index_db_file_entry {
uint64_t hash;
uint32_t size;
uint64_t last_access_time;
uint64_t cache_db_file_offset;
};
} // namespace MesaFOZ
struct __attribute__((packed)) IndexExtraBlobHeader {
XXH128_hash_t GuestHash; // StoreCacheBlob will assume it's here, care if moving
uint64_t GuestFootprint;
uint32_t GuestSize;
uint32_t GuestExtentsCount;
};
static FileMapperFunc FileMapper = nullptr;
bool FOZFile::Open(const fextl::string& FOZFileName, bool ReadOnly) {
FileName = FOZFileName;
this->ReadOnly = ReadOnly;
File::FileModes Modes = File::FileModes::READ;
if (!ReadOnly) {
Modes = Modes | File::FileModes::WRITE | File::FileModes::CREATE;
}
FD = fextl::make_unique<File::File>(FileName.c_str(), Modes, false);
if (!FD->IsValid()) {
FD.reset();
return false;
}
bool Valid = false;
bool TookLock = false;
ssize_t Size = FD->Size();
if (Size < FOZ_REF_MAGIC_SIZE && !ReadOnly) {
if (!FD->Lock(OPEN_LOCK_TIMEOUT_MS)) {
FD.reset();
return false;
}
TookLock = true;
// check size again in case someone else made it while we waited
Size = FD->Size();
}
if (Size == 0 && !ReadOnly) {
Valid = FD->PWrite(MesaFOZ::stream_reference_magic_and_version, FOZ_REF_MAGIC_SIZE, 0) == FOZ_REF_MAGIC_SIZE;
} else {
uint8_t magic[FOZ_REF_MAGIC_SIZE];
if (FD->PRead(magic, FOZ_REF_MAGIC_SIZE, 0) == FOZ_REF_MAGIC_SIZE &&
memcmp(magic, MesaFOZ::stream_reference_magic_and_version, FOZ_REF_MAGIC_SIZE - 1) == 0) {
int version = magic[FOZ_REF_MAGIC_SIZE - 1];
Valid = version <= MesaFOZ::FOSSILIZE_FORMAT_VERSION && version >= MesaFOZ::FOSSILIZE_FORMAT_MIN_COMPAT_VERSION;
}
}
if (TookLock) {
FD->Unlock();
}
if (!Valid) {
FD.reset();
}
return Valid;
}
ssize_t FOZFile::Size() {
return FD ? FD->Size() : -1;
}
bool FOZFile::ReadAll(fextl::vector<uint8_t>& Out) {
ssize_t FileSize = Size();
if (FileSize < FOZ_REF_MAGIC_SIZE) {
return false;
}
Out.resize((size_t)FileSize - FOZ_REF_MAGIC_SIZE);
return FD->PRead(Out.data(), Out.size(), FOZ_REF_MAGIC_SIZE) == (ssize_t)Out.size();
}
bool FOZFile::ReadBlob(uint64_t Offset, std::span<uint8_t> OutBlob) {
if (FD->PRead(OutBlob.data(), OutBlob.size(), Offset) != (ssize_t)OutBlob.size()) {
return false;
}
return true;
}
bool FOZFile::WriteBlob(const MesaFOZ::foz_payload_key& Key, std::span<const std::span<const uint8_t>> BlobChunks, uint64_t& OutBlobOffset) {
ssize_t FileSize = FD->Size();
if (FileSize < 0) {
return false;
}
uint64_t WriteOffset = (uint64_t)FileSize;
if (FD->PWrite(Key.bytes, sizeof(Key.bytes), WriteOffset) != sizeof(Key.bytes)) {
return false;
}
WriteOffset += sizeof(Key.bytes);
uint64_t TotalBlobSize = 0;
for (const std::span<const uint8_t>& Chunk : BlobChunks) {
TotalBlobSize += Chunk.size();
}
MesaFOZ::foz_payload_header ScratchHeader {.payload_size = (uint32_t)TotalBlobSize,
.format = MesaFOZ::FOSSILIZE_COMPRESSION_NONE,
.crc = 0, // todo? maybe
.uncompressed_size = (uint32_t)TotalBlobSize};
if (FD->PWrite(&ScratchHeader, sizeof(ScratchHeader), WriteOffset) != sizeof(ScratchHeader)) {
return false;
}
WriteOffset += sizeof(ScratchHeader);
OutBlobOffset = WriteOffset;
for (const std::span<const uint8_t>& Chunk : BlobChunks) {
if (Chunk.size() == 0) {
continue;
}
if (FD->PWrite(Chunk.data(), Chunk.size(), WriteOffset) != (ssize_t)Chunk.size()) {
return false;
}
WriteOffset += Chunk.size();
}
return true;
}
bool IndexedDB::Open(const fextl::string& CacheDBName, bool ReadOnly) {
if (!CacheFOZ.Open(CacheDBName + ".foz", ReadOnly)) {
return false;
}
if (!IndexFOZ.Open(CacheDBName + "_idx.foz", ReadOnly)) {
return false;
}
File::File::FileHandleType CacheFileHandle = CacheFOZ.GetHandle();
if (FileMapper && CacheFileHandle != (File::File::FileHandleType)-1) {
CacheFileMapping = reinterpret_cast<uint8_t*>(FileMapper(CacheFileHandle, ReadOnly ? CacheFOZ.Size() : BIG_MAPPING_SIZE));
CacheFileSize = CacheFOZ.Size();
}
this->ReadOnly = ReadOnly;
return true;
}
void IndexedDB::PopulateIndex(Index& CacheIndex, bool& FoundMetadata) {
fextl::vector<uint8_t> Data;
if (!IndexFOZ.ReadAll(Data)) {
return;
}
ssize_t CacheFOZSize = CacheFOZ.Size();
if (CacheFOZSize < 0) {
return;
}
const uint8_t* IndexDataStart = Data.data();
const size_t IndexDataSize = Data.size();
size_t ReadOffset = 0;
while (ReadOffset + sizeof(MesaFOZ::foz_payload_key) + sizeof(MesaFOZ::foz_payload_header) <= IndexDataSize) {
const auto* FOZKey = reinterpret_cast<const MesaFOZ::foz_payload_key*>(IndexDataStart + ReadOffset);
ReadOffset += sizeof(MesaFOZ::foz_payload_key);
const auto* FOZHeader = reinterpret_cast<const MesaFOZ::foz_payload_header*>(IndexDataStart + ReadOffset);
ReadOffset += sizeof(MesaFOZ::foz_payload_header);
uint64_t IndexBlobSize = sizeof(MesaFOZ::mesa_index_db_file_entry) + sizeof(IndexExtraBlobHeader);
if (FOZHeader->payload_size < IndexBlobSize || ReadOffset + FOZHeader->payload_size > IndexDataSize) {
break;
}
const auto* IndexBlobCommon = reinterpret_cast<const MesaFOZ::mesa_index_db_file_entry*>(IndexDataStart + ReadOffset);
ReadOffset += sizeof(MesaFOZ::mesa_index_db_file_entry);
const auto* IndexBlobExtra = reinterpret_cast<const IndexExtraBlobHeader*>(IndexDataStart + ReadOffset);
ReadOffset += sizeof(IndexExtraBlobHeader);
// skip corrupt (carefully) so we don't have to figure that out in the hot path later
if (IndexBlobCommon->cache_db_file_offset > (uint64_t)CacheFOZSize ||
IndexBlobCommon->size > (uint64_t)CacheFOZSize - IndexBlobCommon->cache_db_file_offset) {
continue;
}
if (IndexBlobExtra->GuestSize + sizeof(BlobFixedHeader) > IndexBlobCommon->size) {
continue;
}
IndexBlobSize += IndexBlobExtra->GuestExtentsCount * sizeof(uint32_t);
if (FOZHeader->payload_size != IndexBlobSize) {
break;
}
const auto* GuestExtents = reinterpret_cast<const uint32_t*>(IndexDataStart + ReadOffset);
ReadOffset += IndexBlobExtra->GuestExtentsCount * sizeof(uint32_t);
if (FOZKey->bytes[39] != 0xFF) {
IndexEntry NewEntry {this, IndexBlobCommon->cache_db_file_offset, IndexBlobCommon->size, IndexBlobExtra->GuestSize,
IndexBlobExtra->GuestHash};
NewEntry.GuestExtents.resize(IndexBlobExtra->GuestExtentsCount);
memcpy(NewEntry.GuestExtents.data(), GuestExtents, IndexBlobExtra->GuestExtentsCount * sizeof(uint32_t));
bool ExtentsValid = true;
for (uint32_t i = 0; i < NewEntry.GuestExtents.size(); i += 2) {
if ((uint64_t)NewEntry.GuestExtents[i] + NewEntry.GuestExtents[i + 1] > IndexBlobExtra->GuestSize) {
ExtentsValid = false;
break;
}
}
if (!ExtentsValid) {
continue;
}
auto It = CacheIndex.find(IndexBlobCommon->hash);
if (It == CacheIndex.end()) {
CacheIndex.emplace(IndexBlobCommon->hash, IndexCacheHead {std::move(NewEntry), IndexBlobExtra->GuestFootprint, nullptr});
} else {
if (!It->second.MoreEntries.get()) {
It->second.MoreEntries = fextl::make_unique<fextl::multimap<uint64_t, IndexEntry>>();
}
It->second.MoreEntries->insert({IndexBlobExtra->GuestFootprint, std::move(NewEntry)});
}
} else {
FoundMetadata = true;
}
}
// could truncate/delete index if we don't end up perfectly at end here
}
bool IndexedDB::ReadCacheBlob(uint64_t Offset, std::span<uint8_t> OutBlob) {
if (CacheFileMapping && (ReadOnly || Offset + OutBlob.size() <= BIG_MAPPING_SIZE)) {
if (Offset + OutBlob.size() > CacheFileSize) {
return false;
}
// todo could reduce copies by having a private mapping for relocs, etc
memcpy(OutBlob.data(), CacheFileMapping + Offset, OutBlob.size());
return true;
} else {
return CacheFOZ.ReadBlob(Offset, OutBlob);
}
}
bool IndexedDB::StoreCacheBlob(const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey, std::span<const uint8_t> Blob,
MesaFOZ::mesa_index_db_file_entry& IndexEntry, std::span<const uint8_t> IndexBlob) {
if (ReadOnly) {
// shouldn't happen
return false;
}
if (MaxSizeReached || CacheFileSize + Blob.size() >= MaxFileSize) {
MaxSizeReached = true;
return false;
}
if (!CacheFOZ.Lock(STORE_LOCK_TIMEOUT_MS) || !IndexFOZ.Lock(STORE_LOCK_TIMEOUT_MS)) {
CacheFOZ.Unlock();
IndexFOZ.Unlock();
return false;
}
// write cache side first so we get offset for index
std::span<const uint8_t> BlobChunks[] = {Blob};
uint64_t BlobOffset = 0;
if (!CacheFOZ.WriteBlob(UniqueKey, BlobChunks, BlobOffset)) {
CacheFOZ.Unlock();
IndexFOZ.Unlock();
return false;
}
IndexEntry = {.hash = LookupKey,
.size = (uint32_t)Blob.size(),
.last_access_time = 0, // todo..
.cache_db_file_offset = BlobOffset};
std::span<const uint8_t> IndexBlobChunks[] = {
{(const uint8_t*)&IndexEntry, sizeof(IndexEntry)},
IndexBlob,
};
uint64_t UnusedIndexBlobOffset = 0;
if (!IndexFOZ.WriteBlob(UniqueKey, IndexBlobChunks, UnusedIndexBlobOffset)) {
CacheFOZ.Unlock();
IndexFOZ.Unlock();
return false;
}
CacheFOZ.Unlock();
IndexFOZ.Unlock();
// publish new file size for memory-mapped reads
if (CacheFileMapping) {
CacheFileSize = BlobOffset + Blob.size();
}
return true;
}
bool DiskCache::OpenCacheDB(const fextl::string& CacheDBName, bool ReadOnly) {
fextl::unique_ptr<IndexedDB> CurDB;
if (!ReadOnly && RWCacheDB) {
// rw already opened, just support one
return false;
}
CurDB = fextl::make_unique<IndexedDB>();
if (!CurDB) {
return false;
}
if (!CurDB->Open(CacheDBName, ReadOnly)) {
CurDB.reset();
return false;
}
CurDB->PopulateIndex(Index, FoundMetadata);
if (ReadOnly) {
ROCacheDBs.push_back(std::move(CurDB));
} else {
RWCacheDB = std::move(CurDB);
}
return true;
}
FEX_DEFAULT_VISIBILITY void SetFileMapper(FileMapperFunc Func) {
FileMapper = Func;
}
static inline void PruneStaleEntries(std::string_view CacheBase, std::string_view MachineBucketHash) {
FEXCORE_PROFILE_SCOPED("DiskCache::PruneStaleEntries");
FEX_CONFIG_OPT(DiskCachePruneStaleEntries, DISKCACHEPRUNESTALEENTRIES);
if (!DiskCachePruneStaleEntries) {
return;
}
struct SimpleCapture {
std::string_view CacheBase, MachineBucketHash;
} const SimpleCapture {
.CacheBase = CacheBase,
.MachineBucketHash = MachineBucketHash,
};
FEXCore::FileUtils::WalkDirectory(
CacheBase,
[](std::string_view name, bool is_dir, const void* user_data) {
if (!is_dir) {
return;
}
auto capture = reinterpret_cast<const struct SimpleCapture*>(user_data);
// Current behaviour is to remove entries that no longer match the MachineBucketHash.
// This means that if the Disk Cache version no longer matches, or the FEXCore::HostFeatures differ, then they get removed.
if (name != capture->MachineBucketHash) {
FEXCore::FileUtils::RecursiveRemoveDirectory(fextl::fmt::format("{}/{}", capture->CacheBase, name));
}
},
&SimpleCapture);
}
void DiskCache::Init(FEXCore::Context::ContextImpl* CTX) {
this->CTX = CTX;
if (!EnableDiskCache) {
return;
}
fextl::string SerializedConfig = FEXCore::Config::SerializeForCache();
const auto HostFeatureHash = CTX->HostFeatures.HashForCaching();
struct __attribute__((packed)) {
uint16_t FormatVersion;
uint64_t HostFeaturesHash;
} MachineBucketData = {FormatVersion, HostFeatureHash.HostFeaturesHash};
fextl::vector<uint8_t> BucketBytes(sizeof(MachineBucketData) + (sizeof(uint8_t) * 2) + SerializedConfig.size());
memcpy(BucketBytes.data(), &MachineBucketData, sizeof(MachineBucketData));
// 64-bit mode and HostType is in the ProcessBucket hash only instead of the MachineBucketHash
// These effect code-gen, but they aren't part of the MachineBucketHash, as it comes from Process state.
BucketBytes[sizeof(MachineBucketData)] = CTX->Config.Is64BitMode;
BucketBytes[sizeof(MachineBucketData) + 1] = FEXCore::ToUnderlying(HostFeatureHash.HostType);
memcpy(BucketBytes.data() + sizeof(MachineBucketData) + 2, SerializedConfig.data(), SerializedConfig.size());
uint64_t MachineBucketHash = XXH3_64bits(BucketBytes.data(), sizeof(MachineBucketData));
uint64_t ProcessBucketHash = XXH3_64bits(BucketBytes.data() + sizeof(MachineBucketData), 2 + SerializedConfig.size());
BucketHash.high64 = MachineBucketHash;
BucketHash.low64 = ProcessBucketHash;
fextl::string BasePath = BasePathOverride();
if (BasePath.empty()) {
BasePath = FEXCore::Config::GetCacheDirectory() + "DiskCache/";
}
const auto MachineBucketHashAsString = fextl::fmt::format("{:016x}", MachineBucketHash);
PruneStaleEntries(BasePath, MachineBucketHashAsString);
BasePath += MachineBucketHashAsString + "/";
FHU::Filesystem::CreateDirectories(BasePath);
if (!MapDiskCacheFiles) {
FileMapper = nullptr;
}
const auto RWDBBasePath = fextl::fmt::format("{}RWCacheDB_{:016x}", BasePath, ProcessBucketHash);
OpenCacheDB(RWDBBasePath, false);
if (RWCacheDB && !FoundMetadata) {
// we just opened a fresh cache, add a metadata blob
MesaFOZ::foz_payload_key MetadataKey;
memset(MetadataKey.bytes, 0xFF, sizeof(MetadataKey));
IndexExtraBlobHeader MetaDataHeader = {};
MesaFOZ::mesa_index_db_file_entry IndexEntry = {};
RWCacheDB->StoreCacheBlob(MetadataKey, ~0, {BucketBytes.data(), BucketBytes.size()}, IndexEntry,
{reinterpret_cast<uint8_t*>(&MetaDataHeader), sizeof(MetaDataHeader)});
Index.erase(~0);
}
std::string_view RONames = RODBNames();
while (!RONames.empty()) {
const auto Delim = RONames.find(',');
const std::string_view ROName = RONames.substr(0, Delim);
if (!ROName.empty()) {
fextl::string RODBBasePath = BasePath;
RODBBasePath += ROName;
OpenCacheDB(RODBBasePath, true);
}
if (Delim == std::string_view::npos) {
break;
}
// advance to next
RONames.remove_prefix(Delim + 1);
}
WritingDiskCache = (bool)RWCacheDB;
ReadingDiskCache = !ROCacheDBs.empty() || RWCacheDB != nullptr;
if (IsWritingDiskCache()) {
FEXCore::Threads::Flags WriterThreadFlags = {.LowPriority = true, .Internal = true};
Writer = fextl::make_unique<WorkQueueThread>(WriterThreadFlags, "FEX:DiskCache");
}
}
uint64_t DiskCache::MakeLookupKey(Core::InternalThreadState* Thread, const uint64_t CodeKey, bool Writable, bool MonoBackpatcher) {
struct __attribute__((packed)) {
uint64_t CodeKey;
XXH128_hash_t BucketHash;
uint8_t Flags;
} BlobKeyBytes = {CodeKey, BucketHash, 0};
if (Writable) {
BlobKeyBytes.Flags |= 1 << 0;
}
if (CTX->AreMonoHacksActive()) {
BlobKeyBytes.Flags |= 1 << 1;
}
if (Thread->CurrentFrame->State.flags[X86State::RFLAG_TF_RAW_LOC]) {
BlobKeyBytes.Flags |= 1 << 2;
}
if (MonoBackpatcher) {
BlobKeyBytes.Flags |= 1 << 3;
}
return XXH3_64bits(&BlobKeyBytes, sizeof(BlobKeyBytes));
}
struct DiskCache::PruneMemoryLRUWorkItem final : WorkQueueThread::WorkItem {
DiskCache* Self;
PruneMemoryLRUWorkItem(DiskCache* Self)
: Self(Self) {}
void Run() override {
if (Self->MemoryLRUCurrentSize <= Self->MemoryLRUMaxSize + Self->MemoryLRUEvictThreshold) {
return;
}
std::lock_guard IndexGuard(Self->IndexLock);
std::lock_guard LRUGuard(Self->MemoryLRULock);
if (Self->MemoryLRU.empty()) {
return;
}
auto Last = std::prev(Self->MemoryLRU.end());
while (Self->MemoryLRUCurrentSize > Self->MemoryLRUMaxSize && !Self->MemoryLRU.empty()) {
auto LastKey = *Last;
auto IndexEntry = Self->LookupLocked(LastKey.LookupKey, LastKey.GuestHash, LastKey.GuestFootprint);
bool AtFront = (Last == Self->MemoryLRU.begin());
if (IndexEntry && IndexEntry->MemoryBlob.use_count() > 1) {
// being read rn, keep moving
if (AtFront) {
break;
}
Last--;
continue;
}
if (IndexEntry) {
IndexEntry->MemoryBlob.reset();
IndexEntry->LRUEntry.reset();
}
Self->MemoryLRUCurrentSize -= LastKey.Size;
auto Deleted = Last;
if (AtFront) {
Self->MemoryLRU.erase(Deleted);
break;
}
Last--;
Self->MemoryLRU.erase(Deleted);
}
}
};
std::optional<CodeHitData> DiskCache::Lookup(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region,
uint64_t GuestRIP, std::optional<uint64_t>& GuestCodeKey) {
if (!IsReadingDiskCache()) {
return std::nullopt;
}
if (Region && Region->FileStartVA) {
struct __attribute__((packed)) {
uint64_t GuestOffset;
uint64_t FileId;
} FileBackedKey = {GuestRIP - Region->FileStartVA, Region->FileInfo.FileId};
GuestCodeKey = XXH3_64bits(&FileBackedKey, sizeof(FileBackedKey));
} else {
if (!AnonCaching) {
return std::nullopt;
}
Thread->FrontendDecoder->DecodeLoop(reinterpret_cast<const uint8_t*>(GuestRIP), AnonPrefixGuestBytes);
const auto* BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo();
XXH3_state_t HashState;
XXH3_64bits_reset(&HashState);
for (auto& SubBlock : BlockInfo->Blocks) {
if (SubBlock.BlockStatus != Frontend::Decoder::DecodedBlockStatus::SUCCESS) {
return std::nullopt;
}
uint64_t HashStart = SubBlock.Entry;
// skip over masked in-block data and data/etc gaps between blocks
for (auto& DataMask : SubBlock.DataMasks) {
XXH3_64bits_update(&HashState, reinterpret_cast<const uint8_t*>(HashStart), DataMask.FieldAddress - HashStart);
HashStart = DataMask.FieldAddress + DataMask.ValueSize;
}
if (HashStart != SubBlock.Entry + SubBlock.Size) {
XXH3_64bits_update(&HashState, reinterpret_cast<const uint8_t*>(HashStart), SubBlock.Size - (HashStart - SubBlock.Entry));
}
}
GuestCodeKey = XXH3_64bits_digest(&HashState);
// if (TotalSize < AnonPrefixGuestBytes) {
// GuestCodeKey = 0;
// return std::nullopt;
// }
// LogMan::Msg::IFmt("anon lookup! length {:d} {}", GuestCodeKey, TotalSize);
}
auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, GuestRIP);
if (RangeInfo.Size == 0 || RangeInfo.Base > GuestRIP) {
return std::nullopt;
}
uint64_t Available = RangeInfo.Base + RangeInfo.Size - GuestRIP;
uint64_t LookupKey =
MakeLookupKey(Thread, *GuestCodeKey, RangeInfo.Writable, GuestRIP == CTX->GetMonoBackPatcherBlock().load(std::memory_order_relaxed));
XXH128_hash_t LiveGuestHash;
uint64_t LastFootprintHashed = 0;
bool FirstHash = true; // 0 is probably a valid footprint hash so we need an extra bit there
bool TriedMainEntry = false;
IndexEntry MainEntry;
uint64_t MainEntryFootprint;
fextl::multimap<uint64_t, IndexEntry>::iterator MoreEntriesIt;
fextl::multimap<uint64_t, IndexEntry>* MapPointer = nullptr;
IndexEntry* EntryUnderReview;
fextl::shared_ptr<fextl::vector<uint8_t>> BlobRef;
std::optional<fextl::list<MemoryLRUKey>::iterator> LRUIter;
uint64_t CurrentFootprint = 0;
{
std::lock_guard Guard(IndexLock);
auto It = Index.find(LookupKey);
if (It == Index.end()) {
// definite miss
return std::nullopt;
}
MainEntry = It->second.MainEntry;
MainEntryFootprint = It->second.MainEntryFootprint;
MapPointer = It->second.MoreEntries.get();
if (MapPointer) {
MoreEntriesIt = It->second.MoreEntries->begin();
}
if (MapPointer && MoreEntriesIt != It->second.MoreEntries->end()) {
EntryUnderReview = &MoreEntriesIt->second;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MoreEntriesIt->first;
} else {
EntryUnderReview = &MainEntry;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MainEntryFootprint;
TriedMainEntry = true;
}
}
// found a lookup key match, check for guest hash match now
bool FoundMatchingHash = false;
bool Advance = false;
while (!FoundMatchingHash) {
if (Advance) {
std::lock_guard Guard(IndexLock);
EntryUnderReview = nullptr;
if (MapPointer && MoreEntriesIt != MapPointer->end()) {
// if the current footprint is also the main entry's footprint, give main entry a shot next
if (!TriedMainEntry && MoreEntriesIt->first == MainEntryFootprint) {
EntryUnderReview = &MainEntry;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MainEntryFootprint;
TriedMainEntry = true;
} else {
MoreEntriesIt++;
}
}
if (!MapPointer || MoreEntriesIt == MapPointer->end()) {
if (TriedMainEntry) {
// ran out
break;
} else {
EntryUnderReview = &MainEntry;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MainEntryFootprint;
TriedMainEntry = true;
}
}
if (!EntryUnderReview && MapPointer && MoreEntriesIt != MapPointer->end()) {
EntryUnderReview = &MoreEntriesIt->second;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MoreEntriesIt->first;
}
}
Advance = true;
// entry not backed by anything right now - todo prune..
if (!EntryUnderReview->DB && !BlobRef) {
continue;
}
// do we have enough room in our live code to even hash GuestSize worth?
if (Available < EntryUnderReview->GuestSize) {
continue;
}
// if (Entry.GuestExtents.size()) {
// LogMan::Msg::IFmt("lookup! length {:d}", Entry.GuestSize);
// for(uint32_t i = 0; i < Entry.GuestExtents.size(); i+=2 ) {
// LogMan::Msg::IFmt("extent {} {}", Entry.GuestExtents[i], Entry.GuestExtents[i]+Entry.GuestExtents[i+1]);
// }
// }
{
IndexEntry& Entry = *EntryUnderReview;
// only renew the hash if the candidate footprint is different than previous
if (FirstHash || LastFootprintHashed != CurrentFootprint) {
XXH3_state_t HashState;
XXH3_128bits_reset(&HashState);
for (uint32_t i = 0; i < Entry.GuestExtents.size(); i += 2) {
XXH3_128bits_update(&HashState, reinterpret_cast<uint8_t*>(GuestRIP) + Entry.GuestExtents[i], Entry.GuestExtents[i + 1]);
}
LiveGuestHash = XXH3_128bits_digest(&HashState);
LastFootprintHashed = CurrentFootprint;
FirstHash = false;
}
if (XXH128_isEqual(LiveGuestHash, Entry.GuestHash)) {
FoundMatchingHash = true;
break;
} else if (Validation) {
fextl::vector<uint8_t> GuestCode(Entry.GuestSize);
if (Entry.Size >= Entry.GuestSize && Entry.DB && Entry.DB->ReadCacheBlob(Entry.Offset + Entry.Size - Entry.GuestSize, GuestCode)) {
const uint8_t* CachedGuest = GuestCode.data();
const uint8_t* LiveGuest = reinterpret_cast<const uint8_t*>(GuestRIP);
uint64_t DiffCount = 0;
uint64_t DiffOffset = 0;
for (uint32_t i = 0; i < Entry.GuestExtents.size(); i += 2) {
uint32_t Begin = Entry.GuestExtents[i];
uint32_t End = Begin + Entry.GuestExtents[i + 1];
bool PreviousByteDiff = false;
for (uint32_t Offset = Begin; Offset < End; Offset++) {
if (LiveGuest[Offset] != CachedGuest[Offset]) {
if (DiffCount == 0) {
DiffOffset = Offset;
}
if (!PreviousByteDiff) {
DiffCount++;
}
PreviousByteDiff = true;
} else {
PreviousByteDiff = false;
}
}
}
if (DiffCount) {
uint64_t DiffStart = DiffOffset >= 8 ? DiffOffset - 8 : 0;
uint64_t DiffContextBytes = std::min<uint64_t>(16, Entry.GuestSize - DiffStart);
auto LiveDump = fmt::join(std::span<const uint8_t> {LiveGuest + DiffStart, DiffContextBytes}, " ");
auto CacheDump = fmt::join(std::span<const uint8_t> {CachedGuest + DiffStart, DiffContextBytes}, " ");
auto KeyPrefix = (Region && Region->FileStartVA) ? "file" : "anon";
LogMan::Msg::IFmt("DiskCache: lookup guest hash mismatch key={}-{:x} gsize={}, ndiff={}, first diff: offset={} "
"live=[{:02x}] "
"cached=[{:02x}]",
KeyPrefix, *GuestCodeKey, Entry.GuestSize, DiffCount, DiffOffset, LiveDump, CacheDump);
} else {
LogMan::Msg::IFmt("DiskCache: guest hash mismatch but no diff?");
}
}
}
}
}
if (!FoundMatchingHash || !EntryUnderReview) {
return std::nullopt;
}
IndexEntry& Entry = *EntryUnderReview;
fextl::vector<uint64_t> GuestPages;
for (uint32_t i = 0; i < Entry.GuestExtents.size(); i += 2) {
uint64_t FirstPage = (Entry.GuestExtents[i] + GuestRIP) & Utils::FEX_PAGE_MASK;
uint64_t LastPage = (Entry.GuestExtents[i] + GuestRIP + Entry.GuestExtents[i + 1] - 1) & Utils::FEX_PAGE_MASK;
for (uint64_t Page = FirstPage; Page <= LastPage; Page += Utils::FEX_PAGE_SIZE) {
if (GuestPages.size() == 0 || Page != GuestPages.back()) {
GuestPages.push_back(Page);
}
}
}
// this seems to be a full hit, pull from disk and check the entry is big enough to have everything (except GuestCode)
CodeHitData HitData;
uint32_t EntrySizeWithoutGuestCode = Entry.Size - Entry.GuestSize;
HitData.Blob.resize(GuestPages.size() * sizeof(uint64_t) + EntrySizeWithoutGuestCode);
memcpy(HitData.Blob.data(), GuestPages.data(), GuestPages.size() * sizeof(uint64_t));
uint32_t BlobOffset = GuestPages.size() * sizeof(uint64_t);
bool FoundInLRU = false;
if (BlobRef && BlobRef->size() >= EntrySizeWithoutGuestCode) {
FoundInLRU = true;
memcpy(HitData.Blob.data() + BlobOffset, BlobRef->data(), EntrySizeWithoutGuestCode);
if (LRUIter) {
std::lock_guard Guard(MemoryLRULock);
// avoided disk by nabbing from lru, bump to front
// LogMan::Msg::IFmt("lru hit! {}", MemoryLRUCurrentSize);
MemoryLRU.splice(MemoryLRU.begin(), MemoryLRU, *LRUIter);
}
} else if (!Entry.DB->ReadCacheBlob(Entry.Offset, {HitData.Blob.data() + BlobOffset, EntrySizeWithoutGuestCode})) {
return std::nullopt;
}
if (EntrySizeWithoutGuestCode < sizeof(BlobFixedHeader)) {
return std::nullopt;
}
BlobFixedHeader Header;
memcpy(&Header, HitData.Blob.data() + BlobOffset, sizeof(Header));
BlobOffset += sizeof(Header);
uint32_t SizeNeeded = sizeof(Header) + Header.HostSize + Header.EntryPointCount * (sizeof(uint64_t) + sizeof(uint32_t));
SizeNeeded += Header.SmallRelocCount * sizeof(BlobSmallRelocation) + Header.ThunkRelocCount * sizeof(BlobThunkRelocation);
if (EntrySizeWithoutGuestCode != SizeNeeded) {
return std::nullopt;
}
if (Entry.GuestSize != Header.GuestSize || !XXH128_isEqual(Header.GuestHash, Entry.GuestHash)) {
return std::nullopt;
}
bool StoreInMemory = !FoundInLRU;
if (EntrySizeWithoutGuestCode > MemoryLRUMaxSize) {
StoreInMemory = false;
}
if (StoreInMemory) {
auto NewBlobRef = fextl::make_shared<fextl::vector<uint8_t>>(EntrySizeWithoutGuestCode);
memcpy(NewBlobRef->data(), HitData.Blob.data() + BlobOffset, EntrySizeWithoutGuestCode);
std::lock_guard Guard(IndexLock);
auto CurrentIndexEntry = LookupLocked(LookupKey, Header.GuestHash, LastFootprintHashed);
if (CurrentIndexEntry && !CurrentIndexEntry->MemoryBlob) {
std::lock_guard LRUGuard(MemoryLRULock);
MemoryLRU.push_front({LookupKey, Header.GuestHash, LastFootprintHashed, EntrySizeWithoutGuestCode});
CurrentIndexEntry->LRUEntry = MemoryLRU.begin();
CurrentIndexEntry->MemoryBlob = NewBlobRef;
MemoryLRUCurrentSize += EntrySizeWithoutGuestCode;
}
}
if (StoreInMemory && MemoryLRUCurrentSize > MemoryLRUMaxSize + MemoryLRUEvictThreshold) {
Writer->QueueWork(fextl::make_unique<PruneMemoryLRUWorkItem>(this));
}
HitData.HostCode = {HitData.Blob.data() + BlobOffset, Header.HostSize};
BlobOffset += Header.HostSize;
HitData.EntryPointRIPs = {reinterpret_cast<uint64_t*>(HitData.Blob.data() + BlobOffset), Header.EntryPointCount};
BlobOffset += Header.EntryPointCount * sizeof(uint64_t);
HitData.EntryPointHostOffsets = {reinterpret_cast<const uint32_t*>(HitData.Blob.data() + BlobOffset), Header.EntryPointCount};
BlobOffset += Header.EntryPointCount * sizeof(uint32_t);
std::span<const BlobSmallRelocation> SmallRelocs = {reinterpret_cast<const BlobSmallRelocation*>(HitData.Blob.data() + BlobOffset),
Header.SmallRelocCount};
BlobOffset += Header.SmallRelocCount * sizeof(BlobSmallRelocation);
std::span<const BlobThunkRelocation> ThunkRelocs = {reinterpret_cast<const BlobThunkRelocation*>(HitData.Blob.data() + BlobOffset),
Header.ThunkRelocCount};
BlobOffset += Header.ThunkRelocCount * sizeof(BlobThunkRelocation);
HitData.GuestPages = {reinterpret_cast<uint64_t*>(HitData.Blob.data()), GuestPages.size()};
for (auto& EntryPointRip : HitData.EntryPointRIPs) {
EntryPointRip += GuestRIP;
}
if (!CTX->CodeCache.ApplyPackedCodeRelocations(GuestRIP, std::as_writable_bytes(HitData.HostCode), SmallRelocs, ThunkRelocs)) {
return std::nullopt;
}
return HitData;
}
void DiskCache::Validate(uint64_t GuestCodeKey, const CodeHitData& Hit, const CPU::CPUBackend::CompiledCode& CompiledCode,
std::optional<ExecutableFileSectionInfo> Region) {
if (!Validation) {
return;
}
auto KeyPrefix = (Region && Region->FileStartVA) ? "file" : "anon";
if (Hit.HostCode.size() != CompiledCode.Size) {
LogMan::Msg::EFmt("DiskCache: validate host size mismatch key={}-{:x} cached={} live={}", KeyPrefix, GuestCodeKey,
Hit.HostCode.size(), CompiledCode.Size);
} else if (memcmp(Hit.HostCode.data(), CompiledCode.BlockBegin, CompiledCode.Size) != 0) {
bool PreviousByteDiff = false;
size_t FirstDiff = 0;
size_t DiffCount = 0;
for (size_t i = 0; i < CompiledCode.Size; i++) {
if (Hit.HostCode[i] != CompiledCode.BlockBegin[i]) {
if (DiffCount == 0) {
FirstDiff = i;
}
if (!PreviousByteDiff) {
DiffCount++;
}
PreviousByteDiff = true;
} else {
PreviousByteDiff = false;
}
}
// align to 4 bytes to read host arm easier
const size_t DiffStart = (FirstDiff & ~3ULL) >= 8 ? (FirstDiff & ~3ULL) - 8 : 0;
const size_t ContextBytes = std::min<size_t>(16, CompiledCode.Size - DiffStart);
LogMan::Msg::EFmt("DiskCache: validate host code mismatch key={}-{:x} size={} firstoffset={} ndiff={} cached=[{:02x}] live=[{:02x}]",
KeyPrefix, GuestCodeKey, CompiledCode.Size, FirstDiff, DiffCount,
fmt::join(std::span<const uint8_t> {Hit.HostCode.data() + DiffStart, ContextBytes}, " "),
fmt::join(std::span<const uint8_t> {CompiledCode.BlockBegin + DiffStart, ContextBytes}, " "));
}
}
IndexEntry* DiskCache::LookupLocked(const uint64_t LookupKey, const XXH128_hash_t& GuestHash, const uint64_t GuestFootprint) {
auto It = Index.find(LookupKey);
if (It != Index.end()) {
if (XXH128_isEqual(It->second.MainEntry.GuestHash, GuestHash)) {
return &It->second.MainEntry;
}
if (It->second.MoreEntries.get()) {
auto Range = It->second.MoreEntries->equal_range(GuestFootprint);
for (auto MoreEntriesIt = Range.first; MoreEntriesIt != Range.second; MoreEntriesIt++) {
if (XXH128_isEqual(MoreEntriesIt->second.GuestHash, GuestHash)) {
return &MoreEntriesIt->second;
}
}
}
}
return nullptr;
}
struct DiskCache::CacheStoreWorkItem final : WorkQueueThread::WorkItem {
DiskCache* Self;
IndexedDB* DB;
MesaFOZ::foz_payload_key UniqueKey;
uint64_t LookupKey;
std::span<uint8_t> Blob;
fextl::vector<uint8_t> IndexBlob;
bool StoreDisk;
CacheStoreWorkItem(DiskCache* Self, IndexedDB* DB, const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey,
std::span<uint8_t> Blob, fextl::vector<uint8_t>&& IndexBlob, bool StoreDisk)
: Self(Self)
, DB(DB)
, UniqueKey(UniqueKey)
, LookupKey(LookupKey)
, Blob(Blob)
, IndexBlob(std::move(IndexBlob))
, StoreDisk(StoreDisk) {}
void Run() override {
struct MesaFOZ::mesa_index_db_file_entry IndexHeader;
bool DiskSuccess = !StoreDisk || DB->StoreCacheBlob(UniqueKey, LookupKey, Blob, IndexHeader, IndexBlob);
bool KeepEntryInMemory = true;
// todo possible other lru condition here like entry size?
if (Blob.size() > Self->MemoryLRUMaxSize) {
KeepEntryInMemory = false;
} else {
Self->MemoryLRUCurrentSize += Blob.size();
}
const IndexExtraBlobHeader* IndexAfterHeader = reinterpret_cast<const IndexExtraBlobHeader*>(IndexBlob.data());
fextl::list<MemoryLRUKey>::iterator NewLRUEntry;
if (KeepEntryInMemory) {
std::lock_guard Guard(Self->MemoryLRULock);
Self->MemoryLRU.push_front({LookupKey, IndexAfterHeader->GuestHash, IndexAfterHeader->GuestFootprint, (uint32_t)Blob.size()});
NewLRUEntry = Self->MemoryLRU.begin();
}
{
std::lock_guard Guard(Self->IndexLock);
auto IndexEntry = Self->LookupLocked(LookupKey, IndexAfterHeader->GuestHash, IndexAfterHeader->GuestFootprint);
LOGMAN_THROW_A_FMT(IndexEntry != nullptr, "Stored Index entry not found?");
if (IndexEntry) {
if (StoreDisk && DiskSuccess) {
IndexEntry->DB = DB;
IndexEntry->Offset = IndexHeader.cache_db_file_offset;
}
if (!KeepEntryInMemory) {
IndexEntry->MemoryBlob.reset();
} else {
IndexEntry->LRUEntry = NewLRUEntry;
}
}
}
if (KeepEntryInMemory && Self->MemoryLRUCurrentSize > Self->MemoryLRUMaxSize + Self->MemoryLRUEvictThreshold) {
Self->Writer->QueueWork(fextl::make_unique<PruneMemoryLRUWorkItem>(Self));
}
}
};
bool DiskCache::Store(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region, uint64_t GuestRIP,
uint64_t GuestCodeKey, std::span<const uint8_t> GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode,
std::span<const FEXCore::CPU::Relocation> Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo) {
if (!IsWritingDiskCache()) {
return false;
}
if (!DecodedBlockInfo) {
return false;
}
// check for any reloc targets outside of our jurisdiction
// todo what are they exactly? caching those blocks is great when it works, so need to figure this out and make finer-grained if we can
if (RelocationFilter && Region) {
for (const auto& Reloc : Relocations) {
if (Reloc.Header.Type != CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL && Reloc.Header.Type != CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE) {
continue;
}
uint64_t Target = Reloc.GuestRIP.GuestRIP;
if (Target >= Region->BeginVA && Target < Region->EndVA) {
continue;
}
// let it through if it's inside the same ELF image? (like bss)
if (Region->FileInfo.MappedSize && Target >= Region->FileStartVA && Target < Region->FileStartVA + Region->FileInfo.MappedSize) {
continue;
}
auto TargetSection = CTX->SyscallHandler->LookupExecutableFileSection(Thread, Target);
if (!TargetSection || TargetSection->FileInfo.FileId != Region->FileInfo.FileId) {
// we don't know where it's pointing, so we don't know how to encode the offset, so we can't cache atm
return false;
}
}
}
uint32_t SmallRelocCount = 0;
uint32_t ThunkRelocCount = 0;
for (const auto& Reloc : Relocations) {
if (Reloc.Header.Type == CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE) {
ThunkRelocCount++;
} else {
SmallRelocCount++;
}
}
const uint32_t EntryPointCount = (uint32_t)CompiledCode.EntryPoints.size();
const size_t HeaderOffset = 0;
const size_t HostCodeOffset = HeaderOffset + sizeof(BlobFixedHeader);
const size_t EntryPointRIPsOffset = HostCodeOffset + CompiledCode.Size;
const size_t EntryPointHostOffsetsOffset = EntryPointRIPsOffset + EntryPointCount * sizeof(uint64_t);
const size_t SmallRelocsOffset = EntryPointHostOffsetsOffset + EntryPointCount * sizeof(uint32_t);
const size_t ThunkRelocsOffset = SmallRelocsOffset + SmallRelocCount * sizeof(BlobSmallRelocation);
const size_t GuestCodeOffset = ThunkRelocsOffset + ThunkRelocCount * sizeof(BlobThunkRelocation);
const size_t TotalSize = GuestCodeOffset + GuestCode.size();
// we'll copy everything into here and pass it to the Writer, then return to caller quickly
fextl::vector<uint8_t> Blob;
Blob.resize(TotalSize);
uint8_t* BlobData = Blob.data();
BlobFixedHeader Header {
.GuestSize = (uint32_t)GuestCode.size(),
.HostSize = (uint32_t)CompiledCode.Size,
.EntryPointCount = EntryPointCount,
.SmallRelocCount = SmallRelocCount,
.ThunkRelocCount = ThunkRelocCount,
};
memcpy(BlobData + HostCodeOffset, CompiledCode.BlockBegin, CompiledCode.Size);
// pack and relocate entrypoints
auto* EntryRIPs = reinterpret_cast<uint64_t*>(BlobData + EntryPointRIPsOffset);
auto* EntryHostOffsets = reinterpret_cast<uint32_t*>(BlobData + EntryPointHostOffsetsOffset);
uint32_t EntryIdx = 0;
for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) {
EntryRIPs[EntryIdx] = GuestAddr - GuestRIP;
EntryHostOffsets[EntryIdx] = uint32_t(HostAddr - CompiledCode.BlockBegin);
EntryIdx++;
}
fextl::set<uint64_t> DataMasksConsumed;
// pack relocations
auto* SmallRelocs = reinterpret_cast<BlobSmallRelocation*>(BlobData + SmallRelocsOffset);
auto* ThunkRelocs = reinterpret_cast<BlobThunkRelocation*>(BlobData + ThunkRelocsOffset);
uint32_t SmallIdx = 0;
uint32_t ThunkIdx = 0;
for (const auto& Reloc : Relocations) {
switch (Reloc.Header.Type) {
// it's important to zero-init the element completely so we don't have garbage in unused fields
// this way, the caches stay deterministic across machines
case CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: {
BlobSmallRelocation SmallReloc = {};
SmallReloc.Offset = Reloc.Header.Offset;
SmallReloc.Type = uint8_t(Reloc.Header.Type);
SmallReloc.Named.Symbol = uint32_t(Reloc.NamedSymbolLiteral.Symbol);
SmallRelocs[SmallIdx++] = SmallReloc;
break;
}
case CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL: {
BlobSmallRelocation SmallReloc = {};
SmallReloc.Offset = Reloc.Header.Offset;
SmallReloc.Type = uint8_t(Reloc.Header.Type);
SmallReloc.RIPLiteral.GuestRIP = Reloc.GuestRIP.GuestRIP - GuestRIP;
SmallRelocs[SmallIdx++] = SmallReloc;
break;
}
case CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE: {
BlobSmallRelocation SmallReloc = {};
SmallReloc.Offset = Reloc.Header.Offset;
SmallReloc.Type = uint8_t(Reloc.Header.Type);
SmallReloc.RIPMove.RegisterIndex = Reloc.GuestRIP.RegisterIndex;
SmallReloc.RIPMove.GuestRIP = Reloc.GuestRIP.GuestRIP - GuestRIP;
SmallRelocs[SmallIdx++] = SmallReloc;
break;
}
case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE:
case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE:
case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_LITERAL:
case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE: {
// same data for all, relative vs. not and register vs. literal will depend on type on apply
BlobSmallRelocation SmallReloc = {};
SmallReloc.Offset = Reloc.Header.Offset;
SmallReloc.Type = uint8_t(Reloc.Header.Type);
SmallReloc.PatchableData.RegisterIndex = Reloc.GuestPatchableData.RegisterIndex;
SmallReloc.PatchableData.ValueSize = Reloc.GuestPatchableData.ValueSize;
SmallReloc.PatchableData.SiteOffset = uint32_t(Reloc.GuestPatchableData.SiteAddress - GuestRIP);
SmallRelocs[SmallIdx++] = SmallReloc;
// mark the corresponding data mask consumed
DataMasksConsumed.insert(Reloc.GuestPatchableData.SiteAddress);
break;
}
case CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE: {
BlobThunkRelocation BigReloc = {};
BigReloc.Offset = Reloc.Header.Offset;
BigReloc.RegisterIndex = Reloc.NamedThunkMove.RegisterIndex;
memcpy(BigReloc.SymbolHash, &Reloc.NamedThunkMove.Symbol, sizeof(BigReloc.SymbolHash));
ThunkRelocs[ThunkIdx++] = BigReloc;
break;
}
}
}
fextl::vector<uint32_t> ExactGuestCodeExtents;
uint64_t CurStartExtent = 0, CurEndExtent = 0;
const Frontend::Decoder::DecodedBlocks* LastBlock = nullptr;
for (auto& SubBlock : DecodedBlockInfo->Blocks) {
if (SubBlock.BlockStatus != Frontend::Decoder::DecodedBlockStatus::SUCCESS) {
return false;
}
if (!CurStartExtent) {
CurStartExtent = SubBlock.Entry;
CurEndExtent = SubBlock.Entry + SubBlock.Size;
} else {
LOGMAN_THROW_A_FMT(SubBlock.Entry >= CurEndExtent, "DecodedBlocks not sorted or overlapping?");
if (SubBlock.Entry == CurEndExtent) {
CurEndExtent = SubBlock.Entry + SubBlock.Size;
} else {
ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
CurStartExtent = SubBlock.Entry;
CurEndExtent = SubBlock.Entry + SubBlock.Size;
}
}
// split extents according to data masks as well
for (auto& Mask : SubBlock.DataMasks) {
if (Mask.Type == Frontend::Decoder::DataMaskType::NOP || DataMasksConsumed.find(Mask.FieldAddress) != DataMasksConsumed.end()) {
if (Mask.FieldAddress > CurStartExtent) {
ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
ExactGuestCodeExtents.push_back(Mask.FieldAddress - CurStartExtent);
}
CurStartExtent = Mask.FieldAddress + Mask.ValueSize;
}
}
LastBlock = &SubBlock;
}
if (LastBlock && (CurStartExtent != GuestRIP || CurEndExtent != GuestRIP + GuestCode.size())) {
ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
}
if (ExactGuestCodeExtents.size() == 0) {
ExactGuestCodeExtents.reserve(2);
ExactGuestCodeExtents.push_back(0);
ExactGuestCodeExtents.push_back(GuestCode.size());
}
uint64_t GuestFootprint = XXH3_64bits(ExactGuestCodeExtents.data(), ExactGuestCodeExtents.size() * sizeof(uint32_t));
// if (ExactGuestCodeExtents.size()) {
// LogMan::Msg::IFmt("store! length {:d}", GuestCode.size());
// for(uint32_t i = 0; i < ExactGuestCodeExtents.size(); i+=2 ) {
// LogMan::Msg::IFmt("extent {} {}", ExactGuestCodeExtents[i], ExactGuestCodeExtents[i]+ExactGuestCodeExtents[i+1]);
// }
// }
{
XXH3_state_t HashState;
XXH3_128bits_reset(&HashState);
for (uint32_t i = 0; i < ExactGuestCodeExtents.size(); i += 2) {
XXH3_128bits_update(&HashState, GuestCode.data() + ExactGuestCodeExtents[i], ExactGuestCodeExtents[i + 1]);
}
Header.GuestHash = XXH3_128bits_digest(&HashState);
}
memcpy(BlobData + HeaderOffset, &Header, sizeof(Header));
auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, GuestRIP);
uint64_t LookupKey =
MakeLookupKey(Thread, GuestCodeKey, RangeInfo.Writable, GuestRIP == CTX->GetMonoBackPatcherBlock().load(std::memory_order_relaxed));
// blob done, publish to index as in-memory for now, flush to disk below
auto BlobRef = fextl::make_shared<fextl::vector<uint8_t>>(std::move(Blob));
struct IndexEntry NewEntry {nullptr, 0, (uint32_t)TotalSize, Header.GuestSize, Header.GuestHash, BlobRef};
NewEntry.GuestExtents = ExactGuestCodeExtents;
{
std::lock_guard Guard(IndexLock);
auto It = Index.find(LookupKey);
if (It == Index.end()) {
Index.emplace(LookupKey, IndexCacheHead {std::move(NewEntry), GuestFootprint, nullptr});
} else {
bool Dupe = false;
if (It->second.MoreEntries.get()) {
for (auto& [Key, Elem] : *It->second.MoreEntries) {
if (XXH128_isEqual(Elem.GuestHash, Header.GuestHash)) {
Dupe = true;
break;
}
}
}
if (!Dupe && XXH128_isEqual(It->second.MainEntry.GuestHash, Header.GuestHash)) {
Dupe = true;
}
// could happen if it's seen again while in flight in the store queue
if (Dupe) {
return true;
}
if (!It->second.MoreEntries.get()) {
It->second.MoreEntries = fextl::make_unique<fextl::multimap<uint64_t, struct IndexEntry>>();
} else if (It->second.MoreEntries->size() >= LOOKUP_KEY_MAX_BUCKET_DEPTH) {
return false;
}
It->second.MoreEntries->insert({GuestFootprint, std::move(NewEntry)});
}
}
memcpy(BlobData + GuestCodeOffset, GuestCode.data(), GuestCode.size());
MesaFOZ::foz_payload_key Key = {};
{
XXH3_state_t HashState;
XXH3_128bits_reset(&HashState);
XXH3_128bits_update(&HashState, &Header.GuestHash, sizeof(Header.GuestHash));
XXH3_128bits_update(&HashState, &LookupKey, sizeof(LookupKey));
XXH128_hash_t UniqueKey = XXH3_128bits_digest(&HashState);
fextl::string BlobName = fextl::fmt::format("{:016x}{:016x}", UniqueKey.high64, UniqueKey.low64);
memcpy(Key.bytes, BlobName.data(), BlobName.size());
}
fextl::vector<uint8_t> IndexBlob;
IndexBlob.resize(sizeof(IndexExtraBlobHeader) + ExactGuestCodeExtents.size() * sizeof(uint32_t));
IndexExtraBlobHeader IndexBlobHeader {Header.GuestHash, GuestFootprint, Header.GuestSize, (uint32_t)ExactGuestCodeExtents.size()};
memcpy(IndexBlob.data(), &IndexBlobHeader, sizeof(IndexExtraBlobHeader));
memcpy(IndexBlob.data() + sizeof(IndexExtraBlobHeader), ExactGuestCodeExtents.data(), ExactGuestCodeExtents.size() * sizeof(uint32_t));
bool StoreDisk = true;
if (RWCacheDB->Full()) {
StoreDisk = false;
}
// hand the rest off to the writer thread
Writer->QueueWork(fextl::make_unique<CacheStoreWorkItem>(this, RWCacheDB.get(), Key, LookupKey, std::span(BlobData, TotalSize),
std::move(IndexBlob), StoreDisk));
return true;
}
uint16_t GetFormatVersion() {
return FormatVersion;
}
} // namespace DiskCache
} // namespace FEXCore
Reference in new issue View Git Blame Copy Permalink