steam-frame/FEX-Emu--FEX
1
0 Fork 0
mirror of https://github.com/FEX-Emu/FEX.git synced 2026-10-06 10:00:16 +02:00
Code Issues Packages Projects Releases 66 Wiki Activity
Files
83d8317dae95fd2c1c70a5be1a04dfec5a68d865
FEX-Emu--FEX/FEXCore/Source/Interface/Core/DiskCache.cpp
T

1207 lines
47 KiB
C++
Raw Blame History

// SPDX-License-Identifier: MIT
#define XXH_STATIC_LINKING_ONLY
#include "FEXCore/Utils/TypeDefines.h"
#include "Interface/Core/Frontend.h"
#include "FEXCore/Config/Config.h"
#include "FEXCore/fextl/string.h"
#include "FEXHeaderUtils/Filesystem.h"
#include "FEXCore/Core/DiskCache.h"
#include "FEXCore/Core/DiskCacheFileMapper.h"
#include "FEXCore/Utils/LogManager.h"
#include "Interface/Context/Context.h"
#include "FEXCore/HLE/SyscallHandler.h"
#include "FEXCore/Utils/File.h"
#include "FEXCore/fextl/memory.h"
#include <cstdint>
#include <cstring>
#include <atomic>
namespace FEXCore {
namespace DiskCache {
namespace MesaFOZ {
enum { FOSSILIZE_COMPRESSION_NONE = 1, FOSSILIZE_COMPRESSION_DEFLATE = 2 };
enum { FOSSILIZE_FORMAT_VERSION = 6, FOSSILIZE_FORMAT_MIN_COMPAT_VERSION = 5 };
#define FOZ_REF_MAGIC_SIZE 16
static const uint8_t stream_reference_magic_and_version[FOZ_REF_MAGIC_SIZE] = {
0x81, 'F', 'O', 'S', 'S', 'I', 'L', 'I', 'Z', 'E', 'D', 'B', 0, 0, 0, FOSSILIZE_FORMAT_VERSION, /* 4 bytes to use for versioning. */
};
struct __attribute__((packed)) mesa_index_db_file_entry {
uint64_t hash;
uint32_t size;
uint64_t last_access_time;
uint64_t cache_db_file_offset;
};
} // namespace MesaFOZ
struct __attribute__((packed)) IndexExtraBlobHeader {
XXH128_hash_t GuestHash; // StoreCacheBlob will assume it's here, care if moving
uint64_t GuestFootprint;
uint32_t GuestSize;
uint32_t GuestExtentsCount;
};
static FileMapperFunc FileMapper = nullptr;
bool FOZFile::Open(const fextl::string& FOZFileName, bool ReadOnly) {
FileName = FOZFileName;
this->ReadOnly = ReadOnly;
File::FileModes Modes = File::FileModes::READ;
if (!ReadOnly) {
Modes = Modes | File::FileModes::WRITE | File::FileModes::CREATE;
}
FD = fextl::make_unique<File::File>(FileName.c_str(), Modes, false);
if (!FD->IsValid()) {
FD.reset();
return false;
}
bool Valid = false;
bool TookLock = false;
ssize_t Size = FD->Size();
if (Size < FOZ_REF_MAGIC_SIZE && !ReadOnly) {
if (!FD->Lock(OPEN_LOCK_TIMEOUT_MS)) {
FD.reset();
return false;
}
TookLock = true;
// check size again in case someone else made it while we waited
Size = FD->Size();
}
if (Size == 0 && !ReadOnly) {
Valid = FD->PWrite(MesaFOZ::stream_reference_magic_and_version, FOZ_REF_MAGIC_SIZE, 0) == FOZ_REF_MAGIC_SIZE;
} else {
uint8_t magic[FOZ_REF_MAGIC_SIZE];
if (FD->PRead(magic, FOZ_REF_MAGIC_SIZE, 0) == FOZ_REF_MAGIC_SIZE &&
memcmp(magic, MesaFOZ::stream_reference_magic_and_version, FOZ_REF_MAGIC_SIZE - 1) == 0) {
int version = magic[FOZ_REF_MAGIC_SIZE - 1];
Valid = version <= MesaFOZ::FOSSILIZE_FORMAT_VERSION && version >= MesaFOZ::FOSSILIZE_FORMAT_MIN_COMPAT_VERSION;
}
}
if (TookLock) {
FD->Unlock();
}
if (!Valid) {
FD.reset();
}
return Valid;
}
ssize_t FOZFile::Size() {
return FD ? FD->Size() : -1;
}
bool FOZFile::ReadAll(fextl::vector<uint8_t>& Out) {
ssize_t FileSize = Size();
if (FileSize < FOZ_REF_MAGIC_SIZE) {
return false;
}
Out.resize((size_t)FileSize - FOZ_REF_MAGIC_SIZE);
return FD->PRead(Out.data(), Out.size(), FOZ_REF_MAGIC_SIZE) == (ssize_t)Out.size();
}
bool FOZFile::ReadBlob(uint64_t Offset, std::span<uint8_t> OutBlob) {
if (FD->PRead(OutBlob.data(), OutBlob.size(), Offset) != (ssize_t)OutBlob.size()) {
return false;
}
return true;
}
bool FOZFile::WriteBlob(const MesaFOZ::foz_payload_key& Key, std::span<const std::span<const uint8_t>> BlobChunks, uint64_t& OutBlobOffset) {
ssize_t FileSize = FD->Size();
if (FileSize < 0) {
return false;
}
uint64_t WriteOffset = (uint64_t)FileSize;
if (FD->PWrite(Key.bytes, sizeof(Key.bytes), WriteOffset) != sizeof(Key.bytes)) {
return false;
}
WriteOffset += sizeof(Key.bytes);
uint64_t TotalBlobSize = 0;
for (const std::span<const uint8_t>& Chunk : BlobChunks) {
TotalBlobSize += Chunk.size();
}
MesaFOZ::foz_payload_header ScratchHeader {.payload_size = (uint32_t)TotalBlobSize,
.format = MesaFOZ::FOSSILIZE_COMPRESSION_NONE,
.crc = 0, // todo? maybe
.uncompressed_size = (uint32_t)TotalBlobSize};
if (FD->PWrite(&ScratchHeader, sizeof(ScratchHeader), WriteOffset) != sizeof(ScratchHeader)) {
return false;
}
WriteOffset += sizeof(ScratchHeader);
OutBlobOffset = WriteOffset;
for (const std::span<const uint8_t>& Chunk : BlobChunks) {
if (Chunk.size() == 0) {
continue;
}
if (FD->PWrite(Chunk.data(), Chunk.size(), WriteOffset) != (ssize_t)Chunk.size()) {
return false;
}
WriteOffset += Chunk.size();
}
return true;
}
bool IndexedDB::Open(const fextl::string& CacheDBName, bool ReadOnly) {
if (!CacheFOZ.Open(CacheDBName + ".foz", ReadOnly)) {
return false;
}
if (!IndexFOZ.Open(CacheDBName + "_idx.foz", ReadOnly)) {
return false;
}
File::File::FileHandleType CacheFileHandle = CacheFOZ.GetHandle();
if (FileMapper && CacheFileHandle != (File::File::FileHandleType)-1) {
CacheFileMapping = reinterpret_cast<uint8_t*>(FileMapper(CacheFileHandle, ReadOnly ? CacheFOZ.Size() : BIG_MAPPING_SIZE));
CacheFileSize = CacheFOZ.Size();
}
this->ReadOnly = ReadOnly;
return true;
}
void IndexedDB::PopulateIndex(Index& CacheIndex, bool& FoundMetadata) {
fextl::vector<uint8_t> Data;
if (!IndexFOZ.ReadAll(Data)) {
return;
}
ssize_t CacheFOZSize = CacheFOZ.Size();
if (CacheFOZSize < 0) {
return;
}
const uint8_t* IndexDataStart = Data.data();
const size_t IndexDataSize = Data.size();
size_t ReadOffset = 0;
while (ReadOffset + sizeof(MesaFOZ::foz_payload_key) + sizeof(MesaFOZ::foz_payload_header) <= IndexDataSize) {
const auto* FOZKey = reinterpret_cast<const MesaFOZ::foz_payload_key*>(IndexDataStart + ReadOffset);
ReadOffset += sizeof(MesaFOZ::foz_payload_key);
const auto* FOZHeader = reinterpret_cast<const MesaFOZ::foz_payload_header*>(IndexDataStart + ReadOffset);
ReadOffset += sizeof(MesaFOZ::foz_payload_header);
uint64_t IndexBlobSize = sizeof(MesaFOZ::mesa_index_db_file_entry) + sizeof(IndexExtraBlobHeader);
if (FOZHeader->payload_size < IndexBlobSize || ReadOffset + FOZHeader->payload_size > IndexDataSize) {
break;
}
const auto* IndexBlobCommon = reinterpret_cast<const MesaFOZ::mesa_index_db_file_entry*>(IndexDataStart + ReadOffset);
ReadOffset += sizeof(MesaFOZ::mesa_index_db_file_entry);
const auto* IndexBlobExtra = reinterpret_cast<const IndexExtraBlobHeader*>(IndexDataStart + ReadOffset);
ReadOffset += sizeof(IndexExtraBlobHeader);
// skip corrupt (carefully) so we don't have to figure that out in the hot path later
if (IndexBlobCommon->cache_db_file_offset > (uint64_t)CacheFOZSize ||
IndexBlobCommon->size > (uint64_t)CacheFOZSize - IndexBlobCommon->cache_db_file_offset) {
continue;
}
if (IndexBlobExtra->GuestSize + sizeof(BlobFixedHeader) > IndexBlobCommon->size) {
continue;
}
IndexBlobSize += IndexBlobExtra->GuestExtentsCount * sizeof(uint32_t);
if (FOZHeader->payload_size != IndexBlobSize) {
break;
}
const auto* GuestExtents = reinterpret_cast<const uint32_t*>(IndexDataStart + ReadOffset);
ReadOffset += IndexBlobExtra->GuestExtentsCount * sizeof(uint32_t);
if (FOZKey->bytes[39] != 0xFF) {
IndexEntry NewEntry {this, IndexBlobCommon->cache_db_file_offset, IndexBlobCommon->size, IndexBlobExtra->GuestSize,
IndexBlobExtra->GuestHash};
NewEntry.GuestExtents.resize(IndexBlobExtra->GuestExtentsCount);
memcpy(NewEntry.GuestExtents.data(), GuestExtents, IndexBlobExtra->GuestExtentsCount * sizeof(uint32_t));
bool ExtentsValid = true;
for (uint32_t i = 0; i < NewEntry.GuestExtents.size(); i += 2) {
if ((uint64_t)NewEntry.GuestExtents[i] + NewEntry.GuestExtents[i + 1] > IndexBlobExtra->GuestSize) {
ExtentsValid = false;
break;
}
}
if (!ExtentsValid) {
continue;
}
auto It = CacheIndex.find(IndexBlobCommon->hash);
if (It == CacheIndex.end()) {
CacheIndex.emplace(IndexBlobCommon->hash, IndexCacheHead {std::move(NewEntry), IndexBlobExtra->GuestFootprint, nullptr});
} else {
if (!It->second.MoreEntries.get()) {
It->second.MoreEntries = fextl::make_unique<fextl::multimap<uint64_t, IndexEntry>>();
}
It->second.MoreEntries->insert({IndexBlobExtra->GuestFootprint, std::move(NewEntry)});
}
} else {
FoundMetadata = true;
}
}
// could truncate/delete index if we don't end up perfectly at end here
}
bool IndexedDB::ReadCacheBlob(uint64_t Offset, std::span<uint8_t> OutBlob) {
if (CacheFileMapping && (ReadOnly || Offset + OutBlob.size() <= BIG_MAPPING_SIZE)) {
if (Offset + OutBlob.size() > CacheFileSize) {
return false;
}
// todo could reduce copies by having a private mapping for relocs, etc
memcpy(OutBlob.data(), CacheFileMapping + Offset, OutBlob.size());
return true;
} else {
return CacheFOZ.ReadBlob(Offset, OutBlob);
}
}
bool IndexedDB::StoreCacheBlob(const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey, std::span<const uint8_t> Blob,
MesaFOZ::mesa_index_db_file_entry& IndexEntry, std::span<const uint8_t> IndexBlob) {
if (ReadOnly) {
// shouldn't happen
return false;
}
if (MaxSizeReached || CacheFileSize + Blob.size() >= MaxFileSize) {
MaxSizeReached = true;
return false;
}
if (!CacheFOZ.Lock(STORE_LOCK_TIMEOUT_MS) || !IndexFOZ.Lock(STORE_LOCK_TIMEOUT_MS)) {
CacheFOZ.Unlock();
IndexFOZ.Unlock();
return false;
}
// write cache side first so we get offset for index
std::span<const uint8_t> BlobChunks[] = {Blob};
uint64_t BlobOffset = 0;
if (!CacheFOZ.WriteBlob(UniqueKey, BlobChunks, BlobOffset)) {
CacheFOZ.Unlock();
IndexFOZ.Unlock();
return false;
}
IndexEntry = {.hash = LookupKey,
.size = (uint32_t)Blob.size(),
.last_access_time = 0, // todo..
.cache_db_file_offset = BlobOffset};
std::span<const uint8_t> IndexBlobChunks[] = {
{(const uint8_t*)&IndexEntry, sizeof(IndexEntry)},
IndexBlob,
};
uint64_t UnusedIndexBlobOffset = 0;
if (!IndexFOZ.WriteBlob(UniqueKey, IndexBlobChunks, UnusedIndexBlobOffset)) {
CacheFOZ.Unlock();
IndexFOZ.Unlock();
return false;
}
CacheFOZ.Unlock();
IndexFOZ.Unlock();
// publish new file size for memory-mapped reads
if (CacheFileMapping) {
CacheFileSize = BlobOffset + Blob.size();
}
return true;
}
bool DiskCache::OpenCacheDB(const fextl::string& CacheDBName, bool ReadOnly) {
fextl::unique_ptr<IndexedDB> CurDB;
if (!ReadOnly && RWCacheDB) {
// rw already opened, just support one
return false;
}
CurDB = fextl::make_unique<IndexedDB>();
if (!CurDB) {
return false;
}
if (!CurDB->Open(CacheDBName, ReadOnly)) {
CurDB.reset();
return false;
}
CurDB->PopulateIndex(Index, FoundMetadata);
if (ReadOnly) {
ROCacheDBs.push_back(std::move(CurDB));
} else {
RWCacheDB = std::move(CurDB);
}
return true;
}
FEX_DEFAULT_VISIBILITY void SetFileMapper(FileMapperFunc Func) {
FileMapper = Func;
}
void DiskCache::Init(FEXCore::Context::ContextImpl* CTX) {
this->CTX = CTX;
if (!EnableDiskCache) {
return;
}
fextl::string SerializedConfig = FEXCore::Config::SerializeForCache();
struct __attribute__((packed)) {
uint16_t FormatVersion;
uint8_t Is64BitMode;
uint64_t HostFeaturesHash;
} BucketHeader = {FormatVersion, CTX->Config.Is64BitMode, CTX->HostFeatures.HashForCaching()};
fextl::vector<uint8_t> BucketBytes;
BucketBytes.resize(sizeof(BucketHeader) + SerializedConfig.size());
memcpy(BucketBytes.data(), &BucketHeader, sizeof(BucketHeader));
memcpy(BucketBytes.data() + sizeof(BucketHeader), SerializedConfig.data(), SerializedConfig.size());
BucketHash = XXH3_128bits(BucketBytes.data(), BucketBytes.size());
fextl::string BasePath = BasePathOverride();
if (BasePath.empty()) {
BasePath = FEXCore::Config::GetCacheDirectory() + "DiskCache/";
BasePath += fextl::fmt::format("{:016x}{:016x}", BucketHash.high64, BucketHash.low64) + "/";
}
FHU::Filesystem::CreateDirectories(BasePath);
if (!MapDiskCacheFiles) {
FileMapper = nullptr;
}
fextl::string RWDBBasePath = BasePath + "RWCacheDB";
OpenCacheDB(RWDBBasePath, false);
if (RWCacheDB && !FoundMetadata) {
// we just opened a fresh cache, add a metadata blob
MesaFOZ::foz_payload_key MetadataKey;
memset(MetadataKey.bytes, 0xFF, sizeof(MetadataKey));
IndexExtraBlobHeader MetaDataHeader = {};
MesaFOZ::mesa_index_db_file_entry IndexEntry = {};
RWCacheDB->StoreCacheBlob(MetadataKey, ~0, {BucketBytes.data(), BucketBytes.size()}, IndexEntry,
{reinterpret_cast<uint8_t*>(&MetaDataHeader), sizeof(MetaDataHeader)});
Index.erase(~0);
}
std::string_view RONames = RODBNames();
while (!RONames.empty()) {
const auto Delim = RONames.find(',');
const std::string_view ROName = RONames.substr(0, Delim);
if (!ROName.empty()) {
fextl::string RODBBasePath = BasePath;
RODBBasePath += ROName;
OpenCacheDB(RODBBasePath, true);
}
if (Delim == std::string_view::npos) {
break;
}
// advance to next
RONames.remove_prefix(Delim + 1);
}
WritingDiskCache = (bool)RWCacheDB;
ReadingDiskCache = !ROCacheDBs.empty() || RWCacheDB != nullptr;
if (IsWritingDiskCache()) {
FEXCore::Threads::Flags WriterThreadFlags = {.LowPriority = true, .Internal = true};
Writer = fextl::make_unique<WorkQueueThread>(WriterThreadFlags, "FEX:DiskCache");
}
}
uint64_t DiskCache::MakeLookupKey(Core::InternalThreadState* Thread, const uint64_t CodeKey, bool Writable, bool MonoBackpatcher) {
struct __attribute__((packed)) {
uint64_t CodeKey;
XXH128_hash_t BucketHash;
uint8_t Flags;
} BlobKeyBytes = {CodeKey, BucketHash, 0};
if (Writable) {
BlobKeyBytes.Flags |= 1 << 0;
}
if (CTX->AreMonoHacksActive()) {
BlobKeyBytes.Flags |= 1 << 1;
}
if (Thread->CurrentFrame->State.flags[X86State::RFLAG_TF_RAW_LOC]) {
BlobKeyBytes.Flags |= 1 << 2;
}
if (MonoBackpatcher) {
BlobKeyBytes.Flags |= 1 << 3;
}
return XXH3_64bits(&BlobKeyBytes, sizeof(BlobKeyBytes));
}
struct DiskCache::PruneMemoryLRUWorkItem final : WorkQueueThread::WorkItem {
DiskCache* Self;
PruneMemoryLRUWorkItem(DiskCache* Self)
: Self(Self) {}
void Run() override {
if (Self->MemoryLRUCurrentSize <= Self->MemoryLRUMaxSize + Self->MemoryLRUEvictThreshold) {
return;
}
std::lock_guard IndexGuard(Self->IndexLock);
std::lock_guard LRUGuard(Self->MemoryLRULock);
if (Self->MemoryLRU.empty()) {
return;
}
auto Last = std::prev(Self->MemoryLRU.end());
while (Self->MemoryLRUCurrentSize > Self->MemoryLRUMaxSize && !Self->MemoryLRU.empty()) {
auto LastKey = *Last;
auto IndexEntry = Self->LookupLocked(LastKey.LookupKey, LastKey.GuestHash, LastKey.GuestFootprint);
bool AtFront = (Last == Self->MemoryLRU.begin());
if (IndexEntry && IndexEntry->MemoryBlob.use_count() > 1) {
// being read rn, keep moving
if (AtFront) {
break;
}
Last--;
continue;
}
if (IndexEntry) {
IndexEntry->MemoryBlob.reset();
IndexEntry->LRUEntry.reset();
}
Self->MemoryLRUCurrentSize -= LastKey.Size;
auto Deleted = Last;
if (AtFront) {
Self->MemoryLRU.erase(Deleted);
break;
}
Last--;
Self->MemoryLRU.erase(Deleted);
}
}
};
std::optional<CodeHitData> DiskCache::Lookup(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region,
uint64_t GuestRIP, std::optional<uint64_t>& GuestCodeKey) {
if (!IsReadingDiskCache()) {
return std::nullopt;
}
if (Region && Region->FileStartVA) {
struct __attribute__((packed)) {
uint64_t GuestOffset;
uint64_t FileId;
} FileBackedKey = {GuestRIP - Region->FileStartVA, Region->FileInfo.FileId};
GuestCodeKey = XXH3_64bits(&FileBackedKey, sizeof(FileBackedKey));
} else {
if (!AnonCaching) {
return std::nullopt;
}
Thread->FrontendDecoder->DecodeLoop(reinterpret_cast<const uint8_t*>(GuestRIP), AnonPrefixGuestBytes);
const auto* BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo();
XXH3_state_t HashState;
XXH3_64bits_reset(&HashState);
for (auto& SubBlock : BlockInfo->Blocks) {
if (SubBlock.BlockStatus != Frontend::Decoder::DecodedBlockStatus::SUCCESS) {
return std::nullopt;
}
uint64_t HashStart = SubBlock.Entry;
// skip over masked in-block data and data/etc gaps between blocks
for (auto& DataMask : SubBlock.DataMasks) {
XXH3_64bits_update(&HashState, reinterpret_cast<const uint8_t*>(HashStart), DataMask.FieldAddress - HashStart);
HashStart = DataMask.FieldAddress + DataMask.ValueSize;
}
if (HashStart != SubBlock.Entry + SubBlock.Size) {
XXH3_64bits_update(&HashState, reinterpret_cast<const uint8_t*>(HashStart), SubBlock.Size - (HashStart - SubBlock.Entry));
}
}
GuestCodeKey = XXH3_64bits_digest(&HashState);
// if (TotalSize < AnonPrefixGuestBytes) {
// GuestCodeKey = 0;
// return std::nullopt;
// }
// LogMan::Msg::IFmt("anon lookup! length {:d} {}", GuestCodeKey, TotalSize);
}
auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, GuestRIP);
if (RangeInfo.Size == 0 || RangeInfo.Base > GuestRIP) {
return std::nullopt;
}
uint64_t Available = RangeInfo.Base + RangeInfo.Size - GuestRIP;
uint64_t LookupKey =
MakeLookupKey(Thread, *GuestCodeKey, RangeInfo.Writable, GuestRIP == CTX->GetMonoBackPatcherBlock().load(std::memory_order_relaxed));
XXH128_hash_t LiveGuestHash;
uint64_t LastFootprintHashed = 0;
bool FirstHash = true; // 0 is probably a valid footprint hash so we need an extra bit there
bool TriedMainEntry = false;
IndexEntry MainEntry;
uint64_t MainEntryFootprint;
fextl::multimap<uint64_t, IndexEntry>::iterator MoreEntriesIt;
fextl::multimap<uint64_t, IndexEntry>* MapPointer = nullptr;
IndexEntry* EntryUnderReview;
fextl::shared_ptr<fextl::vector<uint8_t>> BlobRef;
std::optional<fextl::list<MemoryLRUKey>::iterator> LRUIter;
uint64_t CurrentFootprint = 0;
{
std::lock_guard Guard(IndexLock);
auto It = Index.find(LookupKey);
if (It == Index.end()) {
// definite miss
return std::nullopt;
}
MainEntry = It->second.MainEntry;
MainEntryFootprint = It->second.MainEntryFootprint;
MapPointer = It->second.MoreEntries.get();
if (MapPointer) {
MoreEntriesIt = It->second.MoreEntries->begin();
}
if (MapPointer && MoreEntriesIt != It->second.MoreEntries->end()) {
EntryUnderReview = &MoreEntriesIt->second;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MoreEntriesIt->first;
} else {
EntryUnderReview = &MainEntry;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MainEntryFootprint;
TriedMainEntry = true;
}
}
// found a lookup key match, check for guest hash match now
bool FoundMatchingHash = false;
bool Advance = false;
while (!FoundMatchingHash) {
if (Advance) {
std::lock_guard Guard(IndexLock);
EntryUnderReview = nullptr;
if (MapPointer && MoreEntriesIt != MapPointer->end()) {
// if the current footprint is also the main entry's footprint, give main entry a shot next
if (!TriedMainEntry && MoreEntriesIt->first == MainEntryFootprint) {
EntryUnderReview = &MainEntry;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MainEntryFootprint;
TriedMainEntry = true;
} else {
MoreEntriesIt++;
}
}
if (!MapPointer || MoreEntriesIt == MapPointer->end()) {
if (TriedMainEntry) {
// ran out
break;
} else {
EntryUnderReview = &MainEntry;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MainEntryFootprint;
TriedMainEntry = true;
}
}
if (!EntryUnderReview && MapPointer && MoreEntriesIt != MapPointer->end()) {
EntryUnderReview = &MoreEntriesIt->second;
BlobRef = EntryUnderReview->MemoryBlob;
LRUIter = EntryUnderReview->LRUEntry;
CurrentFootprint = MoreEntriesIt->first;
}
}
Advance = true;
// entry not backed by anything right now - todo prune..
if (!EntryUnderReview->DB && !BlobRef) {
continue;
}
// do we have enough room in our live code to even hash GuestSize worth?
if (Available < EntryUnderReview->GuestSize) {
continue;
}
// if (Entry.GuestExtents.size()) {
// LogMan::Msg::IFmt("lookup! length {:d}", Entry.GuestSize);
// for(uint32_t i = 0; i < Entry.GuestExtents.size(); i+=2 ) {
// LogMan::Msg::IFmt("extent {} {}", Entry.GuestExtents[i], Entry.GuestExtents[i]+Entry.GuestExtents[i+1]);
// }
// }
{
IndexEntry& Entry = *EntryUnderReview;
// only renew the hash if the candidate footprint is different than previous
if (FirstHash || LastFootprintHashed != CurrentFootprint) {
XXH3_state_t HashState;
XXH3_128bits_reset(&HashState);
for (uint32_t i = 0; i < Entry.GuestExtents.size(); i += 2) {
XXH3_128bits_update(&HashState, reinterpret_cast<uint8_t*>(GuestRIP) + Entry.GuestExtents[i], Entry.GuestExtents[i + 1]);
}
LiveGuestHash = XXH3_128bits_digest(&HashState);
LastFootprintHashed = CurrentFootprint;
FirstHash = false;
}
if (XXH128_isEqual(LiveGuestHash, Entry.GuestHash)) {
FoundMatchingHash = true;
break;
} else if (Validation) {
fextl::vector<uint8_t> GuestCode(Entry.GuestSize);
if (Entry.Size >= Entry.GuestSize && Entry.DB && Entry.DB->ReadCacheBlob(Entry.Offset + Entry.Size - Entry.GuestSize, GuestCode)) {
const uint8_t* CachedGuest = GuestCode.data();
const uint8_t* LiveGuest = reinterpret_cast<const uint8_t*>(GuestRIP);
uint64_t DiffCount = 0;
uint64_t DiffOffset = 0;
for (uint32_t i = 0; i < Entry.GuestExtents.size(); i += 2) {
uint32_t Begin = Entry.GuestExtents[i];
uint32_t End = Begin + Entry.GuestExtents[i + 1];
bool PreviousByteDiff = false;
for (uint32_t Offset = Begin; Offset < End; Offset++) {
if (LiveGuest[Offset] != CachedGuest[Offset]) {
if (DiffCount == 0) {
DiffOffset = Offset;
}
if (!PreviousByteDiff) {
DiffCount++;
}
PreviousByteDiff = true;
} else {
PreviousByteDiff = false;
}
}
}
if (DiffCount) {
uint64_t DiffStart = DiffOffset >= 8 ? DiffOffset - 8 : 0;
uint64_t DiffContextBytes = std::min<uint64_t>(16, Entry.GuestSize - DiffStart);
auto LiveDump = fmt::join(std::span<const uint8_t> {LiveGuest + DiffStart, DiffContextBytes}, " ");
auto CacheDump = fmt::join(std::span<const uint8_t> {CachedGuest + DiffStart, DiffContextBytes}, " ");
auto KeyPrefix = (Region && Region->FileStartVA) ? "file" : "anon";
LogMan::Msg::IFmt("DiskCache: lookup guest hash mismatch key={}-{:x} gsize={}, ndiff={}, first diff: offset={} "
"live=[{:02x}] "
"cached=[{:02x}]",
KeyPrefix, *GuestCodeKey, Entry.GuestSize, DiffCount, DiffOffset, LiveDump, CacheDump);
} else {
LogMan::Msg::IFmt("DiskCache: guest hash mismatch but no diff?");
}
}
}
}
}
if (!FoundMatchingHash || !EntryUnderReview) {
return std::nullopt;
}
IndexEntry& Entry = *EntryUnderReview;
fextl::vector<uint64_t> GuestPages;
for (uint32_t i = 0; i < Entry.GuestExtents.size(); i += 2) {
uint64_t FirstPage = (Entry.GuestExtents[i] + GuestRIP) & Utils::FEX_PAGE_MASK;
uint64_t LastPage = (Entry.GuestExtents[i] + GuestRIP + Entry.GuestExtents[i + 1] - 1) & Utils::FEX_PAGE_MASK;
for (uint64_t Page = FirstPage; Page <= LastPage; Page += Utils::FEX_PAGE_SIZE) {
if (GuestPages.size() == 0 || Page != GuestPages.back()) {
GuestPages.push_back(Page);
}
}
}
// this seems to be a full hit, pull from disk and check the entry is big enough to have everything (except GuestCode)
CodeHitData HitData;
uint32_t EntrySizeWithoutGuestCode = Entry.Size - Entry.GuestSize;
HitData.Blob.resize(GuestPages.size() * sizeof(uint64_t) + EntrySizeWithoutGuestCode);
memcpy(HitData.Blob.data(), GuestPages.data(), GuestPages.size() * sizeof(uint64_t));
uint32_t BlobOffset = GuestPages.size() * sizeof(uint64_t);
bool FoundInLRU = false;
if (BlobRef && BlobRef->size() >= EntrySizeWithoutGuestCode) {
FoundInLRU = true;
memcpy(HitData.Blob.data() + BlobOffset, BlobRef->data(), EntrySizeWithoutGuestCode);
if (LRUIter) {
std::lock_guard Guard(MemoryLRULock);
// avoided disk by nabbing from lru, bump to front
// LogMan::Msg::IFmt("lru hit! {}", MemoryLRUCurrentSize);
MemoryLRU.splice(MemoryLRU.begin(), MemoryLRU, *LRUIter);
}
} else if (!Entry.DB->ReadCacheBlob(Entry.Offset, {HitData.Blob.data() + BlobOffset, EntrySizeWithoutGuestCode})) {
return std::nullopt;
}
if (EntrySizeWithoutGuestCode < sizeof(BlobFixedHeader)) {
return std::nullopt;
}
BlobFixedHeader Header;
memcpy(&Header, HitData.Blob.data() + BlobOffset, sizeof(Header));
BlobOffset += sizeof(Header);
uint32_t SizeNeeded = sizeof(Header) + Header.HostSize + Header.EntryPointCount * (sizeof(uint64_t) + sizeof(uint32_t));
SizeNeeded += Header.SmallRelocCount * sizeof(BlobSmallRelocation) + Header.ThunkRelocCount * sizeof(BlobThunkRelocation);
if (EntrySizeWithoutGuestCode != SizeNeeded) {
return std::nullopt;
}
if (Entry.GuestSize != Header.GuestSize || !XXH128_isEqual(Header.GuestHash, Entry.GuestHash)) {
return std::nullopt;
}
bool StoreInMemory = !FoundInLRU;
if (EntrySizeWithoutGuestCode > MemoryLRUMaxSize) {
StoreInMemory = false;
}
if (StoreInMemory) {
auto NewBlobRef = fextl::make_shared<fextl::vector<uint8_t>>(EntrySizeWithoutGuestCode);
memcpy(NewBlobRef->data(), HitData.Blob.data() + BlobOffset, EntrySizeWithoutGuestCode);
std::lock_guard Guard(IndexLock);
auto CurrentIndexEntry = LookupLocked(LookupKey, Header.GuestHash, LastFootprintHashed);
if (CurrentIndexEntry && !CurrentIndexEntry->MemoryBlob) {
std::lock_guard LRUGuard(MemoryLRULock);
MemoryLRU.push_front({LookupKey, Header.GuestHash, LastFootprintHashed, EntrySizeWithoutGuestCode});
CurrentIndexEntry->LRUEntry = MemoryLRU.begin();
CurrentIndexEntry->MemoryBlob = NewBlobRef;
MemoryLRUCurrentSize += EntrySizeWithoutGuestCode;
}
}
if (StoreInMemory && MemoryLRUCurrentSize > MemoryLRUMaxSize + MemoryLRUEvictThreshold) {
Writer->QueueWork(fextl::make_unique<PruneMemoryLRUWorkItem>(this));
}
HitData.HostCode = {HitData.Blob.data() + BlobOffset, Header.HostSize};
BlobOffset += Header.HostSize;
HitData.EntryPointRIPs = {reinterpret_cast<uint64_t*>(HitData.Blob.data() + BlobOffset), Header.EntryPointCount};
BlobOffset += Header.EntryPointCount * sizeof(uint64_t);
HitData.EntryPointHostOffsets = {reinterpret_cast<const uint32_t*>(HitData.Blob.data() + BlobOffset), Header.EntryPointCount};
BlobOffset += Header.EntryPointCount * sizeof(uint32_t);
std::span<const BlobSmallRelocation> SmallRelocs = {reinterpret_cast<const BlobSmallRelocation*>(HitData.Blob.data() + BlobOffset),
Header.SmallRelocCount};
BlobOffset += Header.SmallRelocCount * sizeof(BlobSmallRelocation);
std::span<const BlobThunkRelocation> ThunkRelocs = {reinterpret_cast<const BlobThunkRelocation*>(HitData.Blob.data() + BlobOffset),
Header.ThunkRelocCount};
BlobOffset += Header.ThunkRelocCount * sizeof(BlobThunkRelocation);
HitData.GuestPages = {reinterpret_cast<uint64_t*>(HitData.Blob.data()), GuestPages.size()};
for (auto& EntryPointRip : HitData.EntryPointRIPs) {
EntryPointRip += GuestRIP;
}
if (!CTX->CodeCache.ApplyPackedCodeRelocations(GuestRIP, std::as_writable_bytes(HitData.HostCode), SmallRelocs, ThunkRelocs)) {
return std::nullopt;
}
return HitData;
}
void DiskCache::Validate(uint64_t GuestCodeKey, const CodeHitData& Hit, const CPU::CPUBackend::CompiledCode& CompiledCode,
std::optional<ExecutableFileSectionInfo> Region) {
if (!Validation) {
return;
}
auto KeyPrefix = (Region && Region->FileStartVA) ? "file" : "anon";
if (Hit.HostCode.size() != CompiledCode.Size) {
LogMan::Msg::EFmt("DiskCache: validate host size mismatch key={}-{:x} cached={} live={}", KeyPrefix, GuestCodeKey,
Hit.HostCode.size(), CompiledCode.Size);
} else if (memcmp(Hit.HostCode.data(), CompiledCode.BlockBegin, CompiledCode.Size) != 0) {
bool PreviousByteDiff = false;
size_t FirstDiff = 0;
size_t DiffCount = 0;
for (size_t i = 0; i < CompiledCode.Size; i++) {
if (Hit.HostCode[i] != CompiledCode.BlockBegin[i]) {
if (DiffCount == 0) {
FirstDiff = i;
}
if (!PreviousByteDiff) {
DiffCount++;
}
PreviousByteDiff = true;
} else {
PreviousByteDiff = false;
}
}
// align to 4 bytes to read host arm easier
const size_t DiffStart = (FirstDiff & ~3ULL) >= 8 ? (FirstDiff & ~3ULL) - 8 : 0;
const size_t ContextBytes = std::min<size_t>(16, CompiledCode.Size - DiffStart);
LogMan::Msg::EFmt("DiskCache: validate host code mismatch key={}-{:x} size={} firstoffset={} ndiff={} cached=[{:02x}] live=[{:02x}]",
KeyPrefix, GuestCodeKey, CompiledCode.Size, FirstDiff, DiffCount,
fmt::join(std::span<const uint8_t> {Hit.HostCode.data() + DiffStart, ContextBytes}, " "),
fmt::join(std::span<const uint8_t> {CompiledCode.BlockBegin + DiffStart, ContextBytes}, " "));
}
}
IndexEntry* DiskCache::LookupLocked(const uint64_t LookupKey, const XXH128_hash_t& GuestHash, const uint64_t GuestFootprint) {
auto It = Index.find(LookupKey);
if (It != Index.end()) {
if (XXH128_isEqual(It->second.MainEntry.GuestHash, GuestHash)) {
return &It->second.MainEntry;
}
if (It->second.MoreEntries.get()) {
auto Range = It->second.MoreEntries->equal_range(GuestFootprint);
for (auto MoreEntriesIt = Range.first; MoreEntriesIt != Range.second; MoreEntriesIt++) {
if (XXH128_isEqual(MoreEntriesIt->second.GuestHash, GuestHash)) {
return &MoreEntriesIt->second;
}
}
}
}
return nullptr;
}
struct DiskCache::CacheStoreWorkItem final : WorkQueueThread::WorkItem {
DiskCache* Self;
IndexedDB* DB;
MesaFOZ::foz_payload_key UniqueKey;
uint64_t LookupKey;
std::span<uint8_t> Blob;
fextl::vector<uint8_t> IndexBlob;
bool StoreDisk;
CacheStoreWorkItem(DiskCache* Self, IndexedDB* DB, const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey,
std::span<uint8_t> Blob, fextl::vector<uint8_t>&& IndexBlob, bool StoreDisk)
: Self(Self)
, DB(DB)
, UniqueKey(UniqueKey)
, LookupKey(LookupKey)
, Blob(Blob)
, IndexBlob(std::move(IndexBlob))
, StoreDisk(StoreDisk) {}
void Run() override {
struct MesaFOZ::mesa_index_db_file_entry IndexHeader;
bool DiskSuccess = !StoreDisk || DB->StoreCacheBlob(UniqueKey, LookupKey, Blob, IndexHeader, IndexBlob);
bool KeepEntryInMemory = true;
// todo possible other lru condition here like entry size?
if (Blob.size() > Self->MemoryLRUMaxSize) {
KeepEntryInMemory = false;
} else {
Self->MemoryLRUCurrentSize += Blob.size();
}
const IndexExtraBlobHeader* IndexAfterHeader = reinterpret_cast<const IndexExtraBlobHeader*>(IndexBlob.data());
fextl::list<MemoryLRUKey>::iterator NewLRUEntry;
if (KeepEntryInMemory) {
std::lock_guard Guard(Self->MemoryLRULock);
Self->MemoryLRU.push_front({LookupKey, IndexAfterHeader->GuestHash, IndexAfterHeader->GuestFootprint, (uint32_t)Blob.size()});
NewLRUEntry = Self->MemoryLRU.begin();
}
{
std::lock_guard Guard(Self->IndexLock);
auto IndexEntry = Self->LookupLocked(LookupKey, IndexAfterHeader->GuestHash, IndexAfterHeader->GuestFootprint);
LOGMAN_THROW_A_FMT(IndexEntry != nullptr, "Stored Index entry not found?");
if (IndexEntry) {
if (StoreDisk && DiskSuccess) {
IndexEntry->DB = DB;
IndexEntry->Offset = IndexHeader.cache_db_file_offset;
}
if (!KeepEntryInMemory) {
IndexEntry->MemoryBlob.reset();
} else {
IndexEntry->LRUEntry = NewLRUEntry;
}
}
}
if (KeepEntryInMemory && Self->MemoryLRUCurrentSize > Self->MemoryLRUMaxSize + Self->MemoryLRUEvictThreshold) {
Self->Writer->QueueWork(fextl::make_unique<PruneMemoryLRUWorkItem>(Self));
}
}
};
bool DiskCache::Store(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region, uint64_t GuestRIP,
uint64_t GuestCodeKey, std::span<const uint8_t> GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode,
std::span<const FEXCore::CPU::Relocation> Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo) {
if (!IsWritingDiskCache()) {
return false;
}
if (!DecodedBlockInfo) {
return false;
}
// check for any reloc targets outside of our jurisdiction
// todo what are they exactly? caching those blocks is great when it works, so need to figure this out and make finer-grained if we can
if (RelocationFilter && Region) {
for (const auto& Reloc : Relocations) {
if (Reloc.Header.Type != CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL && Reloc.Header.Type != CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE) {
continue;
}
uint64_t Target = Reloc.GuestRIP.GuestRIP;
if (Target >= Region->BeginVA && Target < Region->EndVA) {
continue;
}
// let it through if it's inside the same ELF image? (like bss)
if (Region->FileInfo.MappedSize && Target >= Region->FileStartVA && Target < Region->FileStartVA + Region->FileInfo.MappedSize) {
continue;
}
auto TargetSection = CTX->SyscallHandler->LookupExecutableFileSection(Thread, Target);
if (!TargetSection || TargetSection->FileInfo.FileId != Region->FileInfo.FileId) {
// we don't know where it's pointing, so we don't know how to encode the offset, so we can't cache atm
return false;
}
}
}
uint32_t SmallRelocCount = 0;
uint32_t ThunkRelocCount = 0;
for (const auto& Reloc : Relocations) {
if (Reloc.Header.Type == CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE) {
ThunkRelocCount++;
} else {
SmallRelocCount++;
}
}
const uint32_t EntryPointCount = (uint32_t)CompiledCode.EntryPoints.size();
const size_t HeaderOffset = 0;
const size_t HostCodeOffset = HeaderOffset + sizeof(BlobFixedHeader);
const size_t EntryPointRIPsOffset = HostCodeOffset + CompiledCode.Size;
const size_t EntryPointHostOffsetsOffset = EntryPointRIPsOffset + EntryPointCount * sizeof(uint64_t);
const size_t SmallRelocsOffset = EntryPointHostOffsetsOffset + EntryPointCount * sizeof(uint32_t);
const size_t ThunkRelocsOffset = SmallRelocsOffset + SmallRelocCount * sizeof(BlobSmallRelocation);
const size_t GuestCodeOffset = ThunkRelocsOffset + ThunkRelocCount * sizeof(BlobThunkRelocation);
const size_t TotalSize = GuestCodeOffset + GuestCode.size();
// we'll copy everything into here and pass it to the Writer, then return to caller quickly
fextl::vector<uint8_t> Blob;
Blob.resize(TotalSize);
uint8_t* BlobData = Blob.data();
BlobFixedHeader Header {
.GuestSize = (uint32_t)GuestCode.size(),
.HostSize = (uint32_t)CompiledCode.Size,
.EntryPointCount = EntryPointCount,
.SmallRelocCount = SmallRelocCount,
.ThunkRelocCount = ThunkRelocCount,
};
memcpy(BlobData + HostCodeOffset, CompiledCode.BlockBegin, CompiledCode.Size);
// pack and relocate entrypoints
auto* EntryRIPs = reinterpret_cast<uint64_t*>(BlobData + EntryPointRIPsOffset);
auto* EntryHostOffsets = reinterpret_cast<uint32_t*>(BlobData + EntryPointHostOffsetsOffset);
uint32_t EntryIdx = 0;
for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) {
EntryRIPs[EntryIdx] = GuestAddr - GuestRIP;
EntryHostOffsets[EntryIdx] = uint32_t(HostAddr - CompiledCode.BlockBegin);
EntryIdx++;
}
fextl::set<uint64_t> DataMasksConsumed;
// pack relocations
auto* SmallRelocs = reinterpret_cast<BlobSmallRelocation*>(BlobData + SmallRelocsOffset);
auto* ThunkRelocs = reinterpret_cast<BlobThunkRelocation*>(BlobData + ThunkRelocsOffset);
uint32_t SmallIdx = 0;
uint32_t ThunkIdx = 0;
for (const auto& Reloc : Relocations) {
switch (Reloc.Header.Type) {
// it's important to zero-init the element completely so we don't have garbage in unused fields
// this way, the caches stay deterministic across machines
case CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: {
BlobSmallRelocation SmallReloc = {};
SmallReloc.Offset = Reloc.Header.Offset;
SmallReloc.Type = uint8_t(Reloc.Header.Type);
SmallReloc.Named.Symbol = uint32_t(Reloc.NamedSymbolLiteral.Symbol);
SmallRelocs[SmallIdx++] = SmallReloc;
break;
}
case CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL: {
BlobSmallRelocation SmallReloc = {};
SmallReloc.Offset = Reloc.Header.Offset;
SmallReloc.Type = uint8_t(Reloc.Header.Type);
SmallReloc.RIPLiteral.GuestRIP = Reloc.GuestRIP.GuestRIP - GuestRIP;
SmallRelocs[SmallIdx++] = SmallReloc;
break;
}
case CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE: {
BlobSmallRelocation SmallReloc = {};
SmallReloc.Offset = Reloc.Header.Offset;
SmallReloc.Type = uint8_t(Reloc.Header.Type);
SmallReloc.RIPMove.RegisterIndex = Reloc.GuestRIP.RegisterIndex;
SmallReloc.RIPMove.GuestRIP = Reloc.GuestRIP.GuestRIP - GuestRIP;
SmallRelocs[SmallIdx++] = SmallReloc;
break;
}
case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE:
case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE:
case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_LITERAL:
case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE: {
// same data for all, relative vs. not and register vs. literal will depend on type on apply
BlobSmallRelocation SmallReloc = {};
SmallReloc.Offset = Reloc.Header.Offset;
SmallReloc.Type = uint8_t(Reloc.Header.Type);
SmallReloc.PatchableData.RegisterIndex = Reloc.GuestPatchableData.RegisterIndex;
SmallReloc.PatchableData.ValueSize = Reloc.GuestPatchableData.ValueSize;
SmallReloc.PatchableData.SiteOffset = uint32_t(Reloc.GuestPatchableData.SiteAddress - GuestRIP);
SmallRelocs[SmallIdx++] = SmallReloc;
// mark the corresponding data mask consumed
DataMasksConsumed.insert(Reloc.GuestPatchableData.SiteAddress);
break;
}
case CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE: {
BlobThunkRelocation BigReloc = {};
BigReloc.Offset = Reloc.Header.Offset;
BigReloc.RegisterIndex = Reloc.NamedThunkMove.RegisterIndex;
memcpy(BigReloc.SymbolHash, &Reloc.NamedThunkMove.Symbol, sizeof(BigReloc.SymbolHash));
ThunkRelocs[ThunkIdx++] = BigReloc;
break;
}
}
}
fextl::vector<uint32_t> ExactGuestCodeExtents;
uint64_t CurStartExtent = 0, CurEndExtent = 0;
const Frontend::Decoder::DecodedBlocks* LastBlock = nullptr;
for (auto& SubBlock : DecodedBlockInfo->Blocks) {
if (SubBlock.BlockStatus != Frontend::Decoder::DecodedBlockStatus::SUCCESS) {
return false;
}
if (!CurStartExtent) {
CurStartExtent = SubBlock.Entry;
CurEndExtent = SubBlock.Entry + SubBlock.Size;
} else {
LOGMAN_THROW_A_FMT(SubBlock.Entry >= CurEndExtent, "DecodedBlocks not sorted or overlapping?");
if (SubBlock.Entry == CurEndExtent) {
CurEndExtent = SubBlock.Entry + SubBlock.Size;
} else {
ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
CurStartExtent = SubBlock.Entry;
CurEndExtent = SubBlock.Entry + SubBlock.Size;
}
}
// split extents according to data masks as well
for (auto& Mask : SubBlock.DataMasks) {
if (Mask.Type == Frontend::Decoder::DataMaskType::NOP || DataMasksConsumed.find(Mask.FieldAddress) != DataMasksConsumed.end()) {
if (Mask.FieldAddress > CurStartExtent) {
ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
ExactGuestCodeExtents.push_back(Mask.FieldAddress - CurStartExtent);
}
CurStartExtent = Mask.FieldAddress + Mask.ValueSize;
}
}
LastBlock = &SubBlock;
}
if (LastBlock && (CurStartExtent != GuestRIP || CurEndExtent != GuestRIP + GuestCode.size())) {
ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
}
if (ExactGuestCodeExtents.size() == 0) {
ExactGuestCodeExtents.reserve(2);
ExactGuestCodeExtents.push_back(0);
ExactGuestCodeExtents.push_back(GuestCode.size());
}
uint64_t GuestFootprint = XXH3_64bits(ExactGuestCodeExtents.data(), ExactGuestCodeExtents.size() * sizeof(uint32_t));
// if (ExactGuestCodeExtents.size()) {
// LogMan::Msg::IFmt("store! length {:d}", GuestCode.size());
// for(uint32_t i = 0; i < ExactGuestCodeExtents.size(); i+=2 ) {
// LogMan::Msg::IFmt("extent {} {}", ExactGuestCodeExtents[i], ExactGuestCodeExtents[i]+ExactGuestCodeExtents[i+1]);
// }
// }
{
XXH3_state_t HashState;
XXH3_128bits_reset(&HashState);
for (uint32_t i = 0; i < ExactGuestCodeExtents.size(); i += 2) {
XXH3_128bits_update(&HashState, GuestCode.data() + ExactGuestCodeExtents[i], ExactGuestCodeExtents[i + 1]);
}
Header.GuestHash = XXH3_128bits_digest(&HashState);
}
memcpy(BlobData + HeaderOffset, &Header, sizeof(Header));
auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, GuestRIP);
uint64_t LookupKey =
MakeLookupKey(Thread, GuestCodeKey, RangeInfo.Writable, GuestRIP == CTX->GetMonoBackPatcherBlock().load(std::memory_order_relaxed));
// blob done, publish to index as in-memory for now, flush to disk below
auto BlobRef = fextl::make_shared<fextl::vector<uint8_t>>(std::move(Blob));
struct IndexEntry NewEntry {nullptr, 0, (uint32_t)TotalSize, Header.GuestSize, Header.GuestHash, BlobRef};
NewEntry.GuestExtents = ExactGuestCodeExtents;
{
std::lock_guard Guard(IndexLock);
auto It = Index.find(LookupKey);
if (It == Index.end()) {
Index.emplace(LookupKey, IndexCacheHead {std::move(NewEntry), GuestFootprint, nullptr});
} else {
bool Dupe = false;
if (It->second.MoreEntries.get()) {
for (auto& [Key, Elem] : *It->second.MoreEntries) {
if (XXH128_isEqual(Elem.GuestHash, Header.GuestHash)) {
Dupe = true;
break;
}
}
}
if (!Dupe && XXH128_isEqual(It->second.MainEntry.GuestHash, Header.GuestHash)) {
Dupe = true;
}
// could happen if it's seen again while in flight in the store queue
if (Dupe) {
return true;
}
if (!It->second.MoreEntries.get()) {
It->second.MoreEntries = fextl::make_unique<fextl::multimap<uint64_t, struct IndexEntry>>();
} else if (It->second.MoreEntries->size() >= LOOKUP_KEY_MAX_BUCKET_DEPTH) {
return false;
}
It->second.MoreEntries->insert({GuestFootprint, std::move(NewEntry)});
}
}
memcpy(BlobData + GuestCodeOffset, GuestCode.data(), GuestCode.size());
MesaFOZ::foz_payload_key Key = {};
{
XXH3_state_t HashState;
XXH3_128bits_reset(&HashState);
XXH3_128bits_update(&HashState, &Header.GuestHash, sizeof(Header.GuestHash));
XXH3_128bits_update(&HashState, &LookupKey, sizeof(LookupKey));
XXH128_hash_t UniqueKey = XXH3_128bits_digest(&HashState);
fextl::string BlobName = fextl::fmt::format("{:016x}{:016x}", UniqueKey.high64, UniqueKey.low64);
memcpy(Key.bytes, BlobName.data(), BlobName.size());
}
fextl::vector<uint8_t> IndexBlob;
IndexBlob.resize(sizeof(IndexExtraBlobHeader) + ExactGuestCodeExtents.size() * sizeof(uint32_t));
IndexExtraBlobHeader IndexBlobHeader {Header.GuestHash, GuestFootprint, Header.GuestSize, (uint32_t)ExactGuestCodeExtents.size()};
memcpy(IndexBlob.data(), &IndexBlobHeader, sizeof(IndexExtraBlobHeader));
memcpy(IndexBlob.data() + sizeof(IndexExtraBlobHeader), ExactGuestCodeExtents.data(), ExactGuestCodeExtents.size() * sizeof(uint32_t));
bool StoreDisk = true;
if (RWCacheDB->Full()) {
StoreDisk = false;
}
// hand the rest off to the writer thread
Writer->QueueWork(fextl::make_unique<CacheStoreWorkItem>(this, RWCacheDB.get(), Key, LookupKey, std::span(BlobData, TotalSize),
std::move(IndexBlob), StoreDisk));
return true;
}
uint16_t GetFormatVersion() {
return FormatVersion;
}
} // namespace DiskCache
} // namespace FEXCore
Reference in new issue View Git Blame Copy Permalink